Mercurial > treepkg
comparison bin/publishdebianpackages.py @ 417:25ef11a79d7f treepkg-status
incremental copying only changed debian packages
| author | Bjoern Ricks <bricks@intevation.de> |
|---|---|
| date | Mon, 26 Jul 2010 12:58:50 +0000 |
| parents | 4980f8d5014a |
| children | 7780bde2ec68 |
comparison
equal
deleted
inserted
replaced
| 416:4980f8d5014a | 417:25ef11a79d7f |
|---|---|
| 27 from treepkg.info.data import Package | 27 from treepkg.info.data import Package |
| 28 from treepkg.info.data import CacheDb | 28 from treepkg.info.data import CacheDb |
| 29 | 29 |
| 30 config_desc = ["distribution", "section", "num_newest", | 30 config_desc = ["distribution", "section", "num_newest", |
| 31 "build_user", "build_host", "build_listpackages", | 31 "build_user", "build_host", "build_listpackages", |
| 32 "publish_user", "publish_host", "cachedb", | 32 "publish_user", "publish_host", |
| 33 ("architectures", shlex.split, "armel i386 source"), | 33 ("architectures", shlex.split, "armel i386 source"), |
| 34 ("after_upload_hook", shlex.split), | 34 ("after_upload_hook", shlex.split), |
| 35 ("publish_remove_old_packages", convert_bool), | 35 ("publish_remove_old_packages", convert_bool), |
| 36 ("publish_dir", remove_trailing_slashes), | 36 ("publish_dir", remove_trailing_slashes), |
| 37 ("cachedb", | |
| 38 lambda s: expand_filename(remove_trailing_slashes(s))), | |
| 37 ("cachedir", | 39 ("cachedir", |
| 38 lambda s: expand_filename(remove_trailing_slashes(s)))] | 40 lambda s: expand_filename(remove_trailing_slashes(s)))] |
| 39 | 41 |
| 40 | 42 |
| 41 def read_config(filename): | 43 def read_config(filename): |
| 104 def sort_trackname_arch(a, b): | 106 def sort_trackname_arch(a, b): |
| 105 if a.trackname < b.trackname: return -1 | 107 if a.trackname < b.trackname: return -1 |
| 106 if a.trackname > b.trackname: return +1 | 108 if a.trackname > b.trackname: return +1 |
| 107 return cmp(a.arch, b.arch) | 109 return cmp(a.arch, b.arch) |
| 108 | 110 |
| 109 def copy_files_to_destdir(destdir, files, quiet): | 111 def copy_files_to_destdir(destdir, files, variables, quiet = False): |
| 110 scp_flags = [] | 112 scp_flags = [] |
| 111 if quiet: | 113 if quiet: |
| 112 scp_flags.append("-q") | 114 scp_flags.append("-q") |
| 113 | 115 |
| 114 if not os.path.exists(destdir): | 116 if not os.path.exists(destdir): |
| 115 os.makedirs(destdir) | 117 os.makedirs(destdir) |
| 116 | 118 if files: |
| 117 # scp the packages to the cache dir | 119 if variables["build_host"]: |
| 118 call(cmdexpand("scp -p @scp_flags @files $cachedir/", files=files, | 120 userhost = "%(build_user)s@%(build_host)s:" % variables |
| 119 scp_flags=scp_flags, cachedir=destdir)) | 121 files = [userhost + filename for filename in files] |
| 120 | 122 # scp the packages to the cache dir |
| 121 def remove_old_packages(cachedb, newpackages): | 123 call(cmdexpand("scp -p @scp_flags @files $cachedir/", files=files, |
| 124 scp_flags=scp_flags, cachedir=destdir)) | |
| 125 | |
| 126 def remove_old_packages(cachedb, newpackages, quiet): | |
| 122 newfiles = [package.filename for package in newpackages] | 127 newfiles = [package.filename for package in newpackages] |
| 123 oldpackages = cachedb.get_old_packages(newfiles) | 128 oldpackages = cachedb.get_old_packages(newfiles) |
| 124 for package in oldpackages: | 129 for package in oldpackages: |
| 125 # better check if the file really exists | 130 # better check if the file really exists |
| 126 if os.path.isfile(package.filename): | 131 if os.path.isfile(package.filename): |
| 132 if not quiet: | |
| 133 print "removing file %s" % package.filename | |
| 127 os.remove(package.filename) | 134 os.remove(package.filename) |
| 128 cachedb.remove_packages(oldpackages) | 135 cachedb.remove_packages(oldpackages) |
| 129 | 136 |
| 130 def copy_packages_to_destdir(cachedb, dir, packages, quiet = False): | 137 def copy_packages_to_destdir(cachedb, dir, packages, variables, quiet = False): |
| 131 packages.sort(cmp=sort_trackname_arch) | 138 packages.sort(cmp=sort_trackname_arch) |
| 132 package = packages[0] | 139 package = packages[0] |
| 133 trackname = package.name | 140 trackname = package.trackname |
| 134 arch = package.arch | 141 arch = package.arch |
| 135 destdir = os.path.join(dir, arch, trackname) | 142 destdir = os.path.join(dir, arch, trackname) |
| 136 files = [] | 143 files = [] |
| 137 for package in packages: | 144 for package in packages: |
| 138 cachedb.add_package(package) | 145 cachedb.add_package(package) |
| 139 if package.trackname != trackname or \ | 146 if package.trackname != trackname or \ |
| 140 package.arch != arch: | 147 package.arch != arch: |
| 141 copy_files_to_destdir(destdir, files, quiet) | 148 copy_files_to_destdir(destdir, files, variables, quiet) |
| 142 trackname = package.trackname | 149 trackname = package.trackname |
| 143 arch = package.arch | 150 arch = package.arch |
| 144 destdir = os.path.join(dir, arch, trackname) | 151 destdir = os.path.join(dir, arch, trackname) |
| 145 files = [] | 152 files = [] |
| 146 # add only to copy files list if the packages differ | 153 # add only to copy files list if the packages differ |
| 147 if check_package_is_new(package.name, destdir, package.md5sum): | 154 if check_package_is_new(package.name, destdir, package.md5sum): |
| 148 files.append(package.sourcepath) | 155 files.append(package.sourcepath) |
| 149 print "copy package: %s, source: %s, destdir: %s" % (package.name, | 156 if not quiet: |
| 150 package.sourcepath, destdir) | 157 print "copy new file: %s" % package.name |
| 151 | 158 elif not quiet: |
| 159 print "file %s did not changed." % package.name | |
| 160 copy_files_to_destdir(destdir, files, variables, quiet) | |
| 161 | |
| 152 def copy_to_cachedir(variables, track, revision, quiet = False, architectures=None): | 162 def copy_to_cachedir(variables, track, revision, quiet = False, architectures=None): |
| 153 cachedir = variables["cachedir"] | 163 cachedir = variables["cachedir"] |
| 154 cachdebfilename = variables["cachedb"] | 164 cachdebfilename = variables["cachedb"] |
| 165 if not quiet: | |
| 166 print "using cachedb: %s" % cachdebfilename | |
| 155 cachedb = CacheDb(cachdebfilename) | 167 cachedb = CacheDb(cachdebfilename) |
| 156 newpackages = [] | 168 newpackages = [] |
| 157 treepkginfo = get_treepkg_info(variables) | 169 treepkginfo = get_treepkg_info(variables) |
| 158 #allowedarchs = set([]) # contains all wanted architectures (incl. source) | 170 #allowedarchs = set([]) # contains all wanted architectures (incl. source) |
| 159 allarchs = set([]) # contains all present architectures (incl. source) | 171 allarchs = set([]) # contains all present architectures (incl. source) |
| 197 binallarchs = allowedarchs - sourcearch | 209 binallarchs = allowedarchs - sourcearch |
| 198 else: | 210 else: |
| 199 binallarchs = (allowedarchs & allarchs) - sourcearch | 211 binallarchs = (allowedarchs & allarchs) - sourcearch |
| 200 for packageinfo in binaryallpackages: | 212 for packageinfo in binaryallpackages: |
| 201 for arch in binallarchs: | 213 for arch in binallarchs: |
| 202 filename = os.path.join(cachedir, arch, track.name, | 214 filename = os.path.join(cachedir, arch, packageinfo.trackname, |
| 203 packageinfo.name) | 215 packageinfo.name) |
| 204 newpackage = Package(filename, packageinfo.trackname, packageinfo.name, | 216 newpackage = Package(filename, packageinfo.trackname, packageinfo.name, |
| 205 packageinfo.path, arch, get_md5sum(packageinfo)) | 217 packageinfo.path, arch, get_md5sum(packageinfo)) |
| 206 newpackages.append(newpackage) | 218 newpackages.append(newpackage) |
| 207 copy_packages_to_destdir(cachedb, cachedir, newpackages, quiet) | 219 copy_packages_to_destdir(cachedb, cachedir, newpackages, variables, quiet) |
| 208 remove_old_packages(cachedb, newpackages) | 220 remove_old_packages(cachedb, newpackages, quiet) |
| 209 | 221 |
| 210 def publish_packages(config_filename, track, revision, dist, section, quiet): | 222 def publish_packages(config_filename, track, revision, dist, section, quiet): |
| 211 config = read_config(config_filename) | 223 config = read_config(config_filename) |
| 212 | 224 |
| 213 if dist is None: | 225 if dist is None: |
| 214 dist = config["distribution"] | 226 dist = config["distribution"] |
| 215 if section is None: | 227 if section is None: |
| 216 section = config["section"] | 228 section = config["section"] |
| 217 | 229 |
| 218 if not "cachedb" in config.keys(): | |
| 219 print >>sys.stderr, "Error. Please specifiy a cachedb in your config file" | |
| 220 sys.exit(1) | |
| 221 | |
| 222 architectures = config["architectures"] | 230 architectures = config["architectures"] |
| 223 copy_to_cachedir(config, track, revision, quiet, architectures) | 231 copy_to_cachedir(config, track, revision, quiet, architectures) |
| 224 # copy_to_publishdir(config, dist, section, arch, quiet) | 232 copy_to_publishdir(config, dist, section, None, quiet) |
| 225 | 233 |
| 226 # update apt archive | 234 # update apt archive |
| 227 # call(config["after_upload_hook"]) | 235 if config["after_upload_hook"]: |
| 236 if not quiet: | |
| 237 print "running after upload hook" | |
| 238 call(config["after_upload_hook"]) | |
| 228 | 239 |
| 229 def main(): | 240 def main(): |
| 230 options, args = parse_commandline() | 241 options, args = parse_commandline() |
| 231 revision = None # for future use cases | 242 revision = None # for future use cases |
| 232 publish_packages(options.config_file, options.track, revision, | 243 publish_packages(options.config_file, options.track, revision, |