# HG changeset patch # User Bjoern Ricks # Date 1280149130 0 # Node ID 25ef11a79d7fac2124c01f241cf8943234779e63 # Parent 4980f8d5014a50173f54748ea788039c1a185b4f incremental copying only changed debian packages diff -r 4980f8d5014a -r 25ef11a79d7f bin/publishdebianpackages.py --- a/bin/publishdebianpackages.py Mon Jul 26 10:12:40 2010 +0000 +++ b/bin/publishdebianpackages.py Mon Jul 26 12:58:50 2010 +0000 @@ -29,11 +29,13 @@ config_desc = ["distribution", "section", "num_newest", "build_user", "build_host", "build_listpackages", - "publish_user", "publish_host", "cachedb", + "publish_user", "publish_host", ("architectures", shlex.split, "armel i386 source"), ("after_upload_hook", shlex.split), ("publish_remove_old_packages", convert_bool), ("publish_dir", remove_trailing_slashes), + ("cachedb", + lambda s: expand_filename(remove_trailing_slashes(s))), ("cachedir", lambda s: expand_filename(remove_trailing_slashes(s)))] @@ -106,31 +108,36 @@ if a.trackname > b.trackname: return +1 return cmp(a.arch, b.arch) -def copy_files_to_destdir(destdir, files, quiet): +def copy_files_to_destdir(destdir, files, variables, quiet = False): scp_flags = [] if quiet: scp_flags.append("-q") if not os.path.exists(destdir): os.makedirs(destdir) + if files: + if variables["build_host"]: + userhost = "%(build_user)s@%(build_host)s:" % variables + files = [userhost + filename for filename in files] + # scp the packages to the cache dir + call(cmdexpand("scp -p @scp_flags @files $cachedir/", files=files, + scp_flags=scp_flags, cachedir=destdir)) - # scp the packages to the cache dir - call(cmdexpand("scp -p @scp_flags @files $cachedir/", files=files, - scp_flags=scp_flags, cachedir=destdir)) - -def remove_old_packages(cachedb, newpackages): +def remove_old_packages(cachedb, newpackages, quiet): newfiles = [package.filename for package in newpackages] oldpackages = cachedb.get_old_packages(newfiles) for package in oldpackages: # better check if the file really exists if os.path.isfile(package.filename): + if not quiet: + print "removing file %s" % package.filename os.remove(package.filename) cachedb.remove_packages(oldpackages) -def copy_packages_to_destdir(cachedb, dir, packages, quiet = False): +def copy_packages_to_destdir(cachedb, dir, packages, variables, quiet = False): packages.sort(cmp=sort_trackname_arch) package = packages[0] - trackname = package.name + trackname = package.trackname arch = package.arch destdir = os.path.join(dir, arch, trackname) files = [] @@ -138,7 +145,7 @@ cachedb.add_package(package) if package.trackname != trackname or \ package.arch != arch: - copy_files_to_destdir(destdir, files, quiet) + copy_files_to_destdir(destdir, files, variables, quiet) trackname = package.trackname arch = package.arch destdir = os.path.join(dir, arch, trackname) @@ -146,12 +153,17 @@ # add only to copy files list if the packages differ if check_package_is_new(package.name, destdir, package.md5sum): files.append(package.sourcepath) - print "copy package: %s, source: %s, destdir: %s" % (package.name, - package.sourcepath, destdir) - + if not quiet: + print "copy new file: %s" % package.name + elif not quiet: + print "file %s did not changed." % package.name + copy_files_to_destdir(destdir, files, variables, quiet) + def copy_to_cachedir(variables, track, revision, quiet = False, architectures=None): cachedir = variables["cachedir"] cachdebfilename = variables["cachedb"] + if not quiet: + print "using cachedb: %s" % cachdebfilename cachedb = CacheDb(cachdebfilename) newpackages = [] treepkginfo = get_treepkg_info(variables) @@ -199,13 +211,13 @@ binallarchs = (allowedarchs & allarchs) - sourcearch for packageinfo in binaryallpackages: for arch in binallarchs: - filename = os.path.join(cachedir, arch, track.name, + filename = os.path.join(cachedir, arch, packageinfo.trackname, packageinfo.name) newpackage = Package(filename, packageinfo.trackname, packageinfo.name, packageinfo.path, arch, get_md5sum(packageinfo)) newpackages.append(newpackage) - copy_packages_to_destdir(cachedb, cachedir, newpackages, quiet) - remove_old_packages(cachedb, newpackages) + copy_packages_to_destdir(cachedb, cachedir, newpackages, variables, quiet) + remove_old_packages(cachedb, newpackages, quiet) def publish_packages(config_filename, track, revision, dist, section, quiet): config = read_config(config_filename) @@ -215,16 +227,15 @@ if section is None: section = config["section"] - if not "cachedb" in config.keys(): - print >>sys.stderr, "Error. Please specifiy a cachedb in your config file" - sys.exit(1) - architectures = config["architectures"] copy_to_cachedir(config, track, revision, quiet, architectures) -# copy_to_publishdir(config, dist, section, arch, quiet) + copy_to_publishdir(config, dist, section, None, quiet) # update apt archive -# call(config["after_upload_hook"]) + if config["after_upload_hook"]: + if not quiet: + print "running after upload hook" + call(config["after_upload_hook"]) def main(): options, args = parse_commandline() diff -r 4980f8d5014a -r 25ef11a79d7f demopublishpackages.cfg --- a/demopublishpackages.cfg Mon Jul 26 10:12:40 2010 +0000 +++ b/demopublishpackages.cfg Mon Jul 26 12:58:50 2010 +0000 @@ -59,3 +59,4 @@ # ~user at the beginning of the value and environment variable # references of the form $VAR or ${VAR} are expanded. cachedir: /tmp/${LOGNAME}/treepkg-package-cache/%(build_host)s +cachedb: /tmp/${LOGNAME}/treepkg-package-cache/%(build_host)s-cache.db diff -r 4980f8d5014a -r 25ef11a79d7f treepkg/publish.py --- a/treepkg/publish.py Mon Jul 26 10:12:40 2010 +0000 +++ b/treepkg/publish.py Mon Jul 26 12:58:50 2010 +0000 @@ -32,8 +32,11 @@ prefix.extend(["ssh", "%s@%s" % (user, host)]) return prefix -def copy_to_publishdir(variables, dist, section, arch, quiet): - destdir = os.path.join(variables["publish_dir"], dist, section, arch) +def copy_to_publishdir(variables, dist, section, arch=None, quiet=False): + if not arch: + destdir = os.path.join(variables["publish_dir"], dist, section) + else: + destdir = os.path.join(variables["publish_dir"], dist, section, arch) remote_destdir = destdir if variables["publish_host"]: remote_destdir = (("%(publish_user)s@%(publish_host)s:" % variables)