changeset 417:25ef11a79d7f treepkg-status

incremental copying only changed debian packages
author Bjoern Ricks <bricks@intevation.de>
date Mon, 26 Jul 2010 12:58:50 +0000
parents 4980f8d5014a
children acb8dbd506f6
files bin/publishdebianpackages.py demopublishpackages.cfg treepkg/publish.py
diffstat 3 files changed, 39 insertions(+), 24 deletions(-) [+]
line wrap: on
line diff
--- a/bin/publishdebianpackages.py	Mon Jul 26 10:12:40 2010 +0000
+++ b/bin/publishdebianpackages.py	Mon Jul 26 12:58:50 2010 +0000
@@ -29,11 +29,13 @@
 
 config_desc = ["distribution", "section", "num_newest",
                "build_user", "build_host", "build_listpackages",
-               "publish_user", "publish_host", "cachedb",
+               "publish_user", "publish_host", 
                ("architectures", shlex.split, "armel i386 source"),
                ("after_upload_hook", shlex.split),
                ("publish_remove_old_packages", convert_bool),
                ("publish_dir", remove_trailing_slashes),
+               ("cachedb",
+                lambda s: expand_filename(remove_trailing_slashes(s))),
                ("cachedir",
                 lambda s: expand_filename(remove_trailing_slashes(s)))]
 
@@ -106,31 +108,36 @@
     if a.trackname > b.trackname: return +1
     return cmp(a.arch, b.arch)
 
-def copy_files_to_destdir(destdir, files, quiet):
+def copy_files_to_destdir(destdir, files, variables, quiet = False):
     scp_flags = []
     if quiet:
         scp_flags.append("-q")
 
     if not os.path.exists(destdir):
         os.makedirs(destdir)
+    if files:
+        if variables["build_host"]:
+            userhost = "%(build_user)s@%(build_host)s:" % variables
+            files = [userhost + filename for filename in files]
+        # scp the packages to the cache dir 
+        call(cmdexpand("scp -p @scp_flags @files $cachedir/", files=files,
+                        scp_flags=scp_flags, cachedir=destdir))
 
-    # scp the packages to the cache dir 
-    call(cmdexpand("scp -p @scp_flags @files $cachedir/", files=files,
-         scp_flags=scp_flags, cachedir=destdir))
-
-def remove_old_packages(cachedb, newpackages):
+def remove_old_packages(cachedb, newpackages, quiet):
     newfiles = [package.filename for package in newpackages]
     oldpackages = cachedb.get_old_packages(newfiles)
     for package in oldpackages:
         # better check if the file really exists
         if os.path.isfile(package.filename):
+            if not quiet:
+                print "removing file %s" % package.filename
             os.remove(package.filename)
     cachedb.remove_packages(oldpackages)
 
-def copy_packages_to_destdir(cachedb, dir, packages, quiet = False):
+def copy_packages_to_destdir(cachedb, dir, packages, variables, quiet = False):
     packages.sort(cmp=sort_trackname_arch)
     package = packages[0]
-    trackname = package.name
+    trackname = package.trackname
     arch = package.arch
     destdir = os.path.join(dir, arch, trackname)
     files = []
@@ -138,7 +145,7 @@
         cachedb.add_package(package)
         if package.trackname != trackname or \
            package.arch != arch:
-            copy_files_to_destdir(destdir, files, quiet)
+            copy_files_to_destdir(destdir, files, variables, quiet)
             trackname = package.trackname
             arch = package.arch
             destdir = os.path.join(dir, arch, trackname)
@@ -146,12 +153,17 @@
         # add only to copy files list if the packages differ
         if check_package_is_new(package.name, destdir, package.md5sum):
             files.append(package.sourcepath)
-            print "copy package: %s, source: %s,  destdir: %s" % (package.name,
-                        package.sourcepath, destdir)
-
+            if not quiet:
+                print "copy new file: %s" % package.name
+        elif not quiet:
+            print "file %s did not changed." % package.name
+    copy_files_to_destdir(destdir, files, variables, quiet)
+           
 def copy_to_cachedir(variables, track, revision, quiet = False, architectures=None):
     cachedir = variables["cachedir"]
     cachdebfilename = variables["cachedb"]
+    if not quiet:
+        print "using cachedb: %s" % cachdebfilename
     cachedb = CacheDb(cachdebfilename)
     newpackages = []
     treepkginfo = get_treepkg_info(variables)
@@ -199,13 +211,13 @@
         binallarchs = (allowedarchs & allarchs) - sourcearch
     for packageinfo in binaryallpackages:
         for arch in binallarchs:
-            filename = os.path.join(cachedir, arch, track.name,
+            filename = os.path.join(cachedir, arch, packageinfo.trackname,
                                     packageinfo.name)
             newpackage = Package(filename, packageinfo.trackname, packageinfo.name,
                                  packageinfo.path, arch, get_md5sum(packageinfo))
             newpackages.append(newpackage)
-    copy_packages_to_destdir(cachedb, cachedir, newpackages, quiet)
-    remove_old_packages(cachedb, newpackages)
+    copy_packages_to_destdir(cachedb, cachedir, newpackages, variables, quiet)
+    remove_old_packages(cachedb, newpackages, quiet)
 
 def publish_packages(config_filename, track, revision, dist, section, quiet):
     config = read_config(config_filename)
@@ -215,16 +227,15 @@
     if section is None:
         section = config["section"]
 
-    if not "cachedb" in config.keys():
-        print >>sys.stderr, "Error. Please specifiy a cachedb in your config file"
-        sys.exit(1)
-
     architectures = config["architectures"]
     copy_to_cachedir(config, track, revision, quiet, architectures)
-#   copy_to_publishdir(config, dist, section, arch, quiet)
+    copy_to_publishdir(config, dist, section, None, quiet)
 
     # update apt archive
-#    call(config["after_upload_hook"])
+    if config["after_upload_hook"]:
+        if not quiet:
+            print "running after upload hook"
+        call(config["after_upload_hook"])
 
 def main():
     options, args = parse_commandline()
--- a/demopublishpackages.cfg	Mon Jul 26 10:12:40 2010 +0000
+++ b/demopublishpackages.cfg	Mon Jul 26 12:58:50 2010 +0000
@@ -59,3 +59,4 @@
 # ~user at the beginning of the value and environment variable
 # references of the form $VAR or ${VAR} are expanded.
 cachedir: /tmp/${LOGNAME}/treepkg-package-cache/%(build_host)s
+cachedb: /tmp/${LOGNAME}/treepkg-package-cache/%(build_host)s-cache.db
--- a/treepkg/publish.py	Mon Jul 26 10:12:40 2010 +0000
+++ b/treepkg/publish.py	Mon Jul 26 12:58:50 2010 +0000
@@ -32,8 +32,11 @@
         prefix.extend(["ssh", "%s@%s" % (user, host)])
     return prefix
 
-def copy_to_publishdir(variables, dist, section, arch, quiet):
-    destdir = os.path.join(variables["publish_dir"], dist, section, arch)
+def copy_to_publishdir(variables, dist, section, arch=None, quiet=False):
+    if not arch:
+        destdir = os.path.join(variables["publish_dir"], dist, section)
+    else:
+        destdir = os.path.join(variables["publish_dir"], dist, section, arch)
     remote_destdir = destdir
     if variables["publish_host"]:
         remote_destdir = (("%(publish_user)s@%(publish_host)s:" % variables)
This site is hosted by Intevation GmbH (Datenschutzerklärung und Impressum | Privacy Policy and Imprint)