changeset 416:4980f8d5014a treepkg-status

first version of incremental copying debian packages to cachedir
author Bjoern Ricks <bricks@intevation.de>
date Mon, 26 Jul 2010 10:12:40 +0000
parents 2d31eea39f70
children 25ef11a79d7f
files bin/publishdebianpackages.py test/test_info_data.py treepkg/info/data.py
diffstat 3 files changed, 70 insertions(+), 43 deletions(-) [+]
line wrap: on
line diff
--- a/bin/publishdebianpackages.py	Mon Jul 26 09:42:07 2010 +0000
+++ b/bin/publishdebianpackages.py	Mon Jul 26 10:12:40 2010 +0000
@@ -10,6 +10,7 @@
 """Publishes selected packages created by treepkg"""
 
 import os
+import os.path
 import sys
 import shlex
 
@@ -28,7 +29,7 @@
 
 config_desc = ["distribution", "section", "num_newest",
                "build_user", "build_host", "build_listpackages",
-               "publish_user", "publish_host",
+               "publish_user", "publish_host", "cachedb",
                ("architectures", shlex.split, "armel i386 source"),
                ("after_upload_hook", shlex.split),
                ("publish_remove_old_packages", convert_bool),
@@ -105,7 +106,7 @@
     if a.trackname > b.trackname: return +1
     return cmp(a.arch, b.arch)
 
-def copy_files(destdir, files, quiet):
+def copy_files_to_destdir(destdir, files, quiet):
     scp_flags = []
     if quiet:
         scp_flags.append("-q")
@@ -117,7 +118,16 @@
     call(cmdexpand("scp -p @scp_flags @files $cachedir/", files=files,
          scp_flags=scp_flags, cachedir=destdir))
 
-def copy_to_destdir(dir, packages, quiet = False):
+def remove_old_packages(cachedb, newpackages):
+    newfiles = [package.filename for package in newpackages]
+    oldpackages = cachedb.get_old_packages(newfiles)
+    for package in oldpackages:
+        # better check if the file really exists
+        if os.path.isfile(package.filename):
+            os.remove(package.filename)
+    cachedb.remove_packages(oldpackages)
+
+def copy_packages_to_destdir(cachedb, dir, packages, quiet = False):
     packages.sort(cmp=sort_trackname_arch)
     package = packages[0]
     trackname = package.name
@@ -125,20 +135,24 @@
     destdir = os.path.join(dir, arch, trackname)
     files = []
     for package in packages:
+        cachedb.add_package(package)
         if package.trackname != trackname or \
            package.arch != arch:
-            #copy_files(destdir, files, quiet)
+            copy_files_to_destdir(destdir, files, quiet)
             trackname = package.trackname
             arch = package.arch
             destdir = os.path.join(dir, arch, trackname)
             files = []
-        # add only if the packages differ
+        # add only to copy files list if the packages differ
         if check_package_is_new(package.name, destdir, package.md5sum):
             files.append(package.sourcepath)
-        print "package: %s, destdir: %s" % (package.name, destdir)
+            print "copy package: %s, source: %s,  destdir: %s" % (package.name,
+                        package.sourcepath, destdir)
 
-def copy_to_cache(variables, track, revision, quiet = False, architectures=None):
+def copy_to_cachedir(variables, track, revision, quiet = False, architectures=None):
     cachedir = variables["cachedir"]
+    cachdebfilename = variables["cachedb"]
+    cachedb = CacheDb(cachdebfilename)
     newpackages = []
     treepkginfo = get_treepkg_info(variables)
     #allowedarchs = set([]) # contains all wanted architectures (incl. source)
@@ -149,32 +163,31 @@
         allowedarchs = set([get_binary_arch(a) for a in architectures])
     else:
         allowedarchs = set([])
-    print "allowedarchs: %s" % allowedarchs
     for track in treepkginfo.tracks:
         for rev in track.revisions:
-            for package in rev.packages:
-                arch = get_binary_arch(package.arch)
-                if package.type == "binary":
+            for packageinfo in rev.packages:
+                arch = get_binary_arch(packageinfo.arch)
+                if packageinfo.type == "binary":
                     # skip other files
-                    if package.arch is None:
+                    if packageinfo.arch is None:
                         continue
                     # handle binary-all
                     if arch == "binary-all":
                         # add trackname for subdir name
-                        package.trackname = track.name
-                        binaryallpackages.append(package)
+                        packageinfo.trackname = track.name
+                        binaryallpackages.append(packageinfo)
                         continue
                     allarchs.add(arch)
-                elif package.type == "source":
-                    arch = package.type
-                print "package: %s %s" % (package.name, arch)
+                elif packageinfo.type == "source":
+                    arch = packageinfo.type
                 # only copy requested archs
                 if len(allowedarchs) == 0 or \
                    arch in allowedarchs:
                     filename = os.path.join(cachedir, arch, track.name,
-                                            package.name)
-                    newpackage = Package(filename, track.name, package.name,
-                                         package.path, arch, get_md5sum(package))
+                                            packageinfo.name)
+                    newpackage = Package(filename, track.name, packageinfo.name,
+                                         packageinfo.path, arch,
+                                         get_md5sum(packageinfo))
                     newpackages.append(newpackage)
     # copy binary-all packages
     sourcearch = set(["source"])
@@ -184,20 +197,15 @@
         binallarchs = allowedarchs - sourcearch
     else:
         binallarchs = (allowedarchs & allarchs) - sourcearch
-    for package in binaryallpackages:
+    for packageinfo in binaryallpackages:
         for arch in binallarchs:
             filename = os.path.join(cachedir, arch, track.name,
-                                    package.name)
-            newpackage = Package(filename, package.trackname, package.name,
-                                 package.path, arch, get_md5sum(package))
+                                    packageinfo.name)
+            newpackage = Package(filename, packageinfo.trackname, packageinfo.name,
+                                 packageinfo.path, arch, get_md5sum(packageinfo))
             newpackages.append(newpackage)
-    print newpackages
-    copy_to_destdir(cachedir, newpackages, quiet)
-
-def publish_packages_arch(variables, track, revision, dist, section, 
-                          quiet, architectures):
-    copy_to_cache(variables, track, revision, quiet, architectures)
-#    copy_to_publishdir(variables, dist, section, arch, quiet)
+    copy_packages_to_destdir(cachedb, cachedir, newpackages, quiet)
+    remove_old_packages(cachedb, newpackages)
 
 def publish_packages(config_filename, track, revision, dist, section, quiet):
     config = read_config(config_filename)
@@ -207,9 +215,13 @@
     if section is None:
         section = config["section"]
 
+    if not "cachedb" in config.keys():
+        print >>sys.stderr, "Error. Please specifiy a cachedb in your config file"
+        sys.exit(1)
+
     architectures = config["architectures"]
-    publish_packages_arch(config, track, revision, dist, section,
-                          quiet, architectures)
+    copy_to_cachedir(config, track, revision, quiet, architectures)
+#   copy_to_publishdir(config, dist, section, arch, quiet)
 
     # update apt archive
 #    call(config["after_upload_hook"])
--- a/test/test_info_data.py	Mon Jul 26 09:42:07 2010 +0000
+++ b/test/test_info_data.py	Mon Jul 26 10:12:40 2010 +0000
@@ -31,9 +31,18 @@
 
         dbfile = os.path.join(tmpdir, "cachedb2")
         db = CacheDb(dbfile)
+        # insert
+        filename = package.filename
         db.add_package(package)
+        package = db.get_package(filename)
+        self.assertEquals("1234567", package.md5sum)
+        # update
+        package.md5sum = "01234567"
+        db.add_package(package)
+        package = db.get_package(filename)
+        self.assertEquals("01234567", package.md5sum)
 
-        # test get_package and get_timestamp
+        # test get_package
         package2 = db.get_package(package.filename)
         self.assertNotEquals(None, package2)
         self.assertEquals(package.filename, package2.filename)
@@ -42,16 +51,21 @@
         self.assertEquals(package.arch, package2.arch)
         self.assertEquals(package.md5sum, package2.md5sum)
 
-        # test get_old_packages
         package3 = Package("/tmp/foo/foo_0.2.i386.deb", "foo",
-            "foo_0.2_i386.deb", "/tmp/foo/foo_0.2.i386.deb",
+            "foo_0.2_i386.deb", "/tmp/source/foo/foo_0.2.i386.deb",
             "binary-i386", "987654321")
+        package4 = Package("/tmp/john/doe_0.3.i386.deb", "john",
+            "doe_0.3.i386.deb", "/tmp/source/john/doe_0.3.i386.deb",
+            "binary-i386", "5671234")
         db.add_package(package3)
+        db.add_package(package4)
+
+        # test get_old_packages
         oldpackages = db.get_old_packages([package.filename])
-        self.assertEquals(1, len(oldpackages))
+        self.assertEquals(2, len(oldpackages))
         packages = db.get_packages()
-        self.assertEquals(2, len(packages))
-        db.remove_packages([package.filename for package in oldpackages])
+        self.assertEquals(3, len(packages))
+        db.remove_packages(oldpackages)
         packages = db.get_packages()
         self.assertEquals(1, len(packages))
 
--- a/treepkg/info/data.py	Mon Jul 26 09:42:07 2010 +0000
+++ b/treepkg/info/data.py	Mon Jul 26 10:12:40 2010 +0000
@@ -64,7 +64,7 @@
         self.conn.commit()
 
     def update_package(self, package):
-        UPDATE_TMPL = """UPDATE packages set md5sum = '?' 
+        UPDATE_TMPL = """UPDATE packages set md5sum = ? 
                          WHERE filename in (?)"""
         self.cursor.execute(UPDATE_TMPL, (package.md5sum, package.filename))
         self.conn.commit()
@@ -83,11 +83,12 @@
         self.cursor.execute(SELECT_TMPL % tmp, newfiles)
         return [Package(*row) for row in self.cursor.fetchall()]
 
-    def remove_packages(self, files):
+    def remove_packages(self, packages):
         DELET_TMPL = """DELETE FROM packages 
                      WHERE filename in (%s)"""
-        tmp = ", ".join(['?'] * len(files))
-        self.cursor.execute(DELET_TMPL % tmp, files)
+        tmp = ", ".join(['?'] * len(packages))
+        self.cursor.execute(DELET_TMPL % tmp,
+                            [package.filename for package in packages])
         self.conn.commit()
 
     def get_packages(self):
This site is hosted by Intevation GmbH (Datenschutzerklärung und Impressum | Privacy Policy and Imprint)