changeset 413:94a6ae627b31 treepkg-status

implemented CacheDb to store copied Packages from build host
author Bjoern Ricks <bricks@intevation.de>
date Fri, 23 Jul 2010 16:40:38 +0000
parents 58ecf7c0ecba
children e0539b483b04
files bin/publishdebianpackages.py bin/treepkginfo.py test/test_info.py treepkg/info/status.py
diffstat 4 files changed, 207 insertions(+), 32 deletions(-) [+]
line wrap: on
line diff
--- a/bin/publishdebianpackages.py	Thu Jul 22 16:01:43 2010 +0000
+++ b/bin/publishdebianpackages.py	Fri Jul 23 16:40:38 2010 +0000
@@ -12,6 +12,7 @@
 import os
 import sys
 import shlex
+import sqlite3
 
 from optparse import OptionParser
 from ConfigParser import SafeConfigParser
@@ -34,6 +35,91 @@
                ("cachedir",
                 lambda s: expand_filename(remove_trailing_slashes(s)))]
 
+class Package:
+    
+    def __init__(self, filename, trackname, packagename, packagepath, arch, md5sum):
+        self.filename = filename
+        self.trackname = trackname
+        self.name = packagename
+        self.sourcepath = packagepath
+        self.arch = arch
+        self.md5sum = md5sum
+
+
+class CacheDb:
+
+
+    def __init__(self, file):
+        self.SELECT_PACKAGE_TMPL = """SELECT * FROM packages 
+                                   WHERE filename = ?"""
+        self.file = file
+        self.conn = sqlite3.connect(file)
+        self.cursor = self.conn.cursor()
+        self.init_db(file)
+
+    def init_db(self, file):
+        self.cursor.execute(
+               """CREATE TABLE IF NOT EXISTS packages (
+                  filename TEXT PRIMARY KEY,
+                  trackname TEXT,
+                  packagename TEXT,
+                  sourcepath TEXT,
+                  arch TEXT, 
+                  md5sum TEXT )""")
+        self.conn.commit()
+
+
+    def add_package(self, package):
+        self.cursor.execute(self.SELECT_PACKAGE_TMPL, (package.filename,))
+        row = self.cursor.fetchone()
+        if not row:
+            self.insert_package(package)
+        else:
+            self.update_package(package)
+
+    def insert_package(self, package):
+        INSERT_TMPL = """INSERT INTO packages VALUES (
+                         ?, ?, ? ,? ,?, ?)"""
+        self.cursor.execute(INSERT_TMPL, (package.filename,
+                            package.trackname,
+                            package.name,
+                            package.sourcepath,
+                            package.arch,
+                            package.md5sum))
+        self.conn.commit()
+
+    def update_package(self, package):
+        UPDATE_TMPL = """UPDATE packages set md5sum = '?' 
+                         WHERE filename in (?)"""
+        self.cursor.execute(UPDATE_TMPL, (package.md5sum, package.filename))
+        self.conn.commit()
+
+    def get_package(self, filename):
+        self.cursor.execute(self.SELECT_PACKAGE_TMPL, (filename,))
+        row = self.cursor.fetchone()
+        if not row:
+            return None
+        return Package(row[0], row[1], row[2], row[3], row[4], row[5]) 
+
+    def get_old_packages(self, newfiles):
+        SELECT_TMPL = """SELECT * FROM packages 
+                      WHERE filename not in (%s)"""
+        tmp = ", ".join(['?'] * len(newfiles))
+        self.cursor.execute(SELECT_TMPL % tmp, newfiles)
+        return [Package(*row) for row in self.cursor.fetchall()]
+
+    def remove_packages(self, files):
+        DELET_TMPL = """DELETE FROM packages 
+                     WHERE filename in (%s)"""
+        tmp = ", ".join(['?'] * len(files))
+        self.cursor.execute(DELET_TMPL % tmp, files)
+        self.conn.commit()
+
+    def get_packages(self):
+        SELECT_TMPL = "SELECT * FROM packages" 
+        self.cursor.execute(SELECT_TMPL)
+        return [Package(*row) for row in self.cursor.fetchall()]
+
 def read_config(filename):
     if not os.path.exists(filename):
         print >>sys.stderr, "Config file %s does not exist" % filename
@@ -70,8 +156,7 @@
                                           variables["build_host"])
     xml = capture_output(cmdexpand("@runremote $build_listpackages"
                                      " --newest=$num_newest",
-                                     #runremote=runremote,
-                                     runremote="",
+                                     runremote=runremote,
                                      **variables))
     return TreepkgInfo.fromxml(xml)
 
@@ -88,26 +173,55 @@
     destmd5sum = md5sum(destpackage)
     return (destmd5sum != packagemd5sum)
 
-def copy_to_destdir(package, dir, arch, subdir, quiet = False):
+def get_md5sum(packageinfo):
+    md5sum = ""
+    if packageinfo:
+        for checksum in packageinfo.checksums:
+            if checksum.type == "md5":
+                md5sum = checksum.checksum
+                break
+    return md5sum
+
+def sort_trackname_arch(a, b):
+    if a.trackname < b.trackname: return -1
+    if a.trackname > b.trackname: return +1
+    return cmp(a.arch, b.arch)
+
+def copy_files(destdir, files, quiet):
     scp_flags = []
     if quiet:
         scp_flags.append("-q")
-    destdir = os.path.join(dir, arch, subdir)
-    print "package: %s, destdir: %s" % (package.name, destdir)
-    md5sum = ""
-    for checksum in package.checksums:
-        if checksum.type == "md5":
-            md5sum = checksum.checksum
-            break
-    # update only if the packages differ
-    if check_package_is_new(package.name, destdir, md5sum):
-        pass
-        # scp the packages to the cache dir 
-        #call(cmdexpand("scp -p @scp_flags $file $cachedir/", file=package.path,
-        #     scp_flags=scp_flags, cachedir=destdir))
+
+    if not os.path.exists(destdir):
+        os.makedirs(destdir)
+
+    # scp the packages to the cache dir 
+    call(cmdexpand("scp -p @scp_flags @files $cachedir/", files=files,
+         scp_flags=scp_flags, cachedir=destdir))
+
+def copy_to_destdir(dir, packages, quiet = False):
+    packages.sort(cmp=sort_trackname_arch)
+    package = packages[0]
+    trackname = package.name
+    arch = package.arch
+    destdir = os.path.join(dir, arch, trackname)
+    files = []
+    for package in packages:
+        if package.trackname != trackname or \
+           package.arch != arch:
+            #copy_files(destdir, files, quiet)
+            trackname = package.trackname
+            arch = package.arch
+            destdir = os.path.join(dir, arch, trackname)
+            files = []
+        # add only if the packages differ
+        if check_package_is_new(package.name, destdir, package.md5sum):
+            files.append(package.sourcepath)
+        print "package: %s, destdir: %s" % (package.name, destdir)
 
 def copy_to_cache(variables, track, revision, quiet = False, architectures=None):
     cachedir = variables["cachedir"]
+    newpackages = []
     treepkginfo = get_treepkg_info(variables)
     #allowedarchs = set([]) # contains all wanted architectures (incl. source)
     allarchs = set([]) # contains all present architectures (incl. source)
@@ -117,12 +231,11 @@
         allowedarchs = set([get_binary_arch(a) for a in architectures])
     else:
         allowedarchs = set([])
-   
+    print "allowedarchs: %s" % allowedarchs
     for track in treepkginfo.tracks:
         for rev in track.revisions:
             for package in rev.packages:
                 arch = get_binary_arch(package.arch)
-                print "%s %s %s %s" % (track.name, package.name, package.type, package.arch)
                 if package.type == "binary":
                     # skip other files
                     if package.arch is None:
@@ -136,15 +249,15 @@
                     allarchs.add(arch)
                 elif package.type == "source":
                     arch = package.type
+                print "package: %s %s" % (package.name, arch)
                 # only copy requested archs
                 if len(allowedarchs) == 0 or \
                    arch in allowedarchs:
-                    copy_to_destdir(package, cachedir, arch, track.name,
-                                    quiet)
-
-    #print "architectures %s" % architectures
-    #print "allowed %s" % allowedarchs
-    #print "all %s" % allarchs
+                    filename = os.path.join(cachedir, arch, track.name,
+                                            package.name)
+                    newpackage = Package(filename, track.name, package.name,
+                                         package.path, arch, get_md5sum(package))
+                    newpackages.append(newpackage)
     # copy binary-all packages
     sourcearch = set(["source"])
     if len(allowedarchs) == 0:
@@ -153,10 +266,15 @@
         binallarchs = allowedarchs - sourcearch
     else:
         binallarchs = (allowedarchs & allarchs) - sourcearch
-    #print "binallarcgs %s" % binallarchs
     for package in binaryallpackages:
         for arch in binallarchs:
-            copy_to_destdir(package, cachedir, arch, package.trackname, quiet)
+            filename = os.path.join(cachedir, arch, track.name,
+                                    package.name)
+            newpackage = Package(filename, package.trackname, package.name,
+                                 package.path, arch, get_md5sum(package))
+            newpackages.append(newpackage)
+    print newpackages
+    copy_to_destdir(cachedir, newpackages, quiet)
 
 def publish_packages_arch(variables, track, revision, dist, section, 
                           quiet, architectures):
--- a/bin/treepkginfo.py	Thu Jul 22 16:01:43 2010 +0000
+++ b/bin/treepkginfo.py	Fri Jul 23 16:40:38 2010 +0000
@@ -19,6 +19,8 @@
     parser.set_defaults(newest=-1)
     parser.add_option("--newest", type="int",
                       help=("Number of newest revisions to list."))
+    parser.add_option("--only-successful", action="store_true",
+                      help=("Show only successful builds", dest="success"))
     parser.add_option("--pretty", action="store_true", help=("Show XML pretty"
                       " output."))
  
@@ -28,7 +30,8 @@
 def main():
     options, args = parse_commandline()
 
-    tpkginfo = status.TreepkgInfo(options.config_file, options.newest)
+    tpkginfo = status.TreepkgInfo(options.config_file, options.newest,
+                                  option.success)
 
     if options.pretty:
         print tpkginfo.toxml().toprettyxml()
--- a/test/test_info.py	Thu Jul 22 16:01:43 2010 +0000
+++ b/test/test_info.py	Fri Jul 23 16:40:38 2010 +0000
@@ -13,11 +13,16 @@
 
 test_dir = os.path.dirname(__file__)
 sys.path.append(os.path.join(test_dir, os.pardir))
+sys.path.append(os.path.join(test_dir, os.pardir, "bin"))
 
 from treepkg.info.status import *
 from treepkg.report import get_packager_group
 from filesupport import FileTestMixin
 
+from publishdebianpackages import get_binary_arch
+from publishdebianpackages import Package
+from publishdebianpackages import CacheDb
+
 class TreepkgInfoTest(unittest.TestCase, FileTestMixin):
     config_contents = """\
 [DEFAULT]
@@ -69,9 +74,8 @@
     def test_createinfo(self):
         config_file = os.path.join(self.directory, "treepkg.cfg")
         tpkginfo = TreepkgInfo(config_file)
-        print tpkginfo.toxml().toxml()
-        tpkgroot = tpkginfo.tpkgroot
-        self.assertEquals("testtreepkginfo", tpkgroot.name)
+        tpkgrootinfo = tpkginfo.tpkgroot.info
+        self.assertEquals("testtreepkginfo", tpkgrootinfo.name)
  
 
 class TreepkgRootInfoTest(unittest.TestCase):
@@ -80,7 +84,52 @@
         status = TreepkgRootInfo("testtreepkg")
         dom = status.toxml()
         xml = dom.toxml()
-        self.assertEquals("<status><name>testtreepkg</name></status>", xml)
+        self.assertEquals("<info><name>testtreepkg</name></info>", xml)
+
+class TestPublishDebianPackages(unittest.TestCase, FileTestMixin):
+
+    def test_get_binary_arch(self):
+        source = get_binary_arch("source")
+        self.assertEquals("source", source)
+        binary_armel = get_binary_arch("armel")
+        self.assertEquals("binary-armel", binary_armel)
+        binary_armel = get_binary_arch("binary-armel")
+        self.assertEquals("binary-armel", binary_armel)
+
+    def test_cache_db(self):
+        tmpdir = self.create_test_specific_temp_dir()
+        package = Package("/tmp/abc/abc_0.1_i386.deb", "abc",
+            "abc_0.1_i386.deb", "/source/abc/abc_0.1_i386.deb", 
+            "binary-i386", "1234567")
+#        tmpfile = self.create_temp_file("cachedb1", "abc")
+#        with self.assertRaises(Exception):
+#            db = CacheDb(tmpfile)
+
+        dbfile = os.path.join(tmpdir, "cachedb2")
+        db = CacheDb(dbfile)
+        db.add_package(package)
+
+        # test get_package and get_timestamp
+        package2 = db.get_package(package.filename)
+        self.assertNotEquals(None, package2)
+        self.assertEquals(package.filename, package2.filename)
+        self.assertEquals(package.name, package2.name)
+        self.assertEquals(package.sourcepath, package2.sourcepath)
+        self.assertEquals(package.arch, package2.arch)
+        self.assertEquals(package.md5sum, package2.md5sum)
+
+        # test get_old_packages
+        package3 = Package("/tmp/foo/foo_0.2.i386.deb", "foo",
+            "foo_0.2_i386.deb", "/tmp/foo/foo_0.2.i386.deb",
+            "binary-i386", "987654321")
+        db.add_package(package3)
+        oldpackages = db.get_old_packages([package.filename])
+        self.assertEquals(1, len(oldpackages))
+        packages = db.get_packages()
+        self.assertEquals(2, len(packages))
+        db.remove_packages([package.filename for package in oldpackages])
+        packages = db.get_packages()
+        self.assertEquals(1, len(packages))
 
 if __name__ == '__main__':
     unittest.main()
--- a/treepkg/info/status.py	Thu Jul 22 16:01:43 2010 +0000
+++ b/treepkg/info/status.py	Fri Jul 23 16:40:38 2010 +0000
@@ -61,8 +61,9 @@
 
 class TreepkgInfo:
 
-    def __init__(self, config, numnewestrev=-1):
+    def __init__(self, config, numnewestrev=-1, success=False):
         self.numnewestrev = numnewestrev
+        self.success = success
         group = get_packager_group(config)
         treepkgrootinfo = TreepkgRootInfo(group.name, group.treepkg_dir, 
                                         group.tracks_dir)
@@ -96,6 +97,10 @@
         arch = None
 
         for rev in revisions:
+            if self.success: # skip not successful builds
+                if not rev.status.status.finished \
+                    or rev.status.status.error:
+                    continue
             revision = rev.revision
             rules_revision = rev.rules_revision
             status  = rev.status.status.description  # extend status
This site is hosted by Intevation GmbH (Datenschutzerklärung und Impressum | Privacy Policy and Imprint)