Mercurial > treepkg
changeset 413:94a6ae627b31 treepkg-status
implemented CacheDb to store copied Packages from build host
| author | Bjoern Ricks <bricks@intevation.de> |
|---|---|
| date | Fri, 23 Jul 2010 16:40:38 +0000 |
| parents | 58ecf7c0ecba |
| children | e0539b483b04 |
| files | bin/publishdebianpackages.py bin/treepkginfo.py test/test_info.py treepkg/info/status.py |
| diffstat | 4 files changed, 207 insertions(+), 32 deletions(-) [+] |
line wrap: on
line diff
--- a/bin/publishdebianpackages.py Thu Jul 22 16:01:43 2010 +0000 +++ b/bin/publishdebianpackages.py Fri Jul 23 16:40:38 2010 +0000 @@ -12,6 +12,7 @@ import os import sys import shlex +import sqlite3 from optparse import OptionParser from ConfigParser import SafeConfigParser @@ -34,6 +35,91 @@ ("cachedir", lambda s: expand_filename(remove_trailing_slashes(s)))] +class Package: + + def __init__(self, filename, trackname, packagename, packagepath, arch, md5sum): + self.filename = filename + self.trackname = trackname + self.name = packagename + self.sourcepath = packagepath + self.arch = arch + self.md5sum = md5sum + + +class CacheDb: + + + def __init__(self, file): + self.SELECT_PACKAGE_TMPL = """SELECT * FROM packages + WHERE filename = ?""" + self.file = file + self.conn = sqlite3.connect(file) + self.cursor = self.conn.cursor() + self.init_db(file) + + def init_db(self, file): + self.cursor.execute( + """CREATE TABLE IF NOT EXISTS packages ( + filename TEXT PRIMARY KEY, + trackname TEXT, + packagename TEXT, + sourcepath TEXT, + arch TEXT, + md5sum TEXT )""") + self.conn.commit() + + + def add_package(self, package): + self.cursor.execute(self.SELECT_PACKAGE_TMPL, (package.filename,)) + row = self.cursor.fetchone() + if not row: + self.insert_package(package) + else: + self.update_package(package) + + def insert_package(self, package): + INSERT_TMPL = """INSERT INTO packages VALUES ( + ?, ?, ? ,? ,?, ?)""" + self.cursor.execute(INSERT_TMPL, (package.filename, + package.trackname, + package.name, + package.sourcepath, + package.arch, + package.md5sum)) + self.conn.commit() + + def update_package(self, package): + UPDATE_TMPL = """UPDATE packages set md5sum = '?' + WHERE filename in (?)""" + self.cursor.execute(UPDATE_TMPL, (package.md5sum, package.filename)) + self.conn.commit() + + def get_package(self, filename): + self.cursor.execute(self.SELECT_PACKAGE_TMPL, (filename,)) + row = self.cursor.fetchone() + if not row: + return None + return Package(row[0], row[1], row[2], row[3], row[4], row[5]) + + def get_old_packages(self, newfiles): + SELECT_TMPL = """SELECT * FROM packages + WHERE filename not in (%s)""" + tmp = ", ".join(['?'] * len(newfiles)) + self.cursor.execute(SELECT_TMPL % tmp, newfiles) + return [Package(*row) for row in self.cursor.fetchall()] + + def remove_packages(self, files): + DELET_TMPL = """DELETE FROM packages + WHERE filename in (%s)""" + tmp = ", ".join(['?'] * len(files)) + self.cursor.execute(DELET_TMPL % tmp, files) + self.conn.commit() + + def get_packages(self): + SELECT_TMPL = "SELECT * FROM packages" + self.cursor.execute(SELECT_TMPL) + return [Package(*row) for row in self.cursor.fetchall()] + def read_config(filename): if not os.path.exists(filename): print >>sys.stderr, "Config file %s does not exist" % filename @@ -70,8 +156,7 @@ variables["build_host"]) xml = capture_output(cmdexpand("@runremote $build_listpackages" " --newest=$num_newest", - #runremote=runremote, - runremote="", + runremote=runremote, **variables)) return TreepkgInfo.fromxml(xml) @@ -88,26 +173,55 @@ destmd5sum = md5sum(destpackage) return (destmd5sum != packagemd5sum) -def copy_to_destdir(package, dir, arch, subdir, quiet = False): +def get_md5sum(packageinfo): + md5sum = "" + if packageinfo: + for checksum in packageinfo.checksums: + if checksum.type == "md5": + md5sum = checksum.checksum + break + return md5sum + +def sort_trackname_arch(a, b): + if a.trackname < b.trackname: return -1 + if a.trackname > b.trackname: return +1 + return cmp(a.arch, b.arch) + +def copy_files(destdir, files, quiet): scp_flags = [] if quiet: scp_flags.append("-q") - destdir = os.path.join(dir, arch, subdir) - print "package: %s, destdir: %s" % (package.name, destdir) - md5sum = "" - for checksum in package.checksums: - if checksum.type == "md5": - md5sum = checksum.checksum - break - # update only if the packages differ - if check_package_is_new(package.name, destdir, md5sum): - pass - # scp the packages to the cache dir - #call(cmdexpand("scp -p @scp_flags $file $cachedir/", file=package.path, - # scp_flags=scp_flags, cachedir=destdir)) + + if not os.path.exists(destdir): + os.makedirs(destdir) + + # scp the packages to the cache dir + call(cmdexpand("scp -p @scp_flags @files $cachedir/", files=files, + scp_flags=scp_flags, cachedir=destdir)) + +def copy_to_destdir(dir, packages, quiet = False): + packages.sort(cmp=sort_trackname_arch) + package = packages[0] + trackname = package.name + arch = package.arch + destdir = os.path.join(dir, arch, trackname) + files = [] + for package in packages: + if package.trackname != trackname or \ + package.arch != arch: + #copy_files(destdir, files, quiet) + trackname = package.trackname + arch = package.arch + destdir = os.path.join(dir, arch, trackname) + files = [] + # add only if the packages differ + if check_package_is_new(package.name, destdir, package.md5sum): + files.append(package.sourcepath) + print "package: %s, destdir: %s" % (package.name, destdir) def copy_to_cache(variables, track, revision, quiet = False, architectures=None): cachedir = variables["cachedir"] + newpackages = [] treepkginfo = get_treepkg_info(variables) #allowedarchs = set([]) # contains all wanted architectures (incl. source) allarchs = set([]) # contains all present architectures (incl. source) @@ -117,12 +231,11 @@ allowedarchs = set([get_binary_arch(a) for a in architectures]) else: allowedarchs = set([]) - + print "allowedarchs: %s" % allowedarchs for track in treepkginfo.tracks: for rev in track.revisions: for package in rev.packages: arch = get_binary_arch(package.arch) - print "%s %s %s %s" % (track.name, package.name, package.type, package.arch) if package.type == "binary": # skip other files if package.arch is None: @@ -136,15 +249,15 @@ allarchs.add(arch) elif package.type == "source": arch = package.type + print "package: %s %s" % (package.name, arch) # only copy requested archs if len(allowedarchs) == 0 or \ arch in allowedarchs: - copy_to_destdir(package, cachedir, arch, track.name, - quiet) - - #print "architectures %s" % architectures - #print "allowed %s" % allowedarchs - #print "all %s" % allarchs + filename = os.path.join(cachedir, arch, track.name, + package.name) + newpackage = Package(filename, track.name, package.name, + package.path, arch, get_md5sum(package)) + newpackages.append(newpackage) # copy binary-all packages sourcearch = set(["source"]) if len(allowedarchs) == 0: @@ -153,10 +266,15 @@ binallarchs = allowedarchs - sourcearch else: binallarchs = (allowedarchs & allarchs) - sourcearch - #print "binallarcgs %s" % binallarchs for package in binaryallpackages: for arch in binallarchs: - copy_to_destdir(package, cachedir, arch, package.trackname, quiet) + filename = os.path.join(cachedir, arch, track.name, + package.name) + newpackage = Package(filename, package.trackname, package.name, + package.path, arch, get_md5sum(package)) + newpackages.append(newpackage) + print newpackages + copy_to_destdir(cachedir, newpackages, quiet) def publish_packages_arch(variables, track, revision, dist, section, quiet, architectures):
--- a/bin/treepkginfo.py Thu Jul 22 16:01:43 2010 +0000 +++ b/bin/treepkginfo.py Fri Jul 23 16:40:38 2010 +0000 @@ -19,6 +19,8 @@ parser.set_defaults(newest=-1) parser.add_option("--newest", type="int", help=("Number of newest revisions to list.")) + parser.add_option("--only-successful", action="store_true", + help=("Show only successful builds", dest="success")) parser.add_option("--pretty", action="store_true", help=("Show XML pretty" " output.")) @@ -28,7 +30,8 @@ def main(): options, args = parse_commandline() - tpkginfo = status.TreepkgInfo(options.config_file, options.newest) + tpkginfo = status.TreepkgInfo(options.config_file, options.newest, + option.success) if options.pretty: print tpkginfo.toxml().toprettyxml()
--- a/test/test_info.py Thu Jul 22 16:01:43 2010 +0000 +++ b/test/test_info.py Fri Jul 23 16:40:38 2010 +0000 @@ -13,11 +13,16 @@ test_dir = os.path.dirname(__file__) sys.path.append(os.path.join(test_dir, os.pardir)) +sys.path.append(os.path.join(test_dir, os.pardir, "bin")) from treepkg.info.status import * from treepkg.report import get_packager_group from filesupport import FileTestMixin +from publishdebianpackages import get_binary_arch +from publishdebianpackages import Package +from publishdebianpackages import CacheDb + class TreepkgInfoTest(unittest.TestCase, FileTestMixin): config_contents = """\ [DEFAULT] @@ -69,9 +74,8 @@ def test_createinfo(self): config_file = os.path.join(self.directory, "treepkg.cfg") tpkginfo = TreepkgInfo(config_file) - print tpkginfo.toxml().toxml() - tpkgroot = tpkginfo.tpkgroot - self.assertEquals("testtreepkginfo", tpkgroot.name) + tpkgrootinfo = tpkginfo.tpkgroot.info + self.assertEquals("testtreepkginfo", tpkgrootinfo.name) class TreepkgRootInfoTest(unittest.TestCase): @@ -80,7 +84,52 @@ status = TreepkgRootInfo("testtreepkg") dom = status.toxml() xml = dom.toxml() - self.assertEquals("<status><name>testtreepkg</name></status>", xml) + self.assertEquals("<info><name>testtreepkg</name></info>", xml) + +class TestPublishDebianPackages(unittest.TestCase, FileTestMixin): + + def test_get_binary_arch(self): + source = get_binary_arch("source") + self.assertEquals("source", source) + binary_armel = get_binary_arch("armel") + self.assertEquals("binary-armel", binary_armel) + binary_armel = get_binary_arch("binary-armel") + self.assertEquals("binary-armel", binary_armel) + + def test_cache_db(self): + tmpdir = self.create_test_specific_temp_dir() + package = Package("/tmp/abc/abc_0.1_i386.deb", "abc", + "abc_0.1_i386.deb", "/source/abc/abc_0.1_i386.deb", + "binary-i386", "1234567") +# tmpfile = self.create_temp_file("cachedb1", "abc") +# with self.assertRaises(Exception): +# db = CacheDb(tmpfile) + + dbfile = os.path.join(tmpdir, "cachedb2") + db = CacheDb(dbfile) + db.add_package(package) + + # test get_package and get_timestamp + package2 = db.get_package(package.filename) + self.assertNotEquals(None, package2) + self.assertEquals(package.filename, package2.filename) + self.assertEquals(package.name, package2.name) + self.assertEquals(package.sourcepath, package2.sourcepath) + self.assertEquals(package.arch, package2.arch) + self.assertEquals(package.md5sum, package2.md5sum) + + # test get_old_packages + package3 = Package("/tmp/foo/foo_0.2.i386.deb", "foo", + "foo_0.2_i386.deb", "/tmp/foo/foo_0.2.i386.deb", + "binary-i386", "987654321") + db.add_package(package3) + oldpackages = db.get_old_packages([package.filename]) + self.assertEquals(1, len(oldpackages)) + packages = db.get_packages() + self.assertEquals(2, len(packages)) + db.remove_packages([package.filename for package in oldpackages]) + packages = db.get_packages() + self.assertEquals(1, len(packages)) if __name__ == '__main__': unittest.main()
--- a/treepkg/info/status.py Thu Jul 22 16:01:43 2010 +0000 +++ b/treepkg/info/status.py Fri Jul 23 16:40:38 2010 +0000 @@ -61,8 +61,9 @@ class TreepkgInfo: - def __init__(self, config, numnewestrev=-1): + def __init__(self, config, numnewestrev=-1, success=False): self.numnewestrev = numnewestrev + self.success = success group = get_packager_group(config) treepkgrootinfo = TreepkgRootInfo(group.name, group.treepkg_dir, group.tracks_dir) @@ -96,6 +97,10 @@ arch = None for rev in revisions: + if self.success: # skip not successful builds + if not rev.status.status.finished \ + or rev.status.status.error: + continue revision = rev.revision rules_revision = rev.rules_revision status = rev.status.status.description # extend status