# HG changeset patch # User Bjoern Ricks # Date 1281102114 0 # Node ID eadcb1bb54e2c55652101c36c8b919a3cd73f11d # Parent 383d23c6bfdecfe2404b4bf0d87954d2dd1852a4# Parent 8e0c81870e5eedcf7aea582d6862c3f798204968 Merged revisions 388-399 via svnmerge from svn+ssh://svn.wald.intevation.org/treepkg/branches/treepkg-status ........ r388 | bricks | 2010-07-28 10:38:50 +0200 (Mi, 28 Jul 2010) | 3 lines fix if statement upload hook should be called if it's not empty ........ r389 | bricks | 2010-07-28 13:38:00 +0200 (Mi, 28 Jul 2010) | 2 lines only rsync specified architectures from cachedir to publishdir ........ r390 | bricks | 2010-07-29 14:07:57 +0200 (Do, 29 Jul 2010) | 2 lines introduced after_copy_hook and made rsync upload to publishdir optional ........ r391 | bricks | 2010-07-29 15:43:39 +0200 (Do, 29 Jul 2010) | 3 lines publish package must be default to get original behaviour be more verbose at publishing ........ r392 | bricks | 2010-07-29 16:50:53 +0200 (Do, 29 Jul 2010) | 2 lines fix handling of showing only successfull builds together with num packages ........ r393 | bricks | 2010-07-29 16:52:42 +0200 (Do, 29 Jul 2010) | 2 lines fixed typo ........ r394 | bricks | 2010-07-29 16:59:10 +0200 (Do, 29 Jul 2010) | 2 lines fixed another typo ........ r395 | bricks | 2010-07-29 18:05:54 +0200 (Do, 29 Jul 2010) | 3 lines fixed another typo treepkg is really slow with many revisions ........ r396 | bricks | 2010-08-02 12:41:55 +0200 (Mo, 02 Aug 2010) | 2 lines fixed typos ........ r397 | bricks | 2010-08-02 17:36:12 +0200 (Mo, 02 Aug 2010) | 3 lines be more error prone in listing different files not checking if a dir exists caused several errors if a build wasn't successful ........ r398 | bricks | 2010-08-05 18:21:47 +0200 (Do, 05 Aug 2010) | 2 lines review changes ........ r399 | bricks | 2010-08-06 13:06:08 +0200 (Fr, 06 Aug 2010) | 4 lines cleanup modules fix test_info testcases added testcases for remove_trailingslashes and expand_filename ........ diff -r 383d23c6bfde -r eadcb1bb54e2 bin/publishdebianpackages.py --- a/bin/publishdebianpackages.py Fri Aug 06 13:28:47 2010 +0000 +++ b/bin/publishdebianpackages.py Fri Aug 06 13:41:54 2010 +0000 @@ -10,8 +10,6 @@ """Publishes selected packages created by treepkg""" import os -import os.path -import re import sys import shlex @@ -22,23 +20,23 @@ from treepkg.readconfig import read_config_section, convert_bool from treepkg.run import call, capture_output from treepkg.cmdexpand import cmdexpand -from treepkg.publish import * -from treepkg.util import md5sum +from treepkg.publish import copy_arch_to_publishdir, prefix_for_remote_command + +from treepkg.util import md5sum, expand_filename, remove_trailing_slashes from treepkg.info.status import TreepkgInfo from treepkg.info.data import Package from treepkg.info.data import CacheDb -EMPTY = re.compile(r'\s*') - config_desc = ["distribution", "section", "num_newest", "build_user", "build_host", "build_listpackages", "publish_user", "publish_host", ("architectures", shlex.split, "armel i386 source"), - ("after_upload_hook", shlex.split), + ("after_upload_hook", shlex.split, ""), + ("after_copy_hook", shlex.split, ""), + ("publish_packages", convert_bool, "True"), ("publish_remove_old_packages", convert_bool), ("publish_dir", remove_trailing_slashes), - ("cachedb", - lambda s: expand_filename(remove_trailing_slashes(s))), + ("cachedb", lambda s: expand_filename(s)), ("cachedir", lambda s: expand_filename(remove_trailing_slashes(s)))] @@ -78,18 +76,12 @@ runremote = prefix_for_remote_command(variables["build_user"], variables["build_host"]) xml = capture_output(cmdexpand("@runremote $build_listpackages" - " --newest=$num_newest" - " --only-successful", - runremote=runremote, - **variables)) + " --newest=$num_newest" + " --only-successful", + runremote=runremote, + **variables)) return TreepkgInfo.fromxml(xml) -def get_binary_arch(arch): - if not arch is None and not arch.startswith("binary") and \ - not arch == "source": - arch = "binary-" + arch - return arch - def check_package_is_new(packagename, destdir, packagemd5sum): destpackage = os.path.join(destdir, packagename) if not os.path.isfile(destpackage): @@ -107,11 +99,13 @@ return md5sum def sort_trackname_arch(a, b): - if a.trackname < b.trackname: return -1 - if a.trackname > b.trackname: return +1 + if a.trackname < b.trackname: + return -1 + if a.trackname > b.trackname: + return +1 return cmp(a.arch, b.arch) -def copy_files_to_destdir(destdir, files, variables, quiet = False): +def copy_files_to_destdir(destdir, files, variables, quiet=False): scp_flags = [] if quiet: scp_flags.append("-q") @@ -137,7 +131,7 @@ os.remove(package.filename) cachedb.remove_packages(oldpackages) -def copy_packages_to_destdir(cachedb, dir, packages, variables, quiet = False): +def copy_packages_to_destdir(cachedb, dir, packages, variables, quiet=False): packages.sort(cmp=sort_trackname_arch) package = packages[0] trackname = package.trackname @@ -146,8 +140,7 @@ files = [] for package in packages: cachedb.add_package(package) - if package.trackname != trackname or \ - package.arch != arch: + if package.trackname != trackname or package.arch != arch: copy_files_to_destdir(destdir, files, variables, quiet) trackname = package.trackname arch = package.arch @@ -160,7 +153,8 @@ print "copy new file: %s" % package.name copy_files_to_destdir(destdir, files, variables, quiet) -def copy_to_cachedir(variables, track, revision, quiet = False, architectures=None): +def copy_to_cachedir(variables, track, revision, quiet=False, + architectures=()): cachedir = variables["cachedir"] cachdebfilename = variables["cachedb"] if not quiet: @@ -168,14 +162,10 @@ cachedb = CacheDb(cachdebfilename) newpackages = [] treepkginfo = get_treepkg_info(variables) - #allowedarchs = set([]) # contains all wanted architectures (incl. source) - allarchs = set([]) # contains all present architectures (incl. source) + allarchs = set() # contains all present architectures (incl. source) binaryallpackages = [] # change e.g. armel in binary-armel - if not architectures is None: - allowedarchs = set([get_binary_arch(a) for a in architectures]) - else: - allowedarchs = set([]) + allowedarchs = set([get_binary_arch(a) for a in architectures]) for track in treepkginfo.tracks: for rev in track.revisions: for packageinfo in rev.packages: @@ -204,9 +194,9 @@ newpackages.append(newpackage) # copy binary-all packages sourcearch = set(["source"]) - if len(allowedarchs) == 0: + if not allowedarchs: binallarchs = allarchs - sourcearch - elif len(allarchs) == 0: + elif allarchs: binallarchs = allowedarchs - sourcearch else: binallarchs = (allowedarchs & allarchs) - sourcearch @@ -214,8 +204,9 @@ for arch in binallarchs: filename = os.path.join(cachedir, arch, packageinfo.trackname, packageinfo.name) - newpackage = Package(filename, packageinfo.trackname, packageinfo.name, - packageinfo.path, arch, get_md5sum(packageinfo)) + newpackage = Package(filename, packageinfo.trackname, + packageinfo.name, packageinfo.path, arch, + get_md5sum(packageinfo)) newpackages.append(newpackage) copy_packages_to_destdir(cachedb, cachedir, newpackages, variables, quiet) remove_old_packages(cachedb, newpackages, quiet) @@ -230,13 +221,22 @@ section = config["section"] architectures = config["architectures"] - allarchs = copy_to_cachedir(config, track, revision, quiet, architectures) - for arch in allarchs: - copy_to_publishdir(config, dist, section, arch, quiet) + copy_to_cachedir(config, track, revision, quiet, architectures) + + if config["after_copy_hook"]: + if not quiet: + print "running after copy hook" + call(config["after_copy_hook"]) + + if config["publish_packages"]: + for arch in architectures: + if not quiet: + print "publish packages for architecture %s" % arch + copy_arch_to_publishdir(config, dist, section, + get_binary_arch(arch), quiet) # update apt archive - if not config["after_upload_hook"] or \ - not EMPTY.match(config["after_upload_hook"]): + if config["after_upload_hook"]: if not quiet: print "running after upload hook" call(config["after_upload_hook"]) diff -r 383d23c6bfde -r eadcb1bb54e2 bin/publishpackages.py --- a/bin/publishpackages.py Fri Aug 06 13:28:47 2010 +0000 +++ b/bin/publishpackages.py Fri Aug 06 13:41:54 2010 +0000 @@ -19,8 +19,8 @@ from treepkg.readconfig import read_config_section, convert_bool from treepkg.run import call, capture_output from treepkg.cmdexpand import cmdexpand -from treepkg.util import ensure_directory, listdir_abs -from treepkg.publish import remove_trailing_slashes, expand_filename +from treepkg.util import ensure_directory, listdir_abs, \ + remove_trailing_slashes, expand_filename from treepkg.publish import prefix_for_remote_command, copy_to_publishdir config_desc = ["distribution", "section", "num_newest", diff -r 383d23c6bfde -r eadcb1bb54e2 test/test_info.py --- a/test/test_info.py Fri Aug 06 13:28:47 2010 +0000 +++ b/test/test_info.py Fri Aug 06 13:41:54 2010 +0000 @@ -8,17 +8,13 @@ """Tests for treepkg.status.status""" import unittest -import os.path +import os import sys -test_dir = os.path.dirname(__file__) -sys.path.append(os.path.join(test_dir, os.pardir)) +from treepkg.info.status import TreepkgInfo, TreepkgRootInfo -from treepkg.info.status import * -from treepkg.report import get_packager_group from filesupport import FileTestMixin - -from publishdebianpackages import get_binary_arch +from treepkg.publish import get_binary_arch class TreepkgInfoTest(unittest.TestCase, FileTestMixin): config_contents = """\ @@ -83,7 +79,7 @@ xml = dom.toxml() self.assertEquals("testtreepkg", xml) -class TestPublishDebianPackages(unittest.TestCase, FileTestMixin): +class TestPublish(unittest.TestCase, FileTestMixin): def test_get_binary_arch(self): source = get_binary_arch("source") diff -r 383d23c6bfde -r eadcb1bb54e2 test/test_util.py --- a/test/test_util.py Fri Aug 06 13:28:47 2010 +0000 +++ b/test/test_util.py Fri Aug 06 13:41:54 2010 +0000 @@ -12,7 +12,9 @@ from filesupport import FileTestMixin -from treepkg.util import replace_in_file, listdir_abs, md5sum +from treepkg.util import replace_in_file, listdir_abs, md5sum, \ + remove_trailing_slashes, expand_filename + class TestReplaceInFile(unittest.TestCase, FileTestMixin): @@ -82,3 +84,21 @@ def test_md5sum(self): sum = md5sum(self.testfile) self.assertEquals("a12511153555c1f0f0a1eda200733a3f", sum) + +class TestRemoveTrailingSlashes(unittest.TestCase): + + def test_remove_trailing_slashes(self): + dir_w_slash = "/tmp/dir/" + dir_wo_slash = "/tmp/dir" + + self.assertEquals(dir_wo_slash, remove_trailing_slashes(dir_w_slash)) + self.assertEquals(dir_wo_slash, remove_trailing_slashes(dir_wo_slash)) + +class TestExpandFilename(unittest.TestCase): + + def test_expand_filenam(self): + os.environ['MY_TEST_VAR'] = "def" + path = "/abc/${MY_TEST_VAR}/" + + self.assertEquals("/abc/def/", expand_filename(path)) + diff -r 383d23c6bfde -r eadcb1bb54e2 treepkg/info/data.py --- a/treepkg/info/data.py Fri Aug 06 13:28:47 2010 +0000 +++ b/treepkg/info/data.py Fri Aug 06 13:41:54 2010 +0000 @@ -15,7 +15,7 @@ class Package: def __init__(self, filename, trackname, packagename, packagepath, - arch, md5sum): + arch, md5sum): self.filename = filename self.trackname = trackname self.name = packagename @@ -26,7 +26,6 @@ class CacheDb: - def __init__(self, file): self.SELECT_PACKAGE_TMPL = """SELECT * FROM packages WHERE filename = ?""" @@ -77,7 +76,7 @@ row = self.cursor.fetchone() if not row: return None - return Package(row[0], row[1], row[2], row[3], row[4], row[5]) + return Package(*row) def get_old_packages(self, newfiles): SELECT_TMPL = """SELECT * FROM packages diff -r 383d23c6bfde -r eadcb1bb54e2 treepkg/info/status.py --- a/treepkg/info/status.py Fri Aug 06 13:28:47 2010 +0000 +++ b/treepkg/info/status.py Fri Aug 06 13:41:54 2010 +0000 @@ -45,13 +45,14 @@ if not node: if required: raise TreepkgInfoException("Element %s is required as child. But" - "parent element is not available.") + "parent element is not available.") return None childs = node.getElementsByTagName(name) if not childs: if required: raise TreepkgInfoException("Element %s is required as child for %s." - "The XML file must be invalid." % (name, node.nodeName)) + "The XML file must be invalid." + % (name, node.nodeName)) return None return childs @@ -66,7 +67,7 @@ self.success = success group = get_packager_group(config) treepkgrootinfo = TreepkgRootInfo(group.name, group.treepkg_dir, - group.tracks_dir) + group.tracks_dir) version = "1.0" self.tpkgroot = TreepkgRoot(version, treepkgrootinfo) tracks = group.get_package_tracks() @@ -91,16 +92,19 @@ revisions = track.get_revisions() revisions = sorted(revisions, key=lambda r: r.status.start, reverse=True) + candidates = revisions + + if self.success: # skip not successful builds + candidates = [rev for rev in revisions + if rev.status.status.finished + and not rev.status.status.error] + if self.numnewestrev > 0: - revisions = revisions[:self.numnewestrev] + candidates = candidates[:self.numnewestrev] arch = None - for rev in revisions: - if self.success: # skip not successful builds - if not rev.status.status.finished \ - or rev.status.status.error: - continue + for rev in candidates: revision = rev.revision rules_revision = rev.rules_revision status = rev.status.status.description # extend status @@ -131,8 +135,9 @@ pkginfo.add_checksum(checksuminfo) revision.add_package(pkginfo) + class TreepkgRoot: - + def __init__(self, version, info): self.version = version self.info = info @@ -308,8 +313,6 @@ messageele = getChild(statusele, "message") message = getTextFromNode(messageele) treepkgrevisioninfo = TreepkgTrackRevisionInfo(number, rules, message) - treepkgrevisioninfo.packages = [] - treepkgrevisioninfo.logs = [] packagesele = getChild(node, "packages") packageeles = getChilds(packagesele, "package") for packageele in packageeles: @@ -318,8 +321,7 @@ logsele = getChild(node, "logs") logeles = getChilds(logsele, "log") for logele in logeles: - treepkgrevisioninfo.logs.append( - TreepkgLogInfo.fromxml(logele)) + treepkgrevisioninfo.logs.append(TreepkgLogInfo.fromxml(logele)) return treepkgrevisioninfo class TreepkgLogInfo: @@ -391,12 +393,13 @@ path = getTextFromNode(pathele) ptype = node.getAttribute("type") arch = node.getAttribute("arch") - if len(arch) == 0: + if not arch: arch = None packageinfo = TreepkgPackageInfo(name, path, ptype, arch) checksumeles = node.getElementsByTagName("checksum") for checksumele in checksumeles: - packageinfo.checksums.append(TreepkgChecksumInfo.fromxml(checksumele)) + packageinfo.checksums.append( + TreepkgChecksumInfo.fromxml(checksumele)) return packageinfo class TreepkgChecksumInfo: @@ -418,6 +421,6 @@ def fromxml(node): checksum = getTextFromNode(node) ctype = node.getAttribute("type") - if len(ctype) == 0: + if not ctype: return TreepkgChecksumInfo(checksum) return TreepkgChecksumInfo(checksum, ctype) diff -r 383d23c6bfde -r eadcb1bb54e2 treepkg/info/status.xsd --- a/treepkg/info/status.xsd Fri Aug 06 13:28:47 2010 +0000 +++ b/treepkg/info/status.xsd Fri Aug 06 13:41:54 2010 +0000 @@ -79,7 +79,7 @@ - + diff -r 383d23c6bfde -r eadcb1bb54e2 treepkg/packager.py --- a/treepkg/packager.py Fri Aug 06 13:28:47 2010 +0000 +++ b/treepkg/packager.py Fri Aug 06 13:41:54 2010 +0000 @@ -8,6 +8,7 @@ """Classes to automatically build debian packages from subversion checkouts""" import os +import os.path import time import re import logging @@ -327,11 +328,11 @@ files = [] if os.path.isdir(self.log_dir): for f in os.listdir(self.log_dir): - if f in logs: + if logs is None or f in logs: f = os.path.join(self.log_dir,f) if os.path.isfile(f): files.append((self.get_log_title(f),f)) - return files + return files def list_log_files(self, logs): """Returns a list describing the logfiles available for the revision. @@ -350,14 +351,20 @@ The implementation assumes that all files in self.src_dir belong to the source package. """ - return sorted(util.listdir_abs(self.src_dir)) + files = [] + if os.path.isdir(self.src_dir): + files = sorted(util.listdir_abs(self.src_dir)) + return files def list_binary_files(self): """Returns a list with the names of the files of the binary packages. The implementation assumes that all files in self.binary_dir belong to the binary packages. """ - return sorted(util.listdir_abs(self.binary_dir)) + files = [] + if os.path.isdir(self.binary_dir): + files = sorted(util.listdir_abs(self.binary_dir)) + return files def package(self): try: diff -r 383d23c6bfde -r eadcb1bb54e2 treepkg/publish.py --- a/treepkg/publish.py Fri Aug 06 13:28:47 2010 +0000 +++ b/treepkg/publish.py Fri Aug 06 13:41:54 2010 +0000 @@ -9,18 +9,10 @@ import os.path +import util from treepkg.run import call, capture_output from treepkg.cmdexpand import cmdexpand -def remove_trailing_slashes(s): - return s.rstrip("/") - -def expand_filename(filename): - """ - Applies os.path.expanduser and os.path.expandvars to filename - """ - return os.path.expandvars(os.path.expanduser(filename)) - def prefix_for_remote_command(user, host): """Returns the ssh call needed to run a command on a remote host. If host is empty, the function assumes the command is to be run on @@ -32,6 +24,46 @@ prefix.extend(["ssh", "%s@%s" % (user, host)]) return prefix +def copy_arch_to_publishdir(variables, dist, section, arch=None, quiet=False, + create=True): + if not arch: + cachedir = variables["cachedir"] + else: + cachedir = os.path.join(variables["cachedir"], arch) + + # if cachedir does not exist rsync will fail therefore + # it must be created or skipped. if it is created remote + # content will be deleted + if not os.path.exists(cachedir): + if create: + util.ensure_directory(cachedir) + else: + return + + destdir = os.path.join(variables["publish_dir"], dist, section) + remote_destdir = destdir + if variables["publish_host"]: + remote_destdir = (("%(publish_user)s@%(publish_host)s:" % variables) + + remote_destdir) + runremote = prefix_for_remote_command(variables["publish_user"], + variables["publish_host"]) + + call(cmdexpand("@runremote mkdir --parents $destdir", + runremote=runremote, destdir=destdir, **variables)) + rsync_flags = [] + if variables["publish_remove_old_packages"]: + rsync_flags.append("--delete") + if quiet: + rsync_flags.append("--quiet") + cmd = cmdexpand("rsync @rsync_flags -r --perms --times --omit-dir-times" + " $cachedir $remote_destdir/", + rsync_flags=rsync_flags, remote_destdir=remote_destdir, + cachedir=cachedir) + #print "rsync cmd: %s" % cmd + call(cmd) + + + def copy_to_publishdir(variables, dist, section, arch=None, quiet=False): if not arch: destdir = os.path.join(variables["publish_dir"], dist, section) @@ -56,4 +88,8 @@ rsync_flags=rsync_flags, remote_destdir=remote_destdir, **variables)) +def get_binary_arch(arch): + if not arch is None and not arch.startswith("binary") and arch != "source": + arch = "binary-" + arch + return arch diff -r 383d23c6bfde -r eadcb1bb54e2 treepkg/util.py --- a/treepkg/util.py Fri Aug 06 13:28:47 2010 +0000 +++ b/treepkg/util.py Fri Aug 06 13:41:54 2010 +0000 @@ -174,3 +174,13 @@ f.close() return m.hexdigest() +def remove_trailing_slashes(s): + return s.rstrip("/") + +def expand_filename(filename): + """ + Applies os.path.expanduser and os.path.expandvars to filename + """ + return os.path.expandvars(os.path.expanduser(filename)) + +