# HG changeset patch # User Bjoern Ricks # Date 1281101327 0 # Node ID 383d23c6bfdecfe2404b4bf0d87954d2dd1852a4 # Parent 25d702bb13f9bc862c0352c3891bddc2af195d02# Parent 56f7da71d41e739b00b162fa8317c0382bb2b65e Merged revisions 346-386 via svnmerge from svn+ssh://svn.wald.intevation.org/treepkg/branches/treepkg-status ........ r346 | bricks | 2010-07-07 17:40:48 +0200 (Mi, 07 Jul 2010) | 2 lines added classed for treepkg status xml generation ........ r347 | bricks | 2010-07-08 12:07:39 +0200 (Do, 08 Jul 2010) | 10 lines renamed new status dir to info because of a naming conflict with status.py let the user specify a treepkg name in the config the name is propagated to PackagerGroup [treepkg] name: becomes: pg = PackagerGroup(...) pg.name ........ r348 | bricks | 2010-07-08 15:25:07 +0200 (Do, 08 Jul 2010) | 3 lines moved test file to test main dir fixed missing : after def statement ........ r349 | bricks | 2010-07-08 17:59:52 +0200 (Do, 08 Jul 2010) | 2 lines added treepkg_dir andd tracks_dir attributes to PackageGroup ........ r350 | bricks | 2010-07-08 18:00:32 +0200 (Do, 08 Jul 2010) | 2 lines changed to ........ r351 | bricks | 2010-07-08 18:08:39 +0200 (Do, 08 Jul 2010) | 2 lines check if config file exists to get usefull error output ........ r352 | bricks | 2010-07-08 18:11:13 +0200 (Do, 08 Jul 2010) | 2 lines write first info about tracks ........ r353 | bricks | 2010-07-09 12:20:49 +0200 (Fr, 09 Jul 2010) | 2 lines get all log files ........ r354 | bricks | 2010-07-09 14:42:42 +0200 (Fr, 09 Jul 2010) | 2 lines added md5sum function ........ r355 | bricks | 2010-07-09 15:15:15 +0200 (Fr, 09 Jul 2010) | 4 lines implemented nearly all info only arch and os info are missing ........ r356 | bricks | 2010-07-09 16:19:17 +0200 (Fr, 09 Jul 2010) | 2 lines Bugfix: source was not defined ........ r357 | bricks | 2010-07-09 16:44:52 +0200 (Fr, 09 Jul 2010) | 3 lines fixed some typos added missing add_checksum methond ........ r358 | bricks | 2010-07-09 17:06:15 +0200 (Fr, 09 Jul 2010) | 2 lines fixed status line for revision ........ r359 | bricks | 2010-07-12 14:10:34 +0200 (Mo, 12 Jul 2010) | 2 lines let the user specify num revisions that should be shown in the info ........ r360 | bricks | 2010-07-12 14:21:39 +0200 (Mo, 12 Jul 2010) | 2 lines missed in last commit ........ r361 | bricks | 2010-07-12 17:06:22 +0200 (Mo, 12 Jul 2010) | 2 lines add os config statement ........ r362 | bricks | 2010-07-12 17:47:42 +0200 (Mo, 12 Jul 2010) | 3 lines moved packages and logs to revision tag implemented os and arch info ........ r363 | bricks | 2010-07-13 14:23:28 +0200 (Di, 13 Jul 2010) | 2 lines updated test readconfig for os config variable ........ r364 | bricks | 2010-07-13 14:24:14 +0200 (Di, 13 Jul 2010) | 2 lines moved common publish functions to a seperate module ........ r365 | bricks | 2010-07-13 16:26:11 +0200 (Di, 13 Jul 2010) | 2 lines moved arch info to each package info ........ r366 | bricks | 2010-07-13 18:46:17 +0200 (Di, 13 Jul 2010) | 2 lines inital checkin for new publishpackages processing ........ r367 | bricks | 2010-07-21 17:52:15 +0200 (Mi, 21 Jul 2010) | 2 lines implemented parsing from xml string ........ r368 | bricks | 2010-07-22 12:17:42 +0200 (Do, 22 Jul 2010) | 2 lines copy binary-all packages in all binary-xyz dirs ........ r369 | bricks | 2010-07-22 12:26:40 +0200 (Do, 22 Jul 2010) | 2 lines fixed small bugs ........ r370 | bricks | 2010-07-22 13:02:36 +0200 (Do, 22 Jul 2010) | 2 lines added a new root element which is called TreepkgRoot for Treepkg Info classes ........ r371 | bricks | 2010-07-22 13:09:52 +0200 (Do, 22 Jul 2010) | 3 lines fixed some small issues remember: run pychecker BEFORE the commit ........ r372 | bricks | 2010-07-22 18:01:43 +0200 (Do, 22 Jul 2010) | 2 lines improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part ........ r373 | bricks | 2010-07-23 18:40:38 +0200 (Fr, 23 Jul 2010) | 2 lines implemented CacheDb to store copied Packages from build host ........ r374 | bricks | 2010-07-26 10:28:48 +0200 (Mo, 26 Jul 2010) | 2 lines moved data handling in publishdebianpackages into seperate module ........ r375 | bricks | 2010-07-26 11:42:07 +0200 (Mo, 26 Jul 2010) | 2 lines fixed some typos ........ r376 | bricks | 2010-07-26 12:12:40 +0200 (Mo, 26 Jul 2010) | 2 lines first version of incremental copying debian packages to cachedir ........ r377 | bricks | 2010-07-26 14:58:50 +0200 (Mo, 26 Jul 2010) | 2 lines incremental copying only changed debian packages ........ r378 | bricks | 2010-07-26 15:39:10 +0200 (Mo, 26 Jul 2010) | 3 lines use md5 instead of hashlib module (md5 is deprecated in favour of hashlib in python 2.5) ........ r379 | bricks | 2010-07-26 15:46:19 +0200 (Mo, 26 Jul 2010) | 2 lines make treepkg compatible to python 2.4 ........ r380 | bricks | 2010-07-26 15:50:49 +0200 (Mo, 26 Jul 2010) | 2 lines fixed last commit (db instead of sqlite3) ........ r381 | bricks | 2010-07-27 10:54:15 +0200 (Di, 27 Jul 2010) | 2 lines fixed wrong import statement ........ r382 | bricks | 2010-07-27 10:54:34 +0200 (Di, 27 Jul 2010) | 2 lines removed unnecessary debug output ........ r383 | bricks | 2010-07-27 18:28:22 +0200 (Di, 27 Jul 2010) | 2 lines check if upload hook is empty ........ r384 | bricks | 2010-07-28 09:35:21 +0200 (Mi, 28 Jul 2010) | 2 lines run rsync only an arch dirs that are present in the current treepkg ........ r385 | bricks | 2010-07-28 09:35:54 +0200 (Mi, 28 Jul 2010) | 2 lines add helper tool to list content of a cache db ........ r386 | bricks | 2010-07-28 10:33:48 +0200 (Mi, 28 Jul 2010) | 3 lines fixed a typo regex fails if option is empty ........ diff -r 25d702bb13f9 -r 383d23c6bfde bin/listcachedb.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/bin/listcachedb.py Fri Aug 06 13:28:47 2010 +0000 @@ -0,0 +1,31 @@ +#! /usr/bin/python +# Copyright (C) 2010 by Intevation GmbH +# Authors: +# Bjoern Ricks +# +# This program is free software under the GPL (>=v2) +# Read the file COPYING coming with the software for details. + +from optparse import OptionParser + +import treepkgcmd +from treepkg.info.data import CacheDb + +def main(): + parser = OptionParser() + parser.add_option("--db", help=("path to the CacheDb")) + options, args = parser.parse_args() + + cachedb = CacheDb(options.db) + packages = cachedb.get_packages() + for package in packages: + print "=============================" + print "name: %s" % package.name + print "=============================" + print "binary: %s" % package.filename + print "source: %s" % package.sourcepath + print "arch: %s" % package.arch + print "md5sum: %s" % package.md5sum + +if __name__ == "__main__": + main() diff -r 25d702bb13f9 -r 383d23c6bfde bin/publishdebianpackages.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/bin/publishdebianpackages.py Fri Aug 06 13:28:47 2010 +0000 @@ -0,0 +1,251 @@ +#! /usr/bin/python +# Copyright (C) 2007 - 2010 by Intevation GmbH +# Authors: +# Bernhard Herzog +# Bjoern Ricks +# +# This program is free software under the GPL (>=v2) +# Read the file COPYING coming with the software for details. + +"""Publishes selected packages created by treepkg""" + +import os +import os.path +import re +import sys +import shlex + +from optparse import OptionParser +from ConfigParser import SafeConfigParser + +import treepkgcmd +from treepkg.readconfig import read_config_section, convert_bool +from treepkg.run import call, capture_output +from treepkg.cmdexpand import cmdexpand +from treepkg.publish import * +from treepkg.util import md5sum +from treepkg.info.status import TreepkgInfo +from treepkg.info.data import Package +from treepkg.info.data import CacheDb + +EMPTY = re.compile(r'\s*') + +config_desc = ["distribution", "section", "num_newest", + "build_user", "build_host", "build_listpackages", + "publish_user", "publish_host", + ("architectures", shlex.split, "armel i386 source"), + ("after_upload_hook", shlex.split), + ("publish_remove_old_packages", convert_bool), + ("publish_dir", remove_trailing_slashes), + ("cachedb", + lambda s: expand_filename(remove_trailing_slashes(s))), + ("cachedir", + lambda s: expand_filename(remove_trailing_slashes(s)))] + + +def read_config(filename): + if not os.path.exists(filename): + print >>sys.stderr, "Config file %s does not exist" % filename + sys.exit(1) + parser = SafeConfigParser() + parser.read([filename]) + return read_config_section(parser, "publishpackages", config_desc) + +def parse_commandline(): + parser = OptionParser() + parser.set_defaults(config_file=os.path.join(treepkgcmd.topdir, + "publishpackages.cfg"), + quiet=False) + parser.add_option("--config-file", + help=("The configuration file." + " Default is publishpackages.cfg")) + parser.add_option("--dist", + help=("The debian distribution name to use on" + " the publishing system")) + parser.add_option("--section", + help=("The debian distribution section name to use on" + " the publishing system")) + parser.add_option("--track", + help=("The package track whose files are to be" + " published. If not given, files of all tracks" + " will be published")) + parser.add_option("--quiet", action="store_true", + help=("Do not print progress meters or other" + " informational output")) + return parser.parse_args() + +def get_treepkg_info(variables): + runremote = prefix_for_remote_command(variables["build_user"], + variables["build_host"]) + xml = capture_output(cmdexpand("@runremote $build_listpackages" + " --newest=$num_newest" + " --only-successful", + runremote=runremote, + **variables)) + return TreepkgInfo.fromxml(xml) + +def get_binary_arch(arch): + if not arch is None and not arch.startswith("binary") and \ + not arch == "source": + arch = "binary-" + arch + return arch + +def check_package_is_new(packagename, destdir, packagemd5sum): + destpackage = os.path.join(destdir, packagename) + if not os.path.isfile(destpackage): + return True + destmd5sum = md5sum(destpackage) + return (destmd5sum != packagemd5sum) + +def get_md5sum(packageinfo): + md5sum = "" + if packageinfo: + for checksum in packageinfo.checksums: + if checksum.type == "md5": + md5sum = checksum.checksum + break + return md5sum + +def sort_trackname_arch(a, b): + if a.trackname < b.trackname: return -1 + if a.trackname > b.trackname: return +1 + return cmp(a.arch, b.arch) + +def copy_files_to_destdir(destdir, files, variables, quiet = False): + scp_flags = [] + if quiet: + scp_flags.append("-q") + + if not os.path.exists(destdir): + os.makedirs(destdir) + if files: + if variables["build_host"]: + userhost = "%(build_user)s@%(build_host)s:" % variables + files = [userhost + filename for filename in files] + # scp the packages to the cache dir + call(cmdexpand("scp -p @scp_flags @files $cachedir/", files=files, + scp_flags=scp_flags, cachedir=destdir)) + +def remove_old_packages(cachedb, newpackages, quiet): + newfiles = [package.filename for package in newpackages] + oldpackages = cachedb.get_old_packages(newfiles) + for package in oldpackages: + # better check if the file really exists + if os.path.isfile(package.filename): + if not quiet: + print "removing file %s" % package.filename + os.remove(package.filename) + cachedb.remove_packages(oldpackages) + +def copy_packages_to_destdir(cachedb, dir, packages, variables, quiet = False): + packages.sort(cmp=sort_trackname_arch) + package = packages[0] + trackname = package.trackname + arch = package.arch + destdir = os.path.join(dir, arch, trackname) + files = [] + for package in packages: + cachedb.add_package(package) + if package.trackname != trackname or \ + package.arch != arch: + copy_files_to_destdir(destdir, files, variables, quiet) + trackname = package.trackname + arch = package.arch + destdir = os.path.join(dir, arch, trackname) + files = [] + # add only to copy files list if the packages differ + if check_package_is_new(package.name, destdir, package.md5sum): + files.append(package.sourcepath) + if not quiet: + print "copy new file: %s" % package.name + copy_files_to_destdir(destdir, files, variables, quiet) + +def copy_to_cachedir(variables, track, revision, quiet = False, architectures=None): + cachedir = variables["cachedir"] + cachdebfilename = variables["cachedb"] + if not quiet: + print "using cachedb: %s" % cachdebfilename + cachedb = CacheDb(cachdebfilename) + newpackages = [] + treepkginfo = get_treepkg_info(variables) + #allowedarchs = set([]) # contains all wanted architectures (incl. source) + allarchs = set([]) # contains all present architectures (incl. source) + binaryallpackages = [] + # change e.g. armel in binary-armel + if not architectures is None: + allowedarchs = set([get_binary_arch(a) for a in architectures]) + else: + allowedarchs = set([]) + for track in treepkginfo.tracks: + for rev in track.revisions: + for packageinfo in rev.packages: + arch = get_binary_arch(packageinfo.arch) + if packageinfo.type == "binary": + # skip other files + if packageinfo.arch is None: + continue + # handle binary-all + if arch == "binary-all": + # add trackname for subdir name + packageinfo.trackname = track.name + binaryallpackages.append(packageinfo) + continue + allarchs.add(arch) + elif packageinfo.type == "source": + arch = packageinfo.type + # only copy requested archs + if len(allowedarchs) == 0 or \ + arch in allowedarchs: + filename = os.path.join(cachedir, arch, track.name, + packageinfo.name) + newpackage = Package(filename, track.name, packageinfo.name, + packageinfo.path, arch, + get_md5sum(packageinfo)) + newpackages.append(newpackage) + # copy binary-all packages + sourcearch = set(["source"]) + if len(allowedarchs) == 0: + binallarchs = allarchs - sourcearch + elif len(allarchs) == 0: + binallarchs = allowedarchs - sourcearch + else: + binallarchs = (allowedarchs & allarchs) - sourcearch + for packageinfo in binaryallpackages: + for arch in binallarchs: + filename = os.path.join(cachedir, arch, packageinfo.trackname, + packageinfo.name) + newpackage = Package(filename, packageinfo.trackname, packageinfo.name, + packageinfo.path, arch, get_md5sum(packageinfo)) + newpackages.append(newpackage) + copy_packages_to_destdir(cachedb, cachedir, newpackages, variables, quiet) + remove_old_packages(cachedb, newpackages, quiet) + return binallarchs + +def publish_packages(config_filename, track, revision, dist, section, quiet): + config = read_config(config_filename) + + if dist is None: + dist = config["distribution"] + if section is None: + section = config["section"] + + architectures = config["architectures"] + allarchs = copy_to_cachedir(config, track, revision, quiet, architectures) + for arch in allarchs: + copy_to_publishdir(config, dist, section, arch, quiet) + + # update apt archive + if not config["after_upload_hook"] or \ + not EMPTY.match(config["after_upload_hook"]): + if not quiet: + print "running after upload hook" + call(config["after_upload_hook"]) + +def main(): + options, args = parse_commandline() + revision = None # for future use cases + publish_packages(options.config_file, options.track, revision, + options.dist, options.section, options.quiet) + +if __name__ == "__main__": + main() diff -r 25d702bb13f9 -r 383d23c6bfde bin/publishpackages.py --- a/bin/publishpackages.py Wed Jul 28 08:35:07 2010 +0000 +++ b/bin/publishpackages.py Fri Aug 06 13:28:47 2010 +0000 @@ -20,16 +20,8 @@ from treepkg.run import call, capture_output from treepkg.cmdexpand import cmdexpand from treepkg.util import ensure_directory, listdir_abs - - -def remove_trailing_slashes(s): - return s.rstrip("/") - -def expand_filename(filename): - """ - Applies os.path.expanduser and os.path.expandvars to filename - """ - return os.path.expandvars(os.path.expanduser(filename)) +from treepkg.publish import remove_trailing_slashes, expand_filename +from treepkg.publish import prefix_for_remote_command, copy_to_publishdir config_desc = ["distribution", "section", "num_newest", "build_user", "build_host", "build_listpackages", @@ -74,18 +66,6 @@ " informational output")) return parser.parse_args() -def prefix_for_remote_command(user, host): - """Returns the ssh call needed to run a command on a remote host. - If host is empty, the function assumes the command is to be run on - the local host as the same user that exectutes this function, so - that no ssh or other call is needed. - """ - prefix = [] - if host: - prefix.extend(["ssh", "%s@%s" % (user, host)]) - return prefix - - def copy_to_cache(variables, track, revision, arch, quiet): listpackages_vars = variables.copy() @@ -126,29 +106,6 @@ call(cmdexpand("scp -p @scp_flags @files $cachedir/", files=files, scp_flags=scp_flags, **variables)) - -def copy_to_publishdir(variables, dist, section, arch, quiet): - destdir = os.path.join(variables["publish_dir"], dist, section, arch) - remote_destdir = destdir - if variables["publish_host"]: - remote_destdir = (("%(publish_user)s@%(publish_host)s:" % variables) - + remote_destdir) - runremote = prefix_for_remote_command(variables["publish_user"], - variables["publish_host"]) - - call(cmdexpand("@runremote mkdir --parents $destdir", - runremote=runremote, destdir=destdir, **variables)) - rsync_flags = [] - if variables["publish_remove_old_packages"]: - rsync_flags.append("--delete") - if quiet: - rsync_flags.append("--quiet") - call(cmdexpand("rsync @rsync_flags -r --perms --times --omit-dir-times" - " $cachedir/ $remote_destdir/", - rsync_flags=rsync_flags, remote_destdir=remote_destdir, - **variables)) - - def publish_packages_arch(variables, track, revision, dist, section, arch, quiet): copy_to_cache(variables, track, revision, arch, quiet) diff -r 25d702bb13f9 -r 383d23c6bfde bin/treepkginfo.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/bin/treepkginfo.py Fri Aug 06 13:28:47 2010 +0000 @@ -0,0 +1,41 @@ +#! /usr/bin/python +# Copyright (C) 2010 by Intevation GmbH +# Authors: +# Bjoern Ricks +# +# This program is free software under the GPL (>=v2) +# Read the file COPYING coming with the software for details. + +"""Receive the status of a treepkg""" + +import treepkgcmd + +from treepkg.info import status + +from treepkg.options import create_parser + +def parse_commandline(): + parser = create_parser() + parser.set_defaults(newest=-1) + parser.add_option("--newest", type="int", + help=("Number of newest revisions to list.")) + parser.add_option("--only-successful", action="store_true", + help=("Show only successful builds"), dest="success") + parser.add_option("--pretty", action="store_true", help=("Show XML pretty" + " output.")) + + + return parser.parse_args() + +def main(): + options, args = parse_commandline() + + tpkginfo = status.TreepkgInfo(options.config_file, options.newest, + options.success) + + if options.pretty: + print tpkginfo.toxml().toprettyxml() + else: + print tpkginfo.toxml().toxml() + +main() diff -r 25d702bb13f9 -r 383d23c6bfde demopublishpackages.cfg --- a/demopublishpackages.cfg Wed Jul 28 08:35:07 2010 +0000 +++ b/demopublishpackages.cfg Fri Aug 06 13:28:47 2010 +0000 @@ -59,3 +59,4 @@ # ~user at the beginning of the value and environment variable # references of the form $VAR or ${VAR} are expanded. cachedir: /tmp/${LOGNAME}/treepkg-package-cache/%(build_host)s +cachedb: /tmp/${LOGNAME}/treepkg-package-cache/%(build_host)s-cache.db diff -r 25d702bb13f9 -r 383d23c6bfde test/test_info.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test/test_info.py Fri Aug 06 13:28:47 2010 +0000 @@ -0,0 +1,99 @@ +# Copyright (C) 2010 by Intevation GmbH +# Authors: +# Bjoern Ricks +# +# This program is free software under the GPL (>=v2) +# Read the file COPYING coming with the software for details. + +"""Tests for treepkg.status.status""" + +import unittest +import os.path +import sys + +test_dir = os.path.dirname(__file__) +sys.path.append(os.path.join(test_dir, os.pardir)) + +from treepkg.info.status import * +from treepkg.report import get_packager_group +from filesupport import FileTestMixin + +from publishdebianpackages import get_binary_arch + +class TreepkgInfoTest(unittest.TestCase, FileTestMixin): + config_contents = """\ +[DEFAULT] +#treepkg_dir: /home/builder/mill +#tracks_dir: %(treepkg_dir)s/tracks +tracks_dir: /home/builder/mill/tracks +root_cmd: sudo +pbuilder_dir: %(tracks_dir)s/pbuilder +builderconfig: %(pbuilder_dir)s/pbuilderrc +deb_email: treepkg@example.com +deb_fullname: TreePKG +pkg_revision_template: treepkg%%(pkg_revision)s +handle_dependencies: False + +[treepkg] +instructions_file: %(tracks_dir)s/instructions +check_interval: 3600 +name: testtreepkginfo + +[pkg_simple] +pkg_basename: simple1 +svn_url: svn://example.com/%(name)s/trunk +base_dir: %(tracks_dir)s/%(name)s +packager_class: treepkginfo_test.%(name)s +handle_dependencies: True +signing_key_id: abcd1234 +changelog_msg_template: Update to feature branch r%%(revision)s +version_template: 1.0.svn%%(revision)s + +""" + + files = [("treepkg.cfg", config_contents), + ("treepkginfo_test", + [("__init__.py", ""), + ("simple.py", + "\n".join(["class SourcePackager:", + " pass", + ""])), + ])] + + def setUp(self): + self.directory = self.create_files("treepkg", self.files) + self.old_path = sys.path + sys.path = [self.directory] + sys.path + + def tearDown(self): + sys.path = self.old_path + + def test_createinfo(self): + config_file = os.path.join(self.directory, "treepkg.cfg") + tpkginfo = TreepkgInfo(config_file) + tpkgrootinfo = tpkginfo.tpkgroot.info + self.assertEquals("testtreepkginfo", tpkgrootinfo.name) + + +class TreepkgRootInfoTest(unittest.TestCase): + + def test_toxml(self): + status = TreepkgRootInfo("testtreepkg") + dom = status.toxml() + xml = dom.toxml() + self.assertEquals("testtreepkg", xml) + +class TestPublishDebianPackages(unittest.TestCase, FileTestMixin): + + def test_get_binary_arch(self): + source = get_binary_arch("source") + self.assertEquals("source", source) + binary_armel = get_binary_arch("armel") + self.assertEquals("binary-armel", binary_armel) + binary_armel = get_binary_arch("binary-armel") + self.assertEquals("binary-armel", binary_armel) + + +if __name__ == '__main__': + unittest.main() + diff -r 25d702bb13f9 -r 383d23c6bfde test/test_info_data.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test/test_info_data.py Fri Aug 06 13:28:47 2010 +0000 @@ -0,0 +1,74 @@ +# Copyright (C) 2010 by Intevation GmbH +# Authors: +# Bjoern Ricks +# +# This program is free software under the GPL (>=v2) +# Read the file COPYING coming with the software for details. + +"""Tests for treepkg.info.data""" + +import unittest +import os.path +import sys + +test_dir = os.path.dirname(__file__) +sys.path.append(os.path.join(test_dir, os.pardir)) + +from treepkg.info.data import Package +from treepkg.info.data import CacheDb +from filesupport import FileTestMixin + +class TestCacheDb(unittest.TestCase, FileTestMixin): + + def test_cache_db(self): + tmpdir = self.create_test_specific_temp_dir() + package = Package("/tmp/abc/abc_0.1_i386.deb", "abc", + "abc_0.1_i386.deb", "/source/abc/abc_0.1_i386.deb", + "binary-i386", "1234567") +# tmpfile = self.create_temp_file("cachedb1", "abc") +# with self.assertRaises(Exception): +# db = CacheDb(tmpfile) + + dbfile = os.path.join(tmpdir, "cachedb2") + db = CacheDb(dbfile) + # insert + filename = package.filename + db.add_package(package) + package = db.get_package(filename) + self.assertEquals("1234567", package.md5sum) + # update + package.md5sum = "01234567" + db.add_package(package) + package = db.get_package(filename) + self.assertEquals("01234567", package.md5sum) + + # test get_package + package2 = db.get_package(package.filename) + self.assertNotEquals(None, package2) + self.assertEquals(package.filename, package2.filename) + self.assertEquals(package.name, package2.name) + self.assertEquals(package.sourcepath, package2.sourcepath) + self.assertEquals(package.arch, package2.arch) + self.assertEquals(package.md5sum, package2.md5sum) + + package3 = Package("/tmp/foo/foo_0.2.i386.deb", "foo", + "foo_0.2_i386.deb", "/tmp/source/foo/foo_0.2.i386.deb", + "binary-i386", "987654321") + package4 = Package("/tmp/john/doe_0.3.i386.deb", "john", + "doe_0.3.i386.deb", "/tmp/source/john/doe_0.3.i386.deb", + "binary-i386", "5671234") + db.add_package(package3) + db.add_package(package4) + + # test get_old_packages + oldpackages = db.get_old_packages([package.filename]) + self.assertEquals(2, len(oldpackages)) + packages = db.get_packages() + self.assertEquals(3, len(packages)) + db.remove_packages(oldpackages) + packages = db.get_packages() + self.assertEquals(1, len(packages)) + +if __name__ == '__main__': + unittest.main() + diff -r 25d702bb13f9 -r 383d23c6bfde test/test_readconfig.py --- a/test/test_readconfig.py Wed Jul 28 08:35:07 2010 +0000 +++ b/test/test_readconfig.py Fri Aug 06 13:28:47 2010 +0000 @@ -34,6 +34,7 @@ [treepkg] instructions_file: %(treepkg_dir)s/instructions check_interval: 3600 +name: testtreepkg [pkg_simple] pkg_basename: simple1 @@ -44,6 +45,7 @@ signing_key_id: abcd1234 changelog_msg_template: Update to feature branch r%%(revision)s version_template: 1.0.svn%%(revision)s +os: foo os [pkg_extraargs] svn_url: svn://example.com/%(name)s/trunk @@ -55,6 +57,7 @@ orig_tarball: %(base_dir)s/mytarball.tgz rules_svn_url: file:///tmp/my-debian-repository deb_build_options: parallel=2 +os: bar os """ files = [("treepkg.cfg", config_contents), @@ -82,7 +85,10 @@ treepkg_opts, packager_opts = read_config(config_file) self.assertEquals(treepkg_opts, dict(instructions_file="/home/builder/mill/instructions", - check_interval=3600)) + check_interval=3600, + name="testtreepkg", + treepkg_dir="/home/builder/mill", + tracks_dir="/home/builder/mill/tracks")) self.assertEquals(sorted(packager_opts, key=operator.itemgetter("name")), [ @@ -111,7 +117,8 @@ "tracks/extraargs/mytarball.tgz"), git_url="", git_branch="", - builder_cls="PBuilder"), + builder_cls="PBuilder", + os="bar os"), dict(name="simple", base_dir="/home/builder/mill/tracks/simple", changelog_msg_template=("Update to feature branch" @@ -133,4 +140,11 @@ version_template="1.0.svn%(revision)s", git_url="", git_branch="", - builder_cls="PBuilder")]) + builder_cls="PBuilder", + os="foo os")]) + +def main(): + unittest.main() + +if __name__ == "__main__": + main() diff -r 25d702bb13f9 -r 383d23c6bfde test/test_util.py --- a/test/test_util.py Wed Jul 28 08:35:07 2010 +0000 +++ b/test/test_util.py Fri Aug 06 13:28:47 2010 +0000 @@ -12,7 +12,7 @@ from filesupport import FileTestMixin -from treepkg.util import replace_in_file, listdir_abs +from treepkg.util import replace_in_file, listdir_abs, md5sum class TestReplaceInFile(unittest.TestCase, FileTestMixin): @@ -72,3 +72,13 @@ self.assertEquals(sorted(listdir_abs(directory, '*.dsc')), [os.path.join(directory, "foo.dsc")]) +class TestMd5sum(unittest.TestCase, FileTestMixin): + + content = "this is a test content" + + def setUp(self): + self.testfile = self.create_temp_file("testmd5.txt", self.content) + + def test_md5sum(self): + sum = md5sum(self.testfile) + self.assertEquals("a12511153555c1f0f0a1eda200733a3f", sum) diff -r 25d702bb13f9 -r 383d23c6bfde treepkg/info/__init__.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/treepkg/info/__init__.py Fri Aug 06 13:28:47 2010 +0000 @@ -0,0 +1,6 @@ +# Copyright (C) 2007 by Intevation GmbH +# Authors: +# Bernhard Herzog +# +# This program is free software under the GPL (>=v2) +# Read the file COPYING coming with the software for details. diff -r 25d702bb13f9 -r 383d23c6bfde treepkg/info/data.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/treepkg/info/data.py Fri Aug 06 13:28:47 2010 +0000 @@ -0,0 +1,101 @@ +#! /usr/bin/python +# Copyright (C) 2010 by Intevation GmbH +# Authors: +# Bjoern Ricks +# +# This program is free software under the GPL (>=v2) +# Read the file COPYING coming with the software for details. +""" Classes for storing and handling data related to treepkg info """ + +try: + import sqlite3 as db +except ImportError: + from pysqlite2 import dbapi2 as db + +class Package: + + def __init__(self, filename, trackname, packagename, packagepath, + arch, md5sum): + self.filename = filename + self.trackname = trackname + self.name = packagename + self.sourcepath = packagepath + self.arch = arch + self.md5sum = md5sum + + +class CacheDb: + + + def __init__(self, file): + self.SELECT_PACKAGE_TMPL = """SELECT * FROM packages + WHERE filename = ?""" + self.file = file + self.conn = db.connect(file) + self.cursor = self.conn.cursor() + self.init_db() + + def init_db(self): + self.cursor.execute( + """CREATE TABLE IF NOT EXISTS packages ( + filename TEXT PRIMARY KEY, + trackname TEXT, + packagename TEXT, + sourcepath TEXT, + arch TEXT, + md5sum TEXT )""") + self.conn.commit() + + + def add_package(self, package): + self.cursor.execute(self.SELECT_PACKAGE_TMPL, (package.filename,)) + row = self.cursor.fetchone() + if not row: + self.insert_package(package) + else: + self.update_package(package) + + def insert_package(self, package): + INSERT_TMPL = """INSERT INTO packages VALUES ( + ?, ?, ? ,? ,?, ?)""" + self.cursor.execute(INSERT_TMPL, (package.filename, + package.trackname, + package.name, + package.sourcepath, + package.arch, + package.md5sum)) + self.conn.commit() + + def update_package(self, package): + UPDATE_TMPL = """UPDATE packages set md5sum = ? + WHERE filename in (?)""" + self.cursor.execute(UPDATE_TMPL, (package.md5sum, package.filename)) + self.conn.commit() + + def get_package(self, filename): + self.cursor.execute(self.SELECT_PACKAGE_TMPL, (filename,)) + row = self.cursor.fetchone() + if not row: + return None + return Package(row[0], row[1], row[2], row[3], row[4], row[5]) + + def get_old_packages(self, newfiles): + SELECT_TMPL = """SELECT * FROM packages + WHERE filename not in (%s)""" + tmp = ", ".join(['?'] * len(newfiles)) + self.cursor.execute(SELECT_TMPL % tmp, newfiles) + return [Package(*row) for row in self.cursor.fetchall()] + + def remove_packages(self, packages): + DELET_TMPL = """DELETE FROM packages + WHERE filename in (%s)""" + tmp = ", ".join(['?'] * len(packages)) + self.cursor.execute(DELET_TMPL % tmp, + [package.filename for package in packages]) + self.conn.commit() + + def get_packages(self): + SELECT_TMPL = "SELECT * FROM packages" + self.cursor.execute(SELECT_TMPL) + return [Package(*row) for row in self.cursor.fetchall()] + diff -r 25d702bb13f9 -r 383d23c6bfde treepkg/info/package.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/treepkg/info/package.py Fri Aug 06 13:28:47 2010 +0000 @@ -0,0 +1,44 @@ +# Copyright (C) 2010 by Intevation GmbH +# Authors: +# Bjoern Ricks +# +# This program is free software under the GPL (>=v2) +# Read the file COPYING coming with the software for details. + +"""Classes to read information from binary packages """ + +import os.path + +from treepkg import run + +from treepkg.cmdexpand import cmdexpand + + +class DebianPackage: + + def __init__(self, filename): + self.filename = filename + + def is_package(self): + extension = os.path.splitext(self.filename)[1] + return extension.lower() == ".deb" + + def get_architecture(self): + if not self.is_package(): + return None + arch = run.capture_output(cmdexpand("dpkg-deb -f $debfile Architecture", + debfile=self.filename)) + return arch.strip() + +class BinaryPackage: + + def __init__(self, filename): + self.filename = filename + self.packagetypes = [] + self.packagetypes.append(DebianPackage(self.filename)) + + def get_architecture(self): + for packagetype in self.packagetypes: + retval = packagetype.get_architecture() + if not retval is None: + return retval diff -r 25d702bb13f9 -r 383d23c6bfde treepkg/info/status.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/treepkg/info/status.py Fri Aug 06 13:28:47 2010 +0000 @@ -0,0 +1,423 @@ +# Copyright (C) 2010 by Intevation GmbH +# Authors: +# Bjoern Ricks +# +# This program is free software under the GPL (>=v2) +# Read the file COPYING coming with the software for details. + +"""Build treepkg status information""" + +import os.path +import xml.dom.minidom + +from treepkg.report import get_packager_group +from treepkg.util import md5sum +from treepkg.info.package import BinaryPackage + +TREEPKG_NAMESPACE_URI = "http://wald.intevation.org/projects/treepkg/" +TREEPKG_NAMESPACE_PREFIX = "tpkg" + +def createTpkgElement(doc, name): + return doc.createElementNS(TREEPKG_NAMESPACE_URI, name) + +def createTpkgRoot(name): + domimpl = xml.dom.minidom.getDOMImplementation() + doc = domimpl.createDocument(TREEPKG_NAMESPACE_URI, name, None) + root = doc.documentElement + return (doc, root) + +def getTextFromNode(node): + if not node: + return None + textnode = node.firstChild + if not textnode or textnode.nodeType != node.TEXT_NODE: + return None + return textnode.data + +def getChild(node, name, required=False): + childs = getChilds(node, name, required) + if not childs or len(childs) == 0: + return None + else: + return childs[0] + +def getChilds(node, name, required=False): + if not node: + if required: + raise TreepkgInfoException("Element %s is required as child. But" + "parent element is not available.") + return None + childs = node.getElementsByTagName(name) + if not childs: + if required: + raise TreepkgInfoException("Element %s is required as child for %s." + "The XML file must be invalid." % (name, node.nodeName)) + return None + return childs + + +class TreepkgInfoException(Exception): + """ Exception class for TreepkgInfo """ + +class TreepkgInfo: + + def __init__(self, config, numnewestrev=-1, success=False): + self.numnewestrev = numnewestrev + self.success = success + group = get_packager_group(config) + treepkgrootinfo = TreepkgRootInfo(group.name, group.treepkg_dir, + group.tracks_dir) + version = "1.0" + self.tpkgroot = TreepkgRoot(version, treepkgrootinfo) + tracks = group.get_package_tracks() + + for track in tracks: + trackinfo = TreepkgTrackInfo(track.name) + self.tpkgroot.add_track(trackinfo) + self.add_revisions(track, trackinfo) + + def toxml(self): + return self.tpkgroot.toxml() + + @staticmethod + def fromxml(xmlstr): + doc = xml.dom.minidom.parseString(xmlstr) + root = doc.documentElement + if not root.tagName == "treepkg": + raise TreepkgInfoException("XML is not valid for treepkginfo") + return TreepkgRoot.fromxml(root) + + def add_revisions(self, track, trackinfo): + revisions = track.get_revisions() + revisions = sorted(revisions, key=lambda r: r.status.start, + reverse=True) + if self.numnewestrev > 0: + revisions = revisions[:self.numnewestrev] + + arch = None + + for rev in revisions: + if self.success: # skip not successful builds + if not rev.status.status.finished \ + or rev.status.status.error: + continue + revision = rev.revision + rules_revision = rev.rules_revision + status = rev.status.status.description # extend status + + sources = rev.list_source_files() + binaries = rev.list_binary_files() + + revinfo = TreepkgTrackRevisionInfo(revision, rules_revision, + status) + logs = rev.get_log_files() + for (title, filename) in logs: + loginfo = TreepkgLogInfo(title, filename) + revinfo.add_log(loginfo) + for source in sources: + self.add_package(revinfo, source, "source") + for binary in binaries: + binpackage = BinaryPackage(binary) + arch = binpackage.get_architecture() + self.add_package(revinfo, binary, "binary", arch) + + trackinfo.add_revision(revinfo) + + def add_package(self, revision, file, type, arch=None): + name = os.path.basename(file) + checksum = md5sum(file) + checksuminfo = TreepkgChecksumInfo(checksum, "md5") + pkginfo = TreepkgPackageInfo(name, file, type, arch) + pkginfo.add_checksum(checksuminfo) + revision.add_package(pkginfo) + +class TreepkgRoot: + + def __init__(self, version, info): + self.version = version + self.info = info + self.tracks = [] + + def add_track(self, track): + self.tracks.append(track) + + def toxml(self): + (doc, root) = createTpkgRoot("treepkg") + root.setAttributeNS(TREEPKG_NAMESPACE_URI, "version", self.version) + if len(self.tracks) > 0: + tracksele = createTpkgElement(doc, "tracks") + for track in self.tracks: + tracksele.appendChild(track.toxml()) + root.appendChild(tracksele) + root.appendChild(self.info.toxml()) + return root + + @staticmethod + def fromxml(node): + version = node.getAttribute("version") + tracksele = getChild(node, "tracks") + trackeles = getChilds(tracksele, "track") + infoele = getChild(node, "info") + treepkgrootinfo = TreepkgRootInfo.fromxml(infoele) + treepkgroot = TreepkgRoot(version, treepkgrootinfo) + tracks = [] + for trackele in trackeles: + tracks.append(TreepkgTrackInfo.fromxml(trackele)) + treepkgroot.tracks = tracks + return treepkgroot + +class TreepkgRootInfo: + + def __init__(self, name, treepkgpath=None, trackspath=None, version=None): + self.name = name + self.treepkgpath = treepkgpath + self.trackspath = trackspath + self.version = version + + def toxml(self): + (doc, root) = createTpkgRoot("info") + # add + nameele = createTpkgElement(doc, "name") + text = doc.createTextNode(self.name) + nameele.appendChild(text) + root.appendChild(nameele) + # add + if self.treepkgpath: + treepkgpathele = createTpkgElement(doc, "treepkgpath") + text = doc.createTextNode(self.treepkgpath) + treepkgpathele.appendChild(text) + root.appendChild(treepkgpathele) + # add + if self.trackspath: + trackspathele = createTpkgElement(doc, "trackspath") + text = doc.createTextNode(self.trackspath) + trackspathele.appendChild(text) + root.appendChild(trackspathele) + # add + if self.version: + versionele = createTpkgElement(doc, "version") + text = doc.createTextNode(self.version) + versionele.appendChild(text) + root.appendChild(versionele) + return root + + @staticmethod + def fromxml(node): + version = node.getAttribute("version") + nameele = getChild(node, "name", True) + name = getTextFromNode(nameele) + treepkgpathele = getChild(node, "treepkgpath") + treepkgpath = getTextFromNode(treepkgpathele) + trackspathele = getChild(node, "trackspath") + trackspath = getTextFromNode(trackspathele) + return TreepkgRootInfo(name, treepkgpath, trackspath, version) + +class TreepkgTrackInfo: + + def __init__(self, name, os=None): + self.name = name + self.os = os + self.revisions = [] + + def add_revision(self, revision): + self.revisions.append(revision) + + def toxml(self): + (doc, root) = createTpkgRoot("track") + nameele = createTpkgElement(doc, "name") + text = doc.createTextNode(self.name) + nameele.appendChild(text) + root.appendChild(nameele) + for rev in self.revisions: + root.appendChild(rev.toxml()) + if not self.os is None: + osele = createTpkgElement(doc, "os") + text = doc.createTextNode(self.os) + osele.appendChild(text) + root.appendChild(osele) + return root + + @staticmethod + def fromxml(node): + nameele = getChild(node, "name", True) + name = getTextFromNode(nameele) + osele = getChild(node, "os") + os = getTextFromNode(osele) + treepkgtrackinfo = TreepkgTrackInfo(name, os) + treepkgtrackinfo.revisions = [] + revisioneles = node.getElementsByTagName("revision") + for revele in revisioneles: + treepkgrevision = TreepkgTrackRevisionInfo.fromxml(revele) + treepkgtrackinfo.revisions.append(treepkgrevision) + return treepkgtrackinfo + + +class TreepkgTrackRevisionInfo: + + def __init__(self, number, rules, status): + self.number = number + self.rules = rules + self.status = status + self.packages = [] + self.logs = [] + + def add_package(self, package): + self.packages.append(package) + + def add_log(self, log): + self.logs.append(log) + + def toxml(self): + (doc, root) = createTpkgRoot("revision") + # add + numberele = createTpkgElement(doc, "number") + text = doc.createTextNode(self.number) + numberele.appendChild(text) + root.appendChild(numberele) + # add + rulesele = createTpkgElement(doc, "rules") + text = doc.createTextNode(self.rules) + rulesele.appendChild(text) + root.appendChild(rulesele) + # add + statusele = createTpkgElement(doc, "status") + messageele = createTpkgElement(doc, "message") + text = doc.createTextNode(self.status) + messageele.appendChild(text) + statusele.appendChild(messageele) + root.appendChild(statusele) + # add + packagesele = createTpkgElement(doc, "packages") + for package in self.packages: + packagesele.appendChild(package.toxml()) + root.appendChild(packagesele) + # add + logsele = createTpkgElement(doc, "logs") + for log in self.logs: + logsele.appendChild(log.toxml()) + root.appendChild(logsele) + return root + + @staticmethod + def fromxml(node): + numberele = getChild(node, "number", True) + number = getTextFromNode(numberele) + rulesele = getChild(node, "rules", True) + rules = getTextFromNode(rulesele) + statusele = getChild(node, "status", True) + messageele = getChild(statusele, "message") + message = getTextFromNode(messageele) + treepkgrevisioninfo = TreepkgTrackRevisionInfo(number, rules, message) + treepkgrevisioninfo.packages = [] + treepkgrevisioninfo.logs = [] + packagesele = getChild(node, "packages") + packageeles = getChilds(packagesele, "package") + for packageele in packageeles: + treepkgrevisioninfo.packages.append( + TreepkgPackageInfo.fromxml(packageele)) + logsele = getChild(node, "logs") + logeles = getChilds(logsele, "log") + for logele in logeles: + treepkgrevisioninfo.logs.append( + TreepkgLogInfo.fromxml(logele)) + return treepkgrevisioninfo + +class TreepkgLogInfo: + + def __init__(self, name, path): + self.name = name + self.path = path + + def toxml(self): + (doc, root) = createTpkgRoot("log") + # add + nameele = createTpkgElement(doc, "name") + text = doc.createTextNode(self.name) + nameele.appendChild(text) + root.appendChild(nameele) + # add path + pathele = createTpkgElement(doc, "path") + text = doc.createTextNode(self.path) + pathele.appendChild(text) + root.appendChild(pathele) + return root + + @staticmethod + def fromxml(node): + nameele = getChild(node, "name", True) + name = getTextFromNode(nameele) + pathele = getChild(node, "path", True) + path = getTextFromNode(pathele) + return TreepkgLogInfo(name, path) + +class TreepkgPackageInfo: + + def __init__(self, name, path, type, arch): + self.name = name + self.path = path + self.type = type + self.arch = arch + self.checksums = [] + + def toxml(self): + (doc, root) = createTpkgRoot("package") + # add + nameele = createTpkgElement(doc, "name") + text = doc.createTextNode(self.name) + nameele.appendChild(text) + root.appendChild(nameele) + # add + pathele = createTpkgElement(doc, "path") + text = doc.createTextNode(self.path) + pathele.appendChild(text) + root.appendChild(pathele) + # add + for checksum in self.checksums: + root.appendChild(checksum.toxml()) + # add + root.setAttributeNS(TREEPKG_NAMESPACE_URI, "type", self.type) + if not self.arch is None: + root.setAttributeNS(TREEPKG_NAMESPACE_URI, "arch", self.arch) + return root + + def add_checksum(self, checksum): + self.checksums.append(checksum) + + @staticmethod + def fromxml(node): + nameele = getChild(node, "name", True) + name = getTextFromNode(nameele) + pathele = getChild(node, "path", True) + path = getTextFromNode(pathele) + ptype = node.getAttribute("type") + arch = node.getAttribute("arch") + if len(arch) == 0: + arch = None + packageinfo = TreepkgPackageInfo(name, path, ptype, arch) + checksumeles = node.getElementsByTagName("checksum") + for checksumele in checksumeles: + packageinfo.checksums.append(TreepkgChecksumInfo.fromxml(checksumele)) + return packageinfo + +class TreepkgChecksumInfo: + + def __init__(self, checksum, type="md5"): + self.checksum = checksum + self.type = type + + def toxml(self): + (doc, root) = createTpkgRoot("checksum") + text = doc.createTextNode(self.checksum) + root.appendChild(text) + # add attribute type + if type: + root.setAttributeNS(TREEPKG_NAMESPACE_URI, "type", self.type) + return root + + @staticmethod + def fromxml(node): + checksum = getTextFromNode(node) + ctype = node.getAttribute("type") + if len(ctype) == 0: + return TreepkgChecksumInfo(checksum) + return TreepkgChecksumInfo(checksum, ctype) diff -r 25d702bb13f9 -r 383d23c6bfde treepkg/info/status.xsd --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/treepkg/info/status.xsd Fri Aug 06 13:28:47 2010 +0000 @@ -0,0 +1,103 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff -r 25d702bb13f9 -r 383d23c6bfde treepkg/packager.py --- a/treepkg/packager.py Wed Jul 28 08:35:07 2010 +0000 +++ b/treepkg/packager.py Fri Aug 06 13:28:47 2010 +0000 @@ -323,7 +323,7 @@ return self.build_log + ".gz" return self.build_log - def get_log_files(self, logs): + def get_log_files(self, logs=None): files = [] if os.path.isdir(self.log_dir): for f in os.listdir(self.log_dir): @@ -331,7 +331,7 @@ f = os.path.join(self.log_dir,f) if os.path.isfile(f): files.append((self.get_log_title(f),f)) - return files + return files def list_log_files(self, logs): """Returns a list describing the logfiles available for the revision. @@ -404,7 +404,8 @@ handle_dependencies=False, signing_key_id="", do_build=True, rules_svn_url=None, deb_build_options="", pkg_basename="", changelog_msg_template="Update to r%(revision)s", - svn_subset=(), svn_externals=(), git_branch="", git_url=""): + svn_subset=(), svn_externals=(), git_branch="", git_url="", + os=""): self.name = name # Convert the builder_cls option to a class @@ -435,6 +436,7 @@ self.do_build = do_build self.handle_dependencies = handle_dependencies self.dependencies = None + self.os = os self.pkg_dir_template = "%(revision)s-%(rules_revision)s" self.pkg_dir_regex = re.compile(r"(?P[0-9a-f]+)" r"-(?P[0-9a-f]+)$") @@ -642,7 +644,8 @@ def __init__(self, package_tracks, check_interval, revision=None, instructions_file=None, do_svn_update=True, - stop_on_error=False): + stop_on_error=False, name="", treepkg_dir=None, + tracks_dir=None): self.package_tracks = package_tracks self.check_interval = check_interval self.revision = revision @@ -650,6 +653,9 @@ self.stop_on_error = stop_on_error self.instructions_file = instructions_file self.instructions_file_removed = False + self.name = name + self.treepkg_dir = treepkg_dir + self.tracks_dir = tracks_dir self.sort_tracks() def sort_tracks(self): diff -r 25d702bb13f9 -r 383d23c6bfde treepkg/publish.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/treepkg/publish.py Fri Aug 06 13:28:47 2010 +0000 @@ -0,0 +1,59 @@ +#! /usr/bin/python +# Copyright (C) 2007 - 2010 by Intevation GmbH +# Authors: +# Bernhard Herzog +# Bjoern Ricks +# +# This program is free software under the GPL (>=v2) +# Read the file COPYING coming with the software for details. + +import os.path + +from treepkg.run import call, capture_output +from treepkg.cmdexpand import cmdexpand + +def remove_trailing_slashes(s): + return s.rstrip("/") + +def expand_filename(filename): + """ + Applies os.path.expanduser and os.path.expandvars to filename + """ + return os.path.expandvars(os.path.expanduser(filename)) + +def prefix_for_remote_command(user, host): + """Returns the ssh call needed to run a command on a remote host. + If host is empty, the function assumes the command is to be run on + the local host as the same user that exectutes this function, so + that no ssh or other call is needed. + """ + prefix = [] + if host: + prefix.extend(["ssh", "%s@%s" % (user, host)]) + return prefix + +def copy_to_publishdir(variables, dist, section, arch=None, quiet=False): + if not arch: + destdir = os.path.join(variables["publish_dir"], dist, section) + else: + destdir = os.path.join(variables["publish_dir"], dist, section, arch) + remote_destdir = destdir + if variables["publish_host"]: + remote_destdir = (("%(publish_user)s@%(publish_host)s:" % variables) + + remote_destdir) + runremote = prefix_for_remote_command(variables["publish_user"], + variables["publish_host"]) + + call(cmdexpand("@runremote mkdir --parents $destdir", + runremote=runremote, destdir=destdir, **variables)) + rsync_flags = [] + if variables["publish_remove_old_packages"]: + rsync_flags.append("--delete") + if quiet: + rsync_flags.append("--quiet") + call(cmdexpand("rsync @rsync_flags -r --perms --times --omit-dir-times" + " $cachedir/ $remote_destdir/", + rsync_flags=rsync_flags, remote_destdir=remote_destdir, + **variables)) + + diff -r 25d702bb13f9 -r 383d23c6bfde treepkg/readconfig.py --- a/treepkg/readconfig.py Wed Jul 28 08:35:07 2010 +0000 +++ b/treepkg/readconfig.py Fri Aug 06 13:28:47 2010 +0000 @@ -9,6 +9,7 @@ import sys import shlex +import os.path from ConfigParser import SafeConfigParser, NoOptionError import util @@ -78,12 +79,16 @@ ("changelog_msg_template", str, "Update to revision %(revision)s"), ("git_branch", str,""), ("git_url", str,""), + ("os", str, ""), ("builder_cls",str,"PBuilder") ] treepkg_desc = [ ("check_interval", int), "instructions_file", + ("name", str, ""), + ("treepkg_dir", str, None), + ("tracks_dir", str, None) ] @@ -122,6 +127,9 @@ configuration of the tree packager. The packagers list contains one dict with the configuratiin for each packager. """ + if not os.path.exists(filename): + print >>sys.stderr, "Config file %s does not exist" % filename + sys.exit(2) parser = SafeConfigParser() parser.read([filename]) @@ -133,8 +141,8 @@ packager_class = parser.get(section, "packager_class", vars=vars) module = packager.import_packager_module(packager_class) desc = packager_desc + module.PackageTrack.extra_config_desc - packager_options = (read_config_section(parser, section, desc, - defaults=vars)) + packager_options = read_config_section(parser, section, desc, + defaults=vars) if not packager_options.get("svn_url") \ and not packager_options.get('git_url'): print >>sys.stderr, "Missing repository URL in section %r" \ @@ -145,8 +153,7 @@ print >>sys.stderr, \ "Warning: git_url in section %r will be ignored" \ % (section) - packagers.append(read_config_section(parser, section, desc, - defaults=vars)) + packagers.append(packager_options) # main config treepkg = read_config_section(parser, "treepkg", treepkg_desc) diff -r 25d702bb13f9 -r 383d23c6bfde treepkg/util.py --- a/treepkg/util.py Wed Jul 28 08:35:07 2010 +0000 +++ b/treepkg/util.py Fri Aug 06 13:28:47 2010 +0000 @@ -13,6 +13,8 @@ import shutil import fnmatch import pwd +import os.path +import md5 import run @@ -156,3 +158,19 @@ def getuser(): """Returns the login name of the current user owning the proccess""" return pwd.getpwuid(os.getuid())[0] + +def md5sum(filename): + """ calculates the md5sum of a file """ + if not os.path.isfile(filename): + raise RuntimeError("Could not create md5sum. File not found: %s" + % filename) + f = file(filename, 'rb') + m = md5.new() + while True: + d = f.read(8096) + if not d: + break + m.update(d) + f.close() + return m.hexdigest() +