bricks@406: #! /usr/bin/python bricks@406: # Copyright (C) 2007 - 2010 by Intevation GmbH bricks@406: # Authors: bricks@406: # Bernhard Herzog bricks@406: # Bjoern Ricks bricks@406: # bricks@406: # This program is free software under the GPL (>=v2) bricks@406: # Read the file COPYING coming with the software for details. bricks@406: bricks@406: """Publishes selected packages created by treepkg""" bricks@406: bricks@406: import os bricks@406: import sys bricks@406: import shlex bricks@406: bricks@406: from optparse import OptionParser bricks@406: from ConfigParser import SafeConfigParser bricks@406: bricks@406: import treepkgcmd bricks@406: from treepkg.readconfig import read_config_section, convert_bool bricks@406: from treepkg.run import call, capture_output bricks@406: from treepkg.cmdexpand import cmdexpand bricks@439: from treepkg.publish import copy_arch_to_publishdir, prefix_for_remote_command bricks@438: bricks@439: from treepkg.util import md5sum, expand_filename, remove_trailing_slashes bricks@406: from treepkg.info.status import TreepkgInfo bricks@414: from treepkg.info.data import Package bricks@414: from treepkg.info.data import CacheDb bricks@406: bricks@406: config_desc = ["distribution", "section", "num_newest", bricks@406: "build_user", "build_host", "build_listpackages", bricks@417: "publish_user", "publish_host", bricks@412: ("architectures", shlex.split, "armel i386 source"), bricks@438: ("after_upload_hook", shlex.split, ""), bricks@438: ("after_copy_hook", shlex.split, ""), bricks@431: ("publish_packages", convert_bool, "True"), bricks@406: ("publish_remove_old_packages", convert_bool), bricks@406: ("publish_dir", remove_trailing_slashes), bricks@438: ("cachedb", lambda s: expand_filename(s)), bricks@406: ("cachedir", bricks@406: lambda s: expand_filename(remove_trailing_slashes(s)))] bricks@406: bricks@413: bricks@406: def read_config(filename): bricks@406: if not os.path.exists(filename): bricks@406: print >>sys.stderr, "Config file %s does not exist" % filename bricks@406: sys.exit(1) bricks@406: parser = SafeConfigParser() bricks@406: parser.read([filename]) bricks@406: return read_config_section(parser, "publishpackages", config_desc) bricks@406: bricks@406: def parse_commandline(): bricks@406: parser = OptionParser() bricks@406: parser.set_defaults(config_file=os.path.join(treepkgcmd.topdir, bricks@406: "publishpackages.cfg"), bricks@406: quiet=False) bricks@406: parser.add_option("--config-file", bricks@406: help=("The configuration file." bricks@406: " Default is publishpackages.cfg")) bricks@406: parser.add_option("--dist", bricks@406: help=("The debian distribution name to use on" bricks@406: " the publishing system")) bricks@406: parser.add_option("--section", bricks@406: help=("The debian distribution section name to use on" bricks@406: " the publishing system")) bricks@406: parser.add_option("--track", bricks@406: help=("The package track whose files are to be" bricks@406: " published. If not given, files of all tracks" bricks@406: " will be published")) bricks@406: parser.add_option("--quiet", action="store_true", bricks@406: help=("Do not print progress meters or other" bricks@406: " informational output")) bricks@406: return parser.parse_args() bricks@406: bricks@406: def get_treepkg_info(variables): bricks@406: runremote = prefix_for_remote_command(variables["build_user"], bricks@406: variables["build_host"]) bricks@406: xml = capture_output(cmdexpand("@runremote $build_listpackages" bricks@438: " --newest=$num_newest" bricks@438: " --only-successful", bricks@438: runremote=runremote, bricks@438: **variables)) bricks@408: return TreepkgInfo.fromxml(xml) bricks@406: bricks@406: def check_package_is_new(packagename, destdir, packagemd5sum): bricks@406: destpackage = os.path.join(destdir, packagename) bricks@406: if not os.path.isfile(destpackage): bricks@406: return True bricks@406: destmd5sum = md5sum(destpackage) bricks@408: return (destmd5sum != packagemd5sum) bricks@408: bricks@413: def get_md5sum(packageinfo): bricks@413: md5sum = "" bricks@413: if packageinfo: bricks@413: for checksum in packageinfo.checksums: bricks@413: if checksum.type == "md5": bricks@413: md5sum = checksum.checksum bricks@413: break bricks@413: return md5sum bricks@413: bricks@413: def sort_trackname_arch(a, b): bricks@438: if a.trackname < b.trackname: bricks@438: return -1 bricks@438: if a.trackname > b.trackname: bricks@438: return +1 bricks@413: return cmp(a.arch, b.arch) bricks@413: bricks@438: def copy_files_to_destdir(destdir, files, variables, quiet=False): bricks@408: scp_flags = [] bricks@412: if quiet: bricks@412: scp_flags.append("-q") bricks@413: bricks@413: if not os.path.exists(destdir): bricks@413: os.makedirs(destdir) bricks@417: if files: bricks@417: if variables["build_host"]: bricks@417: userhost = "%(build_user)s@%(build_host)s:" % variables bricks@417: files = [userhost + filename for filename in files] bricks@417: # scp the packages to the cache dir bricks@417: call(cmdexpand("scp -p @scp_flags @files $cachedir/", files=files, bricks@417: scp_flags=scp_flags, cachedir=destdir)) bricks@413: bricks@417: def remove_old_packages(cachedb, newpackages, quiet): bricks@416: newfiles = [package.filename for package in newpackages] bricks@416: oldpackages = cachedb.get_old_packages(newfiles) bricks@416: for package in oldpackages: bricks@416: # better check if the file really exists bricks@416: if os.path.isfile(package.filename): bricks@417: if not quiet: bricks@417: print "removing file %s" % package.filename bricks@416: os.remove(package.filename) bricks@416: cachedb.remove_packages(oldpackages) bricks@416: bricks@438: def copy_packages_to_destdir(cachedb, dir, packages, variables, quiet=False): bricks@413: packages.sort(cmp=sort_trackname_arch) bricks@413: package = packages[0] bricks@417: trackname = package.trackname bricks@413: arch = package.arch bricks@413: destdir = os.path.join(dir, arch, trackname) bricks@413: files = [] bricks@413: for package in packages: bricks@416: cachedb.add_package(package) bricks@438: if package.trackname != trackname or package.arch != arch: bricks@417: copy_files_to_destdir(destdir, files, variables, quiet) bricks@413: trackname = package.trackname bricks@413: arch = package.arch bricks@413: destdir = os.path.join(dir, arch, trackname) bricks@413: files = [] bricks@416: # add only to copy files list if the packages differ bricks@413: if check_package_is_new(package.name, destdir, package.md5sum): bricks@413: files.append(package.sourcepath) bricks@417: if not quiet: bricks@417: print "copy new file: %s" % package.name bricks@417: copy_files_to_destdir(destdir, files, variables, quiet) bricks@417: bricks@438: def copy_to_cachedir(variables, track, revision, quiet=False, bricks@438: architectures=()): bricks@408: cachedir = variables["cachedir"] bricks@416: cachdebfilename = variables["cachedb"] bricks@417: if not quiet: bricks@417: print "using cachedb: %s" % cachdebfilename bricks@416: cachedb = CacheDb(cachdebfilename) bricks@413: newpackages = [] bricks@406: treepkginfo = get_treepkg_info(variables) bricks@438: allarchs = set() # contains all present architectures (incl. source) bricks@408: binaryallpackages = [] bricks@406: # change e.g. armel in binary-armel bricks@438: allowedarchs = set([get_binary_arch(a) for a in architectures]) bricks@408: for track in treepkginfo.tracks: bricks@406: for rev in track.revisions: bricks@416: for packageinfo in rev.packages: bricks@416: arch = get_binary_arch(packageinfo.arch) bricks@416: if packageinfo.type == "binary": bricks@412: # skip other files bricks@416: if packageinfo.arch is None: bricks@412: continue bricks@412: # handle binary-all bricks@412: if arch == "binary-all": bricks@412: # add trackname for subdir name bricks@416: packageinfo.trackname = track.name bricks@416: binaryallpackages.append(packageinfo) bricks@412: continue bricks@412: allarchs.add(arch) bricks@416: elif packageinfo.type == "source": bricks@416: arch = packageinfo.type bricks@408: # only copy requested archs bricks@408: if len(allowedarchs) == 0 or \ bricks@412: arch in allowedarchs: bricks@413: filename = os.path.join(cachedir, arch, track.name, bricks@416: packageinfo.name) bricks@416: newpackage = Package(filename, track.name, packageinfo.name, bricks@416: packageinfo.path, arch, bricks@416: get_md5sum(packageinfo)) bricks@413: newpackages.append(newpackage) bricks@408: # copy binary-all packages bricks@412: sourcearch = set(["source"]) bricks@438: if not allowedarchs: bricks@412: binallarchs = allarchs - sourcearch bricks@438: elif allarchs: bricks@412: binallarchs = allowedarchs - sourcearch bricks@408: else: bricks@412: binallarchs = (allowedarchs & allarchs) - sourcearch bricks@416: for packageinfo in binaryallpackages: bricks@408: for arch in binallarchs: bricks@417: filename = os.path.join(cachedir, arch, packageinfo.trackname, bricks@416: packageinfo.name) bricks@438: newpackage = Package(filename, packageinfo.trackname, bricks@438: packageinfo.name, packageinfo.path, arch, bricks@438: get_md5sum(packageinfo)) bricks@413: newpackages.append(newpackage) bricks@417: copy_packages_to_destdir(cachedb, cachedir, newpackages, variables, quiet) bricks@417: remove_old_packages(cachedb, newpackages, quiet) bricks@424: return binallarchs bricks@406: bricks@406: def publish_packages(config_filename, track, revision, dist, section, quiet): bricks@406: config = read_config(config_filename) bricks@406: bricks@406: if dist is None: bricks@406: dist = config["distribution"] bricks@406: if section is None: bricks@406: section = config["section"] bricks@406: bricks@406: architectures = config["architectures"] bricks@429: copy_to_cachedir(config, track, revision, quiet, architectures) bricks@430: bricks@438: if config["after_copy_hook"]: bricks@438: if not quiet: bricks@438: print "running after copy hook" bricks@438: call(config["after_copy_hook"]) bricks@430: bricks@430: if config["publish_packages"]: bricks@430: for arch in architectures: bricks@431: if not quiet: bricks@436: print "publish packages for architecture %s" % arch bricks@438: copy_arch_to_publishdir(config, dist, section, bricks@438: get_binary_arch(arch), quiet) bricks@406: bricks@406: # update apt archive bricks@438: if config["after_upload_hook"]: bricks@417: if not quiet: bricks@417: print "running after upload hook" bricks@417: call(config["after_upload_hook"]) bricks@406: bricks@406: def main(): bricks@406: options, args = parse_commandline() bricks@406: revision = None # for future use cases bricks@406: publish_packages(options.config_file, options.track, revision, bricks@406: options.dist, options.section, options.quiet) bricks@406: bricks@406: if __name__ == "__main__": bricks@406: main()