Mercurial > treepkg > treepkg
view treepkg/packager.py @ 91:3ed079a7174a
Implement a way to stop a running treepackager.
author | Bernhard Herzog <bh@intevation.de> |
---|---|
date | Tue, 11 Sep 2007 17:24:56 +0000 |
parents | 6ed1c881ee1b |
children | 73c67372c7f7 |
line wrap: on
line source
# Copyright (C) 2007 by Intevation GmbH # Authors: # Bernhard Herzog <bh@intevation.de> # # This program is free software under the GPL (>=v2) # Read the file COPYING coming with the software for details. """Classes to automatically build debian packages from subversion checkouts""" import os import time import re import logging import shutil import datetime import util import subversion import run import status from cmdexpand import cmdexpand def _filenameproperty(relative_dir): def get(self): return os.path.join(self.base_dir, relative_dir) return property(get) class SourcePackager(object): # Derived classes must supply the package basename pkg_basename = None def __init__(self, track, status, work_dir, src_dir, revision): self.track = track self.status = status self.work_dir = work_dir self.src_dir = src_dir self.revision = revision assert(self.pkg_basename) def determine_package_version(self, directory): """Returns the version number of the new package as a string The directory parameter is the name of the directory containing the newly exported sources. The sources were exported with the export_sources method. The default implementation simply returns the revision converted to a string. """ return str(self.revision) def export_sources(self): """Export the sources from the subversion working directory This method first exports the sources to a temporary directory and then renames the directory. The new name is of the form <pkg_basename>-<version> Where pkg_basename is the value of self.pkg_basename and version is the return value of the determine_package_version() method. """ temp_dir = os.path.join(self.work_dir, "temp") self.track.export_sources(temp_dir) pkgbaseversion = self.determine_package_version(temp_dir) pkgbasedir = os.path.join(self.work_dir, self.pkg_basename + "-" + pkgbaseversion) os.rename(temp_dir, pkgbasedir) return pkgbaseversion, pkgbasedir def update_version_numbers(self, pkgbasedir): """Updates the version numbers in the code in pkgbasedir. The default implementation does nothing. Derived classes should override this method if necessary. """ def create_tarball(self, tarballname, workdir, basedir): """Creates a new tarball. Parameters: tarballname -- the filename of the new tarball workdir -- The directory into which to change before running tar. (actually this is done with GNUI tar's -C option) basedir -- The basedirectory of the files that are packaged into the tarfile. This should be a relative filename directly in workdir. """ logging.info("Creating tarball %r", tarballname) run.call(cmdexpand("tar czf $tarballname -C $workdir $basedir", **locals())) def copy_debian_directory(self, pkgbasedir, pkgbaseversion, changemsg): """Copies the debian directory and updates the copy's changelog Parameter: pkgbasedir -- The directory holding the unpacked source package pkgbaseversion -- The version to update the changelog to changemsg -- The message for the changelog When determining the actual version for the new package, this function looks at the previous version in the changelog. If it has a prefix separated from the version number by a colon this prefix is prepended to the pkgbaseversion parameter. Debian uses such prefixes for the kde packages. """ debian_dir = os.path.join(pkgbasedir, "debian") changelog = os.path.join(debian_dir, "changelog") self.track.copy_debian_directory(debian_dir) logging.info("Updating %r", changelog) oldversion = util.debian_changelog_version(changelog) if ":" in oldversion: oldversionprefix = oldversion.split(":")[0] + ":" else: oldversionprefix = "" run.call(cmdexpand("debchange -c $changelog" " -v ${oldversionprefix}${pkgbaseversion}-kk1" " $changemsg", **locals()), env=self.track.debian_environment()) def create_source_package(self, pkgbasedir, origtargz): """Creates a new source package from pkgbasedir and origtargz""" logging.info("Creating new source package") run.call(cmdexpand("dpkg-source -b $directory $tarball", directory=os.path.basename(pkgbasedir), tarball=os.path.basename(origtargz)), cwd=os.path.dirname(pkgbasedir), suppress_output=True, env=self.track.debian_environment()) def move_source_package(self, pkgbasename): """Moves the new source package from the work_dir to the src_dir""" logging.info("Moving source package to %r", self.src_dir) util.ensure_directory(self.src_dir) for filename in [filename for filename in os.listdir(self.work_dir) if filename.startswith(pkgbasename)]: os.rename(os.path.join(self.work_dir, filename), os.path.join(self.src_dir, filename)) def package(self): """Creates a source package from a subversion checkout. After setting up the working directory, this method calls the do_package method to do the actual packaging. Afterwards the work directory is removed. """ util.ensure_directory(self.work_dir) try: self.status.creating_source_package() self.do_package() self.status.source_package_created() finally: logging.info("Removing workdir %r", self.work_dir) shutil.rmtree(self.work_dir) def do_package(self): """Does the work of creating a source package This method must be overriden by derived classes. The method should do the work in self.work_dir. When the package is done, the source package files should be in self.src_dir. """ raise NotImplementedError class BinaryPackager(object): def __init__(self, track, status, binary_dir, dsc_file, logfile): self.track = track self.status = status self.binary_dir = binary_dir self.dsc_file = dsc_file self.logfile = logfile def package(self): self.status.creating_binary_package() util.ensure_directory(self.binary_dir) logging.info("Building binary package; logging to %r", self.logfile) run.call(cmdexpand("@rootcmd /usr/sbin/pbuilder build" " --configfile $pbuilderrc" " --logfile $logfile --buildresult $bindir $dsc", rootcmd=self.track.root_cmd, pbuilderrc=self.track.pbuilderrc, logfile=self.logfile, bindir=self.binary_dir, dsc=self.dsc_file), suppress_output=True) # remove the source package files put into the binary directory # by pbuilder for filename in os.listdir(self.binary_dir): if os.path.splitext(filename)[1] not in (".deb", ".changes"): os.remove(os.path.join(self.binary_dir, filename)) self.status.binary_package_created() class RevisionPackager(object): source_packager_cls = SourcePackager binary_packager_cls = BinaryPackager def __init__(self, track, revision): self.track = track self.revision = revision self.base_dir = self.track.pkg_dir_for_revision(self.revision, 1) self.status = status.RevisionStatus(os.path.join(self.base_dir, "status")) work_dir = _filenameproperty("work") binary_dir = _filenameproperty("binary") src_dir = _filenameproperty("src") build_log = _filenameproperty("build.log") def find_dsc_file(self): for filename in os.listdir(self.src_dir): if filename.endswith(".dsc"): return os.path.join(self.src_dir, filename) return None def has_build_log(self): return os.path.exists(self.build_log) def list_source_files(self): """Returns a list with the names of the files of the source package. The implementation assumes that all files in self.src_dir belong to the source package. """ return sorted(util.listdir_abs(self.src_dir)) def list_binary_files(self): """Returns a list with the names of the files of the binary packages. The implementation assumes that all files in self.binary_dir belong to the binary packages. """ return sorted(util.listdir_abs(self.binary_dir)) def package(self): try: util.ensure_directory(self.work_dir) self.status.start = datetime.datetime.utcnow() src_packager = self.source_packager_cls(self.track, self.status, self.work_dir, self.src_dir, self.revision) src_packager.package() dsc_file = self.find_dsc_file() if dsc_file is None: raise RuntimeError("Cannot find dsc File in %r" % self.src_dir) bin_packager = self.binary_packager_cls(self.track, self.status, self.binary_dir, dsc_file, self.build_log) bin_packager.package() self.status.stop = datetime.datetime.utcnow() except: self.status.error() self.status.stop = datetime.datetime.utcnow() raise def remove_package_dir(self): logging.info("Removing pkgdir %r", self.base_dir) shutil.rmtree(self.base_dir) class PackageTrack(object): revision_packager_cls = RevisionPackager svn_external_subdirs = [] extra_config_desc = [] def __init__(self, name, base_dir, svn_url, root_cmd, pbuilderrc, deb_email, deb_fullname, packager_class="treepkg.packager"): self.name = name self.base_dir = base_dir self.svn_url = svn_url self.root_cmd = root_cmd self.pbuilderrc = pbuilderrc self.deb_email = deb_email self.deb_fullname = deb_fullname self.pkg_dir_template = "%(revision)d-%(increment)d" self.pkg_dir_regex \ = re.compile(r"(?P<revision>[0-9]+)-(?P<increment>[0-9]+)$") checkout_dir = _filenameproperty("checkout") debian_dir = _filenameproperty("debian") pkg_dir = _filenameproperty("pkg") def pkg_dir_for_revision(self, revision, increment): return os.path.join(self.pkg_dir, self.pkg_dir_template % locals()) def last_changed_revision(self): revisions = [] for directory in [self.checkout_dir] + self.svn_external_subdirs: directory = os.path.join(self.checkout_dir, directory) revisions.append(subversion.last_changed_revision(directory)) return max(revisions) def get_revision_numbers(self): """Returns a list of the numbers of the packaged revisions""" revisions = [] if os.path.exists(self.pkg_dir): for filename in os.listdir(self.pkg_dir): match = self.pkg_dir_regex.match(filename) if match: revisions.append(int(match.group("revision"))) revisions.sort() return revisions def last_packaged_revision(self): """Returns the revision number of the highest packaged revision. If the revision cannot be determined because no already packaged revisions can be found, the function returns -1. """ return max([-1] + self.get_revision_numbers()) def debian_source(self): return util.extract_value_for_key(open(os.path.join(self.debian_dir, "control")), "Source:") def update_checkout(self, revision=None): """Updates the working copy of self.svn_url in self.checkout_dir. If self.checkout_dir doesn't exist yet, self.svn_url is checked out into that directory. The value of the revision parameter is passed through to subversion.update. """ localdir = self.checkout_dir if os.path.exists(localdir): logging.info("Updating the working copy in %r", localdir) subversion.update(localdir, revision=revision) else: logging.info("The working copy in %r doesn't exist yet." " Checking out from %r", localdir, self.svn_url) subversion.checkout(self.svn_url, localdir) def export_sources(self, to_dir): logging.info("Exporting sources for tarball to %r", to_dir) subversion.export(self.checkout_dir, to_dir) # some versions of svn (notably version 1.4.2 shipped with etch) # do export externals such as the admin subdirectory. We may # have to do that in an extra step. for subdir in self.svn_external_subdirs: absdir = os.path.join(to_dir, subdir) if not os.path.isdir(absdir): subversion.export(os.path.join(self.checkout_dir, subdir), absdir) def copy_debian_directory(self, to_dir): logging.info("Copying debian directory to %r", to_dir) shutil.copytree(self.debian_dir, to_dir) def debian_environment(self): """Returns the environment variables for the debian commands""" env = os.environ.copy() env["DEBFULLNAME"] = self.deb_fullname env["DEBEMAIL"] = self.deb_email return env def package_if_updated(self, revision=None): """Checks if the checkout changed and returns a new packager if so""" self.update_checkout(revision=revision) current_revision = self.last_changed_revision() logging.info("New revision is %d", current_revision) previous_revision = self.last_packaged_revision() logging.info("Previously packaged revision was %d", previous_revision) if current_revision > previous_revision: logging.info("New revision has not been packaged yet") return self.revision_packager_cls(self, current_revision) else: logging.info("New revision already packaged.") def get_revisions(self): """Returns RevisionPackager objects for each packaged revision""" return [self.revision_packager_cls(self, revision) for revision in self.get_revision_numbers()] def create_package_track(packager_class, **kw): module = util.import_dotted_name(packager_class) return module.PackageTrack(**kw) class PackagerGroup(object): def __init__(self, package_tracks, check_interval, revision=None, instructions_file=None): self.package_tracks = package_tracks self.check_interval = check_interval self.revision = revision self.instructions_file = instructions_file self.instructions_file_removed = False def run(self): """Runs the packager group indefinitely""" logging.info("Starting in periodic check mode." " Will check every %d seconds", self.check_interval) last_check = -1 while 1: now = time.time() if now > last_check + self.check_interval: if self.check_package_tracks(): break last_check = now next_check = now + self.check_interval to_sleep = next_check - time.time() if to_sleep > 0: logging.info("Next check at %s", time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(next_check))) time.sleep(to_sleep) else: logging.info("Next check now") if self.should_stop(): logging.info("Received stop instruction. Stopping.") return def check_package_tracks(self): logging.info("Checking package tracks") self.clear_instruction() for track in self.package_tracks: try: packager = track.package_if_updated(revision=self.revision) if packager: packager.package() if self.should_stop(): logging.info("Received stop instruction. Stopping.") return True except: logging.exception("An error occurred while" " checking packager track %r", track.name) logging.info("Checked all package tracks") def get_package_tracks(self): return self.package_tracks def read_instruction(self): if not self.instructions_file: return "" try: f = open(self.instructions_file) except (IOError, OSError): return "" try: return f.read().strip() finally: f.close() self.clear_instruction() def clear_instruction(self, force=False): if self.instructions_file and (not self.instructions_file_removed or force): util.writefile(self.instructions_file, "") self.instructions_file_removed = True def should_stop(self): return self.read_instruction() == "stop"