view treepkg/packager.py @ 0:f78a02e79c84

initial checkin
author Bernhard Herzog <bh@intevation.de>
date Tue, 06 Mar 2007 17:37:32 +0100
parents
children e6a9f4037f68
line wrap: on
line source
# Copyright (C) 2007 by Intevation GmbH
# Authors:
# Bernhard Herzog <bh@intevation.de>
#
# This program is free software under the GPL (>=v2)
# Read the file COPYING coming with the software for details.

"""Classes to automatically build debian packages from subversion checkouts"""

import os
import time
import re
import logging
import shutil
import traceback

import util
import subversion
import run


def _filenameproperty(relative_dir):
    def get(self):
        return os.path.join(self.base_dir, relative_dir)
    return property(get)


class SourcePackager(object):

    def __init__(self, plant, status, work_dir, src_dir, revision):
        self.plant = plant
        self.status = status
        self.work_dir = work_dir
        self.src_dir = src_dir
        self.revision = revision
        self.enterprise_version = (time.strftime("%Y%m%d", time.localtime()) \
                                   + "." + str(self.revision))

    def kdepim_version(self, directory):
        """Determine the kdepim version.

        The version is taken from the kdepim.lsm file in the plants
        checkout dir.
        """
        return util.extract_lsm_version(os.path.join(directory, "kdepim.lsm"))

    def determine_package_version(self, directory):
        enterprise_version = self.enterprise_version
        kdepimversion = self.kdepim_version(directory)
        version_template = "%(kdepimversion)s.enterprise.0.%(enterprise_version)s"
        return version_template % locals()

    def export_sources(self):
        temp_dir = os.path.join(self.work_dir, "temp")
        self.plant.export_sources(temp_dir)

        pkgbaseversion = self.determine_package_version(temp_dir)
        pkgbasedir = os.path.join(self.work_dir, "kdepim-" + pkgbaseversion)

        os.rename(temp_dir, pkgbasedir)
        return pkgbaseversion, pkgbasedir


    def update_version_numbers(self, pkgbasedir):
        versionstring = "(enterprise %s)" % self.enterprise_version
        for versionfile in ["kmail/kmversion.h", "kontact/src/main.cpp",
                            "korganizer/version.h"]:
            filename = os.path.join(pkgbasedir, versionfile)
            patched = re.sub("\(enterprise ([^)]*)\)", versionstring,
                              open(filename).read())
            f = open(filename, "w")
            f.write(patched)
            f.close()

    def create_tarball(self, tarballname, workdir, basedir):
        logging.info("Creating tarball %r", tarballname)
        run.call(["tar", "czf", tarballname, "-C", workdir, basedir])

    def copy_debian_directory(self, pkgbasedir, pkgbaseversion, changemsg):
        debian_dir = os.path.join(pkgbasedir, "debian")
        changelog = os.path.join(debian_dir, "changelog")

        self.plant.copy_debian_directory(debian_dir)

        logging.info("Updating %r", changelog)
        oldversion = util.debian_changelog_version(changelog)
        if ":" in oldversion:
            oldversionprefix = oldversion.split(":")[0] + ":"
        else:
            oldversionprefix = ""
        run.call(["debchange", "-c", changelog,
                  "-v", oldversionprefix + pkgbaseversion + "-kk1",
                  changemsg],
                 env=self.plant.debian_environment())


    def create_source_package(self, pkgbasedir, origtargz):
        logging.info("Creating new source package")
        run.call(["dpkg-source", "-b", os.path.basename(pkgbasedir),
                  os.path.basename(origtargz)],
                 cwd=os.path.dirname(pkgbasedir),
                 suppress_output=True,
                 env=self.plant.debian_environment())

    def package(self):
        util.ensure_directory(self.work_dir)
        try:
            self.status.set("creating_source_package")
            pkgbaseversion, pkgbasedir = self.export_sources()
            self.update_version_numbers(pkgbasedir)

            pkgbasename = "kdepim_" + pkgbaseversion
            origtargz = os.path.join(self.work_dir,
                                     pkgbasename + ".orig.tar.gz")
            self.create_tarball(origtargz, self.work_dir,
                                os.path.basename(pkgbasedir))

            changemsg = ("Update to SVN enterprise branch rev. %d"
                         % (self.revision,))
            self.copy_debian_directory(pkgbasedir, pkgbaseversion,
                                       changemsg)

            self.create_source_package(pkgbasedir, origtargz)

            logging.info("Moving source package to %r", self.src_dir)
            util.ensure_directory(self.src_dir)
            for filename in [filename for filename in os.listdir(self.work_dir)
                                      if filename.startswith(pkgbasename)]:
                os.rename(os.path.join(self.work_dir, filename),
                          os.path.join(self.src_dir, filename))
            self.status.set("source_package_created")
        finally:
            logging.info("Removing workdir %r", self.work_dir)
            shutil.rmtree(self.work_dir)


class BinaryPackager(object):

    def __init__(self, plant, status, binary_dir, dsc_file, logfile):
        self.plant = plant
        self.status = status
        self.binary_dir = binary_dir
        self.dsc_file = dsc_file
        self.logfile = logfile

    def package(self):
        self.status.set("creating_binary_package")
        util.ensure_directory(self.binary_dir)
        logging.info("Building binary package; loging to %r", self.logfile)
        cmd = []
        if self.plant.root_cmd:
            cmd.append(self.plant.root_cmd)
        run.call(cmd + ["/usr/sbin/pbuilder", "build",
                  "--logfile", self.logfile,
                  "--buildresult", self.binary_dir,
                  self.dsc_file],
                 suppress_output=True)
        self.status.set("binary_package_created")


class RevisionPackager(object):

    def __init__(self, plant, revision):
        self.plant = plant
        self.revision = revision
        self.base_dir = self.plant.pkg_dir_for_revision(self.revision, 1)
        self.status = util.StatusFile(os.path.join(self.base_dir, "status"))

    work_dir = _filenameproperty("work")
    binary_dir = _filenameproperty("binary")
    src_dir = _filenameproperty("src")

    def find_dsc_file(self):
        for filename in os.listdir(self.src_dir):
            if filename.endswith(".dsc"):
                return os.path.join(self.src_dir, filename)
        return None

    def package(self):
        try:
            src_packager = SourcePackager(self.plant, self.status,
                                          self.work_dir, self.src_dir,
                                          self.revision)
            src_packager.package()

            dsc_file = self.find_dsc_file()
            if dsc_file is None:
                raise RuntimeError("Cannot find dsc File in %r" % self.src_dir)

            bin_packager = BinaryPackager(self.plant, self.status,
                                          self.binary_dir, dsc_file,
                                          os.path.join(self.base_dir,
                                                       "build.log"))
            bin_packager.package()
        except:
            self.status.set("failed", traceback.format_exc())
            raise

    def remove_package_dir(self):
        logging.info("Removing pkgdir %r", self.base_dir)
        shutil.rmtree(self.base_dir)


class AssemblyLine(object):

    def __init__(self, name, base_dir, svn_url, root_cmd, deb_email,
                 deb_fullname):
        self.name = name
        self.base_dir = base_dir
        self.svn_url = svn_url
        self.root_cmd = root_cmd
        self.deb_email = deb_email
        self.deb_fullname = deb_fullname
        self.pkg_dir_template = "%(revision)d-%(increment)d"
        self.pkg_dir_regex \
                   = re.compile(r"(?P<revision>[0-9]+)-(?P<increment>[0-9]+)$")

    checkout_dir = _filenameproperty("checkout")
    debian_dir = _filenameproperty("debian")
    pkg_dir = _filenameproperty("pkg")

    def pkg_dir_for_revision(self, revision, increment):
        return os.path.join(self.pkg_dir,
                            self.pkg_dir_template % locals())

    def last_changed_revision(self):
        rev1 = subversion.last_changed_revision(self.checkout_dir)
        rev2 = subversion.last_changed_revision(os.path.join(self.checkout_dir,
                                                             "admin"))
        return max(rev1, rev2)

    def last_packaged_revision(self):
        """Returns the revision number of the highest packaged revision.

        If the revision cannot be determined because no already packaged
        revisions can be found, the function returns -1.
        """
        revisions = [-1]
        if os.path.exists(self.pkg_dir):
            for filename in os.listdir(self.pkg_dir):
                match = self.pkg_dir_regex.match(filename)
                if match:
                    revisions.append(int(match.group("revision")))
        return max(revisions)

    def debian_source(self):
        return util.extract_value_for_key(open(os.path.join(self.debian_dir,
                                                            "control")),
                                          "Source:")

    def update_checkout(self):
        """Updates the working copy of self.svn_url in self.checkout_dir.

        If self.checkout_dir doesn't exist yet, self.svn_url is checked
        out into that directory.
        """
        localdir = self.checkout_dir
        if os.path.exists(localdir):
            logging.info("Updating the working copy in %r", localdir)
            subversion.update(localdir)
        else:
            logging.info("The working copy in %r doesn't exist yet."
                         "  Checking out fromo %r", localdir,
                         self.svn_url)
            subversion.checkout(self.svn_url, localdir)

    def export_sources(self, to_dir):
        logging.info("Exporting sources for tarball to %r", to_dir)
        subversion.export(self.checkout_dir, to_dir)
        # some versions of svn (notably version 1.4.2 shipped with etch)
        # do export externals such as the admin subdirectory.  We may
        # have to do that in an extra step.
        admindir = os.path.join(to_dir, "admin")
        if not os.path.isdir(admindir):
            subversion.export(os.path.join(self.checkout_dir, "admin"),
                              admindir)

    def copy_debian_directory(self, to_dir):
        logging.info("Copying debian directory to %r", to_dir)
        shutil.copytree(self.debian_dir, to_dir)

    def debian_environment(self):
        """Returns the environment variables for the debian commands"""
        env = os.environ.copy()
        env["DEBFULLNAME"] = self.deb_fullname
        env["DEBEMAIL"] = self.deb_email
        return env

    def package_if_updated(self):
        """Checks if the checkout changed and returns a new packager if so"""
        self.update_checkout()
        current_revision = self.last_changed_revision()
        logging.info("New revision is %d", current_revision)
        previous_revision = self.last_packaged_revision()
        logging.info("Previously packaged revision was %d", previous_revision)
        if current_revision > previous_revision:
            logging.info("New revision is not packaged yet")
            return RevisionPackager(self, current_revision)
        else:
            logging.info("New revision already packaged.")



class Packager(object):

    def __init__(self, assembly_lines, check_interval):
        self.assembly_lines = assembly_lines
        self.check_interval = check_interval

    def run(self):
        """Runs the plant indefinitely"""
        logging.info("Packager start.  Will check every %d seconds",
                     self.check_interval)
        last_check = -1
        while 1:
            now = time.time()
            if now > last_check + self.check_interval:
                self.check_assembly_lines()
                last_check = now
                next_check = now + self.check_interval
                to_sleep = next_check - time.time()
                if to_sleep > 0:
                    logging.info("Next check at %s",
                                 time.strftime("%Y-%m-%d %H:%M:%S",
                                               time.localtime(next_check)))
                    time.sleep(to_sleep)
                else:
                    logging.info("Next check now")

    def check_assembly_lines(self):
        logging.info("Checking assembly lines")
        for line in self.assembly_lines:
            try:
                packager = line.package_if_updated()
                if packager:
                    packager.package()
            except:
                logging.exception("An error occurred while"
                                  " checking assembly line %r", line.name)
        logging.info("Checked all assembly lines")
This site is hosted by Intevation GmbH (Datenschutzerklärung und Impressum | Privacy Policy and Imprint)