bh@0: # Copyright (C) 2007 by Intevation GmbH
bh@0: # Authors:
bh@0: # Bernhard Herzog <bh@intevation.de>
bh@0: #
bh@0: # This program is free software under the GPL (>=v2)
bh@0: # Read the file COPYING coming with the software for details.
bh@0: 
bh@0: """Classes to automatically build debian packages from subversion checkouts"""
bh@0: 
bh@0: import os
bh@0: import time
bh@0: import re
bh@0: import logging
bh@0: import shutil
bh@16: import datetime
bh@0: 
bh@0: import util
bh@0: import subversion
bh@0: import run
bh@16: import status
bh@45: from cmdexpand import cmdexpand
bh@0: 
bh@0: def _filenameproperty(relative_dir):
bh@0:     def get(self):
bh@0:         return os.path.join(self.base_dir, relative_dir)
bh@0:     return property(get)
bh@0: 
bh@0: 
bh@0: class SourcePackager(object):
bh@0: 
bh@4:     # Derived classes must supply the package basename
bh@4:     pkg_basename = None
bh@4: 
bh@53:     def __init__(self, track, status, work_dir, src_dir, revision):
bh@53:         self.track = track
bh@0:         self.status = status
bh@0:         self.work_dir = work_dir
bh@0:         self.src_dir = src_dir
bh@0:         self.revision = revision
bh@4:         assert(self.pkg_basename)
bh@0: 
bh@0:     def determine_package_version(self, directory):
bh@4:         """Returns the version number of the new package as a string
bh@4: 
bh@4:         The directory parameter is the name of the directory containing
bh@4:         the newly exported sources.  The sources were exported with the
bh@4:         export_sources method.
bh@4: 
bh@4:         The default implementation simply returns the revision converted
bh@4:         to a string.
bh@4:         """
bh@4:         return str(self.revision)
bh@0: 
bh@0:     def export_sources(self):
bh@4:         """Export the sources from the subversion working directory
bh@4: 
bh@4:         This method first exports the sources to a temporary directory
bh@4:         and then renames the directory.  The new name is of the form
bh@4: 
bh@4:           <pkg_basename>-<version>
bh@4: 
bh@4:         Where pkg_basename is the value of self.pkg_basename and version
bh@4:         is the return value of the determine_package_version() method.
bh@4:         """
bh@0:         temp_dir = os.path.join(self.work_dir, "temp")
bh@53:         self.track.export_sources(temp_dir)
bh@0: 
bh@0:         pkgbaseversion = self.determine_package_version(temp_dir)
bh@4:         pkgbasedir = os.path.join(self.work_dir,
bh@4:                                   self.pkg_basename + "-" + pkgbaseversion)
bh@0: 
bh@0:         os.rename(temp_dir, pkgbasedir)
bh@0:         return pkgbaseversion, pkgbasedir
bh@0: 
bh@0: 
bh@0:     def update_version_numbers(self, pkgbasedir):
bh@4:         """Updates the version numbers in the code in pkgbasedir.
bh@4: 
bh@4:         The default implementation does nothing.  Derived classes should
bh@4:         override this method if necessary.
bh@4:         """
bh@0: 
bh@0:     def create_tarball(self, tarballname, workdir, basedir):
bh@4:         """Creates a new tarball.
bh@4: 
bh@4:         Parameters:
bh@4: 
bh@4:           tarballname -- the filename of the new tarball
bh@4:           workdir -- The directory into which to change before running tar.
bh@4:                      (actually this is done with GNUI tar's -C option)
bh@4:           basedir -- The basedirectory of the files that are packaged
bh@4:                      into the tarfile.  This should be a relative
bh@4:                      filename directly in workdir.
bh@4:         """
bh@0:         logging.info("Creating tarball %r", tarballname)
bh@45:         run.call(cmdexpand("tar czf $tarballname -C $workdir $basedir",
bh@45:                            **locals()))
bh@0: 
bh@0:     def copy_debian_directory(self, pkgbasedir, pkgbaseversion, changemsg):
bh@4:         """Copies the debian directory and updates the copy's changelog
bh@4: 
bh@4:         Parameter:
bh@4:           pkgbasedir -- The directory holding the unpacked source package
bh@4:           pkgbaseversion -- The version to update the changelog to
bh@4:           changemsg -- The message for the changelog
bh@4: 
bh@4:         When determining the actual version for the new package, this
bh@4:         function looks at the previous version in the changelog.  If it
bh@4:         has a prefix separated from the version number by a colon this
bh@4:         prefix is prepended to the pkgbaseversion parameter.  Debian
bh@4:         uses such prefixes for the kde packages.
bh@4:         """
bh@0:         debian_dir = os.path.join(pkgbasedir, "debian")
bh@0:         changelog = os.path.join(debian_dir, "changelog")
bh@0: 
bh@53:         self.track.copy_debian_directory(debian_dir)
bh@0: 
bh@0:         logging.info("Updating %r", changelog)
bh@0:         oldversion = util.debian_changelog_version(changelog)
bh@0:         if ":" in oldversion:
bh@0:             oldversionprefix = oldversion.split(":")[0] + ":"
bh@0:         else:
bh@0:             oldversionprefix = ""
bh@45:         run.call(cmdexpand("debchange -c  $changelog"
bh@45:                            " -v ${oldversionprefix}${pkgbaseversion}-kk1"
bh@45:                            " $changemsg", **locals()),
bh@53:                  env=self.track.debian_environment())
bh@0: 
bh@0: 
bh@0:     def create_source_package(self, pkgbasedir, origtargz):
bh@4:         """Creates a new source package from pkgbasedir and origtargz"""
bh@0:         logging.info("Creating new source package")
bh@45:         run.call(cmdexpand("dpkg-source -b $directory $tarball",
bh@45:                            directory=os.path.basename(pkgbasedir),
bh@45:                            tarball=os.path.basename(origtargz)),
bh@0:                  cwd=os.path.dirname(pkgbasedir),
bh@0:                  suppress_output=True,
bh@53:                  env=self.track.debian_environment())
bh@0: 
bh@4:     def move_source_package(self, pkgbasename):
bh@4:         """Moves the new source package from the work_dir to the src_dir"""
bh@4:         logging.info("Moving source package to %r", self.src_dir)
bh@4:         util.ensure_directory(self.src_dir)
bh@4:         for filename in [filename for filename in os.listdir(self.work_dir)
bh@4:                                   if filename.startswith(pkgbasename)]:
bh@4:             os.rename(os.path.join(self.work_dir, filename),
bh@4:                       os.path.join(self.src_dir, filename))
bh@4: 
bh@0:     def package(self):
bh@4:         """Creates a source package from a subversion checkout.
bh@4: 
bh@4:         After setting up the working directory, this method calls the
bh@4:         do_package method to do the actual packaging.  Afterwards the
bh@4:         work directory is removed.
bh@4:         """
bh@0:         util.ensure_directory(self.work_dir)
bh@0:         try:
bh@41:             self.status.creating_source_package()
bh@4:             self.do_package()
bh@41:             self.status.source_package_created()
bh@0:         finally:
bh@0:             logging.info("Removing workdir %r", self.work_dir)
bh@0:             shutil.rmtree(self.work_dir)
bh@0: 
bh@4:     def do_package(self):
bh@4:         """Does the work of creating a source package
bh@4:         This method must be overriden by derived classes.
bh@4: 
bh@4:         The method should do the work in self.work_dir.  When the
bh@4:         package is done, the source package files should be in
bh@4:         self.src_dir.
bh@4:         """
bh@4:         raise NotImplementedError
bh@4: 
bh@0: 
bh@0: class BinaryPackager(object):
bh@0: 
bh@53:     def __init__(self, track, status, binary_dir, dsc_file, logfile):
bh@53:         self.track = track
bh@0:         self.status = status
bh@0:         self.binary_dir = binary_dir
bh@0:         self.dsc_file = dsc_file
bh@0:         self.logfile = logfile
bh@0: 
bh@0:     def package(self):
bh@41:         self.status.creating_binary_package()
bh@0:         util.ensure_directory(self.binary_dir)
bh@54:         logging.info("Building binary package; logging to %r", self.logfile)
bh@45:         run.call(cmdexpand("@rootcmd /usr/sbin/pbuilder build"
bh@51:                            " --configfile $pbuilderrc"
bh@45:                            " --logfile $logfile --buildresult $bindir $dsc",
bh@55:                            rootcmd=self.track.root_cmd,
bh@55:                            pbuilderrc=self.track.pbuilderrc,
bh@51:                            logfile=self.logfile, bindir=self.binary_dir,
bh@51:                            dsc=self.dsc_file),
bh@45:                  suppress_output=True)
bh@84:         # remove the source package files put into the binary directory
bh@84:         # by pbuilder
bh@84:         for filename in os.listdir(self.binary_dir):
bh@84:             if os.path.splitext(filename)[1] not in (".deb", ".changes"):
bh@84:                 os.remove(os.path.join(self.binary_dir, filename))
bh@41:         self.status.binary_package_created()
bh@0: 
bh@0: 
bh@0: class RevisionPackager(object):
bh@0: 
bh@4:     source_packager_cls = SourcePackager
bh@4:     binary_packager_cls = BinaryPackager
bh@4: 
bh@53:     def __init__(self, track, revision):
bh@53:         self.track = track
bh@0:         self.revision = revision
bh@53:         self.base_dir = self.track.pkg_dir_for_revision(self.revision, 1)
bh@36:         self.status = status.RevisionStatus(os.path.join(self.base_dir,
bh@36:                                                          "status"))
bh@0: 
bh@0:     work_dir = _filenameproperty("work")
bh@0:     binary_dir = _filenameproperty("binary")
bh@0:     src_dir = _filenameproperty("src")
bh@18:     build_log = _filenameproperty("build.log")
bh@0: 
bh@0:     def find_dsc_file(self):
bh@0:         for filename in os.listdir(self.src_dir):
bh@0:             if filename.endswith(".dsc"):
bh@0:                 return os.path.join(self.src_dir, filename)
bh@0:         return None
bh@0: 
bh@18:     def has_build_log(self):
bh@18:         return os.path.exists(self.build_log)
bh@18: 
bh@88:     def list_source_files(self):
bh@88:         """Returns a list with the names of the files of the source package.
bh@88:         The implementation assumes that all files in self.src_dir belong
bh@88:         to the source package.
bh@88:         """
bh@88:         return sorted(util.listdir_abs(self.src_dir))
bh@88: 
bh@88:     def list_binary_files(self):
bh@88:         """Returns a list with the names of the files of the binary packages.
bh@88:         The implementation assumes that all files in self.binary_dir belong
bh@88:         to the binary packages.
bh@88:         """
bh@88:         return sorted(util.listdir_abs(self.binary_dir))
bh@88: 
bh@0:     def package(self):
bh@0:         try:
bh@16:             util.ensure_directory(self.work_dir)
bh@16:             self.status.start = datetime.datetime.utcnow()
bh@53:             src_packager = self.source_packager_cls(self.track, self.status,
bh@4:                                                     self.work_dir, self.src_dir,
bh@4:                                                     self.revision)
bh@0:             src_packager.package()
bh@0: 
bh@0:             dsc_file = self.find_dsc_file()
bh@0:             if dsc_file is None:
bh@0:                 raise RuntimeError("Cannot find dsc File in %r" % self.src_dir)
bh@0: 
bh@53:             bin_packager = self.binary_packager_cls(self.track, self.status,
bh@4:                                                     self.binary_dir, dsc_file,
bh@18:                                                     self.build_log)
bh@0:             bin_packager.package()
bh@16:             self.status.stop = datetime.datetime.utcnow()
bh@0:         except:
bh@41:             self.status.error()
bh@16:             self.status.stop = datetime.datetime.utcnow()
bh@0:             raise
bh@0: 
bh@0:     def remove_package_dir(self):
bh@0:         logging.info("Removing pkgdir %r", self.base_dir)
bh@0:         shutil.rmtree(self.base_dir)
bh@0: 
bh@0: 
bh@52: class PackageTrack(object):
bh@0: 
bh@4:     revision_packager_cls = RevisionPackager
bh@4: 
bh@4:     svn_external_subdirs = []
bh@4: 
bh@4:     extra_config_desc = []
bh@4: 
bh@47:     def __init__(self, name, base_dir, svn_url, root_cmd, pbuilderrc, deb_email,
bh@4:                  deb_fullname, packager_class="treepkg.packager"):
bh@0:         self.name = name
bh@0:         self.base_dir = base_dir
bh@0:         self.svn_url = svn_url
bh@0:         self.root_cmd = root_cmd
bh@47:         self.pbuilderrc = pbuilderrc
bh@0:         self.deb_email = deb_email
bh@0:         self.deb_fullname = deb_fullname
bh@0:         self.pkg_dir_template = "%(revision)d-%(increment)d"
bh@0:         self.pkg_dir_regex \
bh@0:                    = re.compile(r"(?P<revision>[0-9]+)-(?P<increment>[0-9]+)$")
bh@0: 
bh@0:     checkout_dir = _filenameproperty("checkout")
bh@0:     debian_dir = _filenameproperty("debian")
bh@0:     pkg_dir = _filenameproperty("pkg")
bh@0: 
bh@0:     def pkg_dir_for_revision(self, revision, increment):
bh@0:         return os.path.join(self.pkg_dir,
bh@0:                             self.pkg_dir_template % locals())
bh@0: 
bh@0:     def last_changed_revision(self):
bh@4:         revisions = []
bh@4:         for directory in [self.checkout_dir] + self.svn_external_subdirs:
bh@4:             directory = os.path.join(self.checkout_dir, directory)
bh@4:             revisions.append(subversion.last_changed_revision(directory))
bh@4:         return max(revisions)
bh@0: 
bh@11:     def get_revision_numbers(self):
bh@11:         """Returns a list of the numbers of the packaged revisions"""
bh@11:         revisions = []
bh@11:         if os.path.exists(self.pkg_dir):
bh@11:             for filename in os.listdir(self.pkg_dir):
bh@11:                 match = self.pkg_dir_regex.match(filename)
bh@11:                 if match:
bh@11:                     revisions.append(int(match.group("revision")))
bh@85:         revisions.sort()
bh@11:         return revisions
bh@11: 
bh@0:     def last_packaged_revision(self):
bh@0:         """Returns the revision number of the highest packaged revision.
bh@0: 
bh@0:         If the revision cannot be determined because no already packaged
bh@0:         revisions can be found, the function returns -1.
bh@0:         """
bh@11:         return max([-1] + self.get_revision_numbers())
bh@0: 
bh@0:     def debian_source(self):
bh@0:         return util.extract_value_for_key(open(os.path.join(self.debian_dir,
bh@0:                                                             "control")),
bh@0:                                           "Source:")
bh@0: 
bh@80:     def update_checkout(self, revision=None):
bh@0:         """Updates the working copy of self.svn_url in self.checkout_dir.
bh@0: 
bh@0:         If self.checkout_dir doesn't exist yet, self.svn_url is checked
bh@80:         out into that directory.  The value of the revision parameter is
bh@80:         passed through to subversion.update.
bh@0:         """
bh@0:         localdir = self.checkout_dir
bh@0:         if os.path.exists(localdir):
bh@0:             logging.info("Updating the working copy in %r", localdir)
bh@80:             subversion.update(localdir, revision=revision)
bh@0:         else:
bh@0:             logging.info("The working copy in %r doesn't exist yet."
bh@26:                          "  Checking out from %r", localdir,
bh@0:                          self.svn_url)
bh@0:             subversion.checkout(self.svn_url, localdir)
bh@0: 
bh@0:     def export_sources(self, to_dir):
bh@0:         logging.info("Exporting sources for tarball to %r", to_dir)
bh@0:         subversion.export(self.checkout_dir, to_dir)
bh@0:         # some versions of svn (notably version 1.4.2 shipped with etch)
bh@0:         # do export externals such as the admin subdirectory.  We may
bh@0:         # have to do that in an extra step.
bh@4:         for subdir in self.svn_external_subdirs:
bh@4:             absdir = os.path.join(to_dir, subdir)
bh@4:             if not os.path.isdir(absdir):
bh@4:                 subversion.export(os.path.join(self.checkout_dir, subdir),
bh@4:                                   absdir)
bh@0: 
bh@0:     def copy_debian_directory(self, to_dir):
bh@0:         logging.info("Copying debian directory to %r", to_dir)
bh@0:         shutil.copytree(self.debian_dir, to_dir)
bh@0: 
bh@0:     def debian_environment(self):
bh@0:         """Returns the environment variables for the debian commands"""
bh@0:         env = os.environ.copy()
bh@0:         env["DEBFULLNAME"] = self.deb_fullname
bh@0:         env["DEBEMAIL"] = self.deb_email
bh@0:         return env
bh@0: 
bh@80:     def package_if_updated(self, revision=None):
bh@0:         """Checks if the checkout changed and returns a new packager if so"""
bh@80:         self.update_checkout(revision=revision)
bh@0:         current_revision = self.last_changed_revision()
bh@0:         logging.info("New revision is %d", current_revision)
bh@0:         previous_revision = self.last_packaged_revision()
bh@0:         logging.info("Previously packaged revision was %d", previous_revision)
bh@0:         if current_revision > previous_revision:
thomas@78:             logging.info("New revision has not been packaged yet")
bh@4:             return self.revision_packager_cls(self, current_revision)
bh@0:         else:
bh@0:             logging.info("New revision already packaged.")
bh@0: 
bh@14:     def get_revisions(self):
bh@14:         """Returns RevisionPackager objects for each packaged revision"""
bh@14:         return [self.revision_packager_cls(self, revision)
bh@14:                 for revision in self.get_revision_numbers()]
bh@14: 
bh@0: 
bh@52: def create_package_track(packager_class, **kw):
bh@4:     module = util.import_dotted_name(packager_class)
bh@52:     return module.PackageTrack(**kw)
bh@4: 
bh@0: 
bh@7: class PackagerGroup(object):
bh@0: 
bh@80:     def __init__(self, package_tracks, check_interval, revision=None):
bh@52:         self.package_tracks = package_tracks
bh@0:         self.check_interval = check_interval
bh@80:         self.revision = revision
bh@0: 
bh@0:     def run(self):
bh@7:         """Runs the packager group indefinitely"""
thomas@78:         logging.info("Starting in periodic check mode."
thomas@78:                      "  Will check every %d seconds", self.check_interval)
bh@0:         last_check = -1
bh@0:         while 1:
bh@0:             now = time.time()
bh@0:             if now > last_check + self.check_interval:
bh@52:                 self.check_package_tracks()
bh@0:                 last_check = now
bh@0:                 next_check = now + self.check_interval
bh@0:                 to_sleep = next_check - time.time()
bh@0:                 if to_sleep > 0:
bh@0:                     logging.info("Next check at %s",
bh@0:                                  time.strftime("%Y-%m-%d %H:%M:%S",
bh@0:                                                time.localtime(next_check)))
bh@0:                     time.sleep(to_sleep)
bh@0:                 else:
bh@0:                     logging.info("Next check now")
bh@0: 
bh@52:     def check_package_tracks(self):
bh@52:         logging.info("Checking package tracks")
bh@52:         for track in self.package_tracks:
bh@0:             try:
bh@80:                 packager = track.package_if_updated(revision=self.revision)
bh@0:                 if packager:
bh@0:                     packager.package()
bh@0:             except:
bh@0:                 logging.exception("An error occurred while"
bh@52:                                   " checking packager track %r", track.name)
bh@52:         logging.info("Checked all package tracks")
bh@14: 
bh@52:     def get_package_tracks(self):
bh@52:         return self.package_tracks