# HG changeset patch # User Bernhard Herzog # Date 1175539705 -7200 # Node ID 38d66dc6a1e50cb7f461c5602dd5c090240ffc4b # Parent aed3869ac04a2415cd402abc3541e3568b298796# Parent b0a4df526d64cfbfa846d55d3f724aa57b24e410 merge diff -r aed3869ac04a -r 38d66dc6a1e5 README --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/README Mon Apr 02 20:48:25 2007 +0200 @@ -0,0 +1,94 @@ +README for TreePackager +======================= +2007-03-19, Bernhard Herzog + + +TreePackager is a tool to automatically build debian packages from SVN. + + +Prerequisites +------------- + +You need the following software to run TreePackager. In the list below, +parentheses contain the name of the corresponding package in Debian Etch +if its not the same as the software. The version required is usually +the one from debian etch. + + Python 2.4 (python2.4-minimal) + Debian devscripts (devscripts) + subversion + pbuilder + sudo + bzip2 + +For the web front-end you also need the following software: + + Genshi (python-genshi) + CherryPy (python-cherrypy) + +Some of the packagers require additional software. The KDEPIM +enterprise branch packagers require the following additional software: + + autoconf2.13 + automake1.9 + + +Installation +------------ + +You can run the tree packager itself directly from the source tree. +However, you need to configure it first and setup pbuilder. + + +Configuration +------------- + +To understand the configuration, first a few notes about the +architecture of TreePackager. The TreePackager consist of one program +that periodically updates svn working directories and if something has +changed, builds a new debian package from the working directory. The +program should run as a normal user. The sample configuration assumes +that it's the user "builder" with a home directory "/home/builder". The +default configuration manages a directory tree under +"/home/builder/enterprise". + +The binary packages are built with pbuilder. Because pbuilder uses a +chroot environment to build the packages, it must be run as root. The +tree packager therefore needs a way to run pbuilder as root even though +itself runs as a non-root user. By default the tree packager uses sudo, +so you have to setup sudo to allow the tree packager user to invoke +pbuilder without a password. This can be accomplished with the +following line in /etc/sudoers (using the default user name): + + builder ALL = NOPASSWD: /usr/sbin/pbuilder + + +Configure TreePackager +~~~~~~~~~~~~~~~~~~~~~~ + +The file demo.cfg contains example configuration that contains most of +what is needed to package KDEPIM and kde-i18n from the KDEPIM enterprise +branch. Copy this file to treepkg.cfg and adapt it to your needs. The +comments in the file should provide enough hints to get you started. + + +Configure pbuilder +~~~~~~~~~~~~~~~~~~ + +Currently the tree packager assumes that pbuilder has already been set up +and that the default configuration is suitable for the purposes the tree +packager instance. Normally, the default way of setting up sudo should +be OK. There is one important interaction with sudo that you need to be +aware of, though: when the user builder starts pbuilder via sudo, +pbuilder uses ~builder/.pbuilderrc as its standard configuration file. + + +Configure the web front-end +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The default configuration should be OK in most cases. If you want you +can customize it in cherrypy.cfg. Start the web front-end with + + ./starttreepkgweb.py + +starttreepkgweb has some options to specify which configuration files to use. diff -r aed3869ac04a -r 38d66dc6a1e5 cherrypy.cfg --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/cherrypy.cfg Mon Apr 02 20:48:25 2007 +0200 @@ -0,0 +1,4 @@ +[global] +server.socketPort = 9090 +server.threadPool = 10 +server.environment = "production" diff -r aed3869ac04a -r 38d66dc6a1e5 enterprise/kdei18n.py --- a/enterprise/kdei18n.py Mon Apr 02 20:47:25 2007 +0200 +++ b/enterprise/kdei18n.py Mon Apr 02 20:48:25 2007 +0200 @@ -84,11 +84,21 @@ return tarball def do_package(self): + # Create a new kde-i18n-de tarball from current SVN and the base + # kde-i18n-de tarball. pkgbaseversion, pkgbasedir = self.export_sources() tarball = self.create_i18n_de_tarball(pkgbasedir, pkgbaseversion) + # We have to reuse the same directory when building the + # orig.tar.gz. However, we need to preserver the scripts + # sub-directory because it's not needed for the kde-i18n-de + # tarball but for the .orig.tar.gz. + pkg_scripts_dir = os.path.join(pkgbasedir, "scripts") + tmp_scripts_dir = os.path.join(self.work_dir, "scripts") + os.rename(pkg_scripts_dir, tmp_scripts_dir) shutil.rmtree(pkgbasedir) os.mkdir(pkgbasedir) + os.rename(tmp_scripts_dir, pkg_scripts_dir) pkgbasename = self.pkg_basename + "_" + pkgbaseversion origtargz = os.path.join(self.work_dir, diff -r aed3869ac04a -r 38d66dc6a1e5 reportstatus.py --- a/reportstatus.py Mon Apr 02 20:47:25 2007 +0200 +++ b/reportstatus.py Mon Apr 02 20:48:25 2007 +0200 @@ -11,8 +11,7 @@ import os from optparse import OptionParser -from treepkg.packager import create_package_line, PackagerGroup -from treepkg.readconfig import read_config +from treepkg.report import get_packager_group, prepare_report def parse_commandline(): parser = OptionParser() @@ -21,17 +20,22 @@ parser.add_option("--config-file") return parser.parse_args() + +def report_text(group): + report = prepare_report(group) + for revno, row in report.revisions: + for col in row: + if col: + print "%s %s: %s" % (col.name, revno, col.status.desc) + if col.status.start: + print " Start:", col.status.start + print " Stop:", col.status.stop + print + def main(): options, args = parse_commandline() + group = get_packager_group(options.config_file) + report_text(group) - treepkg_opts, packager_opts = read_config(options.config_file) - group = PackagerGroup([create_package_line(**opts) - for opts in packager_opts], - **treepkg_opts) - for line in group.get_package_lines(): - for revision in line.get_revisions(): - print line.name, revision.revision, revision.status.status - print " start:", revision.status.start - print " stop:", revision.status.stop main() diff -r aed3869ac04a -r 38d66dc6a1e5 runtreepkg.py --- a/runtreepkg.py Mon Apr 02 20:47:25 2007 +0200 +++ b/runtreepkg.py Mon Apr 02 20:48:25 2007 +0200 @@ -8,6 +8,7 @@ """Starts the tree packager""" +import sys import os import logging from optparse import OptionParser @@ -28,7 +29,13 @@ parser = OptionParser() parser.set_defaults(config_file=os.path.join(os.path.dirname(__file__), "treepkg.cfg")) - parser.add_option("--config-file") + parser.add_option("--config-file", + help=("The tree packager config file." + " Default treepkg.cfg")) + parser.add_option("--once", action="store_true", + help=("Check the packagers only once and exit afterwards." + " Without this option, the tree packager will" + " check periodically.")) return parser.parse_args() def main(): @@ -37,9 +44,24 @@ initialize_logging() treepkg_opts, packager_opts = read_config(options.config_file) - group = PackagerGroup([create_package_line(**opts) - for opts in packager_opts], - **treepkg_opts) - group.run() + + if args: + packager_opts = [opts for opts in packager_opts if opts["name"] in args] + # check whether we got all of the names in args: + for opts in packager_opts: + name = opts["name"] + if name in args: + args.remove(name) + for name in args: + print >>sys.stderr, "No package lines found named %r" % name + + if packager_opts: + group = PackagerGroup([create_package_line(**opts) + for opts in packager_opts], + **treepkg_opts) + if options.once: + group.check_package_lines() + else: + group.run() main() diff -r aed3869ac04a -r 38d66dc6a1e5 starttreepkgweb.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/starttreepkgweb.py Mon Apr 02 20:48:25 2007 +0200 @@ -0,0 +1,33 @@ +#! /usr/bin/python2.4 +# Copyright (C) 2007 by Intevation GmbH +# Authors: +# Bernhard Herzog +# +# This program is free software under the GPL (>=v2) +# Read the file COPYING coming with the software for details. + +"""Starts the tree packager webinterface""" + +import os +from optparse import OptionParser + +from treepkg.web import runserver + +def parse_commandline(): + parser = OptionParser() + dirname = os.path.dirname(__file__) + parser.set_defaults(config_file=os.path.join(dirname, "treepkg.cfg"), + cherrypy_config=os.path.join(dirname, "cherrypy.cfg")) + parser.add_option("--config-file", + help=("The tree packager config file." + " Default treepkg.cfg")) + parser.add_option("--cherrypy-config", + help=("The cherrypy config file for the web interface." + " Default cherrypy.cfg")) + return parser.parse_args() + +def main(): + options, args = parse_commandline() + runserver(options.config_file, options.cherrypy_config) + +main() diff -r aed3869ac04a -r 38d66dc6a1e5 test/test_status.py --- a/test/test_status.py Mon Apr 02 20:47:25 2007 +0200 +++ b/test/test_status.py Mon Apr 02 20:48:25 2007 +0200 @@ -5,13 +5,13 @@ # This program is free software under the GPL (>=v2) # Read the file COPYING coming with the software for details. -"""Tests for the Status class""" +"""Tests for the Status classes""" import os import unittest from datetime import datetime -from treepkg.status import Status +from treepkg.status import RevisionStatus, Status, EnumFieldDesc from treepkg.util import ensure_directory, writefile @@ -29,34 +29,51 @@ os.remove(self.filename) def test_status(self): - status = Status(self.filename) - status.status = "testing" + status = RevisionStatus(self.filename) + status.error() - otherstatus = Status(self.filename) - self.assertEquals(otherstatus.status, "testing") + otherstatus = RevisionStatus(self.filename) + self.assertEquals(otherstatus.status.name, "error") def test_getting_unknown_fields(self): - status = Status(self.filename) + status = RevisionStatus(self.filename) self.assertRaises(AttributeError, getattr, status, "unknown_field") def test_setting_unknown_fields(self): - status = Status(self.filename) + status = RevisionStatus(self.filename) self.assertRaises(AttributeError, setattr, status, "unknown_field", "some value") - def test_default_value(self): - status = Status(self.filename) + def test_default_values(self): + status = RevisionStatus(self.filename) + self.assertEquals(status.status.name, "unknown") self.assertEquals(status.start, None) + self.assertEquals(status.stop, None) def test_date(self): timestamp = datetime(2007, 3, 9, 17, 32, 55) - status = Status(self.filename) + status = RevisionStatus(self.filename) status.start = timestamp - otherstatus = Status(self.filename) + otherstatus = RevisionStatus(self.filename) self.assertEquals(otherstatus.start, timestamp) def test_magic(self): writefile(self.filename, "Some other magic\nstart: 2007-03-09 17:32:55\n") - self.assertRaises(ValueError, Status, self.filename) + self.assertRaises(ValueError, RevisionStatus, self.filename) + + def test_status_enum(self): + + class TestStatus(Status): + status = EnumFieldDesc() + status.add("not_running", "Process is not running", + default=True) + status.add("running", "Process is running") + status.add("error", "An error occurred") + + status = TestStatus(self.filename) + status.running() + + otherstatus = TestStatus(self.filename) + self.assertEquals(otherstatus.status.name, "running") diff -r aed3869ac04a -r 38d66dc6a1e5 treepkg/packager.py --- a/treepkg/packager.py Mon Apr 02 20:47:25 2007 +0200 +++ b/treepkg/packager.py Mon Apr 02 20:48:25 2007 +0200 @@ -154,9 +154,9 @@ """ util.ensure_directory(self.work_dir) try: - self.status.status = "creating_source_package" + self.status.creating_source_package() self.do_package() - self.status.status = "source_package_created" + self.status.source_package_created() finally: logging.info("Removing workdir %r", self.work_dir) shutil.rmtree(self.work_dir) @@ -182,7 +182,7 @@ self.logfile = logfile def package(self): - self.status.status = "creating_binary_package" + self.status.creating_binary_package() util.ensure_directory(self.binary_dir) logging.info("Building binary package; loging to %r", self.logfile) cmd = ["/usr/sbin/pbuilder", "build", @@ -190,7 +190,7 @@ "--logfile", self.logfile, "--buildresult", self.binary_dir, self.dsc_file] run.call(self.pkg_line.root_cmd + cmd, suppress_output=True) - self.status.status = "binary_package_created" + self.status.binary_package_created() class RevisionPackager(object): @@ -202,11 +202,13 @@ self.pkg_line = pkg_line self.revision = revision self.base_dir = self.pkg_line.pkg_dir_for_revision(self.revision, 1) - self.status = status.Status(os.path.join(self.base_dir, "status")) + self.status = status.RevisionStatus(os.path.join(self.base_dir, + "status")) work_dir = _filenameproperty("work") binary_dir = _filenameproperty("binary") src_dir = _filenameproperty("src") + build_log = _filenameproperty("build.log") def find_dsc_file(self): for filename in os.listdir(self.src_dir): @@ -214,6 +216,9 @@ return os.path.join(self.src_dir, filename) return None + def has_build_log(self): + return os.path.exists(self.build_log) + def package(self): try: util.ensure_directory(self.work_dir) @@ -229,12 +234,11 @@ bin_packager = self.binary_packager_cls(self.pkg_line, self.status, self.binary_dir, dsc_file, - os.path.join(self.base_dir, - "build.log")) + self.build_log) bin_packager.package() self.status.stop = datetime.datetime.utcnow() except: - self.status.status = "error" + self.status.error() self.status.stop = datetime.datetime.utcnow() raise @@ -314,7 +318,7 @@ subversion.update(localdir) else: logging.info("The working copy in %r doesn't exist yet." - " Checking out fromo %r", localdir, + " Checking out from %r", localdir, self.svn_url) subversion.checkout(self.svn_url, localdir) diff -r aed3869ac04a -r 38d66dc6a1e5 treepkg/report.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/treepkg/report.py Mon Apr 02 20:48:25 2007 +0200 @@ -0,0 +1,86 @@ +# Copyright (C) 2007 by Intevation GmbH +# Authors: +# Bernhard Herzog +# +# This program is free software under the GPL (>=v2) +# Read the file COPYING coming with the software for details. + +"""Support for status reports""" + +from packager import create_package_line, PackagerGroup +from readconfig import read_config + +class struct(object): + + """Class to create simple struct like objects + + All keyword arguments passed to the constructor are available as + instance variables. + """ + + def __init__(self, **kw): + self.__dict__.update(kw) + + def __repr__(self): + fields = ["%s=%r" % item for item in self.__dict__.items()] + return "struct(" + ", ".join(fields) + ")" + + +def get_packager_group(config_file): + treepkg_opts, packager_opts = read_config(config_file) + return PackagerGroup([create_package_line(**opts) + for opts in packager_opts], + **treepkg_opts) + +def status_class(status): + """Returns the CSS class for a status""" + if status.finished: + return "finished" + elif status.error: + return "error" + else: + return "inprogress" + +def format_time(timestamp): + """Formats a datetime object for a status report + + if the argument is true, the return value is simply str applied to + the argument, which for datetime objects is a string with the format + 'YYYY-MM-DD hh:mm:ss'. If the argument is false, the return value + is ''. + """ + if timestamp: + return str(timestamp) + else: + return "" + + +def prepare_status(status): + return struct(desc=status.status.description, + start=format_time(status.start), + stop=format_time(status.stop), + cls=status_class(status.status)) + +def prepare_report(group): + revisions = {} + columns = [] + pkglines = group.get_package_lines() + num_columns = len(pkglines) + for column, line in enumerate(pkglines): + columns.append((column, line.name)) + for revision in line.get_revisions(): + row = revisions.setdefault(revision.revision, [None] * num_columns) + row[column] = struct(revno=revision.revision, + revision=revision, + column=column, + name=line.name, + status=prepare_status(revision.status)) + + # convert the revisions dict into a sorted list of (revno, row) + # pairs + revisions = revisions.items() + revisions.sort() + revisions.reverse() + + return struct(columns=columns, + revisions=revisions) diff -r aed3869ac04a -r 38d66dc6a1e5 treepkg/status.py --- a/treepkg/status.py Mon Apr 02 20:47:25 2007 +0200 +++ b/treepkg/status.py Mon Apr 02 20:48:25 2007 +0200 @@ -15,90 +15,175 @@ # special object to indicate no default value nodefault = object() -def serialize_datetime(d): - return d.strftime("%Y-%m-%d %H:%M:%S") - -def deserialize_datetime(s): - return datetime.datetime(*time.strptime(s.strip(), "%Y-%m-%d %H:%M:%S")[:6]) - class FieldDesc(object): - def __init__(self, serialize, deserialize, default=nodefault): - self.serialize = serialize - self.deserialize = deserialize + def __init__(self, default=nodefault): self.default = default def has_default(self): return self.default is not nodefault + def set_default(self, value): + self.default = value + + def serialize(self, value): + raise NotImplementedError + + def deserialize(self, string): + raise NotImplementedError + + class StringFieldDesc(FieldDesc): - def __init__(self, **kw): - super(StringFieldDesc, self).__init__(str, lambda s: s.strip(), **kw) + def serialize(self, value): + return str(value) + + def deserialize(self, value): + return value.strip() + class DateFieldDesc(FieldDesc): - def __init__(self, **kw): - super(DateFieldDesc, self).__init__(serialize_datetime, - deserialize_datetime, **kw) + date_format = "%Y-%m-%d %H:%M:%S" + + def serialize(self, value): + return value.strftime(self.date_format) + + def deserialize(self, string): + return datetime.datetime(*time.strptime(string.strip(), + self.date_format)[:6]) + + +class EnumValue(object): + + def __init__(self, name, description, finished=False, error=False): + self.name = name + self.description = description + self.finished = finished + self.error = error + + +class EnumFieldDesc(FieldDesc): + + def __init__(self, *args, **kw): + super(EnumFieldDesc, self).__init__(*args, **kw) + self.values = {} + + def add(self, name, description, default=False, **kw): + enum = EnumValue(name, description, **kw) + self.values[enum.name] = enum + if default: + self.set_default(enum) + + def __iter__(self): + return self.values.itervalues() + + def serialize(self, value): + assert value.name is not None + return value.name + + def deserialize(self, string): + return self.values[string.strip()] + + +def make_setter(fieldname, enum): + def setter(self): + setattr(self, fieldname, enum) + setter.__name__ = enum.name + return setter + + +class StatusMetaClass(type): + + def __new__(cls, name, bases, clsdict): + # Generate the _fields class variable from the field descriptors + # in clsdict and remove the descriptors themselves. Also, add + # one setter method for each enum. + fields = dict() + for key, value in clsdict.items(): + if isinstance(value, FieldDesc): + fields[key] = value + del clsdict[key] + if isinstance(value, EnumFieldDesc): + for enum in value: + clsdict[enum.name] = make_setter(key, enum) + clsdict["_fields"] = fields + return type.__new__(cls, name, bases, clsdict) class Status(object): - fields = dict(status=StringFieldDesc(), - start=DateFieldDesc(default=None), - stop=DateFieldDesc(default=None)) + __metaclass__ = StatusMetaClass - attrs = set(["filename", "values"]) + # Overwrite in derived classes with a different magic string + _magic = "Status 0.0\n" - magic = "TreePackagerStatus 0.0\n" + # Derived classes may extend a copy of this set with more instance + # variables. + _attrs = set(["_filename", "_values"]) def __init__(self, filename): assert os.path.isabs(filename) - self.filename = filename + self._filename = filename self.read() def _init_values(self): - self.values = {} + self._values = {} def read(self): self._init_values() - if not os.path.exists(self.filename): + if not os.path.exists(self._filename): return - f = open(self.filename) + f = open(self._filename) try: magic = f.next() - if magic != self.magic: - raise ValueError("File %r has wrong magic" % self.filename) + if magic != self._magic: + raise ValueError("File %r has wrong magic" % self._filename) for line in f: field, value = line.split(":", 1) - self.values[field] = self.fields[field].deserialize(value) + self._values[field] = self._fields[field].deserialize(value) finally: f.close() def write(self): - lines = [self.magic] - for field, desc in self.fields.items(): - if field in self.values: + lines = [self._magic] + for field, desc in self._fields.items(): + if field in self._values: lines.append("%s: %s\n" - % (field, desc.serialize(self.values[field]))) - util.writefile(self.filename, "".join(lines)) + % (field, desc.serialize(self._values[field]))) + util.writefile(self._filename, "".join(lines)) def __getattr__(self, attr): - desc = self.fields.get(attr) + desc = self._fields.get(attr) if desc is not None: - if attr in self.values: - return self.values[attr] + if attr in self._values: + return self._values[attr] elif desc.has_default(): return desc.default raise AttributeError(attr) def __setattr__(self, attr, value): - if attr in self.fields: - self.values[attr] = value + if attr in self._fields: + self._values[attr] = value self.write() - elif attr in self.attrs: + elif attr in self._attrs: self.__dict__[attr] = value else: raise AttributeError(attr) + + +class RevisionStatus(Status): + + _magic = "TreePackagerStatus 0.0\n" + + status = EnumFieldDesc() + status.add("creating_source_package", "creating source package") + status.add("source_package_created", "source package created") + status.add("creating_binary_package", "building binary packages") + status.add("binary_package_created", "build successful", finished=True) + status.add("error", "error", error=True) + status.add("unknown", "unknown", default=True) + + start = DateFieldDesc(default=None) + stop = DateFieldDesc(default=None) diff -r aed3869ac04a -r 38d66dc6a1e5 treepkg/web-status.html --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/treepkg/web-status.html Mon Apr 02 20:48:25 2007 +0200 @@ -0,0 +1,53 @@ + + + Tree Packager Status + + + +

Tree Packager Status

+ + + + + + + + + + + + + + + + + + + + + + + + + +
Revision${col[1]}
${row[0]} + ${col.status.desc}
+ Start: ${col.status.start}
+ Stop: ${col.status.stop}
+ + build_log + +
+ + All times are given in UTC. + + + diff -r aed3869ac04a -r 38d66dc6a1e5 treepkg/web.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/treepkg/web.py Mon Apr 02 20:48:25 2007 +0200 @@ -0,0 +1,71 @@ +# Copyright (C) 2007 by Intevation GmbH +# Authors: +# Bernhard Herzog +# +# This program is free software under the GPL (>=v2) +# Read the file COPYING coming with the software for details. + +import os + +from genshi.template import TemplateLoader + +import cherrypy +from cherrypy import expose +from cherrypy.lib import cptools + +import report + + +class Status(object): + + """Implements the tree packager status pages""" + + def __init__(self, treepkg_config): + self.treepkg_config = treepkg_config + self.loader = TemplateLoader([os.path.dirname(__file__)]) + + @expose + def index(self): + group = report.get_packager_group(self.treepkg_config) + tmpl = self.loader.load('web-status.html') + stream = tmpl.generate(report=report.prepare_report(group)) + return stream.render('html') + + def build_log_filename(self, package_line_name, revno): + """Returns the name of the build log file of a revision if it exists""" + group = report.get_packager_group(self.treepkg_config) + for line in group.get_package_lines(): + if line.name == package_line_name: + for revision in line.get_revisions(): + if str(revision.revision) == revno: + if revision.has_build_log(): + return revision.build_log + + @expose + def default(self, *rest): + """Handles requests for .../pkg/revno/build.log""" + filename = None + if len(rest) == 3 and rest[2] == "build.log": + filename = self.build_log_filename(*rest[:2]) + if filename is not None: + return cptools.serveFile(filename, contentType="text/plain") + else: + raise cherrypy.HTTPError(status="404") + + + +class TreePKG(object): + + """Root object for the tree packager web interface""" + + @expose + def index(self): + raise cherrypy.HTTPRedirect('/status') + + +def runserver(treepkg_config, cherrypy_config): + cherrypy.root = TreePKG() + cherrypy.root.status = Status(treepkg_config=treepkg_config) + + cherrypy.config.update(file=cherrypy_config) + cherrypy.server.start()