Mercurial > treepkg > treepkg
comparison treepkg/packager.py @ 0:f78a02e79c84
initial checkin
| author | Bernhard Herzog <bh@intevation.de> |
|---|---|
| date | Tue, 06 Mar 2007 17:37:32 +0100 |
| parents | |
| children | e6a9f4037f68 |
comparison
equal
deleted
inserted
replaced
| -1:000000000000 | 0:f78a02e79c84 |
|---|---|
| 1 # Copyright (C) 2007 by Intevation GmbH | |
| 2 # Authors: | |
| 3 # Bernhard Herzog <bh@intevation.de> | |
| 4 # | |
| 5 # This program is free software under the GPL (>=v2) | |
| 6 # Read the file COPYING coming with the software for details. | |
| 7 | |
| 8 """Classes to automatically build debian packages from subversion checkouts""" | |
| 9 | |
| 10 import os | |
| 11 import time | |
| 12 import re | |
| 13 import logging | |
| 14 import shutil | |
| 15 import traceback | |
| 16 | |
| 17 import util | |
| 18 import subversion | |
| 19 import run | |
| 20 | |
| 21 | |
| 22 def _filenameproperty(relative_dir): | |
| 23 def get(self): | |
| 24 return os.path.join(self.base_dir, relative_dir) | |
| 25 return property(get) | |
| 26 | |
| 27 | |
| 28 class SourcePackager(object): | |
| 29 | |
| 30 def __init__(self, plant, status, work_dir, src_dir, revision): | |
| 31 self.plant = plant | |
| 32 self.status = status | |
| 33 self.work_dir = work_dir | |
| 34 self.src_dir = src_dir | |
| 35 self.revision = revision | |
| 36 self.enterprise_version = (time.strftime("%Y%m%d", time.localtime()) \ | |
| 37 + "." + str(self.revision)) | |
| 38 | |
| 39 def kdepim_version(self, directory): | |
| 40 """Determine the kdepim version. | |
| 41 | |
| 42 The version is taken from the kdepim.lsm file in the plants | |
| 43 checkout dir. | |
| 44 """ | |
| 45 return util.extract_lsm_version(os.path.join(directory, "kdepim.lsm")) | |
| 46 | |
| 47 def determine_package_version(self, directory): | |
| 48 enterprise_version = self.enterprise_version | |
| 49 kdepimversion = self.kdepim_version(directory) | |
| 50 version_template = "%(kdepimversion)s.enterprise.0.%(enterprise_version)s" | |
| 51 return version_template % locals() | |
| 52 | |
| 53 def export_sources(self): | |
| 54 temp_dir = os.path.join(self.work_dir, "temp") | |
| 55 self.plant.export_sources(temp_dir) | |
| 56 | |
| 57 pkgbaseversion = self.determine_package_version(temp_dir) | |
| 58 pkgbasedir = os.path.join(self.work_dir, "kdepim-" + pkgbaseversion) | |
| 59 | |
| 60 os.rename(temp_dir, pkgbasedir) | |
| 61 return pkgbaseversion, pkgbasedir | |
| 62 | |
| 63 | |
| 64 def update_version_numbers(self, pkgbasedir): | |
| 65 versionstring = "(enterprise %s)" % self.enterprise_version | |
| 66 for versionfile in ["kmail/kmversion.h", "kontact/src/main.cpp", | |
| 67 "korganizer/version.h"]: | |
| 68 filename = os.path.join(pkgbasedir, versionfile) | |
| 69 patched = re.sub("\(enterprise ([^)]*)\)", versionstring, | |
| 70 open(filename).read()) | |
| 71 f = open(filename, "w") | |
| 72 f.write(patched) | |
| 73 f.close() | |
| 74 | |
| 75 def create_tarball(self, tarballname, workdir, basedir): | |
| 76 logging.info("Creating tarball %r", tarballname) | |
| 77 run.call(["tar", "czf", tarballname, "-C", workdir, basedir]) | |
| 78 | |
| 79 def copy_debian_directory(self, pkgbasedir, pkgbaseversion, changemsg): | |
| 80 debian_dir = os.path.join(pkgbasedir, "debian") | |
| 81 changelog = os.path.join(debian_dir, "changelog") | |
| 82 | |
| 83 self.plant.copy_debian_directory(debian_dir) | |
| 84 | |
| 85 logging.info("Updating %r", changelog) | |
| 86 oldversion = util.debian_changelog_version(changelog) | |
| 87 if ":" in oldversion: | |
| 88 oldversionprefix = oldversion.split(":")[0] + ":" | |
| 89 else: | |
| 90 oldversionprefix = "" | |
| 91 run.call(["debchange", "-c", changelog, | |
| 92 "-v", oldversionprefix + pkgbaseversion + "-kk1", | |
| 93 changemsg], | |
| 94 env=self.plant.debian_environment()) | |
| 95 | |
| 96 | |
| 97 def create_source_package(self, pkgbasedir, origtargz): | |
| 98 logging.info("Creating new source package") | |
| 99 run.call(["dpkg-source", "-b", os.path.basename(pkgbasedir), | |
| 100 os.path.basename(origtargz)], | |
| 101 cwd=os.path.dirname(pkgbasedir), | |
| 102 suppress_output=True, | |
| 103 env=self.plant.debian_environment()) | |
| 104 | |
| 105 def package(self): | |
| 106 util.ensure_directory(self.work_dir) | |
| 107 try: | |
| 108 self.status.set("creating_source_package") | |
| 109 pkgbaseversion, pkgbasedir = self.export_sources() | |
| 110 self.update_version_numbers(pkgbasedir) | |
| 111 | |
| 112 pkgbasename = "kdepim_" + pkgbaseversion | |
| 113 origtargz = os.path.join(self.work_dir, | |
| 114 pkgbasename + ".orig.tar.gz") | |
| 115 self.create_tarball(origtargz, self.work_dir, | |
| 116 os.path.basename(pkgbasedir)) | |
| 117 | |
| 118 changemsg = ("Update to SVN enterprise branch rev. %d" | |
| 119 % (self.revision,)) | |
| 120 self.copy_debian_directory(pkgbasedir, pkgbaseversion, | |
| 121 changemsg) | |
| 122 | |
| 123 self.create_source_package(pkgbasedir, origtargz) | |
| 124 | |
| 125 logging.info("Moving source package to %r", self.src_dir) | |
| 126 util.ensure_directory(self.src_dir) | |
| 127 for filename in [filename for filename in os.listdir(self.work_dir) | |
| 128 if filename.startswith(pkgbasename)]: | |
| 129 os.rename(os.path.join(self.work_dir, filename), | |
| 130 os.path.join(self.src_dir, filename)) | |
| 131 self.status.set("source_package_created") | |
| 132 finally: | |
| 133 logging.info("Removing workdir %r", self.work_dir) | |
| 134 shutil.rmtree(self.work_dir) | |
| 135 | |
| 136 | |
| 137 class BinaryPackager(object): | |
| 138 | |
| 139 def __init__(self, plant, status, binary_dir, dsc_file, logfile): | |
| 140 self.plant = plant | |
| 141 self.status = status | |
| 142 self.binary_dir = binary_dir | |
| 143 self.dsc_file = dsc_file | |
| 144 self.logfile = logfile | |
| 145 | |
| 146 def package(self): | |
| 147 self.status.set("creating_binary_package") | |
| 148 util.ensure_directory(self.binary_dir) | |
| 149 logging.info("Building binary package; loging to %r", self.logfile) | |
| 150 cmd = [] | |
| 151 if self.plant.root_cmd: | |
| 152 cmd.append(self.plant.root_cmd) | |
| 153 run.call(cmd + ["/usr/sbin/pbuilder", "build", | |
| 154 "--logfile", self.logfile, | |
| 155 "--buildresult", self.binary_dir, | |
| 156 self.dsc_file], | |
| 157 suppress_output=True) | |
| 158 self.status.set("binary_package_created") | |
| 159 | |
| 160 | |
| 161 class RevisionPackager(object): | |
| 162 | |
| 163 def __init__(self, plant, revision): | |
| 164 self.plant = plant | |
| 165 self.revision = revision | |
| 166 self.base_dir = self.plant.pkg_dir_for_revision(self.revision, 1) | |
| 167 self.status = util.StatusFile(os.path.join(self.base_dir, "status")) | |
| 168 | |
| 169 work_dir = _filenameproperty("work") | |
| 170 binary_dir = _filenameproperty("binary") | |
| 171 src_dir = _filenameproperty("src") | |
| 172 | |
| 173 def find_dsc_file(self): | |
| 174 for filename in os.listdir(self.src_dir): | |
| 175 if filename.endswith(".dsc"): | |
| 176 return os.path.join(self.src_dir, filename) | |
| 177 return None | |
| 178 | |
| 179 def package(self): | |
| 180 try: | |
| 181 src_packager = SourcePackager(self.plant, self.status, | |
| 182 self.work_dir, self.src_dir, | |
| 183 self.revision) | |
| 184 src_packager.package() | |
| 185 | |
| 186 dsc_file = self.find_dsc_file() | |
| 187 if dsc_file is None: | |
| 188 raise RuntimeError("Cannot find dsc File in %r" % self.src_dir) | |
| 189 | |
| 190 bin_packager = BinaryPackager(self.plant, self.status, | |
| 191 self.binary_dir, dsc_file, | |
| 192 os.path.join(self.base_dir, | |
| 193 "build.log")) | |
| 194 bin_packager.package() | |
| 195 except: | |
| 196 self.status.set("failed", traceback.format_exc()) | |
| 197 raise | |
| 198 | |
| 199 def remove_package_dir(self): | |
| 200 logging.info("Removing pkgdir %r", self.base_dir) | |
| 201 shutil.rmtree(self.base_dir) | |
| 202 | |
| 203 | |
| 204 class AssemblyLine(object): | |
| 205 | |
| 206 def __init__(self, name, base_dir, svn_url, root_cmd, deb_email, | |
| 207 deb_fullname): | |
| 208 self.name = name | |
| 209 self.base_dir = base_dir | |
| 210 self.svn_url = svn_url | |
| 211 self.root_cmd = root_cmd | |
| 212 self.deb_email = deb_email | |
| 213 self.deb_fullname = deb_fullname | |
| 214 self.pkg_dir_template = "%(revision)d-%(increment)d" | |
| 215 self.pkg_dir_regex \ | |
| 216 = re.compile(r"(?P<revision>[0-9]+)-(?P<increment>[0-9]+)$") | |
| 217 | |
| 218 checkout_dir = _filenameproperty("checkout") | |
| 219 debian_dir = _filenameproperty("debian") | |
| 220 pkg_dir = _filenameproperty("pkg") | |
| 221 | |
| 222 def pkg_dir_for_revision(self, revision, increment): | |
| 223 return os.path.join(self.pkg_dir, | |
| 224 self.pkg_dir_template % locals()) | |
| 225 | |
| 226 def last_changed_revision(self): | |
| 227 rev1 = subversion.last_changed_revision(self.checkout_dir) | |
| 228 rev2 = subversion.last_changed_revision(os.path.join(self.checkout_dir, | |
| 229 "admin")) | |
| 230 return max(rev1, rev2) | |
| 231 | |
| 232 def last_packaged_revision(self): | |
| 233 """Returns the revision number of the highest packaged revision. | |
| 234 | |
| 235 If the revision cannot be determined because no already packaged | |
| 236 revisions can be found, the function returns -1. | |
| 237 """ | |
| 238 revisions = [-1] | |
| 239 if os.path.exists(self.pkg_dir): | |
| 240 for filename in os.listdir(self.pkg_dir): | |
| 241 match = self.pkg_dir_regex.match(filename) | |
| 242 if match: | |
| 243 revisions.append(int(match.group("revision"))) | |
| 244 return max(revisions) | |
| 245 | |
| 246 def debian_source(self): | |
| 247 return util.extract_value_for_key(open(os.path.join(self.debian_dir, | |
| 248 "control")), | |
| 249 "Source:") | |
| 250 | |
| 251 def update_checkout(self): | |
| 252 """Updates the working copy of self.svn_url in self.checkout_dir. | |
| 253 | |
| 254 If self.checkout_dir doesn't exist yet, self.svn_url is checked | |
| 255 out into that directory. | |
| 256 """ | |
| 257 localdir = self.checkout_dir | |
| 258 if os.path.exists(localdir): | |
| 259 logging.info("Updating the working copy in %r", localdir) | |
| 260 subversion.update(localdir) | |
| 261 else: | |
| 262 logging.info("The working copy in %r doesn't exist yet." | |
| 263 " Checking out fromo %r", localdir, | |
| 264 self.svn_url) | |
| 265 subversion.checkout(self.svn_url, localdir) | |
| 266 | |
| 267 def export_sources(self, to_dir): | |
| 268 logging.info("Exporting sources for tarball to %r", to_dir) | |
| 269 subversion.export(self.checkout_dir, to_dir) | |
| 270 # some versions of svn (notably version 1.4.2 shipped with etch) | |
| 271 # do export externals such as the admin subdirectory. We may | |
| 272 # have to do that in an extra step. | |
| 273 admindir = os.path.join(to_dir, "admin") | |
| 274 if not os.path.isdir(admindir): | |
| 275 subversion.export(os.path.join(self.checkout_dir, "admin"), | |
| 276 admindir) | |
| 277 | |
| 278 def copy_debian_directory(self, to_dir): | |
| 279 logging.info("Copying debian directory to %r", to_dir) | |
| 280 shutil.copytree(self.debian_dir, to_dir) | |
| 281 | |
| 282 def debian_environment(self): | |
| 283 """Returns the environment variables for the debian commands""" | |
| 284 env = os.environ.copy() | |
| 285 env["DEBFULLNAME"] = self.deb_fullname | |
| 286 env["DEBEMAIL"] = self.deb_email | |
| 287 return env | |
| 288 | |
| 289 def package_if_updated(self): | |
| 290 """Checks if the checkout changed and returns a new packager if so""" | |
| 291 self.update_checkout() | |
| 292 current_revision = self.last_changed_revision() | |
| 293 logging.info("New revision is %d", current_revision) | |
| 294 previous_revision = self.last_packaged_revision() | |
| 295 logging.info("Previously packaged revision was %d", previous_revision) | |
| 296 if current_revision > previous_revision: | |
| 297 logging.info("New revision is not packaged yet") | |
| 298 return RevisionPackager(self, current_revision) | |
| 299 else: | |
| 300 logging.info("New revision already packaged.") | |
| 301 | |
| 302 | |
| 303 | |
| 304 class Packager(object): | |
| 305 | |
| 306 def __init__(self, assembly_lines, check_interval): | |
| 307 self.assembly_lines = assembly_lines | |
| 308 self.check_interval = check_interval | |
| 309 | |
| 310 def run(self): | |
| 311 """Runs the plant indefinitely""" | |
| 312 logging.info("Packager start. Will check every %d seconds", | |
| 313 self.check_interval) | |
| 314 last_check = -1 | |
| 315 while 1: | |
| 316 now = time.time() | |
| 317 if now > last_check + self.check_interval: | |
| 318 self.check_assembly_lines() | |
| 319 last_check = now | |
| 320 next_check = now + self.check_interval | |
| 321 to_sleep = next_check - time.time() | |
| 322 if to_sleep > 0: | |
| 323 logging.info("Next check at %s", | |
| 324 time.strftime("%Y-%m-%d %H:%M:%S", | |
| 325 time.localtime(next_check))) | |
| 326 time.sleep(to_sleep) | |
| 327 else: | |
| 328 logging.info("Next check now") | |
| 329 | |
| 330 def check_assembly_lines(self): | |
| 331 logging.info("Checking assembly lines") | |
| 332 for line in self.assembly_lines: | |
| 333 try: | |
| 334 packager = line.package_if_updated() | |
| 335 if packager: | |
| 336 packager.package() | |
| 337 except: | |
| 338 logging.exception("An error occurred while" | |
| 339 " checking assembly line %r", line.name) | |
| 340 logging.info("Checked all assembly lines") |