|
0
|
1 # Copyright (C) 2007 by Intevation GmbH |
|
|
2 # Authors: |
|
|
3 # Bernhard Herzog <bh@intevation.de> |
|
|
4 # |
|
|
5 # This program is free software under the GPL (>=v2) |
|
|
6 # Read the file COPYING coming with the software for details. |
|
|
7 |
|
|
8 """Classes to automatically build debian packages from subversion checkouts""" |
|
|
9 |
|
|
10 import os |
|
|
11 import time |
|
|
12 import re |
|
|
13 import logging |
|
|
14 import shutil |
|
|
15 import traceback |
|
|
16 |
|
|
17 import util |
|
|
18 import subversion |
|
|
19 import run |
|
|
20 |
|
|
21 |
|
|
22 def _filenameproperty(relative_dir): |
|
|
23 def get(self): |
|
|
24 return os.path.join(self.base_dir, relative_dir) |
|
|
25 return property(get) |
|
|
26 |
|
|
27 |
|
|
28 class SourcePackager(object): |
|
|
29 |
|
|
30 def __init__(self, plant, status, work_dir, src_dir, revision): |
|
|
31 self.plant = plant |
|
|
32 self.status = status |
|
|
33 self.work_dir = work_dir |
|
|
34 self.src_dir = src_dir |
|
|
35 self.revision = revision |
|
|
36 self.enterprise_version = (time.strftime("%Y%m%d", time.localtime()) \ |
|
|
37 + "." + str(self.revision)) |
|
|
38 |
|
|
39 def kdepim_version(self, directory): |
|
|
40 """Determine the kdepim version. |
|
|
41 |
|
|
42 The version is taken from the kdepim.lsm file in the plants |
|
|
43 checkout dir. |
|
|
44 """ |
|
|
45 return util.extract_lsm_version(os.path.join(directory, "kdepim.lsm")) |
|
|
46 |
|
|
47 def determine_package_version(self, directory): |
|
|
48 enterprise_version = self.enterprise_version |
|
|
49 kdepimversion = self.kdepim_version(directory) |
|
|
50 version_template = "%(kdepimversion)s.enterprise.0.%(enterprise_version)s" |
|
|
51 return version_template % locals() |
|
|
52 |
|
|
53 def export_sources(self): |
|
|
54 temp_dir = os.path.join(self.work_dir, "temp") |
|
|
55 self.plant.export_sources(temp_dir) |
|
|
56 |
|
|
57 pkgbaseversion = self.determine_package_version(temp_dir) |
|
|
58 pkgbasedir = os.path.join(self.work_dir, "kdepim-" + pkgbaseversion) |
|
|
59 |
|
|
60 os.rename(temp_dir, pkgbasedir) |
|
|
61 return pkgbaseversion, pkgbasedir |
|
|
62 |
|
|
63 |
|
|
64 def update_version_numbers(self, pkgbasedir): |
|
|
65 versionstring = "(enterprise %s)" % self.enterprise_version |
|
|
66 for versionfile in ["kmail/kmversion.h", "kontact/src/main.cpp", |
|
|
67 "korganizer/version.h"]: |
|
|
68 filename = os.path.join(pkgbasedir, versionfile) |
|
|
69 patched = re.sub("\(enterprise ([^)]*)\)", versionstring, |
|
|
70 open(filename).read()) |
|
|
71 f = open(filename, "w") |
|
|
72 f.write(patched) |
|
|
73 f.close() |
|
|
74 |
|
|
75 def create_tarball(self, tarballname, workdir, basedir): |
|
|
76 logging.info("Creating tarball %r", tarballname) |
|
|
77 run.call(["tar", "czf", tarballname, "-C", workdir, basedir]) |
|
|
78 |
|
|
79 def copy_debian_directory(self, pkgbasedir, pkgbaseversion, changemsg): |
|
|
80 debian_dir = os.path.join(pkgbasedir, "debian") |
|
|
81 changelog = os.path.join(debian_dir, "changelog") |
|
|
82 |
|
|
83 self.plant.copy_debian_directory(debian_dir) |
|
|
84 |
|
|
85 logging.info("Updating %r", changelog) |
|
|
86 oldversion = util.debian_changelog_version(changelog) |
|
|
87 if ":" in oldversion: |
|
|
88 oldversionprefix = oldversion.split(":")[0] + ":" |
|
|
89 else: |
|
|
90 oldversionprefix = "" |
|
|
91 run.call(["debchange", "-c", changelog, |
|
|
92 "-v", oldversionprefix + pkgbaseversion + "-kk1", |
|
|
93 changemsg], |
|
|
94 env=self.plant.debian_environment()) |
|
|
95 |
|
|
96 |
|
|
97 def create_source_package(self, pkgbasedir, origtargz): |
|
|
98 logging.info("Creating new source package") |
|
|
99 run.call(["dpkg-source", "-b", os.path.basename(pkgbasedir), |
|
|
100 os.path.basename(origtargz)], |
|
|
101 cwd=os.path.dirname(pkgbasedir), |
|
|
102 suppress_output=True, |
|
|
103 env=self.plant.debian_environment()) |
|
|
104 |
|
|
105 def package(self): |
|
|
106 util.ensure_directory(self.work_dir) |
|
|
107 try: |
|
|
108 self.status.set("creating_source_package") |
|
|
109 pkgbaseversion, pkgbasedir = self.export_sources() |
|
|
110 self.update_version_numbers(pkgbasedir) |
|
|
111 |
|
|
112 pkgbasename = "kdepim_" + pkgbaseversion |
|
|
113 origtargz = os.path.join(self.work_dir, |
|
|
114 pkgbasename + ".orig.tar.gz") |
|
|
115 self.create_tarball(origtargz, self.work_dir, |
|
|
116 os.path.basename(pkgbasedir)) |
|
|
117 |
|
|
118 changemsg = ("Update to SVN enterprise branch rev. %d" |
|
|
119 % (self.revision,)) |
|
|
120 self.copy_debian_directory(pkgbasedir, pkgbaseversion, |
|
|
121 changemsg) |
|
|
122 |
|
|
123 self.create_source_package(pkgbasedir, origtargz) |
|
|
124 |
|
|
125 logging.info("Moving source package to %r", self.src_dir) |
|
|
126 util.ensure_directory(self.src_dir) |
|
|
127 for filename in [filename for filename in os.listdir(self.work_dir) |
|
|
128 if filename.startswith(pkgbasename)]: |
|
|
129 os.rename(os.path.join(self.work_dir, filename), |
|
|
130 os.path.join(self.src_dir, filename)) |
|
|
131 self.status.set("source_package_created") |
|
|
132 finally: |
|
|
133 logging.info("Removing workdir %r", self.work_dir) |
|
|
134 shutil.rmtree(self.work_dir) |
|
|
135 |
|
|
136 |
|
|
137 class BinaryPackager(object): |
|
|
138 |
|
|
139 def __init__(self, plant, status, binary_dir, dsc_file, logfile): |
|
|
140 self.plant = plant |
|
|
141 self.status = status |
|
|
142 self.binary_dir = binary_dir |
|
|
143 self.dsc_file = dsc_file |
|
|
144 self.logfile = logfile |
|
|
145 |
|
|
146 def package(self): |
|
|
147 self.status.set("creating_binary_package") |
|
|
148 util.ensure_directory(self.binary_dir) |
|
|
149 logging.info("Building binary package; loging to %r", self.logfile) |
|
|
150 cmd = [] |
|
|
151 if self.plant.root_cmd: |
|
|
152 cmd.append(self.plant.root_cmd) |
|
|
153 run.call(cmd + ["/usr/sbin/pbuilder", "build", |
|
|
154 "--logfile", self.logfile, |
|
|
155 "--buildresult", self.binary_dir, |
|
|
156 self.dsc_file], |
|
|
157 suppress_output=True) |
|
|
158 self.status.set("binary_package_created") |
|
|
159 |
|
|
160 |
|
|
161 class RevisionPackager(object): |
|
|
162 |
|
|
163 def __init__(self, plant, revision): |
|
|
164 self.plant = plant |
|
|
165 self.revision = revision |
|
|
166 self.base_dir = self.plant.pkg_dir_for_revision(self.revision, 1) |
|
|
167 self.status = util.StatusFile(os.path.join(self.base_dir, "status")) |
|
|
168 |
|
|
169 work_dir = _filenameproperty("work") |
|
|
170 binary_dir = _filenameproperty("binary") |
|
|
171 src_dir = _filenameproperty("src") |
|
|
172 |
|
|
173 def find_dsc_file(self): |
|
|
174 for filename in os.listdir(self.src_dir): |
|
|
175 if filename.endswith(".dsc"): |
|
|
176 return os.path.join(self.src_dir, filename) |
|
|
177 return None |
|
|
178 |
|
|
179 def package(self): |
|
|
180 try: |
|
|
181 src_packager = SourcePackager(self.plant, self.status, |
|
|
182 self.work_dir, self.src_dir, |
|
|
183 self.revision) |
|
|
184 src_packager.package() |
|
|
185 |
|
|
186 dsc_file = self.find_dsc_file() |
|
|
187 if dsc_file is None: |
|
|
188 raise RuntimeError("Cannot find dsc File in %r" % self.src_dir) |
|
|
189 |
|
|
190 bin_packager = BinaryPackager(self.plant, self.status, |
|
|
191 self.binary_dir, dsc_file, |
|
|
192 os.path.join(self.base_dir, |
|
|
193 "build.log")) |
|
|
194 bin_packager.package() |
|
|
195 except: |
|
|
196 self.status.set("failed", traceback.format_exc()) |
|
|
197 raise |
|
|
198 |
|
|
199 def remove_package_dir(self): |
|
|
200 logging.info("Removing pkgdir %r", self.base_dir) |
|
|
201 shutil.rmtree(self.base_dir) |
|
|
202 |
|
|
203 |
|
|
204 class AssemblyLine(object): |
|
|
205 |
|
|
206 def __init__(self, name, base_dir, svn_url, root_cmd, deb_email, |
|
|
207 deb_fullname): |
|
|
208 self.name = name |
|
|
209 self.base_dir = base_dir |
|
|
210 self.svn_url = svn_url |
|
|
211 self.root_cmd = root_cmd |
|
|
212 self.deb_email = deb_email |
|
|
213 self.deb_fullname = deb_fullname |
|
|
214 self.pkg_dir_template = "%(revision)d-%(increment)d" |
|
|
215 self.pkg_dir_regex \ |
|
|
216 = re.compile(r"(?P<revision>[0-9]+)-(?P<increment>[0-9]+)$") |
|
|
217 |
|
|
218 checkout_dir = _filenameproperty("checkout") |
|
|
219 debian_dir = _filenameproperty("debian") |
|
|
220 pkg_dir = _filenameproperty("pkg") |
|
|
221 |
|
|
222 def pkg_dir_for_revision(self, revision, increment): |
|
|
223 return os.path.join(self.pkg_dir, |
|
|
224 self.pkg_dir_template % locals()) |
|
|
225 |
|
|
226 def last_changed_revision(self): |
|
|
227 rev1 = subversion.last_changed_revision(self.checkout_dir) |
|
|
228 rev2 = subversion.last_changed_revision(os.path.join(self.checkout_dir, |
|
|
229 "admin")) |
|
|
230 return max(rev1, rev2) |
|
|
231 |
|
|
232 def last_packaged_revision(self): |
|
|
233 """Returns the revision number of the highest packaged revision. |
|
|
234 |
|
|
235 If the revision cannot be determined because no already packaged |
|
|
236 revisions can be found, the function returns -1. |
|
|
237 """ |
|
|
238 revisions = [-1] |
|
|
239 if os.path.exists(self.pkg_dir): |
|
|
240 for filename in os.listdir(self.pkg_dir): |
|
|
241 match = self.pkg_dir_regex.match(filename) |
|
|
242 if match: |
|
|
243 revisions.append(int(match.group("revision"))) |
|
|
244 return max(revisions) |
|
|
245 |
|
|
246 def debian_source(self): |
|
|
247 return util.extract_value_for_key(open(os.path.join(self.debian_dir, |
|
|
248 "control")), |
|
|
249 "Source:") |
|
|
250 |
|
|
251 def update_checkout(self): |
|
|
252 """Updates the working copy of self.svn_url in self.checkout_dir. |
|
|
253 |
|
|
254 If self.checkout_dir doesn't exist yet, self.svn_url is checked |
|
|
255 out into that directory. |
|
|
256 """ |
|
|
257 localdir = self.checkout_dir |
|
|
258 if os.path.exists(localdir): |
|
|
259 logging.info("Updating the working copy in %r", localdir) |
|
|
260 subversion.update(localdir) |
|
|
261 else: |
|
|
262 logging.info("The working copy in %r doesn't exist yet." |
|
|
263 " Checking out fromo %r", localdir, |
|
|
264 self.svn_url) |
|
|
265 subversion.checkout(self.svn_url, localdir) |
|
|
266 |
|
|
267 def export_sources(self, to_dir): |
|
|
268 logging.info("Exporting sources for tarball to %r", to_dir) |
|
|
269 subversion.export(self.checkout_dir, to_dir) |
|
|
270 # some versions of svn (notably version 1.4.2 shipped with etch) |
|
|
271 # do export externals such as the admin subdirectory. We may |
|
|
272 # have to do that in an extra step. |
|
|
273 admindir = os.path.join(to_dir, "admin") |
|
|
274 if not os.path.isdir(admindir): |
|
|
275 subversion.export(os.path.join(self.checkout_dir, "admin"), |
|
|
276 admindir) |
|
|
277 |
|
|
278 def copy_debian_directory(self, to_dir): |
|
|
279 logging.info("Copying debian directory to %r", to_dir) |
|
|
280 shutil.copytree(self.debian_dir, to_dir) |
|
|
281 |
|
|
282 def debian_environment(self): |
|
|
283 """Returns the environment variables for the debian commands""" |
|
|
284 env = os.environ.copy() |
|
|
285 env["DEBFULLNAME"] = self.deb_fullname |
|
|
286 env["DEBEMAIL"] = self.deb_email |
|
|
287 return env |
|
|
288 |
|
|
289 def package_if_updated(self): |
|
|
290 """Checks if the checkout changed and returns a new packager if so""" |
|
|
291 self.update_checkout() |
|
|
292 current_revision = self.last_changed_revision() |
|
|
293 logging.info("New revision is %d", current_revision) |
|
|
294 previous_revision = self.last_packaged_revision() |
|
|
295 logging.info("Previously packaged revision was %d", previous_revision) |
|
|
296 if current_revision > previous_revision: |
|
|
297 logging.info("New revision is not packaged yet") |
|
|
298 return RevisionPackager(self, current_revision) |
|
|
299 else: |
|
|
300 logging.info("New revision already packaged.") |
|
|
301 |
|
|
302 |
|
|
303 |
|
|
304 class Packager(object): |
|
|
305 |
|
|
306 def __init__(self, assembly_lines, check_interval): |
|
|
307 self.assembly_lines = assembly_lines |
|
|
308 self.check_interval = check_interval |
|
|
309 |
|
|
310 def run(self): |
|
|
311 """Runs the plant indefinitely""" |
|
|
312 logging.info("Packager start. Will check every %d seconds", |
|
|
313 self.check_interval) |
|
|
314 last_check = -1 |
|
|
315 while 1: |
|
|
316 now = time.time() |
|
|
317 if now > last_check + self.check_interval: |
|
|
318 self.check_assembly_lines() |
|
|
319 last_check = now |
|
|
320 next_check = now + self.check_interval |
|
|
321 to_sleep = next_check - time.time() |
|
|
322 if to_sleep > 0: |
|
|
323 logging.info("Next check at %s", |
|
|
324 time.strftime("%Y-%m-%d %H:%M:%S", |
|
|
325 time.localtime(next_check))) |
|
|
326 time.sleep(to_sleep) |
|
|
327 else: |
|
|
328 logging.info("Next check now") |
|
|
329 |
|
|
330 def check_assembly_lines(self): |
|
|
331 logging.info("Checking assembly lines") |
|
|
332 for line in self.assembly_lines: |
|
|
333 try: |
|
|
334 packager = line.package_if_updated() |
|
|
335 if packager: |
|
|
336 packager.package() |
|
|
337 except: |
|
|
338 logging.exception("An error occurred while" |
|
|
339 " checking assembly line %r", line.name) |
|
|
340 logging.info("Checked all assembly lines") |
