Mercurial > treepkg

comparison bin/publishdebianpackages.py @ 441:eadcb1bb54e2

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Merged revisions 388-399 via svnmerge from svn+ssh://svn.wald.intevation.org/treepkg/branches/treepkg-status ........ r388 | bricks | 2010-07-28 10:38:50 +0200 (Mi, 28 Jul 2010) | 3 lines fix if statement upload hook should be called if it's not empty ........ r389 | bricks | 2010-07-28 13:38:00 +0200 (Mi, 28 Jul 2010) | 2 lines only rsync specified architectures from cachedir to publishdir ........ r390 | bricks | 2010-07-29 14:07:57 +0200 (Do, 29 Jul 2010) | 2 lines introduced after_copy_hook and made rsync upload to publishdir optional ........ r391 | bricks | 2010-07-29 15:43:39 +0200 (Do, 29 Jul 2010) | 3 lines publish package must be default to get original behaviour be more verbose at publishing ........ r392 | bricks | 2010-07-29 16:50:53 +0200 (Do, 29 Jul 2010) | 2 lines fix handling of showing only successfull builds together with num packages ........ r393 | bricks | 2010-07-29 16:52:42 +0200 (Do, 29 Jul 2010) | 2 lines fixed typo ........ r394 | bricks | 2010-07-29 16:59:10 +0200 (Do, 29 Jul 2010) | 2 lines fixed another typo ........ r395 | bricks | 2010-07-29 18:05:54 +0200 (Do, 29 Jul 2010) | 3 lines fixed another typo treepkg is really slow with many revisions ........ r396 | bricks | 2010-08-02 12:41:55 +0200 (Mo, 02 Aug 2010) | 2 lines fixed typos ........ r397 | bricks | 2010-08-02 17:36:12 +0200 (Mo, 02 Aug 2010) | 3 lines be more error prone in listing different files not checking if a dir exists caused several errors if a build wasn't successful ........ r398 | bricks | 2010-08-05 18:21:47 +0200 (Do, 05 Aug 2010) | 2 lines review changes ........ r399 | bricks | 2010-08-06 13:06:08 +0200 (Fr, 06 Aug 2010) | 4 lines cleanup modules fix test_info testcases added testcases for remove_trailingslashes and expand_filename ........
author Bjoern Ricks <bricks@intevation.de>
date Fri, 06 Aug 2010 13:41:54 +0000
parents 8e0c81870e5e
children 12f3642e1756
comparison
equal deleted inserted replaced
440:383d23c6bfde 441:eadcb1bb54e2
8 # Read the file COPYING coming with the software for details. 8 # Read the file COPYING coming with the software for details.
9 9
10 """Publishes selected packages created by treepkg""" 10 """Publishes selected packages created by treepkg"""
11 11
12 import os 12 import os
13 import os.path
14 import re
15 import sys 13 import sys
16 import shlex 14 import shlex
17 15
18 from optparse import OptionParser 16 from optparse import OptionParser
19 from ConfigParser import SafeConfigParser 17 from ConfigParser import SafeConfigParser
20 18
21 import treepkgcmd 19 import treepkgcmd
22 from treepkg.readconfig import read_config_section, convert_bool 20 from treepkg.readconfig import read_config_section, convert_bool
23 from treepkg.run import call, capture_output 21 from treepkg.run import call, capture_output
24 from treepkg.cmdexpand import cmdexpand 22 from treepkg.cmdexpand import cmdexpand
25 from treepkg.publish import * 23 from treepkg.publish import copy_arch_to_publishdir, prefix_for_remote_command
26 from treepkg.util import md5sum 24
25 from treepkg.util import md5sum, expand_filename, remove_trailing_slashes
27 from treepkg.info.status import TreepkgInfo 26 from treepkg.info.status import TreepkgInfo
28 from treepkg.info.data import Package 27 from treepkg.info.data import Package
29 from treepkg.info.data import CacheDb 28 from treepkg.info.data import CacheDb
30
31 EMPTY = re.compile(r'\s*')
32 29
33 config_desc = ["distribution", "section", "num_newest", 30 config_desc = ["distribution", "section", "num_newest",
34 "build_user", "build_host", "build_listpackages", 31 "build_user", "build_host", "build_listpackages",
35 "publish_user", "publish_host", 32 "publish_user", "publish_host",
36 ("architectures", shlex.split, "armel i386 source"), 33 ("architectures", shlex.split, "armel i386 source"),
37 ("after_upload_hook", shlex.split), 34 ("after_upload_hook", shlex.split, ""),
35 ("after_copy_hook", shlex.split, ""),
36 ("publish_packages", convert_bool, "True"),
38 ("publish_remove_old_packages", convert_bool), 37 ("publish_remove_old_packages", convert_bool),
39 ("publish_dir", remove_trailing_slashes), 38 ("publish_dir", remove_trailing_slashes),
40 ("cachedb", 39 ("cachedb", lambda s: expand_filename(s)),
41 lambda s: expand_filename(remove_trailing_slashes(s))),
42 ("cachedir", 40 ("cachedir",
43 lambda s: expand_filename(remove_trailing_slashes(s)))] 41 lambda s: expand_filename(remove_trailing_slashes(s)))]
44 42
45 43
46 def read_config(filename): 44 def read_config(filename):
76 74
77 def get_treepkg_info(variables): 75 def get_treepkg_info(variables):
78 runremote = prefix_for_remote_command(variables["build_user"], 76 runremote = prefix_for_remote_command(variables["build_user"],
79 variables["build_host"]) 77 variables["build_host"])
80 xml = capture_output(cmdexpand("@runremote $build_listpackages" 78 xml = capture_output(cmdexpand("@runremote $build_listpackages"
81 " --newest=$num_newest" 79 " --newest=$num_newest"
82 " --only-successful", 80 " --only-successful",
83 runremote=runremote, 81 runremote=runremote,
84 **variables)) 82 **variables))
85 return TreepkgInfo.fromxml(xml) 83 return TreepkgInfo.fromxml(xml)
86
87 def get_binary_arch(arch):
88 if not arch is None and not arch.startswith("binary") and \
89 not arch == "source":
90 arch = "binary-" + arch
91 return arch
92 84
93 def check_package_is_new(packagename, destdir, packagemd5sum): 85 def check_package_is_new(packagename, destdir, packagemd5sum):
94 destpackage = os.path.join(destdir, packagename) 86 destpackage = os.path.join(destdir, packagename)
95 if not os.path.isfile(destpackage): 87 if not os.path.isfile(destpackage):
96 return True 88 return True
105 md5sum = checksum.checksum 97 md5sum = checksum.checksum
106 break 98 break
107 return md5sum 99 return md5sum
108 100
109 def sort_trackname_arch(a, b): 101 def sort_trackname_arch(a, b):
110 if a.trackname < b.trackname: return -1 102 if a.trackname < b.trackname:
111 if a.trackname > b.trackname: return +1 103 return -1
104 if a.trackname > b.trackname:
105 return +1
112 return cmp(a.arch, b.arch) 106 return cmp(a.arch, b.arch)
113 107
114 def copy_files_to_destdir(destdir, files, variables, quiet = False): 108 def copy_files_to_destdir(destdir, files, variables, quiet=False):
115 scp_flags = [] 109 scp_flags = []
116 if quiet: 110 if quiet:
117 scp_flags.append("-q") 111 scp_flags.append("-q")
118 112
119 if not os.path.exists(destdir): 113 if not os.path.exists(destdir):
135 if not quiet: 129 if not quiet:
136 print "removing file %s" % package.filename 130 print "removing file %s" % package.filename
137 os.remove(package.filename) 131 os.remove(package.filename)
138 cachedb.remove_packages(oldpackages) 132 cachedb.remove_packages(oldpackages)
139 133
140 def copy_packages_to_destdir(cachedb, dir, packages, variables, quiet = False): 134 def copy_packages_to_destdir(cachedb, dir, packages, variables, quiet=False):
141 packages.sort(cmp=sort_trackname_arch) 135 packages.sort(cmp=sort_trackname_arch)
142 package = packages[0] 136 package = packages[0]
143 trackname = package.trackname 137 trackname = package.trackname
144 arch = package.arch 138 arch = package.arch
145 destdir = os.path.join(dir, arch, trackname) 139 destdir = os.path.join(dir, arch, trackname)
146 files = [] 140 files = []
147 for package in packages: 141 for package in packages:
148 cachedb.add_package(package) 142 cachedb.add_package(package)
149 if package.trackname != trackname or \ 143 if package.trackname != trackname or package.arch != arch:
150 package.arch != arch:
151 copy_files_to_destdir(destdir, files, variables, quiet) 144 copy_files_to_destdir(destdir, files, variables, quiet)
152 trackname = package.trackname 145 trackname = package.trackname
153 arch = package.arch 146 arch = package.arch
154 destdir = os.path.join(dir, arch, trackname) 147 destdir = os.path.join(dir, arch, trackname)
155 files = [] 148 files = []
158 files.append(package.sourcepath) 151 files.append(package.sourcepath)
159 if not quiet: 152 if not quiet:
160 print "copy new file: %s" % package.name 153 print "copy new file: %s" % package.name
161 copy_files_to_destdir(destdir, files, variables, quiet) 154 copy_files_to_destdir(destdir, files, variables, quiet)
162 155
163 def copy_to_cachedir(variables, track, revision, quiet = False, architectures=None): 156 def copy_to_cachedir(variables, track, revision, quiet=False,
157 architectures=()):
164 cachedir = variables["cachedir"] 158 cachedir = variables["cachedir"]
165 cachdebfilename = variables["cachedb"] 159 cachdebfilename = variables["cachedb"]
166 if not quiet: 160 if not quiet:
167 print "using cachedb: %s" % cachdebfilename 161 print "using cachedb: %s" % cachdebfilename
168 cachedb = CacheDb(cachdebfilename) 162 cachedb = CacheDb(cachdebfilename)
169 newpackages = [] 163 newpackages = []
170 treepkginfo = get_treepkg_info(variables) 164 treepkginfo = get_treepkg_info(variables)
171 #allowedarchs = set([]) # contains all wanted architectures (incl. source) 165 allarchs = set() # contains all present architectures (incl. source)
172 allarchs = set([]) # contains all present architectures (incl. source)
173 binaryallpackages = [] 166 binaryallpackages = []
174 # change e.g. armel in binary-armel 167 # change e.g. armel in binary-armel
175 if not architectures is None: 168 allowedarchs = set([get_binary_arch(a) for a in architectures])
176 allowedarchs = set([get_binary_arch(a) for a in architectures])
177 else:
178 allowedarchs = set([])
179 for track in treepkginfo.tracks: 169 for track in treepkginfo.tracks:
180 for rev in track.revisions: 170 for rev in track.revisions:
181 for packageinfo in rev.packages: 171 for packageinfo in rev.packages:
182 arch = get_binary_arch(packageinfo.arch) 172 arch = get_binary_arch(packageinfo.arch)
183 if packageinfo.type == "binary": 173 if packageinfo.type == "binary":
202 packageinfo.path, arch, 192 packageinfo.path, arch,
203 get_md5sum(packageinfo)) 193 get_md5sum(packageinfo))
204 newpackages.append(newpackage) 194 newpackages.append(newpackage)
205 # copy binary-all packages 195 # copy binary-all packages
206 sourcearch = set(["source"]) 196 sourcearch = set(["source"])
207 if len(allowedarchs) == 0: 197 if not allowedarchs:
208 binallarchs = allarchs - sourcearch 198 binallarchs = allarchs - sourcearch
209 elif len(allarchs) == 0: 199 elif allarchs:
210 binallarchs = allowedarchs - sourcearch 200 binallarchs = allowedarchs - sourcearch
211 else: 201 else:
212 binallarchs = (allowedarchs & allarchs) - sourcearch 202 binallarchs = (allowedarchs & allarchs) - sourcearch
213 for packageinfo in binaryallpackages: 203 for packageinfo in binaryallpackages:
214 for arch in binallarchs: 204 for arch in binallarchs:
215 filename = os.path.join(cachedir, arch, packageinfo.trackname, 205 filename = os.path.join(cachedir, arch, packageinfo.trackname,
216 packageinfo.name) 206 packageinfo.name)
217 newpackage = Package(filename, packageinfo.trackname, packageinfo.name, 207 newpackage = Package(filename, packageinfo.trackname,
218 packageinfo.path, arch, get_md5sum(packageinfo)) 208 packageinfo.name, packageinfo.path, arch,
209 get_md5sum(packageinfo))
219 newpackages.append(newpackage) 210 newpackages.append(newpackage)
220 copy_packages_to_destdir(cachedb, cachedir, newpackages, variables, quiet) 211 copy_packages_to_destdir(cachedb, cachedir, newpackages, variables, quiet)
221 remove_old_packages(cachedb, newpackages, quiet) 212 remove_old_packages(cachedb, newpackages, quiet)
222 return binallarchs 213 return binallarchs
223 214
228 dist = config["distribution"] 219 dist = config["distribution"]
229 if section is None: 220 if section is None:
230 section = config["section"] 221 section = config["section"]
231 222
232 architectures = config["architectures"] 223 architectures = config["architectures"]
233 allarchs = copy_to_cachedir(config, track, revision, quiet, architectures) 224 copy_to_cachedir(config, track, revision, quiet, architectures)
234 for arch in allarchs: 225
235 copy_to_publishdir(config, dist, section, arch, quiet) 226 if config["after_copy_hook"]:
227 if not quiet:
228 print "running after copy hook"
229 call(config["after_copy_hook"])
230
231 if config["publish_packages"]:
232 for arch in architectures:
233 if not quiet:
234 print "publish packages for architecture %s" % arch
235 copy_arch_to_publishdir(config, dist, section,
236 get_binary_arch(arch), quiet)
236 237
237 # update apt archive 238 # update apt archive
238 if not config["after_upload_hook"] or \ 239 if config["after_upload_hook"]:
239 not EMPTY.match(config["after_upload_hook"]):
240 if not quiet: 240 if not quiet:
241 print "running after upload hook" 241 print "running after upload hook"
242 call(config["after_upload_hook"]) 242 call(config["after_upload_hook"])
243 243
244 def main(): 244 def main():
This site is hosted by Intevation GmbH (Datenschutzerklärung und Impressum | Privacy Policy and Imprint)