Mercurial > treepkg

comparison bin/publishdebianpackages.py @ 416:4980f8d5014a treepkg-status

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
first version of incremental copying debian packages to cachedir
author Bjoern Ricks <bricks@intevation.de>
date Mon, 26 Jul 2010 10:12:40 +0000
parents e0539b483b04
children 25ef11a79d7f
comparison
equal deleted inserted replaced
415:2d31eea39f70 416:4980f8d5014a
8 # Read the file COPYING coming with the software for details. 8 # Read the file COPYING coming with the software for details.
9 9
10 """Publishes selected packages created by treepkg""" 10 """Publishes selected packages created by treepkg"""
11 11
12 import os 12 import os
13 import os.path
13 import sys 14 import sys
14 import shlex 15 import shlex
15 16
16 from optparse import OptionParser 17 from optparse import OptionParser
17 from ConfigParser import SafeConfigParser 18 from ConfigParser import SafeConfigParser
26 from treepkg.info.data import Package 27 from treepkg.info.data import Package
27 from treepkg.info.data import CacheDb 28 from treepkg.info.data import CacheDb
28 29
29 config_desc = ["distribution", "section", "num_newest", 30 config_desc = ["distribution", "section", "num_newest",
30 "build_user", "build_host", "build_listpackages", 31 "build_user", "build_host", "build_listpackages",
31 "publish_user", "publish_host", 32 "publish_user", "publish_host", "cachedb",
32 ("architectures", shlex.split, "armel i386 source"), 33 ("architectures", shlex.split, "armel i386 source"),
33 ("after_upload_hook", shlex.split), 34 ("after_upload_hook", shlex.split),
34 ("publish_remove_old_packages", convert_bool), 35 ("publish_remove_old_packages", convert_bool),
35 ("publish_dir", remove_trailing_slashes), 36 ("publish_dir", remove_trailing_slashes),
36 ("cachedir", 37 ("cachedir",
103 def sort_trackname_arch(a, b): 104 def sort_trackname_arch(a, b):
104 if a.trackname < b.trackname: return -1 105 if a.trackname < b.trackname: return -1
105 if a.trackname > b.trackname: return +1 106 if a.trackname > b.trackname: return +1
106 return cmp(a.arch, b.arch) 107 return cmp(a.arch, b.arch)
107 108
108 def copy_files(destdir, files, quiet): 109 def copy_files_to_destdir(destdir, files, quiet):
109 scp_flags = [] 110 scp_flags = []
110 if quiet: 111 if quiet:
111 scp_flags.append("-q") 112 scp_flags.append("-q")
112 113
113 if not os.path.exists(destdir): 114 if not os.path.exists(destdir):
115 116
116 # scp the packages to the cache dir 117 # scp the packages to the cache dir
117 call(cmdexpand("scp -p @scp_flags @files $cachedir/", files=files, 118 call(cmdexpand("scp -p @scp_flags @files $cachedir/", files=files,
118 scp_flags=scp_flags, cachedir=destdir)) 119 scp_flags=scp_flags, cachedir=destdir))
119 120
120 def copy_to_destdir(dir, packages, quiet = False): 121 def remove_old_packages(cachedb, newpackages):
122 newfiles = [package.filename for package in newpackages]
123 oldpackages = cachedb.get_old_packages(newfiles)
124 for package in oldpackages:
125 # better check if the file really exists
126 if os.path.isfile(package.filename):
127 os.remove(package.filename)
128 cachedb.remove_packages(oldpackages)
129
130 def copy_packages_to_destdir(cachedb, dir, packages, quiet = False):
121 packages.sort(cmp=sort_trackname_arch) 131 packages.sort(cmp=sort_trackname_arch)
122 package = packages[0] 132 package = packages[0]
123 trackname = package.name 133 trackname = package.name
124 arch = package.arch 134 arch = package.arch
125 destdir = os.path.join(dir, arch, trackname) 135 destdir = os.path.join(dir, arch, trackname)
126 files = [] 136 files = []
127 for package in packages: 137 for package in packages:
138 cachedb.add_package(package)
128 if package.trackname != trackname or \ 139 if package.trackname != trackname or \
129 package.arch != arch: 140 package.arch != arch:
130 #copy_files(destdir, files, quiet) 141 copy_files_to_destdir(destdir, files, quiet)
131 trackname = package.trackname 142 trackname = package.trackname
132 arch = package.arch 143 arch = package.arch
133 destdir = os.path.join(dir, arch, trackname) 144 destdir = os.path.join(dir, arch, trackname)
134 files = [] 145 files = []
135 # add only if the packages differ 146 # add only to copy files list if the packages differ
136 if check_package_is_new(package.name, destdir, package.md5sum): 147 if check_package_is_new(package.name, destdir, package.md5sum):
137 files.append(package.sourcepath) 148 files.append(package.sourcepath)
138 print "package: %s, destdir: %s" % (package.name, destdir) 149 print "copy package: %s, source: %s, destdir: %s" % (package.name,
139 150 package.sourcepath, destdir)
140 def copy_to_cache(variables, track, revision, quiet = False, architectures=None): 151
152 def copy_to_cachedir(variables, track, revision, quiet = False, architectures=None):
141 cachedir = variables["cachedir"] 153 cachedir = variables["cachedir"]
154 cachdebfilename = variables["cachedb"]
155 cachedb = CacheDb(cachdebfilename)
142 newpackages = [] 156 newpackages = []
143 treepkginfo = get_treepkg_info(variables) 157 treepkginfo = get_treepkg_info(variables)
144 #allowedarchs = set([]) # contains all wanted architectures (incl. source) 158 #allowedarchs = set([]) # contains all wanted architectures (incl. source)
145 allarchs = set([]) # contains all present architectures (incl. source) 159 allarchs = set([]) # contains all present architectures (incl. source)
146 binaryallpackages = [] 160 binaryallpackages = []
147 # change e.g. armel in binary-armel 161 # change e.g. armel in binary-armel
148 if not architectures is None: 162 if not architectures is None:
149 allowedarchs = set([get_binary_arch(a) for a in architectures]) 163 allowedarchs = set([get_binary_arch(a) for a in architectures])
150 else: 164 else:
151 allowedarchs = set([]) 165 allowedarchs = set([])
152 print "allowedarchs: %s" % allowedarchs
153 for track in treepkginfo.tracks: 166 for track in treepkginfo.tracks:
154 for rev in track.revisions: 167 for rev in track.revisions:
155 for package in rev.packages: 168 for packageinfo in rev.packages:
156 arch = get_binary_arch(package.arch) 169 arch = get_binary_arch(packageinfo.arch)
157 if package.type == "binary": 170 if packageinfo.type == "binary":
158 # skip other files 171 # skip other files
159 if package.arch is None: 172 if packageinfo.arch is None:
160 continue 173 continue
161 # handle binary-all 174 # handle binary-all
162 if arch == "binary-all": 175 if arch == "binary-all":
163 # add trackname for subdir name 176 # add trackname for subdir name
164 package.trackname = track.name 177 packageinfo.trackname = track.name
165 binaryallpackages.append(package) 178 binaryallpackages.append(packageinfo)
166 continue 179 continue
167 allarchs.add(arch) 180 allarchs.add(arch)
168 elif package.type == "source": 181 elif packageinfo.type == "source":
169 arch = package.type 182 arch = packageinfo.type
170 print "package: %s %s" % (package.name, arch)
171 # only copy requested archs 183 # only copy requested archs
172 if len(allowedarchs) == 0 or \ 184 if len(allowedarchs) == 0 or \
173 arch in allowedarchs: 185 arch in allowedarchs:
174 filename = os.path.join(cachedir, arch, track.name, 186 filename = os.path.join(cachedir, arch, track.name,
175 package.name) 187 packageinfo.name)
176 newpackage = Package(filename, track.name, package.name, 188 newpackage = Package(filename, track.name, packageinfo.name,
177 package.path, arch, get_md5sum(package)) 189 packageinfo.path, arch,
190 get_md5sum(packageinfo))
178 newpackages.append(newpackage) 191 newpackages.append(newpackage)
179 # copy binary-all packages 192 # copy binary-all packages
180 sourcearch = set(["source"]) 193 sourcearch = set(["source"])
181 if len(allowedarchs) == 0: 194 if len(allowedarchs) == 0:
182 binallarchs = allarchs - sourcearch 195 binallarchs = allarchs - sourcearch
183 elif len(allarchs) == 0: 196 elif len(allarchs) == 0:
184 binallarchs = allowedarchs - sourcearch 197 binallarchs = allowedarchs - sourcearch
185 else: 198 else:
186 binallarchs = (allowedarchs & allarchs) - sourcearch 199 binallarchs = (allowedarchs & allarchs) - sourcearch
187 for package in binaryallpackages: 200 for packageinfo in binaryallpackages:
188 for arch in binallarchs: 201 for arch in binallarchs:
189 filename = os.path.join(cachedir, arch, track.name, 202 filename = os.path.join(cachedir, arch, track.name,
190 package.name) 203 packageinfo.name)
191 newpackage = Package(filename, package.trackname, package.name, 204 newpackage = Package(filename, packageinfo.trackname, packageinfo.name,
192 package.path, arch, get_md5sum(package)) 205 packageinfo.path, arch, get_md5sum(packageinfo))
193 newpackages.append(newpackage) 206 newpackages.append(newpackage)
194 print newpackages 207 copy_packages_to_destdir(cachedb, cachedir, newpackages, quiet)
195 copy_to_destdir(cachedir, newpackages, quiet) 208 remove_old_packages(cachedb, newpackages)
196
197 def publish_packages_arch(variables, track, revision, dist, section,
198 quiet, architectures):
199 copy_to_cache(variables, track, revision, quiet, architectures)
200 # copy_to_publishdir(variables, dist, section, arch, quiet)
201 209
202 def publish_packages(config_filename, track, revision, dist, section, quiet): 210 def publish_packages(config_filename, track, revision, dist, section, quiet):
203 config = read_config(config_filename) 211 config = read_config(config_filename)
204 212
205 if dist is None: 213 if dist is None:
206 dist = config["distribution"] 214 dist = config["distribution"]
207 if section is None: 215 if section is None:
208 section = config["section"] 216 section = config["section"]
209 217
218 if not "cachedb" in config.keys():
219 print >>sys.stderr, "Error. Please specifiy a cachedb in your config file"
220 sys.exit(1)
221
210 architectures = config["architectures"] 222 architectures = config["architectures"]
211 publish_packages_arch(config, track, revision, dist, section, 223 copy_to_cachedir(config, track, revision, quiet, architectures)
212 quiet, architectures) 224 # copy_to_publishdir(config, dist, section, arch, quiet)
213 225
214 # update apt archive 226 # update apt archive
215 # call(config["after_upload_hook"]) 227 # call(config["after_upload_hook"])
216 228
217 def main(): 229 def main():
This site is hosted by Intevation GmbH (Datenschutzerklärung und Impressum | Privacy Policy and Imprint)