Mercurial > treepkg

comparison bin/publishdebianpackages.py @ 413:94a6ae627b31 treepkg-status

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
implemented CacheDb to store copied Packages from build host
author Bjoern Ricks <bricks@intevation.de>
date Fri, 23 Jul 2010 16:40:38 +0000
parents 58ecf7c0ecba
children e0539b483b04
comparison
equal deleted inserted replaced
412:58ecf7c0ecba 413:94a6ae627b31
10 """Publishes selected packages created by treepkg""" 10 """Publishes selected packages created by treepkg"""
11 11
12 import os 12 import os
13 import sys 13 import sys
14 import shlex 14 import shlex
15 import sqlite3
15 16
16 from optparse import OptionParser 17 from optparse import OptionParser
17 from ConfigParser import SafeConfigParser 18 from ConfigParser import SafeConfigParser
18 19
19 import treepkgcmd 20 import treepkgcmd
31 ("after_upload_hook", shlex.split), 32 ("after_upload_hook", shlex.split),
32 ("publish_remove_old_packages", convert_bool), 33 ("publish_remove_old_packages", convert_bool),
33 ("publish_dir", remove_trailing_slashes), 34 ("publish_dir", remove_trailing_slashes),
34 ("cachedir", 35 ("cachedir",
35 lambda s: expand_filename(remove_trailing_slashes(s)))] 36 lambda s: expand_filename(remove_trailing_slashes(s)))]
37
38 class Package:
39
40 def __init__(self, filename, trackname, packagename, packagepath, arch, md5sum):
41 self.filename = filename
42 self.trackname = trackname
43 self.name = packagename
44 self.sourcepath = packagepath
45 self.arch = arch
46 self.md5sum = md5sum
47
48
49 class CacheDb:
50
51
52 def __init__(self, file):
53 self.SELECT_PACKAGE_TMPL = """SELECT * FROM packages
54 WHERE filename = ?"""
55 self.file = file
56 self.conn = sqlite3.connect(file)
57 self.cursor = self.conn.cursor()
58 self.init_db(file)
59
60 def init_db(self, file):
61 self.cursor.execute(
62 """CREATE TABLE IF NOT EXISTS packages (
63 filename TEXT PRIMARY KEY,
64 trackname TEXT,
65 packagename TEXT,
66 sourcepath TEXT,
67 arch TEXT,
68 md5sum TEXT )""")
69 self.conn.commit()
70
71
72 def add_package(self, package):
73 self.cursor.execute(self.SELECT_PACKAGE_TMPL, (package.filename,))
74 row = self.cursor.fetchone()
75 if not row:
76 self.insert_package(package)
77 else:
78 self.update_package(package)
79
80 def insert_package(self, package):
81 INSERT_TMPL = """INSERT INTO packages VALUES (
82 ?, ?, ? ,? ,?, ?)"""
83 self.cursor.execute(INSERT_TMPL, (package.filename,
84 package.trackname,
85 package.name,
86 package.sourcepath,
87 package.arch,
88 package.md5sum))
89 self.conn.commit()
90
91 def update_package(self, package):
92 UPDATE_TMPL = """UPDATE packages set md5sum = '?'
93 WHERE filename in (?)"""
94 self.cursor.execute(UPDATE_TMPL, (package.md5sum, package.filename))
95 self.conn.commit()
96
97 def get_package(self, filename):
98 self.cursor.execute(self.SELECT_PACKAGE_TMPL, (filename,))
99 row = self.cursor.fetchone()
100 if not row:
101 return None
102 return Package(row[0], row[1], row[2], row[3], row[4], row[5])
103
104 def get_old_packages(self, newfiles):
105 SELECT_TMPL = """SELECT * FROM packages
106 WHERE filename not in (%s)"""
107 tmp = ", ".join(['?'] * len(newfiles))
108 self.cursor.execute(SELECT_TMPL % tmp, newfiles)
109 return [Package(*row) for row in self.cursor.fetchall()]
110
111 def remove_packages(self, files):
112 DELET_TMPL = """DELETE FROM packages
113 WHERE filename in (%s)"""
114 tmp = ", ".join(['?'] * len(files))
115 self.cursor.execute(DELET_TMPL % tmp, files)
116 self.conn.commit()
117
118 def get_packages(self):
119 SELECT_TMPL = "SELECT * FROM packages"
120 self.cursor.execute(SELECT_TMPL)
121 return [Package(*row) for row in self.cursor.fetchall()]
36 122
37 def read_config(filename): 123 def read_config(filename):
38 if not os.path.exists(filename): 124 if not os.path.exists(filename):
39 print >>sys.stderr, "Config file %s does not exist" % filename 125 print >>sys.stderr, "Config file %s does not exist" % filename
40 sys.exit(1) 126 sys.exit(1)
68 def get_treepkg_info(variables): 154 def get_treepkg_info(variables):
69 runremote = prefix_for_remote_command(variables["build_user"], 155 runremote = prefix_for_remote_command(variables["build_user"],
70 variables["build_host"]) 156 variables["build_host"])
71 xml = capture_output(cmdexpand("@runremote $build_listpackages" 157 xml = capture_output(cmdexpand("@runremote $build_listpackages"
72 " --newest=$num_newest", 158 " --newest=$num_newest",
73 #runremote=runremote, 159 runremote=runremote,
74 runremote="",
75 **variables)) 160 **variables))
76 return TreepkgInfo.fromxml(xml) 161 return TreepkgInfo.fromxml(xml)
77 162
78 def get_binary_arch(arch): 163 def get_binary_arch(arch):
79 if not arch is None and not arch.startswith("binary") and \ 164 if not arch is None and not arch.startswith("binary") and \
86 if not os.path.isfile(destpackage): 171 if not os.path.isfile(destpackage):
87 return True 172 return True
88 destmd5sum = md5sum(destpackage) 173 destmd5sum = md5sum(destpackage)
89 return (destmd5sum != packagemd5sum) 174 return (destmd5sum != packagemd5sum)
90 175
91 def copy_to_destdir(package, dir, arch, subdir, quiet = False): 176 def get_md5sum(packageinfo):
177 md5sum = ""
178 if packageinfo:
179 for checksum in packageinfo.checksums:
180 if checksum.type == "md5":
181 md5sum = checksum.checksum
182 break
183 return md5sum
184
185 def sort_trackname_arch(a, b):
186 if a.trackname < b.trackname: return -1
187 if a.trackname > b.trackname: return +1
188 return cmp(a.arch, b.arch)
189
190 def copy_files(destdir, files, quiet):
92 scp_flags = [] 191 scp_flags = []
93 if quiet: 192 if quiet:
94 scp_flags.append("-q") 193 scp_flags.append("-q")
95 destdir = os.path.join(dir, arch, subdir) 194
96 print "package: %s, destdir: %s" % (package.name, destdir) 195 if not os.path.exists(destdir):
97 md5sum = "" 196 os.makedirs(destdir)
98 for checksum in package.checksums: 197
99 if checksum.type == "md5": 198 # scp the packages to the cache dir
100 md5sum = checksum.checksum 199 call(cmdexpand("scp -p @scp_flags @files $cachedir/", files=files,
101 break 200 scp_flags=scp_flags, cachedir=destdir))
102 # update only if the packages differ 201
103 if check_package_is_new(package.name, destdir, md5sum): 202 def copy_to_destdir(dir, packages, quiet = False):
104 pass 203 packages.sort(cmp=sort_trackname_arch)
105 # scp the packages to the cache dir 204 package = packages[0]
106 #call(cmdexpand("scp -p @scp_flags $file $cachedir/", file=package.path, 205 trackname = package.name
107 # scp_flags=scp_flags, cachedir=destdir)) 206 arch = package.arch
207 destdir = os.path.join(dir, arch, trackname)
208 files = []
209 for package in packages:
210 if package.trackname != trackname or \
211 package.arch != arch:
212 #copy_files(destdir, files, quiet)
213 trackname = package.trackname
214 arch = package.arch
215 destdir = os.path.join(dir, arch, trackname)
216 files = []
217 # add only if the packages differ
218 if check_package_is_new(package.name, destdir, package.md5sum):
219 files.append(package.sourcepath)
220 print "package: %s, destdir: %s" % (package.name, destdir)
108 221
109 def copy_to_cache(variables, track, revision, quiet = False, architectures=None): 222 def copy_to_cache(variables, track, revision, quiet = False, architectures=None):
110 cachedir = variables["cachedir"] 223 cachedir = variables["cachedir"]
224 newpackages = []
111 treepkginfo = get_treepkg_info(variables) 225 treepkginfo = get_treepkg_info(variables)
112 #allowedarchs = set([]) # contains all wanted architectures (incl. source) 226 #allowedarchs = set([]) # contains all wanted architectures (incl. source)
113 allarchs = set([]) # contains all present architectures (incl. source) 227 allarchs = set([]) # contains all present architectures (incl. source)
114 binaryallpackages = [] 228 binaryallpackages = []
115 # change e.g. armel in binary-armel 229 # change e.g. armel in binary-armel
116 if not architectures is None: 230 if not architectures is None:
117 allowedarchs = set([get_binary_arch(a) for a in architectures]) 231 allowedarchs = set([get_binary_arch(a) for a in architectures])
118 else: 232 else:
119 allowedarchs = set([]) 233 allowedarchs = set([])
120 234 print "allowedarchs: %s" % allowedarchs
121 for track in treepkginfo.tracks: 235 for track in treepkginfo.tracks:
122 for rev in track.revisions: 236 for rev in track.revisions:
123 for package in rev.packages: 237 for package in rev.packages:
124 arch = get_binary_arch(package.arch) 238 arch = get_binary_arch(package.arch)
125 print "%s %s %s %s" % (track.name, package.name, package.type, package.arch)
126 if package.type == "binary": 239 if package.type == "binary":
127 # skip other files 240 # skip other files
128 if package.arch is None: 241 if package.arch is None:
129 continue 242 continue
130 # handle binary-all 243 # handle binary-all
134 binaryallpackages.append(package) 247 binaryallpackages.append(package)
135 continue 248 continue
136 allarchs.add(arch) 249 allarchs.add(arch)
137 elif package.type == "source": 250 elif package.type == "source":
138 arch = package.type 251 arch = package.type
252 print "package: %s %s" % (package.name, arch)
139 # only copy requested archs 253 # only copy requested archs
140 if len(allowedarchs) == 0 or \ 254 if len(allowedarchs) == 0 or \
141 arch in allowedarchs: 255 arch in allowedarchs:
142 copy_to_destdir(package, cachedir, arch, track.name, 256 filename = os.path.join(cachedir, arch, track.name,
143 quiet) 257 package.name)
144 258 newpackage = Package(filename, track.name, package.name,
145 #print "architectures %s" % architectures 259 package.path, arch, get_md5sum(package))
146 #print "allowed %s" % allowedarchs 260 newpackages.append(newpackage)
147 #print "all %s" % allarchs
148 # copy binary-all packages 261 # copy binary-all packages
149 sourcearch = set(["source"]) 262 sourcearch = set(["source"])
150 if len(allowedarchs) == 0: 263 if len(allowedarchs) == 0:
151 binallarchs = allarchs - sourcearch 264 binallarchs = allarchs - sourcearch
152 elif len(allarchs) == 0: 265 elif len(allarchs) == 0:
153 binallarchs = allowedarchs - sourcearch 266 binallarchs = allowedarchs - sourcearch
154 else: 267 else:
155 binallarchs = (allowedarchs & allarchs) - sourcearch 268 binallarchs = (allowedarchs & allarchs) - sourcearch
156 #print "binallarcgs %s" % binallarchs
157 for package in binaryallpackages: 269 for package in binaryallpackages:
158 for arch in binallarchs: 270 for arch in binallarchs:
159 copy_to_destdir(package, cachedir, arch, package.trackname, quiet) 271 filename = os.path.join(cachedir, arch, track.name,
272 package.name)
273 newpackage = Package(filename, package.trackname, package.name,
274 package.path, arch, get_md5sum(package))
275 newpackages.append(newpackage)
276 print newpackages
277 copy_to_destdir(cachedir, newpackages, quiet)
160 278
161 def publish_packages_arch(variables, track, revision, dist, section, 279 def publish_packages_arch(variables, track, revision, dist, section,
162 quiet, architectures): 280 quiet, architectures):
163 copy_to_cache(variables, track, revision, quiet, architectures) 281 copy_to_cache(variables, track, revision, quiet, architectures)
164 # copy_to_publishdir(variables, dist, section, arch, quiet) 282 # copy_to_publishdir(variables, dist, section, arch, quiet)
This site is hosted by Intevation GmbH (Datenschutzerklärung und Impressum | Privacy Policy and Imprint)