Mercurial > treepkg

annotate bin/publishdebianpackages.py @ 416:4980f8d5014a treepkg-status

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
first version of incremental copying debian packages to cachedir
author Bjoern Ricks <bricks@intevation.de>
date Mon, 26 Jul 2010 10:12:40 +0000
parents e0539b483b04
children 25ef11a79d7f
rev   line source
406
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
1 #! /usr/bin/python
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
2 # Copyright (C) 2007 - 2010 by Intevation GmbH
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
3 # Authors:
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
4 # Bernhard Herzog <bh@intevation.de>
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
5 # Bjoern Ricks <bjoern.ricks@intevation.de>
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
6 #
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
7 # This program is free software under the GPL (>=v2)
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
8 # Read the file COPYING coming with the software for details.
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
9
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
10 """Publishes selected packages created by treepkg"""
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
11
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
12 import os
416
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
13 import os.path
406
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
14 import sys
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
15 import shlex
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
16
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
17 from optparse import OptionParser
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
18 from ConfigParser import SafeConfigParser
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
19
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
20 import treepkgcmd
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
21 from treepkg.readconfig import read_config_section, convert_bool
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
22 from treepkg.run import call, capture_output
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
23 from treepkg.cmdexpand import cmdexpand
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
24 from treepkg.publish import *
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
25 from treepkg.util import md5sum
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
26 from treepkg.info.status import TreepkgInfo
414
e0539b483b04 moved data handling in publishdebianpackages into seperate module
Bjoern Ricks <bricks@intevation.de>
parents: 413
diff changeset
27 from treepkg.info.data import Package
e0539b483b04 moved data handling in publishdebianpackages into seperate module
Bjoern Ricks <bricks@intevation.de>
parents: 413
diff changeset
28 from treepkg.info.data import CacheDb
406
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
29
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
30 config_desc = ["distribution", "section", "num_newest",
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
31 "build_user", "build_host", "build_listpackages",
416
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
32 "publish_user", "publish_host", "cachedb",
412
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
33 ("architectures", shlex.split, "armel i386 source"),
406
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
34 ("after_upload_hook", shlex.split),
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
35 ("publish_remove_old_packages", convert_bool),
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
36 ("publish_dir", remove_trailing_slashes),
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
37 ("cachedir",
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
38 lambda s: expand_filename(remove_trailing_slashes(s)))]
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
39
413
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
40
406
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
41 def read_config(filename):
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
42 if not os.path.exists(filename):
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
43 print >>sys.stderr, "Config file %s does not exist" % filename
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
44 sys.exit(1)
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
45 parser = SafeConfigParser()
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
46 parser.read([filename])
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
47 return read_config_section(parser, "publishpackages", config_desc)
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
48
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
49 def parse_commandline():
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
50 parser = OptionParser()
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
51 parser.set_defaults(config_file=os.path.join(treepkgcmd.topdir,
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
52 "publishpackages.cfg"),
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
53 quiet=False)
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
54 parser.add_option("--config-file",
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
55 help=("The configuration file."
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
56 " Default is publishpackages.cfg"))
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
57 parser.add_option("--dist",
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
58 help=("The debian distribution name to use on"
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
59 " the publishing system"))
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
60 parser.add_option("--section",
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
61 help=("The debian distribution section name to use on"
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
62 " the publishing system"))
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
63 parser.add_option("--track",
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
64 help=("The package track whose files are to be"
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
65 " published. If not given, files of all tracks"
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
66 " will be published"))
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
67 parser.add_option("--quiet", action="store_true",
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
68 help=("Do not print progress meters or other"
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
69 " informational output"))
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
70 return parser.parse_args()
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
71
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
72 def get_treepkg_info(variables):
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
73 runremote = prefix_for_remote_command(variables["build_user"],
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
74 variables["build_host"])
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
75 xml = capture_output(cmdexpand("@runremote $build_listpackages"
414
e0539b483b04 moved data handling in publishdebianpackages into seperate module
Bjoern Ricks <bricks@intevation.de>
parents: 413
diff changeset
76 " --newest=$num_newest"
e0539b483b04 moved data handling in publishdebianpackages into seperate module
Bjoern Ricks <bricks@intevation.de>
parents: 413
diff changeset
77 " --only-successful",
413
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
78 runremote=runremote,
409
ecc671a84a73 fixed small bugs
Bjoern Ricks <bricks@intevation.de>
parents: 408
diff changeset
79 **variables))
408
02d498ee90b8 copy binary-all packages in all binary-xyz dirs
Bjoern Ricks <bricks@intevation.de>
parents: 406
diff changeset
80 return TreepkgInfo.fromxml(xml)
406
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
81
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
82 def get_binary_arch(arch):
412
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
83 if not arch is None and not arch.startswith("binary") and \
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
84 not arch == "source":
406
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
85 arch = "binary-" + arch
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
86 return arch
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
87
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
88 def check_package_is_new(packagename, destdir, packagemd5sum):
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
89 destpackage = os.path.join(destdir, packagename)
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
90 if not os.path.isfile(destpackage):
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
91 return True
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
92 destmd5sum = md5sum(destpackage)
408
02d498ee90b8 copy binary-all packages in all binary-xyz dirs
Bjoern Ricks <bricks@intevation.de>
parents: 406
diff changeset
93 return (destmd5sum != packagemd5sum)
02d498ee90b8 copy binary-all packages in all binary-xyz dirs
Bjoern Ricks <bricks@intevation.de>
parents: 406
diff changeset
94
413
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
95 def get_md5sum(packageinfo):
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
96 md5sum = ""
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
97 if packageinfo:
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
98 for checksum in packageinfo.checksums:
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
99 if checksum.type == "md5":
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
100 md5sum = checksum.checksum
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
101 break
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
102 return md5sum
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
103
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
104 def sort_trackname_arch(a, b):
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
105 if a.trackname < b.trackname: return -1
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
106 if a.trackname > b.trackname: return +1
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
107 return cmp(a.arch, b.arch)
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
108
416
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
109 def copy_files_to_destdir(destdir, files, quiet):
408
02d498ee90b8 copy binary-all packages in all binary-xyz dirs
Bjoern Ricks <bricks@intevation.de>
parents: 406
diff changeset
110 scp_flags = []
412
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
111 if quiet:
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
112 scp_flags.append("-q")
413
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
113
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
114 if not os.path.exists(destdir):
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
115 os.makedirs(destdir)
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
116
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
117 # scp the packages to the cache dir
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
118 call(cmdexpand("scp -p @scp_flags @files $cachedir/", files=files,
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
119 scp_flags=scp_flags, cachedir=destdir))
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
120
416
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
121 def remove_old_packages(cachedb, newpackages):
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
122 newfiles = [package.filename for package in newpackages]
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
123 oldpackages = cachedb.get_old_packages(newfiles)
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
124 for package in oldpackages:
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
125 # better check if the file really exists
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
126 if os.path.isfile(package.filename):
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
127 os.remove(package.filename)
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
128 cachedb.remove_packages(oldpackages)
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
129
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
130 def copy_packages_to_destdir(cachedb, dir, packages, quiet = False):
413
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
131 packages.sort(cmp=sort_trackname_arch)
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
132 package = packages[0]
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
133 trackname = package.name
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
134 arch = package.arch
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
135 destdir = os.path.join(dir, arch, trackname)
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
136 files = []
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
137 for package in packages:
416
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
138 cachedb.add_package(package)
413
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
139 if package.trackname != trackname or \
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
140 package.arch != arch:
416
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
141 copy_files_to_destdir(destdir, files, quiet)
413
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
142 trackname = package.trackname
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
143 arch = package.arch
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
144 destdir = os.path.join(dir, arch, trackname)
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
145 files = []
416
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
146 # add only to copy files list if the packages differ
413
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
147 if check_package_is_new(package.name, destdir, package.md5sum):
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
148 files.append(package.sourcepath)
416
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
149 print "copy package: %s, source: %s, destdir: %s" % (package.name,
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
150 package.sourcepath, destdir)
406
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
151
416
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
152 def copy_to_cachedir(variables, track, revision, quiet = False, architectures=None):
408
02d498ee90b8 copy binary-all packages in all binary-xyz dirs
Bjoern Ricks <bricks@intevation.de>
parents: 406
diff changeset
153 cachedir = variables["cachedir"]
416
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
154 cachdebfilename = variables["cachedb"]
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
155 cachedb = CacheDb(cachdebfilename)
413
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
156 newpackages = []
406
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
157 treepkginfo = get_treepkg_info(variables)
412
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
158 #allowedarchs = set([]) # contains all wanted architectures (incl. source)
408
02d498ee90b8 copy binary-all packages in all binary-xyz dirs
Bjoern Ricks <bricks@intevation.de>
parents: 406
diff changeset
159 allarchs = set([]) # contains all present architectures (incl. source)
02d498ee90b8 copy binary-all packages in all binary-xyz dirs
Bjoern Ricks <bricks@intevation.de>
parents: 406
diff changeset
160 binaryallpackages = []
406
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
161 # change e.g. armel in binary-armel
408
02d498ee90b8 copy binary-all packages in all binary-xyz dirs
Bjoern Ricks <bricks@intevation.de>
parents: 406
diff changeset
162 if not architectures is None:
412
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
163 allowedarchs = set([get_binary_arch(a) for a in architectures])
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
164 else:
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
165 allowedarchs = set([])
408
02d498ee90b8 copy binary-all packages in all binary-xyz dirs
Bjoern Ricks <bricks@intevation.de>
parents: 406
diff changeset
166 for track in treepkginfo.tracks:
406
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
167 for rev in track.revisions:
416
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
168 for packageinfo in rev.packages:
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
169 arch = get_binary_arch(packageinfo.arch)
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
170 if packageinfo.type == "binary":
412
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
171 # skip other files
416
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
172 if packageinfo.arch is None:
412
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
173 continue
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
174 # handle binary-all
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
175 if arch == "binary-all":
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
176 # add trackname for subdir name
416
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
177 packageinfo.trackname = track.name
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
178 binaryallpackages.append(packageinfo)
412
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
179 continue
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
180 allarchs.add(arch)
416
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
181 elif packageinfo.type == "source":
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
182 arch = packageinfo.type
408
02d498ee90b8 copy binary-all packages in all binary-xyz dirs
Bjoern Ricks <bricks@intevation.de>
parents: 406
diff changeset
183 # only copy requested archs
02d498ee90b8 copy binary-all packages in all binary-xyz dirs
Bjoern Ricks <bricks@intevation.de>
parents: 406
diff changeset
184 if len(allowedarchs) == 0 or \
412
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
185 arch in allowedarchs:
413
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
186 filename = os.path.join(cachedir, arch, track.name,
416
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
187 packageinfo.name)
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
188 newpackage = Package(filename, track.name, packageinfo.name,
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
189 packageinfo.path, arch,
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
190 get_md5sum(packageinfo))
413
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
191 newpackages.append(newpackage)
408
02d498ee90b8 copy binary-all packages in all binary-xyz dirs
Bjoern Ricks <bricks@intevation.de>
parents: 406
diff changeset
192 # copy binary-all packages
412
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
193 sourcearch = set(["source"])
408
02d498ee90b8 copy binary-all packages in all binary-xyz dirs
Bjoern Ricks <bricks@intevation.de>
parents: 406
diff changeset
194 if len(allowedarchs) == 0:
412
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
195 binallarchs = allarchs - sourcearch
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
196 elif len(allarchs) == 0:
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
197 binallarchs = allowedarchs - sourcearch
408
02d498ee90b8 copy binary-all packages in all binary-xyz dirs
Bjoern Ricks <bricks@intevation.de>
parents: 406
diff changeset
198 else:
412
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
199 binallarchs = (allowedarchs & allarchs) - sourcearch
416
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
200 for packageinfo in binaryallpackages:
408
02d498ee90b8 copy binary-all packages in all binary-xyz dirs
Bjoern Ricks <bricks@intevation.de>
parents: 406
diff changeset
201 for arch in binallarchs:
413
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
202 filename = os.path.join(cachedir, arch, track.name,
416
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
203 packageinfo.name)
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
204 newpackage = Package(filename, packageinfo.trackname, packageinfo.name,
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
205 packageinfo.path, arch, get_md5sum(packageinfo))
413
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
206 newpackages.append(newpackage)
416
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
207 copy_packages_to_destdir(cachedb, cachedir, newpackages, quiet)
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
208 remove_old_packages(cachedb, newpackages)
406
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
209
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
210 def publish_packages(config_filename, track, revision, dist, section, quiet):
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
211 config = read_config(config_filename)
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
212
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
213 if dist is None:
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
214 dist = config["distribution"]
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
215 if section is None:
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
216 section = config["section"]
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
217
416
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
218 if not "cachedb" in config.keys():
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
219 print >>sys.stderr, "Error. Please specifiy a cachedb in your config file"
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
220 sys.exit(1)
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
221
406
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
222 architectures = config["architectures"]
416
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
223 copy_to_cachedir(config, track, revision, quiet, architectures)
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
224 # copy_to_publishdir(config, dist, section, arch, quiet)
406
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
225
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
226 # update apt archive
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
227 # call(config["after_upload_hook"])
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
228
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
229 def main():
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
230 options, args = parse_commandline()
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
231 revision = None # for future use cases
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
232 publish_packages(options.config_file, options.track, revision,
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
233 options.dist, options.section, options.quiet)
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
234
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
235 if __name__ == "__main__":
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
236 main()
This site is hosted by Intevation GmbH (Datenschutzerklärung und Impressum | Privacy Policy and Imprint)