Mercurial > treepkg

annotate bin/publishdebianpackages.py @ 578:bccd69dedbe5

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
shlex.split all config parameters that may contain a command Return the command as list and use @ operator for the variable in cmdexpand
author Bjoern Ricks <bricks@intevation.de>
date Sat, 03 Sep 2011 11:53:12 +0000
parents b7aad4cb58bb
children
rev   line source
406
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
1 #! /usr/bin/python
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
2 # Copyright (C) 2007 - 2010 by Intevation GmbH
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
3 # Authors:
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
4 # Bernhard Herzog <bh@intevation.de>
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
5 # Bjoern Ricks <bjoern.ricks@intevation.de>
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
6 #
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
7 # This program is free software under the GPL (>=v2)
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
8 # Read the file COPYING coming with the software for details.
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
9
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
10 """Publishes selected packages created by treepkg"""
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
11
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
12 import os
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
13 import sys
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
14 import shlex
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
15
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
16 from optparse import OptionParser
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
17 from ConfigParser import SafeConfigParser
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
18
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
19 import treepkgcmd
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
20 from treepkg.readconfig import read_config_section, convert_bool
451
eacfd3744d16 capture only stdout for treepkginfo
Bjoern Ricks <bricks@intevation.de>
parents: 448
diff changeset
21 from treepkg.run import call, capture_stdout
406
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
22 from treepkg.cmdexpand import cmdexpand
448
12f3642e1756 added missing import
Bjoern Ricks <bricks@intevation.de>
parents: 439
diff changeset
23 from treepkg.publish import copy_arch_to_publishdir, prefix_for_remote_command,\
12f3642e1756 added missing import
Bjoern Ricks <bricks@intevation.de>
parents: 439
diff changeset
24 get_binary_arch
438
3a3cad8f6f60 review changes
Bjoern Ricks <bricks@intevation.de>
parents: 436
diff changeset
25
439
8e0c81870e5e cleanup modules
Bjoern Ricks <bricks@intevation.de>
parents: 438
diff changeset
26 from treepkg.util import md5sum, expand_filename, remove_trailing_slashes
406
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
27 from treepkg.info.status import TreepkgInfo
414
e0539b483b04 moved data handling in publishdebianpackages into seperate module
Bjoern Ricks <bricks@intevation.de>
parents: 413
diff changeset
28 from treepkg.info.data import Package
e0539b483b04 moved data handling in publishdebianpackages into seperate module
Bjoern Ricks <bricks@intevation.de>
parents: 413
diff changeset
29 from treepkg.info.data import CacheDb
406
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
30
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
31 config_desc = ["distribution", "section", "num_newest",
578
bccd69dedbe5 shlex.split all config parameters that may contain a command
Bjoern Ricks <bricks@intevation.de>
parents: 505
diff changeset
32 "build_user", "build_host",
bccd69dedbe5 shlex.split all config parameters that may contain a command
Bjoern Ricks <bricks@intevation.de>
parents: 505
diff changeset
33 ("build_listpackages", shlex.split),
417
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
34 "publish_user", "publish_host",
412
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
35 ("architectures", shlex.split, "armel i386 source"),
438
3a3cad8f6f60 review changes
Bjoern Ricks <bricks@intevation.de>
parents: 436
diff changeset
36 ("after_upload_hook", shlex.split, ""),
3a3cad8f6f60 review changes
Bjoern Ricks <bricks@intevation.de>
parents: 436
diff changeset
37 ("after_copy_hook", shlex.split, ""),
431
0953704dc49e publish package must be default to get original behaviour
Bjoern Ricks <bricks@intevation.de>
parents: 430
diff changeset
38 ("publish_packages", convert_bool, "True"),
406
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
39 ("publish_remove_old_packages", convert_bool),
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
40 ("publish_dir", remove_trailing_slashes),
438
3a3cad8f6f60 review changes
Bjoern Ricks <bricks@intevation.de>
parents: 436
diff changeset
41 ("cachedb", lambda s: expand_filename(s)),
406
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
42 ("cachedir",
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
43 lambda s: expand_filename(remove_trailing_slashes(s)))]
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
44
413
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
45
406
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
46 def read_config(filename):
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
47 if not os.path.exists(filename):
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
48 print >>sys.stderr, "Config file %s does not exist" % filename
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
49 sys.exit(1)
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
50 parser = SafeConfigParser()
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
51 parser.read([filename])
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
52 return read_config_section(parser, "publishpackages", config_desc)
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
53
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
54 def parse_commandline():
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
55 parser = OptionParser()
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
56 parser.set_defaults(config_file=os.path.join(treepkgcmd.topdir,
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
57 "publishpackages.cfg"),
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
58 quiet=False)
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
59 parser.add_option("--config-file",
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
60 help=("The configuration file."
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
61 " Default is publishpackages.cfg"))
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
62 parser.add_option("--dist",
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
63 help=("The debian distribution name to use on"
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
64 " the publishing system"))
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
65 parser.add_option("--section",
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
66 help=("The debian distribution section name to use on"
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
67 " the publishing system"))
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
68 parser.add_option("--track",
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
69 help=("The package track whose files are to be"
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
70 " published. If not given, files of all tracks"
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
71 " will be published"))
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
72 parser.add_option("--quiet", action="store_true",
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
73 help=("Do not print progress meters or other"
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
74 " informational output"))
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
75 return parser.parse_args()
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
76
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
77 def get_treepkg_info(variables):
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
78 runremote = prefix_for_remote_command(variables["build_user"],
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
79 variables["build_host"])
578
bccd69dedbe5 shlex.split all config parameters that may contain a command
Bjoern Ricks <bricks@intevation.de>
parents: 505
diff changeset
80 xml = capture_stdout(cmdexpand("@runremote @build_listpackages"
438
3a3cad8f6f60 review changes
Bjoern Ricks <bricks@intevation.de>
parents: 436
diff changeset
81 " --newest=$num_newest"
3a3cad8f6f60 review changes
Bjoern Ricks <bricks@intevation.de>
parents: 436
diff changeset
82 " --only-successful",
3a3cad8f6f60 review changes
Bjoern Ricks <bricks@intevation.de>
parents: 436
diff changeset
83 runremote=runremote,
3a3cad8f6f60 review changes
Bjoern Ricks <bricks@intevation.de>
parents: 436
diff changeset
84 **variables))
408
02d498ee90b8 copy binary-all packages in all binary-xyz dirs
Bjoern Ricks <bricks@intevation.de>
parents: 406
diff changeset
85 return TreepkgInfo.fromxml(xml)
406
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
86
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
87 def check_package_is_new(packagename, destdir, packagemd5sum):
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
88 destpackage = os.path.join(destdir, packagename)
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
89 if not os.path.isfile(destpackage):
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
90 return True
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
91 destmd5sum = md5sum(destpackage)
408
02d498ee90b8 copy binary-all packages in all binary-xyz dirs
Bjoern Ricks <bricks@intevation.de>
parents: 406
diff changeset
92 return (destmd5sum != packagemd5sum)
02d498ee90b8 copy binary-all packages in all binary-xyz dirs
Bjoern Ricks <bricks@intevation.de>
parents: 406
diff changeset
93
413
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
94 def get_md5sum(packageinfo):
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
95 md5sum = ""
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
96 if packageinfo:
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
97 for checksum in packageinfo.checksums:
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
98 if checksum.type == "md5":
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
99 md5sum = checksum.checksum
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
100 break
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
101 return md5sum
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
102
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
103 def sort_trackname_arch(a, b):
438
3a3cad8f6f60 review changes
Bjoern Ricks <bricks@intevation.de>
parents: 436
diff changeset
104 if a.trackname < b.trackname:
3a3cad8f6f60 review changes
Bjoern Ricks <bricks@intevation.de>
parents: 436
diff changeset
105 return -1
3a3cad8f6f60 review changes
Bjoern Ricks <bricks@intevation.de>
parents: 436
diff changeset
106 if a.trackname > b.trackname:
3a3cad8f6f60 review changes
Bjoern Ricks <bricks@intevation.de>
parents: 436
diff changeset
107 return +1
413
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
108 return cmp(a.arch, b.arch)
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
109
438
3a3cad8f6f60 review changes
Bjoern Ricks <bricks@intevation.de>
parents: 436
diff changeset
110 def copy_files_to_destdir(destdir, files, variables, quiet=False):
408
02d498ee90b8 copy binary-all packages in all binary-xyz dirs
Bjoern Ricks <bricks@intevation.de>
parents: 406
diff changeset
111 scp_flags = []
412
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
112 if quiet:
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
113 scp_flags.append("-q")
413
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
114
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
115 if not os.path.exists(destdir):
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
116 os.makedirs(destdir)
417
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
117 if files:
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
118 if variables["build_host"]:
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
119 userhost = "%(build_user)s@%(build_host)s:" % variables
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
120 files = [userhost + filename for filename in files]
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
121 # scp the packages to the cache dir
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
122 call(cmdexpand("scp -p @scp_flags @files $cachedir/", files=files,
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
123 scp_flags=scp_flags, cachedir=destdir))
413
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
124
417
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
125 def remove_old_packages(cachedb, newpackages, quiet):
505
b7aad4cb58bb don't remove packages if a track is deactivated
Bjoern Ricks <bricks@intevation.de>
parents: 451
diff changeset
126 oldpackages = cachedb.get_old_packages(newpackages)
416
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
127 for package in oldpackages:
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
128 # better check if the file really exists
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
129 if os.path.isfile(package.filename):
417
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
130 if not quiet:
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
131 print "removing file %s" % package.filename
416
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
132 os.remove(package.filename)
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
133 cachedb.remove_packages(oldpackages)
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
134
438
3a3cad8f6f60 review changes
Bjoern Ricks <bricks@intevation.de>
parents: 436
diff changeset
135 def copy_packages_to_destdir(cachedb, dir, packages, variables, quiet=False):
413
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
136 packages.sort(cmp=sort_trackname_arch)
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
137 package = packages[0]
417
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
138 trackname = package.trackname
413
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
139 arch = package.arch
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
140 destdir = os.path.join(dir, arch, trackname)
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
141 files = []
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
142 for package in packages:
416
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
143 cachedb.add_package(package)
438
3a3cad8f6f60 review changes
Bjoern Ricks <bricks@intevation.de>
parents: 436
diff changeset
144 if package.trackname != trackname or package.arch != arch:
417
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
145 copy_files_to_destdir(destdir, files, variables, quiet)
413
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
146 trackname = package.trackname
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
147 arch = package.arch
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
148 destdir = os.path.join(dir, arch, trackname)
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
149 files = []
416
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
150 # add only to copy files list if the packages differ
413
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
151 if check_package_is_new(package.name, destdir, package.md5sum):
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
152 files.append(package.sourcepath)
417
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
153 if not quiet:
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
154 print "copy new file: %s" % package.name
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
155 copy_files_to_destdir(destdir, files, variables, quiet)
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
156
438
3a3cad8f6f60 review changes
Bjoern Ricks <bricks@intevation.de>
parents: 436
diff changeset
157 def copy_to_cachedir(variables, track, revision, quiet=False,
3a3cad8f6f60 review changes
Bjoern Ricks <bricks@intevation.de>
parents: 436
diff changeset
158 architectures=()):
408
02d498ee90b8 copy binary-all packages in all binary-xyz dirs
Bjoern Ricks <bricks@intevation.de>
parents: 406
diff changeset
159 cachedir = variables["cachedir"]
416
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
160 cachdebfilename = variables["cachedb"]
417
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
161 if not quiet:
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
162 print "using cachedb: %s" % cachdebfilename
416
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
163 cachedb = CacheDb(cachdebfilename)
413
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
164 newpackages = []
406
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
165 treepkginfo = get_treepkg_info(variables)
438
3a3cad8f6f60 review changes
Bjoern Ricks <bricks@intevation.de>
parents: 436
diff changeset
166 allarchs = set() # contains all present architectures (incl. source)
408
02d498ee90b8 copy binary-all packages in all binary-xyz dirs
Bjoern Ricks <bricks@intevation.de>
parents: 406
diff changeset
167 binaryallpackages = []
406
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
168 # change e.g. armel in binary-armel
438
3a3cad8f6f60 review changes
Bjoern Ricks <bricks@intevation.de>
parents: 436
diff changeset
169 allowedarchs = set([get_binary_arch(a) for a in architectures])
408
02d498ee90b8 copy binary-all packages in all binary-xyz dirs
Bjoern Ricks <bricks@intevation.de>
parents: 406
diff changeset
170 for track in treepkginfo.tracks:
406
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
171 for rev in track.revisions:
416
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
172 for packageinfo in rev.packages:
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
173 arch = get_binary_arch(packageinfo.arch)
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
174 if packageinfo.type == "binary":
412
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
175 # skip other files
416
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
176 if packageinfo.arch is None:
412
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
177 continue
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
178 # handle binary-all
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
179 if arch == "binary-all":
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
180 # add trackname for subdir name
416
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
181 packageinfo.trackname = track.name
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
182 binaryallpackages.append(packageinfo)
412
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
183 continue
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
184 allarchs.add(arch)
416
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
185 elif packageinfo.type == "source":
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
186 arch = packageinfo.type
408
02d498ee90b8 copy binary-all packages in all binary-xyz dirs
Bjoern Ricks <bricks@intevation.de>
parents: 406
diff changeset
187 # only copy requested archs
02d498ee90b8 copy binary-all packages in all binary-xyz dirs
Bjoern Ricks <bricks@intevation.de>
parents: 406
diff changeset
188 if len(allowedarchs) == 0 or \
412
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
189 arch in allowedarchs:
413
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
190 filename = os.path.join(cachedir, arch, track.name,
416
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
191 packageinfo.name)
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
192 newpackage = Package(filename, track.name, packageinfo.name,
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
193 packageinfo.path, arch,
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
194 get_md5sum(packageinfo))
413
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
195 newpackages.append(newpackage)
408
02d498ee90b8 copy binary-all packages in all binary-xyz dirs
Bjoern Ricks <bricks@intevation.de>
parents: 406
diff changeset
196 # copy binary-all packages
412
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
197 sourcearch = set(["source"])
438
3a3cad8f6f60 review changes
Bjoern Ricks <bricks@intevation.de>
parents: 436
diff changeset
198 if not allowedarchs:
412
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
199 binallarchs = allarchs - sourcearch
438
3a3cad8f6f60 review changes
Bjoern Ricks <bricks@intevation.de>
parents: 436
diff changeset
200 elif allarchs:
412
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
201 binallarchs = allowedarchs - sourcearch
408
02d498ee90b8 copy binary-all packages in all binary-xyz dirs
Bjoern Ricks <bricks@intevation.de>
parents: 406
diff changeset
202 else:
412
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
203 binallarchs = (allowedarchs & allarchs) - sourcearch
416
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
204 for packageinfo in binaryallpackages:
408
02d498ee90b8 copy binary-all packages in all binary-xyz dirs
Bjoern Ricks <bricks@intevation.de>
parents: 406
diff changeset
205 for arch in binallarchs:
417
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
206 filename = os.path.join(cachedir, arch, packageinfo.trackname,
416
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
207 packageinfo.name)
438
3a3cad8f6f60 review changes
Bjoern Ricks <bricks@intevation.de>
parents: 436
diff changeset
208 newpackage = Package(filename, packageinfo.trackname,
3a3cad8f6f60 review changes
Bjoern Ricks <bricks@intevation.de>
parents: 436
diff changeset
209 packageinfo.name, packageinfo.path, arch,
3a3cad8f6f60 review changes
Bjoern Ricks <bricks@intevation.de>
parents: 436
diff changeset
210 get_md5sum(packageinfo))
413
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
211 newpackages.append(newpackage)
417
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
212 copy_packages_to_destdir(cachedb, cachedir, newpackages, variables, quiet)
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
213 remove_old_packages(cachedb, newpackages, quiet)
424
a2ba58ffbfbe run rsync only an arch dirs that are present in the current treepkg
Bjoern Ricks <bricks@intevation.de>
parents: 423
diff changeset
214 return binallarchs
406
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
215
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
216 def publish_packages(config_filename, track, revision, dist, section, quiet):
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
217 config = read_config(config_filename)
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
218
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
219 if dist is None:
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
220 dist = config["distribution"]
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
221 if section is None:
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
222 section = config["section"]
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
223
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
224 architectures = config["architectures"]
429
9cfa9f64387a only rsync specified architectures from cachedir to publishdir
Bjoern Ricks <bricks@intevation.de>
parents: 428
diff changeset
225 copy_to_cachedir(config, track, revision, quiet, architectures)
430
4aaf73a6225e introduced after_copy_hook and made rsync upload to publishdir optional
Bjoern Ricks <bricks@intevation.de>
parents: 429
diff changeset
226
438
3a3cad8f6f60 review changes
Bjoern Ricks <bricks@intevation.de>
parents: 436
diff changeset
227 if config["after_copy_hook"]:
3a3cad8f6f60 review changes
Bjoern Ricks <bricks@intevation.de>
parents: 436
diff changeset
228 if not quiet:
3a3cad8f6f60 review changes
Bjoern Ricks <bricks@intevation.de>
parents: 436
diff changeset
229 print "running after copy hook"
3a3cad8f6f60 review changes
Bjoern Ricks <bricks@intevation.de>
parents: 436
diff changeset
230 call(config["after_copy_hook"])
430
4aaf73a6225e introduced after_copy_hook and made rsync upload to publishdir optional
Bjoern Ricks <bricks@intevation.de>
parents: 429
diff changeset
231
4aaf73a6225e introduced after_copy_hook and made rsync upload to publishdir optional
Bjoern Ricks <bricks@intevation.de>
parents: 429
diff changeset
232 if config["publish_packages"]:
4aaf73a6225e introduced after_copy_hook and made rsync upload to publishdir optional
Bjoern Ricks <bricks@intevation.de>
parents: 429
diff changeset
233 for arch in architectures:
431
0953704dc49e publish package must be default to get original behaviour
Bjoern Ricks <bricks@intevation.de>
parents: 430
diff changeset
234 if not quiet:
436
eab777e1bafd fixed typos
Bjoern Ricks <bricks@intevation.de>
parents: 431
diff changeset
235 print "publish packages for architecture %s" % arch
438
3a3cad8f6f60 review changes
Bjoern Ricks <bricks@intevation.de>
parents: 436
diff changeset
236 copy_arch_to_publishdir(config, dist, section,
3a3cad8f6f60 review changes
Bjoern Ricks <bricks@intevation.de>
parents: 436
diff changeset
237 get_binary_arch(arch), quiet)
406
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
238
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
239 # update apt archive
438
3a3cad8f6f60 review changes
Bjoern Ricks <bricks@intevation.de>
parents: 436
diff changeset
240 if config["after_upload_hook"]:
417
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
241 if not quiet:
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
242 print "running after upload hook"
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
243 call(config["after_upload_hook"])
406
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
244
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
245 def main():
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
246 options, args = parse_commandline()
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
247 revision = None # for future use cases
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
248 publish_packages(options.config_file, options.track, revision,
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
249 options.dist, options.section, options.quiet)
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
250
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
251 if __name__ == "__main__":
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
252 main()
This site is hosted by Intevation GmbH (Datenschutzerklärung und Impressum | Privacy Policy and Imprint)