Mercurial > treepkg

annotate bin/publishdebianpackages.py @ 557:9824e409388b

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Refactor git branching If a checkout is already available and the branch is changed in the config git command would always fail because it doesn't know the branch to track. Therefore always check if the branch is locally available and if not checkout the remote branch
author Bjoern Ricks <bricks@intevation.de>
date Fri, 02 Sep 2011 08:45:28 +0000
parents b7aad4cb58bb
children bccd69dedbe5
rev   line source
406
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
1 #! /usr/bin/python
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
2 # Copyright (C) 2007 - 2010 by Intevation GmbH
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
3 # Authors:
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
4 # Bernhard Herzog <bh@intevation.de>
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
5 # Bjoern Ricks <bjoern.ricks@intevation.de>
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
6 #
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
7 # This program is free software under the GPL (>=v2)
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
8 # Read the file COPYING coming with the software for details.
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
9
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
10 """Publishes selected packages created by treepkg"""
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
11
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
12 import os
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
13 import sys
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
14 import shlex
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
15
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
16 from optparse import OptionParser
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
17 from ConfigParser import SafeConfigParser
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
18
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
19 import treepkgcmd
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
20 from treepkg.readconfig import read_config_section, convert_bool
451
eacfd3744d16 capture only stdout for treepkginfo
Bjoern Ricks <bricks@intevation.de>
parents: 448
diff changeset
21 from treepkg.run import call, capture_stdout
406
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
22 from treepkg.cmdexpand import cmdexpand
448
12f3642e1756 added missing import
Bjoern Ricks <bricks@intevation.de>
parents: 439
diff changeset
23 from treepkg.publish import copy_arch_to_publishdir, prefix_for_remote_command,\
12f3642e1756 added missing import
Bjoern Ricks <bricks@intevation.de>
parents: 439
diff changeset
24 get_binary_arch
438
3a3cad8f6f60 review changes
Bjoern Ricks <bricks@intevation.de>
parents: 436
diff changeset
25
439
8e0c81870e5e cleanup modules
Bjoern Ricks <bricks@intevation.de>
parents: 438
diff changeset
26 from treepkg.util import md5sum, expand_filename, remove_trailing_slashes
406
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
27 from treepkg.info.status import TreepkgInfo
414
e0539b483b04 moved data handling in publishdebianpackages into seperate module
Bjoern Ricks <bricks@intevation.de>
parents: 413
diff changeset
28 from treepkg.info.data import Package
e0539b483b04 moved data handling in publishdebianpackages into seperate module
Bjoern Ricks <bricks@intevation.de>
parents: 413
diff changeset
29 from treepkg.info.data import CacheDb
406
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
30
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
31 config_desc = ["distribution", "section", "num_newest",
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
32 "build_user", "build_host", "build_listpackages",
417
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
33 "publish_user", "publish_host",
412
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
34 ("architectures", shlex.split, "armel i386 source"),
438
3a3cad8f6f60 review changes
Bjoern Ricks <bricks@intevation.de>
parents: 436
diff changeset
35 ("after_upload_hook", shlex.split, ""),
3a3cad8f6f60 review changes
Bjoern Ricks <bricks@intevation.de>
parents: 436
diff changeset
36 ("after_copy_hook", shlex.split, ""),
431
0953704dc49e publish package must be default to get original behaviour
Bjoern Ricks <bricks@intevation.de>
parents: 430
diff changeset
37 ("publish_packages", convert_bool, "True"),
406
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
38 ("publish_remove_old_packages", convert_bool),
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
39 ("publish_dir", remove_trailing_slashes),
438
3a3cad8f6f60 review changes
Bjoern Ricks <bricks@intevation.de>
parents: 436
diff changeset
40 ("cachedb", lambda s: expand_filename(s)),
406
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
41 ("cachedir",
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
42 lambda s: expand_filename(remove_trailing_slashes(s)))]
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
43
413
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
44
406
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
45 def read_config(filename):
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
46 if not os.path.exists(filename):
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
47 print >>sys.stderr, "Config file %s does not exist" % filename
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
48 sys.exit(1)
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
49 parser = SafeConfigParser()
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
50 parser.read([filename])
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
51 return read_config_section(parser, "publishpackages", config_desc)
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
52
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
53 def parse_commandline():
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
54 parser = OptionParser()
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
55 parser.set_defaults(config_file=os.path.join(treepkgcmd.topdir,
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
56 "publishpackages.cfg"),
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
57 quiet=False)
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
58 parser.add_option("--config-file",
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
59 help=("The configuration file."
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
60 " Default is publishpackages.cfg"))
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
61 parser.add_option("--dist",
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
62 help=("The debian distribution name to use on"
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
63 " the publishing system"))
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
64 parser.add_option("--section",
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
65 help=("The debian distribution section name to use on"
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
66 " the publishing system"))
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
67 parser.add_option("--track",
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
68 help=("The package track whose files are to be"
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
69 " published. If not given, files of all tracks"
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
70 " will be published"))
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
71 parser.add_option("--quiet", action="store_true",
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
72 help=("Do not print progress meters or other"
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
73 " informational output"))
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
74 return parser.parse_args()
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
75
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
76 def get_treepkg_info(variables):
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
77 runremote = prefix_for_remote_command(variables["build_user"],
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
78 variables["build_host"])
451
eacfd3744d16 capture only stdout for treepkginfo
Bjoern Ricks <bricks@intevation.de>
parents: 448
diff changeset
79 xml = capture_stdout(cmdexpand("@runremote $build_listpackages"
438
3a3cad8f6f60 review changes
Bjoern Ricks <bricks@intevation.de>
parents: 436
diff changeset
80 " --newest=$num_newest"
3a3cad8f6f60 review changes
Bjoern Ricks <bricks@intevation.de>
parents: 436
diff changeset
81 " --only-successful",
3a3cad8f6f60 review changes
Bjoern Ricks <bricks@intevation.de>
parents: 436
diff changeset
82 runremote=runremote,
3a3cad8f6f60 review changes
Bjoern Ricks <bricks@intevation.de>
parents: 436
diff changeset
83 **variables))
408
02d498ee90b8 copy binary-all packages in all binary-xyz dirs
Bjoern Ricks <bricks@intevation.de>
parents: 406
diff changeset
84 return TreepkgInfo.fromxml(xml)
406
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
85
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
86 def check_package_is_new(packagename, destdir, packagemd5sum):
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
87 destpackage = os.path.join(destdir, packagename)
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
88 if not os.path.isfile(destpackage):
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
89 return True
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
90 destmd5sum = md5sum(destpackage)
408
02d498ee90b8 copy binary-all packages in all binary-xyz dirs
Bjoern Ricks <bricks@intevation.de>
parents: 406
diff changeset
91 return (destmd5sum != packagemd5sum)
02d498ee90b8 copy binary-all packages in all binary-xyz dirs
Bjoern Ricks <bricks@intevation.de>
parents: 406
diff changeset
92
413
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
93 def get_md5sum(packageinfo):
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
94 md5sum = ""
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
95 if packageinfo:
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
96 for checksum in packageinfo.checksums:
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
97 if checksum.type == "md5":
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
98 md5sum = checksum.checksum
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
99 break
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
100 return md5sum
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
101
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
102 def sort_trackname_arch(a, b):
438
3a3cad8f6f60 review changes
Bjoern Ricks <bricks@intevation.de>
parents: 436
diff changeset
103 if a.trackname < b.trackname:
3a3cad8f6f60 review changes
Bjoern Ricks <bricks@intevation.de>
parents: 436
diff changeset
104 return -1
3a3cad8f6f60 review changes
Bjoern Ricks <bricks@intevation.de>
parents: 436
diff changeset
105 if a.trackname > b.trackname:
3a3cad8f6f60 review changes
Bjoern Ricks <bricks@intevation.de>
parents: 436
diff changeset
106 return +1
413
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
107 return cmp(a.arch, b.arch)
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
108
438
3a3cad8f6f60 review changes
Bjoern Ricks <bricks@intevation.de>
parents: 436
diff changeset
109 def copy_files_to_destdir(destdir, files, variables, quiet=False):
408
02d498ee90b8 copy binary-all packages in all binary-xyz dirs
Bjoern Ricks <bricks@intevation.de>
parents: 406
diff changeset
110 scp_flags = []
412
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
111 if quiet:
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
112 scp_flags.append("-q")
413
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
113
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
114 if not os.path.exists(destdir):
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
115 os.makedirs(destdir)
417
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
116 if files:
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
117 if variables["build_host"]:
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
118 userhost = "%(build_user)s@%(build_host)s:" % variables
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
119 files = [userhost + filename for filename in files]
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
120 # scp the packages to the cache dir
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
121 call(cmdexpand("scp -p @scp_flags @files $cachedir/", files=files,
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
122 scp_flags=scp_flags, cachedir=destdir))
413
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
123
417
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
124 def remove_old_packages(cachedb, newpackages, quiet):
505
b7aad4cb58bb don't remove packages if a track is deactivated
Bjoern Ricks <bricks@intevation.de>
parents: 451
diff changeset
125 oldpackages = cachedb.get_old_packages(newpackages)
416
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
126 for package in oldpackages:
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
127 # better check if the file really exists
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
128 if os.path.isfile(package.filename):
417
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
129 if not quiet:
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
130 print "removing file %s" % package.filename
416
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
131 os.remove(package.filename)
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
132 cachedb.remove_packages(oldpackages)
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
133
438
3a3cad8f6f60 review changes
Bjoern Ricks <bricks@intevation.de>
parents: 436
diff changeset
134 def copy_packages_to_destdir(cachedb, dir, packages, variables, quiet=False):
413
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
135 packages.sort(cmp=sort_trackname_arch)
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
136 package = packages[0]
417
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
137 trackname = package.trackname
413
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
138 arch = package.arch
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
139 destdir = os.path.join(dir, arch, trackname)
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
140 files = []
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
141 for package in packages:
416
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
142 cachedb.add_package(package)
438
3a3cad8f6f60 review changes
Bjoern Ricks <bricks@intevation.de>
parents: 436
diff changeset
143 if package.trackname != trackname or package.arch != arch:
417
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
144 copy_files_to_destdir(destdir, files, variables, quiet)
413
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
145 trackname = package.trackname
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
146 arch = package.arch
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
147 destdir = os.path.join(dir, arch, trackname)
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
148 files = []
416
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
149 # add only to copy files list if the packages differ
413
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
150 if check_package_is_new(package.name, destdir, package.md5sum):
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
151 files.append(package.sourcepath)
417
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
152 if not quiet:
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
153 print "copy new file: %s" % package.name
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
154 copy_files_to_destdir(destdir, files, variables, quiet)
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
155
438
3a3cad8f6f60 review changes
Bjoern Ricks <bricks@intevation.de>
parents: 436
diff changeset
156 def copy_to_cachedir(variables, track, revision, quiet=False,
3a3cad8f6f60 review changes
Bjoern Ricks <bricks@intevation.de>
parents: 436
diff changeset
157 architectures=()):
408
02d498ee90b8 copy binary-all packages in all binary-xyz dirs
Bjoern Ricks <bricks@intevation.de>
parents: 406
diff changeset
158 cachedir = variables["cachedir"]
416
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
159 cachdebfilename = variables["cachedb"]
417
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
160 if not quiet:
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
161 print "using cachedb: %s" % cachdebfilename
416
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
162 cachedb = CacheDb(cachdebfilename)
413
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
163 newpackages = []
406
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
164 treepkginfo = get_treepkg_info(variables)
438
3a3cad8f6f60 review changes
Bjoern Ricks <bricks@intevation.de>
parents: 436
diff changeset
165 allarchs = set() # contains all present architectures (incl. source)
408
02d498ee90b8 copy binary-all packages in all binary-xyz dirs
Bjoern Ricks <bricks@intevation.de>
parents: 406
diff changeset
166 binaryallpackages = []
406
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
167 # change e.g. armel in binary-armel
438
3a3cad8f6f60 review changes
Bjoern Ricks <bricks@intevation.de>
parents: 436
diff changeset
168 allowedarchs = set([get_binary_arch(a) for a in architectures])
408
02d498ee90b8 copy binary-all packages in all binary-xyz dirs
Bjoern Ricks <bricks@intevation.de>
parents: 406
diff changeset
169 for track in treepkginfo.tracks:
406
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
170 for rev in track.revisions:
416
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
171 for packageinfo in rev.packages:
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
172 arch = get_binary_arch(packageinfo.arch)
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
173 if packageinfo.type == "binary":
412
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
174 # skip other files
416
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
175 if packageinfo.arch is None:
412
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
176 continue
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
177 # handle binary-all
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
178 if arch == "binary-all":
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
179 # add trackname for subdir name
416
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
180 packageinfo.trackname = track.name
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
181 binaryallpackages.append(packageinfo)
412
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
182 continue
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
183 allarchs.add(arch)
416
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
184 elif packageinfo.type == "source":
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
185 arch = packageinfo.type
408
02d498ee90b8 copy binary-all packages in all binary-xyz dirs
Bjoern Ricks <bricks@intevation.de>
parents: 406
diff changeset
186 # only copy requested archs
02d498ee90b8 copy binary-all packages in all binary-xyz dirs
Bjoern Ricks <bricks@intevation.de>
parents: 406
diff changeset
187 if len(allowedarchs) == 0 or \
412
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
188 arch in allowedarchs:
413
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
189 filename = os.path.join(cachedir, arch, track.name,
416
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
190 packageinfo.name)
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
191 newpackage = Package(filename, track.name, packageinfo.name,
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
192 packageinfo.path, arch,
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
193 get_md5sum(packageinfo))
413
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
194 newpackages.append(newpackage)
408
02d498ee90b8 copy binary-all packages in all binary-xyz dirs
Bjoern Ricks <bricks@intevation.de>
parents: 406
diff changeset
195 # copy binary-all packages
412
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
196 sourcearch = set(["source"])
438
3a3cad8f6f60 review changes
Bjoern Ricks <bricks@intevation.de>
parents: 436
diff changeset
197 if not allowedarchs:
412
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
198 binallarchs = allarchs - sourcearch
438
3a3cad8f6f60 review changes
Bjoern Ricks <bricks@intevation.de>
parents: 436
diff changeset
199 elif allarchs:
412
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
200 binallarchs = allowedarchs - sourcearch
408
02d498ee90b8 copy binary-all packages in all binary-xyz dirs
Bjoern Ricks <bricks@intevation.de>
parents: 406
diff changeset
201 else:
412
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
202 binallarchs = (allowedarchs & allarchs) - sourcearch
416
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
203 for packageinfo in binaryallpackages:
408
02d498ee90b8 copy binary-all packages in all binary-xyz dirs
Bjoern Ricks <bricks@intevation.de>
parents: 406
diff changeset
204 for arch in binallarchs:
417
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
205 filename = os.path.join(cachedir, arch, packageinfo.trackname,
416
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
206 packageinfo.name)
438
3a3cad8f6f60 review changes
Bjoern Ricks <bricks@intevation.de>
parents: 436
diff changeset
207 newpackage = Package(filename, packageinfo.trackname,
3a3cad8f6f60 review changes
Bjoern Ricks <bricks@intevation.de>
parents: 436
diff changeset
208 packageinfo.name, packageinfo.path, arch,
3a3cad8f6f60 review changes
Bjoern Ricks <bricks@intevation.de>
parents: 436
diff changeset
209 get_md5sum(packageinfo))
413
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
210 newpackages.append(newpackage)
417
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
211 copy_packages_to_destdir(cachedb, cachedir, newpackages, variables, quiet)
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
212 remove_old_packages(cachedb, newpackages, quiet)
424
a2ba58ffbfbe run rsync only an arch dirs that are present in the current treepkg
Bjoern Ricks <bricks@intevation.de>
parents: 423
diff changeset
213 return binallarchs
406
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
214
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
215 def publish_packages(config_filename, track, revision, dist, section, quiet):
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
216 config = read_config(config_filename)
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
217
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
218 if dist is None:
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
219 dist = config["distribution"]
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
220 if section is None:
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
221 section = config["section"]
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
222
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
223 architectures = config["architectures"]
429
9cfa9f64387a only rsync specified architectures from cachedir to publishdir
Bjoern Ricks <bricks@intevation.de>
parents: 428
diff changeset
224 copy_to_cachedir(config, track, revision, quiet, architectures)
430
4aaf73a6225e introduced after_copy_hook and made rsync upload to publishdir optional
Bjoern Ricks <bricks@intevation.de>
parents: 429
diff changeset
225
438
3a3cad8f6f60 review changes
Bjoern Ricks <bricks@intevation.de>
parents: 436
diff changeset
226 if config["after_copy_hook"]:
3a3cad8f6f60 review changes
Bjoern Ricks <bricks@intevation.de>
parents: 436
diff changeset
227 if not quiet:
3a3cad8f6f60 review changes
Bjoern Ricks <bricks@intevation.de>
parents: 436
diff changeset
228 print "running after copy hook"
3a3cad8f6f60 review changes
Bjoern Ricks <bricks@intevation.de>
parents: 436
diff changeset
229 call(config["after_copy_hook"])
430
4aaf73a6225e introduced after_copy_hook and made rsync upload to publishdir optional
Bjoern Ricks <bricks@intevation.de>
parents: 429
diff changeset
230
4aaf73a6225e introduced after_copy_hook and made rsync upload to publishdir optional
Bjoern Ricks <bricks@intevation.de>
parents: 429
diff changeset
231 if config["publish_packages"]:
4aaf73a6225e introduced after_copy_hook and made rsync upload to publishdir optional
Bjoern Ricks <bricks@intevation.de>
parents: 429
diff changeset
232 for arch in architectures:
431
0953704dc49e publish package must be default to get original behaviour
Bjoern Ricks <bricks@intevation.de>
parents: 430
diff changeset
233 if not quiet:
436
eab777e1bafd fixed typos
Bjoern Ricks <bricks@intevation.de>
parents: 431
diff changeset
234 print "publish packages for architecture %s" % arch
438
3a3cad8f6f60 review changes
Bjoern Ricks <bricks@intevation.de>
parents: 436
diff changeset
235 copy_arch_to_publishdir(config, dist, section,
3a3cad8f6f60 review changes
Bjoern Ricks <bricks@intevation.de>
parents: 436
diff changeset
236 get_binary_arch(arch), quiet)
406
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
237
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
238 # update apt archive
438
3a3cad8f6f60 review changes
Bjoern Ricks <bricks@intevation.de>
parents: 436
diff changeset
239 if config["after_upload_hook"]:
417
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
240 if not quiet:
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
241 print "running after upload hook"
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
242 call(config["after_upload_hook"])
406
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
243
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
244 def main():
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
245 options, args = parse_commandline()
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
246 revision = None # for future use cases
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
247 publish_packages(options.config_file, options.track, revision,
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
248 options.dist, options.section, options.quiet)
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
249
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
250 if __name__ == "__main__":
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
251 main()
This site is hosted by Intevation GmbH (Datenschutzerklärung und Impressum | Privacy Policy and Imprint)