Mercurial > treepkg > treepkg

annotate bin/publishdebianpackages.py @ 431:0953704dc49e treepkg-status

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
publish package must be default to get original behaviour be more verbose at publishing
author Bjoern Ricks <bricks@intevation.de>
date Thu, 29 Jul 2010 13:43:39 +0000
parents 4aaf73a6225e
children eab777e1bafd
rev   line source
406
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
1 #! /usr/bin/python
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
2 # Copyright (C) 2007 - 2010 by Intevation GmbH
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
3 # Authors:
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
4 # Bernhard Herzog <bh@intevation.de>
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
5 # Bjoern Ricks <bjoern.ricks@intevation.de>
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
6 #
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
7 # This program is free software under the GPL (>=v2)
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
8 # Read the file COPYING coming with the software for details.
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
9
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
10 """Publishes selected packages created by treepkg"""
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
11
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
12 import os
416
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
13 import os.path
406
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
14 import sys
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
15 import shlex
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
16
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
17 from optparse import OptionParser
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
18 from ConfigParser import SafeConfigParser
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
19
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
20 import treepkgcmd
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
21 from treepkg.readconfig import read_config_section, convert_bool
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
22 from treepkg.run import call, capture_output
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
23 from treepkg.cmdexpand import cmdexpand
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
24 from treepkg.publish import *
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
25 from treepkg.util import md5sum
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
26 from treepkg.info.status import TreepkgInfo
414
e0539b483b04 moved data handling in publishdebianpackages into seperate module
Bjoern Ricks <bricks@intevation.de>
parents: 413
diff changeset
27 from treepkg.info.data import Package
e0539b483b04 moved data handling in publishdebianpackages into seperate module
Bjoern Ricks <bricks@intevation.de>
parents: 413
diff changeset
28 from treepkg.info.data import CacheDb
406
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
29
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
30 config_desc = ["distribution", "section", "num_newest",
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
31 "build_user", "build_host", "build_listpackages",
417
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
32 "publish_user", "publish_host",
412
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
33 ("architectures", shlex.split, "armel i386 source"),
406
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
34 ("after_upload_hook", shlex.split),
430
4aaf73a6225e introduced after_copy_hook and made rsync upload to publishdir optional
Bjoern Ricks <bricks@intevation.de>
parents: 429
diff changeset
35 ("after_copy_hook", shlex.split),
431
0953704dc49e publish package must be default to get original behaviour
Bjoern Ricks <bricks@intevation.de>
parents: 430
diff changeset
36 ("publish_packages", convert_bool, "True"),
406
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
37 ("publish_remove_old_packages", convert_bool),
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
38 ("publish_dir", remove_trailing_slashes),
417
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
39 ("cachedb",
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
40 lambda s: expand_filename(remove_trailing_slashes(s))),
406
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
41 ("cachedir",
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
42 lambda s: expand_filename(remove_trailing_slashes(s)))]
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
43
413
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
44
406
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
45 def read_config(filename):
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
46 if not os.path.exists(filename):
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
47 print >>sys.stderr, "Config file %s does not exist" % filename
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
48 sys.exit(1)
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
49 parser = SafeConfigParser()
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
50 parser.read([filename])
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
51 return read_config_section(parser, "publishpackages", config_desc)
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
52
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
53 def parse_commandline():
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
54 parser = OptionParser()
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
55 parser.set_defaults(config_file=os.path.join(treepkgcmd.topdir,
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
56 "publishpackages.cfg"),
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
57 quiet=False)
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
58 parser.add_option("--config-file",
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
59 help=("The configuration file."
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
60 " Default is publishpackages.cfg"))
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
61 parser.add_option("--dist",
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
62 help=("The debian distribution name to use on"
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
63 " the publishing system"))
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
64 parser.add_option("--section",
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
65 help=("The debian distribution section name to use on"
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
66 " the publishing system"))
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
67 parser.add_option("--track",
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
68 help=("The package track whose files are to be"
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
69 " published. If not given, files of all tracks"
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
70 " will be published"))
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
71 parser.add_option("--quiet", action="store_true",
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
72 help=("Do not print progress meters or other"
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
73 " informational output"))
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
74 return parser.parse_args()
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
75
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
76 def get_treepkg_info(variables):
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
77 runremote = prefix_for_remote_command(variables["build_user"],
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
78 variables["build_host"])
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
79 xml = capture_output(cmdexpand("@runremote $build_listpackages"
414
e0539b483b04 moved data handling in publishdebianpackages into seperate module
Bjoern Ricks <bricks@intevation.de>
parents: 413
diff changeset
80 " --newest=$num_newest"
e0539b483b04 moved data handling in publishdebianpackages into seperate module
Bjoern Ricks <bricks@intevation.de>
parents: 413
diff changeset
81 " --only-successful",
413
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
82 runremote=runremote,
409
ecc671a84a73 fixed small bugs
Bjoern Ricks <bricks@intevation.de>
parents: 408
diff changeset
83 **variables))
408
02d498ee90b8 copy binary-all packages in all binary-xyz dirs
Bjoern Ricks <bricks@intevation.de>
parents: 406
diff changeset
84 return TreepkgInfo.fromxml(xml)
406
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
85
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
86 def get_binary_arch(arch):
412
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
87 if not arch is None and not arch.startswith("binary") and \
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
88 not arch == "source":
406
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
89 arch = "binary-" + arch
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
90 return arch
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
91
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
92 def check_package_is_new(packagename, destdir, packagemd5sum):
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
93 destpackage = os.path.join(destdir, packagename)
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
94 if not os.path.isfile(destpackage):
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
95 return True
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
96 destmd5sum = md5sum(destpackage)
408
02d498ee90b8 copy binary-all packages in all binary-xyz dirs
Bjoern Ricks <bricks@intevation.de>
parents: 406
diff changeset
97 return (destmd5sum != packagemd5sum)
02d498ee90b8 copy binary-all packages in all binary-xyz dirs
Bjoern Ricks <bricks@intevation.de>
parents: 406
diff changeset
98
413
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
99 def get_md5sum(packageinfo):
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
100 md5sum = ""
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
101 if packageinfo:
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
102 for checksum in packageinfo.checksums:
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
103 if checksum.type == "md5":
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
104 md5sum = checksum.checksum
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
105 break
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
106 return md5sum
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
107
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
108 def sort_trackname_arch(a, b):
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
109 if a.trackname < b.trackname: return -1
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
110 if a.trackname > b.trackname: return +1
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
111 return cmp(a.arch, b.arch)
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
112
417
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
113 def copy_files_to_destdir(destdir, files, variables, quiet = False):
408
02d498ee90b8 copy binary-all packages in all binary-xyz dirs
Bjoern Ricks <bricks@intevation.de>
parents: 406
diff changeset
114 scp_flags = []
412
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
115 if quiet:
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
116 scp_flags.append("-q")
413
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
117
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
118 if not os.path.exists(destdir):
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
119 os.makedirs(destdir)
417
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
120 if files:
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
121 if variables["build_host"]:
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
122 userhost = "%(build_user)s@%(build_host)s:" % variables
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
123 files = [userhost + filename for filename in files]
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
124 # scp the packages to the cache dir
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
125 call(cmdexpand("scp -p @scp_flags @files $cachedir/", files=files,
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
126 scp_flags=scp_flags, cachedir=destdir))
413
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
127
417
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
128 def remove_old_packages(cachedb, newpackages, quiet):
416
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
129 newfiles = [package.filename for package in newpackages]
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
130 oldpackages = cachedb.get_old_packages(newfiles)
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
131 for package in oldpackages:
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
132 # better check if the file really exists
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
133 if os.path.isfile(package.filename):
417
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
134 if not quiet:
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
135 print "removing file %s" % package.filename
416
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
136 os.remove(package.filename)
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
137 cachedb.remove_packages(oldpackages)
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
138
417
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
139 def copy_packages_to_destdir(cachedb, dir, packages, variables, quiet = False):
413
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
140 packages.sort(cmp=sort_trackname_arch)
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
141 package = packages[0]
417
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
142 trackname = package.trackname
413
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
143 arch = package.arch
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
144 destdir = os.path.join(dir, arch, trackname)
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
145 files = []
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
146 for package in packages:
416
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
147 cachedb.add_package(package)
413
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
148 if package.trackname != trackname or \
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
149 package.arch != arch:
417
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
150 copy_files_to_destdir(destdir, files, variables, quiet)
413
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
151 trackname = package.trackname
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
152 arch = package.arch
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
153 destdir = os.path.join(dir, arch, trackname)
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
154 files = []
416
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
155 # add only to copy files list if the packages differ
413
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
156 if check_package_is_new(package.name, destdir, package.md5sum):
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
157 files.append(package.sourcepath)
417
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
158 if not quiet:
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
159 print "copy new file: %s" % package.name
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
160 copy_files_to_destdir(destdir, files, variables, quiet)
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
161
416
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
162 def copy_to_cachedir(variables, track, revision, quiet = False, architectures=None):
408
02d498ee90b8 copy binary-all packages in all binary-xyz dirs
Bjoern Ricks <bricks@intevation.de>
parents: 406
diff changeset
163 cachedir = variables["cachedir"]
416
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
164 cachdebfilename = variables["cachedb"]
417
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
165 if not quiet:
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
166 print "using cachedb: %s" % cachdebfilename
416
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
167 cachedb = CacheDb(cachdebfilename)
413
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
168 newpackages = []
406
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
169 treepkginfo = get_treepkg_info(variables)
412
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
170 #allowedarchs = set([]) # contains all wanted architectures (incl. source)
408
02d498ee90b8 copy binary-all packages in all binary-xyz dirs
Bjoern Ricks <bricks@intevation.de>
parents: 406
diff changeset
171 allarchs = set([]) # contains all present architectures (incl. source)
02d498ee90b8 copy binary-all packages in all binary-xyz dirs
Bjoern Ricks <bricks@intevation.de>
parents: 406
diff changeset
172 binaryallpackages = []
406
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
173 # change e.g. armel in binary-armel
408
02d498ee90b8 copy binary-all packages in all binary-xyz dirs
Bjoern Ricks <bricks@intevation.de>
parents: 406
diff changeset
174 if not architectures is None:
412
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
175 allowedarchs = set([get_binary_arch(a) for a in architectures])
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
176 else:
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
177 allowedarchs = set([])
408
02d498ee90b8 copy binary-all packages in all binary-xyz dirs
Bjoern Ricks <bricks@intevation.de>
parents: 406
diff changeset
178 for track in treepkginfo.tracks:
406
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
179 for rev in track.revisions:
416
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
180 for packageinfo in rev.packages:
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
181 arch = get_binary_arch(packageinfo.arch)
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
182 if packageinfo.type == "binary":
412
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
183 # skip other files
416
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
184 if packageinfo.arch is None:
412
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
185 continue
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
186 # handle binary-all
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
187 if arch == "binary-all":
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
188 # add trackname for subdir name
416
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
189 packageinfo.trackname = track.name
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
190 binaryallpackages.append(packageinfo)
412
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
191 continue
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
192 allarchs.add(arch)
416
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
193 elif packageinfo.type == "source":
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
194 arch = packageinfo.type
408
02d498ee90b8 copy binary-all packages in all binary-xyz dirs
Bjoern Ricks <bricks@intevation.de>
parents: 406
diff changeset
195 # only copy requested archs
02d498ee90b8 copy binary-all packages in all binary-xyz dirs
Bjoern Ricks <bricks@intevation.de>
parents: 406
diff changeset
196 if len(allowedarchs) == 0 or \
412
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
197 arch in allowedarchs:
413
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
198 filename = os.path.join(cachedir, arch, track.name,
416
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
199 packageinfo.name)
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
200 newpackage = Package(filename, track.name, packageinfo.name,
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
201 packageinfo.path, arch,
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
202 get_md5sum(packageinfo))
413
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
203 newpackages.append(newpackage)
408
02d498ee90b8 copy binary-all packages in all binary-xyz dirs
Bjoern Ricks <bricks@intevation.de>
parents: 406
diff changeset
204 # copy binary-all packages
412
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
205 sourcearch = set(["source"])
408
02d498ee90b8 copy binary-all packages in all binary-xyz dirs
Bjoern Ricks <bricks@intevation.de>
parents: 406
diff changeset
206 if len(allowedarchs) == 0:
412
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
207 binallarchs = allarchs - sourcearch
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
208 elif len(allarchs) == 0:
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
209 binallarchs = allowedarchs - sourcearch
408
02d498ee90b8 copy binary-all packages in all binary-xyz dirs
Bjoern Ricks <bricks@intevation.de>
parents: 406
diff changeset
210 else:
412
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
211 binallarchs = (allowedarchs & allarchs) - sourcearch
416
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
212 for packageinfo in binaryallpackages:
408
02d498ee90b8 copy binary-all packages in all binary-xyz dirs
Bjoern Ricks <bricks@intevation.de>
parents: 406
diff changeset
213 for arch in binallarchs:
417
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
214 filename = os.path.join(cachedir, arch, packageinfo.trackname,
416
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
215 packageinfo.name)
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
216 newpackage = Package(filename, packageinfo.trackname, packageinfo.name,
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
217 packageinfo.path, arch, get_md5sum(packageinfo))
413
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
218 newpackages.append(newpackage)
417
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
219 copy_packages_to_destdir(cachedb, cachedir, newpackages, variables, quiet)
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
220 remove_old_packages(cachedb, newpackages, quiet)
424
a2ba58ffbfbe run rsync only an arch dirs that are present in the current treepkg
Bjoern Ricks <bricks@intevation.de>
parents: 423
diff changeset
221 return binallarchs
406
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
222
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
223 def publish_packages(config_filename, track, revision, dist, section, quiet):
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
224 config = read_config(config_filename)
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
225
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
226 if dist is None:
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
227 dist = config["distribution"]
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
228 if section is None:
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
229 section = config["section"]
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
230
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
231 architectures = config["architectures"]
429
9cfa9f64387a only rsync specified architectures from cachedir to publishdir
Bjoern Ricks <bricks@intevation.de>
parents: 428
diff changeset
232 copy_to_cachedir(config, track, revision, quiet, architectures)
430
4aaf73a6225e introduced after_copy_hook and made rsync upload to publishdir optional
Bjoern Ricks <bricks@intevation.de>
parents: 429
diff changeset
233
4aaf73a6225e introduced after_copy_hook and made rsync upload to publishdir optional
Bjoern Ricks <bricks@intevation.de>
parents: 429
diff changeset
234 if config["after_copy_hook"] and \
4aaf73a6225e introduced after_copy_hook and made rsync upload to publishdir optional
Bjoern Ricks <bricks@intevation.de>
parents: 429
diff changeset
235 len((config["after_copy_hook"][0]).strip()) > 0:
4aaf73a6225e introduced after_copy_hook and made rsync upload to publishdir optional
Bjoern Ricks <bricks@intevation.de>
parents: 429
diff changeset
236 if not quiet:
4aaf73a6225e introduced after_copy_hook and made rsync upload to publishdir optional
Bjoern Ricks <bricks@intevation.de>
parents: 429
diff changeset
237 print "running after copy hook"
4aaf73a6225e introduced after_copy_hook and made rsync upload to publishdir optional
Bjoern Ricks <bricks@intevation.de>
parents: 429
diff changeset
238 call(config["after_copy_hook"])
4aaf73a6225e introduced after_copy_hook and made rsync upload to publishdir optional
Bjoern Ricks <bricks@intevation.de>
parents: 429
diff changeset
239
4aaf73a6225e introduced after_copy_hook and made rsync upload to publishdir optional
Bjoern Ricks <bricks@intevation.de>
parents: 429
diff changeset
240 if config["publish_packages"]:
4aaf73a6225e introduced after_copy_hook and made rsync upload to publishdir optional
Bjoern Ricks <bricks@intevation.de>
parents: 429
diff changeset
241 for arch in architectures:
431
0953704dc49e publish package must be default to get original behaviour
Bjoern Ricks <bricks@intevation.de>
parents: 430
diff changeset
242 if not quiet:
0953704dc49e publish package must be default to get original behaviour
Bjoern Ricks <bricks@intevation.de>
parents: 430
diff changeset
243 print "publish packages for achtitecture %s" % arch
430
4aaf73a6225e introduced after_copy_hook and made rsync upload to publishdir optional
Bjoern Ricks <bricks@intevation.de>
parents: 429
diff changeset
244 copy_arch_to_publishdir(config, dist, section, get_binary_arch(arch),
4aaf73a6225e introduced after_copy_hook and made rsync upload to publishdir optional
Bjoern Ricks <bricks@intevation.de>
parents: 429
diff changeset
245 quiet)
406
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
246
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
247 # update apt archive
428
3d65b3176159 fix if statement
Bjoern Ricks <bricks@intevation.de>
parents: 426
diff changeset
248 if config["after_upload_hook"] and \
429
9cfa9f64387a only rsync specified architectures from cachedir to publishdir
Bjoern Ricks <bricks@intevation.de>
parents: 428
diff changeset
249 len((config["after_upload_hook"][0]).strip()) > 0:
417
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
250 if not quiet:
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
251 print "running after upload hook"
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
252 call(config["after_upload_hook"])
406
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
253
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
254 def main():
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
255 options, args = parse_commandline()
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
256 revision = None # for future use cases
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
257 publish_packages(options.config_file, options.track, revision,
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
258 options.dist, options.section, options.quiet)
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
259
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
260 if __name__ == "__main__":
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
261 main()
This site is hosted by Intevation GmbH (Datenschutzerklärung und Impressum | Privacy Policy and Imprint)