Mercurial > treepkg

annotate bin/publishdebianpackages.py @ 425:b0179962ce26 treepkg-status

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
add helper tool to list content of a cache db
author Bjoern Ricks <bricks@intevation.de>
date Wed, 28 Jul 2010 07:35:54 +0000
parents a2ba58ffbfbe
children 56f7da71d41e
rev   line source
406
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
1 #! /usr/bin/python
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
2 # Copyright (C) 2007 - 2010 by Intevation GmbH
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
3 # Authors:
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
4 # Bernhard Herzog <bh@intevation.de>
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
5 # Bjoern Ricks <bjoern.ricks@intevation.de>
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
6 #
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
7 # This program is free software under the GPL (>=v2)
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
8 # Read the file COPYING coming with the software for details.
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
9
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
10 """Publishes selected packages created by treepkg"""
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
11
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
12 import os
416
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
13 import os.path
423
855829a4a2db check if upload hook is empty
Bjoern Ricks <bricks@intevation.de>
parents: 422
diff changeset
14 import re
406
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
15 import sys
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
16 import shlex
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
17
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
18 from optparse import OptionParser
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
19 from ConfigParser import SafeConfigParser
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
20
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
21 import treepkgcmd
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
22 from treepkg.readconfig import read_config_section, convert_bool
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
23 from treepkg.run import call, capture_output
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
24 from treepkg.cmdexpand import cmdexpand
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
25 from treepkg.publish import *
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
26 from treepkg.util import md5sum
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
27 from treepkg.info.status import TreepkgInfo
414
e0539b483b04 moved data handling in publishdebianpackages into seperate module
Bjoern Ricks <bricks@intevation.de>
parents: 413
diff changeset
28 from treepkg.info.data import Package
e0539b483b04 moved data handling in publishdebianpackages into seperate module
Bjoern Ricks <bricks@intevation.de>
parents: 413
diff changeset
29 from treepkg.info.data import CacheDb
406
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
30
423
855829a4a2db check if upload hook is empty
Bjoern Ricks <bricks@intevation.de>
parents: 422
diff changeset
31 EMPTY = re.compile(r'\s*')
855829a4a2db check if upload hook is empty
Bjoern Ricks <bricks@intevation.de>
parents: 422
diff changeset
32
406
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
33 config_desc = ["distribution", "section", "num_newest",
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
34 "build_user", "build_host", "build_listpackages",
417
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
35 "publish_user", "publish_host",
412
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
36 ("architectures", shlex.split, "armel i386 source"),
406
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
37 ("after_upload_hook", shlex.split),
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
38 ("publish_remove_old_packages", convert_bool),
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
39 ("publish_dir", remove_trailing_slashes),
417
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
40 ("cachedb",
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
41 lambda s: expand_filename(remove_trailing_slashes(s))),
406
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
42 ("cachedir",
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
43 lambda s: expand_filename(remove_trailing_slashes(s)))]
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
44
413
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
45
406
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
46 def read_config(filename):
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
47 if not os.path.exists(filename):
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
48 print >>sys.stderr, "Config file %s does not exist" % filename
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
49 sys.exit(1)
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
50 parser = SafeConfigParser()
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
51 parser.read([filename])
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
52 return read_config_section(parser, "publishpackages", config_desc)
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
53
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
54 def parse_commandline():
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
55 parser = OptionParser()
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
56 parser.set_defaults(config_file=os.path.join(treepkgcmd.topdir,
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
57 "publishpackages.cfg"),
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
58 quiet=False)
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
59 parser.add_option("--config-file",
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
60 help=("The configuration file."
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
61 " Default is publishpackages.cfg"))
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
62 parser.add_option("--dist",
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
63 help=("The debian distribution name to use on"
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
64 " the publishing system"))
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
65 parser.add_option("--section",
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
66 help=("The debian distribution section name to use on"
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
67 " the publishing system"))
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
68 parser.add_option("--track",
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
69 help=("The package track whose files are to be"
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
70 " published. If not given, files of all tracks"
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
71 " will be published"))
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
72 parser.add_option("--quiet", action="store_true",
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
73 help=("Do not print progress meters or other"
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
74 " informational output"))
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
75 return parser.parse_args()
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
76
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
77 def get_treepkg_info(variables):
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
78 runremote = prefix_for_remote_command(variables["build_user"],
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
79 variables["build_host"])
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
80 xml = capture_output(cmdexpand("@runremote $build_listpackages"
414
e0539b483b04 moved data handling in publishdebianpackages into seperate module
Bjoern Ricks <bricks@intevation.de>
parents: 413
diff changeset
81 " --newest=$num_newest"
e0539b483b04 moved data handling in publishdebianpackages into seperate module
Bjoern Ricks <bricks@intevation.de>
parents: 413
diff changeset
82 " --only-successful",
413
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
83 runremote=runremote,
409
ecc671a84a73 fixed small bugs
Bjoern Ricks <bricks@intevation.de>
parents: 408
diff changeset
84 **variables))
408
02d498ee90b8 copy binary-all packages in all binary-xyz dirs
Bjoern Ricks <bricks@intevation.de>
parents: 406
diff changeset
85 return TreepkgInfo.fromxml(xml)
406
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
86
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
87 def get_binary_arch(arch):
412
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
88 if not arch is None and not arch.startswith("binary") and \
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
89 not arch == "source":
406
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
90 arch = "binary-" + arch
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
91 return arch
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
92
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
93 def check_package_is_new(packagename, destdir, packagemd5sum):
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
94 destpackage = os.path.join(destdir, packagename)
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
95 if not os.path.isfile(destpackage):
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
96 return True
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
97 destmd5sum = md5sum(destpackage)
408
02d498ee90b8 copy binary-all packages in all binary-xyz dirs
Bjoern Ricks <bricks@intevation.de>
parents: 406
diff changeset
98 return (destmd5sum != packagemd5sum)
02d498ee90b8 copy binary-all packages in all binary-xyz dirs
Bjoern Ricks <bricks@intevation.de>
parents: 406
diff changeset
99
413
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
100 def get_md5sum(packageinfo):
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
101 md5sum = ""
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
102 if packageinfo:
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
103 for checksum in packageinfo.checksums:
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
104 if checksum.type == "md5":
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
105 md5sum = checksum.checksum
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
106 break
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
107 return md5sum
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
108
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
109 def sort_trackname_arch(a, b):
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
110 if a.trackname < b.trackname: return -1
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
111 if a.trackname > b.trackname: return +1
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
112 return cmp(a.arch, b.arch)
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
113
417
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
114 def copy_files_to_destdir(destdir, files, variables, quiet = False):
408
02d498ee90b8 copy binary-all packages in all binary-xyz dirs
Bjoern Ricks <bricks@intevation.de>
parents: 406
diff changeset
115 scp_flags = []
412
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
116 if quiet:
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
117 scp_flags.append("-q")
413
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
118
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
119 if not os.path.exists(destdir):
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
120 os.makedirs(destdir)
417
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
121 if files:
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
122 if variables["build_host"]:
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
123 userhost = "%(build_user)s@%(build_host)s:" % variables
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
124 files = [userhost + filename for filename in files]
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
125 # scp the packages to the cache dir
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
126 call(cmdexpand("scp -p @scp_flags @files $cachedir/", files=files,
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
127 scp_flags=scp_flags, cachedir=destdir))
413
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
128
417
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
129 def remove_old_packages(cachedb, newpackages, quiet):
416
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
130 newfiles = [package.filename for package in newpackages]
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
131 oldpackages = cachedb.get_old_packages(newfiles)
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
132 for package in oldpackages:
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
133 # better check if the file really exists
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
134 if os.path.isfile(package.filename):
417
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
135 if not quiet:
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
136 print "removing file %s" % package.filename
416
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
137 os.remove(package.filename)
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
138 cachedb.remove_packages(oldpackages)
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
139
417
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
140 def copy_packages_to_destdir(cachedb, dir, packages, variables, quiet = False):
413
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
141 packages.sort(cmp=sort_trackname_arch)
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
142 package = packages[0]
417
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
143 trackname = package.trackname
413
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
144 arch = package.arch
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
145 destdir = os.path.join(dir, arch, trackname)
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
146 files = []
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
147 for package in packages:
416
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
148 cachedb.add_package(package)
413
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
149 if package.trackname != trackname or \
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
150 package.arch != arch:
417
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
151 copy_files_to_destdir(destdir, files, variables, quiet)
413
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
152 trackname = package.trackname
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
153 arch = package.arch
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
154 destdir = os.path.join(dir, arch, trackname)
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
155 files = []
416
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
156 # add only to copy files list if the packages differ
413
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
157 if check_package_is_new(package.name, destdir, package.md5sum):
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
158 files.append(package.sourcepath)
417
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
159 if not quiet:
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
160 print "copy new file: %s" % package.name
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
161 copy_files_to_destdir(destdir, files, variables, quiet)
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
162
416
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
163 def copy_to_cachedir(variables, track, revision, quiet = False, architectures=None):
408
02d498ee90b8 copy binary-all packages in all binary-xyz dirs
Bjoern Ricks <bricks@intevation.de>
parents: 406
diff changeset
164 cachedir = variables["cachedir"]
416
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
165 cachdebfilename = variables["cachedb"]
417
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
166 if not quiet:
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
167 print "using cachedb: %s" % cachdebfilename
416
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
168 cachedb = CacheDb(cachdebfilename)
413
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
169 newpackages = []
406
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
170 treepkginfo = get_treepkg_info(variables)
412
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
171 #allowedarchs = set([]) # contains all wanted architectures (incl. source)
408
02d498ee90b8 copy binary-all packages in all binary-xyz dirs
Bjoern Ricks <bricks@intevation.de>
parents: 406
diff changeset
172 allarchs = set([]) # contains all present architectures (incl. source)
02d498ee90b8 copy binary-all packages in all binary-xyz dirs
Bjoern Ricks <bricks@intevation.de>
parents: 406
diff changeset
173 binaryallpackages = []
406
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
174 # change e.g. armel in binary-armel
408
02d498ee90b8 copy binary-all packages in all binary-xyz dirs
Bjoern Ricks <bricks@intevation.de>
parents: 406
diff changeset
175 if not architectures is None:
412
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
176 allowedarchs = set([get_binary_arch(a) for a in architectures])
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
177 else:
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
178 allowedarchs = set([])
408
02d498ee90b8 copy binary-all packages in all binary-xyz dirs
Bjoern Ricks <bricks@intevation.de>
parents: 406
diff changeset
179 for track in treepkginfo.tracks:
406
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
180 for rev in track.revisions:
416
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
181 for packageinfo in rev.packages:
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
182 arch = get_binary_arch(packageinfo.arch)
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
183 if packageinfo.type == "binary":
412
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
184 # skip other files
416
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
185 if packageinfo.arch is None:
412
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
186 continue
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
187 # handle binary-all
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
188 if arch == "binary-all":
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
189 # add trackname for subdir name
416
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
190 packageinfo.trackname = track.name
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
191 binaryallpackages.append(packageinfo)
412
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
192 continue
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
193 allarchs.add(arch)
416
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
194 elif packageinfo.type == "source":
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
195 arch = packageinfo.type
408
02d498ee90b8 copy binary-all packages in all binary-xyz dirs
Bjoern Ricks <bricks@intevation.de>
parents: 406
diff changeset
196 # only copy requested archs
02d498ee90b8 copy binary-all packages in all binary-xyz dirs
Bjoern Ricks <bricks@intevation.de>
parents: 406
diff changeset
197 if len(allowedarchs) == 0 or \
412
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
198 arch in allowedarchs:
413
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
199 filename = os.path.join(cachedir, arch, track.name,
416
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
200 packageinfo.name)
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
201 newpackage = Package(filename, track.name, packageinfo.name,
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
202 packageinfo.path, arch,
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
203 get_md5sum(packageinfo))
413
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
204 newpackages.append(newpackage)
408
02d498ee90b8 copy binary-all packages in all binary-xyz dirs
Bjoern Ricks <bricks@intevation.de>
parents: 406
diff changeset
205 # copy binary-all packages
412
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
206 sourcearch = set(["source"])
408
02d498ee90b8 copy binary-all packages in all binary-xyz dirs
Bjoern Ricks <bricks@intevation.de>
parents: 406
diff changeset
207 if len(allowedarchs) == 0:
412
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
208 binallarchs = allarchs - sourcearch
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
209 elif len(allarchs) == 0:
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
210 binallarchs = allowedarchs - sourcearch
408
02d498ee90b8 copy binary-all packages in all binary-xyz dirs
Bjoern Ricks <bricks@intevation.de>
parents: 406
diff changeset
211 else:
412
58ecf7c0ecba improved publishdebianpackages and fixed a lot of bugs in the xml info parsing part
Bjoern Ricks <bricks@intevation.de>
parents: 409
diff changeset
212 binallarchs = (allowedarchs & allarchs) - sourcearch
416
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
213 for packageinfo in binaryallpackages:
408
02d498ee90b8 copy binary-all packages in all binary-xyz dirs
Bjoern Ricks <bricks@intevation.de>
parents: 406
diff changeset
214 for arch in binallarchs:
417
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
215 filename = os.path.join(cachedir, arch, packageinfo.trackname,
416
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
216 packageinfo.name)
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
217 newpackage = Package(filename, packageinfo.trackname, packageinfo.name,
4980f8d5014a first version of incremental copying debian packages to cachedir
Bjoern Ricks <bricks@intevation.de>
parents: 414
diff changeset
218 packageinfo.path, arch, get_md5sum(packageinfo))
413
94a6ae627b31 implemented CacheDb to store copied Packages from build host
Bjoern Ricks <bricks@intevation.de>
parents: 412
diff changeset
219 newpackages.append(newpackage)
417
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
220 copy_packages_to_destdir(cachedb, cachedir, newpackages, variables, quiet)
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
221 remove_old_packages(cachedb, newpackages, quiet)
424
a2ba58ffbfbe run rsync only an arch dirs that are present in the current treepkg
Bjoern Ricks <bricks@intevation.de>
parents: 423
diff changeset
222 return binallarchs
406
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
223
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
224 def publish_packages(config_filename, track, revision, dist, section, quiet):
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
225 config = read_config(config_filename)
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
226
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
227 if dist is None:
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
228 dist = config["distribution"]
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
229 if section is None:
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
230 section = config["section"]
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
231
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
232 architectures = config["architectures"]
424
a2ba58ffbfbe run rsync only an arch dirs that are present in the current treepkg
Bjoern Ricks <bricks@intevation.de>
parents: 423
diff changeset
233 allarchs = opy_to_cachedir(config, track, revision, quiet, architectures)
a2ba58ffbfbe run rsync only an arch dirs that are present in the current treepkg
Bjoern Ricks <bricks@intevation.de>
parents: 423
diff changeset
234 for arch in allarchs:
a2ba58ffbfbe run rsync only an arch dirs that are present in the current treepkg
Bjoern Ricks <bricks@intevation.de>
parents: 423
diff changeset
235 copy_to_publishdir(config, dist, section, arch, quiet)
406
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
236
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
237 # update apt archive
423
855829a4a2db check if upload hook is empty
Bjoern Ricks <bricks@intevation.de>
parents: 422
diff changeset
238 if not EMPTY.match(config["after_upload_hook"]):
417
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
239 if not quiet:
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
240 print "running after upload hook"
25ef11a79d7f incremental copying only changed debian packages
Bjoern Ricks <bricks@intevation.de>
parents: 416
diff changeset
241 call(config["after_upload_hook"])
406
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
242
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
243 def main():
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
244 options, args = parse_commandline()
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
245 revision = None # for future use cases
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
246 publish_packages(options.config_file, options.track, revision,
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
247 options.dist, options.section, options.quiet)
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
248
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
249 if __name__ == "__main__":
52e3c3976e53 inital checkin for new publishpackages processing
Bjoern Ricks <bricks@intevation.de>
parents:
diff changeset
250 main()
This site is hosted by Intevation GmbH (Datenschutzerklärung und Impressum | Privacy Policy and Imprint)