Mercurial > treepkg
comparison bin/publishpackages.py @ 256:8052aabada8b
Make publishpackages.py work for automated regular publishing of the
newest built packages. This requires changes in the configuration file
as well, so demopublishpackages.cfg is updated too.
| author | Bernhard Herzog <bh@intevation.de> |
|---|---|
| date | Thu, 16 Apr 2009 09:32:35 +0000 |
| parents | a3f106580525 |
| children | acf6c0ce2014 |
comparison
equal
deleted
inserted
replaced
| 255:70735b398bb0 | 256:8052aabada8b |
|---|---|
| 9 """Publishes selected packages created by treepkg""" | 9 """Publishes selected packages created by treepkg""" |
| 10 | 10 |
| 11 import sys | 11 import sys |
| 12 import os | 12 import os |
| 13 import shutil | 13 import shutil |
| 14 import shlex | |
| 14 from optparse import OptionParser | 15 from optparse import OptionParser |
| 15 from ConfigParser import SafeConfigParser | 16 from ConfigParser import SafeConfigParser |
| 16 | 17 |
| 17 import treepkgcmd | 18 import treepkgcmd |
| 18 from treepkg.readconfig import read_config_section | 19 from treepkg.readconfig import read_config_section, convert_bool |
| 19 from treepkg.run import call, capture_output | 20 from treepkg.run import call, capture_output |
| 20 from treepkg.cmdexpand import cmdexpand | 21 from treepkg.cmdexpand import cmdexpand |
| 21 from treepkg.util import ensure_directory, listdir_abs | 22 from treepkg.util import ensure_directory, listdir_abs |
| 22 | 23 |
| 23 | 24 |
| 28 """ | 29 """ |
| 29 Applies os.path.expanduser and os.path.expandvars to filename | 30 Applies os.path.expanduser and os.path.expandvars to filename |
| 30 """ | 31 """ |
| 31 return os.path.expandvars(os.path.expanduser(filename)) | 32 return os.path.expandvars(os.path.expanduser(filename)) |
| 32 | 33 |
| 33 config_desc = ["distribution", "section", | 34 config_desc = ["distribution", "section", "num_newest", |
| 34 "build_user", "build_host", "build_listpackages", | 35 "build_user", "build_host", "build_listpackages", |
| 35 "publish_user", "publish_host", "publish_apt_archive_update", | 36 "publish_user", "publish_host", |
| 37 ("after_upload_hook", shlex.split), | |
| 38 ("publish_remove_old_packages", convert_bool), | |
| 36 ("publish_dir", remove_trailing_slashes), | 39 ("publish_dir", remove_trailing_slashes), |
| 37 ("cachedir", | 40 ("cachedir", |
| 38 lambda s: expand_filename(remove_trailing_slashes(s)))] | 41 lambda s: expand_filename(remove_trailing_slashes(s)))] |
| 39 | 42 |
| 40 def read_config(filename): | 43 def read_config(filename): |
| 65 help=("The package track whose files are to be" | 68 help=("The package track whose files are to be" |
| 66 " published. If not given, files of all tracks" | 69 " published. If not given, files of all tracks" |
| 67 " will be published")) | 70 " will be published")) |
| 68 return parser.parse_args() | 71 return parser.parse_args() |
| 69 | 72 |
| 73 def prefix_for_remote_command(user, host): | |
| 74 """Returns the ssh call needed to run a command on a remote host. | |
| 75 If host is empty, the function assumes the command is to be run on | |
| 76 the local host as the same user that exectutes this function, so | |
| 77 that no ssh or other call is needed. | |
| 78 """ | |
| 79 prefix = [] | |
| 80 if host: | |
| 81 prefix.extend(["ssh", "%s@%s" % (user, host)]) | |
| 82 return prefix | |
| 70 | 83 |
| 71 def publish_packages_arch(variables, track, revision, dist, section, arch): | 84 |
| 72 # create web-page on build host | 85 def copy_to_cache(variables, track, revision, arch): |
| 73 listpackages_vars = variables.copy() | 86 listpackages_vars = variables.copy() |
| 74 | 87 |
| 75 if arch == "source": | 88 if arch == "source": |
| 76 listpackages_vars["pkgtype"] = "--source" | 89 listpackages_vars["pkgtype"] = "--source" |
| 77 else: | 90 else: |
| 85 if revision: | 98 if revision: |
| 86 listpackages_vars["revision"] = ["--revision", revision] | 99 listpackages_vars["revision"] = ["--revision", revision] |
| 87 else: | 100 else: |
| 88 listpackages_vars["revision"] = [] | 101 listpackages_vars["revision"] = [] |
| 89 | 102 |
| 90 files = capture_output(cmdexpand("ssh $build_user$@$build_host" | 103 runremote = prefix_for_remote_command(variables["build_user"], |
| 91 " $build_listpackages" | 104 variables["build_host"]) |
| 92 " @track @revision $pkgtype", | 105 files = capture_output(cmdexpand("@runremote $build_listpackages" |
| 106 " @track @revision $pkgtype" | |
| 107 " --newest=$num_newest", | |
| 108 runremote=runremote, | |
| 93 **listpackages_vars)).strip().split("\n") | 109 **listpackages_vars)).strip().split("\n") |
| 94 | 110 |
| 95 # scp the packages to the cache dir | 111 # scp the packages to the cache dir |
| 96 cachedir = variables["cachedir"] | 112 cachedir = variables["cachedir"] |
| 97 shutil.rmtree(cachedir, ignore_errors=True) | 113 shutil.rmtree(cachedir, ignore_errors=True) |
| 98 ensure_directory(cachedir) | 114 ensure_directory(cachedir) |
| 99 userhost = "%(build_user)s@%(build_host)s:" % variables | 115 if variables["build_host"]: |
| 100 call(cmdexpand("scp @files $cachedir/", | 116 userhost = "%(build_user)s@%(build_host)s:" % variables |
| 101 files = [userhost + filename for filename in files], | 117 files = [userhost + filename for filename in files] |
| 118 call(cmdexpand("scp -p @files $cachedir/", files = files, **variables)) | |
| 119 | |
| 120 | |
| 121 def copy_to_publishdir(variables, dist, section, arch): | |
| 122 destdir = os.path.join(variables["publish_dir"], dist, section, arch) | |
| 123 remote_destdir = destdir | |
| 124 if variables["publish_host"]: | |
| 125 remote_destdir = (("%(publish_user)s@%(publish_host)s:" % variables) | |
| 126 + remote_destdir) | |
| 127 runremote = prefix_for_remote_command(variables["publish_user"], | |
| 128 variables["publish_host"]) | |
| 129 | |
| 130 call(cmdexpand("@runremote mkdir --parents $destdir", | |
| 131 runremote=runremote, destdir=destdir, **variables)) | |
| 132 rsync_flags = [] | |
| 133 if variables["publish_remove_old_packages"]: | |
| 134 rsync_flags.append("--delete") | |
| 135 call(cmdexpand("rsync @rsync_flags -rpt $cachedir/ $remote_destdir/", | |
| 136 rsync_flags=rsync_flags, remote_destdir=remote_destdir, | |
| 102 **variables)) | 137 **variables)) |
| 103 | 138 |
| 104 # copy the packages to the remote publishing host. Create the | 139 |
| 105 # destination directory if it doesn't exist yet. | 140 def publish_packages_arch(variables, track, revision, dist, section, arch): |
| 106 destdir = os.path.join(variables["publish_dir"], dist, section, arch) | 141 copy_to_cache(variables, track, revision, arch) |
| 107 call(cmdexpand("ssh $publish_user$@$publish_host mkdir --parents $destdir", | 142 copy_to_publishdir(variables, dist, section, arch) |
| 108 destdir=destdir, **variables)) | |
| 109 call(cmdexpand("scp @files $publish_user$@$publish_host:$destdir", | |
| 110 files=listdir_abs(cachedir), destdir=destdir, | |
| 111 **variables)) | |
| 112 | 143 |
| 113 | 144 |
| 114 def publish_packages(config_filename, track, revision, dist, section): | 145 def publish_packages(config_filename, track, revision, dist, section): |
| 115 config = read_config(config_filename) | 146 config = read_config(config_filename) |
| 116 | 147 |
| 121 | 152 |
| 122 for arch in ["binary-i386", "source"]: | 153 for arch in ["binary-i386", "source"]: |
| 123 publish_packages_arch(config, track, revision, dist, section, arch) | 154 publish_packages_arch(config, track, revision, dist, section, arch) |
| 124 | 155 |
| 125 # update apt archive | 156 # update apt archive |
| 126 call(cmdexpand("ssh $publish_user$@$publish_host" | 157 call(config["after_upload_hook"]) |
| 127 " $publish_apt_archive_update", | |
| 128 **config)) | |
| 129 | 158 |
| 130 def main(): | 159 def main(): |
| 131 options, args = parse_commandline() | 160 options, args = parse_commandline() |
| 132 publish_packages(options.config_file, options.track, options.revision, | 161 publish_packages(options.config_file, options.track, options.revision, |
| 133 options.dist, options.section) | 162 options.dist, options.section) |