# HG changeset patch
# User Bernhard Herzog <bh@intevation.de>
# Date 1239874355 0
# Node ID 8052aabada8b5401040bcd9a9f0579f7f3fa9317
# Parent  70735b398bb07f0a51fba0e0c618416120a02fa3
Make publishpackages.py work for automated regular publishing of the
newest built packages.  This requires changes in the configuration file
as well, so demopublishpackages.cfg is updated too.

diff -r 70735b398bb0 -r 8052aabada8b bin/publishpackages.py
--- a/bin/publishpackages.py	Wed Apr 15 13:50:31 2009 +0000
+++ b/bin/publishpackages.py	Thu Apr 16 09:32:35 2009 +0000
@@ -11,11 +11,12 @@
 import sys
 import os
 import shutil
+import shlex
 from optparse import OptionParser
 from ConfigParser import SafeConfigParser
 
 import treepkgcmd
-from treepkg.readconfig import read_config_section
+from treepkg.readconfig import read_config_section, convert_bool
 from treepkg.run import call, capture_output
 from treepkg.cmdexpand import cmdexpand
 from treepkg.util import ensure_directory, listdir_abs
@@ -30,9 +31,11 @@
     """
     return os.path.expandvars(os.path.expanduser(filename))
 
-config_desc = ["distribution", "section",
+config_desc = ["distribution", "section", "num_newest",
                "build_user", "build_host", "build_listpackages",
-               "publish_user", "publish_host", "publish_apt_archive_update",
+               "publish_user", "publish_host",
+               ("after_upload_hook", shlex.split),
+               ("publish_remove_old_packages", convert_bool),
                ("publish_dir", remove_trailing_slashes),
                ("cachedir",
                 lambda s: expand_filename(remove_trailing_slashes(s)))]
@@ -67,9 +70,19 @@
                             " will be published"))
     return parser.parse_args()
 
+def prefix_for_remote_command(user, host):
+    """Returns the ssh call needed to run a command on a remote host.
+    If host is empty, the function assumes the command is to be run on
+    the local host as the same user that exectutes this function, so
+    that no ssh or other call is needed.
+    """
+    prefix = []
+    if host:
+        prefix.extend(["ssh", "%s@%s" % (user, host)])
+    return prefix
 
-def publish_packages_arch(variables, track, revision, dist, section, arch):
-    # create web-page on build host
+
+def copy_to_cache(variables, track, revision, arch):
     listpackages_vars = variables.copy()
 
     if arch == "source":
@@ -87,28 +100,46 @@
     else:
         listpackages_vars["revision"] = []
 
-    files = capture_output(cmdexpand("ssh $build_user$@$build_host"
-                                     " $build_listpackages"
-                                     " @track @revision $pkgtype",
+    runremote = prefix_for_remote_command(variables["build_user"],
+                                          variables["build_host"])
+    files = capture_output(cmdexpand("@runremote $build_listpackages"
+                                     " @track @revision $pkgtype"
+                                     " --newest=$num_newest",
+                                     runremote=runremote,
                                      **listpackages_vars)).strip().split("\n")
 
     # scp the packages to the cache dir
     cachedir = variables["cachedir"]
     shutil.rmtree(cachedir, ignore_errors=True)
     ensure_directory(cachedir)
-    userhost = "%(build_user)s@%(build_host)s:" % variables
-    call(cmdexpand("scp @files $cachedir/",
-                   files = [userhost + filename for filename in files],
+    if variables["build_host"]:
+        userhost = "%(build_user)s@%(build_host)s:" % variables
+        files = [userhost + filename for filename in files]
+    call(cmdexpand("scp -p @files $cachedir/", files = files, **variables))
+
+
+def copy_to_publishdir(variables, dist, section, arch):
+    destdir = os.path.join(variables["publish_dir"], dist, section, arch)
+    remote_destdir = destdir
+    if variables["publish_host"]:
+        remote_destdir = (("%(publish_user)s@%(publish_host)s:" % variables)
+                          + remote_destdir)
+    runremote = prefix_for_remote_command(variables["publish_user"],
+                                          variables["publish_host"])
+
+    call(cmdexpand("@runremote mkdir --parents $destdir",
+                   runremote=runremote, destdir=destdir, **variables))
+    rsync_flags = []
+    if variables["publish_remove_old_packages"]:
+        rsync_flags.append("--delete")
+    call(cmdexpand("rsync @rsync_flags -rpt $cachedir/ $remote_destdir/",
+                   rsync_flags=rsync_flags, remote_destdir=remote_destdir,
                    **variables))
 
-    # copy the packages to the remote publishing host.  Create the
-    # destination directory if it doesn't exist yet.
-    destdir = os.path.join(variables["publish_dir"], dist, section, arch)
-    call(cmdexpand("ssh $publish_user$@$publish_host mkdir --parents $destdir",
-                   destdir=destdir, **variables))
-    call(cmdexpand("scp @files $publish_user$@$publish_host:$destdir",
-                   files=listdir_abs(cachedir), destdir=destdir,
-                   **variables))
+
+def publish_packages_arch(variables, track, revision, dist, section, arch):
+    copy_to_cache(variables, track, revision, arch)
+    copy_to_publishdir(variables, dist, section, arch)
 
 
 def publish_packages(config_filename, track, revision, dist, section):
@@ -123,9 +154,7 @@
         publish_packages_arch(config, track, revision, dist, section, arch)
 
     # update apt archive
-    call(cmdexpand("ssh $publish_user$@$publish_host"
-                   " $publish_apt_archive_update",
-                   **config))
+    call(config["after_upload_hook"])
 
 def main():
     options, args = parse_commandline()
diff -r 70735b398bb0 -r 8052aabada8b demopublishpackages.cfg
--- a/demopublishpackages.cfg	Wed Apr 15 13:50:31 2009 +0000
+++ b/demopublishpackages.cfg	Thu Apr 16 09:32:35 2009 +0000
@@ -9,6 +9,11 @@
 distribution: lenny
 section: experimental
 
+# Number of newest successfully builts revisions to publish for each
+# track.  Ignored when publishpackages is called with an explicit
+# revision.
+num_newest: 3
+
 # Username and host on which the treepackager runs.  publishpackages.py
 # has to be able to connect to that host as the builduser via ssh
 # without knowning the password.  This is best achieved with the
@@ -23,16 +28,25 @@
 
 # Username and host on which to publish the packages.
 # publishpackages.py uses ssh to execute commands on the publish_host to
-# create directories and scp to copy the files from the local cache to
-# the publish_host.
+# create directories and rsync to copy the files from the local cache to
+# the publish_host.  If publish_host is empty, the publishpackages.py
+# assumes that no ssh or other remote login is needed to access the
+# publish host with the right permissions.
 publish_user: builder
 publish_host: localhost
 
 # the directory on the publish_host where the apt directories reside.
 publish_dir: /home/ftp/apt/dists/
 
-# Command to execute on the build host to update the apt archive
-publish_apt_archive_update: /home/ftp/apt/update
+# If true publishpackages.py will remove the packages in publish_dir
+# that were not put there by the current run.  Setting it to True is
+# probably only useful when using publishpackages.py to automatically
+# publish the newest built packages using a cron-job, for instance.
+publish_remove_old_packages: False
+
+# Command to run after the packages have been copied to the publish
+# host.  This command is run on the host running publishpackages.py.
+after_upload_hook: ssh %(publish_user)s@%(publish_host)s /home/ftp/apt/update
 
 # local cache directory.  publishpackages.py may delete it and its
 # contents, so if you intend to run publishpackages.py on the same