--- a/bin/publishdebianpackages.py	Thu Jul 22 16:01:43 2010 +0000
+++ b/bin/publishdebianpackages.py	Fri Jul 23 16:40:38 2010 +0000
@@ -12,6 +12,7 @@
 import os
 import sys
 import shlex
+import sqlite3
 
 from optparse import OptionParser
 from ConfigParser import SafeConfigParser
@@ -34,6 +35,91 @@
                ("cachedir",
                 lambda s: expand_filename(remove_trailing_slashes(s)))]
 
+class Package:
+    
+    def __init__(self, filename, trackname, packagename, packagepath, arch, md5sum):
+        self.filename = filename
+        self.trackname = trackname
+        self.name = packagename
+        self.sourcepath = packagepath
+        self.arch = arch
+        self.md5sum = md5sum
+
+
+class CacheDb:
+
+
+    def __init__(self, file):
+        self.SELECT_PACKAGE_TMPL = """SELECT * FROM packages 
+                                   WHERE filename = ?"""
+        self.file = file
+        self.conn = sqlite3.connect(file)
+        self.cursor = self.conn.cursor()
+        self.init_db(file)
+
+    def init_db(self, file):
+        self.cursor.execute(
+               """CREATE TABLE IF NOT EXISTS packages (
+                  filename TEXT PRIMARY KEY,
+                  trackname TEXT,
+                  packagename TEXT,
+                  sourcepath TEXT,
+                  arch TEXT, 
+                  md5sum TEXT )""")
+        self.conn.commit()
+
+
+    def add_package(self, package):
+        self.cursor.execute(self.SELECT_PACKAGE_TMPL, (package.filename,))
+        row = self.cursor.fetchone()
+        if not row:
+            self.insert_package(package)
+        else:
+            self.update_package(package)
+
+    def insert_package(self, package):
+        INSERT_TMPL = """INSERT INTO packages VALUES (
+                         ?, ?, ? ,? ,?, ?)"""
+        self.cursor.execute(INSERT_TMPL, (package.filename,
+                            package.trackname,
+                            package.name,
+                            package.sourcepath,
+                            package.arch,
+                            package.md5sum))
+        self.conn.commit()
+
+    def update_package(self, package):
+        UPDATE_TMPL = """UPDATE packages set md5sum = '?' 
+                         WHERE filename in (?)"""
+        self.cursor.execute(UPDATE_TMPL, (package.md5sum, package.filename))
+        self.conn.commit()
+
+    def get_package(self, filename):
+        self.cursor.execute(self.SELECT_PACKAGE_TMPL, (filename,))
+        row = self.cursor.fetchone()
+        if not row:
+            return None
+        return Package(row[0], row[1], row[2], row[3], row[4], row[5]) 
+
+    def get_old_packages(self, newfiles):
+        SELECT_TMPL = """SELECT * FROM packages 
+                      WHERE filename not in (%s)"""
+        tmp = ", ".join(['?'] * len(newfiles))
+        self.cursor.execute(SELECT_TMPL % tmp, newfiles)
+        return [Package(*row) for row in self.cursor.fetchall()]
+
+    def remove_packages(self, files):
+        DELET_TMPL = """DELETE FROM packages 
+                     WHERE filename in (%s)"""
+        tmp = ", ".join(['?'] * len(files))
+        self.cursor.execute(DELET_TMPL % tmp, files)
+        self.conn.commit()
+
+    def get_packages(self):
+        SELECT_TMPL = "SELECT * FROM packages" 
+        self.cursor.execute(SELECT_TMPL)
+        return [Package(*row) for row in self.cursor.fetchall()]
+
 def read_config(filename):
     if not os.path.exists(filename):
         print >>sys.stderr, "Config file %s does not exist" % filename
@@ -70,8 +156,7 @@
                                           variables["build_host"])
     xml = capture_output(cmdexpand("@runremote $build_listpackages"
                                      " --newest=$num_newest",
-                                     #runremote=runremote,
-                                     runremote="",
+                                     runremote=runremote,
                                      **variables))
     return TreepkgInfo.fromxml(xml)
 
@@ -88,26 +173,55 @@
     destmd5sum = md5sum(destpackage)
     return (destmd5sum != packagemd5sum)
 
-def copy_to_destdir(package, dir, arch, subdir, quiet = False):
+def get_md5sum(packageinfo):
+    md5sum = ""
+    if packageinfo:
+        for checksum in packageinfo.checksums:
+            if checksum.type == "md5":
+                md5sum = checksum.checksum
+                break
+    return md5sum
+
+def sort_trackname_arch(a, b):
+    if a.trackname < b.trackname: return -1
+    if a.trackname > b.trackname: return +1
+    return cmp(a.arch, b.arch)
+
+def copy_files(destdir, files, quiet):
     scp_flags = []
     if quiet:
         scp_flags.append("-q")
-    destdir = os.path.join(dir, arch, subdir)
-    print "package: %s, destdir: %s" % (package.name, destdir)
-    md5sum = ""
-    for checksum in package.checksums:
-        if checksum.type == "md5":
-            md5sum = checksum.checksum
-            break
-    # update only if the packages differ
-    if check_package_is_new(package.name, destdir, md5sum):
-        pass
-        # scp the packages to the cache dir 
-        #call(cmdexpand("scp -p @scp_flags $file $cachedir/", file=package.path,
-        #     scp_flags=scp_flags, cachedir=destdir))
+
+    if not os.path.exists(destdir):
+        os.makedirs(destdir)
+
+    # scp the packages to the cache dir 
+    call(cmdexpand("scp -p @scp_flags @files $cachedir/", files=files,
+         scp_flags=scp_flags, cachedir=destdir))
+
+def copy_to_destdir(dir, packages, quiet = False):
+    packages.sort(cmp=sort_trackname_arch)
+    package = packages[0]
+    trackname = package.name
+    arch = package.arch
+    destdir = os.path.join(dir, arch, trackname)
+    files = []
+    for package in packages:
+        if package.trackname != trackname or \
+           package.arch != arch:
+            #copy_files(destdir, files, quiet)
+            trackname = package.trackname
+            arch = package.arch
+            destdir = os.path.join(dir, arch, trackname)
+            files = []
+        # add only if the packages differ
+        if check_package_is_new(package.name, destdir, package.md5sum):
+            files.append(package.sourcepath)
+        print "package: %s, destdir: %s" % (package.name, destdir)
 
 def copy_to_cache(variables, track, revision, quiet = False, architectures=None):
     cachedir = variables["cachedir"]
+    newpackages = []
     treepkginfo = get_treepkg_info(variables)
     #allowedarchs = set([]) # contains all wanted architectures (incl. source)
     allarchs = set([]) # contains all present architectures (incl. source)
@@ -117,12 +231,11 @@
         allowedarchs = set([get_binary_arch(a) for a in architectures])
     else:
         allowedarchs = set([])
-   
+    print "allowedarchs: %s" % allowedarchs
     for track in treepkginfo.tracks:
         for rev in track.revisions:
             for package in rev.packages:
                 arch = get_binary_arch(package.arch)
-                print "%s %s %s %s" % (track.name, package.name, package.type, package.arch)
                 if package.type == "binary":
                     # skip other files
                     if package.arch is None:
@@ -136,15 +249,15 @@
                     allarchs.add(arch)
                 elif package.type == "source":
                     arch = package.type
+                print "package: %s %s" % (package.name, arch)
                 # only copy requested archs
                 if len(allowedarchs) == 0 or \
                    arch in allowedarchs:
-                    copy_to_destdir(package, cachedir, arch, track.name,
-                                    quiet)
-
-    #print "architectures %s" % architectures
-    #print "allowed %s" % allowedarchs
-    #print "all %s" % allarchs
+                    filename = os.path.join(cachedir, arch, track.name,
+                                            package.name)
+                    newpackage = Package(filename, track.name, package.name,
+                                         package.path, arch, get_md5sum(package))
+                    newpackages.append(newpackage)
     # copy binary-all packages
     sourcearch = set(["source"])
     if len(allowedarchs) == 0:
@@ -153,10 +266,15 @@
         binallarchs = allowedarchs - sourcearch
     else:
         binallarchs = (allowedarchs & allarchs) - sourcearch
-    #print "binallarcgs %s" % binallarchs
     for package in binaryallpackages:
         for arch in binallarchs:
-            copy_to_destdir(package, cachedir, arch, package.trackname, quiet)
+            filename = os.path.join(cachedir, arch, track.name,
+                                    package.name)
+            newpackage = Package(filename, package.trackname, package.name,
+                                 package.path, arch, get_md5sum(package))
+            newpackages.append(newpackage)
+    print newpackages
+    copy_to_destdir(cachedir, newpackages, quiet)
 
 def publish_packages_arch(variables, track, revision, dist, section, 
                           quiet, architectures):