bh@224: # Copyright (C) 2007, 2008, 2009 by Intevation GmbH bh@0: # Authors: bh@0: # Bernhard Herzog bh@0: # bh@0: # This program is free software under the GPL (>=v2) bh@0: # Read the file COPYING coming with the software for details. bh@0: bh@224: """Collection of subversion utility code""" bh@0: bh@0: import os bh@230: import shutil bh@273: import re bh@282: import StringIO bh@282: bh@282: from lxml import etree bh@0: bh@0: import run bh@45: from cmdexpand import cmdexpand bh@0: from util import extract_value_for_key bh@0: bh@0: bh@269: class SubversionError(Exception): bh@302: bh@302: """Base class for subversion specific errors raised by TreePKG""" bh@302: bh@302: bh@303: class SubversionUrlMismatchError(SubversionError): bh@303: bh@303: """The repository URL does not match the URL of a working copy""" bh@303: bh@269: bh@262: def list_url(url): bh@262: """Runs svn list with the given url and returns files listed as a list""" bh@262: output = run.capture_output(cmdexpand("svn list $url", **locals())) bh@262: return output.splitlines() bh@262: bh@208: def checkout(url, localdir, revision=None, recurse=True): bh@0: """Runs svn to checkout the repository at url into the localdir""" bh@208: args = [] bh@208: if revision: bh@208: args.extend(["--revision", revision]) bh@208: if not recurse: bh@208: args.append("-N") bh@208: run.call(cmdexpand("svn checkout -q @args $url $localdir", **locals())) bh@0: bh@208: def update(localdir, revision=None, recurse=True): bh@79: """Runs svn update on the localdir. bh@79: The parameter revision, if given, is passed to svn as the value of bh@79: the --revision option. bh@79: """ bh@208: args = [] bh@79: if revision: bh@208: args.extend(["--revision", revision]) bh@208: if not recurse: bh@208: args.append("-N") bh@208: run.call(cmdexpand("svn update -q @args $localdir", **locals())) bh@0: bh@266: def export(src, dest, revision=None, recurse=True): bh@0: """Runs svn export src dest""" bh@266: args = [] bh@266: if revision: bh@266: args.extend(["--revision", revision]) bh@266: if not recurse: bh@266: args.append("-N") bh@266: run.call(cmdexpand("svn export -q @args $src $dest", **locals())) bh@0: bh@0: def last_changed_revision(svn_working_copy): bh@0: """return the last changed revision of an SVN working copy as an int""" bh@0: # Make sure we run svn under the C locale to avoid localized bh@0: # messages bh@0: env = os.environ.copy() bh@0: env["LANG"] = "C" bh@0: bh@45: output = run.capture_output(cmdexpand("svn info $svn_working_copy", bh@45: **locals()), bh@45: env=env) bh@269: str_rev = extract_value_for_key(output.splitlines(), "Last Changed Rev:") bh@269: if str_rev is None: bh@269: raise SubversionError("Cannot determine last changed revision for %r" bh@269: % svn_working_copy) aheinecke@333: return str_rev bh@224: bh@303: def svn_url(url_or_working_copy): bh@303: """Returns the URL used for the working copy in svn_working_copy""" bh@303: # Make sure we run svn under the C locale to avoid localized bh@303: # messages bh@303: env = os.environ.copy() bh@303: env["LANG"] = "C" bh@303: bh@303: output = run.capture_output(cmdexpand("svn info $url_or_working_copy", bh@303: **locals()), bh@303: env=env) bh@303: return extract_value_for_key(output.splitlines(), "URL:") bh@303: bh@282: def log_xml(url, base_revision): bh@282: """Return the log in XML of the repository at url from base_revision to HEAD bh@282: """ bh@282: args = ["--revision", str(base_revision) + ":HEAD", bh@282: "--verbose", bh@282: "--xml"] bh@282: return run.capture_output(cmdexpand("svn log @args $url", **locals())) bh@282: bh@282: bh@282: def extract_tag_revisions(xml_log): bh@282: """Extracts the revisions which changed an SVN tag since its creation bh@282: This includes the revision which created the tag and all subsequent bh@282: changes. The xml_log parameter should contain the xml-Version of bh@282: the SVN log of the tag that includes at least the revision that bh@282: created the tag and all the newer revisions. bh@282: """ bh@282: tree = etree.parse(StringIO.StringIO(xml_log)) bh@282: tag_revisions = tree.xpath("logentry/@revision" bh@282: "[.>=../../logentry/@revision" bh@282: "[../paths/path[@copyfrom-path]]]") bh@282: return tag_revisions bh@282: bh@282: bh@224: bh@224: class SvnRepository(object): bh@224: bh@224: """Describes a subversion repository""" bh@224: bh@304: def __init__(self, url, external_subdirs=(), subset=()): bh@224: """Initialize the subversion repository description bh@224: Parameters: bh@224: url -- The url of the repository bh@304: bh@224: external_subdirs -- A list of subdirectories which are managed bh@224: by svn externals definitions bh@304: bh@304: subset -- A sequence of (filename, recurse) pairs where bh@304: filename is a filename (usually a directory name) bh@304: relative to url and recurse should be a boolean bh@304: indicating whether checkout filename with recursion. bh@304: If recurse is False, svn checkout/export will be bh@304: called with the -N option. bh@304: bh@304: The first item in subset should be for '.', which bh@304: indicates the top-level directory under url. If a bh@304: non-empty subset is given this will usually be bh@304: ('.', False) so that the top-level directory is not bh@304: checked out recursively. bh@224: """ bh@224: self.url = url bh@224: self.external_subdirs = external_subdirs bh@304: if not subset: bh@304: # default subset is to checkout the top-level directory at bh@304: # URL recursively. Alwas having a subset makes the code bh@304: # simpler bh@304: subset = [(".", True)] bh@304: self.subset = subset bh@224: bh@224: def checkout(self, localdir, revision=None): bh@304: """Checks out the repository into localdir. bh@304: The revision parameter should be an int and indicates the bh@304: revision to check out or it should be None to indicate that the bh@304: newest version is to be checked out. bh@224: """ bh@304: base_url = self.url bh@304: if not base_url.endswith("/"): bh@304: base_url += "/" bh@304: subdir, recurse = self.subset[0] bh@304: checkout(base_url + subdir, os.path.join(localdir, subdir), bh@304: revision=revision, recurse=recurse) bh@304: for subdir, recurse in self.subset[1:]: bh@304: update(os.path.join(localdir, subdir), revision=revision, bh@304: recurse=recurse) bh@304: if len(self.subset) > 1 and revision is None: bh@304: # do an additional update on the whole working copy after bh@304: # creating a subset checkout so that svn info will show bh@304: # revision numbers that match the entire working copy bh@304: # (externals are handled elsewhere). The repository might bh@304: # have been changed between the initial checkout of the bh@304: # top-level directory and the updates for the bh@304: # subdirectories. bh@304: update(localdir, revision=revision) bh@224: bh@224: def export(self, localdir, destdir): bh@224: """Exports the working copy in localdir to destdir""" bh@224: export(localdir, destdir) bh@224: for subdir in self.external_subdirs: bh@224: absdir = os.path.join(destdir, subdir) bh@224: if not os.path.isdir(absdir): bh@228: export(os.path.join(localdir, subdir), absdir) bh@224: bh@311: def export_tag(self, url, destdir, revision=None): bh@311: """Exports the tag at url to destdir. bh@311: Note: the implementation of this method would work for any URL bh@311: but it really is intended to only be used for URLs to tags of bh@311: the same code as represented by this object. bh@311: """ bh@311: base_url = url bh@311: if not base_url.endswith("/"): bh@311: base_url += "/" bh@311: for subdir, recurse in self.subset: bh@311: export(base_url + "/" + subdir, os.path.join(destdir, subdir), bh@311: revision=revision, recurse=recurse) bh@311: bh@224: def last_changed_revision(self, localdir): bh@224: """Returns the last changed revision of the working copy in localdir""" aheinecke@343: max_rev = max([int(last_changed_revision(os.path.join(localdir, d))) bh@224: for d in [localdir] + list(self.external_subdirs)]) aheinecke@343: return str(max_rev) bh@224: bh@303: def check_working_copy(self, localdir): bh@303: """Checks whether localdir contains a checkout of the bh@303: repository. The check compares the expected URL with the one bh@303: returned by svn info executed in localdir. Raises bh@303: SubversionUrlMismatchError if the URLs do not match. bh@303: """ bh@303: localurl = svn_url(localdir) bh@303: expected_url = svn_url(self.url) bh@303: if localurl != expected_url: bh@303: raise SubversionUrlMismatchError("Working copy in %r has URL %r," bh@303: " expected %r" bh@303: % (localdir, localurl, bh@303: expected_url)) bh@303: bh@224: bh@224: class SvnWorkingCopy(object): bh@224: bh@224: """Represents a checkout of a subversion repository""" bh@224: bh@224: def __init__(self, repository, localdir, logger=None): bh@224: """ bh@224: Initialize the working copy. bh@224: Parameters: bh@224: repository -- The SvnRepository instance describing the bh@224: repository bh@224: localdir -- The directory for the working copy bh@224: logger -- logging object to use for some info/debug messages bh@224: """ bh@224: self.repository = repository bh@224: self.localdir = localdir bh@224: self.logger = logger bh@224: bh@224: def log_info(self, *args): bh@224: if self.logger is not None: bh@224: self.logger.info(*args) bh@224: bh@224: def update_or_checkout(self, revision=None): bh@224: """Updates the working copy or creates by checking out the repository""" bh@224: if os.path.exists(self.localdir): bh@224: self.log_info("Updating the working copy in %r", self.localdir) bh@303: self.repository.check_working_copy(self.localdir) bh@224: update(self.localdir, revision=revision) bh@224: else: bh@224: self.log_info("The working copy in %r doesn't exist yet." bh@224: " Checking out from %r", bh@224: self.localdir, self.repository.url) bh@224: self.repository.checkout(self.localdir, revision=revision) bh@224: bh@224: def export(self, destdir): bh@224: """Exports the working copy to destdir""" bh@224: self.repository.export(self.localdir, destdir) bh@224: bh@311: def export_tag(self, url, destdir, revision=None): bh@311: """Exports the tag at url to destdir. bh@311: The URL is expected to point to the same repository as the one bh@311: used by the working copy and is intended to be used when bh@311: exporting tagged versions of the code in the working copy. It's bh@311: a method on the working copy so that the repository description bh@311: including the subset settings are used. bh@311: """ bh@311: self.repository.export_tag(url, destdir, revision=revision) bh@311: bh@224: def last_changed_revision(self): bh@224: """Returns the last changed rev of the working copy""" bh@224: return self.repository.last_changed_revision(self.localdir) bh@230: bh@230: bh@230: class ManualWorkingCopy(object): bh@230: bh@230: """A manually managed working copy""" bh@230: bh@230: def __init__(self, directory): bh@230: self.directory = directory bh@230: bh@230: def update_or_checkout(self, revision=None, recurse=True): bh@230: """This method does nothing""" bh@230: pass bh@230: bh@230: def export(self, destdir): bh@230: """Copies the entire working copy to destdir""" bh@230: shutil.copytree(self.directory, destdir) bh@230: bh@230: def last_changed_revision(self): bh@230: """Always returns 0""" aheinecke@331: return "0" bh@273: bh@273: bh@273: class TagDetector(object): bh@273: bh@273: """Class to automatically find SVN tags and help package them bh@273: bh@273: The tags are found using three parameters: bh@273: url -- The base url of the SVN tags directory to use bh@273: pattern -- A regular expression matching the subdirectories to bh@273: consider in the tag directory specified by the url bh@273: subdir -- A subdirectory of the directory matched by pattern to bh@273: export and use to determine revision number bh@273: bh@273: The subdir parameter is there to cope with the kdepim enterprise bh@273: tags. The URL for a tag is of the form bh@273: .../tags/kdepim/enterprise4.0.. . Each such tag has bh@273: subdirectories for kdepim, kdelibs, etc. The url and pattern are bh@273: used to match the URL for the tag, and the subdir is used to select bh@273: which part of the tag is meant. bh@273: bh@273: The subdir also determines which SVN directory's revision number is bricks@369: used. bh@273: """ bh@273: bh@273: def __init__(self, url, pattern, subdir): bh@273: self.url = url bh@273: self.pattern = re.compile(pattern) bh@273: self.subdir = subdir bh@273: bh@273: def list_tags(self): bh@273: matches = [] bh@273: if self.url: bh@273: for tag in list_url(self.url): bh@273: if self.pattern.match(tag.rstrip("/")): bh@273: matches.append(tag) bh@273: return sorted(matches) bh@273: bh@273: def newest_tag_revision(self): bh@273: """Determines the newest tag revision and returns (tagurl, revno) bh@273: If no tag can be found, the method returns the tuple (None, None). bh@273: """ bh@273: candidates = self.list_tags() bh@273: urlrev = (None, None) bh@273: if candidates: bh@273: newest = candidates[-1] bh@273: urlrev = self.determine_revision(self.url + "/" + newest, bh@273: self.subdir) bh@273: return urlrev bh@273: bh@273: def determine_revision(self, baseurl, subdir): bh@273: urlrev = (None, None) bricks@369: revision_url = baseurl + "/" + subdir bh@273: try: bh@273: revision = last_changed_revision(revision_url) bh@273: urlrev = (baseurl + "/" + subdir, revision) bh@273: except SubversionError: bh@273: pass bh@273: return urlrev bricks@525: bricks@540: def log_xml(self, url): bricks@540: """Return the log in XML of the repository since the copy bricks@540: """ bricks@540: args = ["--stop-on-copy", bricks@540: "--verbose", bricks@540: "--xml"] bricks@540: return run.capture_output(cmdexpand("svn log @args $url", **locals())) bricks@540: bricks@525: def tag_pkg_parameters(self, tag_url): bricks@525: # FIXME: Don't hardcore svn tag path and regex bricks@525: match = re.search(r"/enterprise[^.]*\.[^.]*\." bricks@525: r"(?P[0-9]{8})\.(?P[0-9]+)/", bricks@525: tag_url) bricks@525: if match: bricks@525: date = match.group("date") bricks@540: # baserev is time since git migration bricks@525: baserev = match.group("baserev") bricks@540: xml_log = self.log_xml(tag_url) bricks@530: revisions = extract_tag_revisions(xml_log) bricks@525: tag_change_count = len(revisions) bricks@525: return (date, tag_change_count) bricks@525: else: bricks@525: raise RuntimeError("Cannot determine tag parameters from %s" bricks@525: % tag_url)