Mercurial > treepkg
changeset 66:3c2e8cb7e658
merge
| author | Bernhard Herzog <bh@intevation.de> |
|---|---|
| date | Wed, 11 Apr 2007 17:05:48 +0200 |
| parents | bebe06ff4bd6 (current diff) 353912e12d53 (diff) |
| children | 517253fa987f |
| files | |
| diffstat | 5 files changed, 270 insertions(+), 19 deletions(-) [+] |
line wrap: on
line diff
--- a/enterprise/kdei18n.py Wed Apr 11 17:03:33 2007 +0200 +++ b/enterprise/kdei18n.py Wed Apr 11 17:05:48 2007 +0200 @@ -13,6 +13,7 @@ import treepkg.packager import treepkg.util import treepkg.run as run +from treepkg.cmdexpand import cmdexpand class SourcePackager(treepkg.packager.SourcePackager): @@ -50,7 +51,8 @@ def unpack_orig_tarball(self): orig_tarball = self.track.orig_tarball - run.call(["tar", "xjf", orig_tarball, "-C", self.work_dir]) + run.call(cmdexpand("tar xjf $tarball -C $directory", + tarball=orig_tarball, directory=self.work_dir)) tarbasename = os.path.basename(orig_tarball) splitext = os.path.splitext return os.path.join(self.work_dir, @@ -71,7 +73,7 @@ treepkg.util.copytree(untarred_dir, de_dir) treepkg.util.copytree(new_de_dir, de_dir) logging.info("Running scripts/autogen.sh for kde-i18n-de tarball") - run.call(["/bin/sh", "scripts/autogen.sh", "de"], cwd=pkgbasedir, + run.call(cmdexpand("/bin/sh scripts/autogen.sh de"), cwd=pkgbasedir, suppress_output=True) tarballdir = "kde-i18n-de-" + pkgbaseversion @@ -79,7 +81,8 @@ tarball = os.path.join(os.path.dirname(pkgbasedir), tarballdir + ".tar.bz2") - run.call(["tar", "cjf", tarball, "-C", pkgbasedir, tarballdir]) + run.call(cmdexpand("tar cjf $tarball -C $pkgbasedir $tarballdir", + **locals())) logging.info("Created kde-i18n-de tarball") return tarball
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test/test_cmdexpand.py Wed Apr 11 17:05:48 2007 +0200 @@ -0,0 +1,124 @@ +# Copyright (C) 2007 by Intevation GmbH +# Authors: +# Bernhard Herzog <bh@intevation.de> +# +# This program is free software under the GPL (>=v2) +# Read the file COPYING coming with the software for details. + +"""Tests for the cmdexpand function""" + +import unittest + +from treepkg.cmdexpand import cmdexpand + + +class TestCMDExpand(unittest.TestCase): + + def test_words(self): + """Test cmdexpand with simple whitespace separated words""" + self.assertEquals(cmdexpand("abc defg xyz zy"), + ['abc', 'defg', 'xyz', 'zy']) + + def test_single_quoted(self): + """Test cmdexpand with some single quoted words""" + self.assertEquals(cmdexpand("abc 'defg xyz' zy"), + ['abc', 'defg xyz', 'zy']) + + def test_double_quoted(self): + """Test cmdexpand with some double quoted words""" + self.assertEquals(cmdexpand('abc "defg xyz" zy'), + ['abc', 'defg xyz', 'zy']) + + def test_word_expansion(self): + """Test cmdexpand with simple word expansion""" + self.assertEquals(cmdexpand('abc $foo ghi', foo="def"), + ['abc', 'def', 'ghi']) + self.assertEquals(cmdexpand('abc $foo ghi $bar', foo="def", bar="X"), + ['abc', 'def', 'ghi', 'X']) + + def test_word_expansion_braced_name(self): + """Test cmdexpand with word expansion using braced names""" + self.assertEquals(cmdexpand('abc ${foo} x${foo}y ghi', foo="def"), + ['abc', 'def', 'xdefy', 'ghi']) + + def test_word_expansion_non_byte_string(self): + """Test cmdexpand quoting of dollar signs""" + self.assertEquals(cmdexpand('abc $foo $bar ghi', foo=123, bar=u"1 2 3"), + ['abc', '123', '1 2 3', 'ghi']) + + def test_word_expansion_non_identifier(self): + """Test cmdexpand word expansion if dollar not followed by identifier""" + # $ immediately followed by a non-identifier character + self.assertRaises(ValueError, cmdexpand, 'abc $#foo bar', foo="def") + + def test_word_expansion_inside_words(self): + """Test cmdexpand word expansions in parts of words""" + self.assertEquals(cmdexpand("$foo x$bar y$baz.", + foo="abc", bar="yz", baz="zx"), + ["abc", "xyz", "yzx."]) + self.assertEquals(cmdexpand("$foo x$bar-$baz.", + foo="abc", bar="yz", baz="zx"), + ["abc", "xyz-zx."]) + + def test_case_sensitivity(self): + """Test case sensitivity of expansion keys""" + self.assertEquals(cmdexpand('abc $foo $Foo $FOO', + foo="def", Foo="DEF", FOO="Def"), + ['abc', 'def', 'DEF', 'Def']) + + def test_list_expansion(self): + """Test cmdexpand with list expansion""" + self.assertEquals(cmdexpand('abc @foo ghi', foo=["d", "e", "f"]), + ['abc', 'd', 'e', 'f', 'ghi']) + + def test_list_expansion_non_string(self): + """Test cmdexpand with list expansion""" + self.assertEquals(cmdexpand('abc @foo ghi', foo=[1, 1.0, None]), + ['abc', '1', '1.0', 'None', 'ghi']) + + def test_list_expansion_with_iterators(self): + """Test cmdexpand with list expansion using an iterator""" + self.assertEquals(cmdexpand('abc @foo ghi', + foo=(i**2 for i in range(3))), + ['abc', '0', '1', '4', 'ghi']) + + def test_list_expansion_non_identifier(self): + """Test cmdexpand with at-sign not followed by identifier""" + # @+identifier do not cover entire word + self.assertRaises(ValueError, cmdexpand, 'abc @foo, ghi', + foo=["d", "e", "f"]) + + # @ immediately followed by a non-identifier character + self.assertRaises(ValueError, cmdexpand, 'abc @. z') + + def test_list_expansion_inside_word(self): + """Test whether cmdexpand raises ValueError for at-signs inside words""" + self.assertRaises(ValueError, cmdexpand, 'abc x@foo ghi', + foo=["d", "e", "f"]) + + + def test_dollar_quoting(self): + """Test cmdexpand quoting of dollar signs""" + self.assertEquals(cmdexpand('abc $$foo $foo g$$hi', foo="def"), + ['abc', '$foo', 'def', 'g$hi']) + + def test_atsign_quoting(self): + """Test cmdexpand quoting of at-signs""" + self.assertEquals(cmdexpand('abc @foo $@foo g$@i', foo=["d", "e", "f"]), + ['abc', 'd', 'e', 'f', '@foo', 'g@i']) + + def test_interaction_with_shlex_quoting(self): + """Test cmdexpand's interaction with shlex's quoting""" + # Unlike unix-shells the expansion isn't influenced much by + # shell quoting as supported by shlex. + self.assertEquals(cmdexpand('abc "@foo" \'@foo\' ghi', + foo=["d", "e", "f"]), + ['abc', 'd', 'e', 'f', 'd', 'e', 'f', 'ghi']) + self.assertEquals(cmdexpand('abc "$foo" \'$foo\' ghi', foo="def"), + ['abc', 'def', 'def', 'ghi']) + self.assertEquals(cmdexpand('abc " $foo" \'a $foo\' ghi', foo="def"), + ['abc', ' def', 'a def', 'ghi']) + + +if __name__ == "__main__": + unittest.main()
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/treepkg/cmdexpand.py Wed Apr 11 17:05:48 2007 +0200 @@ -0,0 +1,116 @@ +# Copyright (C) 2007 by Intevation GmbH +# Authors: +# Bernhard Herzog <bh@intevation.de> +# +# This program is free software under the GPL (>=v2) +# Read the file COPYING coming with the software for details. + +"""Shell like string splitting and expansion""" + +import re +import shlex + + +# helper for the other regular expression matching a python identifier +match_identifier = "[_a-zA-Z][_a-zA-Z0-9]*" + +# regular expression to use for word expansion matching a dollar +# followed by exactly one of these: +# a) another dollar sign or the at-sign (for quoting of these characters) +# b) a python identifier +# c) a python identifier enclosed in braces +# d) something else which indicates invalid use of the dollar sign +rx_word_expansion = re.compile(r"\$((?P<delim>[$@])" + r"|(?P<named>%(identifier)s)" + r"|\{(?P<braced>%(identifier)s)\}" + r"|(?P<invalid>))" + % dict(identifier=match_identifier)) + +# regular expression matching an entire word that has to be list +# expanded. The regex matches if the word starts with an at-sign. The +# part of the word that followes the at-sign either matches an +# identifier with the named group "named" or anything else which +# indicates invalid use the at-sign. +rx_list_expansion = re.compile(r"^@((?P<named>%(identifier)s)|(?P<invalid>.+))$" + % dict(identifier=match_identifier)) + +# match an unquoted at-sign. +rx_unquoted_at = re.compile("[^$]@") + +def expandword(word, mapping): + def replacment(match): + key = match.group("named") or match.group("braced") + if key: + return str(mapping[key]) + + delim = match.group("delim") + if delim: + return delim + + # otherwise invalid has matched and we raise a value error + assert match.group("invalid") != None + raise ValueError + + return rx_word_expansion.sub(replacment, word) + +def cmdexpand(string, **kw): + """Split the string into 'words' and expand variable references. + +The string is first split into words with shlex.split. Each of the +words is then subjected to either word expansion or list expansion. +Word expansion is very similar to what the Template class in Python's +string module provides: + + '$$' is expanded to '$' + + '$@' is expanded to '@' + + '$identifier' is expanded to the value of the variable given by + identifier. The identifier has the same syntax as a normal Python + identifier. The identifier stops at the first non-identifier + character. The value is converted to a string with str. + + '${identifier}' is treated like '$identifier' and provides a way to + delimit the identifier in cases where the identifier is followed by + characters that would otherwise be interpreted as part of the + identifier. + +A word will remain a single word after the expansion even if the +expanded string would be treated as multiple words by shlex. + +A list expansion is applied to words that consist of a '@' followed by +an identifier. Nothing else must be in the word. The variable the +identifier refers to must be a sequence and the word will be replaced by +the sequence with each element of the sequence converted to a string +with str. + +The variables known to the function are the keyword arguments. + +Examples: + + >>> from cmdexpand import cmdexpand + >>> cmdexpand("ssh $user$@$host", user="john", host="python") + ['ssh', 'john@python'] + + >>> cmdexpand("scp @files $user$@$host:$remotedir", user="john", + ... host="python", files=["main.py", "cmdexpand.py"], + ... remotedir="/home/john/files") + ['scp', 'main.py', 'cmdexpand.py', 'john@python:/home/john/files'] +""" + words = shlex.split(string) + for index, word in reversed(list(enumerate(words))): + match = rx_unquoted_at.search(word) + if match: + raise ValueError("%r contains an unquoted '@'" % word) + match = rx_list_expansion.match(word) + if match: + key = match.group("named") + if key: + words[index:index + 1] = (str(item) for item in kw[key]) + else: + assert match.group("invalid") != None + raise ValueError("In %r the characters after the '@'" + " do not match a python identifier" % word) + else: + words[index] = expandword(word, kw) + return words
--- a/treepkg/packager.py Wed Apr 11 17:03:33 2007 +0200 +++ b/treepkg/packager.py Wed Apr 11 17:05:48 2007 +0200 @@ -19,7 +19,7 @@ import subversion import run import status - +from cmdexpand import cmdexpand def _filenameproperty(relative_dir): def get(self): @@ -94,7 +94,8 @@ filename directly in workdir. """ logging.info("Creating tarball %r", tarballname) - run.call(["tar", "czf", tarballname, "-C", workdir, basedir]) + run.call(cmdexpand("tar czf $tarballname -C $workdir $basedir", + **locals())) def copy_debian_directory(self, pkgbasedir, pkgbaseversion, changemsg): """Copies the debian directory and updates the copy's changelog @@ -121,17 +122,18 @@ oldversionprefix = oldversion.split(":")[0] + ":" else: oldversionprefix = "" - run.call(["debchange", "-c", changelog, - "-v", oldversionprefix + pkgbaseversion + "-kk1", - changemsg], + run.call(cmdexpand("debchange -c $changelog" + " -v ${oldversionprefix}${pkgbaseversion}-kk1" + " $changemsg", **locals()), env=self.track.debian_environment()) def create_source_package(self, pkgbasedir, origtargz): """Creates a new source package from pkgbasedir and origtargz""" logging.info("Creating new source package") - run.call(["dpkg-source", "-b", os.path.basename(pkgbasedir), - os.path.basename(origtargz)], + run.call(cmdexpand("dpkg-source -b $directory $tarball", + directory=os.path.basename(pkgbasedir), + tarball=os.path.basename(origtargz)), cwd=os.path.dirname(pkgbasedir), suppress_output=True, env=self.track.debian_environment()) @@ -185,11 +187,14 @@ self.status.creating_binary_package() util.ensure_directory(self.binary_dir) logging.info("Building binary package; logging to %r", self.logfile) - cmd = ["/usr/sbin/pbuilder", "build", - "--configfile", self.track.pbuilderrc, - "--logfile", self.logfile, - "--buildresult", self.binary_dir, self.dsc_file] - run.call(self.track.root_cmd + cmd, suppress_output=True) + run.call(cmdexpand("@rootcmd /usr/sbin/pbuilder build" + " --configfile $pbuilderrc" + " --logfile $logfile --buildresult $bindir $dsc", + rootcmd=self.track.root_cmd, + pbuilderrc=self.track.pbuilderrc, + logfile=self.logfile, bindir=self.binary_dir, + dsc=self.dsc_file), + suppress_output=True) self.status.binary_package_created()
--- a/treepkg/subversion.py Wed Apr 11 17:03:33 2007 +0200 +++ b/treepkg/subversion.py Wed Apr 11 17:05:48 2007 +0200 @@ -10,20 +10,21 @@ import os import run +from cmdexpand import cmdexpand from util import extract_value_for_key def checkout(url, localdir): """Runs svn to checkout the repository at url into the localdir""" - run.call(["svn", "checkout", "-q", url, localdir]) + run.call(cmdexpand("svn checkout -q $url $localdir", **locals())) def update(localdir): """Runs svn update on the localdir""" - run.call(["svn", "update", "-q", localdir]) + run.call(cmdexpand("svn update -q $localdir", **locals())) def export(src, dest): """Runs svn export src dest""" - run.call(["svn", "export", "-q", src, dest]) + run.call(cmdexpand("svn export -q $src $dest", **locals())) def last_changed_revision(svn_working_copy): """return the last changed revision of an SVN working copy as an int""" @@ -32,6 +33,8 @@ env = os.environ.copy() env["LANG"] = "C" - output = run.capture_output(["svn", "info", svn_working_copy], env=env) + output = run.capture_output(cmdexpand("svn info $svn_working_copy", + **locals()), + env=env) return int(extract_value_for_key(output.splitlines(), "Last Changed Rev:"))