diff --git a/master/docs/examples/repo_gerrit.cfg b/master/docs/examples/repo_gerrit.cfg index b707ecad0e7..94f8a8ac148 100644 --- a/master/docs/examples/repo_gerrit.cfg +++ b/master/docs/examples/repo_gerrit.cfg @@ -71,10 +71,10 @@ from buildbot.steps.master import MasterShellCommand from buildbot.steps.transfer import FileUpload from buildbot.steps.python_twisted import Trial from buildbot.config import BuilderConfig -from buildbot.process.properties import WithProperties +from buildbot.process.properties import Interpolate -getOutputDir = WithProperties("/var/www/builds/build-%s-%s","buildername","changenumber") -getWebDir = WithProperties("http://buildmaster.mysite.com/builds/build-%s-%s","buildername","changenumber") +getOutputDir = Interpolate("/var/www/builds/build-%(prop:buildername)s-%(prop:changenumber)s") +getWebDir = Interpolate("http://buildmaster.mysite.com/builds/build-%(prop:buildername)s-%(prop:changenumber)s") builders = [] for board, manifest_file, gerrit_branch in build_branches: diff --git a/master/docs/manual/cfg-builders.rst b/master/docs/manual/cfg-builders.rst index 166c8af1e35..7f2e3fc6f62 100644 --- a/master/docs/manual/cfg-builders.rst +++ b/master/docs/manual/cfg-builders.rst @@ -120,7 +120,7 @@ Other optional keys may be set on each ``BuilderConfig``: ``properties`` A builder may be given a dictionary of :ref:`Build-Properties` specific for this builder in this parameter. Those values can be used - later on like other properties. :ref:`WithProperties`. + later on like other properties. :ref:`Interpolate`. .. index:: Builds; merging diff --git a/master/docs/manual/cfg-buildsteps.rst b/master/docs/manual/cfg-buildsteps.rst index 50e408fd883..008e365c044 100644 --- a/master/docs/manual/cfg-buildsteps.rst +++ b/master/docs/manual/cfg-buildsteps.rst @@ -1376,7 +1376,7 @@ The :bb:step:`ShellCommand` arguments are: Note that environment values must be strings (or lists that are turned into strings). In particular, numeric properties such as ``buildnumber`` must - be substituted using :ref:`WithProperties`. + be substituted using :ref:`Interpolate`. ``want_stdout`` if ``False``, stdout from the child process is discarded rather than being @@ -1856,7 +1856,7 @@ The :bb:step:`MTR` step's arguments are: ``mtr_subdir`` The subdirectory in which to look for server error log files. Defaults to - :file:`mysql-test`, which is usually correct. :ref:`WithProperties` is supported. + :file:`mysql-test`, which is usually correct. :ref:`Interpolate` is supported. .. bb:step:: SubunitShellCommand @@ -2299,7 +2299,7 @@ slave. Instead of having to create a temporary file and then use FileDownload, you can use one of the string download steps. :: from buildbot.steps.transfer import StringDownload - f.append(StringDownload(WithProperties("%(branch)s-%(got_revision)s\n"), + f.append(StringDownload(Interpolate("%(src::branch)s-%(prop:got_revision)s\n"), slavedest="buildid.txt")) :bb:step:`StringDownload` works just like :bb:step:`FileDownload` except it takes a single argument, @@ -2369,7 +2369,7 @@ Variables that don't exist on the master will be replaced by ``""``. :: Note that environment values must be strings (or lists that are turned into strings). In particular, numeric properties such as ``buildnumber`` must -be substituted using :ref:`WithProperties`. +be substituted using :ref:`Interpolate`. ``interruptSignal`` (optional) Signal to use to end the process, if the step is interrupted. @@ -2402,7 +2402,7 @@ This runs ``uname -a`` and captures its stdout, stripped of leading and trailing whitespace, in the property ``uname``. To avoid stripping, add ``strip=False``. -The ``property`` argument can be specified as a :ref:`WithProperties` +The ``property`` argument can be specified as a :ref:`Interpolate` object, allowing the property name to be built from other property values. The more advanced usage allows you to specify a function to extract @@ -2452,7 +2452,7 @@ displayed as :envvar:`TMP` in the Windows GUI. :: from buildbot.steps.shell import Compile f.addStep(SetPropertiesFromEnv(variables=["SOME_JAVA_LIB_HOME", "JAVAC"])) - f.addStep(Compile(commands=[WithProperties("%s","JAVAC"), "-cp", WithProperties("%s", "SOME_JAVA_LIB_HOME"))) + f.addStep(Compile(commands=[Interpolate("%(prop:JAVAC)s"), "-cp", Interpolate("%(prop:SOME_JAVA_LIB_HOME)s"))) Note that this step requires that the Buildslave be at least version 0.8.3. For previous versions, no environment variables are available (the slave @@ -2515,7 +2515,7 @@ Two parameters allow control of the properties that are passed to the triggered scheduler. To simply copy properties verbatim, list them in the ``copy_properties`` parameter. To set properties explicitly, use the more sophisticated ``set_properties``, which takes a dictionary mapping property -names to values. You may use :ref:`WithProperties` here to dynamically +names to values. You may use :ref:`Interpolate` here to dynamically construct new property values. RPM-Related Steps @@ -2701,7 +2701,7 @@ violations. The packages or changes file to test is specified in ``fileloc`` :: from buildbot.steps.package.deb.lintian import DebLintian - f.addStep(DebLintian(fileloc=WithProperties("%(deb-changes)s"))) + f.addStep(DebLintian(fileloc=Interpolate("%(prop:deb-changes)s"))) Miscellaneous BuildSteps ------------------------ diff --git a/master/docs/manual/cfg-properties.rst b/master/docs/manual/cfg-properties.rst index c0a921a7eca..038a57338b2 100644 --- a/master/docs/manual/cfg-properties.rst +++ b/master/docs/manual/cfg-properties.rst @@ -28,9 +28,8 @@ sources for properties are: number of properties on itself. * :bb:cfg:`builders ` -- A builder can set properties on all the builds it runs. -* :ref:`steps ` -- The steps of a build can set properties that - are available to subsequent steps. In particular, source steps set a number - of properties. +* :ref:`steps ` -- The steps of a build can set properties that are available to subsequent steps. + In particular, source steps set the `got_revision` property. If the same property is supplied in multiple places, the final appearance takes precedence. For example, a property set in a builder configuration will @@ -49,31 +48,6 @@ Common Build Properties The following build properties are set when the build is started, and are available to all steps. -.. index:: single: Properties; branch - -``branch`` - This comes from the build's :class:`SourceStamp`, and describes which branch is - being checked out. This will be ``None`` (which interpolates into - ``WithProperties`` as an empty string) if the build is on the - default branch, which is generally the trunk. Otherwise it will be a - string like ``branches/beta1.4``. The exact syntax depends upon the VC - system being used. - -.. index:: single: Properties; revision - -``revision`` - This also comes from the :class:`SourceStamp`, and is the revision of the source code - tree that was requested from the VC system. When a build is requested of a - specific revision (as is generally the case when the build is triggered by - Changes), this will contain the revision specification. This is always a - string, although the syntax depends upon the VC system in use: for SVN it is an - integer, for Mercurial it is a short string, for Darcs it is a rather large - string, etc. - - If the :guilabel:`force build` button was pressed, the revision will be ``None``, - which means to use the most recent revision available. This is a `trunk - build`. This will be interpolated as an empty string. - .. index:: single: Properties; got_revision ``got_revision`` @@ -113,6 +87,45 @@ are available to all steps. If the build was started from a scheduler, then this property will contain the name of that scheduler. + +``workdir`` + The absolute path of the base working directory on the slave, of the current + builder. + +.. index:: single: Properties; workdir + +For single codebase builds, where the codebase is `''`, the following :ref:`Source-Stamp-Attributes` are also available as properties: ``branch``, ``revision``, ``repository``, and ``project`` . + +.. _Source-Stamp-Attributes: + +Source Stamp Attributes +----------------------- + +.. index:: single: Properties; branch + +``branch`` + This comes from the build's :class:`SourceStamp`, and describes which branch is + being checked out. This will be ``None`` (which interpolates into + ``Interpolate`` as an empty string) if the build is on the + default branch, which is generally the trunk. Otherwise it will be a + string like ``branches/beta1.4``. The exact syntax depends upon the VC + system being used. + +.. index:: single: Properties; revision + +``revision`` + This also comes from the :class:`SourceStamp`, and is the revision of the source code + tree that was requested from the VC system. When a build is requested of a + specific revision (as is generally the case when the build is triggered by + Changes), this will contain the revision specification. This is always a + string, although the syntax depends upon the VC system in use: for SVN it is an + integer, for Mercurial it is a short string, for Darcs it is a rather large + string, etc. + + If the :guilabel:`force build` button was pressed, the revision will be ``None``, + which means to use the most recent revision available. This is a `trunk + build`. This will be interpolated as an empty string. + .. index:: single: Properties; repository ``repository`` @@ -123,11 +136,7 @@ are available to all steps. ``project`` The project of the sourcestamp for this build -.. index:: single: Properties; workdir -``workdir`` - The absolute path of the base working directory on the slave, of the current - builder. Using Properties in Steps ------------------------- @@ -183,18 +192,90 @@ The default value can reference other properties, e.g., :: command=Property('command', default=Property('default-command')) +.. Index:: single; Properties; Interpolate + +.. _Interpolate: + +Interpolate ++++++++++++ + +:class:`Property` can only be used to replace an entire argument: in the +example above, it replaces an argument to ``echo``. Often, properties need to +be interpolated into strings, instead. The tool for that job is +:ref:`Interpolate`. + +The more common pattern is to use python dictionary-style string interpolation by using the ``%(prop:)s`` syntax. +In this form, the property name goes in the parentheses, as above. +A common mistake is to omit the trailing "s", leading to a rather obscure error from Python ("ValueError: unsupported format character"). :: -.. Index:: single; Properties; WithProperties + from buildbot.steps.shell import ShellCommand + from buildbot.process.properties import Interpolate + f.addStep(ShellCommand(command=[ 'make', Interpolate('REVISION=%(prop:got_revision)s'), + 'dist' ])) + +This example will result in a ``make`` command with an argument like +``REVISION=12098``. + +.. _Interpolate-DictStyle: + +The syntax of dictionary-style interpolation is a selector, followed by a colon, followed by a selector specific key, optionally followed by a colon and a string indicating how to interpret the value produced by the key. + +The following selectors are supported. + +``prop`` + The key is the name of a property. + +``src`` + The key is a codebase and source stamp attribute, seperated by a colon. + +``kw`` + The key refers to a keyword argument passed to ``Interpolate``. + +The following ways of interpreting the value are available. + +``-replacement`` + If the key exists, substitute its value; otherwise, + substitute ``replacement``. ``replacement`` may be empty + (``%(prop:propname:-)s``). This is the default. + +``~replacement`` + Like ``-replacement``, but only substitutes the value + of the key if it is something Python regards as ``True``. + Python considers ``None``, 0, empty lists, and the empty string to be + false, so such values will be replaced by ``replacement``. + +``+replacement`` + If the key exists, substitute ``replacement``; otherwise, + substitute an empty string. + +``?|sub_if_exists|sub_if_missing`` + +``#?|sub_if_true|sub_if_false`` + Ternary substitution, depending on either the key being present (with + ``?``, similar to ``+``) or being ``True`` (with ``#?``, like ``~``). + Notice that there is a pipe immediately following the question mark *and* + between the two substitution alternatives. The character that follows the + question mark is used as the delimeter between the two alternatives. In the + above examples, it is a pipe, but any character other than ``(`` can be used. + + +Although these are similar to shell substitutions, no other substitutions are currently supported. + +In addition, ``Interpolate`` supports using positional string interpolation. +Here, ``%s`` is used as a placeholder, and the substitutions (which may themselves be placeholders), are given as subsequent arguments:: + +.. note: like python, you can use either positional interpolation *or* dictionary-style interpolation, not both. +Thus you cannot use a string like ``Interpolate("foo-%(src::revision)s-%s", "branch")``. + +.. index:: single; Properties; WithProperties .. _WithProperties: WithProperties ++++++++++++++ -:class:`Property` can only be used to replace an entire argument: in the -example above, it replaces an argument to ``echo``. Often, properties need to -be interpolated into strings, instead. The tool for that job is -:class:`WithProperties`. +This placeholder is deprecated. It is an older version of :ref:`Interpolate`. +It exists for compatability with older configs. The simplest use of this class is with positional string interpolation. Here, ``%s`` is used as a placeholder, and property names are given as subsequent diff --git a/master/docs/manual/cfg-statustargets.rst b/master/docs/manual/cfg-statustargets.rst index abf8983dd0f..3c398f4cdcc 100644 --- a/master/docs/manual/cfg-statustargets.rst +++ b/master/docs/manual/cfg-statustargets.rst @@ -1125,7 +1125,7 @@ MailNotifier arguments ``extraHeaders`` (dictionary) A dictionary containing key/value pairs of extra headers to add - to sent e-mails. Both the keys and the values may be a `WithProperties` instance. + to sent e-mails. Both the keys and the values may be a `Interpolate` instance. As a help to those writing :func:`messageFormatter` functions, the following table describes how to get some useful pieces of information from the various diff --git a/master/docs/manual/customization.rst b/master/docs/manual/customization.rst index 7b3bcb30928..648a8a51c58 100644 --- a/master/docs/manual/customization.rst +++ b/master/docs/manual/customization.rst @@ -510,9 +510,11 @@ repos and workdir, this will work. Advanced Property Interpolation ------------------------------- +TODO + If the simple string substitutions described in :ref:`Properties` are not sufficent, more complex substitutions can be achieved with -:class:`WithProperties` and Python functions. This only works with +:class:`Interpolate` and Python functions. This only works with dictionary-style interpolation. The function should take one argument - a properties object, described below - @@ -884,7 +886,7 @@ arbitrary object. For example:: ShellCommand.start(self) Remember that properties set in a step may not be available until the next step -begins. In particular, any :class:`Property` or :class:`WithProperties` +begins. In particular, any :class:`Property` or :class:`Interpolate` instances for the current step are interpoloated before the ``start`` method begins. @@ -922,7 +924,7 @@ build number. :: class TestWithCodeCoverage(BuildStep): command = ["make", "test", - WithProperties("buildnum=%s", "buildnumber")] + Interpolate("buildnum=%(prop:buildnumber)s")] def createSummary(self, log): buildnumber = self.getProperty("buildnumber") @@ -934,7 +936,7 @@ output by the build process itself:: class TestWithCodeCoverage(BuildStep): command = ["make", "test", - WithProperties("buildnum=%s", "buildnumber")] + Interpolate("buildnum=%(prop:buildnumber)s")] def createSummary(self, log): output = StringIO(log.getText()) diff --git a/master/docs/release-notes.rst b/master/docs/release-notes.rst index 47868596a78..e984e28da6e 100644 --- a/master/docs/release-notes.rst +++ b/master/docs/release-notes.rst @@ -30,8 +30,9 @@ Features * The ``Git`` step has a new ``getDescription`` option, which will run ``git describe`` after checkout normally. See :bb:step:`Git` for details. -* A new ternary substitution operator ``:?:`` and ``:#?:`` is available with the ``Interpolate`` - and ``WithProperties`` classes. +* A new interpolation placeholder :ref:`Interpolate`, with more regular syntax, is available. + +* A new ternary substitution operator ``:?`` and ``:#?`` is available with the ``Interpolate`` class. * ``IRenderable.getRenderingFor`` can now return a deferred. @@ -148,6 +149,34 @@ Deprecations, Removals, and Non-Compatible Changes This allows the ``split_file`` function to distinguish between files and directories. Customized split functions may need to be adjusted accordingly. +* :ref:`WithProperties` has been deprecated in favor of :ref:`Interpolate`. + `Interpolate` doesn't handle functions as keyword arguments. + The following code using ``WithProperties`` :: + + from buildbot.process.properties import WithProperties + def determine_foo(props): + if props.hasProperty('bar'): + return props['bar'] + elif props.hasProperty('baz'): + return props['baz'] + return 'qux' + WithProperties('%(foo)s', foo=determine_foo) + + can be replaced with :: + + from zope.interface import implementer + from buildbot.interfaces import IRenderable + from buildbot.process.properties import Interpolate + @implementer(IRenderable) + class determineFoo(object): + def getRenderingFor(self, props): + if props.hasProperty('bar'): + return props['bar'] + elif props.hasProperty('baz'): + return props['baz'] + return 'qux' + Interpolate('%s(kw:foo)s', foo=determineFoo()) + Changes for Developers ~~~~~~~~~~~~~~~~~~~~~~