Merge branch 'master' into nine

* master: (24 commits)
  update docs regarding empty repository string - refs #913.
  remove duplicate tests
  clarify docs as to how :#? and :? differ
  Apply some editing to the concepts documentation
  ignore detritus from 'python setup.py test'
  ternary-sub: add docs
  Fixes #2165: Add 'ternary' interpolation for WithProperties
  git-describe: update release notes; also fix a misplaced note for the 'descriptionSuffix' change
  move 'Setting Authorized Web Users' down a bit, add a link
  git-describe: update docs; adjust behavior of 'dict()'
  Add tutorial tour section on adjusting the authorized users in the webstatus
  Fixes #2180: Allow for "git describe" in addition to got_revision
  another timezeone-sensitive test
  fix timezone-sensitive test
  add a 'mapping' test for Jinja
  Add support for tox (http://tox.testrun.org/) for quick local testing across multiple Pythons.
  Make `python setup.py test` work (in a fresh virtualenv!) for buildbot/master
  Make `python setup.py test` do something useful
  add 'project' support to svnpoller
  Add docs for the 'exception' and 'all' mail modes
  ...

.gitignore

@@ -39,3 +39,5 @@ apidocs/reference.tgz
 common/googlecode_upload.py
 master/docs/tutorial/_build
 _build
+.tox
+master/setuptools_trial*

master/buildbot/changes/svnpoller.py

@@ -53,7 +53,7 @@ class SVNPoller(base.PollingChangeSource, util.ComparableMixin):
     """
 
     compare_attrs = ["svnurl", "split_file",
-                     "svnuser", "svnpasswd",
+                     "svnuser", "svnpasswd", "project",
                      "pollInterval", "histmax",
                      "svnbin", "category", "cachepath"]
 

master/buildbot/process/properties.py

@@ -185,6 +185,17 @@ class _PropertyMap(object):
     colon_minus_re = re.compile(r"(.*):-(.*)")
     colon_tilde_re = re.compile(r"(.*):~(.*)")
     colon_plus_re = re.compile(r"(.*):\+(.*)")
+
+    colon_ternary_re = re.compile(r"""(?P<prop>.*) # the property to match
+                                      :            # colon
+                                      (?P<alt>\#)? # might have the alt marker '#'
+                                      \?           # question mark
+                                      (?P<delim>.) # the delimiter
+                                      (?P<true>.*) # sub-if-true
+                                      (?P=delim)   # the delimiter again
+                                      (?P<false>.*)# sub-if-false
+                                      """, re.VERBOSE)
+
     def __init__(self, properties):
         # use weakref here to avoid a reference loop
         self.properties = weakref.ref(properties)
@@ -225,10 +236,38 @@ def colon_plus(mo):
             else:
                 return ''
 
+        def colon_ternary(mo):
+            # %(prop:?:T:F)s
+            # if prop exists, use T; otherwise, F
+            # %(prop:#?:T:F)s
+            # if prop is true, use T; otherwise, F
+            groups = mo.groupdict()
+
+            prop = groups['prop']
+
+            if prop in self.temp_vals:
+                if groups['alt']:
+                    use_true = self.temp_vals[prop]
+                else:
+                    use_true = True
+            elif properties.has_key(prop):
+                if groups['alt']:
+                    use_true = properties[prop]
+                else:
+                    use_true = True
+            else:
+                use_true = False
+
+            if use_true:
+                return groups['true']
+            else:
+                return groups['false']
+
         for regexp, fn in [
             ( self.colon_minus_re, colon_minus ),
             ( self.colon_tilde_re, colon_tilde ),
             ( self.colon_plus_re, colon_plus ),
+            ( self.colon_ternary_re, colon_ternary ),
             ]:
             mo = regexp.match(key)
             if mo:
@@ -436,20 +475,45 @@ def _parseColon_plus(self, d, kw, repl):
                defaultWhenFalse=False,
                elideNoneAs='')
 
+    colon_ternary_re = re.compile(r"""(?P<delim>.) # the delimiter
+                                      (?P<true>.*) # sub-if-true
+                                      (?P=delim)   # the delimiter again
+                                      (?P<false>.*)# sub-if-false
+                                      """, re.VERBOSE)
+
+    def _parseColon_ternary(self, d, kw, repl, defaultWhenFalse=False):
+        m = self.colon_ternary_re.match(repl)
+        if not m:
+            config.error("invalid Interpolate ternary expression for selector '%s' and delim '%s'" % (kw, repl[0]))
+            return None
+        m = m.groupdict()
+        return _Lookup(d, kw,
+               hasKey=Interpolate(m['true'], **self.kwargs),
+               default=Interpolate(m['false'], **self.kwargs),
+               defaultWhenFalse=defaultWhenFalse,
+               elideNoneAs='')
+
+    def _parseColon_ternary_hash(self, d, kw, repl):
+        return self._parseColon_ternary(d, kw, repl, defaultWhenFalse=True)
+
     def _parse(self, fmtstring):
         keys = _getInterpolationList(fmtstring)
         for key in keys:
             if not self.interpolations.has_key(key):
                 d, kw, repl = self._parseSubstitution(key)
                 if repl is None:
                     repl = '-'
-                for char, fn in [
+                for pattern, fn in [
                     ( "-", self._parseColon_minus ),
                     ( "~", self._parseColon_tilde ),
                     ( "+", self._parseColon_plus ),
+                    ( "?", self._parseColon_ternary ),
+                    ( "#?", self._parseColon_ternary_hash )
                     ]:
-                    if repl[0] == char:
-                        self.interpolations[key] = fn(d, kw, repl[1:])
+                    junk, matches, tail = repl.partition(pattern)
+                    if not junk and matches:
+                        self.interpolations[key] = fn(d, kw, tail)
+                        break
                 if not self.interpolations.has_key(key):
                     config.error("invalid Interpolate default type '%s'" % repl[0])
 

master/buildbot/status/mail.py

@@ -49,7 +49,7 @@
 from buildbot import interfaces, util, config
 from buildbot.process.users import users
 from buildbot.status import base
-from buildbot.status.results import FAILURE, SUCCESS, WARNINGS, Results
+from buildbot.status.results import FAILURE, SUCCESS, WARNINGS, EXCEPTION, Results
 
 VALID_EMAIL = re.compile("[a-zA-Z0-9\.\_\%\-\+]+@[a-zA-Z0-9\.\_\%\-]+.[a-zA-Z]{2,6}")
 
@@ -92,6 +92,8 @@ def defaultMessage(mode, name, build, results, master_status):
             text += "The Buildbot has detected a restored build"
         else:
             text += "The Buildbot has detected a passing build"
+    elif results == EXCEPTION:
+        text += "The Buildbot has detected a build exception"
 
     projects = []
     if ss_list:
@@ -177,7 +179,7 @@ class MailNotifier(base.StatusReceiverMultiService):
                      "subject", "sendToInterestedUsers", "customMesg",
                      "messageFormatter", "extraHeaders"]
 
-    possible_modes = ("change", "failing", "passing", "problem", "warnings")
+    possible_modes = ("change", "failing", "passing", "problem", "warnings", "exception")
 
     def __init__(self, fromaddr, mode=("failing", "passing", "warnings"),
                  categories=None, builders=None, addLogs=False,
@@ -220,6 +222,8 @@ def __init__(self, fromaddr, mode=("failing", "passing", "warnings"),
                      - "problem": send mail about a build which failed
                                   when the previous build passed
                      - "warnings": send mail if a build contain warnings
+                     - "exception": send mail if a build fails due to an exception
+                     - "all": always send mail
                      Defaults to ("failing", "passing", "warnings").
 
         @type  builders: list of strings
@@ -316,7 +320,7 @@ def __init__(self, fromaddr, mode=("failing", "passing", "warnings"),
         self.fromaddr = fromaddr
         if isinstance(mode, basestring):
             if mode == "all":
-                mode = ("failing", "passing", "warnings")
+                mode = ("failing", "passing", "warnings", "exception")
             elif mode == "warnings":
                 mode = ("failing", "warnings")
             else:
@@ -432,6 +436,8 @@ def isMailNeeded(self, build, results):
                 return True
         if "warnings" in self.mode and results == WARNINGS:
             return True
+        if "exception" in self.mode and results == EXCEPTION:
+            return True
 
         return False
 

master/buildbot/status/web/base.py

@@ -514,6 +514,8 @@ def createJinjaEnv(revlink=None, changecommentlink=None,
 
     env.install_null_translations() # needed until we have a proper i18n backend
 
+    env.tests['mapping'] = lambda obj : isinstance(obj, dict)
+
     env.filters.update(dict(
         urlencode = urllib.quote,
         email = emailfilter,

master/buildbot/steps/source/git.py

@@ -20,14 +20,48 @@
 from buildbot.steps.source.base import Source
 from buildbot.interfaces import BuildSlaveTooOldError
 
+def isTrueOrIsExactlyZero(v):
+    # nonzero values are true...
+    if v:
+        return True
+
+    # ... and True for the number zero, but we have to
+    # explicitly guard against v==False, since
+    # isinstance(False, int) is surprisingly True
+    if isinstance(v, int) and v is not False:
+        return True
+
+    # all other false-ish values are false
+    return False
+
+git_describe_flags = [
+    # on or off
+    ('all',         lambda v: ['--all'] if v else None),
+    ('always',      lambda v: ['--always'] if v else None),
+    ('contains',    lambda v: ['--contains'] if v else None),
+    ('debug',       lambda v: ['--debug'] if v else None),
+    ('long',        lambda v: ['--long'] if v else None),
+    ('exact-match', lambda v: ['--exact-match'] if v else None),
+    ('tags',        lambda v: ['--tags'] if v else None),
+    # string parameter
+    ('match',       lambda v: ['--match', v] if v else None),
+    # numeric parameter
+    ('abbrev',      lambda v: ['--abbrev=%s' % v] if isTrueOrIsExactlyZero(v) else None),
+    ('candidates',  lambda v: ['--candidates=%s' % v] if isTrueOrIsExactlyZero(v) else None),
+    # optional string parameter
+    ('dirty',       lambda v: ['--dirty'] if (v is True or v=='') else None),
+    ('dirty',       lambda v: ['--dirty=%s' % v] if (v and v is not True) else None),
+]
+
 class Git(Source):
     """ Class for Git with all the smarts """
     name='git'
     renderables = [ "repourl"]
 
     def __init__(self, repourl=None, branch='HEAD', mode='incremental',
                  method=None, submodules=False, shallow=False, progress=False,
-                 retryFetch=False, clobberOnFailure=False, **kwargs):
+                 retryFetch=False, clobberOnFailure=False, getDescription=False,
+                 **kwargs):
         """
         @type  repourl: string
         @param repourl: the URL which points at the git repository
@@ -57,7 +91,12 @@ def __init__(self, repourl=None, branch='HEAD', mode='incremental',
 
         @type  retryFetch: boolean
         @param retryFetch: Retry fetching before failing source checkout.
+        
+        @type  getDescription: boolean or dict
+        @param getDescription: Use 'git describe' to describe the fetched revision
         """
+        if not getDescription and not isinstance(getDescription, dict):
+            getDescription = False
 
         self.branch    = branch
         self.method    = method
@@ -69,6 +108,7 @@ def __init__(self, repourl=None, branch='HEAD', mode='incremental',
         self.fetchcount = 0
         self.clobberOnFailure = clobberOnFailure
         self.mode = mode
+        self.getDescription = getDescription
         Source.__init__(self, **kwargs)
         self.addFactoryArguments(branch=branch,
                                  mode=mode,
@@ -80,12 +120,15 @@ def __init__(self, repourl=None, branch='HEAD', mode='incremental',
                                  retryFetch=retryFetch,
                                  clobberOnFailure=
                                  clobberOnFailure,
+                                 getDescription=
+                                 getDescription
                                  )
 
         assert self.mode in ['incremental', 'full']
         assert self.repourl is not None
         if self.mode == 'full':
             assert self.method in ['clean', 'fresh', 'clobber', 'copy', None]
+        assert isinstance(self.getDescription, (bool, dict))
 
     def startVC(self, branch, revision, patch):
         self.branch = branch or 'HEAD'
@@ -107,6 +150,7 @@ def checkInstall(gitInstalled):
         if patch:
             d.addCallback(self.patch, patch)
         d.addCallback(self.parseGotRevision)
+        d.addCallback(self.parseCommitDescription)
         d.addCallback(self.finish)
         d.addErrback(self.failed)
         return d
@@ -224,17 +268,40 @@ def _gotResults(results):
         d.addCallbacks(self.finished, self.checkDisconnect)
         return d
 
-    def parseGotRevision(self, _):
-        d = self._dovccmd(['rev-parse', 'HEAD'], collectStdout=True)
-        def setrev(stdout):
-            revision = stdout.strip()
-            if len(revision) != 40:
-                raise buildstep.BuildStepFailed()
-            log.msg("Got Git revision %s" % (revision, ))
-            self.setProperty('got_revision', revision, 'Source')
-            return 0
-        d.addCallback(setrev)
-        return d
+    @defer.inlineCallbacks
+    def parseGotRevision(self, _=None):
+        stdout = yield self._dovccmd(['rev-parse', 'HEAD'], collectStdout=True)
+        revision = stdout.strip()
+        if len(revision) != 40:
+            raise buildstep.BuildStepFailed()
+        log.msg("Got Git revision %s" % (revision, ))
+        self.setProperty('got_revision', revision, 'Source')
+
+        defer.returnValue(0)
+
+    @defer.inlineCallbacks
+    def parseCommitDescription(self, _=None):
+        if self.getDescription==False: # dict() should not return here
+            defer.returnValue(0)
+            return
+
+        cmd = ['describe']
+        if isinstance(self.getDescription, dict):
+            for opt, arg in git_describe_flags:
+                opt = self.getDescription.get(opt, None)
+                arg = arg(opt)
+                if arg:
+                    cmd.extend(arg)            
+        cmd.append('HEAD')
+
+        try:
+            stdout = yield self._dovccmd(cmd, collectStdout=True)
+            desc = stdout.strip()
+            self.setProperty('commit-description', desc, 'Source')
+        except:
+            pass
+
+        defer.returnValue(0)
 
     def _dovccmd(self, command, abandonOnFailure=True, collectStdout=False, initialStdin=None):
         cmd = buildstep.RemoteShellCommand(self.workdir, ['git'] + command,

master/buildbot/test/unit/test_changes_changes.py

@@ -95,23 +95,24 @@ def test_str(self):
 
     def test_asText(self):
         text = self.change23.asText()
-        self.assertEqual(text, textwrap.dedent(u'''\
+        self.assertTrue(re.match(textwrap.dedent(u'''\
             Files:
              master/README.txt
              slave/README.txt
             On: git://warner
             For: Buildbot
-            At: Thu 15 Jun 1978 01:00:04
+            At: .*
             Changed By: dustin
             Comments: fix whitespaceProperties: 
               notest: no
 
-            '''))
+            '''), text), text)
 
     def test_asDict(self):
         dict = self.change23.asDict()
+        self.assertIn('1978', dict['at']) # timezone-sensitive
+        del dict['at']
         self.assertEqual(dict, {
-            'at': 'Thu 15 Jun 1978 01:00:04',
             'branch': u'warnerdb',
             'category': u'devel',
             'codebase': u'mainapp',