Merge branch 'master' into nine

Conflicts:
	master/buildbot/schedulers/timed.py
	master/buildbot/schedulers/triggerable.py
	master/buildbot/status/web/templates/forms.html
	master/buildbot/test/fake/fakedb.py
	master/buildbot/test/unit/test_schedulers_timed_Nightly.py
	master/buildbot/test/unit/test_schedulers_triggerable.py
	master/docs/manual/cfg-schedulers.rst

.travis.yml

@@ -73,6 +73,8 @@ install:
 
   # zope.interface dropped Python 2.5 support in 4.0.0
   - "[ $TRAVIS_PYTHON_VERSION != '2.5' ] || pip install 'zope.interface<4.0.0'"
+  # Python 2.5 doesn't have multiprocessing natively
+  - "[ $TRAVIS_PYTHON_VERSION != '2.5' ] || pip install multiprocessing"
   - "[ $TWISTED = latest ] || pip install Twisted==$TWISTED"
   - "[ $SQLALCHEMY = latest ] || pip install sqlalchemy==$SQLALCHEMY"
   - "[ $SQLALCHEMY_MIGRATE = latest ] || pip install sqlalchemy-migrate==$SQLALCHEMY_MIGRATE"

master/buildbot/changes/gerritchangesource.py

@@ -124,8 +124,9 @@ def addChange(self, chdict):
         return d
     def eventReceived_patchset_created(self, properties, event):
         change = event["change"]
+        username = change["owner"].get("username","unknown")
         return self.addChange(dict(
-                author="%s <%s>" % (change["owner"]["name"], change["owner"]["email"]),
+                author="%s <%s>" % (change["owner"].get("name",username), change["owner"].get("email","unknown@example.com")),
                 project=change["project"],
                 repository="ssh://%s@%s:%s/%s" % (
                     self.username, self.gerritserver, self.gerritport, change["project"]),
@@ -141,7 +142,8 @@ def eventReceived_ref_updated(self, properties, event):
         author = "gerrit"
 
         if "submitter" in event:
-            author="%s <%s>" % (event["submitter"]["name"], event["submitter"]["email"])
+            username = event["submitter"].get("username","unknown")
+            author="%s <%s>" % (event["submitter"].get("name",username), event["submitter"].get("email","unnkown@example.com"))
 
         return self.addChange(dict(
                 author=author,

master/buildbot/schedulers/basic.py

@@ -35,13 +35,14 @@ class BaseBasicScheduler(base.BaseScheduler):
     _reactor = reactor # for tests
 
     fileIsImportant = None
-    reason = 'scheduler'
+    reason = ''
 
     class NotSet: pass
     def __init__(self, name, shouldntBeSet=NotSet, treeStableTimer=None,
                 builderNames=None, branch=NotABranch, branches=NotABranch,
                 fileIsImportant=None, properties={}, categories=None,
-                reason=None, change_filter=None, onlyImportant=False, **kwargs):
+                reason="The %(classname)s scheduler named '%(name)s' triggered this build",
+                change_filter=None, onlyImportant=False, **kwargs):
         if shouldntBeSet is not self.NotSet:
             config.error(
                 "pass arguments to schedulers using keyword arguments")
@@ -65,8 +66,9 @@ def __init__(self, name, shouldntBeSet=NotSet, treeStableTimer=None,
         self._stable_timers = defaultdict(lambda : None)
         self._stable_timers_lock = defer.DeferredLock()
 
-        if reason is not None:
-            self.reason = reason
+        self.reason = util.ascii2unicode(reason % {
+            'name': name, 'classname': self.__class__.__name__
+        })
 
     def getChangeFilter(self, branch, branches, change_filter, categories):
         raise NotImplementedError

master/buildbot/schedulers/timed.py

@@ -57,7 +57,7 @@ def __init__(self, name, builderNames, properties={}, reason='', **kwargs):
         self.actuateAt = None
         self.actuateAtTimer = None
 
-        self.reason = reason % { 'name': name }
+        self.reason = util.ascii2unicode(reason % { 'name': name })
 
         self._reactor = reactor # patched by tests
 
@@ -267,12 +267,13 @@ def getNextBuildTime(self, lastActuated):
 
 class Nightly(NightlyBase):
     compare_attrs = ('branch', 'onlyIfChanged', 'fileIsImportant',
-               'change_filter', 'onlyImportant',)
+               'change_filter', 'onlyImportant', 'createAbsoluteSourceStamps',)
 
     class NoBranch: pass
     def __init__(self, name, builderNames, minute=0, hour='*',
                  dayOfMonth='*', month='*', dayOfWeek='*',
                  branch=NoBranch, fileIsImportant=None, onlyIfChanged=False,
+                 createAbsoluteSourceStamps=False,
                  properties={}, change_filter=None, onlyImportant=False,
                  reason="The Nightly scheduler named '%(name)s' triggered this build",
                  codebases = base.BaseScheduler.DEFAULT_CODEBASES):
@@ -291,30 +292,74 @@ def __init__(self, name, builderNames, minute=0, hour='*',
             config.error(
                 "Nightly parameter 'branch' is required")
 
+        if createAbsoluteSourceStamps and not onlyIfChanged:
+            config.error(
+                "createAbsoluteSourceStamps can only be used with onlyIfChanged")
+
+        self._lastCodebases = {}
         self.branch = branch
         self.onlyIfChanged = onlyIfChanged
+        self.createAbsoluteSourceStamps = createAbsoluteSourceStamps
         self.fileIsImportant = fileIsImportant
         self.change_filter = filter.ChangeFilter.fromSchedulerConstructorArgs(
                 change_filter=change_filter)
 
     def startTimedSchedulerService(self):
         if self.onlyIfChanged:
-            return self.startConsumingChanges(fileIsImportant=self.fileIsImportant,
+            d = self.preStartConsumingChanges()
+
+            d.addCallback(lambda _ :
+                    self.startConsumingChanges(fileIsImportant=self.fileIsImportant,
                                               change_filter=self.change_filter,
-                                              onlyImportant=self.onlyImportant)
+                                              onlyImportant=self.onlyImportant))
+            return d
         else:
             return self.master.db.schedulers.flushChangeClassifications(self.objectid)
 
+    def preStartConsumingChanges(self):
+        if self.createAbsoluteSourceStamps:
+            # load saved codebases
+            d = self.getState("lastCodebases", {})
+            def setLast(lastCodebases):
+                self._lastCodebases = lastCodebases
+            d.addCallback(setLast)
+            return d
+        else:
+            return defer.succeed(None)
+
     def gotChange(self, change, important):
         # both important and unimportant changes on our branch are recorded, as
         # we will include all such changes in any buildsets we start.  Note
         # that we must check the branch here because it is not included in the
         # change filter. 
         if change.branch != self.branch:
             return defer.succeed(None) # don't care about this change
-        return self.master.db.schedulers.classifyChanges(
+
+        d = self.master.db.schedulers.classifyChanges(
                 self.objectid, { change.number : important })
 
+        if self.createAbsoluteSourceStamps:
+            self._lastCodebases.setdefault(change.codebase, {})
+            lastChange = self._lastCodebases[change.codebase].get('lastChange', -1)
+
+            codebaseDict = dict(repository=change.repository,
+                                branch=change.branch,
+                                revision=change.revision,
+                                lastChange=change.number)
+
+            if change.number > lastChange:
+                self._lastCodebases[change.codebase] = codebaseDict
+                d.addCallback(lambda _ :
+                        self.setState('lastCodebases', self._lastCodebases))
+
+        return d
+
+    def getCodebaseDict(self, codebase):
+        if self.createAbsoluteSourceStamps:
+            return self._lastCodebases.get(codebase, self.codebases[codebase])
+        else:
+            return self.codebases[codebase]
+
     @defer.inlineCallbacks
     def startBuild(self):
         scheds = self.master.db.schedulers

master/buildbot/schedulers/triggerable.py

@@ -31,7 +31,7 @@ def __init__(self, name, builderNames, properties={}, **kwargs):
                                     **kwargs)
         self._waiters = {}
         self._buildset_complete_consumer = None
-        self.reason = u"Triggerable(%s)" % name
+        self.reason = u"The Triggerable scheduler named '%s' triggered this build" % name
 
     def trigger(self, waited_for, sourcestamps = None, set_props=None):
         """Trigger this scheduler with the optional given list of sourcestamps

master/buildbot/status/words.py

@@ -397,11 +397,24 @@ def buildStarted(self, builderName, build):
             r = "build containing revision(s) [%s] on %s started" % \
                 (build.getRevisions(), builder.getName())
         else:
-            r = "build #%d of %s started, including [%s]" % \
-                (build.getNumber(),
-                 builder.getName(),
-                 ", ".join([str(c.revision) for c in build.getChanges()])
-                 )
+            # Abbreviate long lists of changes to simply two
+            # revisions, and the number of additional changes.
+            changes = [str(c.revision) for c in build.getChanges()][:2]
+            changes_str = ""
+
+            if len(build.getChanges()) > 0:
+                changes_str = "including [%s]" % ', '.join(changes)
+
+                if len(build.getChanges()) > 2:
+                    # Append number of truncated changes
+                    changes_str += " and %d more" % (len(build.getChanges())-2)
+
+            r = "build #%d of %s started" % (
+                build.getNumber(),
+                builder.getName())
+
+            if changes_str:
+                r += " (%s)" % changes_str
 
         self.send(r)
 

master/buildbot/steps/source/git.py

@@ -58,7 +58,7 @@ def isTrueOrIsExactlyZero(v):
 class Git(Source):
     """ Class for Git with all the smarts """
     name='git'
-    renderables = [ "repourl", "reference"]
+    renderables = [ "repourl", "reference", "branch", "codebase"]
 
     def __init__(self, repourl=None, branch='HEAD', mode='incremental', method=None,
                  reference=None, submodules=False, shallow=False, progress=False, retryFetch=False,

master/buildbot/steps/source/mercurial.py

@@ -219,8 +219,6 @@ def _checkBranchChange(self, _):
 
     def _pullUpdate(self, res):
         command = ['pull' , self.repourl]
-        if self.revision:
-            command.extend(['--rev', self.revision])
         d = self._dovccmd(command)
         d.addCallback(self._checkBranchChange)
         return d

master/buildbot/test/unit/test_schedulers_basic.py

@@ -39,7 +39,7 @@ def addBuildsetForChanges(
                 waited_for=False, reason='', external_idstring=None, changeids=[],
                 builderNames=None, properties=None):
             self.assertEqual(external_idstring, None)
-            self.assertEqual(reason, 'scheduler') # basic schedulers all use this
+            self.assertEqual(reason, sched.reason)
             self.events.append('B%s@%d' % (`changeids`.replace(' ',''),
                                            self.clock.seconds()))
             return defer.succeed(None)
@@ -290,7 +290,7 @@ def makeFullScheduler(self, **kwargs):
 
     def mkbs(self, **kwargs):
         # create buildset for expected_buildset in assertBuildset.
-        bs = dict(reason='scheduler', external_idstring=None, sourcestampsetid=100,
+        bs = dict(reason=self.sched.reason, external_idstring=None, sourcestampsetid=100,
                   properties=[('scheduler', ('test', 'Scheduler'))])
         bs.update(kwargs)
         return bs
@@ -318,6 +318,14 @@ def setUp(self):
     def tearDown(self):
         self.tearDownScheduler()
 
+    def test_constructor_no_reason(self):
+        sched = self.makeScheduler(basic.SingleBranchScheduler, branch="master")
+        self.assertEqual(sched.reason, "The SingleBranchScheduler scheduler named 'tsched' triggered this build")
+
+    def test_constructor_reason(self):
+        sched = self.makeScheduler(basic.SingleBranchScheduler, branch="master", reason="Changeset")
+        self.assertEqual(sched.reason, "Changeset")
+
     def test_constructor_branch_mandatory(self):
         self.assertRaises(config.ConfigErrors,
                 lambda : basic.SingleBranchScheduler(name="tsched", treeStableTimer=60))