Merge branch 'master' into jinja

buildbot/master.py

@@ -66,6 +66,10 @@ def __init__(self):
         # requests
         self.mergeRequests = None
 
+        # self.prioritizeBuilders is the callable override for builder order
+        # traversal
+        self.prioritizeBuilders = None
+
     # these four are convenience functions for testing
 
     def waitUntilBuilderAttached(self, name):
@@ -194,19 +198,29 @@ def _updateAllSlaves(self):
 
     def maybeStartAllBuilds(self):
         builders = self.builders.values()
-        def _sortfunc(b1, b2):
-            t1 = b1.getOldestRequestTime()
-            t2 = b2.getOldestRequestTime()
-            # If t1 or t2 is None, then there are no build requests,
-            # so sort it at the end
-            if t1 is None:
-                return 1
-            if t2 is None:
-                return -1
-            return cmp(t1, t2)
-        builders.sort(cmp=_sortfunc)
-        for b in builders:
-            b.maybeStartBuild()
+        if self.prioritizeBuilders is not None:
+            try:
+                builders = self.prioritizeBuilders(self.parent, builders)
+            except:
+                log.err(None, "Exception prioritizing builders")
+                return
+        else:
+            def _sortfunc(b1, b2):
+                t1 = b1.getOldestRequestTime()
+                t2 = b2.getOldestRequestTime()
+                # If t1 or t2 is None, then there are no build requests,
+                # so sort it at the end
+                if t1 is None:
+                    return 1
+                if t2 is None:
+                    return -1
+                return cmp(t1, t2)
+            builders.sort(cmp=_sortfunc)
+        try:
+            for b in builders:
+                b.maybeStartBuild()
+        except:
+            log.err(None, "Exception starting builds")
 
     def shouldMergeRequests(self, builder, req1, req2):
         """Determine whether two BuildRequests should be merged for
@@ -523,10 +537,10 @@ def loadConfig(self, f):
 
         known_keys = ("bots", "slaves",
                       "sources", "change_source",
-                      "schedulers", "builders", "mergeRequests", 
+                      "schedulers", "builders", "mergeRequests",
                       "slavePortnum", "debugPassword", "logCompressionLimit",
                       "manhole", "status", "projectName", "projectURL",
-                      "buildbotURL", "properties"
+                      "buildbotURL", "properties", "prioritizeBuilders",
                       )
         for k in config.keys():
             if k not in known_keys:
@@ -562,6 +576,9 @@ def loadConfig(self, f):
             mergeRequests = config.get('mergeRequests')
             if mergeRequests is not None and not callable(mergeRequests):
                 raise ValueError("mergeRequests must be a callable")
+            prioritizeBuilders = config.get('prioritizeBuilders')
+            if prioritizeBuilders is not None and not callable(prioritizeBuilders):
+                raise ValueError("prioritizeBuilders must be callable")
 
         except KeyError, e:
             log.msg("config dictionary is missing a required parameter")
@@ -722,6 +739,8 @@ def loadConfig(self, f):
             self.status.logCompressionLimit = logCompressionLimit
         if mergeRequests is not None:
             self.botmaster.mergeRequests = mergeRequests
+        if prioritizeBuilders is not None:
+            self.botmaster.prioritizeBuilders = prioritizeBuilders
 
         # self.slaves: Disconnect any that were attached and removed from the
         # list. Update self.checker with the new list of passwords, including

buildbot/process/builder.py

@@ -368,13 +368,19 @@ def __init__(self, setup, builder_status):
             self.slavenames.extend(setup['slavenames'])
         self.builddir = setup['builddir']
         self.buildFactory = setup['factory']
+        self.nextSlave = setup.get('nextSlave')
+        if self.nextSlave is not None and not callable(self.nextSlave):
+            raise ValueError("nextSlave must be callable")
         self.locks = setup.get("locks", [])
         self.env = setup.get('env', {})
         assert isinstance(self.env, dict)
         if setup.has_key('periodicBuildTime'):
             raise ValueError("periodicBuildTime can no longer be defined as"
                              " part of the Builder: use scheduler.Periodic"
                              " instead")
+        self.nextBuild = setup.get('nextBuild')
+        if self.nextBuild is not None and not callable(self.nextBuild):
+            raise ValueError("nextBuild must be callable")
 
         # build/wannabuild slots: Build objects move along this sequence
         self.buildable = []
@@ -421,6 +427,10 @@ def compareToSetup(self, setup):
                     for lock in setup.get('locks',[])]
         if oldlocks != newlocks:
             diffs.append('locks changed from %s to %s' % (oldlocks, newlocks))
+        if setup.get('nextSlave') != self.nextSlave:
+            diffs.append('nextSlave changed from %s to %s' % (self.nextSlave, setup['nextSlave']))
+        if setup.get('nextBuild') != self.nextBuild:
+            diffs.append('nextBuild changed from %s to %s' % (self.nextBuild, setup['nextBuild']))
         return diffs
 
     def __repr__(self):
@@ -440,7 +450,8 @@ def submitBuildRequest(self, req):
         self.buildable.append(req)
         req.requestSubmitted(self)
         self.builder_status.addBuildRequest(req.status)
-        self.maybeStartBuild()
+        #self.maybeStartBuild()
+        self.botmaster.maybeStartAllBuilds()
 
     def cancelBuildRequest(self, req):
         if req in self.buildable:
@@ -554,7 +565,7 @@ def addLatentSlave(self, slave):
             self.builder_status.addPointEvent(
                 ['added', 'latent', slave.slavename])
             self.slaves.append(sb)
-            reactor.callLater(0, self.maybeStartBuild)
+            reactor.callLater(0, self.botmaster.maybeStartAllBuilds)
 
     def attached(self, slave, remote, commands):
         """This is invoked by the BuildSlave when the self.slavename bot
@@ -676,16 +687,40 @@ def maybeStartBuild(self):
                     % self)
             self.updateBigStatus()
             return
-        if self.CHOOSE_SLAVES_RANDOMLY:
-            # TODO prefer idle over latent? maybe other sorting preferences?
+        if self.nextSlave:
+            sb = None
+            try:
+                sb = self.nextSlave(self, available_slaves)
+            except:
+                log.err(None, "Exception choosing next slave")
+
+            if not sb:
+                log.msg("%s: want to start build, but we don't have a remote"
+                        % self)
+                self.updateBigStatus()
+                return
+        elif self.CHOOSE_SLAVES_RANDOMLY:
             sb = random.choice(available_slaves)
         else:
             sb = available_slaves[0]
 
         # there is something to build, and there is a slave on which to build
         # it. Grab the oldest request, see if we can merge it with anything
         # else.
-        req = self.buildable.pop(0)
+        if not self.nextBuild:
+            req = self.buildable.pop(0)
+        else:
+            try:
+                req = self.nextBuild(self, self.buildable)
+                if not req:
+                    # Nothing to do
+                    self.updateBigStatus()
+                    return
+                self.buildable.remove(req)
+            except:
+                log.err(None, "Exception choosing next build")
+                self.updateBigStatus()
+                return
         self.builder_status.removeBuildRequest(req.status)
         mergers = []
         botmaster = self.botmaster

buildbot/status/mail.py

@@ -176,13 +176,15 @@ class MailNotifier(base.StatusReceiverMultiService):
 
     compare_attrs = ["extraRecipients", "lookup", "fromaddr", "mode",
                      "categories", "builders", "addLogs", "relayhost",
-                     "subject", "sendToInterestedUsers", "customMesg"]
+                     "subject", "sendToInterestedUsers", "customMesg",
+                     "extraHeaders"]
 
     def __init__(self, fromaddr, mode="all", categories=None, builders=None,
                  addLogs=False, relayhost="localhost",
                  subject="buildbot %(result)s in %(projectName)s on %(builder)s",
                  lookup=None, extraRecipients=[],
-                 sendToInterestedUsers=True, customMesg=message):
+                 sendToInterestedUsers=True, customMesg=message,
+                 extraHeaders=None):
         """
         @type  fromaddr: string
         @param fromaddr: the email address to be used in the 'From' header.
@@ -300,7 +302,11 @@ def __init__(self, fromaddr, mode="all", categories=None, builders=None,
 
                            logs - (List of Tuples) List of tuples that contain the log name, log url,
                                   and log contents as a list of strings.
-
+        @type  extraHeaders: dict
+        @param extraHeaders: A dict of extra headers to add to the mail. It's
+                             best to avoid putting 'To', 'From', 'Date',
+                             'Subject', or 'CC' in here. Both the names and
+                             values may be WithProperties instances.
         """
 
         base.StatusReceiverMultiService.__init__(self)
@@ -324,6 +330,9 @@ def __init__(self, fromaddr, mode="all", categories=None, builders=None,
             assert interfaces.IEmailLookup.providedBy(lookup)
         self.lookup = lookup
         self.customMesg = customMesg
+        if extraHeaders:
+            assert isinstance(extraHeaders, dict)
+        self.extraHeaders = extraHeaders
         self.watched = []
         self.status = None
 
@@ -406,6 +415,8 @@ def buildMessage(self, name, build, results):
                          '%s/steps/%s/logs/%s' % (self.status.getURLForThing(build), stepName, logName),
                          logf.getText().splitlines(),
                          logStatus))
+
+        properties = build.getProperties()
 
         attrs = {'builderName': name,
                  'projectName': self.status.getProjectName(),
@@ -414,7 +425,7 @@ def buildMessage(self, name, build, results):
                  'buildURL': self.status.getURLForThing(build),
                  'buildbotURL': self.status.getBuildbotURL(),
                  'buildText': build.getText(),
-                 'buildProperties': build.getProperties(),
+                 'buildProperties': properties,
                  'slavename': build.getSlavename(),
                  'reason':  build.getReason(),
                  'responsibleUsers': build.getResponsibleUsers(),
@@ -474,6 +485,18 @@ def buildMessage(self, name, build, results):
                                  filename=name)
                     m.attach(a)
 
+        # Add any extra headers that were requested, doing WithProperties
+        # interpolation if necessary
+        if self.extraHeaders:
+            for k,v in self.extraHeaders:
+                k = properties.render(k)
+                if k in m:
+                    twlog("Warning: Got header " + k + " in self.extraHeaders "
+                          "but it already exists in the Message - "
+                          "not adding it.")
+                    continue
+                m[k] = properties.render(v)
+
         # now, who is this message going to?
         dl = []
         recipients = []