Merge branch 'build_set_status' of git://github.com/aslater/buildbot

* 'build_set_status' of git://github.com/aslater/buildbot:
  Corrected order of operations in MailNotifier startService/stopService Added return value to MailNotifier stopService
  Fixed one more long line
  Added buildset subscribe/unsubscribe in MailNotifier start/stopService Fixed long lines Added documentation Removed unnecessary import
  Added the ability to only send a single summary email from a MailNotifier when an entire buildset completes, rather than sending one for each build

master/buildbot/status/mail.py

@@ -156,7 +156,7 @@ class MailNotifier(base.StatusReceiverMultiService):
     possible_modes = ('all', 'failing', 'problem', 'change', 'passing', 'warnings')
 
     def __init__(self, fromaddr, mode="all", categories=None, builders=None,
-                 addLogs=False, relayhost="localhost",
+                 addLogs=False, relayhost="localhost", buildSetSummary=False,
                  subject="buildbot %(result)s in %(title)s on %(builder)s",
                  lookup=None, extraRecipients=[],
                  sendToInterestedUsers=True, customMesg=None,
@@ -221,7 +221,12 @@ def __init__(self, fromaddr, mode="all", categories=None, builders=None,
         @type  relayhost: string
         @param relayhost: the host to which the outbound SMTP connection
                           should be made. Defaults to 'localhost'
-
+                          
+        @type  buildSetSummary: boolean
+        @param buildSetSummary: if True, this notifier will only send a summary
+                                email when a buildset containing any of its
+                                watched builds completes
+                                
         @type  lookup:    implementor of {IEmailLookup}
         @param lookup:    object which provides IEmailLookup, which is
                           responsible for mapping User names for Interested
@@ -304,6 +309,8 @@ def __init__(self, fromaddr, mode="all", categories=None, builders=None,
         self.smtpUser = smtpUser
         self.smtpPassword = smtpPassword
         self.smtpPort = smtpPort
+        self.buildSetSummary = buildSetSummary
+        self.buildSetSubscription = None
         self.watched = []
         self.master_status = None
 
@@ -325,6 +332,22 @@ def setServiceParent(self, parent):
     def setup(self):
         self.master_status = self.parent.getStatus()
         self.master_status.subscribe(self)
+
+
+    def startService(self):
+        if self.buildSetSummary:
+            self.buildSetSubscription = \
+            self.parent.subscribeToBuildsetCompletions(self.buildsetFinished)
+
+        base.StatusReceiverMultiService.startService(self)
+
+
+    def stopService(self):
+        if self.buildSetSubscription is not None:
+            self.buildSetSubscription.unsubscribe()
+            self.buildSetSubscription = None
+
+        return base.StatusReceiverMultiService.stopService(self)
 
     def disownServiceParent(self):
         self.master_status.unsubscribe(self)
@@ -347,38 +370,79 @@ def builderChangedState(self, name, state):
         pass
     def buildStarted(self, name, build):
         pass
-    def buildFinished(self, name, build, results):
+    def isMailNeeded(self, build, results):
         # here is where we actually do something.
         builder = build.getBuilder()
-        if self.builders is not None and name not in self.builders:
-            return # ignore this build
+        if self.builders is not None and builder.name not in self.builders:
+            return False # ignore this build
         if self.categories is not None and \
                builder.category not in self.categories:
-            return # ignore this build
+            return False # ignore this build
 
         if self.mode == "warnings" and results == SUCCESS:
-            return
+            return False
         if self.mode == "failing" and results != FAILURE:
-            return
+            return False
         if self.mode == "passing" and results != SUCCESS:
-            return
+            return False
         if self.mode == "problem":
             if results != FAILURE:
-                return
+                return False
             prev = build.getPreviousBuild()
             if prev and prev.getResults() == FAILURE:
-                return
+                return False
         if self.mode == "change":
             prev = build.getPreviousBuild()
             if not prev or prev.getResults() == results:
-                return
-        # for testing purposes, buildMessage returns a Deferred that fires
-        # when the mail has been sent. To help unit tests, we return that
-        # Deferred here even though the normal IStatusReceiver.buildFinished
-        # signature doesn't do anything with it. If that changes (if
-        # .buildFinished's return value becomes significant), we need to
-        # rearrange this.
-        return self.buildMessage(name, build, results)
+                return False
+
+        return True
+
+    def buildFinished(self, name, build, results):
+        if ( not self.buildSetSummary and
+             self.isMailNeeded(build, results) ):
+            # for testing purposes, buildMessage returns a Deferred that fires
+            # when the mail has been sent. To help unit tests, we return that
+            # Deferred here even though the normal IStatusReceiver.buildFinished
+            # signature doesn't do anything with it. If that changes (if
+            # .buildFinished's return value becomes significant), we need to
+            # rearrange this.
+            return self.buildMessage(name, [build], results)
+        return None
+
+    def _gotBuilds(self, res, builddicts, buildset, builders):
+        builds = []
+        for (builddictlist, builder) in zip(builddicts, builders):
+                for builddict in builddictlist:
+                    build = builder.getBuild(builddict['number'])
+                    if self.isMailNeeded(build, build.results):
+                        builds.append(build)
+
+        self.buildMessage("Buildset Complete: " + buildset['reason'], builds,
+                          buildset['results'])
+
+    def _gotBuildRequests(self, breqs, buildset):
+        builddicts = []
+        builders =[]
+        dl = []
+        for breq in breqs:
+            buildername = breq['buildername']
+            builders.append(self.master_status.getBuilder(buildername))
+            d = self.parent.db.builds.getBuildsForRequest(breq['brid'])
+            d.addCallback(builddicts.append)
+            dl.append(d)
+        d = defer.DeferredList(dl)
+        d.addCallback(self._gotBuilds, builddicts, buildset, builders)
+
+    def _gotBuildSet(self, buildset, bsid):
+        d = self.parent.db.buildrequests.getBuildRequests(bsid=bsid)
+        d.addCallback(self._gotBuildRequests, buildset)
+
+    def buildsetFinished(self, bsid, result):
+        d = self.parent.db.buildsets.getBuildset(bsid=bsid)
+        d.addCallback(self._gotBuildSet, bsid)
+
+        return d
 
     def getCustomMesgData(self, mode, name, build, results, master_status):
         #
@@ -392,7 +456,9 @@ def getCustomMesgData(self, mode, name, build, results, master_status):
             logStatus, dummy = logStep.getResults()
             logName = logf.getName()
             logs.append(('%s.%s' % (stepName, logName),
-                         '%s/steps/%s/logs/%s' % (master_status.getURLForThing(build), stepName, logName),
+                         '%s/steps/%s/logs/%s' % (
+                             master_status.getURLForThing(build),
+                             stepName, logName),
                          logf.getText().splitlines(),
                          logStatus))
 
@@ -422,8 +488,8 @@ def getCustomMesgData(self, mode, name, build, results, master_status):
 
         return attrs
 
-    def createEmail(self, msgdict, builderName, title, results, build,
-                    patch=None, logs=None):
+    def createEmail(self, msgdict, builderName, title, results, builds=None,
+                    patches=None, logs=None):
         text = msgdict['body'].encode(ENCODING)
         type = msgdict['type']
         if 'subject' in msgdict:
@@ -436,9 +502,10 @@ def createEmail(self, msgdict, builderName, title, results, build,
                                        }
 
 
-        assert type in ('plain', 'html'), "'%s' message type must be 'plain' or 'html'." % type
+        assert type in ('plain', 'html'), \
+            "'%s' message type must be 'plain' or 'html'." % type
 
-        if patch or logs:
+        if patches or logs:
             m = MIMEMultipart()
             m.attach(MIMEText(text, type, ENCODING))
         else:
@@ -451,65 +518,92 @@ def createEmail(self, msgdict, builderName, title, results, build,
         m['From'] = self.fromaddr
         # m['To'] is added later
 
-        if patch:
-            a = MIMEText(patch[1].encode(ENCODING), _charset=ENCODING)
-            a.add_header('Content-Disposition', "attachment",
-                         filename="source patch")
-            m.attach(a)
+        if patches:
+            for (i, patch) in enumerate(patches):
+                a = MIMEText(patch[1].encode(ENCODING), _charset=ENCODING)
+                a.add_header('Content-Disposition', "attachment",
+                         filename="source patch " + str(i) )
+                m.attach(a)
         if logs:
             for log in logs:
                 name = "%s.%s" % (log.getStep().getName(),
                                   log.getName())
-                if self._shouldAttachLog(log.getName()) or self._shouldAttachLog(name):
+                if ( self._shouldAttachLog(log.getName()) or
+                     self._shouldAttachLog(name) ):
                     a = MIMEText(log.getText().encode(ENCODING), 
                                  _charset=ENCODING)
                     a.add_header('Content-Disposition', "attachment",
                                  filename=name)
                     m.attach(a)
 
+        #@todo: is there a better way to do this?
         # Add any extra headers that were requested, doing WithProperties
-        # interpolation if necessary
+        # interpolation if only one build was given
         if self.extraHeaders:
-            properties = build.getProperties()
+            if len(builds) == 1:
+                properties = builds[0].getProperties()
+
             for k,v in self.extraHeaders.items():
-                k = properties.render(k)
+                if len(builds == 1):
+                    k = properties.render(k)
                 if k in m:
-                    twlog.msg("Warning: Got header " + k + " in self.extraHeaders "
-                          "but it already exists in the Message - "
-                          "not adding it.")
-                    continue
-                m[k] = properties.render(v)
-
+                    twlog.msg("Warning: Got header " + k +
+                      " in self.extraHeaders "
+                      "but it already exists in the Message - "
+                      "not adding it.")
+                continue
+                if len(builds == 1):
+                    m[k] = properties.render(v)
+                else:
+                    m[k] = v
+
         return m
-
-    def buildMessage(self, name, build, results):
+    
+    def buildMessageDict(self, name, build, results):
         if self.customMesg:
             # the customMesg stuff can be *huge*, so we prefer not to load it
-            attrs = self.getCustomMesgData(self.mode, name, build, results, self.master_status)
+            attrs = self.getCustomMesgData(self.mode, name, build, results,
+                                           self.master_status)
             text, type = self.customMesg(attrs)
             msgdict = { 'body' : text, 'type' : type }
         else:
-            msgdict = self.messageFormatter(self.mode, name, build, results, self.master_status)
+            msgdict = self.messageFormatter(self.mode, name, build, results,
+                                            self.master_status)
+
+        return msgdict
 
-        patch = None
-        ss = build.getSourceStamp()
-        if ss and ss.patch and self.addPatch:
-            patch == ss.patch
-        logs = None
-        if self.addLogs:
-            logs = build.getLogs()
+
+    def buildMessage(self, name, builds, results):
+        patches = []
+        logs = []
+        msgdict = {"body":""}
+
+        for build in builds:
+            ss = build.getSourceStamp()
+            if ss and ss.patch and self.addPatch:
+                patches.append(ss.patch)
+            if self.addLogs:
+                logs.append(build.getLogs())
             twlog.err("LOG: %s" % str(logs))
+
+            tmp = self.buildMessageDict(name=build.getBuilder().name,
+                                        build=build, results=build.results)
+            msgdict['body'] += tmp['body']
+            msgdict['body'] += '\n\n'
+            msgdict['type'] = tmp['type']
+
         m = self.createEmail(msgdict, name, self.master_status.getTitle(),
-                             results, build, patch, logs)
+                             results, builds, patches, logs)
 
         # now, who is this message going to?
         dl = []
         recipients = []
         if self.sendToInterestedUsers and self.lookup:
-            for u in build.getInterestedUsers():
-                d = defer.maybeDeferred(self.lookup.getAddress, u)
-                d.addCallback(recipients.append)
-                dl.append(d)
+            for build in builds:
+                for u in build.getInterestedUsers():
+                    d = defer.maybeDeferred(self.lookup.getAddress, u)
+                    d.addCallback(recipients.append)
+                    dl.append(d)
         d = defer.DeferredList(dl)
         d.addCallback(self._gotRecipients, recipients, m)
         return d

master/buildbot/test/fake/fakedb.py

@@ -963,6 +963,22 @@ def mkdt(epoch):
             number=row.number,
             start_time=mkdt(row.start_time),
             finish_time=mkdt(row.finish_time)))
+
+    def getBuildsForRequest(self, brid):
+        ret = []
+        def mkdt(epoch):
+            if epoch:
+                return epoch2datetime(epoch)
+
+        for (id, row) in self.builds.items():
+            if row.brid == brid:
+                ret.append(dict(bid = row.id,
+                                brid=row.brid,
+                                number=row.number,
+                                start_time=mkdt(row.start_time),
+                                finish_time=mkdt(row.finish_time)))
+
+        return defer.succeed(ret)            
 
     def addBuild(self, brid, number, _reactor=reactor):
         bid = self._newId()