diff --git a/master/buildbot/status/mail.py b/master/buildbot/status/mail.py index f1ee29f9ec1..95e42f7b9e6 100644 --- a/master/buildbot/status/mail.py +++ b/master/buildbot/status/mail.py @@ -156,7 +156,7 @@ class MailNotifier(base.StatusReceiverMultiService): possible_modes = ('all', 'failing', 'problem', 'change', 'passing', 'warnings') def __init__(self, fromaddr, mode="all", categories=None, builders=None, - addLogs=False, relayhost="localhost", + addLogs=False, relayhost="localhost", buildSetSummary=False, subject="buildbot %(result)s in %(title)s on %(builder)s", lookup=None, extraRecipients=[], sendToInterestedUsers=True, customMesg=None, @@ -221,7 +221,12 @@ def __init__(self, fromaddr, mode="all", categories=None, builders=None, @type relayhost: string @param relayhost: the host to which the outbound SMTP connection should be made. Defaults to 'localhost' - + + @type buildSetSummary: boolean + @param buildSetSummary: if True, this notifier will only send a summary + email when a buildset containing any of its + watched builds completes + @type lookup: implementor of {IEmailLookup} @param lookup: object which provides IEmailLookup, which is responsible for mapping User names for Interested @@ -304,6 +309,8 @@ def __init__(self, fromaddr, mode="all", categories=None, builders=None, self.smtpUser = smtpUser self.smtpPassword = smtpPassword self.smtpPort = smtpPort + self.buildSetSummary = buildSetSummary + self.buildSetSubscription = None self.watched = [] self.master_status = None @@ -325,6 +332,22 @@ def setServiceParent(self, parent): def setup(self): self.master_status = self.parent.getStatus() self.master_status.subscribe(self) + + + def startService(self): + if self.buildSetSummary: + self.buildSetSubscription = \ + self.parent.subscribeToBuildsetCompletions(self.buildsetFinished) + + base.StatusReceiverMultiService.startService(self) + + + def stopService(self): + if self.buildSetSubscription is not None: + self.buildSetSubscription.unsubscribe() + self.buildSetSubscription = None + + return base.StatusReceiverMultiService.stopService(self) def disownServiceParent(self): self.master_status.unsubscribe(self) @@ -347,38 +370,79 @@ def builderChangedState(self, name, state): pass def buildStarted(self, name, build): pass - def buildFinished(self, name, build, results): + def isMailNeeded(self, build, results): # here is where we actually do something. builder = build.getBuilder() - if self.builders is not None and name not in self.builders: - return # ignore this build + if self.builders is not None and builder.name not in self.builders: + return False # ignore this build if self.categories is not None and \ builder.category not in self.categories: - return # ignore this build + return False # ignore this build if self.mode == "warnings" and results == SUCCESS: - return + return False if self.mode == "failing" and results != FAILURE: - return + return False if self.mode == "passing" and results != SUCCESS: - return + return False if self.mode == "problem": if results != FAILURE: - return + return False prev = build.getPreviousBuild() if prev and prev.getResults() == FAILURE: - return + return False if self.mode == "change": prev = build.getPreviousBuild() if not prev or prev.getResults() == results: - return - # for testing purposes, buildMessage returns a Deferred that fires - # when the mail has been sent. To help unit tests, we return that - # Deferred here even though the normal IStatusReceiver.buildFinished - # signature doesn't do anything with it. If that changes (if - # .buildFinished's return value becomes significant), we need to - # rearrange this. - return self.buildMessage(name, build, results) + return False + + return True + + def buildFinished(self, name, build, results): + if ( not self.buildSetSummary and + self.isMailNeeded(build, results) ): + # for testing purposes, buildMessage returns a Deferred that fires + # when the mail has been sent. To help unit tests, we return that + # Deferred here even though the normal IStatusReceiver.buildFinished + # signature doesn't do anything with it. If that changes (if + # .buildFinished's return value becomes significant), we need to + # rearrange this. + return self.buildMessage(name, [build], results) + return None + + def _gotBuilds(self, res, builddicts, buildset, builders): + builds = [] + for (builddictlist, builder) in zip(builddicts, builders): + for builddict in builddictlist: + build = builder.getBuild(builddict['number']) + if self.isMailNeeded(build, build.results): + builds.append(build) + + self.buildMessage("Buildset Complete: " + buildset['reason'], builds, + buildset['results']) + + def _gotBuildRequests(self, breqs, buildset): + builddicts = [] + builders =[] + dl = [] + for breq in breqs: + buildername = breq['buildername'] + builders.append(self.master_status.getBuilder(buildername)) + d = self.parent.db.builds.getBuildsForRequest(breq['brid']) + d.addCallback(builddicts.append) + dl.append(d) + d = defer.DeferredList(dl) + d.addCallback(self._gotBuilds, builddicts, buildset, builders) + + def _gotBuildSet(self, buildset, bsid): + d = self.parent.db.buildrequests.getBuildRequests(bsid=bsid) + d.addCallback(self._gotBuildRequests, buildset) + + def buildsetFinished(self, bsid, result): + d = self.parent.db.buildsets.getBuildset(bsid=bsid) + d.addCallback(self._gotBuildSet, bsid) + + return d def getCustomMesgData(self, mode, name, build, results, master_status): # @@ -392,7 +456,9 @@ def getCustomMesgData(self, mode, name, build, results, master_status): logStatus, dummy = logStep.getResults() logName = logf.getName() logs.append(('%s.%s' % (stepName, logName), - '%s/steps/%s/logs/%s' % (master_status.getURLForThing(build), stepName, logName), + '%s/steps/%s/logs/%s' % ( + master_status.getURLForThing(build), + stepName, logName), logf.getText().splitlines(), logStatus)) @@ -422,8 +488,8 @@ def getCustomMesgData(self, mode, name, build, results, master_status): return attrs - def createEmail(self, msgdict, builderName, title, results, build, - patch=None, logs=None): + def createEmail(self, msgdict, builderName, title, results, builds=None, + patches=None, logs=None): text = msgdict['body'].encode(ENCODING) type = msgdict['type'] if 'subject' in msgdict: @@ -436,9 +502,10 @@ def createEmail(self, msgdict, builderName, title, results, build, } - assert type in ('plain', 'html'), "'%s' message type must be 'plain' or 'html'." % type + assert type in ('plain', 'html'), \ + "'%s' message type must be 'plain' or 'html'." % type - if patch or logs: + if patches or logs: m = MIMEMultipart() m.attach(MIMEText(text, type, ENCODING)) else: @@ -451,65 +518,92 @@ def createEmail(self, msgdict, builderName, title, results, build, m['From'] = self.fromaddr # m['To'] is added later - if patch: - a = MIMEText(patch[1].encode(ENCODING), _charset=ENCODING) - a.add_header('Content-Disposition', "attachment", - filename="source patch") - m.attach(a) + if patches: + for (i, patch) in enumerate(patches): + a = MIMEText(patch[1].encode(ENCODING), _charset=ENCODING) + a.add_header('Content-Disposition', "attachment", + filename="source patch " + str(i) ) + m.attach(a) if logs: for log in logs: name = "%s.%s" % (log.getStep().getName(), log.getName()) - if self._shouldAttachLog(log.getName()) or self._shouldAttachLog(name): + if ( self._shouldAttachLog(log.getName()) or + self._shouldAttachLog(name) ): a = MIMEText(log.getText().encode(ENCODING), _charset=ENCODING) a.add_header('Content-Disposition', "attachment", filename=name) m.attach(a) + #@todo: is there a better way to do this? # Add any extra headers that were requested, doing WithProperties - # interpolation if necessary + # interpolation if only one build was given if self.extraHeaders: - properties = build.getProperties() + if len(builds) == 1: + properties = builds[0].getProperties() + for k,v in self.extraHeaders.items(): - k = properties.render(k) + if len(builds == 1): + k = properties.render(k) if k in m: - twlog.msg("Warning: Got header " + k + " in self.extraHeaders " - "but it already exists in the Message - " - "not adding it.") - continue - m[k] = properties.render(v) - + twlog.msg("Warning: Got header " + k + + " in self.extraHeaders " + "but it already exists in the Message - " + "not adding it.") + continue + if len(builds == 1): + m[k] = properties.render(v) + else: + m[k] = v + return m - - def buildMessage(self, name, build, results): + + def buildMessageDict(self, name, build, results): if self.customMesg: # the customMesg stuff can be *huge*, so we prefer not to load it - attrs = self.getCustomMesgData(self.mode, name, build, results, self.master_status) + attrs = self.getCustomMesgData(self.mode, name, build, results, + self.master_status) text, type = self.customMesg(attrs) msgdict = { 'body' : text, 'type' : type } else: - msgdict = self.messageFormatter(self.mode, name, build, results, self.master_status) + msgdict = self.messageFormatter(self.mode, name, build, results, + self.master_status) + + return msgdict - patch = None - ss = build.getSourceStamp() - if ss and ss.patch and self.addPatch: - patch == ss.patch - logs = None - if self.addLogs: - logs = build.getLogs() + + def buildMessage(self, name, builds, results): + patches = [] + logs = [] + msgdict = {"body":""} + + for build in builds: + ss = build.getSourceStamp() + if ss and ss.patch and self.addPatch: + patches.append(ss.patch) + if self.addLogs: + logs.append(build.getLogs()) twlog.err("LOG: %s" % str(logs)) + + tmp = self.buildMessageDict(name=build.getBuilder().name, + build=build, results=build.results) + msgdict['body'] += tmp['body'] + msgdict['body'] += '\n\n' + msgdict['type'] = tmp['type'] + m = self.createEmail(msgdict, name, self.master_status.getTitle(), - results, build, patch, logs) + results, builds, patches, logs) # now, who is this message going to? dl = [] recipients = [] if self.sendToInterestedUsers and self.lookup: - for u in build.getInterestedUsers(): - d = defer.maybeDeferred(self.lookup.getAddress, u) - d.addCallback(recipients.append) - dl.append(d) + for build in builds: + for u in build.getInterestedUsers(): + d = defer.maybeDeferred(self.lookup.getAddress, u) + d.addCallback(recipients.append) + dl.append(d) d = defer.DeferredList(dl) d.addCallback(self._gotRecipients, recipients, m) return d diff --git a/master/buildbot/test/fake/fakedb.py b/master/buildbot/test/fake/fakedb.py index 6688cace3bd..d665a6fd363 100644 --- a/master/buildbot/test/fake/fakedb.py +++ b/master/buildbot/test/fake/fakedb.py @@ -963,6 +963,22 @@ def mkdt(epoch): number=row.number, start_time=mkdt(row.start_time), finish_time=mkdt(row.finish_time))) + + def getBuildsForRequest(self, brid): + ret = [] + def mkdt(epoch): + if epoch: + return epoch2datetime(epoch) + + for (id, row) in self.builds.items(): + if row.brid == brid: + ret.append(dict(bid = row.id, + brid=row.brid, + number=row.number, + start_time=mkdt(row.start_time), + finish_time=mkdt(row.finish_time))) + + return defer.succeed(ret) def addBuild(self, brid, number, _reactor=reactor): bid = self._newId() diff --git a/master/buildbot/test/unit/test_status_mail_MailNotifier.py b/master/buildbot/test/unit/test_status_mail_MailNotifier.py index 0848495821f..25363f19368 100644 --- a/master/buildbot/test/unit/test_status_mail_MailNotifier.py +++ b/master/buildbot/test/unit/test_status_mail_MailNotifier.py @@ -19,6 +19,14 @@ from buildbot.status.results import SUCCESS, FAILURE from buildbot.status.mail import MailNotifier +from buildbot.process.build import Build +from buildbot.process.builder import Builder +from buildbot.db.buildsets import BuildsetsConnectorComponent +from buildbot.db.builds import BuildsConnectorComponent +from buildbot.db.buildrequests import BuildRequestsConnectorComponent +from buildbot.status.master import Status +from twisted.internet import defer +from buildbot.test.fake import fakedb class FakeLog(object): def __init__(self, text): @@ -57,7 +65,7 @@ def test_createEmail_message_with_patch_and_log_contains_unicode(self): mn = MailNotifier('from@example.org', addLogs=True) m = mn.createEmail(msgdict, u'builder-n\u00E5me', u'project-n\u00E5me', SUCCESS, - build, patch, logs) + [build], [patch], logs) try: m.as_string() except UnicodeEncodeError: @@ -93,10 +101,64 @@ def test_builderAdded_subscribes_to_all_builders_by_default(self): def test_buildFinished_ignores_unspecified_builders(self): mn = MailNotifier('from@example.org', builders=['a','b']) - build = Mock() self.assertEqual(None, mn.buildFinished('dummyBuilder', build, SUCCESS)) + + def test_buildsetFinished_sends_email(self): + fakeBuildMessage = Mock() + mn = MailNotifier('from@example.org', + buildSetSummary=True, + mode="all", + builders=["Builder"]) + + mn.buildMessage = fakeBuildMessage + + def fakeGetBuild(number): + return build + + def fakeGetBuilder(buildername): + if buildername == builder.name: + return builder + return None + + def fakeGetBuildRequests(self, bsid): + return defer.succeed([{"buildername":"Builder", "brid":1}]) + + builder = Mock() + builder.getBuild = fakeGetBuild + builder.name = "Builder" + + build = Mock() + build.result = FAILURE + build.finished = True + build.reason = "testReason" + build.builder = builder + build.getBuilder = Mock() + build.getBuilder.return_value = builder + + + self.db = fakedb.FakeDBConnector(self) + self.db.insertTestData([fakedb.Buildset(id=99, sourcestampid=127, + results=SUCCESS, + reason="testReason"), + fakedb.BuildRequest(id=11, buildsetid=99, + buildername='Builder'), + fakedb.Build(number=0, brid=11, results=SUCCESS) + ]) + mn.parent = self + + self.status = Mock() + mn.master_status = Mock() + mn.master_status.getBuilder = fakeGetBuilder + mn.buildMessageDict = Mock() + mn.buildMessageDict.return_value = {"body":"body", "type":"text", + "subject":"subject"} + + mn.buildsetFinished(99, FAILURE) + fakeBuildMessage.assert_called_with("Buildset Complete: testReason", + [build], SUCCESS) + def test_buildFinished_ignores_unspecified_categories(self): mn = MailNotifier('from@example.org', categories=['fast']) @@ -117,7 +179,7 @@ def test_buildFinished_mode_all_always_sends_email(self): build = Mock() mn.buildFinished('dummyBuilder', build, FAILURE) - mock_method.assert_called_with('dummyBuilder', build, FAILURE) + mock_method.assert_called_with('dummyBuilder', [build], FAILURE) def test_buildFinished_mode_failing_ignores_successful_build(self): mn = MailNotifier('from@example.org', mode="failing") diff --git a/master/docs/cfg-statustargets.texinfo b/master/docs/cfg-statustargets.texinfo index e348a03ab51..c81012e37d9 100644 --- a/master/docs/cfg-statustargets.texinfo +++ b/master/docs/cfg-statustargets.texinfo @@ -947,6 +947,9 @@ log names, to send a subset of the logs. Defaults to False. Patches are usually used on a Try server. Defaults to True. +@item buildSetSummary +(boolean). If True, send a single summary email consisting of the concatenation of all build completion messages rather than a completion message for each build. Defaults to False. + @item relayhost (string). The host to which the outbound SMTP connection should be made. Defaults to 'localhost' @@ -994,7 +997,8 @@ text of the message. The @code{type} key is the message type ('plain' or @item extraHeaders (dictionary) A dictionary containing key/value pairs of extra headers to add -to sent e-mails. Both the keys and the values may be a WithProperties instance. +to sent e-mails. If buildSetStatus is False, both the keys and the values may be +a WithProperties instance. @end table