Merge branch '9/use-data-api-6' of git://github.com/djmitche/buildbot…

… into nine

README.md

@@ -312,6 +312,10 @@ Here are the remaining bits:
   * merge rewrites to master
   * remove `step_status` for statistics
 
+* update documentation
+  * lots of changes in `customization.rst`
+  * `cfg-statustargets.rst` references `getText`
+
 ### Importing old Build Pickles ###
 
 The most likely plan for data from 0.8.x implementations is to ignore it.

master/buildbot/process/buildstep.py

@@ -13,8 +13,6 @@
 #
 # Copyright Buildbot Team Members
 
-import re
-
 from twisted.internet import defer
 from twisted.internet import error
 from twisted.python import components
@@ -41,7 +39,6 @@
 from buildbot.status.results import SKIPPED
 from buildbot.status.results import SUCCESS
 from buildbot.status.results import WARNINGS
-from buildbot.status.results import worst_status
 
 
 class BuildStepFailed(Exception):
@@ -808,27 +805,6 @@ def setStatus(self, cmd, results):
         return defer.succeed(None)
 
 
-# Parses the logs for a list of regexs. Meant to be invoked like:
-# regexes = ((re.compile(...), FAILURE), (re.compile(...), WARNINGS))
-# self.addStep(ShellCommand,
-#   command=...,
-#   ...,
-#   log_eval_func=lambda c,s: regex_log_evaluator(c, s, regexs)
-# )
-def regex_log_evaluator(cmd, step_status, regexes):
-    worst = cmd.results()
-    for err, possible_status in regexes:
-        # worst_status returns the worse of the two status' passed to it.
-        # we won't be changing "worst" unless possible_status is worse than it,
-        # so we don't even need to check the log if that's the case
-        if worst_status(worst, possible_status) == possible_status:
-            if isinstance(err, (basestring)):
-                err = re.compile(".*%s.*" % err, re.DOTALL)
-            for l in cmd.logs.values():
-                if err.search(l.getText()):
-                    worst = possible_status
-    return worst
-
 # (WithProperties used to be available in this module)
 from buildbot.process.properties import WithProperties
 _hush_pyflakes = [WithProperties]

master/buildbot/process/logobserver.py

@@ -106,6 +106,37 @@ def headerLineReceived(self, line):
         pass
 
 
+class LineConsumerLogObserver(LogLineObserver):
+
+    def __init__(self, consumerFunction):
+        LogLineObserver.__init__(self)
+        self.generator = None
+        self.consumerFunction = consumerFunction
+
+    def feed(self, input):
+        # note that we defer starting the generator until the first bit of
+        # data, since the observer may be instantiated during configuration as
+        # well as for each execution of the step.
+        self.generator = self.consumerFunction()
+        self.generator.next()
+        # shortcut all remaining feed operations
+        self.feed = self.generator.send
+        self.feed(input)
+
+    def outLineReceived(self, line):
+        self.feed(('o', line))
+
+    def errLineReceived(self, line):
+        self.feed(('e', line))
+
+    def headerLineReceived(self, line):
+        self.feed(('h', line))
+
+    def finishReceived(self):
+        if self.generator:
+            self.generator.close()
+
+
 class OutputProgressObserver(LogObserver):
     length = 0
 

master/buildbot/steps/package/deb/pbuilder.py

@@ -24,6 +24,7 @@
 from twisted.python import log
 
 from buildbot import config
+from buildbot.process import logobserver
 from buildbot.process import remotecommand
 from buildbot.process.buildstep import FAILURE
 from buildbot.steps.shell import WarningCountingShellCommand
@@ -120,6 +121,9 @@ def __init__(self,
 
         self.suppressions.append((None, re.compile(r"\.pbuilderrc does not exist"), None, None))
 
+        self.addLogObserver(
+            'stdio', logobserver.LineConsumerLogObserver(self.logConsumer))
+
     # Check for Basetgz
     def start(self):
         cmd = remotecommand.RemoteCommand('stat', {'file': self.basetgz})
@@ -182,11 +186,13 @@ def startBuild(self, cmd):
         else:
             return WarningCountingShellCommand.start(self)
 
-    def commandComplete(self, cmd):
-        out = cmd.logs['stdio'].getText()
-        m = re.search(r"dpkg-genchanges  >\.\./(.+\.changes)", out)
-        if m:
-            self.setProperty("deb-changes", m.group(1), "DebPbuilder")
+    def logConsumer(self):
+        r = re.compile(r"dpkg-genchanges  >\.\./(.+\.changes)")
+        while True:
+            stream, line = yield
+            mo = r.search(line)
+            if mo:
+                self.setProperty("deb-changes", mo.group(1), "DebPbuilder")
 
 
 class DebCowbuilder(DebPbuilder):

master/buildbot/steps/package/rpm/mock.py

@@ -144,12 +144,16 @@ def __init__(self,
 
         self.command += ['--buildsrpm', '--spec', self.spec,
                          '--sources', self.sources]
-
-    def commandComplete(self, cmd):
-        out = cmd.logs['build.log'].getText()
-        m = re.search(r"Wrote: .*/([^/]*.src.rpm)", out)
-        if m:
-            self.setProperty("srpm", m.group(1), 'MockBuildSRPM')
+        self.addLogObserver(
+            'stdio', logobserver.LineConsumerLogObserver(self.logConsumer))
+
+    def logConsumer(self):
+        r = re.compile(r"Wrote: .*/([^/]*.src.rpm)")
+        while True:
+            stream, line = yield
+            m = r.search(line)
+            if m:
+                self.setProperty("srpm", m.group(1), 'MockBuildSRPM')
 
 
 class MockRebuild(Mock):

master/buildbot/steps/package/rpm/rpmbuild.py

@@ -12,15 +12,16 @@
 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 #
 # Portions Copyright Buildbot Team Members
-
-from __future__ import with_statement
 # Portions Copyright Dan Radez <dradez+buildbot@redhat.com>
 # Portions Copyright Steve 'Ashcrow' Milner <smilner+buildbot@redhat.com>
 
+from __future__ import with_statement
+
 import os
 
 from buildbot import config
 from buildbot.process import buildstep
+from buildbot.process import logobserver
 from buildbot.steps.shell import ShellCommand
 
 
@@ -86,6 +87,9 @@ def __init__(self,
         if not self.specfile:
             config.error("You must specify a specfile")
 
+        self.addLogObserver(
+            'stdio', logobserver.LineConsumerLogObserver(self.logConsumer))
+
     def start(self):
         if self.autoRelease:
             relfile = '%s.release' % (
@@ -116,25 +120,29 @@ def start(self):
         cmd = buildstep.RemoteShellCommand(**kwargs)
         self.setupEnvironment(cmd)
         self.startCommand(cmd)
+        self.addLogObserver(
+            'stdio', logobserver.LineConsumerLogObserver(self.logConsumer))
 
-    def createSummary(self, log):
+    def logConsumer(self):
         rpm_prefixes = ['Provides:', 'Requires(', 'Requires:',
                         'Checking for unpackaged', 'Wrote:',
                         'Executing(%', '+ ', 'Processing files:']
         rpm_err_pfx = ['   ', 'RPM build errors:', 'error: ']
+        self.rpmcmdlog = []
+        self.rpmerrors = []
 
-        rpmcmdlog = []
-        rpmerrors = []
-
-        for line in log.getText().splitlines(True):
+        while True:
+            stream, line = yield
             for pfx in rpm_prefixes:
                 if line.startswith(pfx):
-                    rpmcmdlog.append(line)
+                    self.rpmcmdlog.append(line)
                     break
             for err in rpm_err_pfx:
                 if line.startswith(err):
-                    rpmerrors.append(line)
+                    self.rpmerrors.append(line)
                     break
-        self.addCompleteLog('RPM Command Log', "".join(rpmcmdlog))
-        if rpmerrors:
-            self.addCompleteLog('RPM Errors', "".join(rpmerrors))
+
+    def createSummary(self, log):
+        self.addCompleteLog('RPM Command Log', "".join(self.rpmcmdlog))
+        if self.rpmerrors:
+            self.addCompleteLog('RPM Errors', "".join(self.rpmerrors))