Merge branch '9/use-data-api-4' of git://github.com/djmitche/buildbot…

… into nine

master/buildbot/process/logobserver.py

@@ -18,7 +18,7 @@
 from zope.interface import implements
 
 
-class LogObserver:
+class LogObserver(object):
     implements(interfaces.ILogObserver)
 
     def setStep(self, step):
@@ -113,5 +113,37 @@ def __init__(self, name):
         self.name = name
 
     def gotData(self, stream, data):
-        self.length += len(data)
+        if data:
+            self.length += len(data)
         self.step.setProgress(self.name, self.length)
+
+
+class BufferLogObserver(LogObserver):
+
+    def __init__(self, wantStdout=True, wantStderr=False):
+        LogObserver.__init__(self)
+        self.stdout = [] if wantStdout else None
+        self.stderr = [] if wantStderr else None
+
+    def outReceived(self, data):
+        if self.stdout is not None:
+            self.stdout.append(data)
+
+    def errReceived(self, data):
+        if self.stderr is not None:
+            self.stderr.append(data)
+
+    def _get(self, chunks):
+        if chunks is None:
+            return [u'']
+        if len(chunks) > 1:
+            chunks = [''.join(chunks)]
+        elif not chunks:
+            chunks = [u'']
+        return chunks
+
+    def getStdout(self):
+        return self._get(self.stdout)[0]
+
+    def getStderr(self):
+        return self._get(self.stderr)[0]

master/buildbot/steps/shell.py

@@ -19,9 +19,8 @@
 
 from buildbot import config
 from buildbot.process import buildstep
+from buildbot.process import logobserver
 from buildbot.process import remotecommand
-from buildbot.status.logfile import STDERR
-from buildbot.status.logfile import STDOUT
 from buildbot.status.results import FAILURE
 from buildbot.status.results import SUCCESS
 from buildbot.status.results import WARNINGS
@@ -317,6 +316,11 @@ def __init__(self, property=None, extract_fn=None, strip=True, **kwargs):
 
         ShellCommand.__init__(self, **kwargs)
 
+        if self.extract_fn:
+            self.observer = logobserver.BufferLogObserver(wantStdout=True,
+                                                          wantStderr=True)
+            self.addLogObserver('stdio', self.observer)
+
         self.property_changes = {}
 
     def commandComplete(self, cmd):
@@ -330,10 +334,9 @@ def commandComplete(self, cmd):
             self.setProperty(propname, result, "SetProperty Step")
             self.property_changes[propname] = result
         else:
-            log = cmd.logs['stdio']
             new_props = self.extract_fn(cmd.rc,
-                                        ''.join(log.getChunks([STDOUT], onlyText=True)),
-                                        ''.join(log.getChunks([STDERR], onlyText=True)))
+                                        self.observer.getStdout(),
+                                        self.observer.getStderr())
             for k, v in new_props.items():
                 self.setProperty(k, v, "SetProperty Step")
             self.property_changes = new_props

master/buildbot/test/fake/logfile.py

@@ -95,14 +95,3 @@ def getText(self):
         warnings.warn("step uses removed LogFile method `getText`")
         return ''.join([c for str, c in self.chunks
                         if str in (STDOUT, STDERR)])
-
-    def getChunks(self, channels=[], onlyText=False):
-        warnings.warn("step uses removed LogFile method `getChunks`")
-        if onlyText:
-            return [data
-                    for (ch, data) in self.chunks
-                    if not channels or ch in channels]
-        else:
-            return [(ch, data)
-                    for (ch, data) in self.chunks
-                    if not channels or ch in channels]

master/buildbot/test/unit/test_process_log.py

@@ -299,7 +299,3 @@ def test_signature_getText_removed(self):
     def test_signature_readlines_removed(self):
         InterfaceTests.test_signature_readlines_removed(self)
     test_signature_readlines_removed.todo = "not removed yet"
-
-    def test_signature_getChunks_removed(self):
-        InterfaceTests.test_signature_getChunks_removed(self)
-    test_signature_getChunks_removed.todo = "not removed yet"

master/buildbot/test/unit/test_process_logobserver.py

@@ -13,6 +13,8 @@
 #
 # Copyright Buildbot Team Members
 
+import mock
+
 from buildbot.process import log
 from buildbot.process import logobserver
 from buildbot.test.fake import fakemaster
@@ -120,3 +122,54 @@ def test_old_setMaxLineLength(self):
         # callable.  Just don't fail.
         lo = MyLogLineObserver()
         lo.setMaxLineLength(120939403)
+
+
+class TestOutputProgressObserver(unittest.TestCase):
+
+    def setUp(self):
+        self.master = fakemaster.make_master(testcase=self, wantData=True)
+
+    @defer.inlineCallbacks
+    def test_sequence(self):
+        logid = yield self.master.data.updates.newLog(1, u'mine', u's')
+        l = log.Log.new(self.master, 'mine', 's', logid, 'utf-8')
+        lo = logobserver.OutputProgressObserver('stdio')
+        step = mock.Mock()
+        lo.setStep(step)
+        lo.setLog(l)
+
+        yield l.addStdout(u'hello\n')
+        step.setProgress.assert_called_with('stdio', 6)
+        yield l.finish()
+
+
+class TestBufferObserver(unittest.TestCase):
+
+    def setUp(self):
+        self.master = fakemaster.make_master(testcase=self, wantData=True)
+
+    @defer.inlineCallbacks
+    def do_test_sequence(self, lo):
+        logid = yield self.master.data.updates.newLog(1, u'mine', u's')
+        l = log.Log.new(self.master, 'mine', 's', logid, 'utf-8')
+        lo.setLog(l)
+
+        yield l.addStdout(u'hello\n')
+        yield l.addStderr(u'cruel\n')
+        yield l.addStdout(u'multi\nline\nchunk\n')
+        yield l.addHeader(u'H1\nH2\n')
+        yield l.finish()
+
+    @defer.inlineCallbacks
+    def test_stdout_only(self):
+        lo = logobserver.BufferLogObserver(wantStdout=True, wantStderr=False)
+        yield self.do_test_sequence(lo)
+        self.assertEqual(lo.getStdout(), u'hello\nmulti\nline\nchunk\n')
+        self.assertEqual(lo.getStderr(), u'')
+
+    @defer.inlineCallbacks
+    def test_both(self):
+        lo = logobserver.BufferLogObserver(wantStdout=True, wantStderr=True)
+        yield self.do_test_sequence(lo)
+        self.assertEqual(lo.getStdout(), u'hello\nmulti\nline\nchunk\n')
+        self.assertEqual(lo.getStderr(), u'cruel\n')

master/docs/developer/cls-logobserver.rst

@@ -65,3 +65,23 @@ LogObservers
     .. py:method:: finishReceived()
 
         This method, inherited from :py:class:`LogObserver`, is invoked when the observed log is finished.
+
+.. py:class:: BufferLogObserver(wantStdout=True, wantStderr=False)
+
+    :param boolean wantStdout: true if stdout should be buffered
+    :param boolean wantStderr: true if stderr should be buffered
+
+    This subclass of :py:class:`LogObserver` buffers stdout and/or stderr for analysis after the step is complete.
+    This can cause excessive memory consumption if the output is large.
+
+    .. py:method:: getStdout()
+
+        :returns: unicode string
+
+        Return the accumulated stdout.
+
+    .. py:method:: getStderr()
+
+        :returns: unicode string
+
+        Return the accumulated stderr.

master/docs/manual/cfg-buildsteps.rst

@@ -3061,6 +3061,8 @@ stderr is lost.  For example, given ::
 Then ``my_extract`` will see ``stdout="output1\noutput2\n"``
 and ``stderr="error\n"``.
 
+Avoid using the ``extract_fn`` form of this step with commands that produce a great deal of output, as the output is buffered in memory until complete.
+
 .. bb:step:: SetPropertiesFromEnv
 
 .. py:class:: buildbot.steps.slave.SetPropertiesFromEnv