move log observers to buildbot.process.logobserver

master/buildbot/process/buildstep.py

@@ -27,7 +27,7 @@
 from buildbot.status import progress
 from buildbot.status.results import SUCCESS, WARNINGS, FAILURE, SKIPPED, \
      EXCEPTION, RETRY, worst_status
-from buildbot.process import remotecommand, properties
+from buildbot.process import remotecommand, logobserver, properties
 from buildbot.util.eventual import eventually
 
 class BuildStepFailed(Exception):
@@ -37,7 +37,12 @@ class BuildStepFailed(Exception):
 RemoteCommand = remotecommand.RemoteCommand
 LoggedRemoteCommand = remotecommand.LoggedRemoteCommand
 RemoteShellCommand = remotecommand.RemoteShellCommand
-_hush_pyflakes = [ RemoteCommand, LoggedRemoteCommand, RemoteShellCommand ]
+LogObserver = logobserver.LogObserver
+LogLineObserver = logobserver.LogLineObserver
+OutputProgressObserver = logobserver.OutputProgressObserver
+_hush_pyflakes = [
+    RemoteCommand, LoggedRemoteCommand, RemoteShellCommand,
+    LogObserver, LogLineObserver, OutputProgressObserver ]
 
 class LogObserver:
     implements(interfaces.ILogObserver)

master/buildbot/process/logobserver.py

@@ -0,0 +1,98 @@
+# This file is part of Buildbot.  Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
+from zope.interface import implements
+from twisted.protocols import basic
+from buildbot import interfaces
+
+
+class LogObserver:
+    implements(interfaces.ILogObserver)
+
+    def setStep(self, step):
+        self.step = step
+
+    def setLog(self, loog):
+        assert interfaces.IStatusLog.providedBy(loog)
+        loog.subscribe(self, True)
+
+    def logChunk(self, build, step, log, channel, text):
+        if channel == interfaces.LOG_CHANNEL_STDOUT:
+            self.outReceived(text)
+        elif channel == interfaces.LOG_CHANNEL_STDERR:
+            self.errReceived(text)
+
+    # TODO: add a logEnded method? er, stepFinished?
+
+    def outReceived(self, data):
+        """This will be called with chunks of stdout data. Override this in
+        your observer."""
+        pass
+
+    def errReceived(self, data):
+        """This will be called with chunks of stderr data. Override this in
+        your observer."""
+        pass
+
+
+class LogLineObserver(LogObserver):
+    def __init__(self):
+        self.stdoutParser = basic.LineOnlyReceiver()
+        self.stdoutParser.delimiter = "\n"
+        self.stdoutParser.lineReceived = self.outLineReceived
+        self.stdoutParser.transport = self # for the .disconnecting attribute
+        self.disconnecting = False
+
+        self.stderrParser = basic.LineOnlyReceiver()
+        self.stderrParser.delimiter = "\n"
+        self.stderrParser.lineReceived = self.errLineReceived
+        self.stderrParser.transport = self
+
+    def setMaxLineLength(self, max_length):
+        """
+        Set the maximum line length: lines longer than max_length are
+        dropped.  Default is 16384 bytes.  Use sys.maxint for effective
+        infinity.
+        """
+        self.stdoutParser.MAX_LENGTH = max_length
+        self.stderrParser.MAX_LENGTH = max_length
+
+    def outReceived(self, data):
+        self.stdoutParser.dataReceived(data)
+
+    def errReceived(self, data):
+        self.stderrParser.dataReceived(data)
+
+    def outLineReceived(self, line):
+        """This will be called with complete stdout lines (not including the
+        delimiter). Override this in your observer."""
+        pass
+
+    def errLineReceived(self, line):
+        """This will be called with complete lines of stderr (not including
+        the delimiter). Override this in your observer."""
+        pass
+
+
+class OutputProgressObserver(LogObserver):
+    length = 0
+
+    def __init__(self, name):
+        self.name = name
+
+    def logChunk(self, build, step, log, channel, text):
+        self.length += len(text)
+        self.step.setProgress(self.name, self.length)
+

master/buildbot/process/subunitlogobserver.py

@@ -13,100 +13,6 @@
 #
 # Copyright Buildbot Team Members
 
-
-from unittest import TestResult
-from StringIO import StringIO
-
-from buildbot.process import buildstep
-from buildbot.status.testresult import TestResult as aTestResult
-from buildbot.status.results import SUCCESS, FAILURE, SKIPPED
-
-class SubunitLogObserver(buildstep.LogLineObserver, TestResult):
-    """Observe a log that may contain subunit output.
-
-    This class extends TestResult to receive the callbacks from the subunit
-    parser in the most direct fashion.
-    """
-
-    def __init__(self):
-        buildstep.LogLineObserver.__init__(self)
-        TestResult.__init__(self)
-        try:
-            from subunit import TestProtocolServer, PROGRESS_CUR, PROGRESS_SET
-            from subunit import PROGRESS_PUSH, PROGRESS_POP
-        except ImportError:
-            raise ImportError("subunit is not importable, but is required for "
-                "SubunitLogObserver support.")
-        self.PROGRESS_CUR = PROGRESS_CUR
-        self.PROGRESS_SET = PROGRESS_SET
-        self.PROGRESS_PUSH = PROGRESS_PUSH
-        self.PROGRESS_POP = PROGRESS_POP
-        self.warningio = StringIO()
-        self.protocol = TestProtocolServer(self, self.warningio)
-        self.skips = []
-        self.seen_tags = set() #don't yet know what tags does in subunit
-
-    def outLineReceived(self, line):
-        """Process a received stdout line."""
-        # Impedance mismatch: subunit wants lines, observers get lines-no\n
-        self.protocol.lineReceived(line + '\n')
-
-    def errLineReceived(self, line):
-        """same for stderr line."""
-        self.protocol.lineReceived(line + '\n')
-
-    def stopTest(self, test):
-        TestResult.stopTest(self, test)
-        self.step.setProgress('tests', self.testsRun)
-
-    def addSuccess(self, test):
-        TestResult.addSuccess(self, test)
-        self.addAResult(test, SUCCESS, 'SUCCESS')
-
-    def addSkip(self, test, detail):
-        if hasattr(TestResult,'addSkip'):
-            TestResult.addSkip(self, test, detail)
-        else:
-            self.skips.append((test, detail))
-        self.addAResult(test, SKIPPED, 'SKIPPED', detail)
-
-    def addError(self, test, err):
-        TestResult.addError(self, test, err)
-        self.issue(test, err)
-
-    def addFailure(self, test, err):
-        TestResult.addFailure(self, test, err)
-        self.issue(test, err)
-
-    def addAResult(self, test, result, text, log=""):
-        tr = aTestResult(tuple(test.id().split('.')), result, text, log)
-        self.step.build.build_status.addTestResult(tr)
-
-    def issue(self, test, err):
-        """An issue - failing, erroring etc test."""
-        self.addAResult(test, FAILURE, 'FAILURE', err)
-        self.step.setProgress('tests failed', len(self.failures) + 
-            len(self.errors))
-
-    expectedTests = 0
-    contextLevel = 0
-    def progress(self, offset, whence):
-        if not self.contextLevel:
-            if whence == self.PROGRESS_CUR:
-                self.expectedTests += offset
-            elif whence == self.PROGRESS_SET:
-                self.expectedTests = offset
-            self.step.progress.setExpectations({'tests': self.expectedTests})
-        #TODO: properly support PUSH/POP
-        if whence == self.PROGRESS_PUSH:
-            self.contextLevel += 1
-        elif whence == self.PROGRESS_POP:
-            self.contextLevel -= 1
-
-    def tags(self, new_tags, gone_tags):
-        """Accumulate the seen tags."""
-        self.seen_tags.update(new_tags)
-
 # this used to be referenced here, so we keep a link for old time's sake
 import buildbot.steps.subunit
 SubunitShellCommand = buildbot.steps.subunit.SubunitShellCommand

master/buildbot/steps/package/rpm/mock.py

@@ -20,10 +20,10 @@
 import re
 
 from buildbot.steps.shell import ShellCommand
-from buildbot.process import buildstep, remotecommand
+from buildbot.process import logobserver, remotecommand
 from buildbot import config
 
-class MockStateObserver(buildstep.LogLineObserver):
+class MockStateObserver(logobserver.LogLineObserver):
     _line_re = re.compile(r'^.*State Changed: (.*)$')
 
     def outLineReceived(self, line):

master/buildbot/steps/subunit.py

@@ -14,8 +14,98 @@
 # Copyright Buildbot Team Members
 
 
+from unittest import TestResult
+from StringIO import StringIO
+from buildbot.process import logobserver
+from buildbot.status.testresult import TestResult as aTestResult
+from buildbot.status.results import SUCCESS, FAILURE, SKIPPED
 from buildbot.steps.shell import ShellCommand
-from buildbot.status.results import SUCCESS, FAILURE
+
+class SubunitLogObserver(logobserver.LogLineObserver, TestResult):
+    """Observe a log that may contain subunit output.
+
+    This class extends TestResult to receive the callbacks from the subunit
+    parser in the most direct fashion.
+    """
+
+    def __init__(self):
+        logobserver.LogLineObserver.__init__(self)
+        TestResult.__init__(self)
+        try:
+            from subunit import TestProtocolServer, PROGRESS_CUR, PROGRESS_SET
+            from subunit import PROGRESS_PUSH, PROGRESS_POP
+        except ImportError:
+            raise ImportError("subunit is not importable, but is required for "
+                "SubunitLogObserver support.")
+        self.PROGRESS_CUR = PROGRESS_CUR
+        self.PROGRESS_SET = PROGRESS_SET
+        self.PROGRESS_PUSH = PROGRESS_PUSH
+        self.PROGRESS_POP = PROGRESS_POP
+        self.warningio = StringIO()
+        self.protocol = TestProtocolServer(self, self.warningio)
+        self.skips = []
+        self.seen_tags = set() #don't yet know what tags does in subunit
+
+    def outLineReceived(self, line):
+        """Process a received stdout line."""
+        # Impedance mismatch: subunit wants lines, observers get lines-no\n
+        self.protocol.lineReceived(line + '\n')
+
+    def errLineReceived(self, line):
+        """same for stderr line."""
+        self.protocol.lineReceived(line + '\n')
+
+    def stopTest(self, test):
+        TestResult.stopTest(self, test)
+        self.step.setProgress('tests', self.testsRun)
+
+    def addSuccess(self, test):
+        TestResult.addSuccess(self, test)
+        self.addAResult(test, SUCCESS, 'SUCCESS')
+
+    def addSkip(self, test, detail):
+        if hasattr(TestResult,'addSkip'):
+            TestResult.addSkip(self, test, detail)
+        else:
+            self.skips.append((test, detail))
+        self.addAResult(test, SKIPPED, 'SKIPPED', detail)
+
+    def addError(self, test, err):
+        TestResult.addError(self, test, err)
+        self.issue(test, err)
+
+    def addFailure(self, test, err):
+        TestResult.addFailure(self, test, err)
+        self.issue(test, err)
+
+    def addAResult(self, test, result, text, log=""):
+        tr = aTestResult(tuple(test.id().split('.')), result, text, log)
+        self.step.build.build_status.addTestResult(tr)
+
+    def issue(self, test, err):
+        """An issue - failing, erroring etc test."""
+        self.addAResult(test, FAILURE, 'FAILURE', err)
+        self.step.setProgress('tests failed', len(self.failures) + 
+            len(self.errors))
+
+    expectedTests = 0
+    contextLevel = 0
+    def progress(self, offset, whence):
+        if not self.contextLevel:
+            if whence == self.PROGRESS_CUR:
+                self.expectedTests += offset
+            elif whence == self.PROGRESS_SET:
+                self.expectedTests = offset
+            self.step.progress.setExpectations({'tests': self.expectedTests})
+        #TODO: properly support PUSH/POP
+        if whence == self.PROGRESS_PUSH:
+            self.contextLevel += 1
+        elif whence == self.PROGRESS_POP:
+            self.contextLevel -= 1
+
+    def tags(self, new_tags, gone_tags):
+        """Accumulate the seen tags."""
+        self.seen_tags.update(new_tags)
 
 class SubunitShellCommand(ShellCommand):
     """A ShellCommand that sniffs subunit output.
@@ -25,16 +115,13 @@ def __init__(self, failureOnNoTests=False, *args, **kwargs):
         ShellCommand.__init__(self, *args, **kwargs)
         self.failureOnNoTests = failureOnNoTests
 
-        # importing here gets around an import loop
-        from buildbot.process import subunitlogobserver
-
-        self.ioObverser = subunitlogobserver.SubunitLogObserver()
-        self.addLogObserver('stdio', self.ioObverser)
+        self.ioObserver = SubunitLogObserver()
+        self.addLogObserver('stdio', self.ioObserver)
         self.progressMetrics = self.progressMetrics + ('tests', 'tests failed')
 
     def commandComplete(self, cmd):
         # figure out all statistics about the run
-        ob = self.ioObverser
+        ob = self.ioObserver
         failures = len(ob.failures)
         errors = len(ob.errors)
         skips = len(ob.skips)
@@ -86,7 +173,7 @@ def evaluateCommand(self, cmd):
         return self.results
 
     def createSummary(self, loog):
-        ob = self.ioObverser
+        ob = self.ioObserver
         problems = ""
         for test, err in ob.errors + ob.failures:
             problems += "%s\n%s" % (test.id(), err)

master/buildbot/test/regressions/test_oldpaths.py

@@ -215,3 +215,10 @@ def test_buildstep_remotecommand(self):
         assert RemoteCommand
         assert LoggedRemoteCommand
         assert RemoteShellCommand
+
+    def test_buildstep_logobserver(self):
+        from buildbot.process.buildstep import LogObserver, \
+                LogLineObserver, OutputProgressObserver
+        assert LogObserver
+        assert LogLineObserver
+        assert OutputProgressObserver

master/buildbot/test/unit/test_steps_subunit.py

@@ -19,7 +19,6 @@
 from twisted.trial import unittest
 from buildbot import interfaces
 from buildbot.steps import subunit
-from buildbot.process import subunitlogobserver
 from buildbot.status.results import SUCCESS, FAILURE
 from buildbot.test.fake.remotecommand import ExpectShell
 from buildbot.test.util import steps
@@ -36,7 +35,7 @@ def setUp(self):
         self.logobserver.skips = []
         self.logobserver.testsRun = 0
         self.logobserver.warningio = StringIO.StringIO()
-        self.patch(subunitlogobserver, 'SubunitLogObserver',
+        self.patch(subunit, 'SubunitLogObserver',
                                 lambda : self.logobserver)
         return self.setUpBuildStep()