refactor python steps to not use log.getText

master/buildbot/steps/python.py

@@ -17,48 +17,46 @@
 import re
 
 from buildbot import config
+from buildbot.process import logobserver
 from buildbot.status.results import FAILURE
 from buildbot.status.results import SUCCESS
 from buildbot.status.results import WARNINGS
 from buildbot.steps.shell import ShellCommand
 
-try:
-    import cStringIO
-    StringIO = cStringIO.StringIO
-except ImportError:
-    from StringIO import StringIO
-
 
 class BuildEPYDoc(ShellCommand):
     name = "epydoc"
     command = ["make", "epydocs"]
     description = ["building", "epydocs"]
     descriptionDone = ["epydoc"]
 
-    def createSummary(self, log):
-        import_errors = 0
-        warnings = 0
-        errors = 0
+    def __init__(self, **kwargs):
+        ShellCommand.__init__(self, **kwargs)
+        self.addLogObserver(
+            'stdio', logobserver.LineConsumerLogObserver(self.logConsumer))
+
+    def logConsumer(self):
+        self.import_errors = 0
+        self.warnings = 0
+        self.errors = 0
 
-        for line in StringIO(log.getText()):
+        while True:
+            stream, line = yield
             if line.startswith("Error importing "):
-                import_errors += 1
+                self.import_errors += 1
             if line.find("Warning: ") != -1:
-                warnings += 1
+                self.warnings += 1
             if line.find("Error: ") != -1:
-                errors += 1
+                self.errors += 1
 
+    def createSummary(self, log):
         self.descriptionDone = self.descriptionDone[:]
-        if import_errors:
-            self.descriptionDone.append("ierr=%d" % import_errors)
-        if warnings:
-            self.descriptionDone.append("warn=%d" % warnings)
-        if errors:
-            self.descriptionDone.append("err=%d" % errors)
-
-        self.import_errors = import_errors
-        self.warnings = warnings
-        self.errors = errors
+        if self.import_errors:
+            self.descriptionDone.append("ierr=%d" % self.import_errors)
+        if self.warnings:
+            self.descriptionDone.append("warn=%d" % self.warnings)
+        if self.errors:
+            self.descriptionDone.append("err=%d" % self.errors)
 
     def evaluateCommand(self, cmd):
         if cmd.didFail():
@@ -87,16 +85,21 @@ def __init__(self, *args, **kwargs):
         # evaluateCommand below can inspect the results more closely.
         kwargs['decodeRC'] = {0: SUCCESS, 1: WARNINGS}
         ShellCommand.__init__(self, *args, **kwargs)
+        self.addLogObserver(
+            'stdio', logobserver.LineConsumerLogObserver(self.logConsumer))
 
-    def createSummary(self, log):
-        counts = {}
-        summaries = {}
+        counts = self.counts = {}
+        summaries = self.summaries = {}
         for m in self.MESSAGES:
             counts[m] = 0
             summaries[m] = []
 
+    def logConsumer(self):
+        counts = self.counts
+        summaries = self.summaries
         first = True
-        for line in StringIO(log.getText()).readlines():
+        while True:
+            stream, line = yield
             # the first few lines might contain echoed commands from a 'make
             # pyflakes' step, so don't count these as warnings. Stop ignoring
             # the initial lines as soon as we see one with a colon.
@@ -126,6 +129,8 @@ def createSummary(self, log):
             summaries[m].append(line)
             counts[m] += 1
 
+    def createSummary(self, log):
+        counts, summaries = self.counts, self.summaries
         self.descriptionDone = self.descriptionDone[:]
 
         # we log 'misc' as syntax-error
@@ -195,15 +200,21 @@ class PyLint(ShellCommand):
     _default_line_re = re.compile(r'^%s(\d{4})?: *\d+(,\d+)?:.+' % _msgtypes_re_str)
     _parseable_line_re = re.compile(r'[^:]+:\d+: \[%s(\d{4})?[,\]] .+' % _msgtypes_re_str)
 
-    def createSummary(self, log):
-        counts = {}
-        summaries = {}
+    def __init__(self, **kwargs):
+        ShellCommand.__init__(self, **kwargs)
+        self.addLogObserver(
+            'stdio', logobserver.LineConsumerLogObserver(self.logConsumer))
+
+    def logConsumer(self):
+        self.counts = {}
+        self.summaries = {}
         for m in self.MESSAGES:
-            counts[m] = 0
-            summaries[m] = []
+            self.counts[m] = 0
+            self.summaries[m] = []
 
         line_re = None  # decide after first match
-        for line in StringIO(log.getText()).readlines():
+        while True:
+            stream, line = yield
             if not line_re:
                 # need to test both and then decide on one
                 if self._parseable_line_re.match(line):
@@ -216,9 +227,11 @@ def createSummary(self, log):
             if mo:
                 msgtype = mo.group(self._re_groupname)
                 assert msgtype in self.MESSAGES
-            summaries[msgtype].append(line)
-            counts[msgtype] += 1
+            self.summaries[msgtype].append(line)
+            self.counts[msgtype] += 1
 
+    def createSummary(self, log):
+        counts, summaries = self.counts, self.summaries
         self.descriptionDone = self.descriptionDone[:]
         for msg, fullmsg in self.MESSAGES.items():
             if counts[msg]:
@@ -260,7 +273,6 @@ def __init__(self, sphinx_sourcedir='.', sphinx_builddir=None,
             config.error("Sphinx argument mode has to be 'incremental' or" +
                          "'full' is required")
 
-        self.warnings = 0
         self.success = False
         ShellCommand.__init__(self, **kwargs)
 
@@ -287,28 +299,32 @@ def __init__(self, sphinx_sourcedir='.', sphinx_builddir=None,
         command.extend([sphinx_sourcedir, sphinx_builddir])
         self.setCommand(command)
 
-    def createSummary(self, log):
+        self.addLogObserver(
+            'stdio', logobserver.LineConsumerLogObserver(self.logConsumer))
 
-        msgs = ['WARNING', 'ERROR', 'SEVERE']
+    msgs = ['WARNING', 'ERROR', 'SEVERE']
 
-        warnings = []
-        for line in log.getText().split('\n'):
+    def logConsumer(self):
+        self.warnings = []
+        while True:
+            stream, line = yield
             if (line.startswith('build succeeded')
                     or line.startswith('no targets are out of date.')):
                 self.success = True
             else:
-                for msg in msgs:
+                for msg in self.msgs:
                     if msg in line:
-                        warnings.append(line)
-                        self.warnings += 1
-        if self.warnings > 0:
-            self.addCompleteLog('warnings', "\n".join(warnings))
+                        self.warnings.append(line)
+
+    def createSummary(self, log):
+        if len(self.warnings) > 0:
+            self.addCompleteLog('warnings', "\n".join(self.warnings))
 
-        self.step_status.setStatistic('warnings', self.warnings)
+        self.step_status.setStatistic('warnings', len(self.warnings))
 
     def evaluateCommand(self, cmd):
         if self.success:
-            if self.warnings == 0:
+            if not self.warnings:
                 return SUCCESS
             else:
                 return WARNINGS
@@ -320,5 +336,5 @@ def describe(self, done=False):
             return ["building"]
 
         description = [self.name]
-        description.append('%d warnings' % self.warnings)
+        description.append('%d warnings' % len(self.warnings))
         return description

master/buildbot/test/unit/test_steps_python.py

@@ -78,6 +78,58 @@
 doesn't have a title: no link will be generated\
 '''
 
+# this is from a run of epydoc against the buildbot source..
+epydoc_output = '''\
+  [...............
++---------------------------------------------------------------------
+| In /home/dustin/code/buildbot/t/buildbot/master/buildbot/
+| ec2buildslave.py:
+| Import failed (but source code parsing was successful).
+|     Error: ImportError: No module named boto (line 19)
+|
+  [....
+Warning: Unable to extract the base list for
+         twisted.web.resource.EncodingResourceWrapper: Bad dotted name
+  [......
++---------------------------------------------------------------------
+| In /home/dustin/code/buildbot/t/buildbot/master/buildbot/buildslave/
+| ec2.py:
+| Import failed (but source code parsing was successful).
+|     Error: ImportError: No module named boto (line 28)
+|
+  [...........
++---------------------------------------------------------------------
+| In /home/dustin/code/buildbot/t/buildbot/master/buildbot/status/
+| status_push.py:
+| Import failed (but source code parsing was successful).
+|     Error: ImportError: No module named status_json (line 40)
+|
+  [....................<paragraph>Special descriptor for class __provides__</paragraph>
+'''
+
+
+class BuildEPYDoc(steps.BuildStepMixin, unittest.TestCase):
+
+    def setUp(self):
+        return self.setUpBuildStep()
+
+    def tearDown(self):
+        return self.tearDownBuildStep()
+
+    def test_sample(self):
+        self.setupStep(python.BuildEPYDoc())
+        self.expectCommands(
+            ExpectShell(workdir='wkdir', command=['make', 'epydocs'],
+                        usePTY='slave-config')
+            + ExpectShell.log('stdio',
+                              stdout=epydoc_output)
+            + 1,
+        )
+        self.expectOutcome(result=FAILURE,
+                           status_text=['epydoc', 'warn=1',
+                                        'err=3', 'failed'])
+        return self.runStep()
+
 
 class PyLint(steps.BuildStepMixin, unittest.TestCase):