add line-boundary finder for processing logs

master/buildbot/test/unit/test_util_lineboundaries.py

@@ -0,0 +1,70 @@
+# This file is part of Buildbot.  Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
+from twisted.trial import unittest
+from twisted.internet import defer, reactor
+from buildbot.util import lineboundaries
+
+class LBF(unittest.TestCase):
+
+    def setUp(self):
+        self.callbacks = []
+        self.lbf = lineboundaries.LineBoundaryFinder(self._callback)
+
+    def _callback(self, wholeLines):
+        self.assertEqual(wholeLines[-1], '\n', 'got %r' % (wholeLines))
+        self.callbacks.append(wholeLines)
+        d = defer.Deferred()
+        reactor.callLater(0, d.callback, None)
+        return d
+
+    def assertCallbacks(self, callbacks):
+        self.assertEqual(self.callbacks, callbacks)
+        self.callbacks = []
+
+    # tests
+
+    @defer.inlineCallbacks
+    def test_already_terminated(self):
+        yield self.lbf.append('abcd\ndefg\n')
+        self.assertCallbacks(['abcd\ndefg\n'])
+        yield self.lbf.append('xyz\n')
+        self.assertCallbacks(['xyz\n'])
+        yield self.lbf.flush()
+        self.assertCallbacks([])
+
+    @defer.inlineCallbacks
+    def test_partial_line(self):
+        for c in "hello\nworld":
+            yield self.lbf.append(c)
+        self.assertCallbacks(['hello\n'])
+        yield self.lbf.flush()
+        self.assertCallbacks(['world\n'])
+
+    @defer.inlineCallbacks
+    def test_embedded_newlines(self):
+        yield self.lbf.append('hello, ')
+        self.assertCallbacks([])
+        yield self.lbf.append('cruel\nworld')
+        self.assertCallbacks(['hello, cruel\n'])
+        yield self.lbf.flush()
+        self.assertCallbacks(['world\n'])
+
+    def test_empty_flush(self):
+        d = self.lbf.flush()
+        @d.addCallback
+        def check(_):
+            self.assertEqual(self.callbacks, [])
+        return d

master/buildbot/util/lineboundaries.py

@@ -0,0 +1,44 @@
+# This file is part of Buildbot.  Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
+from twisted.internet import defer
+
+class LineBoundaryFinder(object):
+
+    __slots_ = ['partialLine', 'callback']
+
+    def __init__(self, callback):
+        self.partialLine = None
+        self.callback = callback
+
+    def append(self, text):
+        if self.partialLine:
+            text = self.partialLine + text
+            self.partialLine = None
+        if text[-1] != '\n':
+            i = text.rfind('\n')
+            if i >= 0:
+                i = i + 1
+                text, self.partialLine = text[:i], text[i:]
+            else:
+                self.partialLine = text
+                return defer.succeed(None)
+        return self.callback(text)
+
+    def flush(self):
+        if self.partialLine:
+            return self.append('\n')
+        else:
+            return defer.succeed(None)

master/docs/developer/utils.rst

@@ -668,3 +668,29 @@ This module makes it easy to check argument types.
     :returns: boolean
 
     Is object a :ref:`identifier <type-identifier>`?
+
+buildbot.util.lineboundaries
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. py:module:: buildbot.util.lineboundaries
+
+.. py:class:: LineBoundaryFinder
+
+    This class accepts a sequence of arbitrary strings and invokes a callback only with complete (newline-terminated) substrings.
+    It buffers any partial lines until a subsequent newline is seen.
+
+    :param callback: asynchronous function to call with newline-terminated strings
+
+    .. py:method:: append(text)
+
+        :param text: text to append to the boundary finder
+        :returns: Deferred
+
+        Add additional text to the boundary finder.
+        If the addition of this text completes at least one line, the callback will be invoked with as many complete lines as possible.
+
+    .. py:method:: flush()
+
+        :returns: Deferred
+
+        Flush any remaining partial line by adding a newline and invoking the callback.