Initial implementation of logMaxSize

buildbot/master.py

@@ -545,7 +545,7 @@ def loadConfig(self, f):
                       "manhole", "status", "projectName", "projectURL",
                       "buildbotURL", "properties", "prioritizeBuilders",
                       "eventHorizon", "buildCacheSize", "logHorizon", "buildHorizon",
-                      "changeHorizon",
+                      "changeHorizon", "logMaxSize",
                       )
         for k in config.keys():
             if k not in known_keys:
@@ -582,6 +582,10 @@ def loadConfig(self, f):
             if logCompressionLimit is not None and not \
                     isinstance(logCompressionLimit, int):
                 raise ValueError("logCompressionLimit needs to be bool or int")
+            logMaxSize = config.get('logMaxSize')
+            if logMaxSize is not None and not \
+                    isinstance(logMaxSize, int):
+                raise ValueError("logMaxSize needs to be None or int")
             mergeRequests = config.get('mergeRequests')
             if mergeRequests is not None and not callable(mergeRequests):
                 raise ValueError("mergeRequests must be a callable")
@@ -752,6 +756,8 @@ def loadConfig(self, f):
         self.properties.update(properties, self.configFileName)
         if logCompressionLimit is not None:
             self.status.logCompressionLimit = logCompressionLimit
+        if logMaxSize is not None:
+            self.status.logMaxSize = logMaxSize
         if mergeRequests is not None:
             self.botmaster.mergeRequests = mergeRequests
         if prioritizeBuilders is not None:

buildbot/status/builder.py

@@ -226,6 +226,8 @@ class LogFile:
     length = 0
     chunkSize = 10*1000
     runLength = 0
+    maxLength = None
+    maxLengthExceeded = False
     runEntries = [] # provided so old pickled builds will getChunks() ok
     entries = None
     BUFFERSIZE = 2048
@@ -409,6 +411,17 @@ def merge(self):
 
     def addEntry(self, channel, text):
         assert not self.finished
+
+        if channel != HEADER:
+            # Truncate the log if it's more than maxLength bytes
+            if self.maxLength and self.length > self.maxLength:
+                if not self.maxLengthExceeded:
+                    # Add a message about what's going on
+                    self.addEntry(HEADER,
+                            "\noutput exceeded %i bytes, remaining output has been truncated\n" % self.maxLength)
+                    self.maxLengthExceeded = True
+                return
+
         # we only add to .runEntries here. merge() is responsible for adding
         # merged chunks to .entries
         if self.runEntries and channel != self.runEntries[0][0]:
@@ -894,6 +907,7 @@ def addLog(self, name):
         assert self.started # addLog before stepStarted won't notify watchers
         logfilename = self.build.generateLogfileName(self.name, name)
         log = LogFile(self, name, logfilename)
+        log.maxLength = self.build.builder.logMaxSize
         self.logs.append(log)
         for w in self.watchers:
             receiver = w.logStarted(self.build, self, log)
@@ -1537,6 +1551,9 @@ def determineNextBuildNumber(self):
     def setLogCompressionLimit(self, lowerLimit):
         self.logCompressionLimit = lowerLimit
 
+    def setLogMaxSize(self, upperLimit):
+        self.logMaxSize = upperLimit
+
     def saveYourself(self):
         for b in self.currentBuilds:
             if not b.isFinished:
@@ -2050,6 +2067,8 @@ def __init__(self, botmaster, basedir):
         assert os.path.isdir(basedir)
         # compress logs bigger than 4k, a good default on linux
         self.logCompressionLimit = 4*1024
+        # No default limit to the log size
+        self.logMaxSize = None
 
 
     # methods called by our clients
@@ -2263,6 +2282,7 @@ def builderAdded(self, name, basedir, category=None):
 
         builder_status.setBigState("offline")
         builder_status.setLogCompressionLimit(self.logCompressionLimit)
+        builder_status.setLogMaxSize(self.logMaxSize)
 
         for t in self.watchers:
             self.announceNewBuilder(t, name, builder_status)

buildbot/test/test_limitlogs.py

@@ -10,6 +10,9 @@
 from buildbot.test.runutils import SignalMixin
 import os
 
+from buildbot.test.runutils import RunMixin, rmtree
+from buildbot.changes import changes
+
 '''Testcases to verify that the --log-size and --log-count options to
 create-master and create-slave actually work.
 
@@ -97,3 +100,66 @@ def _stop_master(self):
 
     def _master_stopped(self, res):
         log.msg("master stopped")
+
+builder_cfg = """from buildbot.process import factory
+from buildbot.steps.shell import ShellCommand
+from buildbot.buildslave import BuildSlave
+
+f1 = factory.BuildFactory([
+    ShellCommand(command=['echo', 'really long string'*50]),
+    ])
+
+BuildmasterConfig = c = {}
+c['slaves'] = [BuildSlave('bot1', 'sekrit')]
+from buildbot.scheduler import Scheduler
+c['schedulers'] = [Scheduler('dummy', None, 0.1, ['dummy'])]
+c['builders'] = []
+c['builders'].append({'name':'dummy', 'slavename':'bot1',
+                      'builddir': 'dummy', 'factory': f1})
+c['slavePortnum'] = 0
+c['logMaxSize'] = 150
+"""
+class BuilderLogs(RunMixin, unittest.TestCase):
+    '''Limit builder log size'''
+
+    def testLogMaxSize(self):
+        rmtree("basedir")
+        os.mkdir("basedir")
+        d = self.master.loadConfig(builder_cfg)
+        d.addCallback(lambda res: self.master.startService())
+        d.addCallback(lambda res: self.connectSlave())
+
+        # Trigger a change
+        def _send(res):
+            cm = self.master.change_svc
+            c = changes.Change("bob", ["Makefile", "foo/bar.c"],
+                               "changed stuff")
+            cm.addChange(c)
+        d.addCallback(_send)
+
+        # Delay for a bit, so we know we're building
+        def _delay(res):
+            d1 = defer.Deferred()
+            reactor.callLater(0.5, d1.callback, None)
+            return d1
+        d.addCallback(_delay)
+
+        # Wait until the build is done
+        def _waitForBuild(res):
+            b = self.master.botmaster.builders['dummy']
+            if len(b.builder_status.currentBuilds) > 0:
+                return b.builder_status.currentBuilds[0].waitUntilFinished()
+            else:
+                return defer.succeed(None)
+        d.addCallback(_waitForBuild)
+
+        def _checkLog(res):
+            builder = self.master.botmaster.builders['dummy']
+            build = builder.builder_status.getLastFinishedBuild()
+            text = build.steps[0].logs[0].getText()
+            headers = build.steps[0].logs[0].getTextWithHeaders()
+            self.failIf(len(text) > 150, "Text too long")
+            self.failIf("truncated" not in headers, "Truncated message not found")
+        d.addCallback(_checkLog)
+
+        return d