Merge commit 'ad13a16bbdec535c8edebdbba4f77ae39b19c84c' into jinja

NEWS

@@ -1,5 +1,13 @@
 User visible changes in Buildbot.             -*- outline -*-
 
+** New buildstep `MTR'
+
+A new class buildbot.process.mtrlogobserver.MTR was added. This buildstep is
+used to run test suites using mysql-test-run. It parses the stdio output for
+test failures and summarises them on the waterfall page. It also makes server
+error logs available for debugging failures, and optionally inserts
+information about test runs and test failures into an external database.
+
 * Release 0.7.11p (July 16, 2009)
 
 Fixes a few test failures in 0.7.11, and gives a default value for branchType

buildbot/changes/svnpoller.py

@@ -48,7 +48,7 @@ class SVNPoller(base.ChangeSource, util.ComparableMixin):
     compare_attrs = ["svnurl", "split_file_function",
                      "svnuser", "svnpasswd",
                      "pollinterval", "histmax",
-                     "svnbin"]
+                     "svnbin", "category"]
 
     parent = None # filled in when we're added
     last_change = None
@@ -58,7 +58,7 @@ class SVNPoller(base.ChangeSource, util.ComparableMixin):
     def __init__(self, svnurl, split_file=None,
                  svnuser=None, svnpasswd=None,
                  pollinterval=10*60, histmax=100,
-                 svnbin='svn', revlinktmpl=''):
+                 svnbin='svn', revlinktmpl='', category=None):
         """
         @type  svnurl: string
         @param svnurl: the SVN URL that describes the repository and
@@ -171,6 +171,11 @@ def __init__(self, svnurl, split_file=None,
                              "http://reposerver/websvn/revision.php?rev=%s"
                              would create suitable links on the build pages
                              to information in websvn on each revision.
+
+        @type  category:     string
+        @param category:     A single category associated with the changes that
+                             could be used by schedulers watch for branches of a
+                             certain name AND category.
         """
 
         if svnurl.endswith("/"):
@@ -188,6 +193,7 @@ def __init__(self, svnurl, split_file=None,
         self._prefix = None
         self.overrun_counter = 0
         self.loop = LoopingCall(self.checksvn)
+        self.category = category
 
     def split_file(self, path):
         # use getattr() to avoid turning this function into a bound method,
@@ -457,7 +463,8 @@ def create_changes(self, new_logentries):
                                comments=comments,
                                revision=revision,
                                branch=branch,
-                               revlink=revlink)
+                               revlink=revlink,
+                               category=self.category)
                     changes.append(c)
 
         return changes

buildbot/process/buildstep.py

@@ -256,6 +256,7 @@ class LoggedRemoteCommand(RemoteCommand):
 
     def __init__(self, *args, **kwargs):
         self.logs = {}
+        self.delayedLogs = {}
         self._closeWhenFinished = {}
         RemoteCommand.__init__(self, *args, **kwargs)
 
@@ -290,9 +291,15 @@ def useLog(self, loog, closeWhenFinished=False, logfileName=None):
         if not logfileName:
             logfileName = loog.getName()
         assert logfileName not in self.logs
+        assert logfileName not in self.delayedLogs
         self.logs[logfileName] = loog
         self._closeWhenFinished[logfileName] = closeWhenFinished
 
+    def useLogDelayed(self, logfileName, activateCallBack, closeWhenFinished=False):
+        assert logfileName not in self.logs
+        assert logfileName not in self.delayedLogs
+        self.delayedLogs[logfileName] = (activateCallBack, closeWhenFinished)
+
     def start(self):
         log.msg("LoggedRemoteCommand.start")
         if 'stdio' not in self.logs:
@@ -313,6 +320,14 @@ def addHeader(self, data):
             self.logs['stdio'].addHeader(data)
 
     def addToLog(self, logname, data):
+        # Activate delayed logs on first data.
+        if logname in self.delayedLogs:
+            (activateCallBack, closeWhenFinished) = self.delayedLogs[logname]
+            del self.delayedLogs[logname]
+            loog = activateCallBack(self)
+            self.logs[logname] = loog
+            self._closeWhenFinished[logname] = closeWhenFinished
+
         if logname in self.logs:
             self.logs[logname].addStdout(data)
         else:
@@ -431,10 +446,10 @@ class RemoteShellCommand(LoggedRemoteCommand):
     command is finished, it will fire a Deferred. You can then check the
     results of the command and parse the output however you like."""
 
-    def __init__(self, workdir, command, env=None, 
+    def __init__(self, workdir, command, env=None,
                  want_stdout=1, want_stderr=1,
-                 timeout=20*60, logfiles={}, usePTY="slave-config",
-                 logEnviron=True):
+                 timeout=20*60, maxTime=None, logfiles={},
+                 usePTY="slave-config", logEnviron=True):
         """
         @type  workdir: string
         @param workdir: directory where the command ought to run,
@@ -475,6 +490,11 @@ def __init__(self, workdir, command, env=None,
                         None to disable the timeout.
 
         @param logEnviron: whether to log env vars on the slave side
+
+        @type  maxTime: int
+        @param maxTime: tell the remote that if the command fails to complete
+                        in this number of seconds, the command should be
+                        killed.  Use None to disable maxTime.
         """
 
         self.command = command # stash .command, set it later
@@ -489,6 +509,7 @@ def __init__(self, workdir, command, env=None,
                 'want_stderr': want_stderr,
                 'logfiles': logfiles,
                 'timeout': timeout,
+                'maxTime': maxTime,
                 'usePTY': usePTY,
                 'logEnviron': logEnviron,
                 }
@@ -958,15 +979,17 @@ class LoggingBuildStep(BuildStep):
     progressMetrics = ('output',)
     logfiles = {}
 
-    parms = BuildStep.parms + ['logfiles']
+    parms = BuildStep.parms + ['logfiles', 'lazylogfiles']
 
-    def __init__(self, logfiles={}, *args, **kwargs):
+    def __init__(self, logfiles={}, lazylogfiles=False, *args, **kwargs):
         BuildStep.__init__(self, *args, **kwargs)
-        self.addFactoryArguments(logfiles=logfiles)
+        self.addFactoryArguments(logfiles=logfiles,
+                                 lazylogfiles=lazylogfiles)
         # merge a class-level 'logfiles' attribute with one passed in as an
         # argument
         self.logfiles = self.logfiles.copy()
         self.logfiles.update(logfiles)
+        self.lazylogfiles = lazylogfiles
         self.addLogObserver('stdio', OutputProgressObserver("output"))
 
     def describe(self, done=False):
@@ -1009,10 +1032,20 @@ def setupLogfiles(self, cmd, logfiles):
         """Set up any additional logfiles= logs.
         """
         for logname,remotefilename in logfiles.items():
-            # tell the BuildStepStatus to add a LogFile
-            newlog = self.addLog(logname)
-            # and tell the LoggedRemoteCommand to feed it
-            cmd.useLog(newlog, True)
+            if self.lazylogfiles:
+                # Ask LoggedRemoteCommand to watch a logfile, but only add
+                # it when/if we see any data.
+                #
+                # The dummy default argument local_logname is a work-around for
+                # Python name binding; default values are bound by value, but
+                # captured variables in the body are bound by name.
+                callback = lambda cmd_arg, local_logname=logname: self.addLog(local_logname)
+                cmd.useLogDelayed(logname, callback, True)
+            else:
+                # tell the BuildStepStatus to add a LogFile
+                newlog = self.addLog(logname)
+                # and tell the LoggedRemoteCommand to feed it
+                cmd.useLog(newlog, True)
 
     def interrupt(self, reason):
         # TODO: consider adding an INTERRUPTED or STOPPED status to use