backport ShellMixin and CommandMixin from nine

master/buildbot/process/build.py

@@ -102,6 +102,7 @@ def setLocks(self, lockList):
                       for access in lockList]
 
     def setSlaveEnvironment(self, env):
+        # TODO: remove once we don't have anything depending on this method or attribute
         self.slaveEnvironment = env
 
     def getSourceStamp(self, codebase=''):
@@ -320,6 +321,7 @@ def setupBuild(self, expectations):
             step = factory.buildStep()
             step.setBuild(self)
             step.setBuildSlave(self.slavebuilder.slave)
+            # TODO: remove once we don't have anything depending on setDefaultWorkdir
             if callable(self.workdir):
                 step.setDefaultWorkdir(self.workdir(self.sources))
             else:

master/buildbot/process/buildstep.py

@@ -19,6 +19,7 @@
 from twisted.internet import error
 from twisted.python import components
 from twisted.python import log
+from twisted.python import failure
 from twisted.python.failure import Failure
 from twisted.python.reflect import accumulateClassList
 from twisted.web.util import formatFailure
@@ -27,6 +28,7 @@
 from buildbot import config
 from buildbot import interfaces
 from buildbot import util
+from buildbot.util import flatten
 from buildbot.process import logobserver
 from buildbot.process import properties
 from buildbot.process import remotecommand
@@ -139,6 +141,7 @@ class BuildStep(object, properties.PropertiesMixin):
     buildslave = None
     step_status = None
     progress = None
+    cmd = None
 
     def __init__(self, **kwargs):
         for p in self.__class__.parms:
@@ -494,11 +497,15 @@ def addURL(self, name, url):
         self.step_status.addURL(name, url)
         return defer.succeed(None)
 
+    @defer.inlineCallbacks
     def runCommand(self, command):
         self.cmd = command
         command.buildslave = self.buildslave
-        d = command.run(self, self.remote)
-        return d
+        try:
+            res = yield command.run(self, self.remote)
+        finally:
+            self.cmd = None
+        defer.returnValue(res)
 
     @staticmethod
     def _maybeEvaluate(value, *args, **kwargs):
@@ -669,6 +676,208 @@ def setStatus(self, cmd, results):
         self.step_status.setText2(self.maybeGetText2(cmd, results))
 
 
+class CommandMixin(object):
+
+    @defer.inlineCallbacks
+    def _runRemoteCommand(self, cmd, abandonOnFailure, args, makeResult=None):
+        cmd = remotecommand.RemoteCommand(cmd, args)
+        try:
+            log = self.getLog('stdio')
+        except Exception:
+            log = yield self.addLog('stdio')
+        cmd.useLog(log, False)
+        yield self.runCommand(cmd)
+        if abandonOnFailure and cmd.didFail():
+            raise BuildStepFailed()
+        if makeResult:
+            defer.returnValue(makeResult(cmd))
+        else:
+            defer.returnValue(not cmd.didFail())
+
+    def runRmdir(self, dir, log=None, abandonOnFailure=True):
+        return self._runRemoteCommand('rmdir', abandonOnFailure,
+                                      {'dir': dir, 'logEnviron': False})
+
+    def pathExists(self, path, log=None):
+        return self._runRemoteCommand('stat', False,
+                                      {'file': path, 'logEnviron': False})
+
+    def runMkdir(self, dir, log=None, abandonOnFailure=True):
+        return self._runRemoteCommand('mkdir', abandonOnFailure,
+                                      {'dir': dir, 'logEnviron': False})
+
+    def glob(self, glob):
+        return self._runRemoteCommand(
+            'glob', True, {'glob': glob, 'logEnviron': False},
+            makeResult=lambda cmd: cmd.updates['files'][0])
+
+
+class ShellMixin(object):
+
+    command = None
+    workdir = None
+    env = {}
+    want_stdout = True
+    want_stderr = True
+    usePTY = 'slave-config'
+    logfiles = {}
+    lazylogfiles = {}
+    timeout = 1200
+    maxTime = None
+    logEnviron = True
+    interruptSignal = 'KILL'
+    sigtermTime = None
+    initialStdin = None
+    decodeRC = {0: SUCCESS}
+
+    _shellMixinArgs = [
+        'command',
+        'workdir',
+        'env',
+        'want_stdout',
+        'want_stderr',
+        'usePTY',
+        'logfiles',
+        'lazylogfiles',
+        'timeout',
+        'maxTime',
+        'logEnviron',
+        'interruptSignal',
+        'sigtermTime',
+        'initialStdin',
+        'decodeRC',
+    ]
+    renderables = _shellMixinArgs
+
+    def setupShellMixin(self, constructorArgs, prohibitArgs=[]):
+        constructorArgs = constructorArgs.copy()
+
+        def bad(arg):
+            config.error("invalid %s argument %s" %
+                         (self.__class__.__name__, arg))
+        for arg in self._shellMixinArgs:
+            if arg not in constructorArgs:
+                continue
+            if arg in prohibitArgs:
+                bad(arg)
+            else:
+                setattr(self, arg, constructorArgs[arg])
+            del constructorArgs[arg]
+        for arg in constructorArgs:
+            if arg not in BuildStep.parms:
+                bad(arg)
+                del constructorArgs[arg]
+        return constructorArgs
+
+    @defer.inlineCallbacks
+    def makeRemoteShellCommand(self, collectStdout=False, collectStderr=False,
+                               stdioLogName='stdio',
+                               **overrides):
+        kwargs = dict([(arg, getattr(self, arg))
+                       for arg in self._shellMixinArgs])
+        kwargs.update(overrides)
+        stdio = None
+        if stdioLogName is not None:
+            stdio = yield self.addLog(stdioLogName)
+        kwargs['command'] = flatten(kwargs['command'], (list, tuple))
+
+        # check for the usePTY flag
+        if kwargs['usePTY'] != 'slave-config':
+            if self.slaveVersionIsOlderThan("shell", "2.7"):
+                if stdio is not None:
+                    yield stdio.addHeader(
+                        "NOTE: slave does not allow master to override usePTY\n")
+                del kwargs['usePTY']
+
+        # check for the interruptSignal flag
+        if kwargs["interruptSignal"] and self.slaveVersionIsOlderThan("shell", "2.15"):
+            if stdio is not None:
+                yield stdio.addHeader(
+                    "NOTE: slave does not allow master to specify interruptSignal\n")
+            del kwargs['interruptSignal']
+
+        # lazylogfiles are handled below
+        del kwargs['lazylogfiles']
+
+        # merge the builder's environment with that supplied here
+        builderEnv = self.build.builder.config.env
+        kwargs['env'] = yield self.build.render(builderEnv)
+        kwargs['env'].update(self.env)
+        kwargs['stdioLogName'] = stdioLogName
+        # default the workdir appropriately
+        if not self.workdir:
+            if callable(self.build.workdir):
+                kwargs['workdir'] = self.build.workdir(self.build.sources)
+            else:
+                kwargs['workdir'] = self.build.workdir
+
+        # the rest of the args go to RemoteShellCommand
+        cmd = remotecommand.RemoteShellCommand(**kwargs)
+
+        # set up logging
+        if stdio is not None:
+            cmd.useLog(stdio, False)
+        for logname, remotefilename in self.logfiles.items():
+            if self.lazylogfiles:
+                # it's OK if this does, or does not, return a Deferred
+                callback = lambda cmd_arg, logname=logname: self.addLog(
+                    logname)
+                cmd.useLogDelayed(logname, callback, True)
+            else:
+                # tell the BuildStepStatus to add a LogFile
+                newlog = yield self.addLog(logname)
+                # and tell the RemoteCommand to feed it
+                cmd.useLog(newlog, False)
+
+        defer.returnValue(cmd)
+
+    def _describe(self, done=False):
+        try:
+            if done and self.descriptionDone is not None:
+                return self.descriptionDone
+            if self.description is not None:
+                return self.description
+
+            # if self.cmd is set, then use the RemoteCommand's info
+            if self.cmd:
+                command = self.command.command
+            # otherwise, if we were configured with a command, use that
+            elif self.command:
+                command = self.command
+            else:
+                return super(ShellMixin, self)._describe(done)
+
+            words = command
+            if isinstance(words, (str, unicode)):
+                words = words.split()
+
+            try:
+                len(words)
+            except (AttributeError, TypeError):
+                # WithProperties and Property don't have __len__
+                # For old-style classes instances AttributeError raised,
+                # for new-style classes instances - TypeError.
+                return super(ShellMixin, self)._describe(done)
+
+            # flatten any nested lists
+            words = flatten(words, (list, tuple))
+
+            # strip instances and other detritus (which can happen if a
+            # description is requested before rendering)
+            words = [w for w in words if isinstance(w, (str, unicode))]
+
+            if len(words) < 1:
+                return super(ShellMixin, self)._describe(done)
+            if len(words) == 1:
+                return ["'%s'" % words[0]]
+            if len(words) == 2:
+                return ["'%s" % words[0], "%s'" % words[1]]
+            return ["'%s" % words[0], "%s" % words[1], "...'"]
+
+        except Exception:
+            log.err(failure.Failure(), "Error describing step")
+            return super(ShellMixin, self)._describe(done)
+
 # Parses the logs for a list of regexs. Meant to be invoked like:
 # regexes = ((re.compile(...), FAILURE), (re.compile(...), WARNINGS))
 # self.addStep(ShellCommand,

master/buildbot/process/remotecommand.py

@@ -13,8 +13,8 @@
 #
 # Copyright Buildbot Team Members
 
-from buildbot import interfaces
 from buildbot import util
+from buildbot import interfaces
 from buildbot.process import metrics
 from buildbot.status.results import FAILURE
 from buildbot.status.results import SUCCESS
@@ -36,7 +36,8 @@ class RemoteCommand(pb.Referenceable):
     debug = False
 
     def __init__(self, remote_command, args, ignore_updates=False,
-                 collectStdout=False, collectStderr=False, decodeRC={0: SUCCESS}):
+                 collectStdout=False, collectStderr=False, decodeRC={0: SUCCESS},
+                 stdioLogName='stdio'):
         self.logs = {}
         self.delayedLogs = {}
         self._closeWhenFinished = {}
@@ -45,7 +46,7 @@ def __init__(self, remote_command, args, ignore_updates=False,
         self.stdout = ''
         self.stderr = ''
         self.updates = {}
-
+        self.stdioLogName = stdioLogName
         self._startTime = None
         self._remoteElapsed = None
         self.remote_command = remote_command
@@ -193,22 +194,22 @@ def remote_complete(self, failure=None):
         return None
 
     def addStdout(self, data):
-        if 'stdio' in self.logs:
-            self.logs['stdio'].addStdout(data)
+        if self.stdioLogName is not None and self.stdioLogName in self.logs:
+            self.logs[self.stdioLogName].addStdout(data)
         if self.collectStdout:
             self.stdout += data
         return defer.succeed(None)
 
     def addStderr(self, data):
-        if 'stdio' in self.logs:
-            self.logs['stdio'].addStderr(data)
+        if self.stdioLogName is not None and self.stdioLogName in self.logs:
+            self.logs[self.stdioLogName].addStderr(data)
         if self.collectStderr:
             self.stderr += data
         return defer.succeed(None)
 
     def addHeader(self, data):
-        if 'stdio' in self.logs:
-            self.logs['stdio'].addHeader(data)
+        if self.stdioLogName is not None and self.stdioLogName in self.logs:
+            self.logs[self.stdioLogName].addHeader(data)
         return defer.succeed(None)
 
     def addToLog(self, logname, data):
@@ -290,7 +291,8 @@ def __init__(self, workdir, command, env=None,
                  logfiles={}, usePTY="slave-config", logEnviron=True,
                  collectStdout=False, collectStderr=False,
                  interruptSignal=None,
-                 initialStdin=None, decodeRC={0: SUCCESS}):
+                 initialStdin=None, decodeRC={0: SUCCESS},
+                 stdioLogName='stdio'):
 
         self.command = command  # stash .command, set it later
         if isinstance(self.command, basestring):
@@ -326,7 +328,8 @@ def obfuscate(arg):
             args['interruptSignal'] = interruptSignal
         RemoteCommand.__init__(self, "shell", args, collectStdout=collectStdout,
                                collectStderr=collectStderr,
-                               decodeRC=decodeRC)
+                               decodeRC=decodeRC,
+                               stdioLogName=stdioLogName)
 
     def _start(self):
         self.args['command'] = self.command

master/buildbot/steps/slave.py

@@ -242,8 +242,6 @@ def commandComplete(self, cmd):
 
 class CompositeStepMixin():
 
-    """I define utils for composite steps, factorizing basic remote commands"""
-
     def addLogForRemoteCommands(self, logname):
         """This method must be called by user classes
         composite steps could create several logs, this mixin functions will write