Skip to content

Commit

Permalink
revert LoggingBuildStep and everything that depends on it, and merge …
Browse files Browse the repository at this point in the history 
…the modifications to it into ShellBaseStep
  • Loading branch information
@djmitche
djmitche committed Jan 21, 2014
1 parent e8de289 commit de05e33
Show file tree
Hide file tree
Showing 6 changed files with 392 additions and 120 deletions.
266 changes: 210 additions & 56 deletions master/buildbot/process/buildstep.py
View file
Expand Up @@ -720,82 +720,83 @@ class LoggingBuildStep(BuildStep):
progressMetrics = ('output',)
logfiles = {}

parms = BuildStep.parms
parms = BuildStep.parms + ['logfiles', 'lazylogfiles']
cmd = None

def __init__(self, logfiles=None, lazylogfiles=None, log_eval_func=None,
renderables = ['logfiles', 'lazylogfiles']

def __init__(self, logfiles={}, lazylogfiles=False, log_eval_func=None,
*args, **kwargs):
BuildStep.__init__(self, *args, **kwargs)

if any([x is not None for x in [logfiles, lazylogfiles, log_eval_func]]):
if logfiles and not isinstance(logfiles, dict):
config.error(
"The LoggingBuildStep parameters 'logfiles', 'lazylogfiles' and 'log_eval_func' are no longer available")
"the ShellCommand 'logfiles' parameter must be a dictionary")

# merge a class-level 'logfiles' attribute with one passed in as an
# argument
self.logfiles = self.logfiles.copy()
self.logfiles.update(logfiles)
self.lazylogfiles = lazylogfiles
if log_eval_func is not None:
config.error(
"the 'log_eval_func' paramater is no longer available")
self.addLogObserver('stdio', OutputProgressObserver("output"))

def addLogFile(self, logname, filename):
self.logfiles[logname] = filename

def setupLogsRunCommandAndProcessResults(self, cmd, stdioLog=None, closeLogWhenFinished=True,
errorMessages=None, logfiles=None, lazylogfiles=False):
if errorMessages is None:
errorMessages = []
if logfiles is None:
logfiles = {}
if logfiles:
cmd.setLogFiles(logfiles)
log.msg("%s.setupLogsRunCommandAndProcessResults(cmd=%s)" % (self.__class__.__name__, cmd))
log.msg(" cmd.args = %r" % (cmd.args,))
def buildCommandKwargs(self):
kwargs = dict()
kwargs['logfiles'] = self.logfiles
return kwargs

def startCommand(self, cmd, errorMessages=[]):
"""
@param cmd: a suitable RemoteCommand which will be launched, with
all output being put into our self.stdio_log LogFile
"""
log.msg("ShellCommand.startCommand(cmd=%s)" % (cmd,))
log.msg(" cmd.args = %r" % (cmd.args))
self.cmd = cmd # so we can interrupt it
self.step_status.setText(self.describe(False))

# stdio is the first log
self.stdio_log = stdio_log = self.addLog("stdio")
cmd.useLog(stdio_log, True)
for em in errorMessages:
stdio_log.addHeader(em)
# TODO: consider setting up self.stdio_log earlier, and have the
# code that passes in errorMessages instead call
# self.stdio_log.addHeader() directly.

# there might be other logs
self.setupLogfiles(cmd, self.logfiles)

d = self.runCommand(cmd) # might raise ConnectionLost
d.addCallback(lambda res: self.commandComplete(cmd))

# TODO: when the status.LogFile object no longer exists, then this
# method will a synthetic logfile for old-style steps, and to be called
# without the `logs` parameter for new-style steps. Unfortunately,
# lots of createSummary methods exist, but don't look at the log, so
# it's difficult to optimize when the synthetic logfile is needed.
if stdioLog is not None:
cmd.useLog(stdioLog, closeLogWhenFinished)
for em in errorMessages:
stdioLog.addHeader(em)
# TODO: consider setting up self.stdioLog earlier, and have the
# code that passes in errorMessages instead call
# self.stdioLog.addHeader() directly.
self.setupLogfiles(cmd, logfiles, lazylogfiles=lazylogfiles)
d = self.runCommand(cmd) # might raise ConnectionLost
d.addCallback(lambda _: self.commandComplete(cmd))

if stdioLog is not None:
def _createSummary(res):
self.createSummary(cmd.logs[stdioLog.getName()])
return res
d.addCallback(lambda res: _createSummary(res))
d.addCallback(lambda res: self.createSummary(cmd.logs['stdio']))

d.addCallback(lambda _: self.evaluateCommand(cmd)) # returns results

def _gotResults(results):
if stdioLog is not None:
# finish off the stdio logfile
stdioLog.finish()
return results
d.addCallback(_gotResults) # returns results
d.addErrback(self.checkDisconnect) # handle slave lost
return d

def startCommandAndSetStatus(self, cmd, stdioLog=None, closeLogWhenFinished=True,
errorMessages=None, logfiles=None, lazylogfiles=False):
self.step_status.setText(self.describe(False))
d = self.setupLogsRunCommandAndProcessResults(cmd, stdioLog=stdioLog,
closeLogWhenFinished=closeLogWhenFinished,
errorMessages=errorMessages,
logfiles=logfiles, lazylogfiles=lazylogfiles)
d.addCallback(lambda res: self.evaluateCommand(cmd)) # returns results

def _gotResults(results):
self.setStatus(cmd, results)
# finish off the stdio logfile
stdio_log.finish()
return results
d.addCallback(_gotResults) # returns results
return d
d.addCallback(self.finished)
d.addErrback(self.failed)

def setupLogfiles(self, cmd, logfiles, lazylogfiles=False):
def setupLogfiles(self, cmd, logfiles):
for logname, remotefilename in logfiles.items():
if lazylogfiles:
if self.lazylogfiles:
# Ask RemoteCommand to watch a logfile, but only add
# it when/if we see any data.
#
Expand Down Expand Up @@ -878,7 +879,13 @@ def setStatus(self, cmd, results):
return defer.succeed(None)


class ShellBaseStep(LoggingBuildStep):
class ShellBaseStep(BuildStep):
progressMetrics = ('output',)
logfiles = {}

parms = BuildStep.parms
cmd = None

renderables = [
'slaveEnvironment', 'remote_kwargs', 'description',
'descriptionDone', 'descriptionSuffix', 'haltOnFailure', 'flunkOnFailure']
Expand All @@ -890,6 +897,8 @@ def __init__(self, workdir=None,
# that we create, but first strip out the ones that we pass to
# BuildStep (like haltOnFailure and friends), and a couple that we
# consume ourselves.

# TODO: handle description stuff in BuildStep
if description:
self.description = description
if isinstance(self.description, str):
Expand All @@ -904,13 +913,17 @@ def __init__(self, workdir=None,
if isinstance(self.descriptionSuffix, str):
self.descriptionSuffix = [self.descriptionSuffix]

# pull out the ones that LoggingBuildStep wants, then upcall
if 'log_eval_func' in kwargs:
config.error(
"the 'log_eval_func' paramater is no longer available")

# pull out the ones that BuildStep wants, then upcall
buildstep_kwargs = {}
for k in kwargs.keys()[:]:
if k in self.__class__.parms:
buildstep_kwargs[k] = kwargs[k]
del kwargs[k]
LoggingBuildStep.__init__(self, **buildstep_kwargs)
BuildStep.__init__(self, **buildstep_kwargs)

# check validity of arguments being passed to RemoteShellCommand
invalid_args = []
Expand All @@ -928,13 +941,16 @@ def __init__(self, workdir=None,
kwargs['usePTY'] = usePTY
self.remote_kwargs = kwargs

# use the stdio log to monitor progress
self.addLogObserver('stdio', OutputProgressObserver("output"))

def setBuild(self, build):
LoggingBuildStep.setBuild(self, build)
BuildStep.setBuild(self, build)
# Set this here, so it gets rendered when we start the step
self.slaveEnvironment = self.build.slaveEnvironment

def setStepStatus(self, step_status):
LoggingBuildStep.setStepStatus(self, step_status)
BuildStep.setStepStatus(self, step_status)

def setDefaultWorkdir(self, workdir):
rkw = self.remote_kwargs
Expand All @@ -948,6 +964,81 @@ def getWorkdir(self):
"""
return self.remote_kwargs['workdir']

def setupLogfiles(self, cmd, logfiles, lazylogfiles=False):
for logname, remotefilename in logfiles.items():
if lazylogfiles:
# Ask RemoteCommand to watch a logfile, but only add
# it when/if we see any data.
#
# The dummy default argument local_logname is a work-around for
# Python name binding; default values are bound by value, but
# captured variables in the body are bound by name.
callback = lambda cmd_arg, local_logname=logname: self.addLog(
local_logname)
cmd.useLogDelayed(logname, callback, True)
else:
# tell the BuildStepStatus to add a LogFile
newlog = self.addLog(logname)
# and tell the RemoteCommand to feed it
cmd.useLog(newlog, True)

def setupLogsRunCommandAndProcessResults(self, cmd, stdioLog=None, closeLogWhenFinished=True,
errorMessages=None, logfiles=None, lazylogfiles=False):
if errorMessages is None:
errorMessages = []
if logfiles is None:
logfiles = {}
if logfiles:
cmd.setLogFiles(logfiles)
log.msg("%s.setupLogsRunCommandAndProcessResults(cmd=%s)" % (self.__class__.__name__, cmd))
log.msg(" cmd.args = %r" % (cmd.args,))
self.cmd = cmd # so we can interrupt it
# TODO: when the status.LogFile object no longer exists, then this
# method will a synthetic logfile for old-style steps, and to be called
# without the `logs` parameter for new-style steps. Unfortunately,
# lots of createSummary methods exist, but don't look at the log, so
# it's difficult to optimize when the synthetic logfile is needed.
if stdioLog is not None:
cmd.useLog(stdioLog, closeLogWhenFinished)
for em in errorMessages:
stdioLog.addHeader(em)
# TODO: consider setting up self.stdioLog earlier, and have the
# code that passes in errorMessages instead call
# self.stdioLog.addHeader() directly.
self.setupLogfiles(cmd, logfiles, lazylogfiles=lazylogfiles)
d = self.runCommand(cmd) # might raise ConnectionLost
d.addCallback(lambda _: self.commandComplete(cmd))

if stdioLog is not None:
def _createSummary(res):
self.createSummary(cmd.logs[stdioLog.getName()])
return res
d.addCallback(lambda res: _createSummary(res))

d.addCallback(lambda _: self.evaluateCommand(cmd)) # returns results

def _gotResults(results):
if stdioLog is not None:
# finish off the stdio logfile
stdioLog.finish()
return results
d.addCallback(_gotResults) # returns results
return d

def startCommandAndSetStatus(self, cmd, stdioLog=None, closeLogWhenFinished=True,
errorMessages=None, logfiles=None, lazylogfiles=False):
self.step_status.setText(self.describe(False))
d = self.setupLogsRunCommandAndProcessResults(cmd, stdioLog=stdioLog,
closeLogWhenFinished=closeLogWhenFinished,
errorMessages=errorMessages,
logfiles=logfiles, lazylogfiles=lazylogfiles)

def _gotResults(results):
self.setStatus(cmd, results)
return results
d.addCallback(_gotResults) # returns results
return d

def setupEnvironment(self, cmd):
# merge in anything from Build.slaveEnvironment
# This can be set from a Builder-level environment, or from earlier
Expand Down Expand Up @@ -984,9 +1075,26 @@ def buildCommandKwargs(self, command, warnings, logfiles=None):
return kwargs

def getCurrCommand(self):
# TODO: other code assumes self.cmd
raise NotImplementedError()

def interrupt(self, reason):
# TODO: consider adding an INTERRUPTED or STOPPED status to use
# instead of FAILURE, might make the text a bit more clear.
# 'reason' can be a Failure, or text
BuildStep.interrupt(self, reason)
if self.step_status.isWaitingForLocks():
self.addCompleteLog(
'cancelled while waiting for locks', str(reason))
else:
self.addCompleteLog('cancelled', str(reason))

if self.cmd:
d = self.cmd.interrupt(reason)
d.addErrback(log.err, 'while cancelling command')

def _describe(self, done=False):
# construct a description based on the current command and its status
try:
if done and self.descriptionDone is not None:
return self.descriptionDone
Expand Down Expand Up @@ -1030,6 +1138,52 @@ def _describe(self, done=False):
log.err(failure.Failure(), "Error describing step")
return ["???"]

def commandComplete(self, cmd):
pass

def createSummary(self, stdio):
pass

def evaluateCommand(self, cmd):
return cmd.results()

def setStatus(self, cmd, results):
# this is good enough for most steps, but it can be overridden to
# get more control over the displayed text
self.step_status.setText(self.getText(cmd, results))
self.step_status.setText2(self._maybeGetText2(cmd, results))
return defer.succeed(None)

def getText(self, cmd, results):
if results == SUCCESS:
return self.describe(True)
elif results == WARNINGS:
return self.describe(True) + ["warnings"]
elif results == EXCEPTION:
return self.describe(True) + ["exception"]
elif results == CANCELLED:
return self.describe(True) + ["cancelled"]
else:
return self.describe(True) + ["failed"]

def getText2(self, cmd, results):
return [self.name]

def _maybeGetText2(self, cmd, results):
if results == SUCCESS:
# successful steps do not add anything to the build's text
pass
elif results == WARNINGS:
if (self.flunkOnWarnings or self.warnOnWarnings):
# we're affecting the overall build, so tell them why
return self.getText2(cmd, results)
else:
if (self.haltOnFailure or self.flunkOnFailure
or self.warnOnFailure):
# we're affecting the overall build, so tell them why
return self.getText2(cmd, results)
return []


# (WithProperties used to be available in this module)
from buildbot.process.properties import WithProperties
Expand Down

0 comments on commit de05e33

Please sign in to comment.