diff --git a/README.md b/README.md index b50b8a5bfbe..d56c46f91cc 100644 --- a/README.md +++ b/README.md @@ -312,6 +312,10 @@ Here are the remaining bits: * merge rewrites to master * remove `step_status` for statistics +* update documentation + * lots of changes in `customization.rst` + * `cfg-statustargets.rst` references `getText` + ### Importing old Build Pickles ### The most likely plan for data from 0.8.x implementations is to ignore it. diff --git a/master/buildbot/process/buildstep.py b/master/buildbot/process/buildstep.py index ce8f41d0b9f..817432f78fc 100644 --- a/master/buildbot/process/buildstep.py +++ b/master/buildbot/process/buildstep.py @@ -13,8 +13,6 @@ # # Copyright Buildbot Team Members -import re - from twisted.internet import defer from twisted.internet import error from twisted.python import components @@ -41,7 +39,6 @@ from buildbot.status.results import SKIPPED from buildbot.status.results import SUCCESS from buildbot.status.results import WARNINGS -from buildbot.status.results import worst_status class BuildStepFailed(Exception): @@ -808,27 +805,6 @@ def setStatus(self, cmd, results): return defer.succeed(None) -# Parses the logs for a list of regexs. Meant to be invoked like: -# regexes = ((re.compile(...), FAILURE), (re.compile(...), WARNINGS)) -# self.addStep(ShellCommand, -# command=..., -# ..., -# log_eval_func=lambda c,s: regex_log_evaluator(c, s, regexs) -# ) -def regex_log_evaluator(cmd, step_status, regexes): - worst = cmd.results() - for err, possible_status in regexes: - # worst_status returns the worse of the two status' passed to it. - # we won't be changing "worst" unless possible_status is worse than it, - # so we don't even need to check the log if that's the case - if worst_status(worst, possible_status) == possible_status: - if isinstance(err, (basestring)): - err = re.compile(".*%s.*" % err, re.DOTALL) - for l in cmd.logs.values(): - if err.search(l.getText()): - worst = possible_status - return worst - # (WithProperties used to be available in this module) from buildbot.process.properties import WithProperties _hush_pyflakes = [WithProperties] diff --git a/master/buildbot/process/logobserver.py b/master/buildbot/process/logobserver.py index 69e36ff5623..e3c906a88d2 100644 --- a/master/buildbot/process/logobserver.py +++ b/master/buildbot/process/logobserver.py @@ -106,6 +106,37 @@ def headerLineReceived(self, line): pass +class LineConsumerLogObserver(LogLineObserver): + + def __init__(self, consumerFunction): + LogLineObserver.__init__(self) + self.generator = None + self.consumerFunction = consumerFunction + + def feed(self, input): + # note that we defer starting the generator until the first bit of + # data, since the observer may be instantiated during configuration as + # well as for each execution of the step. + self.generator = self.consumerFunction() + self.generator.next() + # shortcut all remaining feed operations + self.feed = self.generator.send + self.feed(input) + + def outLineReceived(self, line): + self.feed(('o', line)) + + def errLineReceived(self, line): + self.feed(('e', line)) + + def headerLineReceived(self, line): + self.feed(('h', line)) + + def finishReceived(self): + if self.generator: + self.generator.close() + + class OutputProgressObserver(LogObserver): length = 0 diff --git a/master/buildbot/steps/package/deb/pbuilder.py b/master/buildbot/steps/package/deb/pbuilder.py index fc34d4f4066..e66107418a3 100644 --- a/master/buildbot/steps/package/deb/pbuilder.py +++ b/master/buildbot/steps/package/deb/pbuilder.py @@ -24,6 +24,7 @@ from twisted.python import log from buildbot import config +from buildbot.process import logobserver from buildbot.process import remotecommand from buildbot.process.buildstep import FAILURE from buildbot.steps.shell import WarningCountingShellCommand @@ -120,6 +121,9 @@ def __init__(self, self.suppressions.append((None, re.compile(r"\.pbuilderrc does not exist"), None, None)) + self.addLogObserver( + 'stdio', logobserver.LineConsumerLogObserver(self.logConsumer)) + # Check for Basetgz def start(self): cmd = remotecommand.RemoteCommand('stat', {'file': self.basetgz}) @@ -182,11 +186,13 @@ def startBuild(self, cmd): else: return WarningCountingShellCommand.start(self) - def commandComplete(self, cmd): - out = cmd.logs['stdio'].getText() - m = re.search(r"dpkg-genchanges >\.\./(.+\.changes)", out) - if m: - self.setProperty("deb-changes", m.group(1), "DebPbuilder") + def logConsumer(self): + r = re.compile(r"dpkg-genchanges >\.\./(.+\.changes)") + while True: + stream, line = yield + mo = r.search(line) + if mo: + self.setProperty("deb-changes", mo.group(1), "DebPbuilder") class DebCowbuilder(DebPbuilder): diff --git a/master/buildbot/steps/package/rpm/mock.py b/master/buildbot/steps/package/rpm/mock.py index 18459763426..23cdeea856d 100644 --- a/master/buildbot/steps/package/rpm/mock.py +++ b/master/buildbot/steps/package/rpm/mock.py @@ -144,12 +144,16 @@ def __init__(self, self.command += ['--buildsrpm', '--spec', self.spec, '--sources', self.sources] - - def commandComplete(self, cmd): - out = cmd.logs['build.log'].getText() - m = re.search(r"Wrote: .*/([^/]*.src.rpm)", out) - if m: - self.setProperty("srpm", m.group(1), 'MockBuildSRPM') + self.addLogObserver( + 'stdio', logobserver.LineConsumerLogObserver(self.logConsumer)) + + def logConsumer(self): + r = re.compile(r"Wrote: .*/([^/]*.src.rpm)") + while True: + stream, line = yield + m = r.search(line) + if m: + self.setProperty("srpm", m.group(1), 'MockBuildSRPM') class MockRebuild(Mock): diff --git a/master/buildbot/steps/package/rpm/rpmbuild.py b/master/buildbot/steps/package/rpm/rpmbuild.py index f53fbe79994..56c2e2b5b8b 100644 --- a/master/buildbot/steps/package/rpm/rpmbuild.py +++ b/master/buildbot/steps/package/rpm/rpmbuild.py @@ -12,15 +12,16 @@ # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. # # Portions Copyright Buildbot Team Members - -from __future__ import with_statement # Portions Copyright Dan Radez # Portions Copyright Steve 'Ashcrow' Milner +from __future__ import with_statement + import os from buildbot import config from buildbot.process import buildstep +from buildbot.process import logobserver from buildbot.steps.shell import ShellCommand @@ -86,6 +87,9 @@ def __init__(self, if not self.specfile: config.error("You must specify a specfile") + self.addLogObserver( + 'stdio', logobserver.LineConsumerLogObserver(self.logConsumer)) + def start(self): if self.autoRelease: relfile = '%s.release' % ( @@ -116,25 +120,29 @@ def start(self): cmd = buildstep.RemoteShellCommand(**kwargs) self.setupEnvironment(cmd) self.startCommand(cmd) + self.addLogObserver( + 'stdio', logobserver.LineConsumerLogObserver(self.logConsumer)) - def createSummary(self, log): + def logConsumer(self): rpm_prefixes = ['Provides:', 'Requires(', 'Requires:', 'Checking for unpackaged', 'Wrote:', 'Executing(%', '+ ', 'Processing files:'] rpm_err_pfx = [' ', 'RPM build errors:', 'error: '] + self.rpmcmdlog = [] + self.rpmerrors = [] - rpmcmdlog = [] - rpmerrors = [] - - for line in log.getText().splitlines(True): + while True: + stream, line = yield for pfx in rpm_prefixes: if line.startswith(pfx): - rpmcmdlog.append(line) + self.rpmcmdlog.append(line) break for err in rpm_err_pfx: if line.startswith(err): - rpmerrors.append(line) + self.rpmerrors.append(line) break - self.addCompleteLog('RPM Command Log', "".join(rpmcmdlog)) - if rpmerrors: - self.addCompleteLog('RPM Errors', "".join(rpmerrors)) + + def createSummary(self, log): + self.addCompleteLog('RPM Command Log', "".join(self.rpmcmdlog)) + if self.rpmerrors: + self.addCompleteLog('RPM Errors', "".join(self.rpmerrors)) diff --git a/master/buildbot/steps/python.py b/master/buildbot/steps/python.py index f377e04acda..649ff681c4f 100644 --- a/master/buildbot/steps/python.py +++ b/master/buildbot/steps/python.py @@ -17,17 +17,12 @@ import re from buildbot import config +from buildbot.process import logobserver from buildbot.status.results import FAILURE from buildbot.status.results import SUCCESS from buildbot.status.results import WARNINGS from buildbot.steps.shell import ShellCommand -try: - import cStringIO - StringIO = cStringIO.StringIO -except ImportError: - from StringIO import StringIO - class BuildEPYDoc(ShellCommand): name = "epydoc" @@ -35,30 +30,33 @@ class BuildEPYDoc(ShellCommand): description = ["building", "epydocs"] descriptionDone = ["epydoc"] - def createSummary(self, log): - import_errors = 0 - warnings = 0 - errors = 0 + def __init__(self, **kwargs): + ShellCommand.__init__(self, **kwargs) + self.addLogObserver( + 'stdio', logobserver.LineConsumerLogObserver(self.logConsumer)) + + def logConsumer(self): + self.import_errors = 0 + self.warnings = 0 + self.errors = 0 - for line in StringIO(log.getText()): + while True: + stream, line = yield if line.startswith("Error importing "): - import_errors += 1 + self.import_errors += 1 if line.find("Warning: ") != -1: - warnings += 1 + self.warnings += 1 if line.find("Error: ") != -1: - errors += 1 + self.errors += 1 + def createSummary(self, log): self.descriptionDone = self.descriptionDone[:] - if import_errors: - self.descriptionDone.append("ierr=%d" % import_errors) - if warnings: - self.descriptionDone.append("warn=%d" % warnings) - if errors: - self.descriptionDone.append("err=%d" % errors) - - self.import_errors = import_errors - self.warnings = warnings - self.errors = errors + if self.import_errors: + self.descriptionDone.append("ierr=%d" % self.import_errors) + if self.warnings: + self.descriptionDone.append("warn=%d" % self.warnings) + if self.errors: + self.descriptionDone.append("err=%d" % self.errors) def evaluateCommand(self, cmd): if cmd.didFail(): @@ -87,16 +85,21 @@ def __init__(self, *args, **kwargs): # evaluateCommand below can inspect the results more closely. kwargs['decodeRC'] = {0: SUCCESS, 1: WARNINGS} ShellCommand.__init__(self, *args, **kwargs) + self.addLogObserver( + 'stdio', logobserver.LineConsumerLogObserver(self.logConsumer)) - def createSummary(self, log): - counts = {} - summaries = {} + counts = self.counts = {} + summaries = self.summaries = {} for m in self.MESSAGES: counts[m] = 0 summaries[m] = [] + def logConsumer(self): + counts = self.counts + summaries = self.summaries first = True - for line in StringIO(log.getText()).readlines(): + while True: + stream, line = yield # the first few lines might contain echoed commands from a 'make # pyflakes' step, so don't count these as warnings. Stop ignoring # the initial lines as soon as we see one with a colon. @@ -126,6 +129,8 @@ def createSummary(self, log): summaries[m].append(line) counts[m] += 1 + def createSummary(self, log): + counts, summaries = self.counts, self.summaries self.descriptionDone = self.descriptionDone[:] # we log 'misc' as syntax-error @@ -195,15 +200,21 @@ class PyLint(ShellCommand): _default_line_re = re.compile(r'^%s(\d{4})?: *\d+(,\d+)?:.+' % _msgtypes_re_str) _parseable_line_re = re.compile(r'[^:]+:\d+: \[%s(\d{4})?[,\]] .+' % _msgtypes_re_str) - def createSummary(self, log): - counts = {} - summaries = {} + def __init__(self, **kwargs): + ShellCommand.__init__(self, **kwargs) + self.addLogObserver( + 'stdio', logobserver.LineConsumerLogObserver(self.logConsumer)) + + def logConsumer(self): + self.counts = {} + self.summaries = {} for m in self.MESSAGES: - counts[m] = 0 - summaries[m] = [] + self.counts[m] = 0 + self.summaries[m] = [] line_re = None # decide after first match - for line in StringIO(log.getText()).readlines(): + while True: + stream, line = yield if not line_re: # need to test both and then decide on one if self._parseable_line_re.match(line): @@ -216,9 +227,11 @@ def createSummary(self, log): if mo: msgtype = mo.group(self._re_groupname) assert msgtype in self.MESSAGES - summaries[msgtype].append(line) - counts[msgtype] += 1 + self.summaries[msgtype].append(line) + self.counts[msgtype] += 1 + def createSummary(self, log): + counts, summaries = self.counts, self.summaries self.descriptionDone = self.descriptionDone[:] for msg, fullmsg in self.MESSAGES.items(): if counts[msg]: @@ -260,7 +273,6 @@ def __init__(self, sphinx_sourcedir='.', sphinx_builddir=None, config.error("Sphinx argument mode has to be 'incremental' or" + "'full' is required") - self.warnings = 0 self.success = False ShellCommand.__init__(self, **kwargs) @@ -287,28 +299,32 @@ def __init__(self, sphinx_sourcedir='.', sphinx_builddir=None, command.extend([sphinx_sourcedir, sphinx_builddir]) self.setCommand(command) - def createSummary(self, log): + self.addLogObserver( + 'stdio', logobserver.LineConsumerLogObserver(self.logConsumer)) - msgs = ['WARNING', 'ERROR', 'SEVERE'] + msgs = ['WARNING', 'ERROR', 'SEVERE'] - warnings = [] - for line in log.getText().split('\n'): + def logConsumer(self): + self.warnings = [] + while True: + stream, line = yield if (line.startswith('build succeeded') or line.startswith('no targets are out of date.')): self.success = True else: - for msg in msgs: + for msg in self.msgs: if msg in line: - warnings.append(line) - self.warnings += 1 - if self.warnings > 0: - self.addCompleteLog('warnings', "\n".join(warnings)) + self.warnings.append(line) + + def createSummary(self, log): + if len(self.warnings) > 0: + self.addCompleteLog('warnings', "\n".join(self.warnings)) - self.step_status.setStatistic('warnings', self.warnings) + self.step_status.setStatistic('warnings', len(self.warnings)) def evaluateCommand(self, cmd): if self.success: - if self.warnings == 0: + if not self.warnings: return SUCCESS else: return WARNINGS @@ -320,5 +336,5 @@ def describe(self, done=False): return ["building"] description = [self.name] - description.append('%d warnings' % self.warnings) + description.append('%d warnings' % len(self.warnings)) return description diff --git a/master/buildbot/steps/python_twisted.py b/master/buildbot/steps/python_twisted.py index ff74c8d3bda..8146b8caf4c 100644 --- a/master/buildbot/steps/python_twisted.py +++ b/master/buildbot/steps/python_twisted.py @@ -16,8 +16,7 @@ from twisted.python import log -from buildbot.process.buildstep import LogLineObserver -from buildbot.process.buildstep import OutputProgressObserver +from buildbot.process import logobserver from buildbot.status import testresult from buildbot.status.results import FAILURE from buildbot.status.results import SKIPPED @@ -54,6 +53,9 @@ class HLint(ShellCommand): def __init__(self, python=None, **kwargs): ShellCommand.__init__(self, **kwargs) self.python = python + self.warningLines = [] + self.addLogObserver( + 'stdio', logobserver.LineConsumerLogObserver(self.logConsumer)) def start(self): # create the command @@ -77,17 +79,15 @@ def start(self): ShellCommand.start(self) + def logConsumer(self): + while True: + stream, line = yield + if ':' in line: + self.warnings += 1 + self.warningLines.append(line) + def commandComplete(self, cmd): - # TODO: remove the 'files' file (a list of .xhtml files that were - # submitted to hlint) because it is available in the logfile and - # mostly exists to give the user an idea of how long the step will - # take anyway). - lines = cmd.logs['stdio'].getText().split("\n") - warningLines = filter(lambda line: ':' in line, lines) - if warningLines: - self.addCompleteLog("warnings", "".join(warningLines)) - warnings = len(warningLines) - self.warnings = warnings + self.addCompleteLog('warnings', '\n'.join(self.warningLines)) def evaluateCommand(self, cmd): # warnings are in stdout, rc is always 0, unless the tools break @@ -104,64 +104,20 @@ def getText2(self, cmd, results): self.warnings == 1 and 't' or 'ts')] -def countFailedTests(output): - # start scanning 10kb from the end, because there might be a few kb of - # import exception tracebacks between the total/time line and the errors - # line - chunk = output[-10000:] - lines = chunk.split("\n") - lines.pop() # blank line at end - # lines[-3] is "Ran NN tests in 0.242s" - # lines[-2] is blank - # lines[-1] is 'OK' or 'FAILED (failures=1, errors=12)' - # or 'FAILED (failures=1)' - # or "PASSED (skips=N, successes=N)" (for Twisted-2.0) - # there might be other lines dumped here. Scan all the lines. - res = {'total': None, - 'failures': 0, - 'errors': 0, - 'skips': 0, - 'expectedFailures': 0, - 'unexpectedSuccesses': 0, - } - for l in lines: - out = re.search(r'Ran (\d+) tests', l) - if out: - res['total'] = int(out.group(1)) - if (l.startswith("OK") or - l.startswith("FAILED ") or - l.startswith("PASSED")): - # the extra space on FAILED_ is to distinguish the overall - # status from an individual test which failed. The lack of a - # space on the OK is because it may be printed without any - # additional text (if there are no skips,etc) - out = re.search(r'failures=(\d+)', l) - if out: - res['failures'] = int(out.group(1)) - out = re.search(r'errors=(\d+)', l) - if out: - res['errors'] = int(out.group(1)) - out = re.search(r'skips=(\d+)', l) - if out: - res['skips'] = int(out.group(1)) - out = re.search(r'expectedFailures=(\d+)', l) - if out: - res['expectedFailures'] = int(out.group(1)) - out = re.search(r'unexpectedSuccesses=(\d+)', l) - if out: - res['unexpectedSuccesses'] = int(out.group(1)) - # successes= is a Twisted-2.0 addition, and is not currently used - out = re.search(r'successes=(\d+)', l) - if out: - res['successes'] = int(out.group(1)) - - return res - - -class TrialTestCaseCounter(LogLineObserver): +class TrialTestCaseCounter(logobserver.LogLineObserver): _line_re = re.compile(r'^(?:Doctest: )?([\w\.]+) \.\.\. \[([^\]]+)\]$') - numTests = 0 - finished = False + + def __init__(self): + logobserver.LogLineObserver.__init__(self) + self.numTests = 0 + self.finished = False + self.counts = {'total': None, + 'failures': 0, + 'errors': 0, + 'skips': 0, + 'expectedFailures': 0, + 'unexpectedSuccesses': 0, + } def outLineReceived(self, line): # different versions of Twisted emit different per-test lines with @@ -173,18 +129,44 @@ def outLineReceived(self, line): # Note that doctests create lines line this: # Doctest: viff.field.GF ... [OK] - if self.finished: - return if line.startswith("=" * 40): self.finished = True - return - - m = self._line_re.search(line.strip()) - if m: - testname, result = m.groups() - self.numTests += 1 - self.step.setProgress('tests', self.numTests) - + if not self.finished: + m = self._line_re.search(line.strip()) + if m: + testname, result = m.groups() + self.numTests += 1 + self.step.setProgress('tests', self.numTests) + + out = re.search(r'Ran (\d+) tests', line) + if out: + self.counts['total'] = int(out.group(1)) + if (line.startswith("OK") or + line.startswith("FAILED ") or + line.startswith("PASSED")): + # the extra space on FAILED_ is to distinguish the overall + # status from an individual test which failed. The lack of a + # space on the OK is because it may be printed without any + # additional text (if there are no skips,etc) + out = re.search(r'failures=(\d+)', line) + if out: + self.counts['failures'] = int(out.group(1)) + out = re.search(r'errors=(\d+)', line) + if out: + self.counts['errors'] = int(out.group(1)) + out = re.search(r'skips=(\d+)', line) + if out: + self.counts['skips'] = int(out.group(1)) + out = re.search(r'expectedFailures=(\d+)', line) + if out: + self.counts['expectedFailures'] = int(out.group(1)) + out = re.search(r'unexpectedSuccesses=(\d+)', line) + if out: + self.counts['unexpectedSuccesses'] = int(out.group(1)) + # successes= is a Twisted-2.0 addition, and is not currently used + out = re.search(r'successes=(\d+)', line) + if out: + self.counts['successes'] = int(out.group(1)) UNSPECIFIED = () # since None is a valid choice @@ -382,7 +364,15 @@ def __init__(self, reactor=UNSPECIFIED, python=None, trial=None, self.descriptionDone = ["tests"] # this counter will feed Progress along the 'test cases' metric - self.addLogObserver('stdio', TrialTestCaseCounter()) + self.observer = TrialTestCaseCounter() + self.addLogObserver('stdio', self.observer) + + # this observer consumes multiple lines in a go, so it can't be easily + # handled in TrialTestCaseCounter. + self.addLogObserver( + 'stdio', logobserver.LineConsumerLogObserver(self.logConsumer)) + self.problems = "" + self.warnings = {} def setupEnvironment(self, cmd): ShellCommand.setupEnvironment(self, cmd) @@ -403,7 +393,7 @@ def setupEnvironment(self, cmd): def start(self): # choose progressMetrics and logfiles based on whether trial is being # run with multiple workers or not. - output_observer = OutputProgressObserver('test.log') + output_observer = logobserver.OutputProgressObserver('test.log') if self.jobs is not None: self.jobs = int(self.jobs) @@ -436,10 +426,7 @@ def commandComplete(self, cmd): # figure out all status, then let the various hook functions return # different pieces of it - # 'cmd' is the original trial command, so cmd.logs['stdio'] is the - # trial output. We don't have access to test.log from here. - output = cmd.logs['stdio'].getText() - counts = countFailedTests(output) + counts = self.observer.counts total = counts['total'] failures, errors = counts['failures'], counts['errors'] @@ -525,32 +512,31 @@ def addTestResult(self, testname, results, text, tlog): # self.step_status.build.addTestResult(tr) self.build.build_status.addTestResult(tr) - def createSummary(self, loog): - output = loog.getText() - problems = "" - sio = StringIO.StringIO(output) - warnings = {} + def logConsumer(self): while True: - line = sio.readline() - if line == "": - break + stream, line = yield if line.find(" exceptions.DeprecationWarning: ") != -1: # no source warning = line # TODO: consider stripping basedir prefix here - warnings[warning] = warnings.get(warning, 0) + 1 + self.warnings[warning] = self.warnings.get(warning, 0) + 1 elif (line.find(" DeprecationWarning: ") != -1 or line.find(" UserWarning: ") != -1): # next line is the source - warning = line + sio.readline() - warnings[warning] = warnings.get(warning, 0) + 1 + warning = line + "\n" + (yield)[1] + "\n" + self.warnings[warning] = self.warnings.get(warning, 0) + 1 elif line.find("Warning: ") != -1: warning = line - warnings[warning] = warnings.get(warning, 0) + 1 + self.warnings[warning] = self.warnings.get(warning, 0) + 1 if line.find("=" * 60) == 0 or line.find("-" * 60) == 0: - problems += line - problems += sio.read() - break + # read to EOF + while True: + self.problems += line + "\n" + stream, line = yield + + def createSummary(self, loog): + problems = self.problems + warnings = self.warnings if problems: self.addCompleteLog("problems", problems) diff --git a/master/buildbot/steps/shell.py b/master/buildbot/steps/shell.py index 44c0fe65fe6..e7f367928de 100644 --- a/master/buildbot/steps/shell.py +++ b/master/buildbot/steps/shell.py @@ -281,8 +281,14 @@ class TreeSize(ShellCommand): descriptionDone = "tree size measured" kib = None + def __init__(self, **kwargs): + ShellCommand.__init__(self, **kwargs) + self.observer = logobserver.BufferLogObserver(wantStdout=True, + wantStderr=True) + self.addLogObserver('stdio', self.observer) + def commandComplete(self, cmd): - out = cmd.logs['stdio'].getText() + out = self.observer.getStdout() m = re.search(r'^(\d+)', out) if m: self.kib = int(m.group(1)) @@ -316,10 +322,9 @@ def __init__(self, property=None, extract_fn=None, strip=True, **kwargs): ShellCommand.__init__(self, **kwargs) - if self.extract_fn: - self.observer = logobserver.BufferLogObserver(wantStdout=True, - wantStderr=True) - self.addLogObserver('stdio', self.observer) + self.observer = logobserver.BufferLogObserver( + wantStdout=True, wantStderr=self.extract_fn) + self.addLogObserver('stdio', self.observer) self.property_changes = {} @@ -327,7 +332,7 @@ def commandComplete(self, cmd): if self.property: if cmd.didFail(): return - result = cmd.logs['stdio'].getText() + result = self.observer.getStdout() if self.strip: result = result.strip() propname = self.property @@ -433,6 +438,13 @@ def __init__(self, self.suppressions = [] self.directoryStack = [] + self.warnCount = 0 + self.loggedWarnings = [] + + self.addLogObserver( + 'stdio', + logobserver.LineConsumerLogObserver(self.warningLogConsumer)) + def addSuppression(self, suppressionList): """ This method can be used to add patters of warnings that should @@ -478,6 +490,44 @@ def warnExtractFromRegexpGroups(self, line, match): text = match.group(3) return (file, lineNo, text) + def warningLogConsumer(self): + # Now compile a regular expression from whichever warning pattern we're + # using + wre = self.warningPattern + if isinstance(wre, str): + wre = re.compile(wre) + + directoryEnterRe = self.directoryEnterPattern + if (directoryEnterRe is not None + and isinstance(directoryEnterRe, basestring)): + directoryEnterRe = re.compile(directoryEnterRe) + + directoryLeaveRe = self.directoryLeavePattern + if (directoryLeaveRe is not None + and isinstance(directoryLeaveRe, basestring)): + directoryLeaveRe = re.compile(directoryLeaveRe) + + # Check if each line in the output from this command matched our + # warnings regular expressions. If did, bump the warnings count and + # add the line to the collection of lines with warnings + self.loggedWarnings = [] + while True: + stream, line = yield + if directoryEnterRe: + match = directoryEnterRe.search(line) + if match: + self.directoryStack.append(match.group(1)) + continue + if (directoryLeaveRe and + self.directoryStack and + directoryLeaveRe.search(line)): + self.directoryStack.pop() + continue + + match = wre.match(line) + if match: + self.maybeAddWarning(self.loggedWarnings, line, match) + def maybeAddWarning(self, warnings, line, match): if self.suppressions: (file, lineNo, text) = self.warningExtractor(self, line, match) @@ -549,49 +599,11 @@ def createSummary(self, log): Warnings are collected into another log for this step, and the build-wide 'warnings-count' is updated.""" - self.warnCount = 0 - - # Now compile a regular expression from whichever warning pattern we're - # using - wre = self.warningPattern - if isinstance(wre, str): - wre = re.compile(wre) - - directoryEnterRe = self.directoryEnterPattern - if (directoryEnterRe is not None - and isinstance(directoryEnterRe, basestring)): - directoryEnterRe = re.compile(directoryEnterRe) - - directoryLeaveRe = self.directoryLeavePattern - if (directoryLeaveRe is not None - and isinstance(directoryLeaveRe, basestring)): - directoryLeaveRe = re.compile(directoryLeaveRe) - - # Check if each line in the output from this command matched our - # warnings regular expressions. If did, bump the warnings count and - # add the line to the collection of lines with warnings - warnings = [] - for line in log.getText().split("\n"): - if directoryEnterRe: - match = directoryEnterRe.search(line) - if match: - self.directoryStack.append(match.group(1)) - continue - if (directoryLeaveRe and - self.directoryStack and - directoryLeaveRe.search(line)): - self.directoryStack.pop() - continue - - match = wre.match(line) - if match: - self.maybeAddWarning(warnings, line, match) - # If there were any warnings, make the log if lines with warnings # available if self.warnCount: self.addCompleteLog("warnings (%d)" % self.warnCount, - "\n".join(warnings) + "\n") + "\n".join(self.loggedWarnings) + "\n") warnings_stat = self.getStatistic('warnings', 0) self.setStatistic('warnings', warnings_stat + self.warnCount) diff --git a/master/buildbot/test/fake/fakebuild.py b/master/buildbot/test/fake/fakebuild.py index 6439f04c236..2611bae6ad1 100644 --- a/master/buildbot/test/fake/fakebuild.py +++ b/master/buildbot/test/fake/fakebuild.py @@ -55,6 +55,9 @@ def getSourceStamp(self, codebase): return self.sources[codebase] return None + def allFiles(self): + return [] + components.registerAdapter( lambda build: build.build_status.properties, diff --git a/master/buildbot/test/fake/logfile.py b/master/buildbot/test/fake/logfile.py index 7de181d3274..c2acfe94dab 100644 --- a/master/buildbot/test/fake/logfile.py +++ b/master/buildbot/test/fake/logfile.py @@ -13,8 +13,6 @@ # # Copyright Buildbot Team Members -import warnings - from buildbot import util from buildbot.status.logfile import HEADER from buildbot.status.logfile import STDERR @@ -97,10 +95,3 @@ def fakeData(self, header='', stdout='', stderr=''): if stderr: self.stderr += stderr self.chunks.append((STDERR, stderr)) - - # removed methods, here temporarily - - def getText(self): - warnings.warn("step uses removed LogFile method `getText`") - return ''.join([c for str, c in self.chunks - if str in (STDOUT, STDERR)]) diff --git a/master/buildbot/test/unit/test_process_buildstep.py b/master/buildbot/test/unit/test_process_buildstep.py index 0bd24ab3d18..61442743346 100644 --- a/master/buildbot/test/unit/test_process_buildstep.py +++ b/master/buildbot/test/unit/test_process_buildstep.py @@ -14,12 +14,10 @@ # Copyright Buildbot Team Members import mock -import re from buildbot.process import buildstep from buildbot.process import properties from buildbot.process import remotecommand -from buildbot.process.buildstep import regex_log_evaluator from buildbot.status.results import EXCEPTION from buildbot.status.results import FAILURE from buildbot.status.results import SKIPPED @@ -63,53 +61,6 @@ def run(self): pass -class TestRegexLogEvaluator(unittest.TestCase): - - def makeRemoteCommand(self, rc, stdout, stderr=''): - cmd = fakeremotecommand.FakeRemoteCommand('cmd', {}) - cmd.fakeLogData(self, 'stdio', stdout=stdout, stderr=stderr) - cmd.rc = rc - return cmd - - def test_find_worse_status(self): - cmd = self.makeRemoteCommand(0, 'This is a big step') - step_status = FakeStepStatus() - r = [(re.compile("This is"), WARNINGS)] - new_status = regex_log_evaluator(cmd, step_status, r) - self.assertEqual(new_status, WARNINGS, - "regex_log_evaluator returned %d, expected %d" - % (new_status, WARNINGS)) - - def test_multiple_regexes(self): - cmd = self.makeRemoteCommand(0, "Normal stdout text\nan error") - step_status = FakeStepStatus() - r = [(re.compile("Normal stdout"), SUCCESS), - (re.compile("error"), FAILURE)] - new_status = regex_log_evaluator(cmd, step_status, r) - self.assertEqual(new_status, FAILURE, - "regex_log_evaluator returned %d, expected %d" - % (new_status, FAILURE)) - - def test_exception_not_in_stdout(self): - cmd = self.makeRemoteCommand(0, - "Completely normal output", "exception output") - step_status = FakeStepStatus() - r = [(re.compile("exception"), EXCEPTION)] - new_status = regex_log_evaluator(cmd, step_status, r) - self.assertEqual(new_status, EXCEPTION, - "regex_log_evaluator returned %d, expected %d" - % (new_status, EXCEPTION)) - - def test_pass_a_string(self): - cmd = self.makeRemoteCommand(0, "Output", "Some weird stuff on stderr") - step_status = FakeStepStatus() - r = [("weird stuff", WARNINGS)] - new_status = regex_log_evaluator(cmd, step_status, r) - self.assertEqual(new_status, WARNINGS, - "regex_log_evaluator returned %d, expected %d" - % (new_status, WARNINGS)) - - class TestBuildStep(steps.BuildStepMixin, config.ConfigErrorsMixin, unittest.TestCase): class FakeBuildStep(buildstep.BuildStep): diff --git a/master/buildbot/test/unit/test_process_log.py b/master/buildbot/test/unit/test_process_log.py index 98100e4f27f..e2dfdb9d8d4 100644 --- a/master/buildbot/test/unit/test_process_log.py +++ b/master/buildbot/test/unit/test_process_log.py @@ -287,11 +287,3 @@ def setUp(self): step = mock.Mock(name='fake step') step.logobservers = [] self.log = fakelogfile.FakeLogFile('stdio', step) - - # mark these TODO for the fake, for the moment -- leaving these methods in - # place lets the tests pass until all of the built-in steps are rewritten - # to use LogObservers, etc. - - def test_signature_getText_removed(self): - InterfaceTests.test_signature_getText_removed(self) - test_signature_getText_removed.todo = "not removed yet" diff --git a/master/buildbot/test/unit/test_process_logobserver.py b/master/buildbot/test/unit/test_process_logobserver.py index 60b5a130612..065f1a56f08 100644 --- a/master/buildbot/test/unit/test_process_logobserver.py +++ b/master/buildbot/test/unit/test_process_logobserver.py @@ -88,6 +88,70 @@ def finishReceived(self): self.obs.append(('fin',)) +class TestLineConsumerLogObesrver(unittest.TestCase): + + def setUp(self): + self.master = fakemaster.make_master(testcase=self, wantData=True) + + @defer.inlineCallbacks + def do_test_sequence(self, consumer): + logid = yield self.master.data.updates.newLog(1, u'mine', u's') + l = log.Log.new(self.master, 'mine', 's', logid, 'utf-8') + lo = logobserver.LineConsumerLogObserver(consumer) + lo.setLog(l) + + yield l.addStdout(u'hello\n') + yield l.addStderr(u'cruel\n') + yield l.addStdout(u'multi\nline\nchunk\n') + yield l.addHeader(u'H1\nH2\n') + yield l.finish() + + @defer.inlineCallbacks + def test_sequence_finish(self): + results = [] + + def consumer(): + while True: + try: + stream, line = yield + results.append((stream, line)) + except GeneratorExit: + results.append('finish') + raise + yield self.do_test_sequence(consumer) + + self.assertEqual(results, [ + ('o', u'hello'), + ('e', u'cruel'), + ('o', u'multi'), + ('o', u'line'), + ('o', u'chunk'), + ('h', u'H1'), + ('h', u'H2'), + 'finish', + ]) + + @defer.inlineCallbacks + def test_sequence_no_finish(self): + results = [] + + def consumer(): + while True: + stream, line = yield + results.append((stream, line)) + yield self.do_test_sequence(consumer) + + self.assertEqual(results, [ + ('o', u'hello'), + ('e', u'cruel'), + ('o', u'multi'), + ('o', u'line'), + ('o', u'chunk'), + ('h', u'H1'), + ('h', u'H2'), + ]) + + class TestLogLineObserver(unittest.TestCase): def setUp(self): diff --git a/master/buildbot/test/unit/test_steps_package_deb_pbuilder.py b/master/buildbot/test/unit/test_steps_package_deb_pbuilder.py index 1d2b993b021..bd8bb54eb8f 100644 --- a/master/buildbot/test/unit/test_steps_package_deb_pbuilder.py +++ b/master/buildbot/test/unit/test_steps_package_deb_pbuilder.py @@ -71,7 +71,7 @@ def test_update(self): self.expectOutcome(result=SUCCESS, status_text=['pdebuild']) return self.runStep() - def test_buildonly(self): + def test_buildonly_and_property(self): self.setupStep(pbuilder.DebPbuilder()) self.expectCommands( Expect('stat', {'file': '/var/cache/pbuilder/stable-local-buildbot.tgz'}) @@ -81,8 +81,14 @@ def test_buildonly(self): command=['pdebuild', '--buildresult', '.', '--pbuilder', '/usr/sbin/pbuilder', '--', '--buildresult', '.', '--basetgz', '/var/cache/pbuilder/stable-local-buildbot.tgz']) + + ExpectShell.log( + 'stdio', + stdout='blah\ndpkg-genchanges >../somefilename.changes\foo\n') + 0) self.expectOutcome(result=SUCCESS, status_text=['pdebuild']) + self.expectProperty('deb-changes', + 'somefilename.changes', + 'DebPbuilder') return self.runStep() def test_architecture(self): diff --git a/master/buildbot/test/unit/test_steps_python.py b/master/buildbot/test/unit/test_steps_python.py index 55da78345cd..2111d9b6efe 100644 --- a/master/buildbot/test/unit/test_steps_python.py +++ b/master/buildbot/test/unit/test_steps_python.py @@ -78,6 +78,58 @@ doesn't have a title: no link will be generated\ ''' +# this is from a run of epydoc against the buildbot source.. +epydoc_output = '''\ + [............... ++--------------------------------------------------------------------- +| In /home/dustin/code/buildbot/t/buildbot/master/buildbot/ +| ec2buildslave.py: +| Import failed (but source code parsing was successful). +| Error: ImportError: No module named boto (line 19) +| + [.... +Warning: Unable to extract the base list for + twisted.web.resource.EncodingResourceWrapper: Bad dotted name + [...... ++--------------------------------------------------------------------- +| In /home/dustin/code/buildbot/t/buildbot/master/buildbot/buildslave/ +| ec2.py: +| Import failed (but source code parsing was successful). +| Error: ImportError: No module named boto (line 28) +| + [........... ++--------------------------------------------------------------------- +| In /home/dustin/code/buildbot/t/buildbot/master/buildbot/status/ +| status_push.py: +| Import failed (but source code parsing was successful). +| Error: ImportError: No module named status_json (line 40) +| + [....................Special descriptor for class __provides__ +''' + + +class BuildEPYDoc(steps.BuildStepMixin, unittest.TestCase): + + def setUp(self): + return self.setUpBuildStep() + + def tearDown(self): + return self.tearDownBuildStep() + + def test_sample(self): + self.setupStep(python.BuildEPYDoc()) + self.expectCommands( + ExpectShell(workdir='wkdir', command=['make', 'epydocs'], + usePTY='slave-config') + + ExpectShell.log('stdio', + stdout=epydoc_output) + + 1, + ) + self.expectOutcome(result=FAILURE, + status_text=['epydoc', 'warn=1', + 'err=3', 'failed']) + return self.runStep() + class PyLint(steps.BuildStepMixin, unittest.TestCase): diff --git a/master/buildbot/test/unit/test_steps_python_twisted.py b/master/buildbot/test/unit/test_steps_python_twisted.py index a891d38214d..9e81a170a06 100644 --- a/master/buildbot/test/unit/test_steps_python_twisted.py +++ b/master/buildbot/test/unit/test_steps_python_twisted.py @@ -13,13 +13,80 @@ # # Copyright Buildbot Team Members +import textwrap + from buildbot.process.properties import Property +from buildbot.status.results import FAILURE from buildbot.status.results import SUCCESS +from buildbot.status.results import WARNINGS from buildbot.steps import python_twisted from buildbot.test.fake.remotecommand import ExpectShell from buildbot.test.util import steps from twisted.trial import unittest +failureLog = '''\ +buildbot.test.unit.test_steps_python_twisted.Trial.testProperties ... [FAILURE] +buildbot.test.unit.test_steps_python_twisted.Trial.test_run_env ... [FAILURE] +buildbot.test.unit.test_steps_python_twisted.Trial.test_run_env_nodupe ... [FAILURE]/home/dustin/code/buildbot/t/buildbot/master/buildbot/test/fake/logfile.py:92: UserWarning: step uses removed LogFile method `getText` +buildbot.test.unit.test_steps_python_twisted.Trial.test_run_env_supplement ... [FAILURE]/home/dustin/code/buildbot/t/buildbot/master/buildbot/test/fake/logfile.py:92: UserWarning: step uses removed LogFile method `getText` +buildbot.test.unit.test_steps_python_twisted.Trial.test_run_jobs ... [FAILURE]/home/dustin/code/buildbot/t/buildbot/master/buildbot/test/fake/logfile.py:92: UserWarning: step uses removed LogFile method `getText` +buildbot.test.unit.test_steps_python_twisted.Trial.test_run_jobsProperties ... [FAILURE] +buildbot.test.unit.test_steps_python_twisted.Trial.test_run_plural ... [FAILURE] +buildbot.test.unit.test_steps_python_twisted.Trial.test_run_singular ... [FAILURE] + +=============================================================================== +[FAIL] +Traceback (most recent call last): + File "/home/dustin/code/buildbot/t/buildbot/master/buildbot/test/util/steps.py", line 244, in check + "expected step outcome") + File "/home/dustin/code/buildbot/t/buildbot/sandbox/lib/python2.7/site-packages/twisted/trial/_synctest.py", line 356, in assertEqual + % (msg, pformat(first), pformat(second))) +twisted.trial.unittest.FailTest: expected step outcome +not equal: +a = {'result': 3, 'status_text': ['2 tests', 'passed']} +b = {'result': 0, 'status_text': ['2 tests', 'passed']} + + +buildbot.test.unit.test_steps_python_twisted.Trial.testProperties +buildbot.test.unit.test_steps_python_twisted.Trial.test_run_plural +=============================================================================== +[FAIL] +Traceback (most recent call last): + File "/home/dustin/code/buildbot/t/buildbot/master/buildbot/test/util/steps.py", line 244, in check + "expected step outcome") + File "/home/dustin/code/buildbot/t/buildbot/sandbox/lib/python2.7/site-packages/twisted/trial/_synctest.py", line 356, in assertEqual + % (msg, pformat(first), pformat(second))) +twisted.trial.unittest.FailTest: expected step outcome +not equal: +a = {'result': 3, 'status_text': ['no tests', 'run']} +b = {'result': 0, 'status_text': ['no tests', 'run']} + + +buildbot.test.unit.test_steps_python_twisted.Trial.test_run_env +buildbot.test.unit.test_steps_python_twisted.Trial.test_run_env_nodupe +buildbot.test.unit.test_steps_python_twisted.Trial.test_run_env_supplement +=============================================================================== +[FAIL] +Traceback (most recent call last): + File "/home/dustin/code/buildbot/t/buildbot/master/buildbot/test/util/steps.py", line 244, in check + "expected step outcome") + File "/home/dustin/code/buildbot/t/buildbot/sandbox/lib/python2.7/site-packages/twisted/trial/_synctest.py", line 356, in assertEqual + % (msg, pformat(first), pformat(second))) +twisted.trial.unittest.FailTest: expected step outcome +not equal: +a = {'result': 3, 'status_text': ['1 test', 'passed']} +b = {'result': 0, 'status_text': ['1 test', 'passed']} + + +buildbot.test.unit.test_steps_python_twisted.Trial.test_run_jobs +buildbot.test.unit.test_steps_python_twisted.Trial.test_run_jobsProperties +buildbot.test.unit.test_steps_python_twisted.Trial.test_run_singular +------------------------------------------------------------------------------- +Ran 8 tests in 0.101s + +FAILED (failures=8) +''' + class Trial(steps.BuildStepMixin, unittest.TestCase): @@ -115,6 +182,29 @@ def test_run_plural(self): self.expectOutcome(result=SUCCESS, status_text=['2 tests', 'passed']) return self.runStep() + def test_run_failure(self): + self.setupStep( + python_twisted.Trial(workdir='build', + tests='testname', + testpath=None)) + self.expectCommands( + ExpectShell(workdir='build', + command=['trial', '--reporter=bwverbose', 'testname'], + usePTY="slave-config", + logfiles={'test.log': '_trial_temp/test.log'}) + + ExpectShell.log('stdio', stdout=failureLog) + + 1 + ) + self.expectOutcome(result=FAILURE, status_text=['tests', '8 failures']) + self.expectLogfile('problems', failureLog.split('\n\n', 1)[1]) + self.expectLogfile('warnings', textwrap.dedent('''\ + buildbot.test.unit.test_steps_python_twisted.Trial.test_run_env_nodupe ... [FAILURE]/home/dustin/code/buildbot/t/buildbot/master/buildbot/test/fake/logfile.py:92: UserWarning: step uses removed LogFile method `getText` + buildbot.test.unit.test_steps_python_twisted.Trial.test_run_env_supplement ... [FAILURE]/home/dustin/code/buildbot/t/buildbot/master/buildbot/test/fake/logfile.py:92: UserWarning: step uses removed LogFile method `getText` + buildbot.test.unit.test_steps_python_twisted.Trial.test_run_jobs ... [FAILURE]/home/dustin/code/buildbot/t/buildbot/master/buildbot/test/fake/logfile.py:92: UserWarning: step uses removed LogFile method `getText` + buildbot.test.unit.test_steps_python_twisted.Trial.test_run_jobsProperties ... [FAILURE] + ''')) + return self.runStep() + def testProperties(self): self.setupStep(python_twisted.Trial(workdir='build', tests=Property('test_list'), @@ -190,3 +280,40 @@ def test_run_jobsProperties(self): ) self.expectOutcome(result=SUCCESS, status_text=['1 test', 'passed']) return self.runStep() + + +class HLint(steps.BuildStepMixin, unittest.TestCase): + + def setUp(self): + return self.setUpBuildStep() + + def tearDown(self): + return self.tearDownBuildStep() + + def test_run_ok(self): + self.setupStep(python_twisted.HLint(workdir='build'), + buildFiles=['foo.xhtml']) + self.expectCommands( + ExpectShell(workdir='build', + command=['bin/lore', '-p', '--output', 'lint', 'foo.xhtml'], + usePTY="slave-config") + + ExpectShell.log('stdio', stdout="dunno what hlint output looks like..\n") + + 0 + ) + self.expectLogfile('files', 'foo.xhtml\n') + self.expectOutcome(result=SUCCESS, status_text=['hlint']) + return self.runStep() + + def test_run_warnings(self): + self.setupStep(python_twisted.HLint(workdir='build'), + buildFiles=['foo.xhtml']) + self.expectCommands( + ExpectShell(workdir='build', + command=['bin/lore', '-p', '--output', 'lint', 'foo.xhtml'], + usePTY="slave-config") + + ExpectShell.log('stdio', stdout="colon: meaning warning\n") + + 0 + ) + self.expectLogfile('warnings', 'colon: meaning warning') + self.expectOutcome(result=WARNINGS, status_text=['hlint', 'warnings']) + return self.runStep() diff --git a/master/buildbot/test/unit/test_steps_shell.py b/master/buildbot/test/unit/test_steps_shell.py index 70b4ef458cb..56b9b886ec9 100644 --- a/master/buildbot/test/unit/test_steps_shell.py +++ b/master/buildbot/test/unit/test_steps_shell.py @@ -406,7 +406,7 @@ def test_run_property(self): self.expectOutcome(result=SUCCESS, status_text=["property 'res' set"]) self.expectProperty("res", "abcdef") # note: stripped - self.expectLogfile('property changes', r"res: 'abcdef'") + self.expectLogfile('property changes', r"res: u'abcdef'") return self.runStep() def test_run_property_no_strip(self): @@ -421,7 +421,7 @@ def test_run_property_no_strip(self): self.expectOutcome(result=SUCCESS, status_text=["property 'res' set"]) self.expectProperty("res", "\n\nabcdef\n") - self.expectLogfile('property changes', r"res: '\n\nabcdef\n'") + self.expectLogfile('property changes', r"res: u'\n\nabcdef\n'") return self.runStep() def test_run_failure(self): diff --git a/master/buildbot/test/util/steps.py b/master/buildbot/test/util/steps.py index fe0cff14ba0..2b10aee2c33 100644 --- a/master/buildbot/test/util/steps.py +++ b/master/buildbot/test/util/steps.py @@ -63,12 +63,13 @@ def tearDownBuildStep(self): # utilities - def setupStep(self, step, slave_version={'*': "99.99"}, slave_env={}): + def setupStep(self, step, slave_version={'*': "99.99"}, slave_env={}, + buildFiles=[]): """ Set up C{step} for testing. This begins by using C{step} as a factory to create a I{new} step instance, thereby testing that the the factory arguments are handled correctly. It then creates a comfortable - environment for the slave to run in, repleate with a fake build and a + environment for the slave to run in, replete with a fake build and a fake slave. As a convenience, it calls the step's setDefaultWorkdir method with @@ -87,6 +88,7 @@ def setupStep(self, step, slave_version={'*': "99.99"}, slave_env={}): # step.build b = self.build = fakebuild.FakeBuild() + b.allFiles = lambda: buildFiles b.master = self.master def getSlaveVersion(cmd, oldversion): diff --git a/master/docs/developer/cls-logobserver.rst b/master/docs/developer/cls-logobserver.rst index 64fd1f3fd79..8a9e99a4aa7 100644 --- a/master/docs/developer/cls-logobserver.rst +++ b/master/docs/developer/cls-logobserver.rst @@ -66,6 +66,42 @@ LogObservers This method, inherited from :py:class:`LogObserver`, is invoked when the observed log is finished. +.. py:class:: LineConsumerLogObserver + + This subclass of :py:class:`LogObserver` takes a generator function and "sends" each line to that function. + This allows consumers to be written as stateful Python functions, e.g., :: + + def logConsumer(self): + while True: + stream, line = yield + if stream == 'o' and line.startswith('W'): + self.warnings.append(line[1:]) + + def __init__(self): + ... + self.warnings = [] + self.addLogObserver('stdio', logobserver.LineConsumerLogObserver(self.logConsumer)) + + Each ``yield`` expression evaluates to a tuple of (stream, line), where the stream is one of 'o', 'e', or 'h' for stdout, stderr, and header, respectively. + As with any generator function, the ``yield`` expression will raise a ``GeneratorExit`` exception when the generator is complete. + To do something after the log is finished, just catch this exception (but then re-raise it or return) :: + + def logConsumer(self): + while True: + try: + stream, line = yield + if stream == 'o' and line.startswith('W'): + self.warnings.append(line[1:]) + except GeneratorExit: + self.warnings.sort() + return + + .. warning:: + + This use of generator functions is a simple Python idiom first described in [PEP 342](http://www.python.org/dev/peps/pep-0342/). + It is unrelated to the generators used in ``inlineCallbacks``. + In fact, consumers of this type are incompatible with asynchronous programming, as each line must be processed immediately. + .. py:class:: BufferLogObserver(wantStdout=True, wantStderr=False) :param boolean wantStdout: true if stdout should be buffered diff --git a/master/docs/manual/cfg-statustargets.rst b/master/docs/manual/cfg-statustargets.rst index 3ebf9ca9359..390d55b78b0 100644 --- a/master/docs/manual/cfg-statustargets.rst +++ b/master/docs/manual/cfg-statustargets.rst @@ -469,6 +469,7 @@ given below:: break name = "%s.%s" % (log.getStep().getName(), log.getName()) status, dummy = log.getStep().getResults() + # XXX logs no longer have getText methods!! content = log.getText().splitlines() # Note: can be VERY LARGE url = u'%s/steps/%s/logs/%s' % (master_status.getURLForThing(build), log.getStep().getName(), @@ -733,6 +734,7 @@ Log information :: for log in build.getLogs(): log_name = "%s.%s" % (log.getStep().getName(), log.getName()) log_status, dummy = log.getStep().getResults() + # XXX logs no longer have a getText method log_body = log.getText().splitlines() # Note: can be VERY LARGE log_url = '%s/steps/%s/logs/%s' % (master_status.getURLForThing(build), log.getStep().getName(),