@@ -1,5 +1,11 @@ User visible changes in Buildbot. -*- outline -*- +** Suppression of selected compiler warnings + +The WarningCountingShellCommand class has been extended with the ability to +upload from the slave a file contain warnings to be ignored. See the +documentation of the suppressionFile argument to the Compile build step. + ** New buildstep `MTR' A new class buildbot.process.mtrlogobserver.MTR was added. This buildstep is NEWS @@ -2,7 +2,9 @@ import re from twisted.python import log +from twisted.spread import pb from buildbot.process.buildstep import LoggingBuildStep, RemoteShellCommand +from buildbot.process.buildstep import RemoteCommand from buildbot.status.builder import SUCCESS, WARNINGS, FAILURE, STDOUT, STDERR # for existing configurations that import WithProperties from here. We like @@ -302,21 +304,189 @@ class Configure(ShellCommand): descriptionDone = ["configure"] command = ["./configure"] +class StringFileWriter(pb.Referenceable): + """ + FileWriter class that just puts received data into a buffer. + + Used to upload a file from slave for inline processing rather than + writing into a file on master. + """ + def __init__(self): + self.buffer = "" + + def remote_write(self, data): + self.buffer += data + + def remote_close(self): + pass + +class SilentRemoteCommand(RemoteCommand): + """ + Remote command subclass used to run an internal file upload command on the + slave. We do not need any progress updates from such command, so override + remoteUpdate() with an empty method. + """ + def remoteUpdate(self, update): + pass + class WarningCountingShellCommand(ShellCommand): warnCount = 0 warningPattern = '.*warning[: ].*' + # The defaults work for GNU Make. + directoryEnterPattern = "make.*: Entering directory [\"`'](.*)['`\"]" + directoryLeavePattern = "make.*: Leaving directory" + suppressionFile = None + + commentEmptyLineRe = re.compile(r"^\s*(\#.*)?$") + suppressionLineRe = re.compile(r"^\s*(.+?)\s*:\s*(.+?)\s*(?:[:]\s*([0-9]+)(?:-([0-9]+))?\s*)?$") - def __init__(self, warningPattern=None, **kwargs): + def __init__(self, workdir=None, + warningPattern=None, warningExtractor=None, + directoryEnterPattern=None, directoryLeavePattern=None, + suppressionFile=None, **kwargs): + self.workdir = workdir # See if we've been given a regular expression to use to match # warnings. If not, use a default that assumes any line with "warning" # present is a warning. This may lead to false positives in some cases. if warningPattern: self.warningPattern = warningPattern + if directoryEnterPattern: + self.directoryEnterPattern = directoryEnterPattern + if directoryLeavePattern: + self.directoryLeavePattern = directoryLeavePattern + if suppressionFile: + self.suppressionFile = suppressionFile + if warningExtractor: + self.warningExtractor = warningExtractor + else: + self.warningExtractor = WarningCountingShellCommand.warnExtractWholeLine # And upcall to let the base class do its work - ShellCommand.__init__(self, **kwargs) + ShellCommand.__init__(self, workdir=workdir, **kwargs) + + self.addFactoryArguments(warningPattern=warningPattern, + directoryEnterPattern=directoryEnterPattern, + directoryLeavePattern=directoryLeavePattern, + warningExtractor=warningExtractor, + suppressionFile=suppressionFile) + self.suppressions = [] + self.directoryStack = [] - self.addFactoryArguments(warningPattern=warningPattern) + def setDefaultWorkdir(self, workdir): + if self.workdir is None: + self.workdir = workdir + ShellCommand.setDefaultWorkdir(self, workdir) + + def addSuppression(self, suppressionList): + """ + This method can be used to add patters of warnings that should + not be counted. + + It takes a single argument, a list of patterns. + + Each pattern is a 4-tuple (FILE-RE, WARN-RE, START, END). + + FILE-RE is a regular expression (string or compiled regexp), or None. + If None, the pattern matches all files, else only files matching the + regexp. If directoryEnterPattern is specified in the class constructor, + matching is against the full path name, eg. src/main.c. + + WARN-RE is similarly a regular expression matched against the + text of the warning, or None to match all warnings. + + START and END form an inclusive line number range to match against. If + START is None, there is no lower bound, similarly if END is none there + is no upper bound.""" + + for fileRe, warnRe, start, end in suppressionList: + if fileRe != None and isinstance(fileRe, str): + fileRe = re.compile(fileRe) + if warnRe != None and isinstance(warnRe, str): + warnRe = re.compile(warnRe) + self.suppressions.append((fileRe, warnRe, start, end)) + + def warnExtractWholeLine(self, line, match): + """ + Extract warning text as the whole line. + No file names or line numbers.""" + return (None, None, line) + + def warnExtractFromRegexpGroups(self, line, match): + """ + Extract file name, line number, and warning text as groups (1,2,3) + of warningPattern match.""" + file = match.group(1) + lineNo = match.group(2) + if lineNo != None: + lineNo = int(lineNo) + text = match.group(3) + return (file, lineNo, text) + + def maybeAddWarning(self, warnings, line, match): + if self.suppressions: + (file, lineNo, text) = self.warningExtractor(self, line, match) + + if file != None and file != "" and self.directoryStack: + currentDirectory = self.directoryStack[-1] + if currentDirectory != None and currentDirectory != "": + file = "%s/%s" % (currentDirectory, file) + + # Skip adding the warning if any suppression matches. + for fileRe, warnRe, start, end in self.suppressions: + if ( (file == None or fileRe == None or fileRe.search(file)) and + (warnRe == None or warnRe.search(text)) and + lineNo != None and + (start == None or start <= lineNo) and + (end == None or end >= lineNo) ): + return + + warnings.append(line) + self.warnCount += 1 + + def start(self): + if self.suppressionFile == None: + return ShellCommand.start(self) + + version = self.slaveVersion("uploadFile") + if not version: + m = "Slave is too old, does not know about uploadFile" + raise BuildSlaveTooOldError(m) + + self.myFileWriter = StringFileWriter() + + args = { + 'slavesrc': self.suppressionFile, + 'workdir': self.workdir, + 'writer': self.myFileWriter, + 'maxsize': None, + 'blocksize': 32*1024, + } + cmd = SilentRemoteCommand('uploadFile', args) + d = self.runCommand(cmd) + d.addCallback(self.uploadDone) + d.addErrback(self.failed) + + def uploadDone(self, dummy): + lines = self.myFileWriter.buffer.split("\n") + del(self.myFileWriter) + + list = [] + for line in lines: + if self.commentEmptyLineRe.match(line): + continue + match = self.suppressionLineRe.match(line) + if (match): + file, test, start, end = match.groups() + if (end != None): + end = int(end) + if (start != None): + start = int(start) + if end == None: + end = start + list.append((file, test, start, end)) + + self.addSuppression(list) + return ShellCommand.start(self) def createSummary(self, log): self.warnCount = 0 @@ -330,6 +500,14 @@ class WarningCountingShellCommand(ShellCommand): if isinstance(wre, str): wre = re.compile(wre) + directoryEnterRe = self.directoryEnterPattern + if directoryEnterRe != None and isinstance(directoryEnterRe, str): + directoryEnterRe = re.compile(directoryEnterRe) + + directoryLeaveRe = self.directoryLeavePattern + if directoryLeaveRe != None and isinstance(directoryLeaveRe, str): + directoryLeaveRe = re.compile(directoryLeaveRe) + # Check if each line in the output from this command matched our # warnings regular expressions. If did, bump the warnings count and # add the line to the collection of lines with warnings @@ -337,9 +515,18 @@ class WarningCountingShellCommand(ShellCommand): # TODO: use log.readlines(), except we need to decide about stdout vs # stderr for line in log.getText().split("\n"): - if wre.match(line): - warnings.append(line) - self.warnCount += 1 + if directoryEnterRe: + match = directoryEnterRe.match(line) + if match: + self.directoryStack.append(match.group(1)) + if (directoryLeaveRe and + self.directoryStack and + directoryLeaveRe.match(line)): + self.directoryStack.pop() + + match = wre.match(line) + if match: + self.maybeAddWarning(warnings, line, match) # If there were any warnings, make the log if lines with warnings # available buildbot/steps/shell.py @@ -14,6 +14,7 @@ # todo: test batched updates, by invoking remote_update(updates) instead of # statusUpdate(update). Also involves interrupted builds. +import sys import os from twisted.trial import unittest @@ -624,6 +625,112 @@ ending line results = step.evaluateCommand(cmd) self.failUnlessEqual(results, WARNINGS) + def testCompile4(self): + # Test suppression of warnings. + self.masterbase = "Warnings.testCompile4" + step = self.makeStep(shell.Compile, + warningPattern="^(.*?):([0-9]+): [Ww]arning: (.*)$", + warningExtractor=shell.Compile.warnExtractFromRegexpGroups, + directoryEnterPattern="make.*: Entering directory [\"`'](.*)['`\"]", + directoryLeavePattern="make.*: Leaving directory") + step.addSuppression([(r"/subdir/", r"xyzzy", None, None), + (r"foo.c", r".*", None, 20), + (r"foo.c", r".*", 200, None), + (r"foo.c", r".*", 50, 50), + (r"xxx", r".*", None, None), + ]) + log = step.addLog("stdio") + output = \ +"""Making all in . +make[1]: Entering directory `/abs/path/build' +foo.c:10: warning: `bar' defined but not used +foo.c:50: warning: `bar' defined but not used +make[2]: Entering directory `/abs/path/build/subdir' +baz.c:33: warning: `xyzzy' defined but not used +baz.c:34: warning: `magic' defined but not used +make[2]: Leaving directory `/abs/path/build/subdir' +foo.c:100: warning: `xyzzy' defined but not used +foo.c:200: warning: `bar' defined but not used +make[2]: Leaving directory `/abs/path/build' +""" + log.addStdout(output) + log.finish() + step.createSummary(log) + self.failUnlessEqual(step.getProperty("warnings-count"), 2) + logs = {} + for log in step.step_status.getLogs(): + logs[log.getName()] = log + self.failUnless("warnings" in logs) + lines = logs["warnings"].readlines() + self.failUnlessEqual(len(lines), 2) + self.failUnlessEqual(lines[0], "baz.c:34: warning: `magic' defined but not used\n") + self.failUnlessEqual(lines[1], "foo.c:100: warning: `xyzzy' defined but not used\n") + + cmd = buildstep.RemoteCommand(None, {}) + cmd.rc = 0 + results = step.evaluateCommand(cmd) + self.failUnlessEqual(results, WARNINGS) + + def filterArgs(self, args): + if "writer" in args: + args["writer"] = self.wrap(args["writer"]) + return args + + suppressionFileData = """ +# Sample suppressions file for testing + +/subdir/ : xyzzy +foo.c: .* : 0-20 +foo.c: .*: 200-10000 +foo.c :.*: 50 +xxx : .* +""" + def testCompile5(self): + # Test downloading warning suppression file from slave. + self.slavebase = "Warnings.testCompile5.slave" + self.masterbase = "Warnings.testCompile5.master" + sb = self.makeSlaveBuilder() + os.mkdir(os.path.join(self.slavebase, self.slavebuilderbase, + "build")) + output = \ +"""Making all in . +make[1]: Entering directory `/abs/path/build' +foo.c:10: warning: `bar' defined but not used +foo.c:50: warning: `bar' defined but not used +make[2]: Entering directory `/abs/path/build/subdir' +baz.c:33: warning: `xyzzy' defined but not used +baz.c:34: warning: `magic' defined but not used +make[2]: Leaving directory `/abs/path/build/subdir' +foo.c:100: warning: `xyzzy' defined but not used +foo.c:200: warning: `bar' defined but not used +make[2]: Leaving directory `/abs/path/build' +""" + printStatement = ('print """%s"""' % output) + step = self.makeStep(shell.Compile, + warningPattern="^(.*?):([0-9]+): [Ww]arning: (.*)$", + warningExtractor=shell.Compile.warnExtractFromRegexpGroups, + suppressionFile="warnings.supp", + command=[sys.executable, "-c", printStatement]) + slavesrc = os.path.join(self.slavebase, + self.slavebuilderbase, + "build", + "warnings.supp") + open(slavesrc, "w").write(self.suppressionFileData) + + d = self.runStep(step) + def _checkResult(result): + self.failUnlessEqual(step.getProperty("warnings-count"), 2) + logs = {} + for log in step.step_status.getLogs(): + logs[log.getName()] = log + self.failUnless("warnings" in logs) + lines = logs["warnings"].readlines() + self.failUnlessEqual(len(lines), 2) + self.failUnlessEqual(lines[0], "baz.c:34: warning: `magic' defined but not used\n") + self.failUnlessEqual(lines[1], "foo.c:100: warning: `xyzzy' defined but not used\n") + + d.addCallback(_checkResult) + return d class TreeSize(StepTester, unittest.TestCase): def testTreeSize(self): buildbot/test/test_steps.py @@ -5759,6 +5759,66 @@ The @code{warningPattern=} can also be a pre-compiled python regexp object: this makes it possible to add flags like @code{re.I} (to use case-insensitive matching). +The @code{suppressionFile=} argument can be specified as the (relative) path +of a file inside the workdir defining warnings to be suppressed from the +warning counting and log file. The file will be uploaded to the master from +the slave before compiling, and any warning matched by a line in the +suppression file will be ignored. This is useful to accept certain warnings +(eg. in some special module of the source tree or in cases where the compiler +is being particularly stupid), yet still be able to easily detect and fix the +introduction of new warnings. + +The file must contain one line per pattern of warnings to ignore. Empty lines +and lines beginning with @code{#} are ignored. Other lines must consist of a +regexp matching the file name, followed by a colon (@code{:}), followed by a +regexp matching the text of the warning. Optionally this may be followed by +another colon and a line number range. For example: + +@example +# Sample warning suppression file + +mi_packrec.c : .*result of 32-bit shift implicitly converted to 64 bits.* : 560-600 +DictTabInfo.cpp : .*invalid access to non-static.* +kernel_types.h : .*only defines private constructors and has no friends.* : 51 +@end example + +If no line number range is specified, the pattern matches the whole file; if +only one number is given it matches only on that line. + +The default warningPattern regexp only matches the warning text, so line +numbers and file names are ignored. To enable line number and file name +matching, privide a different regexp and provide a function (callable) as the +argument of @code{warningExtractor=}. The function is called with three +arguments: the BuildStep object, the line in the log file with the warning, +and the @code{SRE_Match} object of the regexp search for @code{warningPattern}. It +should return a tuple @code{(filename, linenumber, warning_test)}. For +example: + +@example +f.addStep(Compile(command=["make"], + warningPattern="^(.*?):([0-9]+): [Ww]arning: (.*)$", + warningExtractor=Compile.warnExtractFromRegexpGroups, + suppressionFile="support-files/compiler_warnings.supp")) +@end example + +(@code{Compile.warnExtractFromRegexpGroups} is a pre-defined function that +returns the filename, linenumber, and text from groups (1,2,3) of the regexp +match). + +In projects with source files in multiple directories, it is possible to get +full path names for file names matched in the suppression file, as long as the +build command outputs the names of directories as they are entered into and +left again. For this, specify regexps for the arguments +@code{directoryEnterPattern=} and @code{directoryLeavePattern=}. The +@code{directoryEnterPattern=} regexp should return the name of the directory +entered into in the first matched group. The defaults, which are suitable for +GNU Make, are these: + +@example +directoryEnterPattern = "make.*: Entering directory [\"`'](.*)['`\"]" +directoryLeavePattern = "make.*: Leaving directory" +@end example + (TODO: this step needs to be extended to look for GCC error messages as well, and collect them into a separate logfile, along with the source code filenames involved). docs/buildbot.texinfo 98578726636b4b401724fde0a553596e6064724c Kristian Nielsen knmeister@gmail.com http://github.com/djmitche/buildbot/commit/5565d333475d1a904f40e1c37a5c8b561cd03c47 5565d333475d1a904f40e1c37a5c8b561cd03c47 2009-08-18T04:08:38-07:00 2009-08-18T04:08:38-07:00 Implement suppression of specific warnings in WarningCountingShellCommand Warnings to be suppressed can be matched by regexp against filename or warning text, and against line number range. Suppressions can be parsed from a file downloaded from the slave build directory (so warnings to suppress are tracked in revision control along with the source code being compiled). 14ea5eb19967356c67030bcb8835c62e7d697cd1 Kristian Nielsen knmeister@gmail.com