ChangeLog-0.7.11 buildbot/process/mtrlogobserver.py buildbot/process/subunitlogobserver.py buildbot/status/web/console.py buildbot/status/web/console_html.py buildbot/status/web/console_js.py buildbot/test/test_console.py contrib/bitbucket_buildbot.py @@ -1,6 +1,25 @@ User visible changes in Buildbot. -*- outline -*- -* Release 0.7.11 (XXX) +** Suppression of selected compiler warnings + +The WarningCountingShellCommand class has been extended with the ability to +upload from the slave a file contain warnings to be ignored. See the +documentation of the suppressionFile argument to the Compile build step. + +** New buildstep `MTR' + +A new class buildbot.process.mtrlogobserver.MTR was added. This buildstep is +used to run test suites using mysql-test-run. It parses the stdio output for +test failures and summarises them on the waterfall page. It also makes server +error logs available for debugging failures, and optionally inserts +information about test runs and test failures into an external database. + +* Release 0.7.11p (July 16, 2009) + +Fixes a few test failures in 0.7.11, and gives a default value for branchType +if it is not specified by the master. + +* Release 0.7.11 (July 5, 2009) Developers too numerous to mention contributed to this release. Buildbot has truly become a community-maintained application. Much hard work is not NEWS @@ -1 +1 @@ -version = "0.7.11-rc2" +version = "latest" buildbot/__init__.py @@ -31,6 +31,9 @@ class BuildSet: def waitUntilFinished(self): return self.status.waitUntilFinished() + def getProperties(self): + return self.properties + def start(self, builders): """This is called by the BuildMaster to actually create and submit the BuildRequests.""" buildbot/buildset.py @@ -205,6 +205,17 @@ class AbstractBuildSlave(NewCredPerspective, service.MultiService): return d1 d.addCallback(_get_info) + def _get_version(res): + d1 = bot.callRemote("getVersion") + def _got_version(version): + state["version"] = version + def _version_unavailable(why): + # probably an old slave + log.msg("BuildSlave.version_unavailable") + log.err(why) + d1.addCallbacks(_got_version, _version_unavailable) + d.addCallback(_get_version) + def _get_commands(res): d1 = bot.callRemote("getCommands") def _got_commands(commands): @@ -222,6 +233,7 @@ class AbstractBuildSlave(NewCredPerspective, service.MultiService): def _accept_slave(res): self.slave_status.setAdmin(state.get("admin")) self.slave_status.setHost(state.get("host")) + self.slave_status.setVersion(state.get("version")) self.slave_status.setConnected(True) self.slave_commands = state.get("slave_commands") self.slave = bot @@ -309,7 +321,7 @@ class AbstractBuildSlave(NewCredPerspective, service.MultiService): def sendBuilderList(self): our_builders = self.botmaster.getBuildersForSlave(self.slavename) - blist = [(b.name, b.builddir) for b in our_builders] + blist = [(b.name, b.slavebuilddir) for b in our_builders] d = self.slave.callRemote("setBuilderList", blist) return d buildbot/buildslave.py @@ -1,4 +1,3 @@ - import sys, os, time from cPickle import dump @@ -9,6 +8,7 @@ from twisted.application import service from twisted.web import html from buildbot import interfaces, util +from buildbot.process.properties import Properties html_tmpl = """ <p>Changed by: <b>%(who)s</b><br /> @@ -22,6 +22,9 @@ Changed files: Comments: %(comments)s + +Properties: +%(properties)s </p> """ @@ -55,7 +58,7 @@ class Change: def __init__(self, who, files, comments, isdir=0, links=None, revision=None, when=None, branch=None, category=None, - revlink=''): + revlink='', properties={}): self.who = who self.comments = comments self.isdir = isdir @@ -69,17 +72,26 @@ class Change: self.branch = branch self.category = category self.revlink = revlink + self.properties = Properties() + self.properties.update(properties, "Change") # keep a sorted list of the files, for easier display self.files = files[:] self.files.sort() + def __setstate__(self, dict): + self.__dict__ = dict + # Older Changes won't have a 'properties' attribute in them + if not hasattr(self, 'properties'): + self.properties = Properties() + def asText(self): data = "" data += self.getFileContents() data += "At: %s\n" % self.getTime() data += "Changed By: %s\n" % self.who - data += "Comments: %s\n\n" % self.comments + data += "Comments: %s" % self.comments + data += "Properties: \n%s\n\n" % self.getProperties() return data def asHTML(self): @@ -91,24 +103,30 @@ class Change: links.append('<a href="%s"><b>%s</b></a>' % (link[0], file)) else: links.append('<b>%s</b>' % file) - revlink = "" if self.revision: if getattr(self, 'revlink', ""): revision = 'Revision: <a href="%s"><b>%s</b></a>\n' % ( self.revlink, self.revision) else: revision = "Revision: <b>%s</b><br />\n" % self.revision + else: + revision = '' branch = "" if self.branch: branch = "Branch: <b>%s</b><br />\n" % self.branch - kwargs = { 'who' : html.escape(self.who), - 'at' : self.getTime(), - 'files' : html.UL(links) + '\n', - 'revision': revision, - 'branch' : branch, - 'comments': html.PRE(self.comments) } + properties = [] + for prop in self.properties.asList(): + properties.append("%s: %s<br />" % (prop[0], prop[1])) + + kwargs = { 'who' : html.escape(self.who), + 'at' : self.getTime(), + 'files' : html.UL(links) + '\n', + 'revision' : revision, + 'branch' : branch, + 'comments' : html.PRE(self.comments), + 'properties': html.UL(properties) + '\n' } return html_tmpl % kwargs def get_HTML_box(self, url): @@ -161,6 +179,12 @@ class Change: data += " %s\n" % f return data + def getProperties(self): + data = "" + for prop in self.properties.asList(): + data += " %s: %s" % (prop[0], prop[1]) + return data + class ChangeMaster(service.MultiService): """This is the master-side service which receives file change buildbot/changes/changes.py @@ -34,8 +34,10 @@ class ChangePerspective(NewCredPerspective): changedict['comments'], branch=changedict.get('branch'), revision=changedict.get('revision'), + revlink=changedict.get('revlink', ''), category=changedict.get('category'), when=changedict.get('when'), + properties=changedict.get('properties', {}) ) self.changemaster.addChange(change) buildbot/changes/pb.py @@ -48,7 +48,7 @@ class SVNPoller(base.ChangeSource, util.ComparableMixin): compare_attrs = ["svnurl", "split_file_function", "svnuser", "svnpasswd", "pollinterval", "histmax", - "svnbin"] + "svnbin", "category"] parent = None # filled in when we're added last_change = None @@ -58,7 +58,7 @@ class SVNPoller(base.ChangeSource, util.ComparableMixin): def __init__(self, svnurl, split_file=None, svnuser=None, svnpasswd=None, pollinterval=10*60, histmax=100, - svnbin='svn', revlinktmpl=''): + svnbin='svn', revlinktmpl='', category=None): """ @type svnurl: string @param svnurl: the SVN URL that describes the repository and @@ -171,6 +171,11 @@ class SVNPoller(base.ChangeSource, util.ComparableMixin): "http://reposerver/websvn/revision.php?rev=%s" would create suitable links on the build pages to information in websvn on each revision. + + @type category: string + @param category: A single category associated with the changes that + could be used by schedulers watch for branches of a + certain name AND category. """ if svnurl.endswith("/"): @@ -188,6 +193,7 @@ class SVNPoller(base.ChangeSource, util.ComparableMixin): self._prefix = None self.overrun_counter = 0 self.loop = LoopingCall(self.checksvn) + self.category = category def split_file(self, path): # use getattr() to avoid turning this function into a bound method, @@ -457,7 +463,8 @@ class SVNPoller(base.ChangeSource, util.ComparableMixin): comments=comments, revision=revision, branch=branch, - revlink=revlink) + revlink=revlink, + category=self.category) changes.append(c) return changes buildbot/changes/svnpoller.py @@ -10,12 +10,13 @@ class Sender: self.port = int(self.port) self.num_changes = 0 - def send(self, branch, revision, comments, files, user=None, category=None, when=None): + def send(self, branch, revision, comments, files, user=None, category=None, + when=None, properties={}): if user is None: user = self.user change = {'who': user, 'files': files, 'comments': comments, 'branch': branch, 'revision': revision, 'category': category, - 'when': when} + 'when': when, 'properties': properties} self.num_changes += 1 f = pb.PBClientFactory() buildbot/clients/sendchange.py @@ -6,6 +6,7 @@ try: import signal except ImportError: pass +import string from cPickle import load import warnings @@ -320,18 +321,12 @@ class DebugPerspective(NewCredPerspective): def perspective_print(self, msg): print "debug", msg -class Dispatcher(styles.Versioned): +class Dispatcher: implements(portal.IRealm) - persistenceVersion = 2 def __init__(self): self.names = {} - def upgradeToVersion1(self): - self.master = self.botmaster.parent - def upgradeToVersion2(self): - self.names = {} - def register(self, name, afactory): self.names[name] = afactory def unregister(self, name): @@ -377,9 +372,8 @@ class Dispatcher(styles.Versioned): # UNIXServer(ResourcePublisher(self.site)) -class BuildMaster(service.MultiService, styles.Versioned): +class BuildMaster(service.MultiService): debug = 0 - persistenceVersion = 3 manhole = None debugPassword = None projectName = "(unspecified)" @@ -426,23 +420,6 @@ class BuildMaster(service.MultiService, styles.Versioned): self.readConfig = False - def upgradeToVersion1(self): - self.dispatcher = self.slaveFactory.root.portal.realm - - def upgradeToVersion2(self): # post-0.4.3 - self.webServer = self.webTCPPort - del self.webTCPPort - self.webDistribServer = self.webUNIXPort - del self.webUNIXPort - self.configFileName = "master.cfg" - - def upgradeToVersion3(self): - # post 0.6.3, solely to deal with the 0.6.3 breakage. Starting with - # 0.6.5 I intend to do away with .tap files altogether - self.services = [] - self.namedServices = {} - del self.change_svc - def startService(self): service.MultiService.startService(self) self.loadChanges() # must be done before loading the config file @@ -545,7 +522,7 @@ class BuildMaster(service.MultiService, styles.Versioned): "manhole", "status", "projectName", "projectURL", "buildbotURL", "properties", "prioritizeBuilders", "eventHorizon", "buildCacheSize", "logHorizon", "buildHorizon", - "changeHorizon", + "changeHorizon", "logMaxSize", "logMaxTailSize", ) for k in config.keys(): if k not in known_keys: @@ -578,10 +555,18 @@ class BuildMaster(service.MultiService, styles.Versioned): eventHorizon = config.get('eventHorizon', None) logHorizon = config.get('logHorizon', None) buildHorizon = config.get('buildHorizon', None) - logCompressionLimit = config.get('logCompressionLimit') + logCompressionLimit = config.get('logCompressionLimit', 4*1024) if logCompressionLimit is not None and not \ isinstance(logCompressionLimit, int): raise ValueError("logCompressionLimit needs to be bool or int") + logMaxSize = config.get('logMaxSize') + if logMaxSize is not None and not \ + isinstance(logMaxSize, int): + raise ValueError("logMaxSize needs to be None or int") + logMaxTailSize = config.get('logMaxTailSize') + if logMaxTailSize is not None and not \ + isinstance(logMaxTailSize, int): + raise ValueError("logMaxTailSize needs to be None or int") mergeRequests = config.get('mergeRequests') if mergeRequests is not None and not callable(mergeRequests): raise ValueError("mergeRequests must be a callable") @@ -658,6 +643,8 @@ class BuildMaster(service.MultiService, styles.Versioned): slavenames = [s.slavename for s in slaves] buildernames = [] dirnames = [] + badchars_map = string.maketrans("\t !#$%&'()*+,./:;<=>?@[\\]^{|}~", + "______________________________") for b in builders: if type(b) is tuple: raise ValueError("builder %s must be defined with a dict, " @@ -673,6 +660,10 @@ class BuildMaster(service.MultiService, styles.Versioned): raise ValueError("duplicate builder name %s" % b['name']) buildernames.append(b['name']) + + # Fix the dictionnary with default values. + b.setdefault('builddir', b['name'].translate(badchars_map)) + b.setdefault('slavebuilddir', b['builddir']) if b['builddir'] in dirnames: raise ValueError("builder %s reuses builddir %s" % (b['name'], b['builddir'])) @@ -750,8 +741,18 @@ class BuildMaster(service.MultiService, styles.Versioned): self.properties = Properties() self.properties.update(properties, self.configFileName) - if logCompressionLimit is not None: - self.status.logCompressionLimit = logCompressionLimit + + self.status.logCompressionLimit = logCompressionLimit + self.status.logMaxSize = logMaxSize + self.status.logMaxTailSize = logMaxTailSize + # Update any of our existing builders with the current log parameters. + # This is required so that the new value is picked up after a + # reconfig. + for builder in self.botmaster.builders.values(): + builder.builder_status.setLogCompressionLimit(logCompressionLimit) + builder.builder_status.setLogMaxSize(logMaxSize) + builder.builder_status.setLogMaxTailSize(logMaxTailSize) + if mergeRequests is not None: self.botmaster.mergeRequests = mergeRequests if prioritizeBuilders is not None: @@ -898,7 +899,7 @@ class BuildMaster(service.MultiService, styles.Versioned): # everything in newList is either unchanged, changed, or new for name, data in newList.items(): old = self.botmaster.builders.get(name) - basedir = data['builddir'] # used on both master and slave + basedir = data['builddir'] #name, slave, builddir, factory = data if not old: # new # category added after 0.6.2 buildbot/master.py @@ -72,13 +72,6 @@ class ReconnectingPBClientFactory(PBClientFactory, self.doGetPerspective(self._root) self.gotRootObject(self._root) - def __getstate__(self): - # this should get folded into ReconnectingClientFactory - d = self.__dict__.copy() - d['connector'] = None - d['_callID'] = None - return d - # oldcred methods def getPerspective(self, *args): buildbot/pbutil.py @@ -240,12 +240,6 @@ class Build: def __repr__(self): return "<Build %s>" % (self.builder.name,) - def __getstate__(self): - d = self.__dict__.copy() - if d.has_key('remote'): - del d['remote'] - return d - def blamelist(self): blamelist = [] for c in self.allChanges(): @@ -291,6 +285,10 @@ class Build: for rq in self.requests: props.updateFromProperties(rq.properties) + # and finally, from the SourceStamp, which has properties via Change + for change in self.source.changes: + props.updateFromProperties(change.properties) + # now set some properties of our own, corresponding to the # build itself props.setProperty("buildername", self.builder.name, "Build") buildbot/process/base.py @@ -116,6 +116,9 @@ class AbstractSlaveBuilder(pb.Referenceable): log.err(why) return why + def prepare(self, builder_status): + return defer.succeed(None) + def ping(self, timeout, status=None): """Ping the slave to make sure it is still there. Returns a Deferred that fires with True if it is. @@ -260,7 +263,19 @@ class LatentSlaveBuilder(AbstractSlaveBuilder): log.msg("Latent buildslave %s attached to %s" % (slave.slavename, self.builder_name)) - def substantiate(self, build): + def prepare(self, builder_status): + log.msg("substantiating slave %s" % (self,)) + d = self.substantiate() + def substantiation_failed(f): + builder_status.addPointEvent(['removing', 'latent', + self.slave.slavename]) + self.slave.disconnect() + # TODO: should failover to a new Build + return f + d.addErrback(substantiation_failed) + return d + + def substantiate(self): self.state = SUBSTANTIATING d = self.slave.substantiate(self) if not self.slave.substantiated: @@ -335,11 +350,6 @@ class Builder(pb.Referenceable): I also manage forced builds, progress expectation (ETA) management, and some status delivery chores. - I am persisted in C{BASEDIR/BUILDERNAME/builder}, so I can remember how - long a build usually takes to run (in my C{expectations} attribute). This - pickle also includes the L{buildbot.status.builder.BuilderStatus} object, - which remembers the set of historic builds. - @type buildable: list of L{buildbot.process.base.BuildRequest} @ivar buildable: BuildRequests that are ready to build, but which are waiting for a buildslave to be available. @@ -360,7 +370,7 @@ class Builder(pb.Referenceable): @type setup: dict @param setup: builder setup data, as stored in BuildmasterConfig['builders']. Contains name, - slavename(s), builddir, factory, locks. + slavename(s), builddir, slavebuilddir, factory, locks. @type builder_status: L{buildbot.status.builder.BuilderStatus} """ self.name = setup['name'] @@ -370,6 +380,7 @@ class Builder(pb.Referenceable): if setup.has_key('slavenames'): self.slavenames.extend(setup['slavenames']) self.builddir = setup['builddir'] + self.slavebuilddir = setup['slavebuilddir'] self.buildFactory = setup['factory'] self.nextSlave = setup.get('nextSlave') if self.nextSlave is not None and not callable(self.nextSlave): @@ -422,6 +433,9 @@ class Builder(pb.Referenceable): if setup['builddir'] != self.builddir: diffs.append('builddir changed from %s to %s' \ % (self.builddir, setup['builddir'])) + if setup['slavebuilddir'] != self.slavebuilddir: + diffs.append('slavebuilddir changed from %s to %s' \ + % (self.slavebuilddir, setup['slavebuilddir'])) if setup['factory'] != self.buildFactory: # compare objects diffs.append('factory changed') oldlocks = [(lock.__class__, lock.name) @@ -462,18 +476,6 @@ class Builder(pb.Referenceable): return True return False - def __getstate__(self): - d = self.__dict__.copy() - # TODO: note that d['buildable'] can contain Deferreds - del d['building'] # TODO: move these back to .buildable? - del d['slaves'] - return d - - def __setstate__(self, d): - self.__dict__ = d - self.building = [] - self.slaves = [] - def consumeTheSoulOfYourPredecessor(self, old): """Suck the brain out of an old Builder. @@ -758,27 +760,17 @@ class Builder(pb.Referenceable): self.building.append(build) self.updateBigStatus() - if isinstance(sb, LatentSlaveBuilder): - log.msg("starting build %s.. substantiating the slave %s" % - (build, sb)) - d = sb.substantiate(build) - def substantiated(res): - return sb.ping(self.START_BUILD_TIMEOUT) - def substantiation_failed(res): - self.builder_status.addPointEvent( - ['removing', 'latent', sb.slave.slavename]) - sb.slave.disconnect() - # TODO: should failover to a new Build - #self.retryBuild(sb.build) - d.addCallbacks(substantiated, substantiation_failed) - else: + log.msg("starting build %s using slave %s" % (build, sb)) + d = sb.prepare(self.builder_status) + def _ping(ign): + # ping the slave to make sure they're still there. If they're + # fallen off the map (due to a NAT timeout or something), this + # will fail in a couple of minutes, depending upon the TCP + # timeout. TODO: consider making this time out faster, or at + # least characterize the likely duration. log.msg("starting build %s.. pinging the slave %s" % (build, sb)) - d = sb.ping(self.START_BUILD_TIMEOUT) - # ping the slave to make sure they're still there. If they're fallen - # off the map (due to a NAT timeout or something), this will fail in - # a couple of minutes, depending upon the TCP timeout. TODO: consider - # making this time out faster, or at least characterize the likely - # duration. + return sb.ping(self.START_BUILD_TIMEOUT) + d.addCallback(_ping) d.addCallback(self._startBuild_1, build, sb) return d @@ -882,7 +874,8 @@ class BuilderControl(components.Adapter): return ss = bs.getSourceStamp(absolute=True) - req = base.BuildRequest(reason, ss, self.original.name) + req = base.BuildRequest(reason, ss, self.original.name, + properties=bs.getProperties()) self.requestBuild(req) def getPendingBuilds(self): @@ -930,5 +923,3 @@ class BuildRequestControl: def cancel(self): self.original_builder.cancelBuildRequest(self.original_request) - - buildbot/process/builder.py @@ -62,14 +62,6 @@ class RemoteCommand(pb.Referenceable): self.remote_command = remote_command self.args = args - def __getstate__(self): - dict = self.__dict__.copy() - # Remove the remote ref: if necessary (only for resumed builds), it - # will be reattached at resume time - if dict.has_key("remote"): - del dict["remote"] - return dict - def run(self, step, remote): self.active = True self.step = step @@ -103,12 +95,20 @@ class RemoteCommand(pb.Referenceable): @returns: a deferred that will fire when the remote command is done (with None as the result) """ + + # Allow use of WithProperties in logfile path names. + cmd_args = self.args + if cmd_args.has_key("logfiles") and cmd_args["logfiles"]: + cmd_args = cmd_args.copy() + properties = self.step.build.getProperties() + cmd_args["logfiles"] = properties.render(cmd_args["logfiles"]) + # This method only initiates the remote command. # We will receive remote_update messages as the command runs. # We will get a single remote_complete when it finishes. # We should fire self.deferred when the command is done. d = self.remote.callRemote("startCommand", self, self.commandID, - self.remote_command, self.args) + self.remote_command, cmd_args) return d def interrupt(self, why): @@ -256,6 +256,7 @@ class LoggedRemoteCommand(RemoteCommand): def __init__(self, *args, **kwargs): self.logs = {} + self.delayedLogs = {} self._closeWhenFinished = {} RemoteCommand.__init__(self, *args, **kwargs) @@ -290,9 +291,15 @@ class LoggedRemoteCommand(RemoteCommand): if not logfileName: logfileName = loog.getName() assert logfileName not in self.logs + assert logfileName not in self.delayedLogs self.logs[logfileName] = loog self._closeWhenFinished[logfileName] = closeWhenFinished + def useLogDelayed(self, logfileName, activateCallBack, closeWhenFinished=False): + assert logfileName not in self.logs + assert logfileName not in self.delayedLogs + self.delayedLogs[logfileName] = (activateCallBack, closeWhenFinished) + def start(self): log.msg("LoggedRemoteCommand.start") if 'stdio' not in self.logs: @@ -313,6 +320,14 @@ class LoggedRemoteCommand(RemoteCommand): self.logs['stdio'].addHeader(data) def addToLog(self, logname, data): + # Activate delayed logs on first data. + if logname in self.delayedLogs: + (activateCallBack, closeWhenFinished) = self.delayedLogs[logname] + del self.delayedLogs[logname] + loog = activateCallBack(self) + self.logs[logname] = loog + self._closeWhenFinished[logname] = closeWhenFinished + if logname in self.logs: self.logs[logname].addStdout(data) else: @@ -431,10 +446,10 @@ class RemoteShellCommand(LoggedRemoteCommand): command is finished, it will fire a Deferred. You can then check the results of the command and parse the output however you like.""" - def __init__(self, workdir, command, env=None, + def __init__(self, workdir, command, env=None, want_stdout=1, want_stderr=1, - timeout=20*60, logfiles={}, usePTY="slave-config", - logEnviron=True): + timeout=20*60, maxTime=None, logfiles={}, + usePTY="slave-config", logEnviron=True): """ @type workdir: string @param workdir: directory where the command ought to run, @@ -475,6 +490,11 @@ class RemoteShellCommand(LoggedRemoteCommand): None to disable the timeout. @param logEnviron: whether to log env vars on the slave side + + @type maxTime: int + @param maxTime: tell the remote that if the command fails to complete + in this number of seconds, the command should be + killed. Use None to disable maxTime. """ self.command = command # stash .command, set it later @@ -489,6 +509,7 @@ class RemoteShellCommand(LoggedRemoteCommand): 'want_stderr': want_stderr, 'logfiles': logfiles, 'timeout': timeout, + 'maxTime': maxTime, 'usePTY': usePTY, 'logEnviron': logEnviron, } @@ -958,20 +979,29 @@ class LoggingBuildStep(BuildStep): progressMetrics = ('output',) logfiles = {} - parms = BuildStep.parms + ['logfiles'] + parms = BuildStep.parms + ['logfiles', 'lazylogfiles'] - def __init__(self, logfiles={}, *args, **kwargs): + def __init__(self, logfiles={}, lazylogfiles=False, *args, **kwargs): BuildStep.__init__(self, *args, **kwargs) - self.addFactoryArguments(logfiles=logfiles) + self.addFactoryArguments(logfiles=logfiles, + lazylogfiles=lazylogfiles) # merge a class-level 'logfiles' attribute with one passed in as an # argument self.logfiles = self.logfiles.copy() self.logfiles.update(logfiles) + self.lazylogfiles = lazylogfiles self.addLogObserver('stdio', OutputProgressObserver("output")) def describe(self, done=False): raise NotImplementedError("implement this in a subclass") + def addLogFile(self, logname, filename): + """ + This allows to add logfiles after construction, but before calling + startCommand(). + """ + self.logfiles[logname] = filename + def startCommand(self, cmd, errorMessages=[]): """ @param cmd: a suitable RemoteCommand which will be launched, with @@ -1009,10 +1039,20 @@ class LoggingBuildStep(BuildStep): """Set up any additional logfiles= logs. """ for logname,remotefilename in logfiles.items(): - # tell the BuildStepStatus to add a LogFile - newlog = self.addLog(logname) - # and tell the LoggedRemoteCommand to feed it - cmd.useLog(newlog, True) + if self.lazylogfiles: + # Ask LoggedRemoteCommand to watch a logfile, but only add + # it when/if we see any data. + # + # The dummy default argument local_logname is a work-around for + # Python name binding; default values are bound by value, but + # captured variables in the body are bound by name. + callback = lambda cmd_arg, local_logname=logname: self.addLog(local_logname) + cmd.useLogDelayed(logname, callback, True) + else: + # tell the BuildStepStatus to add a LogFile + newlog = self.addLog(logname) + # and tell the LoggedRemoteCommand to feed it + cmd.useLog(newlog, True) def interrupt(self, reason): # TODO: consider adding an INTERRUPTED or STOPPED status to use @@ -1114,7 +1154,7 @@ class LoggingBuildStep(BuildStep): self.step_status.setText(self.getText(cmd, results)) self.step_status.setText2(self.maybeGetText2(cmd, results)) -# (WithProeprties used to be available in this module) +# (WithProperties used to be available in this module) from buildbot.process.properties import WithProperties _hush_pyflakes = [WithProperties] del _hush_pyflakes buildbot/process/buildstep.py @@ -12,6 +12,19 @@ def s(steptype, **kwargs): # specification tuples return (steptype, kwargs) +class ArgumentsInTheWrongPlace(Exception): + """When calling BuildFactory.addStep(stepinstance), addStep() only takes + one argument. You passed extra arguments to addStep(), which you probably + intended to pass to your BuildStep constructor instead. For example, you + should do:: + + f.addStep(ShellCommand(command=['echo','stuff'], haltOnFailure=True)) + + instead of:: + + f.addStep(ShellCommand(command=['echo','stuff']), haltOnFailure=True) + """ + class BuildFactory(util.ComparableMixin): """ @cvar buildClass: class to use when creating builds @@ -42,6 +55,8 @@ class BuildFactory(util.ComparableMixin): def addStep(self, step_or_factory, **kwargs): if isinstance(step_or_factory, BuildStep): + if kwargs: + raise ArgumentsInTheWrongPlace() s = step_or_factory.getStepFactory() else: s = (step_or_factory, dict(kwargs)) buildbot/process/factory.py @@ -77,13 +77,6 @@ class RunUnitTestsJelly(RunUnitTests): ResultTypes.SUCCESS: tests.SUCCESS, } - def __getstate__(self): - #d = RunUnitTests.__getstate__(self) - d = self.__dict__.copy() - # Banana subclasses are Ephemeral - if d.has_key("decoder"): - del d['decoder'] - return d def start(self): self.decoder = remote.DecodeReport(self) # don't accept anything unpleasant from the (untrusted) build slave buildbot/process/step_twisted2.py @@ -134,7 +134,7 @@ class Scheduler(BaseUpstreamScheduler): self.fileIsImportant = fileIsImportant self.importantChanges = [] - self.unimportantChanges = [] + self.allChanges = [] self.nextBuildTime = None self.timer = None self.categories = categories @@ -164,13 +164,14 @@ class Scheduler(BaseUpstreamScheduler): def addImportantChange(self, change): log.msg("%s: change is important, adding %s" % (self, change)) self.importantChanges.append(change) + self.allChanges.append(change) self.nextBuildTime = max(self.nextBuildTime, change.when + self.treeStableTimer) self.setTimer(self.nextBuildTime) def addUnimportantChange(self, change): log.msg("%s: change is not important, adding %s" % (self, change)) - self.unimportantChanges.append(change) + self.allChanges.append(change) def setTimer(self, when): log.msg("%s: setting timer to %s" % @@ -191,9 +192,9 @@ class Scheduler(BaseUpstreamScheduler): # clear out our state self.timer = None self.nextBuildTime = None - changes = self.importantChanges + self.unimportantChanges + changes = self.allChanges self.importantChanges = [] - self.unimportantChanges = [] + self.allChanges = [] # create a BuildSet, submit it to the BuildMaster bs = buildset.BuildSet(self.builderNames, @@ -215,10 +216,10 @@ class AnyBranchScheduler(BaseUpstreamScheduler): fileIsImportant = None compare_attrs = ('name', 'branches', 'treeStableTimer', 'builderNames', - 'fileIsImportant', 'properties') + 'fileIsImportant', 'properties', 'categories') def __init__(self, name, branches, treeStableTimer, builderNames, - fileIsImportant=None, properties={}): + fileIsImportant=None, properties={}, categories=None): """ @param name: the name of this Scheduler @param branches: The branch names that the Scheduler should pay @@ -246,6 +247,7 @@ class AnyBranchScheduler(BaseUpstreamScheduler): @param properties: properties to apply to all builds started from this scheduler + @param categories: A list of categories of changes to accept """ BaseUpstreamScheduler.__init__(self, name, properties) @@ -266,6 +268,7 @@ class AnyBranchScheduler(BaseUpstreamScheduler): assert callable(fileIsImportant) self.fileIsImportant = fileIsImportant self.schedulers = {} # one per branch + self.categories = categories def __repr__(self): return "<AnyBranchScheduler '%s'>" % self.name @@ -290,6 +293,9 @@ class AnyBranchScheduler(BaseUpstreamScheduler): if self.branches is not None and branch not in self.branches: log.msg("%s ignoring off-branch %s" % (self, change)) return + if self.categories is not None and change.category not in self.categories: + log.msg("%s ignoring non-matching categories %s" % (self, change)) + return s = self.schedulers.get(branch) if not s: if branch: @@ -487,7 +493,7 @@ class Nightly(BaseUpstreamScheduler): % name) self.importantChanges = [] - self.unimportantChanges = [] + self.allChanges = [] self.fileIsImportant = None if fileIsImportant: assert callable(fileIsImportant) @@ -584,7 +590,7 @@ class Nightly(BaseUpstreamScheduler): if self.onlyIfChanged: if len(self.importantChanges) > 0: - changes = self.importantChanges + self.unimportantChanges + changes = self.allChanges # And trigger a build log.msg("Nightly Scheduler <%s>: triggering build" % self.name) bs = buildset.BuildSet(self.builderNames, @@ -594,7 +600,7 @@ class Nightly(BaseUpstreamScheduler): self.submitBuildSet(bs) # Reset the change lists self.importantChanges = [] - self.unimportantChanges = [] + self.allChanges = [] else: log.msg("Nightly Scheduler <%s>: skipping build - No important change" % self.name) else: @@ -622,11 +628,12 @@ class Nightly(BaseUpstreamScheduler): def addImportantChange(self, change): log.msg("Nightly Scheduler <%s>: change %s from %s is important, adding it" % (self.name, change.revision, change.who)) + self.allChanges.append(change) self.importantChanges.append(change) def addUnimportantChange(self, change): log.msg("Nightly Scheduler <%s>: change %s from %s is not important, adding it" % (self.name, change.revision, change.who)) - self.unimportantChanges.append(change) + self.allChanges.append(change) class TryBase(BaseScheduler): buildbot/scheduler.py @@ -719,6 +719,10 @@ def statusgui(config): c.run() class SendChangeOptions(usage.Options): + def __init__(self): + usage.Options.__init__(self) + self['properties'] = {} + optParameters = [ ("master", "m", None, "Location of the buildmaster's PBListener (host:port)"), @@ -728,6 +732,8 @@ class SendChangeOptions(usage.Options): ("revision", "r", None, "Revision specifier (string)"), ("revision_number", "n", None, "Revision specifier (integer)"), ("revision_file", None, None, "Filename containing revision spec"), + ("property", "p", None, + "A property for the change, in the format: name:value"), ("comments", "m", None, "log message"), ("logfile", "F", None, "Read the log messages from this file (- for stdin)"), @@ -737,6 +743,9 @@ class SendChangeOptions(usage.Options): return "Usage: buildbot sendchange [options] filenames.." def parseArgs(self, *args): self['files'] = args + def opt_property(self, property): + name,value = property.split(':') + self['properties'][name] = value def sendchange(config, runReactor=False): @@ -750,6 +759,7 @@ def sendchange(config, runReactor=False): branch = config.get('branch', opts.get('branch')) category = config.get('category', opts.get('category')) revision = config.get('revision') + properties = config.get('properties', {}) if config.get('when'): when = float(config.get('when')) else: @@ -776,7 +786,8 @@ def sendchange(config, runReactor=False): assert master, "you must provide the master location" s = Sender(master, user) - d = s.send(branch, revision, comments, files, category=category, when=when) + d = s.send(branch, revision, comments, files, category=category, when=when, + properties=properties) if runReactor: d.addCallbacks(s.printSuccess, s.printFailure) d.addBoth(s.stop) @@ -837,6 +848,12 @@ class TryOptions(usage.Options): "Run the trial build on this Builder. Can be used multiple times."], ["properties", None, None, "A set of properties made available in the build environment, format:prop=value,propb=valueb..."], + + ["try-topfile", None, None, + "Name of a file at the top of the tree, used to find the top. Only needed for SVN and CVS."], + ["try-topdir", None, None, + "Path to the top of the working copy. Only needed for SVN and CVS."], + ] optFlags = [ buildbot/scripts/runner.py @@ -156,21 +156,17 @@ class BzrExtractor(SourceStampExtractor): patchlevel = 0 vcexe = "bzr" def getBaseRevision(self): - d = self.dovc(["version-info"]) + d = self.dovc(["revision-info","-rsubmit:"]) d.addCallback(self.get_revision_number) return d + def get_revision_number(self, out): - for line in out.split("\n"): - colon = line.find(":") - if colon != -1: - key, value = line[:colon], line[colon+2:] - if key == "revno": - self.baserev = int(value) - return - raise ValueError("unable to find revno: in bzr output: '%s'" % out) + revno, revid= out.split() + self.baserev = 'revid:' + revid + return def getPatch(self, res): - d = self.dovc(["diff"]) + d = self.dovc(["diff","-r%s.." % self.baserev]) d.addCallback(self.readPatch, self.patchlevel) return d @@ -419,7 +415,7 @@ class Try(pb.Referenceable): vc = self.getopt("vc", "try_vc") if vc in ("cvs", "svn"): # we need to find the tree-top - topdir = self.getopt("try_topdir", "try_topdir") + topdir = self.getopt("try-topdir", "try_topdir") if topdir: treedir = os.path.expanduser(topdir) else: buildbot/scripts/tryclient.py @@ -339,6 +339,12 @@ class Bot(pb.Referenceable, service.MultiService): files[f] = open(filename, "r").read() return files + def remote_getVersion(self): + """Send our version back to the Master""" + return buildbot.version + + + class BotFactory(ReconnectingPBClientFactory): # 'keepaliveInterval' serves two purposes. The first is to keep the # connection alive: it guarantees that there will be at least some buildbot/slave/bot.py @@ -285,9 +285,9 @@ class ShellCommand: def __init__(self, builder, command, workdir, environ=None, sendStdout=True, sendStderr=True, sendRC=True, - timeout=None, initialStdin=None, keepStdinOpen=False, - keepStdout=False, keepStderr=False, logEnviron=True, - logfiles={}, usePTY="slave-config"): + timeout=None, maxTime=None, initialStdin=None, + keepStdinOpen=False, keepStdout=False, keepStderr=False, + logEnviron=True, logfiles={}, usePTY="slave-config"): """ @param keepStdout: if True, we keep a copy of all the stdout text @@ -336,6 +336,8 @@ class ShellCommand: self.logEnviron = logEnviron self.timeout = timeout self.timer = None + self.maxTime = maxTime + self.maxTimer = None self.keepStdout = keepStdout self.keepStderr = keepStderr @@ -518,6 +520,9 @@ class ShellCommand: if self.timeout: self.timer = reactor.callLater(self.timeout, self.doTimeout) + if self.maxTime: + self.maxTimer = reactor.callLater(self.maxTime, self.doMaxTimeout) + for w in self.logFileWatchers: w.start() @@ -570,6 +575,9 @@ class ShellCommand: if self.timer: self.timer.cancel() self.timer = None + if self.maxTimer: + self.maxTimer.cancel() + self.maxTimer = None d = self.deferred self.deferred = None if d: @@ -582,6 +590,9 @@ class ShellCommand: if self.timer: self.timer.cancel() self.timer = None + if self.maxTimer: + self.maxTimer.cancel() + self.maxTimer = None d = self.deferred self.deferred = None if d: @@ -594,12 +605,20 @@ class ShellCommand: msg = "command timed out: %d seconds without output" % self.timeout self.kill(msg) + def doMaxTimeout(self): + self.maxTimer = None + msg = "command timed out: %d seconds elapsed" % self.maxTime + self.kill(msg) + def kill(self, msg): # This may be called by the timeout, or when the user has decided to # abort this build. if self.timer: self.timer.cancel() self.timer = None + if self.maxTimer: + self.maxTimer.cancel() + self.maxTimer = None if hasattr(self.process, "pid") and self.process.pid is not None: msg += ", killing pid %s" % self.process.pid log.msg(msg) @@ -1192,6 +1211,7 @@ class SlaveShellCommand(Command): configuration of the slave) - ['not_really']: 1 to skip execution and return rc=0 - ['timeout']: seconds of silence to tolerate before killing command + - ['maxTime']: seconds before killing command - ['logfiles']: dict mapping LogFile name to the workdir-relative filename of a local log file. This local file will be watched just like 'tail -f', and all changes will be @@ -1215,6 +1235,7 @@ class SlaveShellCommand(Command): c = ShellCommand(self.builder, args['command'], workdir, environ=args.get('env'), timeout=args.get('timeout', None), + maxTime=args.get('maxTime', None), sendStdout=args.get('want_stdout', True), sendStderr=args.get('want_stderr', True), sendRC=True, @@ -1357,6 +1378,8 @@ class SourceBase(Command): - ['timeout']: seconds of silence tolerated before we kill off the command + - ['maxTime']: seconds before we kill off the command + - ['retry']: If not None, this is a tuple of (delay, repeats) which means that any failed VC updates should be reattempted, up to REPEATS times, after a delay of @@ -1378,6 +1401,7 @@ class SourceBase(Command): self.revision = args.get('revision') self.patch = args.get('patch') self.timeout = args.get('timeout', 120) + self.maxTime = args.get('maxTime', None) self.retry = args.get('retry') # VC-specific subclasses should override this to extract more args. # Make sure to upcall! @@ -1564,7 +1588,8 @@ class SourceBase(Command): return defer.succeed(0) command = ["rm", "-rf", d] c = ShellCommand(self.builder, command, self.builder.basedir, - sendRC=0, timeout=self.timeout, usePTY=False) + sendRC=0, timeout=self.timeout, maxTime=self.maxTime, + usePTY=False) self.command = c # sendRC=0 means the rm command will send stdout/stderr to the @@ -1594,22 +1619,41 @@ class SourceBase(Command): command = ['cp', '-R', '-P', '-p', fromdir, todir] c = ShellCommand(self.builder, command, self.builder.basedir, - sendRC=False, timeout=self.timeout, usePTY=False) + sendRC=False, timeout=self.timeout, maxTime=self.maxTime, + usePTY=False) self.command = c d = c.start() d.addCallback(self._abandonOnFailure) return d def doPatch(self, res): - patchlevel, diff = self.patch - command = [getCommand("patch"), '-p%d' % patchlevel] + patchlevel = self.patch[0] + diff = self.patch[1] + root = None + if len(self.patch) >= 3: + root = self.patch[2] + command = [ + getCommand("patch"), + '-p%d' % patchlevel, + '--remove-empty-files', + '--force', + '--forward', + ] dir = os.path.join(self.builder.basedir, self.workdir) # mark the directory so we don't try to update it later open(os.path.join(dir, ".buildbot-patched"), "w").write("patched\n") + + # Update 'dir' with the 'root' option. Make sure it is a subdirectory + # of dir. + if (root and + os.path.abspath(os.path.join(dir, root) + ).startswith(os.path.abspath(dir))): + dir = os.path.join(dir, root) + # now apply the patch c = ShellCommand(self.builder, command, dir, sendRC=False, timeout=self.timeout, - initialStdin=diff, usePTY=False) + maxTime=self.maxTime, initialStdin=diff, usePTY=False) self.command = c d = c.start() d.addCallback(self._abandonOnFailure) @@ -1658,6 +1702,7 @@ class CVS(SourceBase): + ['login']) c = ShellCommand(self.builder, command, d, sendRC=False, timeout=self.timeout, + maxTime=self.maxTime, initialStdin=self.login+"\n", usePTY=False) self.command = c d = c.start() @@ -1679,7 +1724,8 @@ class CVS(SourceBase): if self.revision: command += ['-D', self.revision] c = ShellCommand(self.builder, command, d, - sendRC=False, timeout=self.timeout, usePTY=False) + sendRC=False, timeout=self.timeout, + maxTime=self.maxTime, usePTY=False) self.command = c return c.start() @@ -1702,7 +1748,8 @@ class CVS(SourceBase): command += [self.cvsmodule] c = ShellCommand(self.builder, command, d, - sendRC=False, timeout=self.timeout, usePTY=False) + sendRC=False, timeout=self.timeout, + maxTime=self.maxTime, usePTY=False) self.command = c return c.start() @@ -1757,7 +1804,7 @@ class SVN(SourceBase): '--non-interactive', '--no-auth-cache'] c = ShellCommand(self.builder, command, d, sendRC=False, timeout=self.timeout, - keepStdout=True, usePTY=False) + maxTime=self.maxTime, keepStdout=True, usePTY=False) self.command = c return c.start() @@ -1779,7 +1826,7 @@ class SVN(SourceBase): self.svnurl, self.srcdir] c = ShellCommand(self.builder, command, d, sendRC=False, timeout=self.timeout, - keepStdout=True, usePTY=False) + maxTime=self.maxTime, keepStdout=True, usePTY=False) self.command = c return c.start() @@ -1858,7 +1905,8 @@ class Darcs(SourceBase): d = os.path.join(self.builder.basedir, self.srcdir) command = [self.vcexe, 'pull', '--all', '--verbose'] c = ShellCommand(self.builder, command, d, - sendRC=False, timeout=self.timeout, usePTY=False) + sendRC=False, timeout=self.timeout, + maxTime=self.maxTime, usePTY=False) self.command = c return c.start() @@ -1879,7 +1927,8 @@ class Darcs(SourceBase): command.append(self.repourl) c = ShellCommand(self.builder, command, d, - sendRC=False, timeout=self.timeout, usePTY=False) + sendRC=False, timeout=self.timeout, + maxTime=self.maxTime, usePTY=False) self.command = c d = c.start() if self.revision: @@ -1951,7 +2000,8 @@ class Monotone(SourceBase): "-r", self.revision, "-b", self.branch] c = ShellCommand(self.builder, command, self.full_srcdir, - sendRC=False, timeout=self.timeout, usePTY=False) + sendRC=False, timeout=self.timeout, + maxTime=self.maxTime, usePTY=False) self.command = c return c.start() @@ -1965,7 +2015,8 @@ class Monotone(SourceBase): "-b", self.branch, self.full_srcdir] c = ShellCommand(self.builder, command, self.builder.basedir, - sendRC=False, timeout=self.timeout, usePTY=False) + sendRC=False, timeout=self.timeout, + maxTime=self.maxTime, usePTY=False) self.command = c return c.start() @@ -1986,7 +2037,8 @@ class Monotone(SourceBase): command = [self.monotone, "db", "init", "--db=" + self.full_db_path] c = ShellCommand(self.builder, command, self.builder.basedir, - sendRC=False, timeout=self.timeout, usePTY=False) + sendRC=False, timeout=self.timeout, + maxTime=self.maxTime, usePTY=False) self.command = c d = c.start() d.addCallback(self._abandonOnFailure) @@ -1998,7 +2050,8 @@ class Monotone(SourceBase): command = [self.monotone, "--db=" + self.full_db_path, "pull", "--ticker=dot", self.server_addr, self.branch] c = ShellCommand(self.builder, command, self.builder.basedir, - sendRC=False, timeout=self._pull_timeout, usePTY=False) + sendRC=False, timeout=self._pull_timeout, + maxTime=self.maxTime, usePTY=False) self.sendStatus({"header": "pulling %s from %s\n" % (self.branch, self.server_addr)}) self.command = c @@ -2025,6 +2078,7 @@ class Git(SourceBase): def setup(self, args): SourceBase.setup(self, args) + self.vcexe = getCommand("git") self.repourl = args['repourl'] self.branch = args.get('branch') if not self.branch: @@ -2046,6 +2100,17 @@ class Git(SourceBase): def readSourcedata(self): return open(self.sourcedatafile, "r").read() + def _dovccmd(self, command, cb=None, **kwargs): + c = ShellCommand(self.builder, [self.vcexe] + command, self._fullSrcdir(), + sendRC=False, timeout=self.timeout, + maxTime=self.maxTime, usePTY=False, **kwargs) + self.command = c + d = c.start() + if cb: + d.addCallback(self._abandonOnFailure) + d.addCallback(cb) + return d + # If the repourl matches the sourcedata file, then # we can say that the sourcedata matches. We can # ignore branch changes, since Git can work with @@ -2060,12 +2125,17 @@ class Git(SourceBase): return False return True - def _didSubmodules(self, res): - command = ['git', 'submodule', 'update', '--init'] - c = ShellCommand(self.builder, command, self._fullSrcdir(), - sendRC=False, timeout=self.timeout, usePTY=False) - self.command = c - return c.start() + def _cleanSubmodules(self, res): + return self._dovccmd(['submodule', 'foreach', 'git', 'clean', '-dfx']) + + def _updateSubmodules(self, res): + return self._dovccmd(['submodule', 'update'], self._cleanSubmodules) + + def _initSubmodules(self, res): + if self.submodules: + return self._dovccmd(['submodule', 'init'], self._updateSubmodules) + else: + return defer.succeed(0) def _didFetch(self, res): if self.revision: @@ -2073,66 +2143,51 @@ class Git(SourceBase): else: head = 'FETCH_HEAD' - command = ['git', 'reset', '--hard', head] - c = ShellCommand(self.builder, command, self._fullSrcdir(), - sendRC=False, timeout=self.timeout, usePTY=False) - self.command = c - d = c.start() - if self.submodules: - d.addCallback(self._abandonOnFailure) - d.addCallback(self._didSubmodules) - return d + command = ['reset', '--hard', head] + return self._dovccmd(command, self._initSubmodules) # Update first runs "git clean", removing local changes, This, # combined with the later "git reset" equates clobbering the repo, # but it's much more efficient. def doVCUpdate(self): - command = ['git', 'clean', '-f', '-d', '-x'] - c = ShellCommand(self.builder, command, self._fullSrcdir(), - sendRC=False, timeout=self.timeout, usePTY=False) - self.command = c - d = c.start() - d.addCallback(self._abandonOnFailure) - d.addCallback(self._didClean) - return d + command = ['clean', '-f', '-d', '-x'] + return self._dovccmd(command, self._didClean) - def _didClean(self, dummy): - command = ['git', 'fetch', '-t', self.repourl, self.branch] + def _doFetch(self, dummy): + command = ['fetch', '-t', self.repourl, self.branch] self.sendStatus({"header": "fetching branch %s from %s\n" % (self.branch, self.repourl)}) - c = ShellCommand(self.builder, command, self._fullSrcdir(), - sendRC=False, timeout=self.timeout, usePTY=False) - self.command = c - d = c.start() - d.addCallback(self._abandonOnFailure) - d.addCallback(self._didFetch) - return d + return self._dovccmd(command, self._didFetch) + + def _didClean(self, dummy): + # After a clean, try to use the given revision if we have one. + if self.revision: + # We know what revision we want. See if we have it. + d = self._dovccmd(['reset', '--hard', self.revision], + self._initSubmodules) + # If we are unable to reset to the specified version, we + # must do a fetch first and retry. + d.addErrback(self._doFetch) + return d + else: + # No known revision, go grab the latest. + return self._doFetch(None) def _didInit(self, res): return self.doVCUpdate() def doVCFull(self): - os.mkdir(self._fullSrcdir()) - c = ShellCommand(self.builder, ['git', 'init'], self._fullSrcdir(), - sendRC=False, timeout=self.timeout, usePTY=False) - self.command = c - d = c.start() - d.addCallback(self._abandonOnFailure) - d.addCallback(self._didInit) - return d + os.makedirs(self._fullSrcdir()) + return self._dovccmd(['init'], self._didInit) def parseGotRevision(self): - command = ['git', 'rev-parse', 'HEAD'] - c = ShellCommand(self.builder, command, self._fullSrcdir(), - sendRC=False, keepStdout=True, usePTY=False) - d = c.start() + command = ['rev-parse', 'HEAD'] def _parse(res): - hash = c.stdout.strip() + hash = self.command.stdout.strip() if len(hash) != 40: return None return hash - d.addCallback(_parse) - return d + return self._dovccmd(command, _parse, keepStdout=True) registerSlaveCommand("git", Git, command_version) @@ -2182,7 +2237,8 @@ class Arch(SourceBase): if self.revision: command.append(self.revision) c = ShellCommand(self.builder, command, d, - sendRC=False, timeout=self.timeout, usePTY=False) + sendRC=False, timeout=self.timeout, + maxTime=self.maxTime, usePTY=False) self.command = c return c.start() @@ -2194,8 +2250,8 @@ class Arch(SourceBase): command = [self.vcexe, 'register-archive', '--force', self.url] c = ShellCommand(self.builder, command, self.builder.basedir, - sendRC=False, keepStdout=True, - timeout=self.timeout, usePTY=False) + sendRC=False, keepStdout=True, timeout=self.timeout, + maxTime=self.maxTime, usePTY=False) self.command = c d = c.start() d.addCallback(self._abandonOnFailure) @@ -2228,7 +2284,8 @@ class Arch(SourceBase): '--no-pristine', ver, self.srcdir] c = ShellCommand(self.builder, command, self.builder.basedir, - sendRC=False, timeout=self.timeout, usePTY=False) + sendRC=False, timeout=self.timeout, + maxTime=self.maxTime, usePTY=False) self.command = c d = c.start() d.addCallback(self._abandonOnFailure) @@ -2240,7 +2297,8 @@ class Arch(SourceBase): d = os.path.join(self.builder.basedir, self.srcdir) command = [self.vcexe, 'build-config', self.buildconfig] c = ShellCommand(self.builder, command, d, - sendRC=False, timeout=self.timeout, usePTY=False) + sendRC=False, timeout=self.timeout, + maxTime=self.maxTime, usePTY=False) self.command = c d = c.start() d.addCallback(self._abandonOnFailure) @@ -2300,7 +2358,8 @@ class Bazaar(Arch): command = [self.vcexe, 'get', '--no-pristine', ver, self.srcdir] c = ShellCommand(self.builder, command, self.builder.basedir, - sendRC=False, timeout=self.timeout, usePTY=False) + sendRC=False, timeout=self.timeout, + maxTime=self.maxTime, usePTY=False) self.command = c d = c.start() d.addCallback(self._abandonOnFailure) @@ -2375,7 +2434,8 @@ class Bzr(SourceBase): srcdir = os.path.join(self.builder.basedir, self.srcdir) command = [self.vcexe, 'update'] c = ShellCommand(self.builder, command, srcdir, - sendRC=False, timeout=self.timeout, usePTY=False) + sendRC=False, timeout=self.timeout, + maxTime=self.maxTime, usePTY=False) self.command = c return c.start() @@ -2403,7 +2463,8 @@ class Bzr(SourceBase): command.append(self.srcdir) c = ShellCommand(self.builder, command, d, - sendRC=False, timeout=self.timeout, usePTY=False) + sendRC=False, timeout=self.timeout, + maxTime=self.maxTime, usePTY=False) self.command = c d = c.start() return d @@ -2418,13 +2479,15 @@ class Bzr(SourceBase): command.append(self.repourl) command.append(tmpdir) c = ShellCommand(self.builder, command, self.builder.basedir, - sendRC=False, timeout=self.timeout, usePTY=False) + sendRC=False, timeout=self.timeout, + maxTime=self.maxTime, usePTY=False) self.command = c d = c.start() def _export(res): command = [self.vcexe, 'export', srcdir] c = ShellCommand(self.builder, command, tmpdir, - sendRC=False, timeout=self.timeout, usePTY=False) + sendRC=False, timeout=self.timeout, + maxTime=self.maxTime, usePTY=False) self.command = c return c.start() d.addCallback(_export) @@ -2514,7 +2577,7 @@ class Mercurial(SourceBase): command = [self.vcexe, 'pull', '--verbose', self.repourl] c = ShellCommand(self.builder, command, d, sendRC=False, timeout=self.timeout, - keepStdout=True, usePTY=False) + maxTime=self.maxTime, keepStdout=True, usePTY=False) self.command = c d = c.start() d.addCallback(self._handleEmptyUpdate) @@ -2542,7 +2605,8 @@ class Mercurial(SourceBase): command.extend([self.repourl, d]) c = ShellCommand(self.builder, command, self.builder.basedir, - sendRC=False, timeout=self.timeout, usePTY=False) + sendRC=False, timeout=self.timeout, + maxTime=self.maxTime, usePTY=False) self.command = c cmd1 = c.start() cmd1.addCallback(self._update) @@ -2707,7 +2771,7 @@ class Mercurial(SourceBase): updatecmd.extend(['--rev', self.args.get('branch', 'default')]) self.command = ShellCommand(self.builder, updatecmd, self.builder.basedir, sendRC=False, - timeout=self.timeout, usePTY=False) + timeout=self.timeout, maxTime=self.maxTime, usePTY=False) return self.command.start() def parseGotRevision(self): @@ -2758,8 +2822,9 @@ class P4Base(SourceBase): command.extend(['changes', '-m', '1', '#have']) c = ShellCommand(self.builder, command, self.builder.basedir, environ=self.env, timeout=self.timeout, - sendStdout=True, sendStderr=False, sendRC=False, - keepStdout=True, usePTY=False) + maxTime=self.maxTime, sendStdout=True, + sendStderr=False, sendRC=False, keepStdout=True, + usePTY=False) self.command = c d = c.start() @@ -2845,7 +2910,7 @@ class P4(P4Base): env = {} c = ShellCommand(self.builder, command, self.builder.basedir, environ=env, sendRC=False, timeout=self.timeout, - keepStdout=True, usePTY=False) + maxTime=self.maxTime, keepStdout=True, usePTY=False) self.command = c d = c.start() d.addCallback(self._abandonOnFailure) @@ -2882,7 +2947,8 @@ class P4(P4Base): log.msg(client_spec) c = ShellCommand(self.builder, command, self.builder.basedir, environ=env, sendRC=False, timeout=self.timeout, - initialStdin=client_spec, usePTY=False) + maxTime=self.maxTime, initialStdin=client_spec, + usePTY=False) self.command = c d = c.start() d.addCallback(self._abandonOnFailure) @@ -2936,7 +3002,8 @@ class P4Sync(P4Base): command.extend(['@' + self.revision]) env = {} c = ShellCommand(self.builder, command, d, environ=env, - sendRC=False, timeout=self.timeout, usePTY=False) + sendRC=False, timeout=self.timeout, + maxTime=self.maxTime, usePTY=False) self.command = c return c.start() buildbot/slave/commands.py @@ -1,4 +1,3 @@ - from zope.interface import implements from buildbot import util, interfaces buildbot/sourcestamp.py @@ -12,7 +12,11 @@ import os, shutil, sys, re, urllib, itertools import gc from cPickle import load, dump from cStringIO import StringIO -from bz2 import BZ2File + +try: # bz2 is not available on py23 + from bz2 import BZ2File +except ImportError: + BZ2File = None # sibling imports from buildbot import interfaces, util, sourcestamp @@ -220,8 +224,15 @@ class LogFile: finished = False length = 0 + nonHeaderLength = 0 + tailLength = 0 chunkSize = 10*1000 runLength = 0 + # No max size by default + logMaxSize = None + # Don't keep a tail buffer by default + logMaxTailSize = None + maxLengthExceeded = False runEntries = [] # provided so old pickled builds will getChunks() ok entries = None BUFFERSIZE = 2048 @@ -251,6 +262,7 @@ class LogFile: self.runEntries = [] self.watchers = [] self.finishedWatchers = [] + self.tailBuffer = [] def getFilename(self): return os.path.join(self.step.build.builder.basedir, self.filename) @@ -282,10 +294,11 @@ class LogFile: return self.openfile # otherwise they get their own read-only handle # try a compressed log first - try: - return BZ2File(self.getFilename() + ".bz2", "r") - except IOError: - pass + if BZ2File is not None: + try: + return BZ2File(self.getFilename() + ".bz2", "r") + except IOError: + pass return open(self.getFilename(), "r") def getText(self): @@ -404,6 +417,31 @@ class LogFile: def addEntry(self, channel, text): assert not self.finished + + if channel != HEADER: + # Truncate the log if it's more than logMaxSize bytes + if self.logMaxSize and self.nonHeaderLength > self.logMaxSize: + # Add a message about what's going on + if not self.maxLengthExceeded: + msg = "\nOutput exceeded %i bytes, remaining output has been truncated\n" % self.logMaxSize + self.addEntry(HEADER, msg) + self.merge() + self.maxLengthExceeded = True + + if self.logMaxTailSize: + # Update the tail buffer + self.tailBuffer.append((channel, text)) + self.tailLength += len(text) + while self.tailLength > self.logMaxTailSize: + # Drop some stuff off the beginning of the buffer + c,t = self.tailBuffer.pop(0) + n = len(t) + self.tailLength -= n + assert self.tailLength >= 0 + return + + self.nonHeaderLength += len(text) + # we only add to .runEntries here. merge() is responsible for adding # merged chunks to .entries if self.runEntries and channel != self.runEntries[0][0]: @@ -425,7 +463,19 @@ class LogFile: self.addEntry(HEADER, text) def finish(self): - self.merge() + if self.tailBuffer: + msg = "\nFinal %i bytes follow below:\n" % self.tailLength + tmp = self.runEntries + self.runEntries = [(HEADER, msg)] + self.merge() + self.runEntries = self.tailBuffer + self.merge() + self.runEntries = tmp + self.merge() + self.tailBuffer = [] + else: + self.merge() + if self.openfile: # we don't do an explicit close, because there might be readers # shareing the filehandle. As soon as they stop reading, the @@ -444,6 +494,10 @@ class LogFile: def compressLog(self): + # bail out if there's no compression support + if BZ2File is None: + return + compressed = self.getFilename() + ".bz2.tmp" d = threads.deferToThread(self._compressLog, compressed) d.addCallback(self._renameCompressedLog, compressed) @@ -885,6 +939,8 @@ class BuildStepStatus(styles.Versioned): assert self.started # addLog before stepStarted won't notify watchers logfilename = self.build.generateLogfileName(self.name, name) log = LogFile(self, name, logfilename) + log.logMaxSize = self.build.builder.logMaxSize + log.logMaxTailSize = self.build.builder.logMaxTailSize self.logs.append(log) for w in self.watchers: receiver = w.logStarted(self.build, self, log) @@ -943,7 +999,9 @@ class BuildStepStatus(styles.Versioned): if logCompressionLimit is not False and \ isinstance(loog, LogFile): if os.path.getsize(loog.getFilename()) > logCompressionLimit: - cld.append(loog.compressLog()) + loog_deferred = loog.compressLog() + if loog_deferred: + cld.append(loog_deferred) for r in self.updates.keys(): if self.updates[r] is not None: @@ -1454,6 +1512,8 @@ class BuilderStatus(styles.Versioned): self.buildCache = weakref.WeakValueDictionary() self.buildCache_LRU = [] self.logCompressionLimit = False # default to no compression for tests + self.logMaxSize = None # No default limit + self.logMaxTailSize = None # No tail buffering # persistence @@ -1526,6 +1586,12 @@ class BuilderStatus(styles.Versioned): def setLogCompressionLimit(self, lowerLimit): self.logCompressionLimit = lowerLimit + def setLogMaxSize(self, upperLimit): + self.logMaxSize = upperLimit + + def setLogMaxTailSize(self, tailSize): + self.logMaxTailSize = tailSize + def saveYourself(self): for b in self.currentBuilds: if not b.isFinished: @@ -1726,6 +1792,11 @@ class BuilderStatus(styles.Versioned): for Nb in range(1, self.nextBuildNumber+1): b = self.getBuild(-Nb) if not b: + # HACK: If this is the first build we are looking at, it is + # possible it's in progress but locked before it has written a + # pickle; in this case keep looking. + if Nb == 1: + continue break if branches and not b.getSourceStamp().branch in branches: continue @@ -1952,6 +2023,7 @@ class SlaveStatus: admin = None host = None + version = None connected = False graceful_shutdown = False @@ -1967,6 +2039,8 @@ class SlaveStatus: return self.admin def getHost(self): return self.host + def getVersion(self): + return self.version def isConnected(self): return self.connected def lastMessageReceived(self): @@ -1978,6 +2052,8 @@ class SlaveStatus: self.admin = admin def setHost(self, host): self.host = host + def setVersion(self, version): + self.version = version def setConnected(self, isConnected): self.connected = isConnected def setLastMessageReceived(self, when): @@ -2034,6 +2110,9 @@ class Status: assert os.path.isdir(basedir) # compress logs bigger than 4k, a good default on linux self.logCompressionLimit = 4*1024 + # No default limit to the log size + self.logMaxSize = None + self.logMaxTailSize = None # methods called by our clients @@ -2247,6 +2326,8 @@ class Status: builder_status.setBigState("offline") builder_status.setLogCompressionLimit(self.logCompressionLimit) + builder_status.setLogMaxSize(self.logMaxSize) + builder_status.setLogMaxTailSize(self.logMaxTailSize) for t in self.watchers: self.announceNewBuilder(t, name, builder_status) buildbot/status/builder.py @@ -190,7 +190,7 @@ class MailNotifier(base.StatusReceiverMultiService): subject="buildbot %(result)s in %(projectName)s on %(builder)s", lookup=None, extraRecipients=[], sendToInterestedUsers=True, customMesg=message, - extraHeaders=None): + extraHeaders=None, addPatch=True): """ @type fromaddr: string @param fromaddr: the email address to be used in the 'From' header. @@ -233,12 +233,16 @@ class MailNotifier(base.StatusReceiverMultiService): categories). Use either builders or categories, but not both. - @type addLogs: boolean. + @type addLogs: boolean @param addLogs: if True, include all build logs as attachments to the messages. These can be quite large. This can also be set to a list of log names, to send a subset of the logs. Defaults to False. + @type addPatch: boolean + @param addPatch: if True, include the patch when the source stamp + includes one. + @type relayhost: string @param relayhost: the host to which the outbound SMTP connection should be made. Defaults to 'localhost' @@ -340,6 +344,7 @@ class MailNotifier(base.StatusReceiverMultiService): if extraHeaders: assert isinstance(extraHeaders, dict) self.extraHeaders = extraHeaders + self.addPatch = addPatch self.watched = [] self.status = None @@ -461,7 +466,7 @@ class MailNotifier(base.StatusReceiverMultiService): assert type in ('plain', 'html'), "'%s' message type must be 'plain' or 'html'." % type haveAttachments = False - if attrs['patch'] or self.addLogs: + if (attrs['patch'] and self.addPatch) or self.addLogs: haveAttachments = True if not canDoAttachments: twlog.msg("warning: I want to send mail with attachments, " @@ -485,7 +490,7 @@ class MailNotifier(base.StatusReceiverMultiService): m['From'] = self.fromaddr # m['To'] is added later - if attrs['patch']: + if attrs['patch'] and self.addPatch: a = MIMEText(attrs['patch'][1]) a.add_header('Content-Disposition', "attachment", filename="source patch") buildbot/status/mail.py @@ -84,6 +84,22 @@ def make_stop_form(stopURL, useUserPasswd, on_all=False, label="Build"): data += '<input type="submit" value="Stop %s" /></form>\n' % label return data +def make_extra_property_row(N): + """helper function to create the html for adding extra build + properties to a forced (or resubmitted) build. "N" is an integer + inserted into the form names so that more than one property can be + used in the form. + """ + prop_html = ''' + <div class="row">Property %(N)i + <span class="label">Name:</span> + <span class="field"><input type="text" name="property%(N)iname" /></span> + <span class="label">Value:</span> + <span class="field"><input type="text" name="property%(N)ivalue" /></span> + </div> + ''' % {"N": N} + return prop_html + def make_force_build_form(forceURL, useUserPasswd, on_all=False): if on_all: data = """<form method="post" action="%s" class="command forcebuild"> @@ -101,6 +117,9 @@ def make_force_build_form(forceURL, useUserPasswd, on_all=False): "<input type='text' name='branch' />") + make_row("Revision to build:", "<input type='text' name='revision' />") + + make_extra_property_row(1) + + make_extra_property_row(2) + + make_extra_property_row(3) + '<input type="submit" value="Force Build" /></form>\n') def td(text="", parms={}, **props): @@ -291,10 +310,10 @@ class HtmlResource(resource.Resource): def path_to_root(self, request): return path_to_root(request) - def footer(self, s, req): + def footer(self, status, req): # TODO: this stuff should be generated by a template of some sort - projectURL = s.getProjectURL() - projectName = s.getProjectName() + projectURL = status.getProjectURL() + projectName = status.getProjectName() data = '<hr /><div class="footer">\n' welcomeurl = self.path_to_root(req) + "index.html" @@ -338,7 +357,7 @@ class HtmlResource(resource.Resource): data += "<head>\n" for he in s.head_elements: data += " " + self.fillTemplate(he, request) + "\n" - data += self.head(request) + data += self.head(request) data += "</head>\n\n" data += '<body %s>\n' % " ".join(['%s="%s"' % (k,v) buildbot/status/web/base.py @@ -16,6 +16,7 @@ from buildbot.status.web.base import HtmlResource, Box, \ from buildbot.status.web.feeds import Rss20StatusResource, \ Atom10StatusResource from buildbot.status.web.waterfall import WaterfallStatusResource +from buildbot.status.web.console import ConsoleStatusResource from buildbot.status.web.grid import GridStatusResource, TransposedGridStatusResource from buildbot.status.web.changes import ChangesResource from buildbot.status.web.builder import BuildersResource @@ -123,10 +124,12 @@ class OneLinePerBuild(HtmlResource, OneLineMixin): data = "" # really this is "up to %d builds" + html_branches = map(html.escape, branches) data += "<h1>Last %d finished builds: %s</h1>\n" % \ - (numbuilds, ", ".join(branches)) + (numbuilds, ", ".join(html_branches)) if builders: - data += ("<p>of builders: %s</p>\n" % (", ".join(builders))) + html_builders = map(html.escape, builders) + data += ("<p>of builders: %s</p>\n" % (", ".join(html_builders))) data += "<ul>\n" got = 0 building = False @@ -179,8 +182,9 @@ class OneLinePerBuildOneBuilder(HtmlResource, OneLineMixin): numbuilds) data = "" + html_branches = map(html.escape, branches) data += ("<h1>Last %d builds of builder %s: %s</h1>\n" % - (numbuilds, self.builder_name, ", ".join(branches))) + (numbuilds, self.builder_name, ", ".join(html_branches))) data += "<ul>\n" got = 0 for build in g: @@ -215,7 +219,8 @@ class OneBoxPerBuilder(HtmlResource): data = "" - data += "<h2>Latest builds: %s</h2>\n" % ", ".join(branches) + html_branches = map(html.escape, branches) + data += "<h2>Latest builds: %s</h2>\n" % ", ".join(html_branches) data += "<table>\n" building = False @@ -510,6 +515,7 @@ class WebStatus(service.MultiService): #self.putChild("", IndexOrWaterfallRedirection()) self.putChild("waterfall", WaterfallStatusResource()) self.putChild("grid", GridStatusResource()) + self.putChild("console", ConsoleStatusResource()) self.putChild("tgrid", TransposedGridStatusResource()) self.putChild("builders", BuildersResource()) # has builds/steps/logs self.putChild("changes", ChangesResource()) buildbot/status/web/baseweb.py @@ -6,7 +6,9 @@ from twisted.internet import defer, reactor import urllib, time from twisted.python import log from buildbot.status.web.base import HtmlResource, make_row, make_stop_form, \ - css_classes, path_to_builder, path_to_slave, make_name_user_passwd_form + make_extra_property_row, css_classes, path_to_builder, path_to_slave, \ + make_name_user_passwd_form + from buildbot.status.web.tests import TestsResource from buildbot.status.web.step import StepsResource @@ -30,8 +32,8 @@ class StatusResourceBuild(HtmlResource): def body(self, req): b = self.build_status status = self.getStatus(req) - projectURL = status.getProjectURL() projectName = status.getProjectName() + projectURL = status.getProjectURL() data = ('<div class="title"><a href="%s">%s</a></div>\n' % (self.path_to_root(req), projectName)) builder_name = b.getBuilder().getName() @@ -123,8 +125,8 @@ class StatusResourceBuild(HtmlResource): name, " ".join(s.getText()), time_to_run)) + data += " <ol>\n" if s.getLogs(): - data += " <ol>\n" for logfile in s.getLogs(): logname = logfile.getName() logurl = req.childLink("steps/%s/logs/%s" % @@ -132,8 +134,15 @@ class StatusResourceBuild(HtmlResource): urllib.quote(logname))) data += (" <li><a href=\"%s\">%s</a></li>\n" % (logurl, logfile.getName())) - data += " </ol>\n" + if s.getURLs(): + for url in s.getURLs().items(): + logname = url[0] + logurl = url[1] + data += (' <li><a href="%s">%s</a></li>\n' % + (logurl, html.escape(logname))) + data += "</ol>\n" data += " </li>\n" + data += "</ol>\n" data += "<h2>Build Properties:</h2>\n" @@ -199,6 +208,9 @@ class StatusResourceBuild(HtmlResource): data += ('<form method="post" action="%s" class="command rebuild">\n' % rebuildURL) data += make_name_user_passwd_form(self.isUsingUserPasswd(req)) + data += make_extra_property_row(1) + data += make_extra_property_row(2) + data += make_extra_property_row(3) data += make_row("Reason for re-running build:", "<input type='text' name='comments' />") data += '<input type="submit" value="Rebuild" />\n' @@ -239,14 +251,14 @@ class StatusResourceBuild(HtmlResource): (b.getBuilder().getName(), b.getNumber())) name = req.args.get("username", ["<unknown>"])[0] comments = req.args.get("comments", ["<no reason specified>"])[0] + # html-quote both the username and comments, just to be safe reason = ("The web-page 'stop build' button was pressed by " - "'%s': %s\n" % (name, comments)) + "'%s': %s\n" % (html.escape(name), html.escape(comments))) if c: c.stopBuild(reason) # we're at http://localhost:8080/svn-hello/builds/5/stop?[args] and # we want to go to: http://localhost:8080/svn-hello - url = req.args.get('url', ['../..'])[0] - r = Redirect(url) + r = Redirect("../..") d = defer.Deferred() reactor.callLater(1, d.callback, r) return DeferredResource(d) @@ -262,7 +274,7 @@ class StatusResourceBuild(HtmlResource): name = req.args.get("username", ["<unknown>"])[0] comments = req.args.get("comments", ["<no reason specified>"])[0] reason = ("The web-page 'rebuild' button was pressed by " - "'%s': %s\n" % (name, comments)) + "'%s': %s\n" % (html.escape(name), html.escape(comments))) if not bc or not b.isFinished(): log.msg("could not rebuild: bc=%s, isFinished=%s" % (bc, b.isFinished())) buildbot/status/web/build.py @@ -10,6 +10,7 @@ from buildbot.status.web.base import HtmlResource, make_row, \ make_force_build_form, OneLineMixin, path_to_build, path_to_slave, \ path_to_builder, path_to_change from buildbot.process.base import BuildRequest +from buildbot.process.properties import Properties from buildbot.sourcestamp import SourceStamp from buildbot.status.web.build import BuildsResource, StatusResourceBuild @@ -115,7 +116,8 @@ class StatusResourceBuilder(HtmlResource, OneLineMixin): data += "</ul>\n" cancelURL = path_to_builder(req, self.builder_status) + '/cancelbuild' - data += ''' + if self.builder_control is not None: + data += ''' <form action="%s" class="command cancelbuild" style="display:inline" method="post"> <input type="hidden" name="id" value="all" /> <input type="submit" value="Cancel All" /> @@ -129,7 +131,7 @@ class StatusResourceBuilder(HtmlResource, OneLineMixin): data += "<h2>Recent Builds:</h2>\n" data += "(<a href=\"%s\">view in waterfall</a>)\n" % (self.path_to_root(req)+"waterfall?show="+html.escape(b.getName())) data += "<ul>\n" - numbuilds = req.args.get('numbuilds', ['5'])[0] + numbuilds = int(req.args.get('numbuilds', ['5'])[0]) for i,build in enumerate(b.generateFinishedBuilds(num_builds=int(numbuilds))): data += " <li>" + self.make_line(req, build, False) + "</li>\n" if i == 0: @@ -192,9 +194,15 @@ class StatusResourceBuilder(HtmlResource, OneLineMixin): reason = req.args.get("comments", ["<no reason specified>"])[0] branch = req.args.get("branch", [""])[0] revision = req.args.get("revision", [""])[0] + properties = Properties() + for i in (1,2,3): + pname = req.args.get("prop%dname" % i, [""])[0] + pvalue = req.args.get("prop%dvalue" % i, [""])[0] + if pname and pvalue: + properties.setProperty(pname, pvalue, "Force Build Form") r = "The web-page 'force build' button was pressed by '%s': %s\n" \ - % (name, reason) + % (html.escape(name), html.escape(reason)) log.msg("web forcebuild of builder '%s', branch='%s', revision='%s'" " by user '%s'" % (self.builder_status.getName(), branch, revision, name)) @@ -208,14 +216,21 @@ class StatusResourceBuilder(HtmlResource, OneLineMixin): if not self.authUser(req): return Redirect("../../authfail") - # keep weird stuff out of the branch and revision strings. TODO: - # centralize this somewhere. + # keep weird stuff out of the branch revision, and property strings. + # TODO: centralize this somewhere. if not re.match(r'^[\w\.\-\/]*$', branch): log.msg("bad branch '%s'" % branch) return Redirect("..") if not re.match(r'^[\w\.\-\/]*$', revision): log.msg("bad revision '%s'" % revision) return Redirect("..") + for p in properties.asList(): + key = p[0] + value = p[1] + if not re.match(r'^[\w\.\-\/]*$', key) \ + or not re.match(r'^[\w\.\-\/]*$', value): + log.msg("bad property name='%s', value='%s'" % (key, value)) + return Redirect("..") if not branch: branch = None if not revision: @@ -229,7 +244,8 @@ class StatusResourceBuilder(HtmlResource, OneLineMixin): # buildbot.changes.changes.Change instance which is tedious at this # stage to compute s = SourceStamp(branch=branch, revision=revision) - req = BuildRequest(r, s, builderName=self.builder_status.getName()) + req = BuildRequest(r, s, builderName=self.builder_status.getName(), + properties=properties) try: self.builder_control.requestBuildSoon(req) except interfaces.NoSlaveError: buildbot/status/web/builder.py @@ -1,3 +1,4 @@ +from twisted.web import html, resource from buildbot.status.web.base import Box from buildbot.status.web.base import HtmlResource from buildbot.status.web.base import IBox @@ -22,6 +23,7 @@ class BuildStatusStatusResource(HtmlResource): number = request.args.get("number", [None])[0] if not name or not number: return "builder and number parameter missing" + number = int(number) # Main table for the build status. data += '<table>\n' buildbot/status/web/buildstatus.py @@ -85,6 +85,221 @@ td.idle, td.waiting, td.offline, td.building { -moz-border-radius-topright: 5px; } +/* Console view styles */ + +td.DevRev { + padding: 4px 8px 4px 8px; + color: #333333; + border-top-left-radius: 5px; + -webkit-border-top-left-radius: 5px; + -moz-border-radius-topleft: 5px; + background-color: #eee; + width: 1%; +} + +td.DevRevCollapse { + border-bottom-left-radius: 5px; + -webkit-border-bottom-left-radius: 5px; + -moz-border-radius-bottomleft: 5px; +} + +td.DevName { + padding: 4px 8px 4px 8px; + color: #333333; + background-color: #eee; + width: 1%; + text-align: left; +} + +td.DevStatus { + padding: 4px 4px 4px 4px; + color: #333333; + background-color: #eee; +} + +td.DevSlave { + padding: 4px 4px 4px 4px; + color: #333333; + background-color: #eee; +} + +td.first { + border-top-left-radius: 5px; + -webkit-border-top-left-radius: 5px; + -moz-border-radius-topleft: 5px; +} + +td.last { + border-top-right-radius: 5px; + -webkit-border-top-right-radius: 5px; + -moz-border-radius-topright: 5px; +} + +td.DevStatusCategory { + border-radius: 5px; + -webkit-border-radius: 5px; + -moz-border-radius: 5px; + border-width:1px; + border-style:solid; +} + +td.DevStatusCollapse { + border-bottom-right-radius: 5px; + -webkit-border-bottom-right-radius: 5px; + -moz-border-radius-bottomright: 5px; +} + +td.DevDetails { + font-weight: normal; + padding: 8px 8px 8px 8px; + color: #333333; + background-color: #eee; + text-align: left; +} + +td.DevComment { + font-weight: normal; + padding: 8px 8px 8px 8px; + color: #333333; + border-bottom-right-radius: 5px; + -webkit-border-bottom-right-radius: 5px; + -moz-border-radius-bottomright: 5px; + border-bottom-left-radius: 5px; + -webkit-border-bottom-left-radius: 5px; + -moz-border-radius-bottomleft: 5px; + background-color: #eee; + text-align: left; +} + +td.Alt { + background-color: #CCCCCC; +} + +.legend { + border-radius: 5px; + -webkit-border-radius: 5px; + -moz-border-radius: 5px; + width: 100px; + max-width: 100px; + text-align:center; + padding: 2px 2px 2px 2px; + height:14px; + white-space:nowrap; +} + +.DevStatusBox { + text-align:center; + height:20px; + padding:0 2px; + line-height:0; + white-space:nowrap; +} + +.DevStatusBox a { + opacity: 0.85; + border-width:1px; + border-style:solid; + border-radius: 4px; + -webkit-border-radius: 4px; + -moz-border-radius: 4px; + display:block; + width:90%; + height:20px; + line-height:20px; + margin-left: auto; + margin-right: auto; +} + +.DevSlaveBox { + text-align:center; + height:10px; + padding:0 2px; + line-height:0; + white-space:nowrap; +} + +.DevSlaveBox a { + opacity: 0.85; + border-width:1px; + border-style:solid; + border-radius: 4px; + -webkit-border-radius: 4px; + -moz-border-radius: 4px; + display:block; + width:90%; + height:10px; + line-height:20px; + margin-left: auto; + margin-right: auto; +} + +a.noround { + border-radius: 0px; + -webkit-border-radius: 0px; + -moz-border-radius: 0px; + position: relative; + margin-top: -8px; + margin-bottom: -8px; + height: 36px; + border-top-width: 0; + border-bottom-width: 0; +} + +a.begin { + border-top-width:1px; + position: relative; + margin-top: 0px; + margin-bottom: -7px; + height: 27px; + border-top-left-radius: 4px; + -webkit-border-top-left-radius: 4px; + -moz-border-radius-topleft: 4px; + border-top-right-radius: 4px; + -webkit-border-top-right-radius: 4px; + -moz-border-radius-topright: 4px; +} + +a.end { + border-bottom-width:1px; + position: relative; + margin-top: -7px; + margin-bottom: 0px; + height: 27px; + border-bottom-left-radius: 4px; + -webkit-border-bottom-left-radius: 4px; + -moz-border-radius-bottomleft: 4px; + border-bottom-right-radius: 4px; + -webkit-border-bottom-right-radius: 4px; + -moz-border-radius-bottomright: 4px; +} + +.center_align { + text-align: center; +} + +.right_align { + text-align: right; +} + +.left_align { + text-align: left; +} + +div.BuildWaterfall { + border-radius: 7px; + -webkit-border-radius: 7px; + -moz-border-radius: 7px; + position: absolute; + left: 0px; + top: 0px; + background-color: #FFFFFF; + padding: 4px 4px 4px 4px; + float: left; + display: none; + border-width: 1px; + border-style: solid; +} + /* LastBuild, BuildStep states */ .success { color: #FFFFFF; @@ -165,7 +380,3 @@ table.Grid tr td.builder { table.Grid tr td.build { border: 1px gray solid; } - -div.footer { - font-size: 80%; -} buildbot/status/web/extended.css @@ -27,7 +27,7 @@ import os import re import sys import time -from twisted.web import resource +from twisted.web import resource, html from buildbot.status.builder import SUCCESS, WARNINGS, FAILURE, EXCEPTION class XmlResource(resource.Resource): @@ -186,22 +186,6 @@ class FeedResource(XmlResource): title = ('%s failed on "%s"' % (source, build.getBuilder().getName())) - # get name of the failed step and the last 30 lines of its log. - if build.getLogs(): - log = build.getLogs()[-1] - laststep = log.getStep().getName() - try: - lastlog = log.getText() - except IOError: - # Probably the log file has been removed - lastlog='<b>log file not available</b>' - - lines = re.split('\n', lastlog) - lastlog = '' - for logline in lines[max(0, len(lines)-30):]: - lastlog = lastlog + logline + '<br/>' - lastlog = lastlog.replace('\n', '<br/>') - description = '' description += ('Date: %s<br/><br/>' % time.strftime("%a, %d %b %Y %H:%M:%S GMT", @@ -218,8 +202,26 @@ class FeedResource(XmlResource): (link, link)) description += ('Author list: <b>%s</b><br/><br/>' % ",".join(build.getResponsibleUsers())) - description += ('Failed step: <b>%s</b><br/><br/>' % laststep) - description += 'Last lines of the build log:<br/>' + + # Add information about the failing steps. + lastlog = '' + for s in build.getSteps(): + if s.getResults()[0] == FAILURE: + description += ('Failed step: <b>%s</b><br/>' % s.getName()) + + # Add the last 30 lines of each log. + for log in s.getLogs(): + lastlog += ('Last lines of build log "%s":<br/>' % log.getName()) + try: + logdata = log.getText() + except IOError: + # Probably the log file has been removed + logdata ='<b>log file not available</b>' + + lastlines = logdata.split('\n')[-30:] + lastlog += '<br/>'.join(lastlines) + lastlog += '<br/>' + description += '<br/>' data += self.item(title, description=description, lastlog=lastlog, link=link, pubDate=finishedTime) @@ -263,12 +265,8 @@ class Rss20StatusResource(FeedResource): if link is not None: data += (' <link>%s</link>\n' % link) if (description is not None and lastlog is not None): - lastlog = re.sub(r'<br/>', "\n", lastlog) - lastlog = re.sub(r'&', "&amp;", lastlog) - lastlog = re.sub(r"'", "&apos;", lastlog) - lastlog = re.sub(r'"', "&quot;", lastlog) - lastlog = re.sub(r'<', '&lt;', lastlog) - lastlog = re.sub(r'>', '&gt;', lastlog) + lastlog = lastlog.replace('<br/>', '\n') + lastlog = html.escape(lastlog) lastlog = lastlog.replace('\n', '<br/>') content = '<![CDATA[' content += description @@ -328,12 +326,9 @@ class Atom10StatusResource(FeedResource): if link is not None: data += (' <link href="%s"/>\n' % link) if (description is not None and lastlog is not None): - lastlog = re.sub(r'<br/>', "\n", lastlog) - lastlog = re.sub(r'&', "&amp;", lastlog) - lastlog = re.sub(r"'", "&apos;", lastlog) - lastlog = re.sub(r'"', "&quot;", lastlog) - lastlog = re.sub(r'<', '&lt;', lastlog) - lastlog = re.sub(r'>', '&gt;', lastlog) + lastlog = lastlog.replace('<br/>', '\n') + lastlog = html.escape(lastlog) + lastlog = lastlog.replace('\n', '<br/>') data += (' <content type="xhtml">\n') data += (' <div xmlns="http://www.w3.org/1999/xhtml">\n') data += (' %s\n' % description) buildbot/status/web/feeds.py @@ -3,6 +3,8 @@ from __future__ import generators import sys, time, os.path import urllib +from twisted.web import html, resource + from buildbot import util from buildbot import version from buildbot.status.web.base import HtmlResource @@ -194,12 +196,13 @@ class GridStatusResource(HtmlResource, GridStatusMixin): data += '<tr>\n' data += '<td class="title"><a href="%s">%s</a>' % (projectURL, projectName) if categories: + html_categories = map(html.escape(categories)) if len(categories) > 1: - data += '\n<br /><b>Categories:</b><br/>%s' % ('<br/>'.join(categories)) + data += '\n<br /><b>Categories:</b><br/>%s' % ('<br/>'.join(html_categories)) else: - data += '\n<br /><b>Category:</b> %s' % categories[0] + data += '\n<br /><b>Category:</b> %s' % html_categories[0] if branch != ANYBRANCH: - data += '\n<br /><b>Branch:</b> %s' % (branch or 'trunk') + data += '\n<br /><b>Branch:</b> %s' % (html.escape(branch or 'trunk')) data += '</td>\n' for stamp in stamps: data += self.stamp_td(stamp) @@ -289,12 +292,13 @@ class TransposedGridStatusResource(HtmlResource, GridStatusMixin): data += '<tr>\n' data += '<td class="title"><a href="%s">%s</a>' % (projectURL, projectName) if categories: + html_categories = map(html.escape(categories)) if len(categories) > 1: - data += '\n<br /><b>Categories:</b><br/>%s' % ('<br/>'.join(categories)) + data += '\n<br /><b>Categories:</b><br/>%s' % ('<br/>'.join(html_categories)) else: - data += '\n<br /><b>Category:</b> %s' % categories[0] + data += '\n<br /><b>Category:</b> %s' % html_categories[0] if branch != ANYBRANCH: - data += '\n<br /><b>Branch:</b> %s' % (branch or 'trunk') + data += '\n<br /><b>Branch:</b> %s' % (html.escape(branch or 'trunk')) data += '</td>\n' sortedBuilderNames = status.getBuilderNames()[:] buildbot/status/web/grid.py @@ -9,15 +9,15 @@ <h1>Welcome to the Buildbot!</h1> <ul> - <li>the <a href="waterfall">Waterfall Display</a> will give you a - time-oriented summary of recent buildbot activity.</li> - <li>the <a href="grid">Grid Display</a> will give you a developer-oriented summary of recent buildbot activity.</li> <li>the <a href="tgrid">Transposed Grid Display</a> presents the same information as the grid, but lists the revisions down the side.</li> + <li>the <a href="waterfall">Waterfall Display</a> will give you a + time-oriented summary of recent buildbot activity.</li> + <li>The <a href="one_box_per_builder">Latest Build</a> for each builder is here.</li> buildbot/status/web/index.html @@ -77,7 +77,7 @@ class OneBuildSlaveResource(HtmlResource, OneLineMixin): data.append("<a href=\"%s\">%s</a>\n" % (self.path_to_root(req), projectName)) - data.append("<h1>Build Slave: %s</h1>\n" % self.slavename) + data.append("<h1>Build Slave: %s</h1>\n" % html.escape(self.slavename)) shutdown_url = req.childLink("shutdown") @@ -170,6 +170,8 @@ class BuildSlavesResource(HtmlResource): isBusy = len(slave_status.getRunningBuilds()) data += " <li><a href=\"%s\">%s</a>:\n" % (req.childLink(urllib.quote(name,'')), name) data += " <ul>\n" + version = slave.getVersion() + data += "<li>Running Buildbot version: %s" % version builder_links = ['<a href="%s">%s</a>' % (req.childLink("../builders/%s" % bname),bname) for bname in used_by_builder.get(name, [])] @@ -213,4 +215,4 @@ class BuildSlavesResource(HtmlResource): slave = self.getStatus(req).getSlave(path) return OneBuildSlaveResource(path) except KeyError: - return NoResource("No such slave '%s'" % path) + return NoResource("No such slave '%s'" % html.escape(path)) buildbot/status/web/slaves.py @@ -61,4 +61,4 @@ class TestsResource(HtmlResource): result = self.test_results[name] return TestResult(name, result) except KeyError: - return NoResource("No such test name '%s'" % path) + return NoResource("No such test name '%s'" % html.escape(path)) buildbot/status/web/tests.py @@ -263,7 +263,7 @@ be displayed that occurred <b>before</b> this timestamp. Instead of providing given interval, where each timestamp on the left hand edge counts as a single event. You can add a <tt>num_events=</tt> argument to override this this.</p> -<h2>Hiding non-Build events</h2> +<h2>Showing non-Build events</h2> <p>By passing <tt>show_events=true</tt>, you can add the "buildslave attached", "buildslave detached", and "builder reconfigured" events that @@ -327,13 +327,13 @@ class WaterfallHelp(HtmlResource): data = '' status = self.getStatus(request) - showEvents_checked = 'checked="checked"' + showEvents_checked = '' if request.args.get("show_events", ["false"])[0].lower() == "true": - showEvents_checked = '' + showEvents_checked = 'checked="checked"' show_events_input = ('<p>' '<input type="checkbox" name="show_events" ' 'value="true" %s>' - 'Hide non-Build events' + 'Show non-Build events' '</p>\n' ) % showEvents_checked @@ -358,7 +358,7 @@ class WaterfallHelp(HtmlResource): '<input type="text" name="branch" ' 'value="%s">' '</td></tr>\n' - ) % (b,) + ) % (html.escape(b),) show_branches_input += '</table>\n' # this has a set of toggle-buttons to let the user choose the @@ -401,7 +401,7 @@ class WaterfallHelp(HtmlResource): '<td><input type="radio" name="reload" ' 'value="%s" %s></td> ' '<td>%s</td></tr>\n' - ) % (value, checked, name) + ) % (html.escape(value), checked, html.escape(name)) show_reload_input += '</table>\n' fields = {"show_events_input": show_events_input, @@ -586,7 +586,7 @@ class WaterfallStatusResource(HtmlResource): newargs[k].append(v) else: newargs[k] = [v] - newquery = "&".join(["%s=%s" % (k, v) + newquery = "&".join(["%s=%s" % (urllib.quote(k), urllib.quote(v)) for k in newargs for v in newargs[k] ]) buildbot/status/web/waterfall.py @@ -53,7 +53,6 @@ class IrcBuildRequest: d = s.waitUntilFinished() d.addCallback(self.parent.watchedBuildFinished) - class Contact: """I hold the state for a single user's interaction with the buildbot. @@ -307,16 +306,16 @@ class Contact: self.send(r) + results_descriptions = { + SUCCESS: "Success", + WARNINGS: "Warnings", + FAILURE: "Failure", + EXCEPTION: "Exception", + } + def buildFinished(self, builderName, build, results): builder = build.getBuilder() - results_descriptions = { - SUCCESS: "Success", - WARNINGS: "Warnings", - FAILURE: "Failure", - EXCEPTION: "Exception", - } - # only notify about builders we are interested in log.msg('[Contact] builder %r in category %s finished' % (builder, builder.category)) @@ -324,39 +323,47 @@ class Contact: builder.category not in self.channel.categories): return - results = build.getResults() + if not self.notify_for_finished(build): + return r = "build #%d of %s is complete: %s" % \ (build.getNumber(), builder.getName(), - results_descriptions.get(results, "??")) + self.results_descriptions.get(build.getResults(), "??")) r += " [%s]" % " ".join(build.getText()) buildurl = self.channel.status.getURLForThing(build) if buildurl: r += " Build details are at %s" % buildurl - if self.notify_for('finished') or self.notify_for(lower(results_descriptions.get(results))): - self.send(r) - return + if self.channel.showBlameList and build.getResults() != SUCCESS and len(build.changes) != 0: + r += ' blamelist: ' + ', '.join([c.who for c in build.changes]) + + self.send(r) + + def notify_for_finished(self, build): + results = build.getResults() + + if self.notify_for('finished'): + return True + + if self.notify_for(lower(self.results_descriptions.get(results))): + return True prevBuild = build.getPreviousBuild() if prevBuild: prevResult = prevBuild.getResults() - required_notification_control_string = join((lower(results_descriptions.get(prevResult)), \ + required_notification_control_string = join((lower(self.results_descriptions.get(prevResult)), \ 'To', \ - capitalize(results_descriptions.get(results))), \ + capitalize(self.results_descriptions.get(results))), \ '') if (self.notify_for(required_notification_control_string)): - self.send(r) + return True + + return False def watchedBuildFinished(self, b): - results = {SUCCESS: "Success", - WARNINGS: "Warnings", - FAILURE: "Failure", - EXCEPTION: "Exception", - } # only notify about builders we are interested in builder = b.getBuilder() @@ -369,7 +376,7 @@ class Contact: r = "Hey! build %s #%d is complete: %s" % \ (b.getBuilder().getName(), b.getNumber(), - results.get(b.getResults(), "??")) + self.results_descriptions.get(b.getResults(), "??")) r += " [%s]" % " ".join(b.getText()) self.send(r) buildurl = self.channel.status.getURLForThing(b) @@ -581,14 +588,15 @@ class IRCContact(Contact): self.dest = dest def describeUser(self, user): - if self.dest[0] == "#": + if self.dest[0] == '#': return "IRC user <%s> on channel %s" % (user, self.dest) return "IRC user <%s> (privmsg)" % user # userJoined(self, user, channel) def send(self, message): - self.channel.msg(self.dest, message.encode("ascii", "replace")) + self.channel.msgOrNotice(self.dest, message.encode("ascii", "replace")) + def act(self, action): self.channel.me(self.dest, action.encode("ascii", "replace")) @@ -661,8 +669,9 @@ class IrcStatusBot(irc.IRCClient): """I represent the buildbot to an IRC server. """ implements(IChannel) + contactClass = IRCContact - def __init__(self, nickname, password, channels, status, categories, notify_events): + def __init__(self, nickname, password, channels, status, categories, notify_events, noticeOnChannel = False, showBlameList = False): """ @type nickname: string @param nickname: the nickname by which this bot should be known @@ -683,6 +692,14 @@ class IrcStatusBot(irc.IRCClient): self.counter = 0 self.hasQuit = 0 self.contacts = {} + self.noticeOnChannel = noticeOnChannel + self.showBlameList = showBlameList + + def msgOrNotice(self, dest, message): + if self.noticeOnChannel and dest[0] == '#': + self.notice(dest, message) + else: + self.msg(dest, message) def addContact(self, name, contact): self.contacts[name] = contact @@ -690,7 +707,7 @@ class IrcStatusBot(irc.IRCClient): def getContact(self, name): if name in self.contacts: return self.contacts[name] - new_contact = IRCContact(self, name) + new_contact = self.contactClass(self, name) self.contacts[name] = new_contact return new_contact @@ -742,6 +759,9 @@ class IrcStatusBot(irc.IRCClient): def joined(self, channel): self.log("I have joined %s" % (channel,)) + # trigger contact contructor, which in turn subscribes to notify events + self.getContact(channel) + def left(self, channel): self.log("I have left %s" % (channel,)) def kickedFrom(self, channel, kicker, message): @@ -774,7 +794,7 @@ class IrcStatusFactory(ThrottledClientFactory): shuttingDown = False p = None - def __init__(self, nickname, password, channels, categories, notify_events): + def __init__(self, nickname, password, channels, categories, notify_events, noticeOnChannel = False, showBlameList = False): #ThrottledClientFactory.__init__(self) # doesn't exist self.status = None self.nickname = nickname @@ -782,6 +802,8 @@ class IrcStatusFactory(ThrottledClientFactory): self.channels = channels self.categories = categories self.notify_events = notify_events + self.noticeOnChannel = noticeOnChannel + self.showBlameList = showBlameList def __getstate__(self): d = self.__dict__.copy() @@ -796,7 +818,9 @@ class IrcStatusFactory(ThrottledClientFactory): def buildProtocol(self, address): p = self.protocol(self.nickname, self.password, self.channels, self.status, - self.categories, self.notify_events) + self.categories, self.notify_events, + noticeOnChannel = self.noticeOnChannel, + showBlameList = self.showBlameList) p.factory = self p.status = self.status p.control = self.control @@ -829,7 +853,8 @@ class IRC(base.StatusReceiverMultiService): "categories"] def __init__(self, host, nick, channels, port=6667, allowForce=True, - categories=None, password=None, notify_events={}): + categories=None, password=None, notify_events={}, + noticeOnChannel = False, showBlameList = True): base.StatusReceiverMultiService.__init__(self) assert allowForce in (True, False) # TODO: implement others @@ -843,12 +868,12 @@ class IRC(base.StatusReceiverMultiService): self.allowForce = allowForce self.categories = categories self.notify_events = notify_events - - # need to stash the factory so we can give it the status object + log.msg('Notify events %s' % notify_events) self.f = IrcStatusFactory(self.nick, self.password, - self.channels, self.categories, self.notify_events) - - c = internet.TCPClient(host, port, self.f) + self.channels, self.categories, self.notify_events, + noticeOnChannel = noticeOnChannel, + showBlameList = showBlameList) + c = internet.TCPClient(self.host, self.port, self.f) c.setServiceParent(self) def setServiceParent(self, parent): buildbot/status/words.py @@ -35,30 +35,33 @@ class MasterShellCommand(BuildStep): self.step.processEnded(status_object) def start(self): + # render properties + properties = self.build.getProperties() + command = properties.render(self.command) # set up argv - if type(self.command) in types.StringTypes: + if type(command) in types.StringTypes: if runtime.platformType == 'win32': argv = os.environ['COMSPEC'].split() # allow %COMSPEC% to have args if '/c' not in argv: argv += ['/c'] - argv += [self.command] + argv += [command] else: # for posix, use /bin/sh. for other non-posix, well, doesn't # hurt to try - argv = ['/bin/sh', '-c', self.command] + argv = ['/bin/sh', '-c', command] else: if runtime.platformType == 'win32': argv = os.environ['COMSPEC'].split() # allow %COMSPEC% to have args if '/c' not in argv: argv += ['/c'] - argv += list(self.command) + argv += list(command) else: - argv = self.command + argv = command self.stdio_log = stdio_log = self.addLog("stdio") - if type(self.command) in types.StringTypes: - stdio_log.addHeader(self.command.strip() + "\n\n") + if type(command) in types.StringTypes: + stdio_log.addHeader(command.strip() + "\n\n") else: - stdio_log.addHeader(" ".join(self.command) + "\n\n") + stdio_log.addHeader(" ".join(command) + "\n\n") stdio_log.addHeader("** RUNNING ON BUILDMASTER **\n") stdio_log.addHeader(" in dir %s\n" % os.getcwd()) stdio_log.addHeader(" argv: %s\n" % (argv,)) buildbot/steps/master.py @@ -2,7 +2,9 @@ import re from twisted.python import log +from twisted.spread import pb from buildbot.process.buildstep import LoggingBuildStep, RemoteShellCommand +from buildbot.process.buildstep import RemoteCommand from buildbot.status.builder import SUCCESS, WARNINGS, FAILURE, STDOUT, STDERR # for existing configurations that import WithProperties from here. We like @@ -39,6 +41,12 @@ class ShellCommand(LoggingBuildStep): something approximating real-time. (note that logfiles= is actually handled by our parent class LoggingBuildStep) + @ivar lazylogfiles: Defaults to False. If True, logfiles will be tracked + `lazily', meaning they will only be added when and if + they are written to. Empty or nonexistent logfiles + will be omitted. (Also handled by class + LoggingBuildStep.) + """ name = "shell" @@ -296,21 +304,191 @@ class Configure(ShellCommand): descriptionDone = ["configure"] command = ["./configure"] +class StringFileWriter(pb.Referenceable): + """ + FileWriter class that just puts received data into a buffer. + + Used to upload a file from slave for inline processing rather than + writing into a file on master. + """ + def __init__(self): + self.buffer = "" + + def remote_write(self, data): + self.buffer += data + + def remote_close(self): + pass + +class SilentRemoteCommand(RemoteCommand): + """ + Remote command subclass used to run an internal file upload command on the + slave. We do not need any progress updates from such command, so override + remoteUpdate() with an empty method. + """ + def remoteUpdate(self, update): + pass + class WarningCountingShellCommand(ShellCommand): warnCount = 0 warningPattern = '.*warning[: ].*' + # The defaults work for GNU Make. + directoryEnterPattern = "make.*: Entering directory [\"`'](.*)['`\"]" + directoryLeavePattern = "make.*: Leaving directory" + suppressionFile = None - def __init__(self, warningPattern=None, **kwargs): + commentEmptyLineRe = re.compile(r"^\s*(\#.*)?$") + suppressionLineRe = re.compile(r"^\s*(.+?)\s*:\s*(.+?)\s*(?:[:]\s*([0-9]+)(?:-([0-9]+))?\s*)?$") + + def __init__(self, workdir=None, + warningPattern=None, warningExtractor=None, + directoryEnterPattern=None, directoryLeavePattern=None, + suppressionFile=None, **kwargs): + self.workdir = workdir # See if we've been given a regular expression to use to match # warnings. If not, use a default that assumes any line with "warning" # present is a warning. This may lead to false positives in some cases. if warningPattern: self.warningPattern = warningPattern + if directoryEnterPattern: + self.directoryEnterPattern = directoryEnterPattern + if directoryLeavePattern: + self.directoryLeavePattern = directoryLeavePattern + if suppressionFile: + self.suppressionFile = suppressionFile + if warningExtractor: + self.warningExtractor = warningExtractor + else: + self.warningExtractor = WarningCountingShellCommand.warnExtractWholeLine # And upcall to let the base class do its work - ShellCommand.__init__(self, **kwargs) + ShellCommand.__init__(self, workdir=workdir, **kwargs) + + self.addFactoryArguments(warningPattern=warningPattern, + directoryEnterPattern=directoryEnterPattern, + directoryLeavePattern=directoryLeavePattern, + warningExtractor=warningExtractor, + suppressionFile=suppressionFile) + self.suppressions = [] + self.directoryStack = [] + + def setDefaultWorkdir(self, workdir): + if self.workdir is None: + self.workdir = workdir + ShellCommand.setDefaultWorkdir(self, workdir) + + def addSuppression(self, suppressionList): + """ + This method can be used to add patters of warnings that should + not be counted. - self.addFactoryArguments(warningPattern=warningPattern) + It takes a single argument, a list of patterns. + + Each pattern is a 4-tuple (FILE-RE, WARN-RE, START, END). + + FILE-RE is a regular expression (string or compiled regexp), or None. + If None, the pattern matches all files, else only files matching the + regexp. If directoryEnterPattern is specified in the class constructor, + matching is against the full path name, eg. src/main.c. + + WARN-RE is similarly a regular expression matched against the + text of the warning, or None to match all warnings. + + START and END form an inclusive line number range to match against. If + START is None, there is no lower bound, similarly if END is none there + is no upper bound.""" + + for fileRe, warnRe, start, end in suppressionList: + if fileRe != None and isinstance(fileRe, str): + fileRe = re.compile(fileRe) + if warnRe != None and isinstance(warnRe, str): + warnRe = re.compile(warnRe) + self.suppressions.append((fileRe, warnRe, start, end)) + + def warnExtractWholeLine(self, line, match): + """ + Extract warning text as the whole line. + No file names or line numbers.""" + return (None, None, line) + + def warnExtractFromRegexpGroups(self, line, match): + """ + Extract file name, line number, and warning text as groups (1,2,3) + of warningPattern match.""" + file = match.group(1) + lineNo = match.group(2) + if lineNo != None: + lineNo = int(lineNo) + text = match.group(3) + return (file, lineNo, text) + + def maybeAddWarning(self, warnings, line, match): + if self.suppressions: + (file, lineNo, text) = self.warningExtractor(self, line, match) + + if file != None and file != "" and self.directoryStack: + currentDirectory = self.directoryStack[-1] + if currentDirectory != None and currentDirectory != "": + file = "%s/%s" % (currentDirectory, file) + + # Skip adding the warning if any suppression matches. + for fileRe, warnRe, start, end in self.suppressions: + if ( (file == None or fileRe == None or fileRe.search(file)) and + (warnRe == None or warnRe.search(text)) and + lineNo != None and + (start == None or start <= lineNo) and + (end == None or end >= lineNo) ): + return + + warnings.append(line) + self.warnCount += 1 + + def start(self): + if self.suppressionFile == None: + return ShellCommand.start(self) + + version = self.slaveVersion("uploadFile") + if not version: + m = "Slave is too old, does not know about uploadFile" + raise BuildSlaveTooOldError(m) + + self.myFileWriter = StringFileWriter() + + properties = self.build.getProperties() + + args = { + 'slavesrc': properties.render(self.suppressionFile), + 'workdir': self.workdir, + 'writer': self.myFileWriter, + 'maxsize': None, + 'blocksize': 32*1024, + } + cmd = SilentRemoteCommand('uploadFile', args) + d = self.runCommand(cmd) + d.addCallback(self.uploadDone) + d.addErrback(self.failed) + + def uploadDone(self, dummy): + lines = self.myFileWriter.buffer.split("\n") + del(self.myFileWriter) + + list = [] + for line in lines: + if self.commentEmptyLineRe.match(line): + continue + match = self.suppressionLineRe.match(line) + if (match): + file, test, start, end = match.groups() + if (end != None): + end = int(end) + if (start != None): + start = int(start) + if end == None: + end = start + list.append((file, test, start, end)) + + self.addSuppression(list) + return ShellCommand.start(self) def createSummary(self, log): self.warnCount = 0 @@ -324,6 +502,14 @@ class WarningCountingShellCommand(ShellCommand): if isinstance(wre, str): wre = re.compile(wre) + directoryEnterRe = self.directoryEnterPattern + if directoryEnterRe != None and isinstance(directoryEnterRe, str): + directoryEnterRe = re.compile(directoryEnterRe) + + directoryLeaveRe = self.directoryLeavePattern + if directoryLeaveRe != None and isinstance(directoryLeaveRe, str): + directoryLeaveRe = re.compile(directoryLeaveRe) + # Check if each line in the output from this command matched our # warnings regular expressions. If did, bump the warnings count and # add the line to the collection of lines with warnings @@ -331,9 +517,18 @@ class WarningCountingShellCommand(ShellCommand): # TODO: use log.readlines(), except we need to decide about stdout vs # stderr for line in log.getText().split("\n"): - if wre.match(line): - warnings.append(line) - self.warnCount += 1 + if directoryEnterRe: + match = directoryEnterRe.search(line) + if match: + self.directoryStack.append(match.group(1)) + if (directoryLeaveRe and + self.directoryStack and + directoryLeaveRe.search(line)): + self.directoryStack.pop() + + match = wre.match(line) + if match: + self.maybeAddWarning(warnings, line, match) # If there were any warnings, make the log if lines with warnings # available buildbot/steps/shell.py @@ -42,7 +42,10 @@ class Source(LoggingBuildStep): files are deleted but generated files can influence test behavior (e.g. python's .pyc files), or when source directories are deleted but generated files prevent CVS from - removing them. + removing them. When used with a patched checkout, from a + previous buildbot try for instance, it will try to "revert" + the changes first and will do a clobber if it is unable to + get a clean checkout. The behavior is SCM-dependent. - 'copy': specifies that the source-controlled workspace should be maintained in a separate directory (called the @@ -176,11 +179,14 @@ class Source(LoggingBuildStep): revision = self.computeSourceRevision(s.changes) # if patch is None, then do not patch the tree after checkout - # 'patch' is None or a tuple of (patchlevel, diff) + # 'patch' is None or a tuple of (patchlevel, diff, root) + # root is optional. patch = s.patch if patch: self.addCompleteLog("patch", patch[1]) + if self.alwaysUseLatest: + revision = None self.startVC(branch, revision, patch) def commandComplete(self, cmd): @@ -1155,4 +1161,3 @@ class Monotone(Source): assert slavever, "slave is too old, does not know about monotone" cmd = LoggedRemoteCommand("monotone", self.args) self.startCommand(cmd) - buildbot/steps/source.py @@ -6,6 +6,8 @@ from twisted.trial import unittest from buildbot import interfaces from buildbot.process import buildstep +from buildbot.process import mtrlogobserver +from buildbot.process import subunitlogobserver # have to subclass LogObserver in order to test it, since the default # implementations of outReceived() and errReceived() do nothing @@ -133,6 +135,133 @@ class LogLineObserver(ObserverTestCase): self._assertStdout([chunk*4 + "12345"]) self._assertStderr([]) +class MyMtrLogObserver(mtrlogobserver.MtrLogObserver): + def __init__(self): + mtrlogobserver.MtrLogObserver.__init__(self, textLimit=3, testNameLimit=15) + self.testFails = [] + self.testWarnLists = [] + # We don't have a buildstep in self.step. + # So we'll just install ourself there, so we can check the call of + # setProgress(). + # Same for self.step.step_status.setText() + self.step = self + self.step_status = self + self.progresses = [] + self.text = [] + + def setProgress(self, type, value): + self.progresses.append((type, value)) + + def setText(self, text): + self.text = text + + def collectTestFail(self, testname, variant, result, info, text): + self.testFails.append((testname, variant, result, info, text)) + + def collectWarningTests(self, testList): + self.testWarnLists.append(testList) + +class MtrLogObserver(ObserverTestCase): + observer_cls = MyMtrLogObserver + + def test1(self): + self._logStdout(""" +MySQL Version 5.1.35 +============================================================================== +TEST RESULT TIME (ms) +------------------------------------------------------------ +worker[3] Using MTR_BUILD_THREAD 252, with reserved ports 12520..12529 +binlog.binlog_multi_engine [ skipped ] No ndbcluster tests(--skip-ndbcluster) +rpl.rpl_ssl 'row' [ pass ] 13976 +***Warnings generated in error logs during shutdown after running tests: rpl.rpl_ssl +rpl.rpl_ssl 'mix' [ pass ] 13308 +main.pool_of_threads w1 [ skipped ] Test requires: 'have_pool_of_threads' +------------------------------------------------------------ +The servers were restarted 613 times +mysql-test-run: *** ERROR: There were errors/warnings in server logs after running test cases. +All 1002 tests were successful. + +Errors/warnings were found in logfiles during server shutdown after running the +following sequence(s) of tests: + rpl.rpl_ssl +""") + self.assertEqual(self.observer.progresses, + map((lambda (x): ('tests', x)), [1,2])) + self.assertEqual(self.observer.testWarnLists, [["rpl.rpl_ssl"]]) + self.assertEqual(self.observer.testFails, []) + self.assertEqual(self.observer.text[1:], ["W:rpl_ssl"]) + + def test2(self): + self._logStdout(""" +Logging: mysql-test-run.pl --force --skip-ndb +============================================================================== +TEST RESULT TIME (ms) +------------------------------------------------------------ +binlog.binlog_multi_engine [ skipped ] No ndbcluster tests(--skip-ndbcluster) +rpl.rpl_sp 'mix' [ pass ] 8117 + +MTR's internal check of the test case 'rpl.rpl_sp' failed. +This is the diff of the states of the servers before and after the +test case was executed: +mysqltest: Logging to '/home/archivist/archivist-cnc/archivist-cnc/build/mysql-test/var/tmp/check-mysqld_2.log'. +--- /home/archivist/archivist-cnc/archivist-cnc/build/mysql-test/var/tmp/check-mysqld_2.result 2009-06-18 16:49:19.000000000 +0300 ++++ /home/archivist/archivist-cnc/archivist-cnc/build/mysql-test/var/tmp/check-mysqld_2.reject 2009-06-18 16:49:29.000000000 +0300 +@@ -523,7 +523,7 @@ + mysql.help_keyword 864336512 + mysql.help_relation 2554468794 + mysql.host 0 +-mysql.proc 3342691386 ++mysql.proc 3520745907 + +not ok + +rpl.rpl_sp_effects 'row' [ pass ] 3789 +rpl.rpl_temporary_errors 'mix' w2 [ fail ] + Test ended at 2009-06-18 16:21:28 + +CURRENT_TEST: rpl.rpl_temporary_errors +Retrying test, attempt(2/3)... + +***Warnings generated in error logs during shutdown after running tests: rpl.rpl_temporary_errors +rpl.rpl_temporary_errors 'mix' [ retry-pass ] 2108 +rpl.rpl_trunc_temp 'stmt' [ pass ] 2576 +main.information_schema [ pass ] 106092 +timer 5953: expired after 900 seconds +worker[1] Trying to dump core for [mysqltest - pid: 5975, winpid: 5975] +main.information_schema_all_engines [ fail ] timeout after 900 seconds + Test ended at 2009-06-18 18:37:25 +Retrying test, attempt(2/3)... + +***Warnings generated in error logs during shutdown after running tests: main.handler_myisam main.ctype_ujis_ucs2 main.ctype_recoding +main.information_schema_chmod [ pass ] 84 +rpl.rpl_circular_for_4_hosts 'stmt' [ pass ] 344547 +timer 21612: expired after 21600 seconds +Test suite timeout! Terminating... +mysql-test-run: *** ERROR: Not all tests completed +""") + self.assertEqual(self.observer.progresses, + map((lambda (x): ('tests', x)), [1,2,3,4,5,6,7,8])) + self.assertEqual(self.observer.testWarnLists, + [["rpl.rpl_temporary_errors"], + ["main.handler_myisam", "main.ctype_ujis_ucs2", "main.ctype_recoding"]]) + failtext1 = """rpl.rpl_temporary_errors 'mix' w2 [ fail ] + Test ended at 2009-06-18 16:21:28 + +CURRENT_TEST: rpl.rpl_temporary_errors +Retrying test, attempt(2/3)... + +""" + failtext2 = """main.information_schema_all_engines [ fail ] timeout after 900 seconds + Test ended at 2009-06-18 18:37:25 +Retrying test, attempt(2/3)... + +""" + self.assertEqual(self.observer.testFails, + [ ("rpl.rpl_temporary_errors", "mix", "fail", "", failtext1), + ("main.information_schema_all_engines", "", "fail", "timeout after 900 seconds", failtext2) + ]) + self.assertEqual(self.observer.text[1:], ["F:information_s...", "F:rpl_temporary...", "W:ctype_recoding", "W:ctype_ujis_ucs2", "W:handler_myisam"]) + class RemoteShellTest(unittest.TestCase): def testRepr(self): # Test for #352 @@ -142,3 +271,47 @@ class RemoteShellTest(unittest.TestCase): testval = repr(rsc) rsc = buildstep.RemoteShellCommand('.', 'make') testval = repr(rsc) + + +class SubunitLogObserver(subunitlogobserver.SubunitLogObserver): + """Subclassed to allow testing behaviour without a real buildstep.""" + + def __init__(self): + subunitlogobserver.SubunitLogObserver.__init__(self) + self.testFails = [] + self.testWarnLists = [] + # We don't have a buildstep in self.step. + # So we'll just install ourself there, so we can check the call of + # setProgress(). + # Same for self.step.step_status.setText() + self.step = self + self.step_status = self + self.progresses = [] + self.text = [] + + def setProgress(self, type, value): + self.progresses.append((type, value)) + + def setText(self, text): + self.text = text + + +class SubunitLogObserverTests(ObserverTestCase): + observer_cls = SubunitLogObserver + + def test1(self): + self._logStdout(""" +test: foo +success: foo +test: bar +failure: bar [ +string +] +test: gam +skip: gam +test: quux +xfail: quux +""") + self.assertEqual(self.observer.progresses, + [('tests', 1), ('tests', 2), ('tests failed', 1), ('tests', 3), + ('tests failed', 2), ('tests', 4)]) buildbot/test/test_buildstep.py @@ -13,7 +13,7 @@ from twisted.spread import pb from twisted.web.server import Site from twisted.web.distrib import ResourcePublisher from buildbot.process.builder import Builder -from buildbot.process.factory import BasicBuildFactory +from buildbot.process.factory import BasicBuildFactory, ArgumentsInTheWrongPlace from buildbot.changes.pb import PBChangeSource from buildbot.changes.mail import SyncmailMaildirSource from buildbot.steps.source import CVS, Darcs @@ -1209,6 +1209,28 @@ c['builders'] = [{'name':'builder1', 'slavename':'bot1', 'builddir':'workdir', 'factory':f1}] """ +cfg1_bad = \ +""" +from buildbot.process.factory import BuildFactory, s +from buildbot.steps.shell import ShellCommand +from buildbot.steps.source import Darcs +from buildbot.buildslave import BuildSlave +BuildmasterConfig = c = {} +c['slaves'] = [BuildSlave('bot1', 'pw1')] +c['schedulers'] = [] +c['slavePortnum'] = 9999 +f1 = BuildFactory([ShellCommand(command='echo yes'), + s(ShellCommand, command='old-style'), + ]) +# it should be this: +#f1.addStep(ShellCommand(command='echo args')) +# but an easy mistake is to do this: +f1.addStep(ShellCommand(), command='echo args') +# this test makes sure this error doesn't get ignored +c['builders'] = [{'name':'builder1', 'slavename':'bot1', + 'builddir':'workdir', 'factory':f1}] +""" + class Factories(unittest.TestCase): def printExpecting(self, factory, args): factory_keys = factory[1].keys() @@ -1231,6 +1253,7 @@ class Factories(unittest.TestCase): 'description': None, 'workdir': None, 'logfiles': {}, + 'lazylogfiles': False, 'usePTY': "slave-config", }) shell_args.update(kwargs) @@ -1248,6 +1271,7 @@ class Factories(unittest.TestCase): 'baseURL': None, 'defaultBranch': None, 'logfiles': {}, + 'lazylogfiles' : False, } darcs_args.update(kwargs) self.failUnlessIdentical(factory[0], Darcs) @@ -1270,6 +1294,10 @@ class Factories(unittest.TestCase): self.failUnlessExpectedShell(steps[3], defaults=False, command="echo old-style") + def testBadAddStepArguments(self): + m = BuildMaster(".") + self.failUnlessRaises(ArgumentsInTheWrongPlace, m.loadConfig, cfg1_bad) + def _loop(self, orig): step_class, kwargs = orig.getStepFactory() newstep = step_class(**kwargs) buildbot/test/test_config.py @@ -10,6 +10,9 @@ from twisted.python import log, logfile from buildbot.test.runutils import SignalMixin import os +from buildbot.test.runutils import RunMixin, rmtree +from buildbot.changes import changes + '''Testcases to verify that the --log-size and --log-count options to create-master and create-slave actually work. buildbot/test/test_limitlogs.py @@ -418,7 +418,7 @@ class Locks(RunMixin, unittest.TestCase): [("start", 1), ("done", 1), ("start", 2), ("done", 2)]) - def testLock1a(self): + def dont_testLock1a(self): ## disabled -- test itself is buggy # just like testLock1, but we reload the config file first, with a # change that causes full1b to be changed. This tickles a design bug # in which full1a and full1b wind up with distinct Lock instances. @@ -455,7 +455,7 @@ class Locks(RunMixin, unittest.TestCase): self.failUnless(self.events[:2] == [("start", 1), ("start", 2)] or self.events[:2] == [("start", 2), ("start", 1)]) - def testLock3(self): + def dont_testLock3(self): ## disabled -- test fails sporadically # two builds run on separate slaves with master-scoped locks should # not overlap self.control.getBuilder("full1c").requestBuild(self.req1) @@ -475,21 +475,22 @@ class Locks(RunMixin, unittest.TestCase): ("start", 1), ("done", 1)] ) - def testLock4(self): - self.control.getBuilder("full1a").requestBuild(self.req1) - self.control.getBuilder("full1c").requestBuild(self.req2) - self.control.getBuilder("full1d").requestBuild(self.req3) - d = defer.DeferredList([self.req1.waitUntilFinished(), - self.req2.waitUntilFinished(), - self.req3.waitUntilFinished()]) - d.addCallback(self._testLock4_1) - return d - - def _testLock4_1(self, res): - # full1a starts, then full1d starts (because they do not interfere). - # Once both are done, full1c can run. - self.failUnlessEqual(self.events, - [("start", 1), ("start", 3), - ("done", 1), ("done", 3), - ("start", 2), ("done", 2)]) + # This test has been disabled due to flakeyness/intermittentness +# def testLock4(self): +# self.control.getBuilder("full1a").requestBuild(self.req1) +# self.control.getBuilder("full1c").requestBuild(self.req2) +# self.control.getBuilder("full1d").requestBuild(self.req3) +# d = defer.DeferredList([self.req1.waitUntilFinished(), +# self.req2.waitUntilFinished(), +# self.req3.waitUntilFinished()]) +# d.addCallback(self._testLock4_1) +# return d +# +# def _testLock4_1(self, res): +# # full1a starts, then full1d starts (because they do not interfere). +# # Once both are done, full1c can run. +# self.failUnlessEqual(self.events, +# [("start", 1), ("start", 3), +# ("done", 1), ("done", 3), +# ("start", 2), ("done", 2)]) buildbot/test/test_locks.py @@ -30,7 +30,8 @@ c['slaves'] = [BuildSlave('bot1', 'sekrit')] c['schedulers'] = [] c['builders'] = [] c['builders'].append({'name':'quick', 'slavename':'bot1', - 'builddir': 'quickdir', 'factory': f1}) + 'builddir': 'quickdir', 'factory': f1, + 'slavebuilddir': 'slavequickdir'}) c['slavePortnum'] = 0 """ @@ -64,7 +65,7 @@ from buildbot.scheduler import Scheduler c['schedulers'] = [Scheduler('dummy', None, 0.1, ['dummy', 'dummy2'])] c['builders'].append({'name': 'dummy', 'slavename': 'bot1', - 'builddir': 'dummy', 'factory': f2}) + 'factory': f2}) c['builders'].append({'name': 'dummy2', 'slavename': 'bot1', 'builddir': 'dummy2', 'factory': f2}) """ @@ -72,8 +73,8 @@ c['builders'].append({'name': 'dummy2', 'slavename': 'bot1', config_2 = config_base + """ c['builders'] = [{'name': 'dummy', 'slavename': 'bot1', 'builddir': 'dummy1', 'factory': f2}, - {'name': 'testdummy', 'slavename': 'bot1', - 'builddir': 'dummy2', 'factory': f2, 'category': 'test'}] + {'name': 'test dummy', 'slavename': 'bot1', + 'factory': f2, 'category': 'test'}] """ config_3 = config_2 + """ @@ -86,7 +87,7 @@ c['builders'].append({'name': 'bdummy', 'slavename': 'bot1', config_4 = config_base + """ c['builders'] = [{'name': 'dummy', 'slavename': 'bot1', - 'builddir': 'dummy', 'factory': f2}] + 'slavebuilddir': 'sdummy', 'factory': f2}] """ config_4_newbasedir = config_4 + """ @@ -244,9 +245,9 @@ class BuilderNames(unittest.TestCase): m.readConfig = True self.failUnlessEqual(s.getBuilderNames(), - ["dummy", "testdummy", "adummy", "bdummy"]) + ["dummy", "test dummy", "adummy", "bdummy"]) self.failUnlessEqual(s.getBuilderNames(categories=['test']), - ["testdummy", "bdummy"]) + ["test dummy", "bdummy"]) class Disconnect(RunMixin, unittest.TestCase): @@ -263,7 +264,7 @@ class Disconnect(RunMixin, unittest.TestCase): m.readConfig = True m.startService() - self.failUnlessEqual(s.getBuilderNames(), ["dummy", "testdummy"]) + self.failUnlessEqual(s.getBuilderNames(), ["dummy", "test dummy"]) self.s1 = s1 = s.getBuilder("dummy") self.failUnlessEqual(s1.getName(), "dummy") self.failUnlessEqual(s1.getState(), ("offline", [])) @@ -531,7 +532,7 @@ class Disconnect2(RunMixin, unittest.TestCase): m.readConfig = True m.startService() - self.failUnlessEqual(s.getBuilderNames(), ["dummy", "testdummy"]) + self.failUnlessEqual(s.getBuilderNames(), ["dummy", "test dummy"]) self.s1 = s1 = s.getBuilder("dummy") self.failUnlessEqual(s1.getName(), "dummy") self.failUnlessEqual(s1.getState(), ("offline", [])) @@ -605,9 +606,10 @@ class Basedir(RunMixin, unittest.TestCase): self.bot = bot = self.slaves['bot1'].bot self.builder = builder = bot.builders.get("dummy") self.failUnless(builder) - self.failUnlessEqual(builder.builddir, "dummy") + # slavebuilddir value. + self.failUnlessEqual(builder.builddir, "sdummy") self.failUnlessEqual(builder.basedir, - os.path.join("slavebase-bot1", "dummy")) + os.path.join("slavebase-bot1", "sdummy")) d = self.master.loadConfig(config_4_newbasedir) d.addCallback(self._testChangeBuilddir_2) @@ -716,14 +718,15 @@ c['builders'] = [{'name': 'triggerer', 'slavename': 'bot1', self.assertEqual(self.getFlag("props"), "lit:dyn") class PropertyPropagation(RunMixin, TestFlagMixin, unittest.TestCase): - def setupTest(self, config, builders, checkFn): + def setupTest(self, config, builders, checkFn, changeProps={}): self.clearFlags() m = self.master m.loadConfig(config) m.readConfig = True m.startService() - c = changes.Change("bob", ["Makefile", "foo/bar.c"], "changed stuff") + c = changes.Change("bob", ["Makefile", "foo/bar.c"], "changed stuff", + properties=changeProps) m.change_svc.addChange(c) d = self.connectSlave(builders=builders) @@ -759,6 +762,30 @@ c['builders'] = [{'name': 'flagcolor', 'slavename': 'bot1', 'color=red sched=mysched') return self.setupTest(self.config_schprop, ['flagcolor'], _check) + config_changeprop = config_base + """ +from buildbot.scheduler import Scheduler +from buildbot.steps.dummy import Dummy +from buildbot.test.runutils import SetTestFlagStep +from buildbot.process.properties import WithProperties +c['schedulers'] = [ + Scheduler('mysched', None, 0.1, ['flagcolor'], properties={'color':'red'}), +] +factory = factory.BuildFactory([ + s(SetTestFlagStep, flagname='testresult', + value=WithProperties('color=%(color)s sched=%(scheduler)s prop1=%(prop1)s')), + ]) +c['builders'] = [{'name': 'flagcolor', 'slavename': 'bot1', + 'builddir': 'test', 'factory': factory}, + ] +""" + + def testChangeProp(self): + def _check(res): + self.failUnlessEqual(self.getFlag('testresult'), + 'color=blue sched=mysched prop1=prop1') + return self.setupTest(self.config_changeprop, ['flagcolor'], _check, + changeProps={'color': 'blue', 'prop1': 'prop1'}) + config_slaveprop = config_base + """ from buildbot.scheduler import Scheduler from buildbot.steps.dummy import Dummy buildbot/test/test_run.py @@ -108,7 +108,7 @@ class Scheduling(unittest.TestCase): s.addChange(c3) self.failUnlessEqual(s.importantChanges, [c1,c3]) - self.failUnlessEqual(s.unimportantChanges, [c2]) + self.failUnlessEqual(s.allChanges, [c1,c2,c3]) self.failUnless(s.timer) d = defer.Deferred() buildbot/test/test_scheduler.py @@ -391,6 +391,48 @@ class Mail(unittest.TestCase): self.assertFalse(mailer._shouldAttachLog('anything')) self.assertTrue(mailer._shouldAttachLog('something')) + def testShouldAttachPatches(self): + basedir = "test_should_attach_patches" + os.mkdir(basedir) + b1 = self.makeBuild(4, builder.FAILURE) + b1.setProperty('buildnumber', 1, 'Build') + b1.setText(["snarkleack", "polarization", "failed"]) + b1.blamelist = ["dev3", "dev3", "dev3", "dev4", + "Thomas_Walters"] + b1.source.changes = (Change(who = 'author1', files = ['file1'], comments = 'comment1', revision = 123), + Change(who = 'author2', files = ['file2'], comments = 'comment2', revision = 456)) + b1.testlogs = [MyLog(basedir, 'compile', "Compile log here\n"), + MyLog(basedir, 'test', "Test log here\nTest 1 failed\nTest 2 failed\nTest 3 failed\nTest 4 failed\n")] + b1.source.patch = (0, '--- /dev/null\n+++ a_file\n', None) + + mailer = MyMailer(fromaddr="buildbot@example.com", addPatch=True) + mailer.parent = self + mailer.status = self + self.messages = [] + mailer.buildFinished("builder1", b1, b1.results) + m,r = self.messages.pop() + self.assertTrue(m.is_multipart()) + self.assertEqual(len([True for i in m.walk()]), 3) + + mailer = MyMailer(fromaddr="buildbot@example.com", addPatch=False) + mailer.parent = self + mailer.status = self + self.messages = [] + mailer.buildFinished("builder1", b1, b1.results) + m,r = self.messages.pop() + self.assertFalse(m.is_multipart()) + self.assertEqual(len([True for i in m.walk()]), 1) + + mailer = MyMailer(fromaddr="buildbot@example.com") + mailer.parent = self + mailer.status = self + self.messages = [] + mailer.buildFinished("builder1", b1, b1.results) + m,r = self.messages.pop() + self.assertTrue(m.is_multipart()) + self.assertEqual(len([True for i in m.walk()]), 3) + + def testFailure(self): mailer = MyMailer(fromaddr="buildbot@example.com", mode="problem", extraRecipients=["recip@example.com", @@ -899,8 +941,26 @@ class Log(unittest.TestCase): return d testLargeSummary.timeout = 5 + def testLimit(self): + l = MyLog(self.basedir, "limit") + l.logMaxSize = 150 + for i in range(1000): + l.addStdout("Some data") + l.finish() + t = l.getText() + # Compare against 175 since we truncate logs based on chunks, so we may + # go slightly over the limit + self.failIf(len(t) > 175, "Text too long (%i)" % len(t)) + self.failUnless("truncated" in l.getTextWithHeaders(), + "No truncated message found") class CompressLog(unittest.TestCase): + # compression is not supported unless bz2 is installed + try: + import bz2 + except: + skip = "compression not supported (no bz2 module available)" + def testCompressLogs(self): bss = setupBuildStepStatus("test-compress") bss.build.builder.setLogCompressionLimit(1024) @@ -1320,8 +1380,10 @@ class ContactTester(unittest.TestCase): self.failUnlessEqual(irc.message, "", "No started notification with notify_events=['failed']") irc.message = "" + irc.channel.showBlameList = True irc.buildFinished(my_builder.getName(), my_build, None) - self.failUnlessEqual(irc.message, "build #862 of builder834 is complete: Failure [step1 step2] Build details are at http://myserver/mypath?build=765", "Finish notification generated on failure with notify_events=['failed']") + self.failUnlessEqual(irc.message, "build #862 of builder834 is complete: Failure [step1 step2] Build details are at http://myserver/mypath?build=765 blamelist: author1", "Finish notification generated on failure with notify_events=['failed']") + irc.channel.showBlameList = False irc.message = "" my_build.results = builder.SUCCESS @@ -1353,6 +1415,12 @@ class ContactTester(unittest.TestCase): self.failUnlessEqual(irc.message, "build #862 of builder834 is complete: Exception [step1 step2] Build details are at http://myserver/mypath?build=765", "Finish notification generated on failure with notify_events=['exception']") irc.message = "" + irc.channel.showBlameList = True + irc.buildFinished(my_builder.getName(), my_build, None) + self.failUnlessEqual(irc.message, "build #862 of builder834 is complete: Exception [step1 step2] Build details are at http://myserver/mypath?build=765 blamelist: author1", "Finish notification generated on failure with notify_events=['exception']") + irc.channel.showBlameList = False + + irc.message = "" my_build.results = builder.SUCCESS irc.buildFinished(my_builder.getName(), my_build, None) self.failUnlessEqual(irc.message, "", "No finish notification generated on success with notify_events=['exception']") @@ -1591,6 +1659,7 @@ class MyChannel: def __init__(self, notify_events = {}): self.notify_events = notify_events + self.showBlameList = False class MyContact(words.Contact): message = "" @@ -1608,6 +1677,31 @@ class MyContact(words.Contact): def send(self, msg): self.message += msg +class MyIrcStatusBot(words.IrcStatusBot): + def msg(self, dest, message): + self.message = ['msg', dest, message] + + def notice(self, dest, message): + self.message = ['notice', dest, message] + +class IrcStatusBotTester(unittest.TestCase): + def testMsgOrNotice(self): + channel = MyIrcStatusBot('alice', 'pa55w0od', ['#here'], + builder.SUCCESS, None, {}) + channel.msgOrNotice('bob', 'hello') + self.failUnlessEqual(channel.message, ['msg', 'bob', 'hello']) + + channel.msgOrNotice('#here', 'hello') + self.failUnlessEqual(channel.message, ['msg', '#here', 'hello']) + + channel.noticeOnChannel = True + + channel.msgOrNotice('bob', 'hello') + self.failUnlessEqual(channel.message, ['msg', 'bob', 'hello']) + + channel.msgOrNotice('#here', 'hello') + self.failUnlessEqual(channel.message, ['notice', '#here', 'hello']) + class StepStatistics(unittest.TestCase): def testStepStatistics(self): status = builder.BuildStatus(builder.BuilderStatus("test"), 123) buildbot/test/test_status.py @@ -14,6 +14,7 @@ # todo: test batched updates, by invoking remote_update(updates) instead of # statusUpdate(update). Also involves interrupted builds. +import sys import os from twisted.trial import unittest @@ -21,6 +22,7 @@ from twisted.internet import reactor, defer from buildbot.sourcestamp import SourceStamp from buildbot.process import buildstep, base, factory +from buildbot.process.properties import Properties, WithProperties from buildbot.buildslave import BuildSlave from buildbot.steps import shell, source, python, master from buildbot.status import builder @@ -134,6 +136,7 @@ class BuildStep(unittest.TestCase): 'want_stderr': 1, 'logfiles': {}, 'timeout': 10, + 'maxTime': None, 'usePTY': 'slave-config', 'env': None}) ] ) self.assertEqual(self.remote.events, self.expectedEvents) @@ -622,6 +625,112 @@ ending line results = step.evaluateCommand(cmd) self.failUnlessEqual(results, WARNINGS) + def testCompile4(self): + # Test suppression of warnings. + self.masterbase = "Warnings.testCompile4" + step = self.makeStep(shell.Compile, + warningPattern="^(.*?):([0-9]+): [Ww]arning: (.*)$", + warningExtractor=shell.Compile.warnExtractFromRegexpGroups, + directoryEnterPattern="make.*: Entering directory [\"`'](.*)['`\"]", + directoryLeavePattern="make.*: Leaving directory") + step.addSuppression([(r"/subdir/", r"xyzzy", None, None), + (r"foo.c", r".*", None, 20), + (r"foo.c", r".*", 200, None), + (r"foo.c", r".*", 50, 50), + (r"xxx", r".*", None, None), + ]) + log = step.addLog("stdio") + output = \ +"""Making all in . +make[1]: Entering directory `/abs/path/build' +foo.c:10: warning: `bar' defined but not used +foo.c:50: warning: `bar' defined but not used +make[2]: Entering directory `/abs/path/build/subdir' +baz.c:33: warning: `xyzzy' defined but not used +baz.c:34: warning: `magic' defined but not used +make[2]: Leaving directory `/abs/path/build/subdir' +foo.c:100: warning: `xyzzy' defined but not used +foo.c:200: warning: `bar' defined but not used +make[2]: Leaving directory `/abs/path/build' +""" + log.addStdout(output) + log.finish() + step.createSummary(log) + self.failUnlessEqual(step.getProperty("warnings-count"), 2) + logs = {} + for log in step.step_status.getLogs(): + logs[log.getName()] = log + self.failUnless("warnings" in logs) + lines = logs["warnings"].readlines() + self.failUnlessEqual(len(lines), 2) + self.failUnlessEqual(lines[0], "baz.c:34: warning: `magic' defined but not used\n") + self.failUnlessEqual(lines[1], "foo.c:100: warning: `xyzzy' defined but not used\n") + + cmd = buildstep.RemoteCommand(None, {}) + cmd.rc = 0 + results = step.evaluateCommand(cmd) + self.failUnlessEqual(results, WARNINGS) + + def filterArgs(self, args): + if "writer" in args: + args["writer"] = self.wrap(args["writer"]) + return args + + suppressionFileData = """ +# Sample suppressions file for testing + +/subdir/ : xyzzy +foo.c: .* : 0-20 +foo.c: .*: 200-10000 +foo.c :.*: 50 +xxx : .* +""" + def testCompile5(self): + # Test downloading warning suppression file from slave. + self.slavebase = "Warnings.testCompile5.slave" + self.masterbase = "Warnings.testCompile5.master" + sb = self.makeSlaveBuilder() + os.mkdir(os.path.join(self.slavebase, self.slavebuilderbase, + "build")) + output = \ +"""Making all in . +make[1]: Entering directory `/abs/path/build' +foo.c:10: warning: `bar' defined but not used +foo.c:50: warning: `bar' defined but not used +make[2]: Entering directory `/abs/path/build/subdir' +baz.c:33: warning: `xyzzy' defined but not used +baz.c:34: warning: `magic' defined but not used +make[2]: Leaving directory `/abs/path/build/subdir' +foo.c:100: warning: `xyzzy' defined but not used +foo.c:200: warning: `bar' defined but not used +make[2]: Leaving directory `/abs/path/build' +""" + printStatement = ('print """%s"""' % output) + step = self.makeStep(shell.Compile, + warningPattern="^(.*?):([0-9]+): [Ww]arning: (.*)$", + warningExtractor=shell.Compile.warnExtractFromRegexpGroups, + suppressionFile="warnings.supp", + command=[sys.executable, "-c", printStatement]) + slavesrc = os.path.join(self.slavebase, + self.slavebuilderbase, + "build", + "warnings.supp") + open(slavesrc, "w").write(self.suppressionFileData) + + d = self.runStep(step) + def _checkResult(result): + self.failUnlessEqual(step.getProperty("warnings-count"), 2) + logs = {} + for log in step.step_status.getLogs(): + logs[log.getName()] = log + self.failUnless("warnings" in logs) + lines = logs["warnings"].readlines() + self.failUnlessEqual(len(lines), 2) + self.failUnlessEqual(lines[0], "baz.c:34: warning: `magic' defined but not used\n") + self.failUnlessEqual(lines[1], "foo.c:100: warning: `xyzzy' defined but not used\n") + + d.addCallback(_checkResult) + return d class TreeSize(StepTester, unittest.TestCase): def testTreeSize(self): @@ -752,7 +861,9 @@ class MasterShellCommand(StepTester, unittest.TestCase): self.slavebase = "testMasterShellCommand.slave" self.masterbase = "testMasterShellCommand.master" sb = self.makeSlaveBuilder() - step = self.makeStep(master.MasterShellCommand, command=['echo', 'hi']) + step = self.makeStep(master.MasterShellCommand, command=['echo', + WithProperties("hi build-%(other)s.tar.gz")]) + step.build.setProperty("other", "foo", "test") # we can't invoke runStep until the reactor is started .. hence this # little dance @@ -764,7 +875,7 @@ class MasterShellCommand(StepTester, unittest.TestCase): def _check(results): self.failUnlessEqual(results, SUCCESS) logtxt = step.getLog("stdio").getText() - self.failUnlessEqual(logtxt.strip(), "hi") + self.failUnlessEqual(logtxt.strip(), "hi build-foo.tar.gz") d.addCallback(_check) reactor.callLater(0, d.callback, None) return d buildbot/test/test_steps.py @@ -90,6 +90,15 @@ diff -u -r1.1.1.1 subdir.c } """ +# Test buildbot try --root support. +subdir_diff = p0_diff.replace('subdir/subdir.c', 'subdir.c') + +# Test --patchlevel support. +p2_diff = p0_diff.replace('subdir/subdir.c', 'foo/bar/subdir/subdir.c') + +# Used in do_patch() test. +PATCHLEVEL0, SUBDIR_ROOT, PATCHLEVEL2 = range(3) + # this patch does not include the filename headers, so it is # patchlevel-neutral TRY_PATCH = ''' @@ -286,7 +295,8 @@ class BaseHelper: # you must call this from createRepository self.repbase = os.path.abspath(os.path.join("test_vc", "repositories")) - _makedirsif(self.repbase) + rmdirRecursive(self.repbase) + os.makedirs(self.repbase) def createRepository(self): # this will only be called once per process @@ -725,7 +735,7 @@ class VCBase(SignalMixin): #self.checkGotRevisionIsLatest(bs) # VC 'export' is not required to have a got_revision - def do_patch(self): + def do_patch(self, type): vctype = self.vctype args = self.helper.vcargs m = self.master @@ -736,12 +746,19 @@ class VCBase(SignalMixin): s += ", %s=%s" % (k, repr(v)) s += ")" self.config = config_vc % s - + if type == PATCHLEVEL0: + self.patch = (0, p0_diff) + elif type == SUBDIR_ROOT: + self.patch = (0, subdir_diff, 'subdir') + elif type == PATCHLEVEL2: + self.patch = (2, p2_diff) + else: + raise NotImplementedError m.loadConfig(self.config % "clobber") m.readConfig = True m.startService() - ss = SourceStamp(revision=self.helper.trunk[-1], patch=(0, p0_diff)) + ss = SourceStamp(revision=self.helper.trunk[-1], patch=self.patch) d = self.connectSlave() d.addCallback(lambda res: self.doBuild(ss=ss)) @@ -782,7 +799,7 @@ class VCBase(SignalMixin): return self._doPatch_3() def _doPatch_3(self, res=None): - ss = SourceStamp(revision=self.helper.trunk[-2], patch=(0, p0_diff)) + ss = SourceStamp(revision=self.helper.trunk[-2], patch=self.patch) d = self.doBuild(ss=ss) d.addCallback(self._doPatch_4) return d @@ -801,7 +818,7 @@ class VCBase(SignalMixin): # now check that we can patch a branch ss = SourceStamp(branch=self.helper.branchname, revision=self.helper.branch[-1], - patch=(0, p0_diff)) + patch=self.patch) d = self.doBuild(ss=ss) d.addCallback(self._doPatch_5) return d @@ -1040,6 +1057,8 @@ class VCBase(SignalMixin): def dumpPatch(self, patch): + # FIXME: This function is never called. + raise NotImplementedError # this exists to help me figure out the right 'patchlevel' value # should be returned by tryclient.getSourceStamp n = self.mktemp() @@ -1211,7 +1230,15 @@ class CVS(VCBase, unittest.TestCase): return d def testPatch(self): - d = self.do_patch() + d = self.do_patch(PATCHLEVEL0) + return d + + def testPatchSubDir(self): + d = self.do_patch(SUBDIR_ROOT) + return d + + def testPatchP2(self): + d = self.do_patch(PATCHLEVEL2) return d def testCheckoutBranch(self): @@ -1418,7 +1445,21 @@ class SVN(VCBase, unittest.TestCase): self.helper.vcargs = { 'baseURL': self.helper.svnurl + "/", 'defaultBranch': "sample/trunk", } - d = self.do_patch() + d = self.do_patch(PATCHLEVEL0) + return d + + def testPatchSubDir(self): + self.helper.vcargs = { 'baseURL': self.helper.svnurl + "/", + 'defaultBranch': "sample/trunk", + } + d = self.do_patch(SUBDIR_ROOT) + return d + + def testPatchP2(self): + self.helper.vcargs = { 'baseURL': self.helper.svnurl + "/", + 'defaultBranch': "sample/trunk", + } + d = self.do_patch(PATCHLEVEL2) return d def testCheckoutBranch(self): @@ -1607,7 +1648,21 @@ class P4(VCBase, unittest.TestCase): self.helper.vcargs = { 'p4port': self.helper.p4port, 'p4base': '//depot/', 'defaultBranch': 'trunk' } - d = self.do_patch() + d = self.do_patch(PATCHLEVEL0) + return d + + def testPatchSubDir(self): + self.helper.vcargs = { 'p4port': self.helper.p4port, + 'p4base': '//depot/', + 'defaultBranch': 'trunk' } + d = self.do_patch(SUBDIR_ROOT) + return d + + def testPatchP2(self): + self.helper.vcargs = { 'p4port': self.helper.p4port, + 'p4base': '//depot/', + 'defaultBranch': 'trunk' } + d = self.do_patch(PATCHLEVEL2) return d VCS.registerVC(P4.vc_name, P4Helper()) @@ -1724,7 +1779,19 @@ class Darcs(VCBase, unittest.TestCase): def testPatch(self): self.helper.vcargs = { 'baseURL': self.helper.darcs_base + "/", 'defaultBranch': "trunk" } - d = self.do_patch() + d = self.do_patch(PATCHLEVEL0) + return d + + def testPatchSubDir(self): + self.helper.vcargs = { 'baseURL': self.helper.darcs_base + "/", + 'defaultBranch': "trunk" } + d = self.do_patch(SUBDIR_ROOT) + return d + + def testPatchP2(self): + self.helper.vcargs = { 'baseURL': self.helper.darcs_base + "/", + 'defaultBranch': "trunk" } + d = self.do_patch(PATCHLEVEL2) return d def testCheckoutBranch(self): @@ -2026,7 +2093,19 @@ class Arch(VCBase, unittest.TestCase): def testPatch(self): self.helper.vcargs = {'url': self.helper.archrep, 'version': self.helper.defaultbranch } - d = self.do_patch() + d = self.do_patch(PATCHLEVEL0) + return d + + def testPatchSubDir(self): + self.helper.vcargs = {'url': self.helper.archrep, + 'version': self.helper.defaultbranch } + d = self.do_patch(SUBDIR_ROOT) + return d + + def testPatchP2(self): + self.helper.vcargs = {'url': self.helper.archrep, + 'version': self.helper.defaultbranch } + d = self.do_patch(PATCHLEVEL2) return d def testCheckoutBranch(self): @@ -2104,7 +2183,25 @@ class Bazaar(Arch): 'archive': self.helper.archname, 'version': self.helper.defaultbranch, } - d = self.do_patch() + d = self.do_patch(PATCHLEVEL0) + return d + + def testPatchSubDir(self): + self.helper.vcargs = {'url': self.helper.archrep, + # Baz adds the required 'archive' argument + 'archive': self.helper.archname, + 'version': self.helper.defaultbranch, + } + d = self.do_patch(SUBDIR_ROOT) + return d + + def testPatchP2(self): + self.helper.vcargs = {'url': self.helper.archrep, + # Baz adds the required 'archive' argument + 'archive': self.helper.archname, + 'version': self.helper.defaultbranch, + } + d = self.do_patch(PATCHLEVEL2) return d def testCheckoutBranch(self): @@ -2292,7 +2389,7 @@ class BzrHelper(BaseHelper): rep = self.rep_trunk else: rep = os.path.join(self.bzr_base, branch) - w = self.dovc(self.bzr_base, ["checkout", rep, workdir]) + w = self.dovc(self.bzr_base, ["clone", rep, workdir]) yield w; w.getResult() open(os.path.join(workdir, "subdir", "subdir.c"), "w").write(TRY_C) vc_try_checkout = deferredGenerator(vc_try_checkout) @@ -2321,11 +2418,25 @@ class Bzr(VCBase, unittest.TestCase): # TODO: testRetry has the same problem with Bzr as it does for # Arch return d + # Bzr is *slow*, and the testCheckout step takes a *very* long time + testCheckout.timeout = 480 def testPatch(self): self.helper.vcargs = { 'baseURL': self.helper.bzr_base + "/", 'defaultBranch': "trunk" } - d = self.do_patch() + d = self.do_patch(PATCHLEVEL0) + return d + + def testPatchSubDir(self): + self.helper.vcargs = { 'baseURL': self.helper.bzr_base + "/", + 'defaultBranch': "trunk" } + d = self.do_patch(SUBDIR_ROOT) + return d + + def testPatchP2(self): + self.helper.vcargs = { 'baseURL': self.helper.bzr_base + "/", + 'defaultBranch': "trunk" } + d = self.do_patch(PATCHLEVEL2) return d def testCheckoutBranch(self): @@ -2439,7 +2550,7 @@ class MercurialHelper(BaseHelper): yield w; w.getResult() w = self.dovc(tmp, "add") yield w; w.getResult() - w = self.dovc(tmp, ['commit', '-m', 'initial_import']) + w = self.dovc(tmp, ['commit', '-m', 'initial_import', '--user' ,'bbtests@localhost' ]) yield w; w.getResult() w = self.dovc(tmp, ['push', self.rep_trunk]) # note that hg-push does not actually update the working directory @@ -2449,7 +2560,7 @@ class MercurialHelper(BaseHelper): self.addTrunkRev(self.extract_id(out)) self.populate_branch(tmp) - w = self.dovc(tmp, ['commit', '-m', 'commit_on_branch']) + w = self.dovc(tmp, ['commit', '-m', 'commit_on_branch', '--user' ,'bbtests@localhost' ]) yield w; w.getResult() w = self.dovc(tmp, ['push', self.rep_branch]) yield w; w.getResult() @@ -2472,7 +2583,7 @@ class MercurialHelper(BaseHelper): # force the mtime forward a little bit future = time.time() + 2*self.version os.utime(version_c_filename, (future, future)) - w = self.dovc(tmp, ['commit', '-m', 'revised_to_%d' % self.version]) + w = self.dovc(tmp, ['commit', '-m', 'revised_to_%d' % self.version, '--user' ,'bbtests@localhost' ]) yield w; w.getResult() w = self.dovc(tmp, ['push', self.rep_trunk]) yield w; w.getResult() @@ -2545,7 +2656,19 @@ class Mercurial(VCBase, unittest.TestCase): def testPatch(self): self.helper.vcargs = { 'baseURL': self.helper.hg_base + "/", 'defaultBranch': "trunk" } - d = self.do_patch() + d = self.do_patch(PATCHLEVEL0) + return d + + def testPatchSubDir(self): + self.helper.vcargs = { 'baseURL': self.helper.hg_base + "/", + 'defaultBranch': "trunk" } + d = self.do_patch(SUBDIR_ROOT) + return d + + def testPatchP2(self): + self.helper.vcargs = { 'baseURL': self.helper.hg_base + "/", + 'defaultBranch': "trunk" } + d = self.do_patch(PATCHLEVEL2) return d def testCheckoutBranch(self): @@ -2741,7 +2864,17 @@ class MercurialInRepo(Mercurial): def testPatch(self): self.helper.vcargs = self.default_args() - d = self.do_patch() + d = self.do_patch(PATCHLEVEL0) + return d + + def testPatchSubDir(self): + self.helper.vcargs = self.default_args() + d = self.do_patch(SUBDIR_ROOT) + return d + + def testPatchP2(self): + self.helper.vcargs = self.default_args() + d = self.do_patch(PATCHLEVEL2) return d def testCheckoutBranch(self): @@ -2880,6 +3013,12 @@ class GitHelper(BaseHelper): w = self.dovc(self.repbase, "init", env=env) yield w; w.getResult() + # NetBSD pkgsrc uses templates that stupidly enable the update hook, requiring + # a non-default description. This is broken, but easily worked around. + # http://www.netbsd.org/cgi-bin/query-pr-single.pl?number=41683 + descrfile = os.path.join(self.gitrepo, "description") + open(descrfile, "w").write("NetBSD workaround") + self.populate(tmp) w = self.dovc(tmp, "init") yield w; w.getResult() @@ -2983,7 +3122,19 @@ class Git(VCBase, unittest.TestCase): def testPatch(self): self.helper.vcargs = { 'repourl': self.helper.gitrepo, 'branch': "master" } - d = self.do_patch() + d = self.do_patch(PATCHLEVEL0) + return d + + def testPatchSubDir(self): + self.helper.vcargs = { 'repourl': self.helper.gitrepo, + 'branch': "master" } + d = self.do_patch(SUBDIR_ROOT) + return d + + def testPatchP2(self): + self.helper.vcargs = { 'repourl': self.helper.gitrepo, + 'branch': "master" } + d = self.do_patch(PATCHLEVEL2) return d def testCheckoutBranch(self): buildbot/test/test_vc.py @@ -5,7 +5,7 @@ github_buildbot.py is based on git_buildbot.py github_buildbot.py will determine the repository information from the JSON HTTP POST it receives from github.com and build the appropriate repository. If your github repository is private, you must add a ssh key to the github -repository for the user who initiated the buildslave. +repository for the user who initiated the build on the buildslave. """ @@ -55,7 +55,7 @@ class GitHubBuildBot(resource.Resource): for msg in traceback.format_exception(*sys.exc_info()): logging.error(msg.strip()) - def process_change(self, payload): + def process_change(self, payload, user, repo, github_url): """ Consumes the JSON as a python object and actually starts the build. @@ -85,12 +85,15 @@ class GitHubBuildBot(resource.Resource): files.extend(commit['modified']) files.extend(commit['removed']) change = {'revision': commit['id'], + 'revlink': commit['url'], 'comments': commit['message'], 'branch': branch, 'who': commit['author']['name'] + " <" + commit['author']['email'] + ">", 'files': files, 'links': [commit['url']], + 'properties': {'repository': + self.repo_url(user, repo, github_url)}, } changes.append(change) @@ -152,7 +155,7 @@ def main(): parser.add_option("-p", "--port", help="Port the HTTP server listens to for the GitHub Service Hook" - + " [default: %default]", default=4000, dest="port") + + " [default: %default]", default=4000, type=int, dest="port") parser.add_option("-m", "--buildmaster", help="Buildbot Master host and port. ie: localhost:9989 [default:" @@ -169,7 +172,9 @@ def main(): + " %default]", default='warn', dest="level") parser.add_option("-g", "--github", - help="The github serve [default: %default]", default='github.com', + help="The github server. Changing this is useful if you've specified" + + " a specific HOST handle in ~/.ssh/config for github " + + "[default: %default]", default='github.com', dest="github") (options, _) = parser.parse_args() contrib/github_buildbot.py @@ -162,7 +162,7 @@ class ChangeSender: who = commands.getoutput('svnlook author %s "%s"' % (rev_arg, repo)) revision = opts.get('revision') if revision is not None: - revision = int(revision) + revision = str(int(revision)) # see if we even need to notify buildbot by looking at filters first changestring = '\n'.join(changed) contrib/svn_buildbot.py @@ -223,7 +223,9 @@ Simple ShellCommand Subclasses * Test:: * TreeSize:: * PerlModuleTest:: +* Testing with mysql-test-run:: * SetProperty:: +* SubunitShellCommand:: Python BuildSteps @@ -1633,6 +1635,12 @@ branch='buildbot--usebranches--0', files=['buildbot/master.py'] branch='warner-newfeature', files=['src/foo.c'] @end table +@heading Build Properties + +A Change may have one or more properties attached to it, usually specified +through the Force Build form or @pxref{sendchange}. Properties are discussed +in detail in the @pxref{Build Properties} section. + @heading Links @c TODO: who is using 'links'? how is it being used? @@ -1746,11 +1754,13 @@ SourceStamp, which will result in same behavior as this. This builds the most recent code on the given BRANCH. Again, this is generally triggered by a user request or Periodic build. -@item (revision=REV, changes=None, patch=(LEVEL, DIFF)) +@item (revision=REV, changes=None, patch=(LEVEL, DIFF, SUBDIR_ROOT)) This checks out the tree at the given revision REV, then applies a -patch (using @code{patch -pLEVEL <DIFF}). The @ref{try} feature uses -this kind of @code{SourceStamp}. If @code{patch} is None, the patching -step is bypassed. +patch (using @code{patch -pLEVEL <DIFF}) from inside the relative +directory SUBDIR_ROOT. Item SUBDIR_ROOT is optional and defaults to the +builder working directory. The @ref{try} feature uses this kind of +@code{SourceStamp}. If @code{patch} is None, the patching step is +bypassed. @end table @@ -1836,7 +1846,7 @@ f.addStep(Compile()) c['builders'] = [ @{'name': 'test', 'slavenames': ['slave1', 'slave2', 'slave3', 'slave4', 'slave5', 'slave6'], - 'builddir': 'test', 'factory': f', + 'builddir': 'test', 'factory': f', 'slavebuilddir': 'test', 'env': @{'PATH': '/opt/local/bin:/opt/app/bin:/usr/local/bin:/usr/bin'@}@} @end example @@ -1998,6 +2008,9 @@ These properties apply to all builds. @item schedulers -- A scheduler can specify properties available to all the builds it starts. +@item changes -- +A change can have properties attached to it. These are usually specified +through Force Build or sendchange. @item buildslaves -- A buildslave can pass properties on to the builds it performs. @item builds -- @@ -2255,6 +2268,20 @@ be a reasonable default on most file systems. This setting has no impact on status plugins, and merely affects the required disk space on the master for build logs. +@bcindex c['logMaxSize'] +The @code{logMaxSize} parameter sets an upper limit (in bytes) to how large +logs from an individual build step can be. The default value is None, meaning +no upper limit to the log size. Any output exceeding @code{logMaxSize} will be +truncated, and a message to this effect will be added to the log's HEADER +channel. + +@bcindex c['logMaxTailSize'] +If @code{logMaxSize} is set, and the output from a step exceeds the maximum, +the @code{logMaxTailSize} parameter controls how much of the end of the build +log will be kept. The effect of setting this parameter is that the log will +contain the first @code{logMaxSize} bytes and the last @code{logMaxTailSize} +bytes of output. Don't set this value too high, as the the tail of the log is +kept in memory. @node Change Sources and Schedulers, Merging BuildRequests, Defining the Project, Configuration @section Change Sources and Schedulers @@ -2442,6 +2469,11 @@ A callable which takes one argument, a Change instance, and returns it is not. Unimportant Changes are accumulated until the build is triggered by an important change. The default value of None means that all Changes are important. + +@item categories +A list of categories of changes that this scheduler will respond to. If this +is specified, then any non-matching changes are ignored. + @end table @node Dependent Scheduler, Periodic Scheduler, AnyBranchScheduler, Change Sources and Schedulers @@ -2482,7 +2514,7 @@ build, then those changes will be included in the downstream build. See the @pxref{Triggerable Scheduler} for a more flexible dependency mechanism that can avoid this problem. -The arguments to this scheduler are: +The keyword arguments to this scheduler are: @table @code @item name @@ -2502,9 +2534,9 @@ Example: from buildbot import scheduler tests = scheduler.Scheduler("just-tests", None, 5*60, ["full-linux", "full-netbsd", "full-OSX"]) -package = scheduler.Dependent("build-package", - tests, # upstream scheduler -- no quotes! - ["make-tarball", "make-deb", "make-rpm"]) +package = scheduler.Dependent(name="build-package", + upstream=tests, # <- no quotes! + builderNames=["make-tarball", "make-deb", "make-rpm"]) c['schedulers'] = [tests, package] @end example @@ -2700,7 +2732,7 @@ test = scheduler.Triggerable(name="distributed-test", package = scheduler.Triggerable(name="package-all-platforms", builderNames=["package-all-platforms"]) -c['schedulers'] = [checkin, nightly, build, test, package] +c['schedulers'] = [mktarball, checkin, nightly, build, test, package] # on checkin, make a tarball, build it, and test it checkin_factory = factory.BuildFactory() @@ -3293,12 +3325,6 @@ occurs on some of the buildslaves and not the others. Different platforms, operating systems, versions of major programs or libraries, all these things mean you should use separate Builders. -@item builddir -This specifies the name of a subdirectory (under the base directory) -in which everything related to this builder will be placed. On the -buildmaster, this holds build status information. On the buildslave, -this is where checkouts, compiles, and tests are run. - @item factory This is a @code{buildbot.process.factory.BuildFactory} instance which controls how the build is performed. Full details appear in their own @@ -3312,6 +3338,22 @@ Other optional keys may be set on each Builder: @table @code +@item builddir +Specifies the name of a subdirectory (under the base directory) in which +everything related to this builder will be placed on the buildmaster. +This holds build status information. If not set, defaults to @code{name} +with some characters escaped. Each builder must have a unique build +directory. + +@item slavebuilddir +Specifies the name of a subdirectory (under the base directory) in which +everything related to this builder will be placed on the buildslave. +This is where checkouts, compiles, and tests are run. If not set, +defaults to @code{builddir}. If a slave is connected to multiple builders +that shares the same @code{slavebuilddir}, make sure the slave is set to +run one build at a time or ensure this is fine to run multiple builds from +the same directory simultaneously. + @item category If provided, this is a string that identifies a category for the builder to be a part of. Status clients can limit themselves to a @@ -5655,10 +5697,19 @@ f.addStep(ShellCommand( @end example +@item lazylogfiles +If set to @code{True}, logfiles will be tracked lazily, meaning that they will +only be added when and if something is written to them. This can be used to +suppress the display of empty or missing log files. The default is @code{False}. + + @item timeout if the command fails to produce any output for this many seconds, it is assumed to be locked up and will be killed. +@item maxTime +if the command takes longer than this many seconds, it will be killed. + @item description This will be used to describe the command (on the Waterfall display) while the command is still running. It should be a single @@ -5705,7 +5756,9 @@ file less verbose. * Test:: * TreeSize:: * PerlModuleTest:: +* Testing with mysql-test-run:: * SetProperty:: +* SubunitShellCommand:: @end menu @node Configure, Compile, Simple ShellCommand Subclasses, Simple ShellCommand Subclasses @@ -5748,6 +5801,66 @@ The @code{warningPattern=} can also be a pre-compiled python regexp object: this makes it possible to add flags like @code{re.I} (to use case-insensitive matching). +The @code{suppressionFile=} argument can be specified as the (relative) path +of a file inside the workdir defining warnings to be suppressed from the +warning counting and log file. The file will be uploaded to the master from +the slave before compiling, and any warning matched by a line in the +suppression file will be ignored. This is useful to accept certain warnings +(eg. in some special module of the source tree or in cases where the compiler +is being particularly stupid), yet still be able to easily detect and fix the +introduction of new warnings. + +The file must contain one line per pattern of warnings to ignore. Empty lines +and lines beginning with @code{#} are ignored. Other lines must consist of a +regexp matching the file name, followed by a colon (@code{:}), followed by a +regexp matching the text of the warning. Optionally this may be followed by +another colon and a line number range. For example: + +@example +# Sample warning suppression file + +mi_packrec.c : .*result of 32-bit shift implicitly converted to 64 bits.* : 560-600 +DictTabInfo.cpp : .*invalid access to non-static.* +kernel_types.h : .*only defines private constructors and has no friends.* : 51 +@end example + +If no line number range is specified, the pattern matches the whole file; if +only one number is given it matches only on that line. + +The default warningPattern regexp only matches the warning text, so line +numbers and file names are ignored. To enable line number and file name +matching, privide a different regexp and provide a function (callable) as the +argument of @code{warningExtractor=}. The function is called with three +arguments: the BuildStep object, the line in the log file with the warning, +and the @code{SRE_Match} object of the regexp search for @code{warningPattern}. It +should return a tuple @code{(filename, linenumber, warning_test)}. For +example: + +@example +f.addStep(Compile(command=["make"], + warningPattern="^(.*?):([0-9]+): [Ww]arning: (.*)$", + warningExtractor=Compile.warnExtractFromRegexpGroups, + suppressionFile="support-files/compiler_warnings.supp")) +@end example + +(@code{Compile.warnExtractFromRegexpGroups} is a pre-defined function that +returns the filename, linenumber, and text from groups (1,2,3) of the regexp +match). + +In projects with source files in multiple directories, it is possible to get +full path names for file names matched in the suppression file, as long as the +build command outputs the names of directories as they are entered into and +left again. For this, specify regexps for the arguments +@code{directoryEnterPattern=} and @code{directoryLeavePattern=}. The +@code{directoryEnterPattern=} regexp should return the name of the directory +entered into in the first matched group. The defaults, which are suitable for +GNU Make, are these: + +@example +directoryEnterPattern = "make.*: Entering directory [\"`'](.*)['`\"]" +directoryLeavePattern = "make.*: Leaving directory" +@end example + (TODO: this step needs to be extended to look for GCC error messages as well, and collect them into a separate logfile, along with the source code filenames involved). @@ -5771,7 +5884,7 @@ of the code tree. It puts the size (as a count of 1024-byte blocks, aka 'KiB' or 'kibibytes') on the step's status text, and sets a build property named 'tree-size-KiB' with the same value. -@node PerlModuleTest, SetProperty, TreeSize, Simple ShellCommand Subclasses +@node PerlModuleTest, Testing with mysql-test-run, TreeSize, Simple ShellCommand Subclasses @subsubsection PerlModuleTest @bsindex buildbot.steps.shell.PerlModuleTest @@ -5780,7 +5893,95 @@ This is a simple command that knows how to run tests of perl modules. It parses the output to determine the number of tests passed and failed and total number executed, saving the results for later query. -@node SetProperty, , PerlModuleTest, Simple ShellCommand Subclasses +@node Testing with mysql-test-run, SetProperty, PerlModuleTest, Simple ShellCommand Subclasses +@subsection Testing with mysql-test-run + +The @code{process.mtrlogobserver.MTR} class is a subclass of @code{Test} +(@ref{Test}). It is used to run test suites using the mysql-test-run program, +as used in MySQL, Drizzle, MariaDB, and MySQL storage engine plugins. + +The shell command to run the test suite is specified in the same way as for +the Test class. The MTR class will parse the output of running the test suite, +and use the count of tests executed so far to provide more accurate completion +time estimates. Any test failures that occur during the test are summarized on +the Waterfall Display. + +Server error logs are added as additional log files, useful to debug test +failures. + +Optionally, data about the test run and any test failures can be inserted into +a database for further analysis and report generation. To use this facility, +create an instance of @code{twisted.enterprise.adbapi.ConnectionPool} with +connections to the database. The necessary tables can be created automatically +by setting @code{autoCreateTables} to @code{True}, or manually using the SQL +found in the @file{mtrlogobserver.py} source file. + +One problem with specifying a database is that each reload of the +configuration will get a new instance of @code{ConnectionPool} (even if the +connection parameters are the same). To avoid that Buildbot thinks the builder +configuration has changed because of this, use the +@code{process.mtrlogobserver.EqConnectionPool} subclass of +@code{ConnectionPool}, which implements an equiality operation that avoids +this problem. + +Example use: + +@example +from buildbot.process.mtrlogobserver import MTR, EqConnectionPool +myPool = EqConnectionPool("MySQLdb", "host", "buildbot", "password", "db") +myFactory.addStep(MTR(workdir="mysql-test", dbpool=myPool, + command=["perl", "mysql-test-run.pl", "--force"])) +@end example + +@code{MTR} arguments: + +@table @code + +@item textLimit +Maximum number of test failures to show on the waterfall page (to not flood +the page in case of a large number of test failures. Defaults to 5. + +@item testNameLimit +Maximum length of test names to show unabbreviated in the waterfall page, to +avoid excessive column width. Defaults to 16. + +@item parallel +Value of @code{--parallel} option used for mysql-test-run.pl (number of processes +used to run the test suite in parallel). Defaults to 4. This is used to +determine the number of server error log files to download from the +slave. Specifying a too high value does not hurt (as nonexisting error logs +will be ignored), however if using @code{--parallel} value greater than the default +it needs to be specified, or some server error logs will be missing. + +@item dbpool +An instance of twisted.enterprise.adbapi.ConnectionPool, or None. Defaults to +None. If specified, results are inserted into the database using the +ConnectionPool. + +@item autoCreateTables +Boolean, defaults to False. If True (and @code{dbpool} is specified), the +necessary database tables will be created automatically if they do not exist +already. Alternatively, the tables can be created manually from the SQL +statements found in the mtrlogobserver.py source file. + +@item test_type +Short string that will be inserted into the database in the row for the test +run. Defaults to the empty string, but can be specified to identify different +types of test runs. + +@item test_info +Descriptive string that will be inserted into the database in the row for the test +run. Defaults to the empty string, but can be specified as a user-readable +description of this particular test run. + +@item mtr_subdir +The subdirectory in which to look for server error log files. Defaults to +``mysql-test'', which is usually correct. WithProperties is supported. + +@end table + + +@node SetProperty, SubunitShellCommand , Testing with mysql-test-run, Simple ShellCommand Subclasses @subsubsection SetProperty @bsindex buildbot.steps.shell.SetProperty @@ -5823,6 +6024,21 @@ f.addStep(SetProperty( Then @code{my_extract} will see @code{stdout="output1\noutput2\n"} and @code{stderr="error\n"}. +@node SubunitShellCommand, , SetProperty, Simple ShellCommand Subclasses +@subsubsection SubunitShellCommand + +@bsindex buildbot.process.subunitlogger.SubunitShellCommand + +This buildstep is similar to ShellCommand, except that it runs the log content +through a subunit filter to extract test and failure counts. + +@example +f.addStep(SubunitShellCommand(command="make test")) +@end example + +This runs @code{make test} and filters it through subunit. The 'tests' and +'test failed' progress metrics will now accumulate test data from the test run. + @node Python BuildSteps, Transferring Files, Simple ShellCommand Subclasses, Build Steps @subsection Python BuildSteps @@ -6641,7 +6857,7 @@ build number. @example class TestWithCodeCoverage(BuildStep): command = ["make", "test", - WithProperties("buildnum=%s" % "buildnumber")] + WithProperties("buildnum=%s", "buildnumber")] def createSummary(self, log): buildnumber = self.getProperty("buildnumber") @@ -6655,7 +6871,7 @@ output by the build process itself: @example class TestWithCodeCoverage(BuildStep): command = ["make", "test", - WithProperties("buildnum=%s" % "buildnumber")] + WithProperties("buildnum=%s", "buildnumber")] def createSummary(self, log): output = StringIO(log.getText()) @@ -7615,6 +7831,40 @@ of the page, and the build hosts are listed across the top. It accepts the same query arguments. The exception being that instead of ``width'' the argument is named ``length.'' +@item /console + +EXPERIMENTAL: This provides a developer-oriented display of the the last +changes and how they affected the builders. + +It allows a developer to quickly see the status of each builder for the +first build including his or her change. A green box means that the change +succeeded for all the steps for a given builder. A red box means that +the changed introduced a new regression on a builder. An orange box +means that at least one of the test failed, but it was also failing +in the previous build, so it is not possible to see if there was any +regressions from this change. Finally a yellow box means that the test +is in progress. + +By adding one or more ``builder='' query arguments, the Console view is +restricted to only showing information about the given Builders. By +adding one or more ``branch='' query arguments, the display is +restricted to showing information about the given branches. In +addition, adding one or more ``category='' query arguments to the URL +will limit the display to Builders that were defined with one of the +given categories. + +By adding one or more ``name='' query arguments, the console view is +restricted to only showing changes made by the given users. + +NOTE: To use this page, your buildbot.css file in public_html +must be the one found in buildbot/status/web/extended.css. + +The console view is still in development. At this moment it supports +only the source control managers that have an integer based revision id, +like svn. It also has some issues with displaying multiple braches at the +same time. If you do have multiple branches, you should use the +``branch='' query argument. + @item /rss This provides a rss feed summarizing all failed builds. The same @@ -7956,6 +8206,11 @@ builders or categories, but not both. messages. These can be quite large. This can also be set to a list of log names, to send a subset of the logs. Defaults to False. +@item addPatch +(boolean). If True, include the patch content if a patch was present. +Patches are usually used on a Try server. +Defaults to True. + @item relayhost (string). The host to which the outbound SMTP connection should be made. Defaults to 'localhost' @@ -8767,6 +9022,11 @@ This context file can be a couple of kilobytes long, spanning a couple lines per patch, and would be a hassle to pass as a command-line argument. +@item --property +This parameter is used to set a property on the Change generated by sendchange. +Properties are specified as a name:value pair, separated by a colon. You may +specify many properties by passing this parameter multiple times. + @item --comments This provides the change comments as a single argument. You may want to use @option{--logfile} instead. docs/buildbot.texinfo 84a0c8acccbf6074cd62904542b6467bb75741d4 2b00a49451bad833f609b383747dcd1f9f947f3f Matt Heitzenroder mheitzenroder@gmail.com http://github.com/djmitche/buildbot/commit/1688df9a0576dbcf63be2bb9c81d8898c393d830 1688df9a0576dbcf63be2bb9c81d8898c393d830 2009-11-04T18:22:38-08:00 2009-11-04T18:22:38-08:00 simplified the code so it doesn't have to clone locally 696d6e1ac6d21924ce0c10baa9d38b21e8b9b99d Matt Heitzenroder mheitzenroder@gmail.com