diff --git a/.gitignore b/.gitignore index 80390732195..8e9dffb034c 100644 --- a/.gitignore +++ b/.gitignore @@ -39,3 +39,5 @@ apidocs/reference.tgz common/googlecode_upload.py master/docs/tutorial/_build _build +.tox +master/setuptools_trial* diff --git a/master/MANIFEST.in b/master/MANIFEST.in index d0cd257f8f6..6e30f3be76d 100644 --- a/master/MANIFEST.in +++ b/master/MANIFEST.in @@ -1,4 +1,4 @@ -include MANIFEST.in README NEWS CREDITS COPYING UPGRADING +include MANIFEST.in README CREDITS COPYING UPGRADING include docs/examples/*.cfg include docs/conf.py diff --git a/master/buildbot/buildslave.py b/master/buildbot/buildslave.py index f7c39047532..7c605713887 100644 --- a/master/buildbot/buildslave.py +++ b/master/buildbot/buildslave.py @@ -754,6 +754,7 @@ class AbstractLatentBuildSlave(AbstractBuildSlave): substantiated = False substantiation_deferred = None substantiation_build = None + insubstantiating = False build_wait_timer = None _shutdown_callback_handle = None @@ -824,7 +825,7 @@ def clean_up(failure): return d def attached(self, bot): - if self.substantiation_deferred is None: + if self.substantiation_deferred is None and self.build_wait_timeout >= 0: msg = 'Slave %s received connection while not trying to ' \ 'substantiate. Disconnecting.' % (self.slavename,) log.msg(msg) @@ -866,6 +867,11 @@ def _substantiation_failed(self, failure): subject = "Buildbot: buildslave %s never substantiated" % self.slavename return self._mail_missing_message(subject, text) + def canStartBuild(self): + if self.insubstantiating: + return False + return AbstractBuildSlave.canStartBuild(self) + def buildStarted(self, sb): assert self.substantiated self._clearBuildWaitTimer() @@ -876,7 +882,10 @@ def buildFinished(self, sb): self.building.remove(sb.builder_name) if not self.building: - self._setBuildWaitTimer() + if self.build_wait_timeout == 0: + self.insubstantiate() + else: + self._setBuildWaitTimer() def _clearBuildWaitTimer(self): if self.build_wait_timer is not None: @@ -886,10 +895,14 @@ def _clearBuildWaitTimer(self): def _setBuildWaitTimer(self): self._clearBuildWaitTimer() + if self.build_wait_timeout < 0: + return self.build_wait_timer = reactor.callLater( self.build_wait_timeout, self._soft_disconnect) + @defer.inlineCallbacks def insubstantiate(self, fast=False): + self.insubstantiating = True self._clearBuildWaitTimer() d = self.stop_instance(fast) if self._shutdown_callback_handle is not None: @@ -898,9 +911,13 @@ def insubstantiate(self, fast=False): reactor.removeSystemEventTrigger(handle) self.substantiated = False self.building.clear() # just to be sure - return d + yield d + self.insubstantiating = False def _soft_disconnect(self, fast=False): + if not self.build_wait_timeout < 0: + return AbstractBuildSlave.disconnect(self) + d = AbstractBuildSlave.disconnect(self) if self.slave is not None: # this could be called when the slave needs to shut down, such as diff --git a/master/buildbot/changes/base.py b/master/buildbot/changes/base.py index 07eefeedbcc..3af353e9b1e 100644 --- a/master/buildbot/changes/base.py +++ b/master/buildbot/changes/base.py @@ -35,6 +35,8 @@ class PollingChangeSource(ChangeSource): Utility subclass for ChangeSources that use some kind of periodic polling operation. Subclasses should define C{poll} and set C{self.pollInterval}. The rest is taken care of. + + Any subclass will be available via the "poller" webhook. """ pollInterval = 60 @@ -42,6 +44,24 @@ class PollingChangeSource(ChangeSource): _loop = None + def __init__(self, name=None, pollInterval=60*10): + if name: + self.setName(name) + self.pollInterval = pollInterval + + self.doPoll = util.misc.SerializedInvocation(self.doPoll) + + def doPoll(self): + """ + This is the method that is called by LoopingCall to actually poll. + It may also be called by change hooks to request a poll. + It is serialiazed - if you call it while a poll is in progress + then the 2nd invocation won't start until the 1st has finished. + """ + d = defer.maybeDeferred(self.poll) + d.addErrback(log.err, 'while polling for changes') + return d + def poll(self): """ Perform the polling operation, and return a deferred that will fire @@ -49,22 +69,27 @@ def poll(self): method will be called again after C{pollInterval} seconds. """ + def startLoop(self): + self._loop = task.LoopingCall(self.doPoll) + self._loop.start(self.pollInterval, now=False) + + def stopLoop(self): + if self._loop and self._loop.running: + self._loop.stop() + self._loop = None + def startService(self): ChangeSource.startService(self) - def do_poll(): - d = defer.maybeDeferred(self.poll) - d.addErrback(log.err, 'while polling for changes') - return d - - # delay starting the loop until the reactor is running, and do not - # run it immediately - if services are still starting up, they may - # miss an initial flood of changes - def start_loop(): - self._loop = task.LoopingCall(do_poll) - self._loop.start(self.pollInterval, now=False) - reactor.callWhenRunning(start_loop) + + # delay starting doing anything until the reactor is running - if + # services are still starting up, they may miss an initial flood of + # changes + if self.pollInterval: + reactor.callWhenRunning(self.startLoop) + else: + reactor.callWhenRunning(self.doPoll) def stopService(self): - if self._loop and self._loop.running: - self._loop.stop() + self.stopLoop() return ChangeSource.stopService(self) + diff --git a/master/buildbot/changes/bonsaipoller.py b/master/buildbot/changes/bonsaipoller.py index 56f5dab174a..9f252ee5c6a 100644 --- a/master/buildbot/changes/bonsaipoller.py +++ b/master/buildbot/changes/bonsaipoller.py @@ -207,14 +207,16 @@ class BonsaiPoller(base.PollingChangeSource): "module", "branch", "cvsroot"] def __init__(self, bonsaiURL, module, branch, tree="default", - cvsroot="/cvsroot", pollInterval=30, project=''): + cvsroot="/cvsroot", pollInterval=30, project='', name=None): + + base.PollingChangeSource.__init__(self, name=name, pollInterval=pollInterval) + self.bonsaiURL = bonsaiURL self.module = module self.branch = branch self.tree = tree self.cvsroot = cvsroot self.repository = module != 'all' and module or '' - self.pollInterval = pollInterval self.lastChange = time.time() self.lastPoll = time.time() diff --git a/master/buildbot/changes/gitpoller.py b/master/buildbot/changes/gitpoller.py index c9e2e79e7c1..ee394b48f21 100644 --- a/master/buildbot/changes/gitpoller.py +++ b/master/buildbot/changes/gitpoller.py @@ -36,15 +36,18 @@ def __init__(self, repourl, branch='master', gitbin='git', usetimestamps=True, category=None, project=None, pollinterval=-2, fetch_refspec=None, - encoding='utf-8'): + encoding='utf-8', name=None): + # for backward compatibility; the parameter used to be spelled with 'i' if pollinterval != -2: pollInterval = pollinterval + + base.PollingChangeSource.__init__(self, name=name, pollInterval=pollInterval) + if project is None: project = '' self.repourl = repourl self.branch = branch - self.pollInterval = pollInterval self.fetch_refspec = fetch_refspec self.encoding = encoding self.lastChange = time.time() diff --git a/master/buildbot/changes/hgbuildbot.py b/master/buildbot/changes/hgbuildbot.py index 909c427ba24..b01a13ca805 100644 --- a/master/buildbot/changes/hgbuildbot.py +++ b/master/buildbot/changes/hgbuildbot.py @@ -47,8 +47,8 @@ def hook(ui, repo, hooktype, node=None, source=None, **kwargs): # read config parameters baseurl = ui.config('hgbuildbot', 'baseurl', ui.config('web', 'baseurl', '')) - master = ui.config('hgbuildbot', 'master') - if master: + masters = ui.configlist('hgbuildbot', 'master') + if masters: branchtype = ui.config('hgbuildbot', 'branchtype', 'inrepo') branch = ui.config('hgbuildbot', 'branch') fork = ui.configbool('hgbuildbot', 'fork', False) @@ -88,11 +88,8 @@ def hook(ui, repo, hooktype, node=None, source=None, **kwargs): auth = 'change:changepw' auth = auth.split(':', 1) - s = sendchange.Sender(master, auth=auth) - d = defer.Deferred() - reactor.callLater(0, d.callback, None) # process changesets - def _send(res, c): + def _send(res, s, c): if not fork: ui.status("rev %s sent\n" % c['revision']) return s.send(c['branch'], c['revision'], c['comments'], @@ -110,30 +107,35 @@ def _send(res, c): repository = strip(repo.root, stripcount) repository = baseurl + repository - for rev in xrange(start, end): - # send changeset - node = repo.changelog.node(rev) - manifest, user, (time, timezone), files, desc, extra = repo.changelog.read(node) - parents = filter(lambda p: not p == nullid, repo.changelog.parents(node)) - if branchtype == 'inrepo': - branch = extra['branch'] - is_merge = len(parents) > 1 - # merges don't always contain files, but at least one file is required by buildbot - if is_merge and not files: - files = ["merge"] - properties = {'is_merge': is_merge} - if branch: - branch = fromlocal(branch) - change = { - 'master': master, - 'username': fromlocal(user), - 'revision': hex(node), - 'comments': fromlocal(desc), - 'files': files, - 'branch': branch, - 'properties':properties - } - d.addCallback(_send, change) + for master in masters: + s = sendchange.Sender(master, auth=auth) + d = defer.Deferred() + reactor.callLater(0, d.callback, None) + + for rev in xrange(start, end): + # send changeset + node = repo.changelog.node(rev) + manifest, user, (time, timezone), files, desc, extra = repo.changelog.read(node) + parents = filter(lambda p: not p == nullid, repo.changelog.parents(node)) + if branchtype == 'inrepo': + branch = extra['branch'] + is_merge = len(parents) > 1 + # merges don't always contain files, but at least one file is required by buildbot + if is_merge and not files: + files = ["merge"] + properties = {'is_merge': is_merge} + if branch: + branch = fromlocal(branch) + change = { + 'master': master, + 'username': fromlocal(user), + 'revision': hex(node), + 'comments': fromlocal(desc), + 'files': files, + 'branch': branch, + 'properties':properties + } + d.addCallback(_send, s, change) def _printSuccess(res): ui.status(s.getSuccessString(res) + '\n') diff --git a/master/buildbot/changes/p4poller.py b/master/buildbot/changes/p4poller.py index ffae1ce52b2..255570f0e9f 100644 --- a/master/buildbot/changes/p4poller.py +++ b/master/buildbot/changes/p4poller.py @@ -66,11 +66,14 @@ def __init__(self, p4port=None, p4user=None, p4passwd=None, p4base='//', p4bin='p4', split_file=lambda branchfile: (None, branchfile), pollInterval=60 * 10, histmax=None, pollinterval=-2, - encoding='utf8', project=None): + encoding='utf8', project=None, name=None): + # for backward compatibility; the parameter used to be spelled with 'i' if pollinterval != -2: pollInterval = pollinterval + base.PollingChangeSource.__init__(self, name=name, pollInterval=pollInterval) + if project is None: project = '' @@ -80,7 +83,6 @@ def __init__(self, p4port=None, p4user=None, p4passwd=None, self.p4base = p4base self.p4bin = p4bin self.split_file = split_file - self.pollInterval = pollInterval self.encoding = encoding self.project = project diff --git a/master/buildbot/changes/svnpoller.py b/master/buildbot/changes/svnpoller.py index d59849fab2c..fde04097856 100644 --- a/master/buildbot/changes/svnpoller.py +++ b/master/buildbot/changes/svnpoller.py @@ -53,7 +53,7 @@ class SVNPoller(base.PollingChangeSource, util.ComparableMixin): """ compare_attrs = ["svnurl", "split_file", - "svnuser", "svnpasswd", + "svnuser", "svnpasswd", "project", "pollInterval", "histmax", "svnbin", "category", "cachepath"] @@ -66,11 +66,14 @@ def __init__(self, svnurl, split_file=None, pollInterval=10*60, histmax=100, svnbin='svn', revlinktmpl='', category=None, project='', cachepath=None, pollinterval=-2, - extra_args=None): + extra_args=None, name=None): + # for backward compatibility; the parameter used to be spelled with 'i' if pollinterval != -2: pollInterval = pollinterval + base.PollingChangeSource.__init__(self, name=name, pollInterval=pollInterval) + if svnurl.endswith("/"): svnurl = svnurl[:-1] # strip the trailing slash self.svnurl = svnurl @@ -85,7 +88,6 @@ def __init__(self, svnurl, split_file=None, # required for ssh-agent auth self.svnbin = svnbin - self.pollInterval = pollInterval self.histmax = histmax self._prefix = None self.category = category diff --git a/master/buildbot/config.py b/master/buildbot/config.py index 8411de6bb55..c33db5da10a 100644 --- a/master/buildbot/config.py +++ b/master/buildbot/config.py @@ -15,9 +15,10 @@ from __future__ import with_statement -import re import os +import re import sys +import warnings from buildbot.util import safeTranslate from buildbot import interfaces from buildbot import locks @@ -360,6 +361,11 @@ def load_caches(self, filename, config_dict, errors): if not isinstance(caches, dict): errors.addError("c['caches'] must be a dictionary") else: + valPairs = caches.items() + for (x, y) in valPairs: + if (not isinstance(y, int)): + errors.addError( + "value for cache size '%s' must be an integer" % x) self.caches.update(caches) if 'buildCacheSize' in config_dict: @@ -424,6 +430,12 @@ def mapper(b): errors.addError("c['builders'] must be a list of builder configs") return + for builder in builders: + if os.path.isabs(builder.builddir): + warnings.warn("Absolute path '%s' for builder may cause " + "mayhem. Perhaps you meant to specify slavebuilddir " + "instead.") + self.builders = builders @@ -547,15 +559,6 @@ def check_lock(l): for l in b.locks: check_lock(l) - # factories don't necessarily need to implement a .steps attribute - # but in practice most do, so we'll check that if it exists - if not hasattr(b.factory, 'steps'): - continue - for s in b.factory.steps: - for l in s[1].get('locks', []): - check_lock(l) - - def check_builders(self, errors): # look both for duplicate builder names, and for builders pointing # to unknown slaves diff --git a/master/buildbot/db/changes.py b/master/buildbot/db/changes.py index 236b140c1ed..4288be532cc 100644 --- a/master/buildbot/db/changes.py +++ b/master/buildbot/db/changes.py @@ -196,9 +196,12 @@ def thd(conn): for table_name in ('scheduler_changes', 'sourcestamp_changes', 'change_files', 'change_properties', 'changes', 'change_users'): - table = self.db.model.metadata.tables[table_name] - conn.execute( - table.delete(table.c.changeid.in_(ids_to_delete))) + remaining = ids_to_delete[:] + while remaining: + batch, remaining = remaining[:100], remaining[100:] + table = self.db.model.metadata.tables[table_name] + conn.execute( + table.delete(table.c.changeid.in_(batch))) return self.db.pool.do(thd) def _chdict_from_change_row_thd(self, conn, ch_row): diff --git a/master/buildbot/db/connector.py b/master/buildbot/db/connector.py index dff7652f9b1..bc5716495e0 100644 --- a/master/buildbot/db/connector.py +++ b/master/buildbot/db/connector.py @@ -33,12 +33,7 @@ class DatabaseNotReadyError(Exception): buildbot upgrade-master path/to/master to upgrade the database, and try starting the buildmaster again. You may - want to make a backup of your buildmaster before doing so. If you are - using MySQL, you must specify the connector string on the upgrade-master - command line: - - buildbot upgrade-master --db= path/to/master - + want to make a backup of your buildmaster before doing so. """).strip() class DBConnector(config.ReconfigurableServiceMixin, service.MultiService): diff --git a/master/buildbot/interfaces.py b/master/buildbot/interfaces.py index a7763ed5ccc..1cdbc0873ae 100644 --- a/master/buildbot/interfaces.py +++ b/master/buildbot/interfaces.py @@ -220,10 +220,6 @@ def getBuilderNames(): @returns: list of names via Deferred""" def isFinished(): pass - def waitUntilSuccess(): - """Return a Deferred that fires (with this IBuildSetStatus object) - when the outcome of the BuildSet is known, i.e., upon the first - failure, or after all builds complete successfully.""" def waitUntilFinished(): """Return a Deferred that fires (with this IBuildSetStatus object) when all builds have finished.""" @@ -1204,3 +1200,8 @@ def render(value): class IScheduler(Interface): pass + +class IBuildStepFactory(Interface): + def buildStep(): + """ + """ diff --git a/master/buildbot/libvirtbuildslave.py b/master/buildbot/libvirtbuildslave.py index 52f716fcd06..285ad23eac2 100644 --- a/master/buildbot/libvirtbuildslave.py +++ b/master/buildbot/libvirtbuildslave.py @@ -17,10 +17,15 @@ import os from twisted.internet import defer, utils, reactor, threads -from twisted.python import log +from twisted.python import log, failure from buildbot.buildslave import AbstractBuildSlave, AbstractLatentBuildSlave +from buildbot import config -import libvirt +try: + import libvirt + libvirt = libvirt +except ImportError: + libvirt = None class WorkQueue(object): @@ -93,6 +98,9 @@ def __init__(self, connection, domain): self.connection = connection self.domain = domain + def name(self): + return queue.executeInThread(self.domain.name) + def create(self): return queue.executeInThread(self.domain.create) @@ -109,25 +117,34 @@ class Connection(object): I am a wrapper around a libvirt Connection object. """ + DomainClass = Domain + def __init__(self, uri): self.uri = uri self.connection = libvirt.open(uri) + @defer.inlineCallbacks def lookupByName(self, name): """ I lookup an existing prefined domain """ - d = queue.executeInThread(self.connection.lookupByName, name) - def _(res): - return Domain(self, res) - d.addCallback(_) - return d + res = yield queue.executeInThread(self.connection.lookupByName, name) + defer.returnValue(self.DomainClass(self, res)) + @defer.inlineCallbacks def create(self, xml): """ I take libvirt XML and start a new VM """ - d = queue.executeInThread(self.connection.createXML, xml, 0) - def _(res): - return Domain(self, res) - d.addCallback(_) - return d + res = yield queue.executeInThread(self.connection.createXML, xml, 0) + defer.returnValue(self.DomainClass(self, res)) + + @defer.inlineCallbacks + def all(self): + domains = [] + domain_ids = yield queue.executeInThread(self.connection.listDomainsID) + + for did in domain_ids: + domain = yield queue.executeInThread(self.connection.lookupByID, did) + domains.append(self.DomainClass(self, domain)) + + defer.returnValue(domains) class LibVirtSlave(AbstractLatentBuildSlave): @@ -136,18 +153,53 @@ def __init__(self, name, password, connection, hd_image, base_image = None, xml= missing_timeout=60*20, build_wait_timeout=60*10, properties={}, locks=None): AbstractLatentBuildSlave.__init__(self, name, password, max_builds, notify_on_missing, missing_timeout, build_wait_timeout, properties, locks) + + if not libvirt: + config.error("The python module 'libvirt' is needed to use a LibVirtSlave") + self.name = name self.connection = connection self.image = hd_image self.base_image = base_image self.xml = xml - self.insubstantiate_after_build = True self.cheap_copy = True self.graceful_shutdown = False self.domain = None + self.ready = False + self._find_existing_deferred = self._find_existing_instance() + + @defer.inlineCallbacks + def _find_existing_instance(self): + """ + I find existing VMs that are already running that might be orphaned instances of this slave. + """ + if not self.connection: + defer.returnValue(None) + + domains = yield self.connection.all() + for d in domains: + name = yield d.name() + if name.startswith(self.name): + self.domain = d + self.substantiated = True + break + + self.ready = True + + def canStartBuild(self): + if not self.ready: + log.msg("Not accepting builds as existing domains not iterated") + return False + + if self.domain and not self.isConnected(): + log.msg("Not accepting builds as existing domain but slave not connected") + return False + + return AbstractLatentBuildSlave.canStartBuild(self) + def _prepare_base_image(self): """ I am a private method for creating (possibly cheap) copies of a @@ -178,6 +230,7 @@ def _log_result(res): d.addBoth(_log_result) return d + @defer.inlineCallbacks def start_instance(self, build): """ I start a new instance of a VM. @@ -189,38 +242,25 @@ def start_instance(self, build): in the list of defined virtual machines and start that. """ if self.domain is not None: - raise ValueError('domain active') + log.msg("Cannot start_instance '%s' as already active" % self.name) + defer.returnValue(False) - d = self._prepare_base_image() + yield self._prepare_base_image() - def _start(res): + try: if self.xml: - d = self.connection.create(self.xml) - def _xml_start(res): - self.domain = res - return - d.addCallback(_xml_start) - return d - d = self.connection.lookupByName(self.name) - def _really_start(res): - self.domain = res - return self.domain.create() - d.addCallback(_really_start) - return d - d.addCallback(_start) - - def _started(res): - return True - d.addCallback(_started) - - def _start_failed(failure): - log.msg("Cannot start a VM (%s), failing gracefully and triggering a new build check" % self.name) - log.err(failure) + self.domain = yield self.connection.create(self.xml) + else: + self.domain = yield self.connection.lookupByName(self.name) + yield self.domain.create() + except: + log.err(failure.Failure(), + "Cannot start a VM (%s), failing gracefully and triggering" + "a new build check" % self.name) self.domain = None - return False - d.addErrback(_start_failed) + defer.returnValue(False) - return d + defer.returnValue(True) def stop_instance(self, fast=False): """ @@ -231,7 +271,7 @@ def stop_instance(self, fast=False): """ log.msg("Attempting to stop '%s'" % self.name) if self.domain is None: - log.msg("I don't think that domain is evening running, aborting") + log.msg("I don't think that domain is even running, aborting") return defer.succeed(None) domain = self.domain @@ -258,14 +298,3 @@ def _disconnected(res): return d - def buildFinished(self, *args, **kwargs): - """ - I insubstantiate a slave after it has done a build, if that is - desired behaviour. - """ - AbstractLatentBuildSlave.buildFinished(self, *args, **kwargs) - if self.insubstantiate_after_build: - log.msg("Got buildFinished notification - attempting to insubstantiate") - self.insubstantiate() - - diff --git a/master/buildbot/process/build.py b/master/buildbot/process/build.py index c747e576874..42fc4f18080 100644 --- a/master/buildbot/process/build.py +++ b/master/buildbot/process/build.py @@ -187,13 +187,15 @@ def setupProperties(self): def setupSlaveBuilder(self, slavebuilder): self.slavebuilder = slavebuilder + self.path_module = slavebuilder.slave.path_module + # navigate our way back to the L{buildbot.buildslave.BuildSlave} # object that came from the config, and get its properties buildslave_properties = slavebuilder.slave.properties self.getProperties().updateFromProperties(buildslave_properties) if slavebuilder.slave.slave_basedir: self.setProperty("workdir", - slavebuilder.slave.path_module.join( + self.path_module.join( slavebuilder.slave.slave_basedir, self.builder.config.slavebuilddir), "slave") @@ -300,15 +302,8 @@ def setupBuild(self, expectations): stepnames = {} sps = [] - for factory, args in self.stepFactories: - args = args.copy() - try: - step = factory(**args) - except: - log.msg("error while creating step, factory=%s, args=%s" - % (factory, args)) - raise - + for factory in self.stepFactories: + step = factory.buildStep() step.setBuild(self) step.setBuildSlave(self.slavebuilder.slave) if callable (self.workdir): diff --git a/master/buildbot/process/builder.py b/master/buildbot/process/builder.py index 066e88378c4..1a48327a05f 100644 --- a/master/buildbot/process/builder.py +++ b/master/buildbot/process/builder.py @@ -692,21 +692,21 @@ def _breakBrdictRefloops(self, requests): class BuilderControl: implements(interfaces.IBuilderControl) - def __init__(self, builder, master): + def __init__(self, builder, control): self.original = builder - self.master = master + self.control = control def submitBuildRequest(self, ss, reason, props=None): - d = ss.getSourceStampSetId(self.master.master) + d = ss.getSourceStampSetId(self.control.master) def add_buildset(sourcestampsetid): - return self.master.master.addBuildset( + return self.control.master.addBuildset( builderNames=[self.original.name], sourcestampsetid=sourcestampsetid, reason=reason, properties=props) d.addCallback(add_buildset) def get_brs((bsid,brids)): brs = BuildRequestStatus(self.original.name, brids[self.original.name], - self.master.master.status) + self.control.master.status) return brs d.addCallback(get_brs) return d @@ -727,14 +727,14 @@ def rebuildBuild(self, bs, reason="", extraProperties= ssList = bs.getSourceStamps(absolute=True) if ssList: - sourcestampsetid = yield ssList[0].getSourceStampSetId(self.master.master) + sourcestampsetid = yield ssList[0].getSourceStampSetId(self.control.master) dl = [] for ss in ssList[1:]: # add defered to the list - dl.append(ss.addSourceStampToDatabase(self.master.master, sourcestampsetid)) + dl.append(ss.addSourceStampToDatabase(self.control.master, sourcestampsetid)) yield defer.gatherResults(dl) - bsid, brids = yield self.master.master.addBuildset( + bsid, brids = yield self.control.master.addBuildset( builderNames=[self.original.name], sourcestampsetid=sourcestampsetid, reason=reason, @@ -755,7 +755,7 @@ def getPendingBuildRequestControls(self): buildrequests = [ ] for brdict in brdicts: br = yield buildrequest.BuildRequest.fromBrdict( - self.master.master, brdict) + self.control.master, brdict) buildrequests.append(br) # and return the corresponding control objects diff --git a/master/buildbot/process/buildrequest.py b/master/buildbot/process/buildrequest.py index a6d26a7d36e..b65160b30ff 100644 --- a/master/buildbot/process/buildrequest.py +++ b/master/buildbot/process/buildrequest.py @@ -91,7 +91,7 @@ def fromBrdict(cls, master, brdict): return cache.get(brdict['brid'], brdict=brdict, master=master) @classmethod - @defer.deferredGenerator + @defer.inlineCallbacks def _make_br(cls, brid, brdict, master): buildrequest = cls() buildrequest.id = brid @@ -103,18 +103,12 @@ def _make_br(cls, brid, brdict, master): buildrequest.master = master # fetch the buildset to get the reason - wfd = defer.waitForDeferred( - master.db.buildsets.getBuildset(brdict['buildsetid'])) - yield wfd - buildset = wfd.getResult() + buildset = yield master.db.buildsets.getBuildset(brdict['buildsetid']) assert buildset # schema should guarantee this buildrequest.reason = buildset['reason'] # fetch the buildset properties, and convert to Properties - wfd = defer.waitForDeferred( - master.db.buildsets.getBuildsetProperties(brdict['buildsetid'])) - yield wfd - buildset_properties = wfd.getResult() + buildset_properties = yield master.db.buildsets.getBuildsetProperties(brdict['buildsetid']) pr = properties.Properties() for name, (value, source) in buildset_properties.iteritems(): @@ -122,10 +116,7 @@ def _make_br(cls, brid, brdict, master): buildrequest.properties = pr # fetch the sourcestamp dictionary - wfd = defer.waitForDeferred( - master.db.sourcestamps.getSourceStamps(buildset['sourcestampsetid'])) - yield wfd - sslist = wfd.getResult() + sslist = yield master.db.sourcestamps.getSourceStamps(buildset['sourcestampsetid']) assert len(sslist) > 0, "Empty sourcestampset: db schema enforces set to exist but cannot enforce a non empty set" # and turn it into a SourceStamps @@ -139,15 +130,12 @@ def store_source(source): d.addCallback(store_source) dlist.append(d) - dl = defer.gatherResults(dlist) - wfd = defer.waitForDeferred(dl) - yield wfd - wfd.getResult() + yield defer.gatherResults(dlist) if buildrequest.sources: buildrequest.source = buildrequest.sources.values()[0] - yield buildrequest # return value + defer.returnValue(buildrequest) def requestsHaveSameCodebases(self, other): self_codebases = set(self.sources.iterkeys()) @@ -230,15 +218,12 @@ def mergeReasons(self, others): def getSubmitTime(self): return self.submittedAt - @defer.deferredGenerator + @defer.inlineCallbacks def cancelBuildRequest(self): # first, try to claim the request; if this fails, then it's too late to # cancel the build anyway try: - wfd = defer.waitForDeferred( - self.master.db.buildrequests.claimBuildRequests([self.id])) - yield wfd - wfd.getResult() + yield self.master.db.buildrequests.claimBuildRequests([self.id]) except buildrequests.AlreadyClaimedError: log.msg("build request already claimed; cannot cancel") return @@ -246,17 +231,11 @@ def cancelBuildRequest(self): # then complete it with 'FAILURE'; this is the closest we can get to # cancelling a request without running into trouble with dangling # references. - wfd = defer.waitForDeferred( - self.master.db.buildrequests.completeBuildRequests([self.id], - FAILURE)) - yield wfd - wfd.getResult() + yield self.master.db.buildrequests.completeBuildRequests([self.id], + FAILURE) # and let the master know that the enclosing buildset may be complete - wfd = defer.waitForDeferred( - self.master.maybeBuildsetComplete(self.bsid)) - yield wfd - wfd.getResult() + yield self.master.maybeBuildsetComplete(self.bsid) class BuildRequestControl: implements(interfaces.IBuildRequestControl) diff --git a/master/buildbot/process/buildstep.py b/master/buildbot/process/buildstep.py index 68f515f3de2..6512bf43de3 100644 --- a/master/buildbot/process/buildstep.py +++ b/master/buildbot/process/buildstep.py @@ -42,7 +42,8 @@ class RemoteCommand(pb.Referenceable): rc = None debug = False - def __init__(self, remote_command, args, ignore_updates=False, collectStdout=False): + def __init__(self, remote_command, args, ignore_updates=False, + collectStdout=False, successfulRC=(0,)): self.logs = {} self.delayedLogs = {} self._closeWhenFinished = {} @@ -54,6 +55,7 @@ def __init__(self, remote_command, args, ignore_updates=False, collectStdout=Fal self.remote_command = remote_command self.args = args self.ignore_updates = ignore_updates + self.successfulRC = successfulRC def __repr__(self): return "" % (self.remote_command, id(self)) @@ -268,6 +270,9 @@ def remoteComplete(self, maybeFailure): log.msg("closing log %s" % loog) loog.finish() return maybeFailure + + def didFail(self): + return self.rc not in self.successfulRC LoggedRemoteCommand = RemoteCommand @@ -345,7 +350,7 @@ def __init__(self, workdir, command, env=None, timeout=20*60, maxTime=None, logfiles={}, usePTY="slave-config", logEnviron=True, collectStdout=False, interruptSignal=None, - initialStdin=None): + initialStdin=None, successfulRC=(0,)): self.command = command # stash .command, set it later if env is not None: @@ -366,7 +371,8 @@ def __init__(self, workdir, command, env=None, } if interruptSignal is not None: args['interruptSignal'] = interruptSignal - RemoteCommand.__init__(self, "shell", args, collectStdout=collectStdout) + RemoteCommand.__init__(self, "shell", args, collectStdout=collectStdout, + successfulRC=successfulRC) def _start(self): self.args['command'] = self.command @@ -383,7 +389,29 @@ def _start(self): def __repr__(self): return "" % repr(self.command) -class BuildStep(properties.PropertiesMixin): +class _BuildStepFactory(util.ComparableMixin): + """ + This is a wrapper to record the arguments passed to as BuildStep subclass. + We use an instance of this class, rather than a closure mostly to make it + easier to test that the right factories are getting created. + """ + compare_attrs = ['factory', 'args', 'kwargs' ] + implements(interfaces.IBuildStepFactory) + + def __init__(self, factory, *args, **kwargs): + self.factory = factory + self.args = args + self.kwargs = kwargs + + def buildStep(self): + try: + return self.factory(*self.args, **self.kwargs) + except: + log.msg("error while creating step, factory=%s, args=%s, kwargs=%s" + % (self.factory, self.args, self.kwargs)) + raise + +class BuildStep(object, properties.PropertiesMixin): haltOnFailure = False flunkOnWarnings = False @@ -426,7 +454,6 @@ class BuildStep(properties.PropertiesMixin): progress = None def __init__(self, **kwargs): - self.factory = (self.__class__, dict(kwargs)) for p in self.__class__.parms: if kwargs.has_key(p): setattr(self, p, kwargs[p]) @@ -440,6 +467,11 @@ def __init__(self, **kwargs): self._acquiringLock = None self.stopped = False + def __new__(klass, *args, **kwargs): + self = object.__new__(klass) + self._factory = _BuildStepFactory(klass, *args, **kwargs) + return self + def describe(self, done=False): return [self.name] @@ -453,10 +485,11 @@ def setDefaultWorkdir(self, workdir): pass def addFactoryArguments(self, **kwargs): - self.factory[1].update(kwargs) + # this is here for backwards compatability + pass - def getStepFactory(self): - return self.factory + def _getStepFactory(self): + return self._factory def setStepStatus(self, step_status): self.step_status = step_status @@ -613,11 +646,19 @@ def _finishFinished(self, results): # from finished() so that subclasses can override finished() if self.progress: self.progress.finish() + + try: + hidden = self._maybeEvaluate(self.hideStepIf, results, self) + except Exception: + why = Failure() + self.addHTMLLog("err.html", formatFailure(why)) + self.addCompleteLog("err.text", why.getTraceback()) + results = EXCEPTION + hidden = False + self.step_status.stepFinished(results) - - hidden = self._maybeEvaluate(self.hideStepIf, results, self) self.step_status.setHidden(hidden) - + self.releaseLocks() self.deferred.callback(results) @@ -731,6 +772,9 @@ def _maybeEvaluate(value, *args, **kwargs): value = value(*args, **kwargs) return value +components.registerAdapter( + BuildStep._getStepFactory, + BuildStep, interfaces.IBuildStepFactory) components.registerAdapter( lambda step : interfaces.IProperties(step.build), BuildStep, interfaces.IProperties) @@ -759,9 +803,6 @@ class LoggingBuildStep(BuildStep): def __init__(self, logfiles={}, lazylogfiles=False, log_eval_func=None, *args, **kwargs): BuildStep.__init__(self, *args, **kwargs) - self.addFactoryArguments(logfiles=logfiles, - lazylogfiles=lazylogfiles, - log_eval_func=log_eval_func) if logfiles and not isinstance(logfiles, dict): config.error( @@ -866,7 +907,7 @@ def createSummary(self, stdio): def evaluateCommand(self, cmd): if self.log_eval_func: return self.log_eval_func(cmd, self.step_status) - if cmd.rc != 0: + if cmd.didFail(): return FAILURE return SUCCESS @@ -914,7 +955,7 @@ def setStatus(self, cmd, results): # ) def regex_log_evaluator(cmd, step_status, regexes): worst = SUCCESS - if cmd.rc != 0: + if cmd.didFail(): worst = FAILURE for err, possible_status in regexes: # worst_status returns the worse of the two status' passed to it. diff --git a/master/buildbot/process/factory.py b/master/buildbot/process/factory.py index e44e46ca430..7516386b38c 100644 --- a/master/buildbot/process/factory.py +++ b/master/buildbot/process/factory.py @@ -13,23 +13,11 @@ # # Copyright Buildbot Team Members -import warnings - -from twisted.python import deprecate, versions - -from buildbot import util +from buildbot import interfaces, util from buildbot.process.build import Build -from buildbot.process.buildstep import BuildStep from buildbot.steps.source import CVS, SVN from buildbot.steps.shell import Configure, Compile, Test, PerlModuleTest -# deprecated, use BuildFactory.addStep -@deprecate.deprecated(versions.Version("buildbot", 0, 8, 6)) -def s(steptype, **kwargs): - # convenience function for master.cfg files, to create step - # specification tuples - return (steptype, kwargs) - class ArgumentsInTheWrongPlace(Exception): """When calling BuildFactory.addStep(stepinstance), addStep() only takes one argument. You passed extra arguments to addStep(), which you probably @@ -54,18 +42,9 @@ class BuildFactory(util.ComparableMixin): compare_attrs = ['buildClass', 'steps', 'useProgress', 'workdir'] def __init__(self, steps=None): - if steps is None: - steps = [] - self.steps = [self._makeStepFactory(s) for s in steps] - - def _makeStepFactory(self, step_or_factory): - if isinstance(step_or_factory, BuildStep): - return step_or_factory.getStepFactory() - warnings.warn( - "Passing a BuildStep subclass to factory.addStep is deprecated. " + - "Please pass a BuildStep instance instead. Support will be dropped in v0.8.7.", - DeprecationWarning, stacklevel=3) - return step_or_factory + self.steps = [] + if steps: + self.addSteps(steps) def newBuild(self, requests): """Create a new Build instance. @@ -79,22 +58,8 @@ def newBuild(self, requests): b.setStepFactories(self.steps) return b - def addStep(self, step_or_factory, **kwargs): - if isinstance(step_or_factory, BuildStep): - if kwargs: - raise ArgumentsInTheWrongPlace() - s = step_or_factory.getStepFactory() - elif type(step_or_factory) == type(BuildStep) and \ - issubclass(step_or_factory, BuildStep): - s = (step_or_factory, dict(kwargs)) - warnings.warn( - "Passing a BuildStep subclass to factory.addStep is deprecated. " + - "Please pass a BuildStep instance instead. Support will be dropped in v0.8.7.", - DeprecationWarning, stacklevel=2) - - else: - raise ValueError('%r is not a BuildStep nor BuildStep subclass' % step_or_factory) - self.steps.append(s) + def addStep(self, step): + self.steps.append(interfaces.IBuildStepFactory(step)) def addSteps(self, steps): for s in steps: @@ -199,7 +164,7 @@ def __init__(self, cvsroot, cvsmodule, mode = "clobber" if cvsCopy: mode = "copy" - source = s(CVS, cvsroot=cvsroot, cvsmodule=cvsmodule, mode=mode) + source = CVS(cvsroot=cvsroot, cvsmodule=cvsmodule, mode=mode) GNUAutoconf.__init__(self, source, configure=configure, configureEnv=configureEnv, compile=compile, @@ -213,7 +178,7 @@ def __init__(self, cvsroot, cvsmodule, compile="make all", test="make check", cvsCopy=False): mode = "update" - source = s(CVS, cvsroot=cvsroot, cvsmodule=cvsmodule, mode=mode) + source = CVS(cvsroot=cvsroot, cvsmodule=cvsmodule, mode=mode) GNUAutoconf.__init__(self, source, configure=configure, configureEnv=configureEnv, compile=compile, @@ -225,7 +190,7 @@ def __init__(self, svnurl, configure=None, configureEnv={}, compile="make all", test="make check"): - source = s(SVN, svnurl=svnurl, mode="update") + source = SVN(svnurl=svnurl, mode="update") GNUAutoconf.__init__(self, source, configure=configure, configureEnv=configureEnv, compile=compile, diff --git a/master/buildbot/process/mtrlogobserver.py b/master/buildbot/process/mtrlogobserver.py index 43168a6bb60..a49b90e2129 100644 --- a/master/buildbot/process/mtrlogobserver.py +++ b/master/buildbot/process/mtrlogobserver.py @@ -296,15 +296,6 @@ def __init__(self, dbpool=None, test_type=None, test_info="", self.mtr_subdir = mtr_subdir self.progressMetrics += ('tests',) - self.addFactoryArguments(dbpool=self.dbpool, - test_type=self.test_type, - test_info=self.test_info, - autoCreateTables=self.autoCreateTables, - textLimit=self.textLimit, - testNameLimit=self.testNameLimit, - parallel=self.parallel, - mtr_subdir=self.mtr_subdir) - def start(self): # Add mysql server logfiles. for mtr in range(0, self.parallel+1): diff --git a/master/buildbot/process/properties.py b/master/buildbot/process/properties.py index 99796a6dd70..b9d3a9c6a99 100644 --- a/master/buildbot/process/properties.py +++ b/master/buildbot/process/properties.py @@ -15,8 +15,10 @@ import collections import re +import warnings import weakref from buildbot import config, util +from buildbot.util import json from buildbot.interfaces import IRenderable, IProperties from twisted.internet import defer from twisted.python.components import registerAdapter @@ -94,9 +96,7 @@ def __repr__(self): def update(self, dict, source, runtime=False): """Update this object from a dictionary, with an explicit source specified.""" for k, v in dict.items(): - self.properties[k] = (v, source) - if runtime: - self.runtime.add(k) + self.setProperty(k, v, source, runtime=runtime) def updateFromProperties(self, other): """Update this object based on another object; the other object's """ @@ -121,6 +121,14 @@ def hasProperty(self, name): has_key = hasProperty def setProperty(self, name, value, source, runtime=False): + try: + json.dumps(value) + except TypeError: + warnings.warn( + "Non jsonable properties are not explicitly supported and" + + "will be explicitly disallowed in a future version.", + DeprecationWarning, stacklevel=2) + self.properties[name] = (value, source) if runtime: self.runtime.add(name) @@ -185,6 +193,17 @@ class _PropertyMap(object): colon_minus_re = re.compile(r"(.*):-(.*)") colon_tilde_re = re.compile(r"(.*):~(.*)") colon_plus_re = re.compile(r"(.*):\+(.*)") + + colon_ternary_re = re.compile(r"""(?P.*) # the property to match + : # colon + (?P\#)? # might have the alt marker '#' + \? # question mark + (?P.) # the delimiter + (?P.*) # sub-if-true + (?P=delim) # the delimiter again + (?P.*)# sub-if-false + """, re.VERBOSE) + def __init__(self, properties): # use weakref here to avoid a reference loop self.properties = weakref.ref(properties) @@ -225,10 +244,38 @@ def colon_plus(mo): else: return '' + def colon_ternary(mo): + # %(prop:?:T:F)s + # if prop exists, use T; otherwise, F + # %(prop:#?:T:F)s + # if prop is true, use T; otherwise, F + groups = mo.groupdict() + + prop = groups['prop'] + + if prop in self.temp_vals: + if groups['alt']: + use_true = self.temp_vals[prop] + else: + use_true = True + elif properties.has_key(prop): + if groups['alt']: + use_true = properties[prop] + else: + use_true = True + else: + use_true = False + + if use_true: + return groups['true'] + else: + return groups['false'] + for regexp, fn in [ ( self.colon_minus_re, colon_minus ), ( self.colon_tilde_re, colon_tilde ), ( self.colon_plus_re, colon_plus ), + ( self.colon_ternary_re, colon_ternary ), ]: mo = regexp.match(key) if mo: @@ -363,6 +410,8 @@ class Interpolate(util.ComparableMixin): implements(IRenderable) compare_attrs = ('fmtstring', 'args', 'kwargs') + + identifier_re = re.compile('^[\w-]*$') def __init__(self, fmtstring, *args, **kwargs): self.fmtstring = fmtstring @@ -380,6 +429,9 @@ def _parse_prop(arg): prop, repl = arg.split(":", 1) except ValueError: prop, repl = arg, None + if not Interpolate.identifier_re.match(prop): + config.error("Property name must be alphanumeric for prop Interpolation '%s'" % arg) + prop = repl = None return _thePropertyDict, prop, repl @staticmethod @@ -394,6 +446,12 @@ def _parse_src(arg): except ValueError: config.error("Must specify both codebase and attribute for src Interpolation '%s'" % arg) codebase = attr = repl = None + if not Interpolate.identifier_re.match(codebase): + config.error("Codebase must be alphanumeric for src Interpolation '%s'" % arg) + codebase = attr = repl = None + if not Interpolate.identifier_re.match(attr): + config.error("Attribute must be alphanumeric for src Interpolation '%s'" % arg) + codebase = attr = repl = None return _SourceStampDict(codebase), attr, repl def _parse_kw(self, arg): @@ -401,6 +459,9 @@ def _parse_kw(self, arg): kw, repl = arg.split(":", 1) except ValueError: kw, repl = arg, None + if not Interpolate.identifier_re.match(kw): + config.error("Keyword must be alphanumeric for kw Interpolation '%s'" % arg) + kw = repl = None return _Lazy(self.kwargs), kw, repl def _parseSubstitution(self, fmt): @@ -417,6 +478,20 @@ def _parseSubstitution(self, fmt): else: return fn(arg) + @staticmethod + def _splitBalancedParen(delim, arg): + parenCount = 0 + for i in range(0, len(arg)): + if arg[i] == "(": + parenCount += 1 + if arg[i] == ")": + parenCount -= 1 + if parenCount < 0: + raise ValueError + if parenCount == 0 and arg[i] == delim: + return arg[0:i], arg[i+1:] + return arg + def _parseColon_minus(self, d, kw, repl): return _Lookup(d, kw, default=Interpolate(repl, **self.kwargs), @@ -436,6 +511,25 @@ def _parseColon_plus(self, d, kw, repl): defaultWhenFalse=False, elideNoneAs='') + def _parseColon_ternary(self, d, kw, repl, defaultWhenFalse=False): + delim = repl[0] + if delim == '(': + config.error("invalid Interpolate ternary delimiter '('") + return None + try: + truePart, falsePart = self._splitBalancedParen(delim, repl[1:]) + except ValueError: + config.error("invalid Interpolate ternary expression '%s' with delimiter '%s'" % (repl[1:], repl[0])) + return None + return _Lookup(d, kw, + hasKey=Interpolate(truePart, **self.kwargs), + default=Interpolate(falsePart, **self.kwargs), + defaultWhenFalse=defaultWhenFalse, + elideNoneAs='') + + def _parseColon_ternary_hash(self, d, kw, repl): + return self._parseColon_ternary(d, kw, repl, defaultWhenFalse=True) + def _parse(self, fmtstring): keys = _getInterpolationList(fmtstring) for key in keys: @@ -443,13 +537,17 @@ def _parse(self, fmtstring): d, kw, repl = self._parseSubstitution(key) if repl is None: repl = '-' - for char, fn in [ + for pattern, fn in [ ( "-", self._parseColon_minus ), ( "~", self._parseColon_tilde ), ( "+", self._parseColon_plus ), + ( "?", self._parseColon_ternary ), + ( "#?", self._parseColon_ternary_hash ) ]: - if repl[0] == char: - self.interpolations[key] = fn(d, kw, repl[1:]) + junk, matches, tail = repl.partition(pattern) + if not junk and matches: + self.interpolations[key] = fn(d, kw, tail) + break if not self.interpolations.has_key(key): config.error("invalid Interpolate default type '%s'" % repl[0]) diff --git a/master/buildbot/schedulers/base.py b/master/buildbot/schedulers/base.py index f7180e06f70..4c9089f48f1 100644 --- a/master/buildbot/schedulers/base.py +++ b/master/buildbot/schedulers/base.py @@ -127,7 +127,7 @@ def stopService(self): ## state management - @defer.deferredGenerator + @defer.inlineCallbacks def getState(self, *args, **kwargs): """ For use by subclasses; get a named state value from the scheduler's @@ -141,18 +141,14 @@ def getState(self, *args, **kwargs): """ # get the objectid, if not known if self._objectid is None: - wfd = defer.waitForDeferred( - self.master.db.state.getObjectId(self.name, - self.__class__.__name__)) - yield wfd - self._objectid = wfd.getResult() - - wfd = defer.waitForDeferred( - self.master.db.state.getState(self._objectid, *args, **kwargs)) - yield wfd - yield wfd.getResult() - - @defer.deferredGenerator + self._objectid = yield self.master.db.state.getObjectId(self.name, + self.__class__.__name__) + + rv = yield self.master.db.state.getState(self._objectid, *args, + **kwargs) + defer.returnValue(rv) + + @defer.inlineCallbacks def setState(self, key, value): """ For use by subclasses; set a named state value in the scheduler's @@ -165,16 +161,10 @@ def setState(self, key, value): """ # get the objectid, if not known if self._objectid is None: - wfd = defer.waitForDeferred( - self.master.db.state.getObjectId(self.name, - self.__class__.__name__)) - yield wfd - self._objectid = wfd.getResult() + self._objectid = yield self.master.db.state.getObjectId(self.name, + self.__class__.__name__) - wfd = defer.waitForDeferred( - self.master.db.state.setState(self._objectid, key, value)) - yield wfd - wfd.getResult() + yield self.master.db.state.setState(self._objectid, key, value) ## status queries @@ -334,7 +324,7 @@ def addBuildsetForLatest(self, reason='', external_idstring=None, defer.returnValue((bsid,brids)) - @defer.deferredGenerator + @defer.inlineCallbacks def addBuildSetForSourceStampDetails(self, reason='', external_idstring=None, branch=None, repository='', project='', revision=None, builderNames=None, properties=None): @@ -357,23 +347,18 @@ def addBuildSetForSourceStampDetails(self, reason='', external_idstring=None, @returns: (buildset ID, buildrequest IDs) via Deferred """ # Define setid for this set of changed repositories - wfd = defer.waitForDeferred(self.master.db.sourcestampsets.addSourceStampSet()) - yield wfd - setid = wfd.getResult() + setid = yield self.master.db.sourcestampsets.addSourceStampSet() - wfd = defer.waitForDeferred(self.master.db.sourcestamps.addSourceStamp( + yield self.master.db.sourcestamps.addSourceStamp( branch=branch, revision=revision, repository=repository, - project=project, sourcestampsetid=setid)) - yield wfd - wfd.getResult() + project=project, sourcestampsetid=setid) - wfd = defer.waitForDeferred(self.addBuildsetForSourceStamp( + rv = yield self.addBuildsetForSourceStamp( setid=setid, reason=reason, external_idstring=external_idstring, builderNames=builderNames, - properties=properties)) - yield wfd - yield wfd.getResult() + properties=properties) + defer.returnValue(rv) @defer.inlineCallbacks @@ -419,7 +404,7 @@ def get_last_change_for_codebase(codebase): defer.returnValue((bsid,brids)) - @defer.deferredGenerator + @defer.inlineCallbacks def addBuildsetForSourceStamp(self, ssid=None, setid=None, reason='', external_idstring=None, properties=None, builderNames=None): """ @@ -460,19 +445,15 @@ def addBuildsetForSourceStamp(self, ssid=None, setid=None, reason='', external_i if setid == None: if ssid is not None: - wfd = defer.waitForDeferred(self.master.db.sourcestamps.getSourceStamp(ssid)) - yield wfd - ssdict = wfd.getResult() + ssdict = yield self.master.db.sourcestamps.getSourceStamp(ssid) setid = ssdict['sourcestampsetid'] else: # no sourcestamp and no sets yield None - wfd = defer.waitForDeferred(self.master.addBuildset( - sourcestampsetid=setid, reason=reason, - properties=properties_dict, - builderNames=builderNames, - external_idstring=external_idstring)) - yield wfd - yield wfd.getResult() + rv = yield self.master.addBuildset(sourcestampsetid=setid, + reason=reason, properties=properties_dict, + builderNames=builderNames, + external_idstring=external_idstring) + defer.returnValue(rv) diff --git a/master/buildbot/schedulers/basic.py b/master/buildbot/schedulers/basic.py index 19823123918..594f9dd6a15 100644 --- a/master/buildbot/schedulers/basic.py +++ b/master/buildbot/schedulers/basic.py @@ -96,14 +96,12 @@ def startService(self, _returnDeferred=False): def stopService(self): # the base stopService will unsubscribe from new changes d = base.BaseScheduler.stopService(self) - d.addCallback(lambda _ : - self._stable_timers_lock.acquire()) + @util.deferredLocked(self._stable_timers_lock) def cancel_timers(_): for timer in self._stable_timers.values(): if timer: timer.cancel() - self._stable_timers = {} - self._stable_timers_lock.release() + self._stable_timers.clear() d.addCallback(cancel_timers) return d @@ -195,6 +193,11 @@ def stableTimerFired(self, timer_name): yield self.master.db.schedulers.flushChangeClassifications( self.objectid, less_than=max_changeid+1) + def getPendingBuildTimes(self): + # This isn't locked, since the caller expects and immediate value, + # and in any case, this is only an estimate. + return [timer.getTime() for timer in self._stable_timers.values() if timer and timer.active()] + class SingleBranchScheduler(BaseBasicScheduler): def getChangeFilter(self, branch, branches, change_filter, categories): if branch is NotABranch and not change_filter and self.codebases is not None: diff --git a/master/buildbot/status/build.py b/master/buildbot/status/build.py index 912bf75122d..9f55922a5fc 100644 --- a/master/buildbot/status/build.py +++ b/master/buildbot/status/build.py @@ -89,7 +89,7 @@ def getPreviousBuild(self): return self.builder.getBuild(self.number-1) def getAllGotRevisions(self): - all_got_revisions = self.properties.getProperty('got_revision', None) + all_got_revisions = self.properties.getProperty('got_revision', {}) # For backwards compatibility all_got_revisions is a string if codebases # are not used. Convert to the default internal type (dict) if isinstance(all_got_revisions, str): @@ -101,7 +101,7 @@ def getSourceStamps(self, absolute=False): if not absolute: sourcestamps.extend(self.sources) else: - all_got_revisions = self.getAllGotRevisions() + all_got_revisions = self.getAllGotRevisions() or {} # always make a new instance for ss in self.sources: if ss.codebase in all_got_revisions: @@ -211,9 +211,6 @@ def getTestResults(self): return self.testResults def getLogs(self): - # TODO: steps should contribute significant logs instead of this - # hack, which returns every log from every step. The logs should get - # names like "compile" and "test" instead of "compile.output" logs = [] for s in self.steps: for loog in s.getLogs(): @@ -456,7 +453,6 @@ def asDict(self): # Constant result['builderName'] = self.builder.name result['number'] = self.getNumber() - # TODO: enable multiple sourcestamps to outside the buildstatus result['sourceStamps'] = [ss.asDict() for ss in self.getSourceStamps()] result['reason'] = self.getReason() result['blame'] = self.getResponsibleUsers() diff --git a/master/buildbot/status/builder.py b/master/buildbot/status/builder.py index c2d003c07a0..8c38b5f6675 100644 --- a/master/buildbot/status/builder.py +++ b/master/buildbot/status/builder.py @@ -219,7 +219,7 @@ def prune(self, events_only=False): # get the horizons straight buildHorizon = self.master.config.buildHorizon if buildHorizon is not None: - earliest_build = self.nextBuildNumber - self.buildHorizon + earliest_build = self.nextBuildNumber - buildHorizon else: earliest_build = 0 diff --git a/master/buildbot/status/logfile.py b/master/buildbot/status/logfile.py index 3b34c563f48..abb6efd7ac3 100644 --- a/master/buildbot/status/logfile.py +++ b/master/buildbot/status/logfile.py @@ -677,6 +677,8 @@ def finish(self): def __getstate__(self): d = self.__dict__.copy() del d['step'] + if d.has_key('master'): + del d['master'] return d diff --git a/master/buildbot/status/mail.py b/master/buildbot/status/mail.py index 1114ad000e9..781db39ea6f 100644 --- a/master/buildbot/status/mail.py +++ b/master/buildbot/status/mail.py @@ -49,7 +49,7 @@ from buildbot import interfaces, util, config from buildbot.process.users import users from buildbot.status import base -from buildbot.status.results import FAILURE, SUCCESS, WARNINGS, Results +from buildbot.status.results import FAILURE, SUCCESS, WARNINGS, EXCEPTION, Results VALID_EMAIL = re.compile("[a-zA-Z0-9\.\_\%\-\+]+@[a-zA-Z0-9\.\_\%\-]+.[a-zA-Z]{2,6}") @@ -92,6 +92,8 @@ def defaultMessage(mode, name, build, results, master_status): text += "The Buildbot has detected a restored build" else: text += "The Buildbot has detected a passing build" + elif results == EXCEPTION: + text += "The Buildbot has detected a build exception" projects = [] if ss_list: @@ -177,7 +179,7 @@ class MailNotifier(base.StatusReceiverMultiService): "subject", "sendToInterestedUsers", "customMesg", "messageFormatter", "extraHeaders"] - possible_modes = ("change", "failing", "passing", "problem", "warnings") + possible_modes = ("change", "failing", "passing", "problem", "warnings", "exception") def __init__(self, fromaddr, mode=("failing", "passing", "warnings"), categories=None, builders=None, addLogs=False, @@ -220,6 +222,8 @@ def __init__(self, fromaddr, mode=("failing", "passing", "warnings"), - "problem": send mail about a build which failed when the previous build passed - "warnings": send mail if a build contain warnings + - "exception": send mail if a build fails due to an exception + - "all": always send mail Defaults to ("failing", "passing", "warnings"). @type builders: list of strings @@ -316,7 +320,7 @@ def __init__(self, fromaddr, mode=("failing", "passing", "warnings"), self.fromaddr = fromaddr if isinstance(mode, basestring): if mode == "all": - mode = ("failing", "passing", "warnings") + mode = ("failing", "passing", "warnings", "exception") elif mode == "warnings": mode = ("failing", "warnings") else: @@ -434,6 +438,8 @@ def isMailNeeded(self, build, results): return True if "warnings" in self.mode and results == WARNINGS: return True + if "exception" in self.mode and results == EXCEPTION: + return True return False diff --git a/master/buildbot/status/master.py b/master/buildbot/status/master.py index 862ed8b63aa..a7c62c11a25 100644 --- a/master/buildbot/status/master.py +++ b/master/buildbot/status/master.py @@ -69,7 +69,15 @@ def reconfigService(self, new_config): for sr in list(self): yield defer.maybeDeferred(lambda : sr.disownServiceParent()) - sr.master = None + + # WebStatus instances tend to "hang around" longer than we'd like - + # if there's an ongoing HTTP request, or even a connection held + # open by keepalive, then users may still be talking to an old + # WebStatus. So WebStatus objects get to keep their `master` + # attribute, but all other status objects lose theirs. And we want + # to test this without importing WebStatus, so we use name + if not sr.__class__.__name__.endswith('WebStatus'): + sr.master = None for sr in new_config.status: sr.master = self.master @@ -336,6 +344,7 @@ def builderAdded(self, name, basedir, category=None): log.msg("added builder %s in category %s" % (name, category)) # an unpickled object might not have category set from before, # so set it here to make sure + builder_status.category = category builder_status.master = self.master builder_status.basedir = os.path.join(self.basedir, basedir) builder_status.name = name # it might have been updated diff --git a/master/buildbot/status/web/base.py b/master/buildbot/status/web/base.py index ab36161593b..c4447ef32bd 100644 --- a/master/buildbot/status/web/base.py +++ b/master/buildbot/status/web/base.py @@ -436,10 +436,12 @@ def get_line_values(self, req, build, include_builder=True): css_class = css_classes.get(results, "") ss_list = build.getSourceStamps() if ss_list: - # TODO: support multiple sourcestamps in web interface repo = ss_list[0].repository if all_got_revision: - rev = all_got_revision[ss_list[0].codebase] + if len(ss_list) == 1: + rev = all_got_revision.get(ss_list[0].codebase, "??") + else: + rev = "multiple rev." else: rev = "??" else: @@ -479,7 +481,7 @@ def map_branches(branches): # jinja utilities def createJinjaEnv(revlink=None, changecommentlink=None, - repositories=None, projects=None): + repositories=None, projects=None, jinja_loaders=None): ''' Create a jinja environment changecommentlink is used to render HTML in the WebStatus and for mail changes @@ -501,10 +503,12 @@ def createJinjaEnv(revlink=None, changecommentlink=None, # See http://buildbot.net/trac/ticket/658 assert not hasattr(sys, "frozen"), 'Frozen config not supported with jinja (yet)' - default_loader = jinja2.PackageLoader('buildbot.status.web', 'templates') - root = os.path.join(os.getcwd(), 'templates') - loader = jinja2.ChoiceLoader([jinja2.FileSystemLoader(root), - default_loader]) + all_loaders = [jinja2.FileSystemLoader(os.path.join(os.getcwd(), 'templates'))] + if jinja_loaders: + all_loaders.extend(jinja_loaders) + all_loaders.append(jinja2.PackageLoader('buildbot.status.web', 'templates')) + loader = jinja2.ChoiceLoader(all_loaders) + env = jinja2.Environment(loader=loader, extensions=['jinja2.ext.i18n'], trim_blocks=True, @@ -512,6 +516,8 @@ def createJinjaEnv(revlink=None, changecommentlink=None, env.install_null_translations() # needed until we have a proper i18n backend + env.tests['mapping'] = lambda obj : isinstance(obj, dict) + env.filters.update(dict( urlencode = urllib.quote, email = emailfilter, @@ -786,3 +792,14 @@ class AlmostStrictUndefined(jinja2.StrictUndefined): fully as strict as StrictUndefined ''' def __nonzero__(self): return False + +_charsetRe = re.compile('charset=([^;]*)', re.I) +def getRequestCharset(req): + """Get the charset for an x-www-form-urlencoded request""" + # per http://stackoverflow.com/questions/708915/detecting-the-character-encoding-of-an-http-post-request + hdr = req.getHeader('Content-Type') + if hdr: + mo = _charsetRe.search(hdr) + if mo: + return mo.group(1).strip() + return 'utf-8' # reasonable guess, works for ascii diff --git a/master/buildbot/status/web/baseweb.py b/master/buildbot/status/web/baseweb.py index 2d9fa42e60d..ec5eaad77c4 100644 --- a/master/buildbot/status/web/baseweb.py +++ b/master/buildbot/status/web/baseweb.py @@ -148,7 +148,7 @@ def __init__(self, http_port=None, distrib_port=None, allowForce=None, order_console_by_time=False, changecommentlink=None, revlink=None, projects=None, repositories=None, authz=None, logRotateLength=None, maxRotatedFiles=None, - change_hook_dialects = {}, provide_feeds=None): + change_hook_dialects = {}, provide_feeds=None, jinja_loaders=None): """Run a web server that provides Buildbot status. @type http_port: int or L{twisted.application.strports} string @@ -266,6 +266,10 @@ def __init__(self, http_port=None, distrib_port=None, allowForce=None, Otherwise, a dictionary of strings of the type of feeds provided. Current possibilities are "atom", "json", and "rss" + + @type jinja_loaders: None or list + @param jinja_loaders: If not empty, a list of additional Jinja2 loader + objects to search for templates. """ service.MultiService.__init__(self) @@ -344,6 +348,8 @@ def __init__(self, http_port=None, distrib_port=None, allowForce=None, else: self.provide_feeds = provide_feeds + self.jinja_loaders = jinja_loaders + def setupUsualPages(self, numbuilds, num_events, num_events_max): #self.putChild("", IndexOrWaterfallRedirection()) self.putChild("waterfall", WaterfallStatusResource(num_events=num_events, @@ -377,8 +383,6 @@ def __repr__(self): (self.http_port, self.distrib_port, hex(id(self)))) def setServiceParent(self, parent): - service.MultiService.setServiceParent(self, parent) - # this class keeps a *separate* link to the buildmaster, rather than # just using self.parent, so that when we are "disowned" (and thus # parent=None), any remaining HTTP clients of this WebStatus will still @@ -404,7 +408,7 @@ def either(a,b): # a if a else b for py2.4 else: revlink = self.master.config.revlink self.templates = createJinjaEnv(revlink, self.changecommentlink, - self.repositories, self.projects) + self.repositories, self.projects, self.jinja_loaders) if not self.site: @@ -444,6 +448,8 @@ def _openLogFile(self, path): self.setupSite() + service.MultiService.setServiceParent(self, parent) + def setupSite(self): # this is responsible for creating the root resource. It isn't done # at __init__ time because we need to reference the parent's basedir. diff --git a/master/buildbot/status/web/build.py b/master/buildbot/status/web/build.py index 8ebb2075968..13f43eba7cb 100644 --- a/master/buildbot/status/web/build.py +++ b/master/buildbot/status/web/build.py @@ -22,7 +22,8 @@ from twisted.python import log from buildbot.status.web.base import HtmlResource, \ css_classes, path_to_build, path_to_builder, path_to_slave, \ - getAndCheckProperties, ActionResource, path_to_authzfail + getAndCheckProperties, ActionResource, path_to_authzfail, \ + getRequestCharset from buildbot.schedulers.forcesched import ForceScheduler, TextParameter from buildbot.status.web.step import StepsResource from buildbot.status.web.tests import TestsResource @@ -53,6 +54,7 @@ def performAction(self, req): log.msg("web rebuild of build %s:%s" % (builder_name, b.getNumber())) name =authz.getUsernameFull(req) comments = req.args.get("comments", [""])[0] + comments.decode(getRequestCharset(req)) reason = ("The web-page 'rebuild' button was pressed by " "'%s': %s\n" % (name, comments)) msg = "" @@ -105,6 +107,7 @@ def performAction(self, req): (b.getBuilder().getName(), b.getNumber())) name = authz.getUsernameFull(req) comments = req.args.get("comments", [""])[0] + comments.decode(getRequestCharset(req)) # html-quote both the username and comments, just to be safe reason = ("The web-page 'stop build' button was pressed by " "'%s': %s\n" % (html.escape(name), html.escape(comments))) @@ -160,16 +163,10 @@ def content(self, req, cxt): cxt['tests_link'] = req.childLink("tests") ssList = b.getSourceStamps() - # TODO: support multiple sourcestamps - ss = cxt['ss'] = ssList[0] - - if ss.branch is None and ss.revision is None and ss.patch is None and not ss.changes: - cxt['most_recent_rev_build'] = True + sourcestamps = cxt['sourcestamps'] = ssList all_got_revisions = b.getAllGotRevisions() - if all_got_revisions: - got_revision = all_got_revisions.get(ss.codebase, "??") - cxt['got_revision'] = str(got_revision) + cxt['got_revisions'] = all_got_revisions try: cxt['slave_url'] = path_to_slave(req, status.getSlave(b.getSlavename())) @@ -224,10 +221,13 @@ def content(self, req, cxt): ps = cxt['properties'] = [] for name, value, source in b.getProperties().asList(): - uvalue = unicode(value) - p = { 'name': name, 'value': uvalue, 'source': source} - if len(uvalue) > 500: - p['short_value'] = uvalue[:500] + if not isinstance(value, dict): + cxt_value = unicode(value) + else: + cxt_value = value + p = { 'name': name, 'value': cxt_value, 'source': source} + if len(cxt_value) > 500: + p['short_value'] = cxt_value[:500] if name in parameters: param = parameters[name] if isinstance(param, TextParameter): @@ -249,8 +249,13 @@ def content(self, req, cxt): now = util.now() cxt['elapsed'] = util.formatInterval(now - start) - cxt['exactly'] = (ss.revision is not None) or b.getChanges() - + exactly = True + has_changes = False + for ss in sourcestamps: + exactly = exactly and (ss.revision is not None) + has_changes = has_changes or ss.changes + cxt['exactly'] = (exactly) or b.getChanges() + cxt['has_changes'] = has_changes cxt['build_url'] = path_to_build(req, b) cxt['authz'] = self.getAuthz(req) @@ -268,6 +273,7 @@ def stop(self, req, auth_ok=False): name = self.getAuthz(req).getUsernameFull(req) comments = req.args.get("comments", [""])[0] + comments.decode(getRequestCharset(req)) # html-quote both the username and comments, just to be safe reason = ("The web-page 'stop build' button was pressed by " "'%s': %s\n" % (html.escape(name), html.escape(comments))) diff --git a/master/buildbot/status/web/builder.py b/master/buildbot/status/web/builder.py index 8052eb18855..21b0278c21e 100644 --- a/master/buildbot/status/web/builder.py +++ b/master/buildbot/status/web/builder.py @@ -22,7 +22,8 @@ from buildbot.status.web.base import HtmlResource, BuildLineMixin, \ path_to_build, path_to_slave, path_to_builder, path_to_change, \ path_to_root, ICurrentBox, build_get_class, \ - map_branches, path_to_authzfail, ActionResource + map_branches, path_to_authzfail, ActionResource, \ + getRequestCharset from buildbot.schedulers.forcesched import ForceScheduler from buildbot.schedulers.forcesched import InheritBuildParameter from buildbot.schedulers.forcesched import ValidationError @@ -141,11 +142,16 @@ def performAction(self, req): "forcescheduler arg not found")) return - args = {} + args = req.args.copy() + + # decode all of the args + encoding = getRequestCharset(req) + for name, argl in args.iteritems(): + args[name] = [ arg.decode(encoding) for arg in argl ] + # damn html's ungeneric checkbox implementation... - for cb in req.args.get("checkbox", []): + for cb in args.get("checkbox", []): args[cb] = True - args.update(req.args) builder_name = self.builder_status.getName() @@ -183,7 +189,7 @@ def buildForceContext(cxt, req, master, buildername=None): else: # filter out unicode chars, and html stuff if type(default)==unicode: - default = html.escape(default.encode('ascii','ignore')) + default = html.escape(default.encode('utf-8','ignore')) default_props[pname] = default cxt['force_schedulers'] = force_schedulers cxt['default_props'] = default_props @@ -481,9 +487,12 @@ class BuildersResource(HtmlResource): @defer.inlineCallbacks def content(self, req, cxt): status = self.getStatus(req) + encoding = getRequestCharset(req) builders = req.args.get("builder", status.getBuilderNames()) - branches = [b for b in req.args.get("branch", []) if b] + branches = [ b.decode(encoding) + for b in req.args.get("branch", []) + if b ] # get counts of pending builds for each builder brstatus_ds = [] @@ -515,7 +524,7 @@ def keep_count(statuses, builderName): b = builds[0] bld['build_url'] = (bld['link'] + "/builds/%d" % b.getNumber()) label = None - all_got_revisions = b.getAllGotRevisions() or {} + all_got_revisions = b.getAllGotRevisions() # If len = 1 then try if revision can be used as label. if len(all_got_revisions) == 1: label = all_got_revisions[all_got_revisions.keys()[0]] diff --git a/master/buildbot/status/web/console.py b/master/buildbot/status/web/console.py index 01490831451..368941693f7 100644 --- a/master/buildbot/status/web/console.py +++ b/master/buildbot/status/web/console.py @@ -234,30 +234,33 @@ def getBuildsForRevision(self, request, builder, builderName, lastRevision, number = 0 while build and number < numBuilds: debugInfo["builds_scanned"] += 1 - number += 1 - - # Get the last revision in this build. - # We first try "got_revision", but if it does not work, then - # we try "revision". - got_rev = build.getProperty("got_revision", build.getProperty("revision", -1)) - if got_rev != -1 and not self.comparator.isValidRevision(got_rev): - got_rev = -1 - - # We ignore all builds that don't have last revisions. - # TODO(nsylvain): If the build is over, maybe it was a problem - # with the update source step. We need to find a way to tell the - # user that his change might have broken the source update. - if got_rev != -1: - details = self.getBuildDetails(request, builderName, build) - devBuild = DevBuild(got_rev, build, details) - builds.append(devBuild) - - # Now break if we have enough builds. - current_revision = self.getChangeForBuild( - build, revision) - if self.comparator.isRevisionEarlier( - devBuild, current_revision): - break + + # The console page cannot handle builds that have more than 1 revision + if len(build.getSourceStamps()) == 1: + number += 1 + # Get the last revision in this build. + # We first try "got_revision", but if it does not work, then + # we try "revision". + got_rev = build.getProperty("got_revision", build.getProperty("revision", -1)) + if got_rev != -1 and not self.comparator.isValidRevision(got_rev): + got_rev = -1 + + + # We ignore all builds that don't have last revisions. + # TODO(nsylvain): If the build is over, maybe it was a problem + # with the update source step. We need to find a way to tell the + # user that his change might have broken the source update. + if got_rev != -1: + details = self.getBuildDetails(request, builderName, build) + devBuild = DevBuild(got_rev, build, details) + builds.append(devBuild) + + # Now break if we have enough builds. + current_revision = self.getChangeForBuild( + build, revision) + if self.comparator.isRevisionEarlier( + devBuild, current_revision): + break build = build.getPreviousBuild() diff --git a/master/buildbot/status/web/feeds.py b/master/buildbot/status/web/feeds.py index 39f49a33618..701339a8af0 100644 --- a/master/buildbot/status/web/feeds.py +++ b/master/buildbot/status/web/feeds.py @@ -179,7 +179,7 @@ def content(self, request): # title: trunk r22191 (plus patch) failed on # 'i686-debian-sarge1 shared gcc-3.3.5' ss_list = build.getSourceStamps() - all_got_revisions = build.getAllGotRevisions() or {} + all_got_revisions = build.getAllGotRevisions() src_cxts = [] for ss in ss_list: sc = {} diff --git a/master/buildbot/status/web/grid.py b/master/buildbot/status/web/grid.py index 49c7193a792..81ad44b9411 100644 --- a/master/buildbot/status/web/grid.py +++ b/master/buildbot/status/web/grid.py @@ -193,17 +193,19 @@ def content(self, request, cxt): for build in self.getRecentBuilds(builder, numBuilds, branch): #TODO: support multiple sourcestamps - ss = build.getSourceStamps(absolute=True)[0] - key= self.getSourceStampKey(ss) - for i in range(len(stamps)): - if key == self.getSourceStampKey(stamps[i]) and builds[i] is None: - builds[i] = build + if len(build.getSourceStamps()) == 1: + ss = build.getSourceStamps(absolute=True)[0] + key= self.getSourceStampKey(ss) + for i in range(len(stamps)): + if key == self.getSourceStampKey(stamps[i]) and builds[i] is None: + builds[i] = build b = yield self.builder_cxt(request, builder) b['builds'] = [] for build in builds: b['builds'].append(self.build_cxt(request, build)) + cxt['builders'].append(b) template = request.site.buildbot_service.templates.get_template("grid.html") @@ -263,11 +265,12 @@ def content(self, request, cxt): for build in self.getRecentBuilds(builder, numBuilds, branch): #TODO: support multiple sourcestamps - ss = build.getSourceStamps(absolute=True)[0] - key = self.getSourceStampKey(ss) - for i in range(len(stamps)): - if key == self.getSourceStampKey(stamps[i]) and builds[i] is None: - builds[i] = build + if len(build.getSourceStamps()) == 1: + ss = build.getSourceStamps(absolute=True)[0] + key = self.getSourceStampKey(ss) + for i in range(len(stamps)): + if key == self.getSourceStampKey(stamps[i]) and builds[i] is None: + builds[i] = build b = yield self.builder_cxt(request, builder) builders.append(b) @@ -275,5 +278,4 @@ def content(self, request, cxt): builder_builds.append(map(lambda b: self.build_cxt(request, b), builds)) template = request.site.buildbot_service.templates.get_template('grid_transposed.html') - yield template.render(**cxt) - + defer.returnValue(template.render(**cxt)) \ No newline at end of file diff --git a/master/buildbot/status/web/hooks/poller.py b/master/buildbot/status/web/hooks/poller.py new file mode 100644 index 00000000000..1ac29a4f599 --- /dev/null +++ b/master/buildbot/status/web/hooks/poller.py @@ -0,0 +1,54 @@ +# This file is part of Buildbot. Buildbot is free software: you can +# redistribute it and/or modify it under the terms of the GNU General Public +# License as published by the Free Software Foundation, version 2. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +# details. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, write to the Free Software Foundation, Inc., 51 +# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Copyright Buildbot Team Members + +# This change hook allows GitHub or a hand crafted curl inovcation to "knock on +# the door" and trigger a change source to poll. + +from buildbot.changes.base import PollingChangeSource + + +def getChanges(req, options=None): + change_svc = req.site.buildbot_service.master.change_svc + poll_all = not "poller" in req.args + + allow_all = True + allowed = [] + if isinstance(options, dict) and "allowed" in options: + allow_all = False + allowed = options["allowed"] + + pollers = [] + + for source in change_svc: + if not isinstance(source, PollingChangeSource): + continue + if not hasattr(source, "name"): + continue + if not poll_all and not source.name in req.args['poller']: + continue + if not allow_all and not source.name in allowed: + continue + pollers.append(source) + + if not poll_all: + missing = set(req.args['poller']) - set(s.name for s in pollers) + if missing: + raise ValueError("Could not find pollers: %s" % ",".join(missing)) + + for p in pollers: + p.doPoll() + + return [], None + diff --git a/master/buildbot/status/web/templates/build.html b/master/buildbot/status/web/templates/build.html index 9a669f7fc21..8b60b90b53a 100644 --- a/master/buildbot/status/web/templates/build.html +++ b/master/buildbot/status/web/templates/build.html @@ -36,44 +36,52 @@

{% endif %} {% endif %} -

SourceStamp:

- - -{% set ss_class = cycler('alt','') %} - -{% if ss.project %} - +

+{% if sourcestamps|count == 1 %} +SourceStamp: +{% else %} +SourceStamps: {% endif %} +

-{% if ss.repository %} - -{% endif %} +{% for ss in sourcestamps %} +

{{ ss.codebase }}

+
Project{{ ss.project|projectlink }}
Repository{{ ss.repository|repolink }}
+ {% set ss_class = cycler('alt','') %} -{% if ss.branch %} - -{% endif %} + {% if ss.project %} + + {% endif %} -{% if ss.revision %} - -{% endif %} + {% if ss.repository %} + + {% endif %} -{% if got_revision %} - -{% endif %} + {% if ss.branch %} + + {% endif %} -{% if ss.patch %} - -{% endif %} + {% if ss.revision %} + + {% endif %} -{% if ss.changes %} - -{% endif %} + {% if got_revisions[ss.codebase] %} + + {% endif %} -{% if most_recent_rev_build %} - -{% endif %} + {% if ss.patch %} + + {% endif %} -
Branch{{ ss.branch|e }}
Project{{ ss.project|projectlink }}
Revision{{ ss.revision|revlink(ss.repository) }}
Repository{{ ss.repository|repolink }}
Got Revision{{ got_revision|revlink(ss.repository) }}
Branch{{ ss.branch|e }}
PatchYES
Revision{{ ss.revision|revlink(ss.repository) }}
Changessee below
Got Revision{{ got_revisions[ss.codebase]|revlink(ss.repository) }}
Build of most recent revision
PatchYES
+ {% if ss.changes %} + Changes{{ ss.changes|count }} change{{ 's' if ss.changes|count > 1 else '' }} + {% endif %} + + {% if not ss.branch and not ss.revision and not ss.patch and not ss.changes %} + Build of most recent revision + {% endif %} + +{% endfor %} {# # TODO: turn this into a table, or some other sort of definition-list @@ -141,9 +149,19 @@

Build Properties:

{{ p.name|e }} {% if p.short_value %} - {{ p.short_value|e }} .. [property value too long] + {{ p.short_value|e }} .. [property value too long] {% else %} - {{ p.value|e }} + {% if p.value is not mapping %} + {{ p.value|e }} + {% else %} + + + {%- for key, value in p.value.items() recursive %} + + {% endfor %} +
{{ key|e }}{{ value|e }}
+ + {% endif %} {% endif %} {{ p.source|e }} @@ -173,7 +191,7 @@

Forced Build Properties:

{% endfor %} -

Blamelist:

+

Responsible Users:

{% if responsible_users %}
    @@ -197,26 +215,29 @@

    Timing:

    {% if authz.advertiseAction('forceBuild', request) %}

    Resubmit Build:

    - {{ forms.rebuild_build(build_url+"/rebuild", authz, exactly, ss) }} + {{ forms.rebuild_build(build_url+"/rebuild", authz, exactly, sourcestamps[0]) }} {% endif %}
    -{% if ss.changes %} -
    -

    All Changes:

    -
      - {% for c in ss.changes %} -

    1. Change #{{ c.number }}

      - {{ change(c.asDict()) }} -
      2. - {% else %} -
    2. no changes
      3. - {% endfor %} -
    -
    +{% if has_changes %} +
    +

    All Changes:

    + {% for ss in sourcestamps %} + {% if ss.changes %} +

    {{ ss.codebase }}:

    +
      + {% for c in ss.changes %} +

    1. Change #{{ c.number }}

      + {{ change(c.asDict()) }} +
      2. + {% endfor %} +
    + {% endif %} + {% endfor %} +
    {% endif %} {% endblock %} diff --git a/master/buildbot/steps/master.py b/master/buildbot/steps/master.py index 42680f0767f..508980460c7 100644 --- a/master/buildbot/steps/master.py +++ b/master/buildbot/steps/master.py @@ -18,6 +18,7 @@ from twisted.internet import reactor from buildbot.process.buildstep import BuildStep from buildbot.process.buildstep import SUCCESS, FAILURE +from twisted.internet import error from twisted.internet.protocol import ProcessProtocol class MasterShellCommand(BuildStep): @@ -29,19 +30,16 @@ class MasterShellCommand(BuildStep): name='MasterShellCommand' description='Running' descriptionDone='Ran' - renderables = [ 'command', 'env' ] + descriptionSuffix = None + renderables = [ 'command', 'env', 'description', 'descriptionDone', 'descriptionSuffix' ] haltOnFailure = True flunkOnFailure = True def __init__(self, command, - description=None, descriptionDone=None, - env=None, path=None, usePTY=0, + description=None, descriptionDone=None, descriptionSuffix=None, + env=None, path=None, usePTY=0, interruptSignal="KILL", **kwargs): BuildStep.__init__(self, **kwargs) - self.addFactoryArguments(description=description, - descriptionDone=descriptionDone, - env=env, path=path, usePTY=usePTY, - command=command) self.command=command if description: @@ -52,9 +50,14 @@ def __init__(self, command, self.descriptionDone = descriptionDone if isinstance(self.descriptionDone, str): self.descriptionDone = [self.descriptionDone] + if descriptionSuffix: + self.descriptionSuffix = descriptionSuffix + if isinstance(self.descriptionSuffix, str): + self.descriptionSuffix = [self.descriptionSuffix] self.env=env self.path=path self.usePTY=usePTY + self.interruptSignal = interruptSignal class LocalPP(ProcessProtocol): def __init__(self, step): @@ -67,7 +70,10 @@ def errReceived(self, data): self.step.stdio_log.addStderr(data) def processEnded(self, status_object): - self.step.stdio_log.addHeader("exit status %d\n" % status_object.value.exitCode) + if status_object.value.exitCode is not None: + self.step.stdio_log.addHeader("exit status %d\n" % status_object.value.exitCode) + if status_object.value.signal is not None: + self.step.stdio_log.addHeader("signal %s\n" % status_object.value.signal) self.step.processEnded(status_object) def start(self): @@ -100,7 +106,7 @@ def start(self): stdio_log.addHeader("** RUNNING ON BUILDMASTER **\n") stdio_log.addHeader(" in dir %s\n" % os.getcwd()) stdio_log.addHeader(" argv: %s\n" % (argv,)) - self.step_status.setText(list(self.description)) + self.step_status.setText(self.describe()) if self.env is None: env = os.environ @@ -120,14 +126,35 @@ def subst(match): stdio_log.addHeader(" env: %r\n" % (env,)) # TODO add a timeout? - reactor.spawnProcess(self.LocalPP(self), argv[0], argv, + self.process = reactor.spawnProcess(self.LocalPP(self), argv[0], argv, path=self.path, usePTY=self.usePTY, env=env ) # (the LocalPP object will call processEnded for us) def processEnded(self, status_object): - if status_object.value.exitCode != 0: - self.step_status.setText(["failed (%d)" % status_object.value.exitCode]) + if status_object.value.signal is not None: + self.descriptionDone = ["killed (%s)" % status_object.value.signal] + self.step_status.setText(self.describe(done=True)) + self.finished(FAILURE) + elif status_object.value.exitCode != 0: + self.descriptionDone = ["failed (%d)" % status_object.value.exitCode] + self.step_status.setText(self.describe(done=True)) self.finished(FAILURE) else: - self.step_status.setText(list(self.descriptionDone)) + self.step_status.setText(self.describe(done=True)) self.finished(SUCCESS) + + def describe(self, done=False): + desc = self.descriptionDone if done else self.description + if self.descriptionSuffix: + desc = desc[:] + desc.extend(self.descriptionSuffix) + return desc + + def interrupt(self, reason): + try: + self.process.signalProcess(self.interruptSignal) + except KeyError: # Process not started yet + pass + except error.ProcessExitedAlready: + pass + BuildStep.interrupt(self, reason) diff --git a/master/buildbot/steps/maxq.py b/master/buildbot/steps/maxq.py index 1d7a1409ea2..bb40677af4b 100644 --- a/master/buildbot/steps/maxq.py +++ b/master/buildbot/steps/maxq.py @@ -26,7 +26,6 @@ def __init__(self, testdir=None, **kwargs): config.error("please pass testdir") kwargs['command'] = 'run_maxq.py %s' % (testdir,) ShellCommand.__init__(self, **kwargs) - self.addFactoryArguments(testdir=testdir) def commandComplete(self, cmd): output = cmd.logs['stdio'].getText() @@ -35,7 +34,7 @@ def commandComplete(self, cmd): def evaluateCommand(self, cmd): # treat a nonzero exit status as a failure, if no other failures are # detected - if not self.failures and cmd.rc != 0: + if not self.failures and cmd.didFail(): self.failures = 1 if self.failures: return FAILURE diff --git a/master/buildbot/steps/package/rpm/rpmbuild.py b/master/buildbot/steps/package/rpm/rpmbuild.py index fa54956507f..b43583584ac 100644 --- a/master/buildbot/steps/package/rpm/rpmbuild.py +++ b/master/buildbot/steps/package/rpm/rpmbuild.py @@ -41,16 +41,6 @@ def __init__(self, vcsRevision=False, **kwargs): ShellCommand.__init__(self, **kwargs) - self.addFactoryArguments(topdir=topdir, - builddir=builddir, - rpmdir=rpmdir, - sourcedir=sourcedir, - specdir=specdir, - srcrpmdir=srcrpmdir, - specfile=specfile, - dist=dist, - autoRelease=autoRelease, - vcsRevision=vcsRevision) self.rpmbuild = ( 'rpmbuild --define "_topdir %s" --define "_builddir %s"' ' --define "_rpmdir %s" --define "_sourcedir %s"' diff --git a/master/buildbot/steps/python.py b/master/buildbot/steps/python.py index 81f4538a88e..e94d4209ff3 100644 --- a/master/buildbot/steps/python.py +++ b/master/buildbot/steps/python.py @@ -58,7 +58,7 @@ def createSummary(self, log): self.errors = errors def evaluateCommand(self, cmd): - if cmd.rc != 0: + if cmd.didFail(): return FAILURE if self.warnings or self.errors: return WARNINGS @@ -118,7 +118,7 @@ def createSummary(self, log): def evaluateCommand(self, cmd): - if cmd.rc != 0: + if cmd.didFail(): return FAILURE for m in self.flunkingIssues: if self.getProperty("pyflakes-%s" % m): @@ -260,17 +260,6 @@ def __init__(self, sphinx_sourcedir='.', sphinx_builddir=None, command.extend([sphinx_sourcedir, sphinx_builddir]) self.setCommand(command) - self.addFactoryArguments( - sphinx = sphinx, - sphinx_sourcedir = sphinx_sourcedir, - sphinx_builddir = sphinx_builddir, - sphinx_builder = sphinx_builder, - tags = tags, - defines = defines, - mode = mode, - ) - - def createSummary(self, log): msgs = ['WARNING', 'ERROR', 'SEVERE'] diff --git a/master/buildbot/steps/python_twisted.py b/master/buildbot/steps/python_twisted.py index d49da432462..0123e54277a 100644 --- a/master/buildbot/steps/python_twisted.py +++ b/master/buildbot/steps/python_twisted.py @@ -47,7 +47,6 @@ class HLint(ShellCommand): def __init__(self, python=None, **kwargs): ShellCommand.__init__(self, **kwargs) - self.addFactoryArguments(python=python) self.python = python def start(self): @@ -87,14 +86,14 @@ def commandComplete(self, cmd): def evaluateCommand(self, cmd): # warnings are in stdout, rc is always 0, unless the tools break - if cmd.rc != 0: + if cmd.didFail(): return FAILURE if self.warnings: return WARNINGS return SUCCESS def getText2(self, cmd, results): - if cmd.rc != 0: + if cmd.didFail(): return ["hlint"] return ["%d hlin%s" % (self.warnings, self.warnings == 1 and 't' or 'ts')] @@ -287,17 +286,6 @@ def __init__(self, reactor=UNSPECIFIED, python=None, trial=None, timeout. """ ShellCommand.__init__(self, **kwargs) - self.addFactoryArguments(reactor=reactor, - python=python, - trial=trial, - testpath=testpath, - tests=tests, - testChanges=testChanges, - recurse=recurse, - randomly=randomly, - trialMode=trialMode, - trialArgs=trialArgs, - ) if python: self.python = python @@ -421,7 +409,7 @@ def commandComplete(self, cmd): text = [] text2 = "" - if cmd.rc == 0: + if not cmd.didFail(): if parsed: results = SUCCESS if total: diff --git a/master/buildbot/steps/shell.py b/master/buildbot/steps/shell.py index 71b4bc20ec5..ee5f15143e2 100644 --- a/master/buildbot/steps/shell.py +++ b/master/buildbot/steps/shell.py @@ -65,9 +65,13 @@ class ShellCommand(buildstep.LoggingBuildStep): """ name = "shell" - renderables = [ 'description', 'descriptionDone', 'slaveEnvironment', 'remote_kwargs', 'command', 'logfiles' ] + renderables = buildstep.LoggingBuildStep.renderables + [ + 'slaveEnvironment', 'remote_kwargs', 'command'] + description = None # set this to a list of short strings to override descriptionDone = None # alternate description when the step is complete + descriptionSuffix = None # extra information to append to suffix + command = None # set this to a command, or set in kwargs # logfiles={} # you can also set 'logfiles' to a dictionary, and it # will be merged with any logfiles= argument passed in @@ -78,7 +82,7 @@ class ShellCommand(buildstep.LoggingBuildStep): flunkOnFailure = True def __init__(self, workdir=None, - description=None, descriptionDone=None, + description=None, descriptionDone=None, descriptionSuffix=None, command=None, usePTY="slave-config", **kwargs): @@ -95,6 +99,12 @@ def __init__(self, workdir=None, self.descriptionDone = descriptionDone if isinstance(self.descriptionDone, str): self.descriptionDone = [self.descriptionDone] + + if descriptionSuffix: + self.descriptionSuffix = descriptionSuffix + if isinstance(self.descriptionSuffix, str): + self.descriptionSuffix = [self.descriptionSuffix] + if command: self.setCommand(command) @@ -105,17 +115,11 @@ def __init__(self, workdir=None, buildstep_kwargs[k] = kwargs[k] del kwargs[k] buildstep.LoggingBuildStep.__init__(self, **buildstep_kwargs) - self.addFactoryArguments(workdir=workdir, - description=description, - descriptionDone=descriptionDone, - command=command) # everything left over goes to the RemoteShellCommand kwargs['workdir'] = workdir # including a copy of 'workdir' kwargs['usePTY'] = usePTY self.remote_kwargs = kwargs - # we need to stash the RemoteShellCommand's args too - self.addFactoryArguments(**kwargs) def setBuild(self, build): buildstep.LoggingBuildStep.setBuild(self, build) @@ -148,6 +152,13 @@ def _flattenList(self, mainlist, commands): self._flattenList(mainlist, x) def describe(self, done=False): + desc = self._describe(done) + if self.descriptionSuffix: + desc = desc[:] + desc.extend(self.descriptionSuffix) + return desc + + def _describe(self, done=False): """Return a list of short strings to describe this step, for the status display. This uses the first few words of the shell command. You can replace this by setting .description in your subclass, or by @@ -276,7 +287,7 @@ def commandComplete(self, cmd): self.setProperty("tree-size-KiB", self.kib, "treesize") def evaluateCommand(self, cmd): - if cmd.rc != 0: + if cmd.didFail(): return FAILURE if self.kib is None: return WARNINGS # not sure how 'du' could fail, but whatever @@ -303,15 +314,11 @@ def __init__(self, property=None, extract_fn=None, strip=True, **kwargs): ShellCommand.__init__(self, **kwargs) - self.addFactoryArguments(property=self.property) - self.addFactoryArguments(extract_fn=self.extract_fn) - self.addFactoryArguments(strip=self.strip) - self.property_changes = {} def commandComplete(self, cmd): if self.property: - if cmd.rc != 0: + if cmd.didFail(): return result = cmd.logs['stdio'].getText() if self.strip: result = result.strip() @@ -405,12 +412,6 @@ def __init__(self, # And upcall to let the base class do its work ShellCommand.__init__(self, **kwargs) - self.addFactoryArguments(warningPattern=warningPattern, - directoryEnterPattern=directoryEnterPattern, - directoryLeavePattern=directoryLeavePattern, - warningExtractor=warningExtractor, - maxWarnCount=maxWarnCount, - suppressionFile=suppressionFile) self.suppressions = [] self.directoryStack = [] @@ -584,7 +585,7 @@ def createSummary(self, log): def evaluateCommand(self, cmd): - if ( cmd.rc != 0 or + if ( cmd.didFail() or ( self.maxWarnCount != None and self.warnCount > self.maxWarnCount ) ): return FAILURE if self.warnCount: @@ -660,7 +661,7 @@ def evaluateCommand(self, cmd): passed = 0 failed = 0 rc = SUCCESS - if cmd.rc > 0: + if cmd.didFail(): rc = FAILURE # New version of Test::Harness? diff --git a/master/buildbot/steps/slave.py b/master/buildbot/steps/slave.py index 7e1eedaacf1..4d9b969e64c 100644 --- a/master/buildbot/steps/slave.py +++ b/master/buildbot/steps/slave.py @@ -30,8 +30,6 @@ class SetPropertiesFromEnv(buildstep.BuildStep): def __init__(self, variables, source="SlaveEnvironment", **kwargs): buildstep.BuildStep.__init__(self, **kwargs) - self.addFactoryArguments(variables = variables, - source = source) self.variables = variables self.source = source @@ -74,7 +72,6 @@ class FileExists(buildstep.BuildStep): def __init__(self, file, **kwargs): buildstep.BuildStep.__init__(self, **kwargs) - self.addFactoryArguments(file = file) self.file = file def start(self): @@ -88,7 +85,7 @@ def start(self): d.addErrback(self.failed) def commandComplete(self, cmd): - if cmd.rc != 0: + if cmd.didFail(): self.step_status.setText(["File not found."]) self.finished(FAILURE) return @@ -115,7 +112,6 @@ class RemoveDirectory(buildstep.BuildStep): def __init__(self, dir, **kwargs): buildstep.BuildStep.__init__(self, **kwargs) - self.addFactoryArguments(dir = dir) self.dir = dir def start(self): @@ -129,7 +125,7 @@ def start(self): d.addErrback(self.failed) def commandComplete(self, cmd): - if cmd.rc != 0: + if cmd.didFail(): self.step_status.setText(["Delete failed."]) self.finished(FAILURE) return @@ -150,7 +146,6 @@ class MakeDirectory(buildstep.BuildStep): def __init__(self, dir, **kwargs): buildstep.BuildStep.__init__(self, **kwargs) - self.addFactoryArguments(dir = dir) self.dir = dir def start(self): @@ -164,7 +159,7 @@ def start(self): d.addErrback(self.failed) def commandComplete(self, cmd): - if cmd.rc != 0: + if cmd.didFail(): self.step_status.setText(["Create failed."]) self.finished(FAILURE) return diff --git a/master/buildbot/steps/source/__init__.py b/master/buildbot/steps/source/__init__.py index 9dcb31ef310..fa414452231 100644 --- a/master/buildbot/steps/source/__init__.py +++ b/master/buildbot/steps/source/__init__.py @@ -15,7 +15,7 @@ from buildbot.steps.source.base import Source from buildbot.steps.source.oldsource import CVS, \ - SVN, Git, Darcs, Repo, Bzr, Mercurial, P4, P4Sync, Monotone + SVN, Git, Darcs, Repo, Bzr, Mercurial, P4, Monotone _hush_pyflakes = [ Source, CVS, SVN, \ - Git, Darcs, Repo, Bzr, Mercurial, P4, P4Sync, Monotone ] + Git, Darcs, Repo, Bzr, Mercurial, P4, Monotone ] diff --git a/master/buildbot/steps/source/base.py b/master/buildbot/steps/source/base.py index 9dc89fa2522..ba15a85153c 100644 --- a/master/buildbot/steps/source/base.py +++ b/master/buildbot/steps/source/base.py @@ -26,9 +26,13 @@ class Source(LoggingBuildStep): starts a RemoteCommand with those arguments. """ - renderables = [ 'workdir', 'description', 'descriptionDone' ] + renderables = LoggingBuildStep.renderables + [ + 'description', 'descriptionDone', 'descriptionSuffix', + 'workdir' ] + description = None # set this to a list of short strings to override descriptionDone = None # alternate description when the step is complete + descriptionSuffix = None # extra information to append to suffix # if the checkout fails, there's no point in doing anything else haltOnFailure = True @@ -39,58 +43,13 @@ class Source(LoggingBuildStep): def __init__(self, workdir=None, mode='update', alwaysUseLatest=False, timeout=20*60, retry=None, env=None, logEnviron=True, - description=None, descriptionDone=None, codebase='', - **kwargs): + description=None, descriptionDone=None, descriptionSuffix=None, + codebase='', **kwargs): """ @type workdir: string @param workdir: local directory (relative to the Builder's root) where the tree should be placed - @type mode: string - @param mode: the kind of VC operation that is desired: - - 'update': specifies that the checkout/update should be - performed directly into the workdir. Each build is performed - in the same directory, allowing for incremental builds. This - minimizes disk space, bandwidth, and CPU time. However, it - may encounter problems if the build process does not handle - dependencies properly (if you must sometimes do a 'clean - build' to make sure everything gets compiled), or if source - files are deleted but generated files can influence test - behavior (e.g. python's .pyc files), or when source - directories are deleted but generated files prevent CVS from - removing them. When used with a patched checkout, from a - previous buildbot try for instance, it will try to "revert" - the changes first and will do a clobber if it is unable to - get a clean checkout. The behavior is SCM-dependent. - - - 'copy': specifies that the source-controlled workspace - should be maintained in a separate directory (called the - 'copydir'), using checkout or update as necessary. For each - build, a new workdir is created with a copy of the source - tree (rm -rf workdir; cp -R -P -p copydir workdir). This - doubles the disk space required, but keeps the bandwidth low - (update instead of a full checkout). A full 'clean' build - is performed each time. This avoids any generated-file - build problems, but is still occasionally vulnerable to - problems such as a CVS repository being manually rearranged - (causing CVS errors on update) which are not an issue with - a full checkout. - - - 'clobber': specifies that the working directory should be - deleted each time, necessitating a full checkout for each - build. This insures a clean build off a complete checkout, - avoiding any of the problems described above, but is - bandwidth intensive, as the whole source tree must be - pulled down for each build. - - - 'export': is like 'clobber', except that e.g. the 'cvs - export' command is used to create the working directory. - This command removes all VC metadata files (the - CVS/.svn/{arch} directories) from the tree, which is - sometimes useful for creating source tarballs (to avoid - including the metadata in the tar file). Not all VC systems - support export. - @type alwaysUseLatest: boolean @param alwaysUseLatest: whether to always update to the most recent available sources for this build. @@ -113,22 +72,13 @@ def __init__(self, workdir=None, mode='update', alwaysUseLatest=False, is can result in an incoherent set of sources (splitting a non-atomic commit) which may not build at all. - @type retry: tuple of ints (delay, repeats) (or None) - @param retry: if provided, VC update failures are re-attempted up - to REPEATS times, with DELAY seconds between each - attempt. Some users have slaves with poor connectivity - to their VC repository, and they say that up to 80% of - their build failures are due to transient network - failures that could be handled by simply retrying a - couple times. - @type logEnviron: boolean @param logEnviron: If this option is true (the default), then the step's logfile will describe the environment variables on the slave. In situations where the environment is not relevant and is long, it may be easier to set logEnviron=False. -+ + @type codebase: string @param codebase: Specifies which changes in a build are processed by the step. The default codebase value is ''. The codebase must correspond @@ -138,39 +88,21 @@ def __init__(self, workdir=None, mode='update', alwaysUseLatest=False, """ LoggingBuildStep.__init__(self, **kwargs) - self.addFactoryArguments(workdir=workdir, - mode=mode, - alwaysUseLatest=alwaysUseLatest, - timeout=timeout, - retry=retry, - logEnviron=logEnviron, - env=env, - description=description, - descriptionDone=descriptionDone, - codebase=codebase, - ) - - assert mode in ("update", "copy", "clobber", "export") - if retry: - delay, repeats = retry - assert isinstance(repeats, int) - assert repeats > 0 - self.args = {'mode': mode, - 'timeout': timeout, - 'retry': retry, - 'patch': None, # set during .start - } + # This will get added to args later, after properties are rendered self.workdir = workdir self.sourcestamp = None - # Codebase cannot be set yet + self.codebase = codebase + if self.codebase: + self.name = ' '.join((self.name, self.codebase)) self.alwaysUseLatest = alwaysUseLatest self.logEnviron = logEnviron self.env = env + self.timeout = timeout descriptions_for_mode = { "clobber": "checkout", @@ -183,8 +115,6 @@ def __init__(self, workdir=None, mode='update', alwaysUseLatest=False, else: self.description = [ descriptions_for_mode.get(mode, "updating")] - if self.codebase: - self.description.append(self.codebase) if isinstance(self.description, str): self.description = [self.description] @@ -193,11 +123,16 @@ def __init__(self, workdir=None, mode='update', alwaysUseLatest=False, else: self.descriptionDone = [ descriptionDones_for_mode.get(mode, "update")] - if self.codebase: - self.descriptionDone.append(self.codebase) if isinstance(self.descriptionDone, str): self.descriptionDone = [self.descriptionDone] + if descriptionSuffix: + self.descriptionSuffix = descriptionSuffix + else: + self.descriptionSuffix = self.codebase or None # want None in lieu of '' + if isinstance(self.descriptionSuffix, str): + self.descriptionSuffix = [self.descriptionSuffix] + def setProperty(self, name, value , source): if self.codebase != '': assert not isinstance(self.getProperty(name, None), str), \ @@ -219,9 +154,11 @@ def setDefaultWorkdir(self, workdir): self.workdir = self.workdir or workdir def describe(self, done=False): - if done: - return self.descriptionDone - return self.description + desc = self.descriptionDone if done else self.description + if self.descriptionSuffix: + desc = desc[:] + desc.extend(self.descriptionSuffix) + return desc def computeSourceRevision(self, changes): """Each subclass must implement this method to do something more @@ -242,9 +179,6 @@ def start(self): % self.name) return SKIPPED - # Allow workdir to be WithProperties - self.args['workdir'] = self.workdir - if not self.alwaysUseLatest: # what source stamp would this step like to use? s = self.build.getSourceStamp(self.codebase) @@ -283,8 +217,6 @@ def start(self): branch = self.branch patch = None - self.args['logEnviron'] = self.logEnviron - self.args['env'] = self.env self.startVC(branch, revision, patch) def commandComplete(self, cmd): diff --git a/master/buildbot/steps/source/bzr.py b/master/buildbot/steps/source/bzr.py index 2f9f1983290..8a0dee5d5bd 100644 --- a/master/buildbot/steps/source/bzr.py +++ b/master/buildbot/steps/source/bzr.py @@ -34,12 +34,6 @@ def __init__(self, repourl=None, baseURL=None, mode='incremental', self.mode = mode self.method = method Source.__init__(self, **kwargs) - self.addFactoryArguments(repourl=repourl, - mode=mode, - method=method, - baseURL=baseURL, - defaultBranch=defaultBranch, - ) if repourl and baseURL: raise ValueError("you must provide exactly one of repourl and" " baseURL") @@ -186,7 +180,7 @@ def _sourcedirIsUpdatable(self): cmd.useLog(self.stdio_log, False) d = self.runCommand(cmd) def _fail(tmp): - if cmd.rc != 0: + if cmd.didFail(): return False return True d.addCallback(_fail) @@ -202,11 +196,12 @@ def _dovccmd(self, command, abandonOnFailure=True, collectStdout=False): cmd = buildstep.RemoteShellCommand(self.workdir, ['bzr'] + command, env=self.env, logEnviron=self.logEnviron, + timeout=self.timeout, collectStdout=collectStdout) cmd.useLog(self.stdio_log, False) d = self.runCommand(cmd) def evaluateCommand(cmd): - if abandonOnFailure and cmd.rc != 0: + if abandonOnFailure and cmd.didFail(): log.msg("Source step failed while running command %s" % cmd) raise buildstep.BuildStepFailed() if collectStdout: diff --git a/master/buildbot/steps/source/cvs.py b/master/buildbot/steps/source/cvs.py index 66585c08fe2..968e8b0af20 100644 --- a/master/buildbot/steps/source/cvs.py +++ b/master/buildbot/steps/source/cvs.py @@ -20,6 +20,7 @@ from twisted.internet import defer from buildbot.process import buildstep +from buildbot.steps.shell import StringFileWriter from buildbot.steps.source.base import Source from buildbot.interfaces import BuildSlaveTooOldError @@ -43,16 +44,9 @@ def __init__(self, cvsroot=None, cvsmodule='', mode='incremental', self.method = method self.srcdir = 'source' Source.__init__(self, **kwargs) - self.addFactoryArguments(cvsroot=cvsroot, - cvsmodule=cvsmodule, - mode=mode, - method=method, - global_options=global_options, - extra_options=extra_options, - login=login, - ) def startVC(self, branch, revision, patch): + self.branch = branch self.revision = revision self.stdio_log = self.addLog("stdio") self.method = self._getMethod() @@ -80,7 +74,7 @@ def incremental(self): if updatable: rv = yield self.doUpdate() else: - rv = yield self.doCheckout(self.workdir) + rv = yield self.clobber() defer.returnValue(rv) @defer.inlineCallbacks @@ -158,24 +152,25 @@ def purge(self, ignore_ignores): command += ['--ignore'] cmd = buildstep.RemoteShellCommand(self.workdir, command, env=self.env, - logEnviron=self.logEnviron) + logEnviron=self.logEnviron, + timeout=self.timeout) cmd.useLog(self.stdio_log, False) d = self.runCommand(cmd) - def evaluate(rc): - if rc != 0: + def evaluate(cmd): + if cmd.didFail(): raise buildstep.BuildStepFailed() - return rc - d.addCallback(lambda _: evaluate(cmd.rc)) + return cmd.rc + d.addCallback(evaluate) return d def doCheckout(self, dir): - command = ['-d', self.cvsroot, '-z3', 'checkout', '-d', dir, - self.cvsmodule] + command = ['-d', self.cvsroot, '-z3', 'checkout', '-d', dir ] command = self.global_options + command + self.extra_options if self.branch: command += ['-r', self.branch] if self.revision: command += ['-D', self.revision] + command += [ self.cvsmodule ] d = self._dovccmd(command, '') return d @@ -218,6 +213,7 @@ def _dovccmd(self, command, workdir=None): cmd = buildstep.RemoteShellCommand(workdir, ['cvs'] + command, env=self.env, + timeout=self.timeout, logEnviron=self.logEnviron) cmd.useLog(self.stdio_log, False) d = self.runCommand(cmd) @@ -229,17 +225,40 @@ def evaluateCommand(cmd): d.addCallback(lambda _: evaluateCommand(cmd)) return d + @defer.inlineCallbacks def _sourcedirIsUpdatable(self): - cmd = buildstep.RemoteCommand('stat', {'file': self.workdir + '/CVS', - 'logEnviron': self.logEnviron}) - cmd.useLog(self.stdio_log, False) - d = self.runCommand(cmd) - def _fail(tmp): - if cmd.rc != 0: - return False - return True - d.addCallback(_fail) - return d + myFileWriter = StringFileWriter() + args = { + 'workdir': self.build.path_module.join(self.workdir, 'CVS'), + 'writer': myFileWriter, + 'maxsize': None, + 'blocksize': 32*1024, + } + + cmd = buildstep.RemoteCommand('uploadFile', + dict(slavesrc='Root', **args), + ignore_updates=True) + yield self.runCommand(cmd) + if cmd.rc is not None and cmd.rc != 0: + defer.returnValue(False) + return + if myFileWriter.buffer.strip() != self.cvsroot: + defer.returnValue(False) + return + + myFileWriter.buffer = "" + cmd = buildstep.RemoteCommand('uploadFile', + dict(slavesrc='Repository', **args), + ignore_updates=True) + yield self.runCommand(cmd) + if cmd.rc is not None and cmd.rc != 0: + defer.returnValue(False) + return + if myFileWriter.buffer.strip() != self.cvsmodule: + defer.returnValue(False) + return + + defer.returnValue(True) def parseGotRevision(self, res): revision = time.strftime("%Y-%m-%d %H:%M:%S +0000", time.gmtime()) diff --git a/master/buildbot/steps/source/git.py b/master/buildbot/steps/source/git.py index e2d150e3677..173dbaa6017 100644 --- a/master/buildbot/steps/source/git.py +++ b/master/buildbot/steps/source/git.py @@ -20,6 +20,39 @@ from buildbot.steps.source.base import Source from buildbot.interfaces import BuildSlaveTooOldError +def isTrueOrIsExactlyZero(v): + # nonzero values are true... + if v: + return True + + # ... and True for the number zero, but we have to + # explicitly guard against v==False, since + # isinstance(False, int) is surprisingly True + if isinstance(v, int) and v is not False: + return True + + # all other false-ish values are false + return False + +git_describe_flags = [ + # on or off + ('all', lambda v: ['--all'] if v else None), + ('always', lambda v: ['--always'] if v else None), + ('contains', lambda v: ['--contains'] if v else None), + ('debug', lambda v: ['--debug'] if v else None), + ('long', lambda v: ['--long'] if v else None), + ('exact-match', lambda v: ['--exact-match'] if v else None), + ('tags', lambda v: ['--tags'] if v else None), + # string parameter + ('match', lambda v: ['--match', v] if v else None), + # numeric parameter + ('abbrev', lambda v: ['--abbrev=%s' % v] if isTrueOrIsExactlyZero(v) else None), + ('candidates', lambda v: ['--candidates=%s' % v] if isTrueOrIsExactlyZero(v) else None), + # optional string parameter + ('dirty', lambda v: ['--dirty'] if (v is True or v=='') else None), + ('dirty', lambda v: ['--dirty=%s' % v] if (v and v is not True) else None), +] + class Git(Source): """ Class for Git with all the smarts """ name='git' @@ -27,7 +60,8 @@ class Git(Source): def __init__(self, repourl=None, branch='HEAD', mode='incremental', method=None, submodules=False, shallow=False, progress=False, - retryFetch=False, clobberOnFailure=False, **kwargs): + retryFetch=False, clobberOnFailure=False, getDescription=False, + **kwargs): """ @type repourl: string @param repourl: the URL which points at the git repository @@ -57,7 +91,12 @@ def __init__(self, repourl=None, branch='HEAD', mode='incremental', @type retryFetch: boolean @param retryFetch: Retry fetching before failing source checkout. + + @type getDescription: boolean or dict + @param getDescription: Use 'git describe' to describe the fetched revision """ + if not getDescription and not isinstance(getDescription, dict): + getDescription = False self.branch = branch self.method = method @@ -69,23 +108,14 @@ def __init__(self, repourl=None, branch='HEAD', mode='incremental', self.fetchcount = 0 self.clobberOnFailure = clobberOnFailure self.mode = mode + self.getDescription = getDescription Source.__init__(self, **kwargs) - self.addFactoryArguments(branch=branch, - mode=mode, - method=method, - progress=progress, - repourl=repourl, - submodules=submodules, - shallow=shallow, - retryFetch=retryFetch, - clobberOnFailure= - clobberOnFailure, - ) assert self.mode in ['incremental', 'full'] assert self.repourl is not None if self.mode == 'full': assert self.method in ['clean', 'fresh', 'clobber', 'copy', None] + assert isinstance(self.getDescription, (bool, dict)) def startVC(self, branch, revision, patch): self.branch = branch or 'HEAD' @@ -107,6 +137,7 @@ def checkInstall(gitInstalled): if patch: d.addCallback(self.patch, patch) d.addCallback(self.parseGotRevision) + d.addCallback(self.parseCommitDescription) d.addCallback(self.finish) d.addErrback(self.failed) return d @@ -224,29 +255,53 @@ def _gotResults(results): d.addCallbacks(self.finished, self.checkDisconnect) return d - def parseGotRevision(self, _): - d = self._dovccmd(['rev-parse', 'HEAD'], collectStdout=True) - def setrev(stdout): - revision = stdout.strip() - if len(revision) != 40: - raise buildstep.BuildStepFailed() - log.msg("Got Git revision %s" % (revision, )) - self.setProperty('got_revision', revision, 'Source') - return 0 - d.addCallback(setrev) - return d + @defer.inlineCallbacks + def parseGotRevision(self, _=None): + stdout = yield self._dovccmd(['rev-parse', 'HEAD'], collectStdout=True) + revision = stdout.strip() + if len(revision) != 40: + raise buildstep.BuildStepFailed() + log.msg("Got Git revision %s" % (revision, )) + self.setProperty('got_revision', revision, 'Source') + + defer.returnValue(0) + + @defer.inlineCallbacks + def parseCommitDescription(self, _=None): + if self.getDescription==False: # dict() should not return here + defer.returnValue(0) + return + + cmd = ['describe'] + if isinstance(self.getDescription, dict): + for opt, arg in git_describe_flags: + opt = self.getDescription.get(opt, None) + arg = arg(opt) + if arg: + cmd.extend(arg) + cmd.append('HEAD') + + try: + stdout = yield self._dovccmd(cmd, collectStdout=True) + desc = stdout.strip() + self.setProperty('commit-description', desc, 'Source') + except: + pass + + defer.returnValue(0) def _dovccmd(self, command, abandonOnFailure=True, collectStdout=False, initialStdin=None): cmd = buildstep.RemoteShellCommand(self.workdir, ['git'] + command, env=self.env, logEnviron=self.logEnviron, + timeout=self.timeout, collectStdout=collectStdout, initialStdin=initialStdin) cmd.useLog(self.stdio_log, False) log.msg("Starting git command : git %s" % (" ".join(command), )) d = self.runCommand(cmd) def evaluateCommand(cmd): - if abandonOnFailure and cmd.rc != 0: + if abandonOnFailure and cmd.didFail(): log.msg("Source step failed while running command %s" % cmd) raise buildstep.BuildStepFailed() if collectStdout: @@ -356,7 +411,7 @@ def _sourcedirIsUpdatable(self): cmd.useLog(self.stdio_log, False) d = self.runCommand(cmd) def _fail(tmp): - if cmd.rc != 0: + if cmd.didFail(): return False return True d.addCallback(_fail) diff --git a/master/buildbot/steps/source/mercurial.py b/master/buildbot/steps/source/mercurial.py index dd2c45384cf..e9cfd34fbfe 100644 --- a/master/buildbot/steps/source/mercurial.py +++ b/master/buildbot/steps/source/mercurial.py @@ -71,14 +71,6 @@ def __init__(self, repourl=None, mode='incremental', self.clobberOnBranchChange = clobberOnBranchChange self.mode = mode Source.__init__(self, **kwargs) - self.addFactoryArguments(repourl=repourl, - mode=mode, - method=method, - defaultBranch=defaultBranch, - branchType=branchType, - clobberOnBranchChange= - clobberOnBranchChange, - ) errors = [] if self.mode not in self.possible_modes: @@ -232,12 +224,13 @@ def _dovccmd(self, command, collectStdout=False): cmd = buildstep.RemoteShellCommand(self.workdir, ['hg', '--verbose'] + command, env=self.env, logEnviron=self.logEnviron, + timeout=self.timeout, collectStdout=collectStdout) cmd.useLog(self.stdio_log, False) log.msg("Starting mercurial command : hg %s" % (" ".join(command), )) d = self.runCommand(cmd) def evaluateCommand(cmd): - if cmd.rc != 0: + if cmd.didFail(): log.msg("Source step failed while running command %s" % cmd) raise buildstep.BuildStepFailed() if collectStdout: @@ -284,7 +277,7 @@ def _sourcedirIsUpdatable(self): cmd.useLog(self.stdio_log, False) d = self.runCommand(cmd) def _fail(tmp): - if cmd.rc != 0: + if cmd.didFail(): return False return True d.addCallback(_fail) diff --git a/master/buildbot/steps/source/oldsource.py b/master/buildbot/steps/source/oldsource.py index 6f8c1d004df..e085bf7cd95 100644 --- a/master/buildbot/steps/source/oldsource.py +++ b/master/buildbot/steps/source/oldsource.py @@ -26,7 +26,8 @@ class _ComputeRepositoryURL(object): implements(IRenderable) - def __init__(self, repository): + def __init__(self, step, repository): + self.step = step self.repository = repository def getRenderingFor(self, props): @@ -37,7 +38,7 @@ def getRenderingFor(self, props): build = props.getBuild() assert build is not None, "Build should be available *during* a build?" - s = build.getSourceStamp('') # TODO: use correct codebase + s = build.getSourceStamp(self.step.codebase) repository = self.repository @@ -60,10 +61,85 @@ def getRenderingFor(self, props): d.addCallback(str) return d +class SlaveSource(Source): + def __init__(self, mode='update', retry=None, + codebase='', **kwargs): + """ + @type mode: string + @param mode: the kind of VC operation that is desired: + - 'update': specifies that the checkout/update should be + performed directly into the workdir. Each build is performed + in the same directory, allowing for incremental builds. This + minimizes disk space, bandwidth, and CPU time. However, it + may encounter problems if the build process does not handle + dependencies properly (if you must sometimes do a 'clean + build' to make sure everything gets compiled), or if source + files are deleted but generated files can influence test + behavior (e.g. python's .pyc files), or when source + directories are deleted but generated files prevent CVS from + removing them. When used with a patched checkout, from a + previous buildbot try for instance, it will try to "revert" + the changes first and will do a clobber if it is unable to + get a clean checkout. The behavior is SCM-dependent. + + - 'copy': specifies that the source-controlled workspace + should be maintained in a separate directory (called the + 'copydir'), using checkout or update as necessary. For each + build, a new workdir is created with a copy of the source + tree (rm -rf workdir; cp -R -P -p copydir workdir). This + doubles the disk space required, but keeps the bandwidth low + (update instead of a full checkout). A full 'clean' build + is performed each time. This avoids any generated-file + build problems, but is still occasionally vulnerable to + problems such as a CVS repository being manually rearranged + (causing CVS errors on update) which are not an issue with + a full checkout. + + - 'clobber': specifies that the working directory should be + deleted each time, necessitating a full checkout for each + build. This insures a clean build off a complete checkout, + avoiding any of the problems described above, but is + bandwidth intensive, as the whole source tree must be + pulled down for each build. + + - 'export': is like 'clobber', except that e.g. the 'cvs + export' command is used to create the working directory. + This command removes all VC metadata files (the + CVS/.svn/{arch} directories) from the tree, which is + sometimes useful for creating source tarballs (to avoid + including the metadata in the tar file). Not all VC systems + support export. + + @type retry: tuple of ints (delay, repeats) (or None) + @param retry: if provided, VC update failures are re-attempted up + to REPEATS times, with DELAY seconds between each + attempt. Some users have slaves with poor connectivity + to their VC repository, and they say that up to 80% of + their build failures are due to transient network + failures that could be handled by simply retrying a + couple times. + """ + Source.__init__(self, **kwargs) + + assert mode in ("update", "copy", "clobber", "export") + if retry: + delay, repeats = retry + assert isinstance(repeats, int) + assert repeats > 0 + self.args = {'mode': mode, + 'retry': retry, + } + + def start(self): + self.args['workdir'] = self.workdir + self.args['logEnviron'] = self.logEnviron + self.args['env'] = self.env + self.args['timeout'] = self.timeout + Source.start(self) -class CVS(Source): +class CVS(SlaveSource): """I do CVS checkout/update operations. Note: if you are doing anonymous/pserver CVS operations, you will need @@ -161,19 +237,9 @@ def __init__(self, cvsroot=None, cvsmodule="", self.checkoutDelay = checkoutDelay self.branch = branch - self.cvsroot = _ComputeRepositoryURL(cvsroot) + self.cvsroot = _ComputeRepositoryURL(self, cvsroot) - Source.__init__(self, **kwargs) - self.addFactoryArguments(cvsroot=cvsroot, - cvsmodule=cvsmodule, - global_options=global_options, - checkout_options=checkout_options, - export_options=export_options, - extra_options=extra_options, - branch=branch, - checkoutDelay=checkoutDelay, - login=login, - ) + SlaveSource.__init__(self, **kwargs) self.args.update({'cvsmodule': cvsmodule, 'global_options': global_options, @@ -255,7 +321,7 @@ def startVC(self, branch, revision, patch): self.startCommand(cmd, warnings) -class SVN(Source): +class SVN(SlaveSource): """I perform Subversion checkout/update operations.""" name = 'svn' @@ -298,8 +364,8 @@ def __init__(self, svnurl=None, baseURL=None, defaultBranch=None, warn("Please use workdir=, not directory=", DeprecationWarning) kwargs['workdir'] = directory - self.svnurl = svnurl and _ComputeRepositoryURL(svnurl) - self.baseURL = _ComputeRepositoryURL(baseURL) + self.svnurl = svnurl and _ComputeRepositoryURL(self, svnurl) + self.baseURL = _ComputeRepositoryURL(self, baseURL) self.branch = defaultBranch self.username = username self.password = password @@ -309,19 +375,7 @@ def __init__(self, svnurl=None, baseURL=None, defaultBranch=None, self.always_purge = always_purge self.depth = depth - Source.__init__(self, **kwargs) - self.addFactoryArguments(svnurl=svnurl, - baseURL=baseURL, - defaultBranch=defaultBranch, - directory=directory, - username=username, - password=password, - extra_args=extra_args, - keep_on_purge=keep_on_purge, - ignore_ignores=ignore_ignores, - always_purge=always_purge, - depth=depth, - ) + SlaveSource.__init__(self, **kwargs) if svnurl and baseURL: raise ValueError("you must use either svnurl OR baseURL") @@ -427,7 +481,7 @@ def startVC(self, branch, revision, patch): self.startCommand(cmd, warnings) -class Darcs(Source): +class Darcs(SlaveSource): """Check out a source tree from a Darcs repository at 'repourl'. Darcs has no concept of file modes. This means the eXecute-bit will be @@ -462,14 +516,10 @@ def __init__(self, repourl=None, baseURL=None, defaultBranch=None, C{baseURL} and the result handed to the 'darcs pull' command. """ - self.repourl = _ComputeRepositoryURL(repourl) - self.baseURL = _ComputeRepositoryURL(baseURL) + self.repourl = _ComputeRepositoryURL(self, repourl) + self.baseURL = _ComputeRepositoryURL(self, baseURL) self.branch = defaultBranch - Source.__init__(self, **kwargs) - self.addFactoryArguments(repourl=repourl, - baseURL=baseURL, - defaultBranch=defaultBranch, - ) + SlaveSource.__init__(self, **kwargs) assert self.args['mode'] != "export", \ "Darcs does not have an 'export' mode" if repourl and baseURL: @@ -521,7 +571,7 @@ def startVC(self, branch, revision, patch): self.startCommand(cmd) -class Git(Source): +class Git(SlaveSource): """Check out a source tree from a git repository 'repourl'.""" name = "git" @@ -561,17 +611,9 @@ def __init__(self, repourl=None, can solve long fetches getting killed due to lack of output, but requires Git 1.7.2+. """ - Source.__init__(self, **kwargs) - self.repourl = _ComputeRepositoryURL(repourl) + SlaveSource.__init__(self, **kwargs) + self.repourl = _ComputeRepositoryURL(self, repourl) self.branch = branch - self.addFactoryArguments(repourl=repourl, - branch=branch, - submodules=submodules, - ignore_ignores=ignore_ignores, - reference=reference, - shallow=shallow, - progress=progress, - ) self.args.update({'submodules': submodules, 'ignore_ignores': ignore_ignores, 'reference': reference, @@ -614,7 +656,7 @@ def startVC(self, branch, revision, patch): self.startCommand(cmd) -class Repo(Source): +class Repo(SlaveSource): """Check out a source tree from a repo repository described by manifest.""" name = "repo" @@ -626,6 +668,7 @@ def __init__(self, manifest_branch="master", manifest_file="default.xml", tarball=None, + jobs=None, **kwargs): """ @type manifest_url: string @@ -638,17 +681,13 @@ def __init__(self, @param manifest_file: The manifest to use for sync. """ - Source.__init__(self, **kwargs) - self.manifest_url = _ComputeRepositoryURL(manifest_url) - self.addFactoryArguments(manifest_url=manifest_url, - manifest_branch=manifest_branch, - manifest_file=manifest_file, - tarball=tarball, - ) + SlaveSource.__init__(self, **kwargs) + self.manifest_url = _ComputeRepositoryURL(self, manifest_url) self.args.update({'manifest_branch': manifest_branch, 'manifest_file': manifest_file, 'tarball': tarball, - 'manifest_override_url': None + 'manifest_override_url': None, + 'jobs': jobs }) def computeSourceRevision(self, changes): @@ -744,7 +783,7 @@ def commandComplete(self, cmd): self.step_status.setText(["repo download issues"]) -class Bzr(Source): +class Bzr(SlaveSource): """Check out a source tree from a bzr (Bazaar) repository at 'repourl'. """ @@ -784,15 +823,10 @@ def __init__(self, repourl=None, baseURL=None, defaultBranch=None, if not using update/copy mode, or if using update/copy mode with multiple branches. """ - self.repourl = _ComputeRepositoryURL(repourl) - self.baseURL = _ComputeRepositoryURL(baseURL) + self.repourl = _ComputeRepositoryURL(self, repourl) + self.baseURL = _ComputeRepositoryURL(self, baseURL) self.branch = defaultBranch - Source.__init__(self, **kwargs) - self.addFactoryArguments(repourl=repourl, - baseURL=baseURL, - defaultBranch=defaultBranch, - forceSharedRepo=forceSharedRepo - ) + SlaveSource.__init__(self, **kwargs) self.args.update({'forceSharedRepo': forceSharedRepo}) if repourl and baseURL: raise ValueError("you must provide exactly one of repourl and" @@ -830,7 +864,7 @@ def startVC(self, branch, revision, patch): self.startCommand(cmd) -class Mercurial(Source): +class Mercurial(SlaveSource): """Check out a source tree from a mercurial repository 'repourl'.""" name = "hg" @@ -872,18 +906,12 @@ def __init__(self, repourl=None, baseURL=None, defaultBranch=None, at each branch change. Otherwise, just update to the branch. """ - self.repourl = _ComputeRepositoryURL(repourl) - self.baseURL = _ComputeRepositoryURL(baseURL) + self.repourl = _ComputeRepositoryURL(self, repourl) + self.baseURL = _ComputeRepositoryURL(self, baseURL) self.branch = defaultBranch self.branchType = branchType self.clobberOnBranchChange = clobberOnBranchChange - Source.__init__(self, **kwargs) - self.addFactoryArguments(repourl=repourl, - baseURL=baseURL, - defaultBranch=defaultBranch, - branchType=branchType, - clobberOnBranchChange=clobberOnBranchChange, - ) + SlaveSource.__init__(self, **kwargs) if repourl and baseURL: raise ValueError("you must provide exactly one of repourl and" " baseURL") @@ -930,7 +958,7 @@ def computeSourceRevision(self, changes): return changes[-1].revision -class P4(Source): +class P4(SlaveSource): """ P4 is a class for accessing perforce revision control""" name = "p4" @@ -972,18 +1000,9 @@ def __init__(self, p4base=None, defaultBranch=None, p4port=None, p4user=None, @param p4client: The perforce client to use for this buildslave. """ - self.p4base = _ComputeRepositoryURL(p4base) + self.p4base = _ComputeRepositoryURL(self, p4base) self.branch = defaultBranch - Source.__init__(self, **kwargs) - self.addFactoryArguments(p4base=p4base, - defaultBranch=defaultBranch, - p4port=p4port, - p4user=p4user, - p4passwd=p4passwd, - p4extra_views=p4extra_views, - p4line_end=p4line_end, - p4client=p4client, - ) + SlaveSource.__init__(self, **kwargs) self.args['p4port'] = p4port self.args['p4user'] = p4user self.args['p4passwd'] = p4passwd @@ -992,7 +1011,7 @@ def __init__(self, p4base=None, defaultBranch=None, p4port=None, p4user=None, self.p4client = p4client def setBuild(self, build): - Source.setBuild(self, build) + SlaveSource.setBuild(self, build) self.args['p4client'] = self.p4client % { 'slave': build.slavename, 'builder': build.builder.name, @@ -1015,57 +1034,7 @@ def startVC(self, branch, revision, patch): cmd = RemoteCommand("p4", args) self.startCommand(cmd) -class P4Sync(Source): - """ - DEPRECATED - will be removed in 0.8.5. - - This is a partial solution for using a P4 source repository. You are - required to manually set up each build slave with a useful P4 - environment, which means setting various per-slave environment variables, - and creating a P4 client specification which maps the right files into - the slave's working directory. Once you have done that, this step merely - performs a 'p4 sync' to update that workspace with the newest files. - - Each slave needs the following environment: - - - PATH: the 'p4' binary must be on the slave's PATH - - P4USER: each slave needs a distinct user account - - P4CLIENT: each slave needs a distinct client specification - - You should use 'p4 client' (?) to set up a client view spec which maps - the desired files into $SLAVEBASE/$BUILDERBASE/source . - """ - - name = "p4sync" - - def __init__(self, p4port, p4user, p4passwd, p4client, **kwargs): - assert kwargs['mode'] == "copy", "P4Sync can only be used in mode=copy" - self.branch = None - Source.__init__(self, **kwargs) - self.addFactoryArguments(p4port=p4port, - p4user=p4user, - p4passwd=p4passwd, - p4client=p4client, - ) - self.args['p4port'] = p4port - self.args['p4user'] = p4user - self.args['p4passwd'] = p4passwd - self.args['p4client'] = p4client - - def computeSourceRevision(self, changes): - if not changes: - return None - lastChange = max([int(c.revision) for c in changes]) - return lastChange - - def startVC(self, branch, revision, patch): - slavever = self.slaveVersion("p4sync") - assert slavever, "slave is too old, does not know about p4" - cmd = RemoteCommand("p4sync", self.args) - self.startCommand(cmd) - - -class Monotone(Source): +class Monotone(SlaveSource): """Check out a source tree from a monotone repository 'repourl'.""" name = "mtn" @@ -1087,16 +1056,12 @@ def __init__(self, repourl=None, branch=None, progress=False, **kwargs): can solve long fetches getting killed due to lack of output. """ - Source.__init__(self, **kwargs) - self.repourl = _ComputeRepositoryURL(repourl) + SlaveSource.__init__(self, **kwargs) + self.repourl = _ComputeRepositoryURL(self, repourl) if (not repourl): raise ValueError("you must provide a repository uri in 'repourl'") if (not branch): raise ValueError("you must provide a default branch in 'branch'") - self.addFactoryArguments(repourl=repourl, - branch=branch, - progress=progress, - ) self.args.update({'branch': branch, 'progress': progress, }) diff --git a/master/buildbot/steps/source/svn.py b/master/buildbot/steps/source/svn.py index 26b1c31d877..54a63e2873b 100644 --- a/master/buildbot/steps/source/svn.py +++ b/master/buildbot/steps/source/svn.py @@ -48,15 +48,6 @@ def __init__(self, repourl=None, mode='incremental', self.method=method self.mode = mode Source.__init__(self, **kwargs) - self.addFactoryArguments(repourl=repourl, - mode=mode, - method=method, - password=password, - username=username, - extra_args=extra_args, - keep_on_purge=keep_on_purge, - depth=depth, - ) errors = [] if self.mode not in self.possible_modes: errors.append("mode %s is not one of %s" % (self.mode, self.possible_modes)) @@ -139,7 +130,7 @@ def clobber(self): 'logEnviron': self.logEnviron,}) cmd.useLog(self.stdio_log, False) yield self.runCommand(cmd) - if cmd.rc != 0: + if cmd.didFail(): raise buildstep.BuildStepFailed() checkout_cmd = ['checkout', self.repourl, '.'] @@ -171,7 +162,7 @@ def copy(self): cmd.useLog(self.stdio_log, False) yield self.runCommand(cmd) - if cmd.rc != 0: + if cmd.didFail(): raise buildstep.BuildStepFailed() # temporarily set workdir = 'source' and do an incremental checkout @@ -196,12 +187,12 @@ def copy(self): export_cmd.extend(['source', self.workdir]) cmd = buildstep.RemoteShellCommand('', export_cmd, - env=self.env, logEnviron=self.logEnviron) + env=self.env, logEnviron=self.logEnviron, timeout=self.timeout) cmd.useLog(self.stdio_log, False) yield self.runCommand(cmd) - if cmd.rc != 0: + if cmd.didFail(): raise buildstep.BuildStepFailed() def finish(self, res): @@ -219,7 +210,7 @@ def _rmdir(self, dir): {'dir': dir, 'logEnviron': self.logEnviron }) cmd.useLog(self.stdio_log, False) yield self.runCommand(cmd) - if cmd.rc != 0: + if cmd.didFail(): raise buildstep.BuildStepFailed() def _dovccmd(self, command, collectStdout=False): @@ -237,12 +228,13 @@ def _dovccmd(self, command, collectStdout=False): cmd = buildstep.RemoteShellCommand(self.workdir, ['svn'] + command, env=self.env, logEnviron=self.logEnviron, + timeout=self.timeout, collectStdout=collectStdout) cmd.useLog(self.stdio_log, False) log.msg("Starting SVN command : svn %s" % (" ".join(command), )) d = self.runCommand(cmd) def evaluateCommand(cmd): - if cmd.rc != 0: + if cmd.didFail(): log.msg("Source step failed while running command %s" % cmd) raise buildstep.BuildStepFailed() if collectStdout: @@ -268,7 +260,7 @@ def _sourcedirIsUpdatable(self): cmd.useLog(self.stdio_log, False) yield self.runCommand(cmd) - if cmd.rc != 0: + if cmd.didFail(): defer.returnValue(False) return @@ -291,6 +283,7 @@ def parseGotRevision(self, _): cmd = buildstep.RemoteShellCommand(svnversion_dir, ['svnversion'], env=self.env, logEnviron=self.logEnviron, + timeout=self.timeout, collectStdout=True) cmd.useLog(self.stdio_log, False) d = self.runCommand(cmd) @@ -379,7 +372,8 @@ def removeFiles(self, files): def checkSvn(self): cmd = buildstep.RemoteShellCommand(self.workdir, ['svn', '--version'], env=self.env, - logEnviron=self.logEnviron) + logEnviron=self.logEnviron, + timeout=self.timeout) cmd.useLog(self.stdio_log, False) d = self.runCommand(cmd) def evaluate(cmd): diff --git a/master/buildbot/steps/subunit.py b/master/buildbot/steps/subunit.py index f4f0ca19c4c..f1e5749ca9d 100644 --- a/master/buildbot/steps/subunit.py +++ b/master/buildbot/steps/subunit.py @@ -24,7 +24,6 @@ class SubunitShellCommand(ShellCommand): def __init__(self, failureOnNoTests=False, *args, **kwargs): ShellCommand.__init__(self, *args, **kwargs) self.failureOnNoTests = failureOnNoTests - self.addFactoryArguments(failureOnNoTests=failureOnNoTests) # importing here gets around an import loop from buildbot.process import subunitlogobserver @@ -82,7 +81,7 @@ def commandComplete(self, cmd): self.text2 = [text2] def evaluateCommand(self, cmd): - if cmd.rc != 0: + if cmd.didFail(): return FAILURE return self.results diff --git a/master/buildbot/steps/transfer.py b/master/buildbot/steps/transfer.py index b460fa8bacb..01e49dcecd2 100644 --- a/master/buildbot/steps/transfer.py +++ b/master/buildbot/steps/transfer.py @@ -172,7 +172,7 @@ def remote_unpack(self): def makeStatusRemoteCommand(step, remote_command, args): - self = buildstep.RemoteCommand(remote_command, args) + self = buildstep.RemoteCommand(remote_command, args, successfulRC=(None, 0)) callback = lambda arg: step.step_status.addLog('stdio') self.useLogDelayed('stdio', callback, True) return self @@ -213,9 +213,9 @@ def finished(self, result): if result == SKIPPED: return BuildStep.finished(self, SKIPPED) - if self.cmd.rc is None or self.cmd.rc == 0: - return BuildStep.finished(self, SUCCESS) - return BuildStep.finished(self, FAILURE) + if self.cmd.didFail(): + return BuildStep.finished(self, FAILURE) + return BuildStep.finished(self, SUCCESS) class FileUpload(_TransferBuildStep): @@ -229,15 +229,6 @@ def __init__(self, slavesrc, masterdest, keepstamp=False, url=None, **buildstep_kwargs): BuildStep.__init__(self, **buildstep_kwargs) - self.addFactoryArguments(slavesrc=slavesrc, - masterdest=masterdest, - workdir=workdir, - maxsize=maxsize, - blocksize=blocksize, - mode=mode, - keepstamp=keepstamp, - url=url, - ) self.slavesrc = slavesrc self.masterdest = masterdest @@ -309,14 +300,6 @@ def __init__(self, slavesrc, masterdest, workdir=None, maxsize=None, blocksize=16*1024, compress=None, url=None, **buildstep_kwargs): BuildStep.__init__(self, **buildstep_kwargs) - self.addFactoryArguments(slavesrc=slavesrc, - masterdest=masterdest, - workdir=workdir, - maxsize=maxsize, - blocksize=blocksize, - compress=compress, - url=url, - ) self.slavesrc = slavesrc self.masterdest = masterdest @@ -378,11 +361,9 @@ def finished(self, result): if result == SKIPPED: return BuildStep.finished(self, SKIPPED) - if self.cmd.rc is None or self.cmd.rc == 0: - return BuildStep.finished(self, SUCCESS) - return BuildStep.finished(self, FAILURE) - - + if self.cmd.didFail(): + return BuildStep.finished(self, FAILURE) + return BuildStep.finished(self, SUCCESS) class _FileReader(pb.Referenceable): @@ -428,13 +409,6 @@ def __init__(self, mastersrc, slavedest, workdir=None, maxsize=None, blocksize=16*1024, mode=None, **buildstep_kwargs): BuildStep.__init__(self, **buildstep_kwargs) - self.addFactoryArguments(mastersrc=mastersrc, - slavedest=slavedest, - workdir=workdir, - maxsize=maxsize, - blocksize=blocksize, - mode=mode, - ) self.mastersrc = mastersrc self.slavedest = slavedest @@ -499,13 +473,6 @@ def __init__(self, s, slavedest, workdir=None, maxsize=None, blocksize=16*1024, mode=None, **buildstep_kwargs): BuildStep.__init__(self, **buildstep_kwargs) - self.addFactoryArguments(s=s, - slavedest=slavedest, - workdir=workdir, - maxsize=maxsize, - blocksize=blocksize, - mode=mode, - ) self.s = s self.slavedest = slavedest @@ -558,7 +525,6 @@ def __init__(self, o, slavedest, **buildstep_kwargs): del buildstep_kwargs['s'] s = json.dumps(o) StringDownload.__init__(self, s=s, slavedest=slavedest, **buildstep_kwargs) - self.addFactoryArguments(o=o) class JSONPropertiesDownload(StringDownload): diff --git a/master/buildbot/steps/trigger.py b/master/buildbot/steps/trigger.py index f1d959afe66..d41e73933a1 100644 --- a/master/buildbot/steps/trigger.py +++ b/master/buildbot/steps/trigger.py @@ -60,14 +60,6 @@ def __init__(self, schedulerNames=[], sourceStamp = None, sourceStamps = None, self.running = False self.ended = False LoggingBuildStep.__init__(self, **kwargs) - self.addFactoryArguments(schedulerNames=schedulerNames, - sourceStamp=sourceStamp, - sourceStamps=sourceStamps, - updateSourceStamp=updateSourceStamp, - alwaysUseLatest=alwaysUseLatest, - waitForFinish=waitForFinish, - set_properties=set_properties, - copy_properties=copy_properties) def interrupt(self, reason): if self.running and not self.ended: @@ -224,4 +216,4 @@ def add_links(res): dl.addCallback(add_links) self.end(result) - return \ No newline at end of file + return diff --git a/master/buildbot/steps/vstudio.py b/master/buildbot/steps/vstudio.py index 65cecbad206..592a5574195 100644 --- a/master/buildbot/steps/vstudio.py +++ b/master/buildbot/steps/vstudio.py @@ -131,17 +131,6 @@ def __init__(self, self.PATH = PATH # always upcall ! ShellCommand.__init__(self, **kwargs) - self.addFactoryArguments( - installdir = installdir, - mode = mode, - projectfile = projectfile, - config = config, - useenv = useenv, - project = project, - INCLUDE = INCLUDE, - LIB = LIB, - PATH = PATH - ) def setupLogfiles(self, cmd, logfiles): logwarnings = self.addLog("warnings") @@ -189,7 +178,7 @@ def createSummary(self, log): self.step_status.setStatistic('errors', self.logobserver.nbErrors) def evaluateCommand(self, cmd): - if cmd.rc != 0: + if cmd.didFail(): return FAILURE if self.logobserver.nbErrors > 0: return FAILURE @@ -305,7 +294,6 @@ def __init__(self, arch = "x86", **kwargs): # always upcall ! VisualStudio.__init__(self, **kwargs) - self.addFactoryArguments(arch = arch) def setupEnvironment(self, cmd): VisualStudio.setupEnvironment(self, cmd) diff --git a/master/buildbot/test/fake/fakebuild.py b/master/buildbot/test/fake/fakebuild.py index 02ee1b14e7d..50251a28dbf 100644 --- a/master/buildbot/test/fake/fakebuild.py +++ b/master/buildbot/test/fake/fakebuild.py @@ -14,6 +14,7 @@ # Copyright Buildbot Team Members import mock +import posixpath from twisted.python import components from buildbot.process import properties from buildbot import interfaces @@ -34,6 +35,7 @@ class FakeBuild(mock.Mock, properties.PropertiesMixin): def __init__(self, *args, **kwargs): mock.Mock.__init__(self, *args, **kwargs) self.build_status = FakeBuildStatus() + self.path_module = posixpath pr = self.build_status.properties = properties.Properties() pr.build = self diff --git a/master/buildbot/test/fake/fakemaster.py b/master/buildbot/test/fake/fakemaster.py index 4494513912c..2c0a7be5c51 100644 --- a/master/buildbot/test/fake/fakemaster.py +++ b/master/buildbot/test/fake/fakemaster.py @@ -16,7 +16,7 @@ import weakref from twisted.internet import defer from buildbot.test.fake import fakedb -from buildbot.test.fake.pbmanager import FakePBManager +from buildbot.test.fake import pbmanager from buildbot import config import mock @@ -37,7 +37,36 @@ def mkref(x): return d -class FakeMaster(mock.Mock): +class FakeCaches(object): + + def get_cache(self, name, miss_fn): + return FakeCache(name, miss_fn) + + +class FakeBotMaster(object): + + pass + + +class FakeStatus(object): + + def builderAdded(self, name, basedir, category=None): + return FakeBuilderStatus() + + +class FakeBuilderStatus(object): + + def setSlavenames(self, names): + pass + + def setCacheSize(self, size): + pass + + def setBigState(self, state): + pass + + +class FakeMaster(object): """ Create a fake Master instance: a Mock with some convenience implementations: @@ -46,15 +75,22 @@ class FakeMaster(mock.Mock): """ def __init__(self, master_id=fakedb.FakeBuildRequestsComponent.MASTER_ID): - mock.Mock.__init__(self, name="fakemaster") self._master_id = master_id self.config = config.MasterConfig() - self.caches.get_cache = FakeCache - self.pbmanager = FakePBManager() + self.caches = FakeCaches() + self.pbmanager = pbmanager.FakePBManager() + self.basedir = 'basedir' + self.botmaster = FakeBotMaster() + self.botmaster.parent = self + self.status = FakeStatus() + self.status.master = self def getObjectId(self): return defer.succeed(self._master_id) + def subscribeToBuildRequests(self, callback): + pass + # work around http://code.google.com/p/mock/issues/detail?id=105 def _get_child_mock(self, **kw): return mock.Mock(**kw) diff --git a/master/buildbot/test/fake/libvirt.py b/master/buildbot/test/fake/libvirt.py new file mode 100644 index 00000000000..2a32f6e71c8 --- /dev/null +++ b/master/buildbot/test/fake/libvirt.py @@ -0,0 +1,67 @@ +# This file is part of Buildbot. Buildbot is free software: you can +# redistribute it and/or modify it under the terms of the GNU General Public +# License as published by the Free Software Foundation, version 2. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +# details. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, write to the Free Software Foundation, Inc., 51 +# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Copyright Buildbot Team Members + +class Domain(object): + + def __init__(self, name, conn): + self.conn = conn + self._name = name + self.running = False + + def name(self): + return self._name + + def create(self): + self.running = True + + def shutdown(self): + self.running = False + + def destroy(self): + self.running = False + del self.conn[self._name] + + +class Connection(object): + + def __init__(self, uri): + self.uri = uri + self.domains = {} + + def createXML(self, xml, flags): + #FIXME: This should really parse the name out of the xml, i guess + d = self.fake_add("instance") + d.running = True + return d + + def listDomainsID(self): + return self.domains.keys() + + def lookupByName(self, name): + return self.domains[name] + + def lookupByID(self, ID): + return self.domains[ID] + + def fake_add(self, name): + d = Domain(name, self) + self.domains[name] = d + return d + + +def open(uri): + return Connection(uri) + + diff --git a/master/buildbot/test/fake/remotecommand.py b/master/buildbot/test/fake/remotecommand.py index b3d176e621d..97ddedc505b 100644 --- a/master/buildbot/test/fake/remotecommand.py +++ b/master/buildbot/test/fake/remotecommand.py @@ -19,13 +19,15 @@ from cStringIO import StringIO -DEFAULT_TIMEOUT="DEFAULT_TIMEOUT" -DEFAULT_MAXTIME="DEFAULT_MAXTIME" -DEFAULT_USEPTY="DEFAULT_USEPTY" +class FakeRemoteCommand(object): -class FakeRemoteCommand: + # callers should set this to the running TestCase instance + testcase = None - def __init__(self, remote_command, args, collectStdout=False, ignore_updates=False): + active = False + + def __init__(self, remote_command, args, + ignore_updates=False, collectStdout=False, successfulRC=(0,)): # copy the args and set a few defaults self.remote_command = remote_command self.args = args.copy() @@ -34,9 +36,14 @@ def __init__(self, remote_command, args, collectStdout=False, ignore_updates=Fal self.rc = -999 self.collectStdout = collectStdout self.updates = {} + self.successfulRC = successfulRC if collectStdout: self.stdout = '' + def run(self, step, remote): + # delegate back to the test case + return self.testcase._remotecommand_run(self, step, remote) + def useLog(self, log, closeWhenFinished=False, logfileName=None): if not logfileName: logfileName = log.getName() @@ -45,25 +52,32 @@ def useLog(self, log, closeWhenFinished=False, logfileName=None): def useLogDelayed(self, logfileName, activateCallBack, closeWhenFinished=False): self.delayedLogs[logfileName] = (activateCallBack, closeWhenFinished) - def run(self, step, remote): - # delegate back to the test case - return self.testcase._remotecommand_run(self, step, remote) + def interrupt(self, why): + raise NotImplementedError + + def didFail(self): + return self.rc not in self.successfulRC + + def fakeLogData(self, step, log, header='', stdout='', stderr=''): + # note that this should not be used in the same test as useLog(Delayed) + self.logs[log] = l = FakeLogFile(log, step) + l.fakeData(header=header, stdout=stdout, stderr=stderr) class FakeRemoteShellCommand(FakeRemoteCommand): def __init__(self, workdir, command, env=None, want_stdout=1, want_stderr=1, - timeout=DEFAULT_TIMEOUT, maxTime=DEFAULT_MAXTIME, logfiles={}, - initialStdin=None, - usePTY=DEFAULT_USEPTY, logEnviron=True, collectStdout=False): + timeout=20*60, maxTime=None, logfiles={}, + usePTY="slave-config", logEnviron=True, collectStdout=False, + interruptSignal=None, initialStdin=None, successfulRC=(0,)): args = dict(workdir=workdir, command=command, env=env or {}, want_stdout=want_stdout, want_stderr=want_stderr, initial_stdin=initialStdin, timeout=timeout, maxTime=maxTime, logfiles=logfiles, usePTY=usePTY, logEnviron=logEnviron) FakeRemoteCommand.__init__(self, "shell", args, - collectStdout=collectStdout) + collectStdout=collectStdout, successfulRC=successfulRC) class FakeLogFile(object): @@ -118,6 +132,16 @@ def getChunks(self, channels=[], onlyText=False): def finish(self): pass + def fakeData(self, header='', stdout='', stderr=''): + if header: + self.header += header + self.chunks.append((HEADER, header)) + if stdout: + self.stdout += stdout + self.chunks.append((STDOUT, stdout)) + if stderr: + self.stderr += stderr + self.chunks.append((STDERR, stderr)) class ExpectRemoteRef(object): """ @@ -240,8 +264,8 @@ class ExpectShell(Expect): """ def __init__(self, workdir, command, env={}, want_stdout=1, want_stderr=1, initialStdin=None, - timeout=DEFAULT_TIMEOUT, maxTime=DEFAULT_MAXTIME, logfiles={}, - usePTY=DEFAULT_USEPTY, logEnviron=True): + timeout=20*60, maxTime=None, logfiles={}, + usePTY="slave-config", logEnviron=True): args = dict(workdir=workdir, command=command, env=env, want_stdout=want_stdout, want_stderr=want_stderr, initial_stdin=initialStdin, diff --git a/master/buildbot/test/integration/test_slave_comm.py b/master/buildbot/test/integration/test_slave_comm.py index 4ebc451190b..9de9035bfb6 100644 --- a/master/buildbot/test/integration/test_slave_comm.py +++ b/master/buildbot/test/integration/test_slave_comm.py @@ -23,6 +23,7 @@ from buildbot.test.util import compat from buildbot.process import botmaster, builder from buildbot import pbmanager, buildslave, config +from buildbot.status import master from buildbot.test.fake import fakemaster class FakeSlaveBuilder(pb.Referenceable): @@ -115,6 +116,8 @@ def setUp(self): self.botmaster = botmaster.BotMaster(self.master) self.botmaster.startService() + self.master.status = master.Status(self.master) + self.buildslave = None self.port = None self.slavebuildslave = None diff --git a/master/buildbot/test/integration/test_upgrade.py b/master/buildbot/test/integration/test_upgrade.py index e41b14053f0..6fdc1ad7e10 100644 --- a/master/buildbot/test/integration/test_upgrade.py +++ b/master/buildbot/test/integration/test_upgrade.py @@ -107,7 +107,7 @@ def setUpUpgradeTest(self): os.makedirs("basedir") self.basedir = os.path.abspath("basedir") - master = fakemaster.make_master() + self.master = master = fakemaster.make_master() master.config.db['db_url'] = self.db_url self.db = connector.DBConnector(master, self.basedir) yield self.db.setup(check_version=False) @@ -382,7 +382,21 @@ def verify_thd(self, conn): ]) def test_upgrade(self): - return self.do_test_upgrade() + d = self.do_test_upgrade() + @d.addCallback + def check_pickles(_): + # try to unpickle things down to the level of a logfile + filename = os.path.join(self.basedir, 'builder', 'builder') + with open(filename, "rb") as f: + builder_status = cPickle.load(f) + builder_status.master = self.master + builder_status.basedir = os.path.join(self.basedir, 'builder') + b0 = builder_status.loadBuildFromFile(0) + logs = b0.getLogs() + log = logs[0] + text = log.getText() + self.assertIn('HEAD is now at', text) + return d class UpgradeTestV083(UpgradeTestMixin, unittest.TestCase): @@ -490,6 +504,99 @@ def test_upgrade(self): return self.do_test_upgrade() +class UpgradeTestV085(UpgradeTestMixin, unittest.TestCase): + + source_tarball = "v085.tgz" + + def verify_thd(self, conn): + "partially verify the contents of the db - run in a thread" + model = self.db.model + + tbl = model.buildrequests + r = conn.execute(tbl.select(order_by=tbl.c.id)) + buildreqs = [ (br.id, br.buildsetid, + br.complete, br.results) + for br in r.fetchall() ] + self.assertEqual(buildreqs, [(1, 1, 1, 0), (2, 2, 1, 0)]) + + br_claims = model.buildrequest_claims + objects = model.objects + r = conn.execute(sa.select([ br_claims.outerjoin(objects, + br_claims.c.objectid == objects.c.id)])) + buildreqs = [ (brc.brid, int(brc.claimed_at), brc.name, brc.class_name) + for brc in r.fetchall() ] + self.assertEqual(buildreqs, [ + (1, 1338226540, u'euclid.r.igoro.us:/A/bbrun', + u'buildbot.master.BuildMaster'), + (2, 1338226574, u'euclid.r.igoro.us:/A/bbrun', + u'buildbot.master.BuildMaster') + ]) + + def test_upgrade(self): + d = self.do_test_upgrade() + @d.addCallback + def check_pickles(_): + # try to unpickle things down to the level of a logfile + filename = os.path.join(self.basedir, 'builder', 'builder') + with open(filename, "rb") as f: + builder_status = cPickle.load(f) + builder_status.master = self.master + builder_status.basedir = os.path.join(self.basedir, 'builder') + b1 = builder_status.loadBuildFromFile(1) + logs = b1.getLogs() + log = logs[0] + text = log.getText() + self.assertIn('HEAD is now at', text) + b2 = builder_status.loadBuildFromFile(1) + self.assertEqual(b2.getReason(), + "The web-page 'rebuild' button was pressed by '': \n") + return d + + +class UpgradeTestV086p1(UpgradeTestMixin, unittest.TestCase): + + source_tarball = "v086p1.tgz" + + def verify_thd(self, conn): + "partially verify the contents of the db - run in a thread" + model = self.db.model + + tbl = model.buildrequests + r = conn.execute(tbl.select(order_by=tbl.c.id)) + buildreqs = [ (br.id, br.buildsetid, + br.complete, br.results) + for br in r.fetchall() ] + self.assertEqual(buildreqs, [(1, 1, 1, 4)]) # note EXCEPTION status + + br_claims = model.buildrequest_claims + objects = model.objects + r = conn.execute(sa.select([ br_claims.outerjoin(objects, + br_claims.c.objectid == objects.c.id)])) + buildreqs = [ (brc.brid, int(brc.claimed_at), brc.name, brc.class_name) + for brc in r.fetchall() ] + self.assertEqual(buildreqs, [ + (1, 1338229046, u'euclid.r.igoro.us:/A/bbrun', + u'buildbot.master.BuildMaster'), + ]) + + def test_upgrade(self): + d = self.do_test_upgrade() + @d.addCallback + def check_pickles(_): + # try to unpickle things down to the level of a logfile + filename = os.path.join(self.basedir, 'builder', 'builder') + with open(filename, "rb") as f: + builder_status = cPickle.load(f) + builder_status.master = self.master + builder_status.basedir = os.path.join(self.basedir, 'builder') + b0 = builder_status.loadBuildFromFile(0) + logs = b0.getLogs() + log = logs[0] + text = log.getText() + self.assertIn('HEAD is now at', text) + return d + + class TestWeirdChanges(change_import.ChangeImportMixin, unittest.TestCase): def setUp(self): d = self.setUpChangeImport() diff --git a/master/buildbot/test/integration/v085-README.txt b/master/buildbot/test/integration/v085-README.txt new file mode 100644 index 00000000000..1e2747ed069 --- /dev/null +++ b/master/buildbot/test/integration/v085-README.txt @@ -0,0 +1,4 @@ +-- Basic v0.8.5 tarball -- + +This tarball is the result of a few runs from a single incarnation of a master +that was running Buildbot-0.8.5. diff --git a/master/buildbot/test/integration/v085.tgz b/master/buildbot/test/integration/v085.tgz new file mode 100644 index 00000000000..90056d9e4e0 Binary files /dev/null and b/master/buildbot/test/integration/v085.tgz differ diff --git a/master/buildbot/test/integration/v086p1-README.txt b/master/buildbot/test/integration/v086p1-README.txt new file mode 100644 index 00000000000..e6f508c9ff4 --- /dev/null +++ b/master/buildbot/test/integration/v086p1-README.txt @@ -0,0 +1,5 @@ +-- Basic v0.8.6p1 tarball -- + +This tarball is the result of a few runs from a single incarnation of a master +that was running Buildbot-0.8.6p1. It has only one build, which was +interrupted. diff --git a/master/buildbot/test/integration/v086p1.tgz b/master/buildbot/test/integration/v086p1.tgz new file mode 100644 index 00000000000..ea8b7af5ab9 Binary files /dev/null and b/master/buildbot/test/integration/v086p1.tgz differ diff --git a/master/buildbot/test/interfaces/test_remotecommand.py b/master/buildbot/test/interfaces/test_remotecommand.py index 72b863f036b..7fa6513fcaf 100644 --- a/master/buildbot/test/interfaces/test_remotecommand.py +++ b/master/buildbot/test/interfaces/test_remotecommand.py @@ -25,41 +25,75 @@ class Tests(interfaces.InterfaceTests): - def makeRemoteCommand(self, name, args): - raise NotImplementedError + remoteCommandClass = None + + def makeRemoteCommand(self): + return self.remoteCommandClass('ping', {'arg':'val'}) + + def test_signature_RemoteCommand_constructor(self): + @self.assertArgSpecMatches(self.remoteCommandClass.__init__) + def __init__(self, remote_command, args, ignore_updates=False, + collectStdout=False, successfulRC=(0,)): + pass + + def test_signature_RemoteShellCommand_constructor(self): + @self.assertArgSpecMatches(self.remoteShellCommandClass.__init__) + def __init__(self, workdir, command, env=None, want_stdout=1, + want_stderr=1, timeout=20*60, maxTime=None, logfiles={}, + usePTY="slave-config", logEnviron=True, collectStdout=False, + interruptSignal=None, initialStdin=None, successfulRC=(0,)): + pass + + def test_signature_run(self): + cmd = self.makeRemoteCommand() + @self.assertArgSpecMatches(cmd.run) + def run(self, step, remote): + pass def test_signature_useLog(self): - rc = self.makeRemoteCommand('ping', {'arg':'val'}) - @self.assertArgSpecMatches(rc.useLog) + cmd = self.makeRemoteCommand() + @self.assertArgSpecMatches(cmd.useLog) def useLog(self, log, closeWhenFinished=False, logfileName=None): pass def test_signature_useLogDelayed(self): - rc = self.makeRemoteCommand('ping', {'arg':'val'}) - @self.assertArgSpecMatches(rc.useLogDelayed) + cmd = self.makeRemoteCommand() + @self.assertArgSpecMatches(cmd.useLogDelayed) def useLogDelayed(self, logfileName, activateCallBack, closeWhenFinished=False): pass - def test_signature_run(self): - rc = self.makeRemoteCommand('ping', {'arg':'val'}) - @self.assertArgSpecMatches(rc.run) - def run(self, step, remote): + def test_signature_interrupt(self): + cmd = self.makeRemoteCommand() + @self.assertArgSpecMatches(cmd.interrupt) + def useLogDelayed(self, why): pass + def test_signature_didFail(self): + cmd = self.makeRemoteCommand() + @self.assertArgSpecMatches(cmd.didFail) + def useLogDelayed(self): + pass -class RealTests(Tests): - pass + def test_signature_logs(self): + cmd = self.makeRemoteCommand() + self.assertIsInstance(cmd.logs, dict) + def test_signature_active(self): + cmd = self.makeRemoteCommand() + self.assertIsInstance(cmd.active, bool) -class TestRunCommand(unittest.TestCase, RealTests): + def test_RemoteShellCommand_constructor(self): + self.remoteShellCommandClass('wkdir', 'some-command') - def makeRemoteCommand(self, name, args): - return buildstep.RemoteCommand(name, args) +class TestRunCommand(unittest.TestCase, Tests): + + remoteCommandClass = buildstep.RemoteCommand + remoteShellCommandClass = buildstep.RemoteShellCommand -class TestFakeRunCommand(unittest.TestCase, Tests): - def makeRemoteCommand(self, name, args): - return remotecommand.FakeRemoteCommand(name, args) +class TestFakeRunCommand(unittest.TestCase, Tests): + remoteCommandClass = remotecommand.FakeRemoteCommand + remoteShellCommandClass = remotecommand.FakeRemoteShellCommand diff --git a/master/buildbot/test/regressions/test_oldpaths.py b/master/buildbot/test/regressions/test_oldpaths.py index c42fc2a39ed..54f4ae0992b 100644 --- a/master/buildbot/test/regressions/test_oldpaths.py +++ b/master/buildbot/test/regressions/test_oldpaths.py @@ -181,10 +181,6 @@ def test_steps_source_P4(self): from buildbot.steps.source import P4 assert P4 - def test_steps_source_P4Sync(self): - from buildbot.steps.source import P4Sync - assert P4Sync - def test_steps_source_Monotone(self): from buildbot.steps.source import Monotone assert Monotone diff --git a/master/buildbot/test/unit/test_buildslave.py b/master/buildbot/test/unit/test_buildslave.py index 043bdefae39..078c729ff9f 100644 --- a/master/buildbot/test/unit/test_buildslave.py +++ b/master/buildbot/test/unit/test_buildslave.py @@ -107,26 +107,16 @@ def test_reconfigService_attrs(self): self.assertEqual(self.master.pbmanager._registrations, []) self.assertTrue(old.updateSlave.called) - @defer.deferredGenerator + @defer.inlineCallbacks def test_reconfigService_has_properties(self): old = self.ConcreteBuildSlave('bot', 'pass') - - wfd = defer.waitForDeferred( - self.do_test_reconfigService(old, 'tcp:1234', old, 'tcp:1234')) - yield wfd - wfd.getResult() - + yield self.do_test_reconfigService(old, 'tcp:1234', old, 'tcp:1234') self.assertTrue(old.properties.getProperty('slavename'), 'bot') - @defer.deferredGenerator + @defer.inlineCallbacks def test_reconfigService_initial_registration(self): old = self.ConcreteBuildSlave('bot', 'pass') - - wfd = defer.waitForDeferred( - self.do_test_reconfigService(old, None, old, 'tcp:1234')) - yield wfd - wfd.getResult() - + yield self.do_test_reconfigService(old, None, old, 'tcp:1234') self.assertEqual(self.master.pbmanager._registrations, [('tcp:1234', 'bot', 'pass')]) @defer.inlineCallbacks diff --git a/master/buildbot/test/unit/test_changes_changes.py b/master/buildbot/test/unit/test_changes_changes.py new file mode 100644 index 00000000000..6dcee925731 --- /dev/null +++ b/master/buildbot/test/unit/test_changes_changes.py @@ -0,0 +1,113 @@ +# This file is part of Buildbot. Buildbot is free software: you can +# redistribute it and/or modify it under the terms of the GNU General Public +# License as published by the Free Software Foundation, version 2. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +# details. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, write to the Free Software Foundation, Inc., 51 +# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Copyright Buildbot Team Members + +import textwrap +import re +from twisted.trial import unittest +from buildbot.test.fake import fakedb +from buildbot.changes import changes + +class Change(unittest.TestCase): + + change23_rows = [ + fakedb.Change(changeid=23, author="dustin", comments="fix whitespace", + is_dir=0, branch="warnerdb", revision="deadbeef", + when_timestamp=266738404, revlink='http://warner/0e92a098b', + category='devel', repository='git://warner', codebase='mainapp', + project='Buildbot'), + + fakedb.ChangeFile(changeid=23, filename='master/README.txt'), + fakedb.ChangeFile(changeid=23, filename='slave/README.txt'), + + fakedb.ChangeProperty(changeid=23, property_name='notest', + property_value='["no","Change"]'), + + fakedb.ChangeUser(changeid=23, uid=27), + ] + + def setUp(self): + self.change23 = changes.Change(**dict( # using **dict(..) forces kwargs + category='devel', + isdir=0, + repository=u'git://warner', + codebase=u'mainapp', + who=u'dustin', + when=266738404, + comments=u'fix whitespace', + project=u'Buildbot', + branch=u'warnerdb', + revlink=u'http://warner/0e92a098b', + properties={'notest':"no"}, + files=[u'master/README.txt', u'slave/README.txt'], + revision=u'deadbeef')) + self.change23.number = 23 + + def test_str(self): + string = str(self.change23) + self.assertTrue(re.match(r"Change\(.*\)", string), string) + + def test_asText(self): + text = self.change23.asText() + self.assertTrue(re.match(textwrap.dedent(u'''\ + Files: + master/README.txt + slave/README.txt + On: git://warner + For: Buildbot + At: .* + Changed By: dustin + Comments: fix whitespaceProperties: + notest: no + + '''), text), text) + + def test_asDict(self): + dict = self.change23.asDict() + self.assertIn('1978', dict['at']) # timezone-sensitive + del dict['at'] + self.assertEqual(dict, { + 'branch': u'warnerdb', + 'category': u'devel', + 'codebase': u'mainapp', + 'comments': u'fix whitespace', + 'files': [{'name': u'master/README.txt'}, + {'name': u'slave/README.txt'}], + 'number': 23, + 'project': u'Buildbot', + 'properties': [('notest', 'no', 'Change')], + 'repository': u'git://warner', + 'rev': u'deadbeef', + 'revision': u'deadbeef', + 'revlink': u'http://warner/0e92a098b', + 'when': 266738404, + 'who': u'dustin'}) + + def test_getShortAuthor(self): + self.assertEqual(self.change23.getShortAuthor(), 'dustin') + + def test_getTime(self): + # careful, or timezones will hurt here + self.assertIn('Jun 1978', self.change23.getTime()) + + def test_getTimes(self): + self.assertEqual(self.change23.getTimes(), (266738404, None)) + + def test_getText(self): + self.change23.who = 'nasty < nasty' # test the html escaping (ugh!) + self.assertEqual(self.change23.getText(), ['nasty < nasty']) + + def test_getLogs(self): + self.assertEqual(self.change23.getLogs(), {}) + diff --git a/master/buildbot/test/unit/test_config.py b/master/buildbot/test/unit/test_config.py index 426b6d98690..45da51bffff 100644 --- a/master/buildbot/test/unit/test_config.py +++ b/master/buildbot/test/unit/test_config.py @@ -546,6 +546,12 @@ def test_load_caches(self): self.errors) self.assertResults(caches=dict(Changes=10, Builds=15, foo=1)) + def test_load_caches_entries_test(self): + self.cfg.load_caches(self.filename, + dict(caches=dict(foo="1")), + self.errors) + self.assertConfigError(self.errors, + "value for cache size 'foo' must be an integer") def test_load_schedulers_defaults(self): self.cfg.load_schedulers(self.filename, {}, self.errors) @@ -607,6 +613,15 @@ def test_load_builders_dict(self): self.assertIsInstance(self.cfg.builders[0], config.BuilderConfig) self.assertEqual(self.cfg.builders[0].name, 'x') + @compat.usesFlushWarnings + def test_load_builders_abs_builddir(self): + bldr = dict(name='x', factory=mock.Mock(), slavename='x', + builddir=os.path.abspath('.')) + self.cfg.load_builders(self.filename, + dict(builders=[bldr]), self.errors) + self.assertEqual( + len(self.flushWarnings([self.cfg.load_builders])), + 1) def test_load_slaves_defaults(self): self.cfg.load_slaves(self.filename, {}, self.errors) @@ -700,13 +715,12 @@ def setup_basic_attrs(self): self.cfg.slaves = [ mock.Mock() ] self.cfg.builders = [ b1, b2 ] - def setup_builder_locks(self, builder_lock=None, dup_builder_lock=False, - step_lock=None, dup_step_lock=False): + def setup_builder_locks(self, builder_lock=None, dup_builder_lock=False): def bldr(name): b = mock.Mock() b.name = name b.locks = [] - b.factory.steps = [ ('cls', dict(locks=[])) ] + b.factory.steps = [ ('cls', (), dict(locks=[])) ] return b def lock(name): @@ -720,11 +734,6 @@ def lock(name): b1.locks.append(lock(builder_lock)) if dup_builder_lock: b2.locks.append(lock(builder_lock)) - if step_lock: - s1, s2 = b1.factory.steps[0][1], b2.factory.steps[0][1] - s1['locks'].append(lock(step_lock)) - if dup_step_lock: - s2['locks'].append(lock(step_lock)) # tests @@ -767,23 +776,13 @@ def test_check_schedulers(self): self.assertNoConfigErrors(self.errors) - def test_check_locks_step_and_builder(self): - self.setup_builder_locks(builder_lock='l', step_lock='l') - self.cfg.check_locks(self.errors) - self.assertConfigError(self.errors, "Two locks share") - def test_check_locks_dup_builder_lock(self): self.setup_builder_locks(builder_lock='l', dup_builder_lock=True) self.cfg.check_locks(self.errors) self.assertConfigError(self.errors, "Two locks share") - def test_check_locks_dup_step_lock(self): - self.setup_builder_locks(step_lock='l', dup_step_lock=True) - self.cfg.check_locks(self.errors) - self.assertConfigError(self.errors, "Two locks share") - def test_check_locks(self): - self.setup_builder_locks(builder_lock='bl', step_lock='sl') + self.setup_builder_locks(builder_lock='bl') self.cfg.check_locks(self.errors) self.assertNoConfigErrors(self.errors) diff --git a/master/buildbot/test/unit/test_db_changes.py b/master/buildbot/test/unit/test_db_changes.py index 4ab8472c8d9..f7427875814 100644 --- a/master/buildbot/test/unit/test_db_changes.py +++ b/master/buildbot/test/unit/test_db_changes.py @@ -438,6 +438,33 @@ def thd(conn): d.addCallback(check) return d + def test_pruneChanges_lots(self): + d = self.insertTestData([ + fakedb.Change(changeid=n) + for n in xrange(1, 151) + ]) + + d.addCallback(lambda _ : self.db.changes.pruneChanges(1)) + def check(_): + def thd(conn): + results = {} + for tbl_name in ('scheduler_changes', 'sourcestamp_changes', + 'change_files', 'change_properties', + 'changes'): + tbl = self.db.model.metadata.tables[tbl_name] + r = conn.execute(sa.select([tbl.c.changeid])) + results[tbl_name] = len([ r for r in r.fetchall() ]) + self.assertEqual(results, { + 'scheduler_changes': 0, + 'sourcestamp_changes': 0, + 'change_files': 0, + 'change_properties': 0, + 'changes': 1, + }) + return self.db.pool.do(thd) + d.addCallback(check) + return d + def test_pruneChanges_None(self): d = self.insertTestData(self.change13_rows) diff --git a/master/buildbot/test/unit/test_libvirtbuildslave.py b/master/buildbot/test/unit/test_libvirtbuildslave.py new file mode 100644 index 00000000000..fa04413e32a --- /dev/null +++ b/master/buildbot/test/unit/test_libvirtbuildslave.py @@ -0,0 +1,281 @@ +# This file is part of Buildbot. Buildbot is free software: you can +# redistribute it and/or modify it under the terms of the GNU General Public +# License as published by the Free Software Foundation, version 2. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +# details. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, write to the Free Software Foundation, Inc., 51 +# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Copyright Buildbot Team Members + +import mock +from twisted.trial import unittest +from twisted.internet import defer, reactor, utils +from twisted.python import failure +from buildbot import libvirtbuildslave, config +from buildbot.test.fake import libvirt +from buildbot.test.util import compat + + +class TestLibVirtSlave(unittest.TestCase): + + class ConcreteBuildSlave(libvirtbuildslave.LibVirtSlave): + pass + + def setUp(self): + self.patch(libvirtbuildslave, "libvirt", libvirt) + self.conn = libvirtbuildslave.Connection("test://") + self.lvconn = self.conn.connection + + def test_constructor_nolibvirt(self): + self.patch(libvirtbuildslave, "libvirt", None) + self.assertRaises(config.ConfigErrors, self.ConcreteBuildSlave, + 'bot', 'pass', None, 'path', 'path') + + def test_constructor_minimal(self): + bs = self.ConcreteBuildSlave('bot', 'pass', self.conn, 'path', 'otherpath') + yield bs._find_existing_deferred + self.assertEqual(bs.slavename, 'bot') + self.assertEqual(bs.password, 'pass') + self.assertEqual(bs.connection, self.conn) + self.assertEqual(bs.image, 'path') + self.assertEqual(bs.base_image, 'otherpath') + self.assertEqual(bs.keepalive_interval, 3600) + + @defer.inlineCallbacks + def test_find_existing(self): + d = self.lvconn.fake_add("bot") + + bs = self.ConcreteBuildSlave('bot', 'pass', self.conn, 'p', 'o') + yield bs._find_existing_deferred + + self.assertEqual(bs.domain.domain, d) + self.assertEqual(bs.substantiated, True) + + @defer.inlineCallbacks + def test_prepare_base_image_none(self): + self.patch(utils, "getProcessValue", mock.Mock()) + utils.getProcessValue.side_effect = lambda x,y: defer.succeed(0) + + bs = self.ConcreteBuildSlave('bot', 'pass', self.conn, 'p', None) + yield bs._find_existing_deferred + yield bs._prepare_base_image() + + self.assertEqual(utils.getProcessValue.call_count, 0) + + @defer.inlineCallbacks + def test_prepare_base_image_cheap(self): + self.patch(utils, "getProcessValue", mock.Mock()) + utils.getProcessValue.side_effect = lambda x,y: defer.succeed(0) + + bs = self.ConcreteBuildSlave('bot', 'pass', self.conn, 'p', 'o') + yield bs._find_existing_deferred + yield bs._prepare_base_image() + + utils.getProcessValue.assert_called_with( + "qemu-img", ["create", "-b", "o", "-f", "qcow2", "p"]) + + @defer.inlineCallbacks + def test_prepare_base_image_full(self): + pass + self.patch(utils, "getProcessValue", mock.Mock()) + utils.getProcessValue.side_effect = lambda x,y: defer.succeed(0) + + bs = self.ConcreteBuildSlave('bot', 'pass', self.conn, 'p', 'o') + yield bs._find_existing_deferred + bs.cheap_copy = False + yield bs._prepare_base_image() + + utils.getProcessValue.assert_called_with( + "cp", ["o", "p"]) + + @defer.inlineCallbacks + def test_start_instance(self): + bs = self.ConcreteBuildSlave('b', 'p', self.conn, 'p', 'o', + xml='') + + prep = mock.Mock() + prep.side_effect = lambda: defer.succeed(0) + self.patch(bs, "_prepare_base_image", prep) + + yield bs._find_existing_deferred + started = yield bs.start_instance(mock.Mock()) + + self.assertEqual(started, True) + + @compat.usesFlushLoggedErrors + @defer.inlineCallbacks + def test_start_instance_create_fails(self): + bs = self.ConcreteBuildSlave('b', 'p', self.conn, 'p', 'o', + xml='') + + prep = mock.Mock() + prep.side_effect = lambda: defer.succeed(0) + self.patch(bs, "_prepare_base_image", prep) + + create = mock.Mock() + create.side_effect = lambda self : defer.fail( + failure.Failure(RuntimeError('oh noes'))) + self.patch(libvirtbuildslave.Connection, 'create', create) + + yield bs._find_existing_deferred + started = yield bs.start_instance(mock.Mock()) + + self.assertEqual(bs.domain, None) + self.assertEqual(started, False) + self.assertEqual(len(self.flushLoggedErrors(RuntimeError)), 1) + + @defer.inlineCallbacks + def setup_canStartBuild(self): + bs = self.ConcreteBuildSlave('b', 'p', self.conn, 'p', 'o') + yield bs._find_existing_deferred + bs.updateLocks() + defer.returnValue(bs) + + @defer.inlineCallbacks + def test_canStartBuild(self): + bs = yield self.setup_canStartBuild() + self.assertEqual(bs.canStartBuild(), True) + + @defer.inlineCallbacks + def test_canStartBuild_notready(self): + """ + If a LibVirtSlave hasnt finished scanning for existing VMs then we shouldn't + start builds on it as it might create a 2nd VM when we want to reuse the existing + one. + """ + bs = yield self.setup_canStartBuild() + bs.ready = False + self.assertEqual(bs.canStartBuild(), False) + + @defer.inlineCallbacks + def test_canStartBuild_domain_and_not_connected(self): + """ + If we've found that the VM this slave would instance already exists but hasnt + connected then we shouldn't start builds or we'll end up with a dupe. + """ + bs = yield self.setup_canStartBuild() + bs.domain = mock.Mock() + self.assertEqual(bs.canStartBuild(), False) + + @defer.inlineCallbacks + def test_canStartBuild_domain_and_connected(self): + """ + If we've found an existing VM and it is connected then we should start builds + """ + bs = yield self.setup_canStartBuild() + bs.domain = mock.Mock() + isconnected = mock.Mock() + isconnected.return_value = True + self.patch(bs, "isConnected", isconnected) + self.assertEqual(bs.canStartBuild(), True) + + +class TestWorkQueue(unittest.TestCase): + + def setUp(self): + self.queue = libvirtbuildslave.WorkQueue() + + def delayed_success(self): + def work(): + d = defer.Deferred() + reactor.callLater(0, d.callback, True) + return d + return work + + def delayed_errback(self): + def work(): + d = defer.Deferred() + reactor.callLater(0, d.errback, + failure.Failure(RuntimeError("Test failure"))) + return d + return work + + def expect_errback(self, d): + def shouldnt_get_called(f): + self.failUnlessEqual(True, False) + d.addCallback(shouldnt_get_called) + def errback(f): + #log.msg("errback called?") + pass + d.addErrback(errback) + return d + + def test_handle_exceptions(self): + def work(): + raise ValueError + return self.expect_errback(self.queue.execute(work)) + + def test_handle_immediate_errback(self): + def work(): + return defer.fail(RuntimeError("Sad times")) + return self.expect_errback(self.queue.execute(work)) + + def test_handle_delayed_errback(self): + work = self.delayed_errback() + return self.expect_errback(self.queue.execute(work)) + + def test_handle_immediate_success(self): + def work(): + return defer.succeed(True) + return self.queue.execute(work) + + def test_handle_delayed_success(self): + work = self.delayed_success() + return self.queue.execute(work) + + def test_single_pow_fires(self): + return self.queue.execute(self.delayed_success()) + + def test_single_pow_errors_gracefully(self): + d = self.queue.execute(self.delayed_errback()) + return self.expect_errback(d) + + def test_fail_doesnt_break_further_work(self): + self.expect_errback(self.queue.execute(self.delayed_errback())) + return self.queue.execute(self.delayed_success()) + + def test_second_pow_fires(self): + self.queue.execute(self.delayed_success()) + return self.queue.execute(self.delayed_success()) + + def test_work(self): + # We want these deferreds to fire in order + flags = {1: False, 2: False, 3: False } + + # When first deferred fires, flags[2] and flags[3] should still be false + # flags[1] shouldnt already be set, either + d1 = self.queue.execute(self.delayed_success()) + def cb1(res): + self.failUnlessEqual(flags[1], False) + flags[1] = True + self.failUnlessEqual(flags[2], False) + self.failUnlessEqual(flags[3], False) + d1.addCallback(cb1) + + # When second deferred fires, only flags[3] should be set + # flags[2] should definitely be False + d2 = self.queue.execute(self.delayed_success()) + def cb2(res): + assert flags[2] == False + flags[2] = True + assert flags[1] == True + assert flags[3] == False + d2.addCallback(cb2) + + # When third deferred fires, only flags[3] should be unset + d3 = self.queue.execute(self.delayed_success()) + def cb3(res): + assert flags[3] == False + flags[3] = True + assert flags[1] == True + assert flags[2] == True + d3.addCallback(cb3) + + return defer.DeferredList([d1, d2, d3], fireOnOneErrback=True) + diff --git a/master/buildbot/test/unit/test_process_build.py b/master/buildbot/test/unit/test_process_build.py index 373521ca878..4edc1be0cc5 100644 --- a/master/buildbot/test/unit/test_process_build.py +++ b/master/buildbot/test/unit/test_process_build.py @@ -84,6 +84,15 @@ class FakeBuildStatus(Mock): class FakeBuilderStatus: implements(interfaces.IBuilderStatus) +class FakeStepFactory(object): + """Fake step factory that just returns a fixed step object.""" + implements(interfaces.IBuildStepFactory) + def __init__(self, step): + self.step = step + + def buildStep(self): + return self.step + class TestBuild(unittest.TestCase): def setUp(self): @@ -103,7 +112,7 @@ def testRunSuccessfulBuild(self): step = Mock() step.return_value = step step.startStep.return_value = SUCCESS - b.setStepFactories([(step, {})]) + b.setStepFactories([FakeStepFactory(step)]) slavebuilder = Mock() @@ -118,7 +127,7 @@ def testStopBuild(self): step = Mock() step.return_value = step - b.setStepFactories([(step, {})]) + b.setStepFactories([FakeStepFactory(step)]) slavebuilder = Mock() @@ -149,8 +158,8 @@ def testAlwaysRunStepStopBuild(self): step2.return_value = step2 step2.alwaysRun = True b.setStepFactories([ - (step1, {}), - (step2, {}), + FakeStepFactory(step1), + FakeStepFactory(step2), ]) slavebuilder = Mock() @@ -197,7 +206,7 @@ def claim(owner, access): step = Mock() step.return_value = step step.startStep.return_value = SUCCESS - b.setStepFactories([(step, {})]) + b.setStepFactories([FakeStepFactory(step)]) b.startBuild(FakeBuildStatus(), None, slavebuilder) @@ -226,7 +235,7 @@ def claim(owner, access): step = Mock() step.return_value = step step.startStep.return_value = SUCCESS - b.setStepFactories([(step, {})]) + b.setStepFactories([FakeStepFactory(step)]) real_lock.claim(Mock(), l.access('counting')) @@ -253,7 +262,7 @@ def testStopBuildWaitingForLocks(self): step.return_value = step step.startStep.return_value = SUCCESS step.alwaysRun = False - b.setStepFactories([(step, {})]) + b.setStepFactories([FakeStepFactory(step)]) real_lock.claim(Mock(), l.access('counting')) @@ -286,7 +295,7 @@ def testStopBuildWaitingForLocks_lostRemote(self): step.return_value = step step.startStep.return_value = SUCCESS step.alwaysRun = False - b.setStepFactories([(step, {})]) + b.setStepFactories([FakeStepFactory(step)]) real_lock.claim(Mock(), l.access('counting')) @@ -317,9 +326,7 @@ def testStopBuildWaitingForStepLocks(self): real_lock = b.builder.botmaster.getLockByID(l).getLock(slavebuilder) step = LoggingBuildStep(locks=[lock_access]) - def factory(*args): - return step - b.setStepFactories([(factory, {})]) + b.setStepFactories([FakeStepFactory(step)]) real_lock.claim(Mock(), l.access('counting')) diff --git a/master/buildbot/test/unit/test_process_builder.py b/master/buildbot/test/unit/test_process_builder.py index e6d379a9b93..00823b741a8 100644 --- a/master/buildbot/test/unit/test_process_builder.py +++ b/master/buildbot/test/unit/test_process_builder.py @@ -19,6 +19,7 @@ from twisted.python import failure from twisted.internet import defer from buildbot import config +from buildbot.status import master from buildbot.test.fake import fakedb, fakemaster from buildbot.process import builder from buildbot.db import buildrequests @@ -322,12 +323,9 @@ def test_maybeStartBuild_builder_stopped(self): yield self.bldr.stopService() yield self.bldr.maybeStartBuild() - @defer.deferredGenerator + @defer.inlineCallbacks def test_maybeStartBuild_merge_ordering(self): - wfd = defer.waitForDeferred( - self.makeBuilder(patch_random=True)) - yield wfd - wfd.getResult() + yield self.makeBuilder(patch_random=True) self.setSlaveBuilders({'bldr':1}) @@ -347,12 +345,9 @@ def test_maybeStartBuild_merge_ordering(self): fakedb.BuildRequest(id=42922, buildsetid=1981, buildername="bldr", submitted_at=1332025495.19141), ] - wfd = defer.waitForDeferred( - self.do_test_maybeStartBuild(rows=rows, + yield self.do_test_maybeStartBuild(rows=rows, exp_claims=[42880, 42922], - exp_builds=[('bldr', [42880, 42922])])) - yield wfd - wfd.getResult() + exp_builds=[('bldr', [42880, 42922])]) # _chooseSlave @@ -711,6 +706,7 @@ def makeBuilder(self, name): self.bstatus = mock.Mock() self.factory = mock.Mock() self.master = fakemaster.make_master() + self.master.status = master.Status(self.master) # only include the necessary required config builder_config = config.BuilderConfig( name=name, slavename="slv", builddir="bdir", @@ -760,7 +756,8 @@ def makeBuilder(self, name, sourcestamps): self.bldr = builder.Builder(builder_config.name) self.master.db = self.db = fakedb.FakeDBConnector(self) self.bldr.master = self.master - self.bldr.master.master.addBuildset.return_value = (1, [100]) + self.master.addBuildset = addBuildset = mock.Mock() + addBuildset.return_value = (1, [100]) def do_test_rebuild(self, sourcestampsetid, @@ -784,7 +781,9 @@ def getSourceStampSetId(master): sslist.append(ssx) self.makeBuilder(name='bldr1', sourcestamps = sslist) - self.bldrctrl = builder.BuilderControl(self.bldr, self.master) + control = mock.Mock(spec=['master']) + control.master = self.master + self.bldrctrl = builder.BuilderControl(self.bldr, control) d = self.bldrctrl.rebuildBuild(self.bstatus, reason = 'unit test', extraProperties = {}) @@ -799,7 +798,7 @@ def test_rebuild_with_no_sourcestamps(self): def test_rebuild_with_single_sourcestamp(self): yield self.do_test_rebuild(101, 1) self.assertEqual(self.sslist, {1:101}) - self.master.master.addBuildset.assert_called_with(builderNames=['bldr1'], + self.master.addBuildset.assert_called_with(builderNames=['bldr1'], sourcestampsetid=101, reason = 'unit test', properties = {}) @@ -809,7 +808,7 @@ def test_rebuild_with_single_sourcestamp(self): def test_rebuild_with_multiple_sourcestamp(self): yield self.do_test_rebuild(101, 3) self.assertEqual(self.sslist, {1:101, 2:101, 3:101}) - self.master.master.addBuildset.assert_called_with(builderNames=['bldr1'], + self.master.addBuildset.assert_called_with(builderNames=['bldr1'], sourcestampsetid=101, reason = 'unit test', properties = {}) diff --git a/master/buildbot/test/unit/test_process_buildstep.py b/master/buildbot/test/unit/test_process_buildstep.py index b5fe86f3308..83c946413e1 100644 --- a/master/buildbot/test/unit/test_process_buildstep.py +++ b/master/buildbot/test/unit/test_process_buildstep.py @@ -21,7 +21,7 @@ from buildbot.process import buildstep from buildbot.process.buildstep import regex_log_evaluator from buildbot.status.results import FAILURE, SUCCESS, WARNINGS, EXCEPTION -from buildbot.test.fake import fakebuild +from buildbot.test.fake import fakebuild, remotecommand from buildbot.test.util import steps, compat class FakeLogFile: @@ -31,44 +31,54 @@ def __init__(self, text): def getText(self): return self.text -class FakeCmd: - def __init__(self, stdout, stderr, rc=0): - self.logs = {'stdout': FakeLogFile(stdout), - 'stderr': FakeLogFile(stderr)} - self.rc = rc - class FakeStepStatus: pass class TestRegexLogEvaluator(unittest.TestCase): + + def makeRemoteCommand(self, rc, stdout, stderr=''): + cmd = remotecommand.FakeRemoteCommand('cmd', {}) + cmd.fakeLogData(self, 'stdio', stdout=stdout, stderr=stderr) + cmd.rc = rc + return cmd + def test_find_worse_status(self): - cmd = FakeCmd("This is log text", "") + cmd = self.makeRemoteCommand(0, 'This is a big step') step_status = FakeStepStatus() - r = [(re.compile("This is"), FAILURE)] + r = [(re.compile("This is"), WARNINGS)] new_status = regex_log_evaluator(cmd, step_status, r) - self.assertEqual(new_status, FAILURE, "regex_log_evaluator returned %d, should've returned %d" % (new_status, FAILURE)) + self.assertEqual(new_status, WARNINGS, + "regex_log_evaluator returned %d, expected %d" + % (new_status, WARNINGS)) def test_multiple_regexes(self): - cmd = FakeCmd("Normal stdout text\nan error", "") + cmd = self.makeRemoteCommand(0, "Normal stdout text\nan error") step_status = FakeStepStatus() r = [(re.compile("Normal stdout"), SUCCESS), (re.compile("error"), FAILURE)] new_status = regex_log_evaluator(cmd, step_status, r) - self.assertEqual(new_status, FAILURE, "regex_log_evaluator returned %d, should've returned %d" % (new_status, FAILURE)) + self.assertEqual(new_status, FAILURE, + "regex_log_evaluator returned %d, expected %d" + % (new_status, FAILURE)) def test_exception_not_in_stdout(self): - cmd = FakeCmd("Completely normal output", "exception output") + cmd = self.makeRemoteCommand(0, + "Completely normal output", "exception output") step_status = FakeStepStatus() r = [(re.compile("exception"), EXCEPTION)] new_status = regex_log_evaluator(cmd, step_status, r) - self.assertEqual(new_status, EXCEPTION, "regex_log_evaluator returned %d, should've returned %d" % (new_status, EXCEPTION)) + self.assertEqual(new_status, EXCEPTION, + "regex_log_evaluator returned %d, expected %d" + % (new_status, EXCEPTION)) def test_pass_a_string(self): - cmd = FakeCmd("Output", "Some weird stuff on stderr") + cmd = self.makeRemoteCommand(0, "Output", "Some weird stuff on stderr") step_status = FakeStepStatus() r = [("weird stuff", WARNINGS)] new_status = regex_log_evaluator(cmd, step_status, r) - self.assertEqual(new_status, WARNINGS, "regex_log_evaluator returned %d, should've returned %d" % (new_status, WARNINGS)) + self.assertEqual(new_status, WARNINGS, + "regex_log_evaluator returned %d, expected %d" + % (new_status, WARNINGS)) class TestBuildStep(steps.BuildStepMixin, unittest.TestCase): @@ -85,9 +95,9 @@ def tearDown(self): # support - def _setupWaterfallTest(self, hideStepIf, expect): + def _setupWaterfallTest(self, hideStepIf, expect, expectedResult=SUCCESS): self.setupStep(TestBuildStep.FakeBuildStep(hideStepIf=hideStepIf)) - self.expectOutcome(result=SUCCESS, status_text=["generic"]) + self.expectOutcome(result=expectedResult, status_text=["generic"]) self.expectHidden(expect) # tests @@ -146,6 +156,11 @@ def shouldHide(result, step): d.addCallback(lambda _ : self.assertTrue(called[0])) return d + def test_hideStepIf_fails(self): + # 0/0 causes DivideByZeroError, which should be flagged as an exception + self._setupWaterfallTest(lambda : 0/0, False, expectedResult=EXCEPTION) + return self.runStep() + @compat.usesFlushLoggedErrors def test_hideStepIf_Callable_Exception(self): called = [False] @@ -175,20 +190,27 @@ def createException(*args, **kwargs): class TestLoggingBuildStep(unittest.TestCase): + + def makeRemoteCommand(self, rc, stdout, stderr=''): + cmd = remotecommand.FakeRemoteCommand('cmd', {}) + cmd.fakeLogData(self, 'stdio', stdout=stdout, stderr=stderr) + cmd.rc = rc + return cmd + def test_evaluateCommand_success(self): - cmd = FakeCmd("Log text", "Log text") + cmd = self.makeRemoteCommand(0, "Log text", "Log text") lbs = buildstep.LoggingBuildStep() status = lbs.evaluateCommand(cmd) self.assertEqual(status, SUCCESS, "evaluateCommand returned %d, should've returned %d" % (status, SUCCESS)) def test_evaluateCommand_failed(self): - cmd = FakeCmd("Log text", "", 23) + cmd = self.makeRemoteCommand(23, "Log text", "") lbs = buildstep.LoggingBuildStep() status = lbs.evaluateCommand(cmd) self.assertEqual(status, FAILURE, "evaluateCommand returned %d, should've returned %d" % (status, FAILURE)) def test_evaluateCommand_log_eval_func(self): - cmd = FakeCmd("Log text", "") + cmd = self.makeRemoteCommand(0, "Log text") def eval(cmd, step_status): return WARNINGS lbs = buildstep.LoggingBuildStep(log_eval_func=eval) @@ -223,6 +245,10 @@ def test_step_raining_buildstepfailed_in_start(self): def test_step_raising_exception_in_start(self): self.setupStep(FailingCustomStep(exception=ValueError)) - self.expectOutcome(result=FAILURE, status_text=["generic"]) - return self.runStep() + self.expectOutcome(result=EXCEPTION, status_text=["generic", "exception"]) + d = self.runStep() + @d.addCallback + def cb(_): + self.assertEqual(len(self.flushLoggedErrors(ValueError)), 1) + return d diff --git a/master/buildbot/test/unit/test_process_factory.py b/master/buildbot/test/unit/test_process_factory.py index 6d57a38d589..c60d7636da7 100644 --- a/master/buildbot/test/unit/test_process_factory.py +++ b/master/buildbot/test/unit/test_process_factory.py @@ -14,42 +14,33 @@ # Copyright Buildbot Team Members from twisted.trial import unittest -from mock import Mock -from buildbot.process.factory import BuildFactory, ArgumentsInTheWrongPlace, s -from buildbot.process.buildstep import BuildStep +from buildbot.process.factory import BuildFactory +from buildbot.process.buildstep import BuildStep, _BuildStepFactory class TestBuildFactory(unittest.TestCase): def test_init(self): step = BuildStep() factory = BuildFactory([step]) - self.assertEqual(factory.steps, [(BuildStep, {})]) - - def test_init_deprecated(self): - factory = BuildFactory([s(BuildStep)]) - self.assertEqual(factory.steps, [(BuildStep, {})]) + self.assertEqual(factory.steps, [_BuildStepFactory(BuildStep)]) def test_addStep(self): step = BuildStep() factory = BuildFactory() factory.addStep(step) - self.assertEqual(factory.steps, [(BuildStep, {})]) - - def test_addStep_deprecated(self): - factory = BuildFactory() - factory.addStep(BuildStep) - self.assertEqual(factory.steps, [(BuildStep, {})]) + self.assertEqual(factory.steps, [_BuildStepFactory(BuildStep)]) def test_addStep_notAStep(self): factory = BuildFactory() - self.assertRaises(ValueError, factory.addStep, Mock()) + # This fails because object isn't adaptable to IBuildStepFactory + self.assertRaises(TypeError, factory.addStep, object()) def test_addStep_ArgumentsInTheWrongPlace(self): factory = BuildFactory() - self.assertRaises(ArgumentsInTheWrongPlace, factory.addStep, BuildStep(), name="name") + self.assertRaises(TypeError, factory.addStep, BuildStep(), name="name") def test_addSteps(self): factory = BuildFactory() factory.addSteps([BuildStep(), BuildStep()]) - self.assertEqual(factory.steps, [(BuildStep, {}), (BuildStep, {})]) + self.assertEqual(factory.steps, [_BuildStepFactory(BuildStep), _BuildStepFactory(BuildStep)]) diff --git a/master/buildbot/test/unit/test_process_properties.py b/master/buildbot/test/unit/test_process_properties.py index d86a1395512..837ba451c71 100644 --- a/master/buildbot/test/unit/test_process_properties.py +++ b/master/buildbot/test/unit/test_process_properties.py @@ -24,6 +24,7 @@ from buildbot.interfaces import IRenderable, IProperties from buildbot.test.util.config import ConfigErrorsMixin from buildbot.test.util.properties import FakeRenderable +from buildbot.test.util import compat class FakeSource: def __init__(self): @@ -197,6 +198,58 @@ def testColonPlusEmpty(self): def testColonPlusUnset(self): return self.doTestSimpleWithProperties('%(prop_nosuch:+present)s', '') + + def testColonTernarySet(self): + return self.doTestSimpleWithProperties('%(prop_str:?:present:missing)s', 'present') + + def testColonTernaryNone(self): + return self.doTestSimpleWithProperties('%(prop_none:?:present:missing)s', 'present') + + def testColonTernaryZero(self): + return self.doTestSimpleWithProperties('%(prop_zero:?|present|missing)s', 'present') + + def testColonTernaryOne(self): + return self.doTestSimpleWithProperties('%(prop_one:?:present:missing)s', 'present') + + def testColonTernaryFalse(self): + return self.doTestSimpleWithProperties('%(prop_false:?|present|missing)s', 'present') + + def testColonTernaryTrue(self): + return self.doTestSimpleWithProperties('%(prop_true:?:present:missing)s', 'present') + + def testColonTernaryEmpty(self): + return self.doTestSimpleWithProperties('%(prop_empty:?ApresentAmissing)s', 'present') + + def testColonTernaryUnset(self): + return self.doTestSimpleWithProperties('%(prop_nosuch:?#present#missing)s', 'missing') + + + def testColonTernaryHashSet(self): + return self.doTestSimpleWithProperties('%(prop_str:#?:truish:falsish)s', 'truish') + + def testColonTernaryHashNone(self): + # None is special-cased *differently* for '#?' + return self.doTestSimpleWithProperties('%(prop_none:#?|truish|falsish)s', 'falsish') + + def testColonTernaryHashZero(self): + return self.doTestSimpleWithProperties('%(prop_zero:#?:truish:falsish)s', 'falsish') + + def testColonTernaryHashOne(self): + return self.doTestSimpleWithProperties('%(prop_one:#?:truish:falsish)s', 'truish') + + def testColonTernaryHashFalse(self): + return self.doTestSimpleWithProperties('%(prop_false:#?:truish:falsish)s', 'falsish') + + def testColonTernaryHashTrue(self): + return self.doTestSimpleWithProperties('%(prop_true:#?|truish|falsish)s', 'truish') + + def testColonTernaryHashEmpty(self): + return self.doTestSimpleWithProperties('%(prop_empty:#?:truish:falsish)s', 'falsish') + + def testColonTernaryHashUnset(self): + return self.doTestSimpleWithProperties('%(prop_nosuch:#?.truish.falsish)s', 'falsish') + + def testClearTempValues(self): d = self.doTestSimpleWithProperties('', '', prop_temp=lambda b: 'present') @@ -274,6 +327,23 @@ def testTempValuePlusUnsetSet(self): prop_nosuch=lambda b: 1) + def testTempValueColonTernaryTrue(self): + return self.doTestSimpleWithProperties('%(prop_temp:?:present:missing)s', 'present', + prop_temp=lambda b: True) + + def testTempValueColonTernaryFalse(self): + return self.doTestSimpleWithProperties('%(prop_temp:?|present|missing)s', 'present', + prop_temp=lambda b: False) + + def testTempValueColonTernaryHashTrue(self): + return self.doTestSimpleWithProperties('%(prop_temp:#?|truish|falsish)s', 'truish', + prop_temp=lambda b: 1) + + def testTempValueColonTernaryHashFalse(self): + return self.doTestSimpleWithProperties('%(prop_temp:#?|truish|falsish)s', 'falsish', + prop_nosuch=lambda b: 0) + + class TestInterpolateConfigure(unittest.TestCase, ConfigErrorsMixin): """ Test that Interpolate reports erros in the interpolation string @@ -313,6 +383,33 @@ def test_nested_invalid_selector(self): self.assertRaisesConfigError("invalid Interpolate selector 'garbage'", lambda: Interpolate("%(prop:some_prop:~%(garbage:test)s)s")) + def test_colon_ternary_missing_delimeter(self): + self.assertRaisesConfigError("invalid Interpolate ternary expression 'one' with delimiter ':'", + lambda: Interpolate("echo '%(prop:P:?:one)s'")) + + def test_colon_ternary_paren_delimiter(self): + self.assertRaisesConfigError("invalid Interpolate ternary expression 'one(:)' with delimiter ':'", + lambda: Interpolate("echo '%(prop:P:?:one(:))s'")) + + def test_colon_ternary_hash_bad_delimeter(self): + self.assertRaisesConfigError("invalid Interpolate ternary expression 'one' with delimiter '|'", + lambda: Interpolate("echo '%(prop:P:#?|one)s'")) + + def test_prop_invalid_character(self): + self.assertRaisesConfigError("Property name must be alphanumeric for prop Interpolation 'a+a'", + lambda: Interpolate("echo '%(prop:a+a)s'")) + + def test_kw_invalid_character(self): + self.assertRaisesConfigError("Keyword must be alphanumeric for kw Interpolation 'a+a'", + lambda: Interpolate("echo '%(kw:a+a)s'")) + + def test_src_codebase_invalid_character(self): + self.assertRaisesConfigError("Codebase must be alphanumeric for src Interpolation 'a+a:a'", + lambda: Interpolate("echo '%(src:a+a:a)s'")) + + def test_src_attr_invalid_character(self): + self.assertRaisesConfigError("Attribute must be alphanumeric for src Interpolation 'a:a+a'", + lambda: Interpolate("echo '%(src:a:a+a)s'")) class TestInterpolatePositional(unittest.TestCase): @@ -401,14 +498,6 @@ def test_property_colon_plus(self): "echo projectdefined") return d - def test_property_renderable(self): - self.props.setProperty("project", FakeRenderable('testing'), "test") - command = Interpolate("echo '%(prop:project)s'") - d = self.build.render(command) - d.addCallback(self.failUnlessEqual, - "echo 'testing'") - return d - def test_nested_property(self): self.props.setProperty("project", "so long!", "test") command = Interpolate("echo '%(prop:missing:~%(prop:project)s)s'") @@ -417,6 +506,84 @@ def test_nested_property(self): "echo 'so long!'") return d + def test_property_substitute_recursively(self): + self.props.setProperty("project", "proj1", "test") + command = Interpolate("echo '%(prop:no_such:-%(prop:project)s)s'") + d = self.build.render(command) + d.addCallback(self.failUnlessEqual, + "echo 'proj1'") + return d + + def test_property_colon_ternary_present(self): + self.props.setProperty("project", "proj1", "test") + command = Interpolate("echo %(prop:project:?:defined:missing)s") + d = self.build.render(command) + d.addCallback(self.failUnlessEqual, + "echo defined") + return d + + def test_property_colon_ternary_missing(self): + command = Interpolate("echo %(prop:project:?|defined|missing)s") + d = self.build.render(command) + d.addCallback(self.failUnlessEqual, + "echo missing") + return d + + def test_property_colon_ternary_hash_true(self): + self.props.setProperty("project", "winbld", "test") + command = Interpolate("echo buildby-%(prop:project:#?:T:F)s") + d = self.build.render(command) + d.addCallback(self.failUnlessEqual, + "echo buildby-T") + return d + + def test_property_colon_ternary_hash_false(self): + self.props.setProperty("project", "", "test") + command = Interpolate("echo buildby-%(prop:project:#?|T|F)s") + d = self.build.render(command) + d.addCallback(self.failUnlessEqual, + "echo buildby-F") + return d + + def test_property_colon_ternary_substitute_recursively_true(self): + self.props.setProperty("P", "present", "test") + self.props.setProperty("one", "proj1", "test") + self.props.setProperty("two", "proj2", "test") + command = Interpolate("echo '%(prop:P:?|%(prop:one)s|%(prop:two)s)s'") + d = self.build.render(command) + d.addCallback(self.failUnlessEqual, + "echo 'proj1'") + return d + + def test_property_colon_ternary_substitute_recursively_false(self): + self.props.setProperty("one", "proj1", "test") + self.props.setProperty("two", "proj2", "test") + command = Interpolate("echo '%(prop:P:?|%(prop:one)s|%(prop:two)s)s'") + d = self.build.render(command) + d.addCallback(self.failUnlessEqual, + "echo 'proj2'") + return d + + def test_property_colon_ternary_substitute_recursively_delimited_true(self): + self.props.setProperty("P", "present", "test") + self.props.setProperty("one", "proj1", "test") + self.props.setProperty("two", "proj2", "test") + command = Interpolate("echo '%(prop:P:?|%(prop:one:?|true|false)s|%(prop:two:?|false|true)s)s'") + d = self.build.render(command) + d.addCallback(self.failUnlessEqual, + "echo 'true'") + return d + + def test_property_colon_ternary_substitute_recursively_delimited_false(self): + self.props.setProperty("one", "proj1", "test") + self.props.setProperty("two", "proj2", "test") + command = Interpolate("echo '%(prop:P:?|%(prop:one:?|true|false)s|%(prop:two:?|false|true)s)s'") + d = self.build.render(command) + d.addCallback(self.failUnlessEqual, + "echo 'false'") + return d + + class TestInterpolateSrc(unittest.TestCase): def setUp(self): self.props = Properties() @@ -716,6 +883,15 @@ def testDictColonPlus(self): "build-exists-.tar.gz") return d + def testDictColonTernary(self): + # test dict-style substitution with WithProperties + self.props.setProperty("prop1", "foo", "test") + command = WithProperties("build-%(prop1:?:exists:missing)s-%(prop2:?:exists:missing)s.tar.gz") + d = self.build.render(command) + d.addCallback(self.failUnlessEqual, + "build-exists-missing.tar.gz") + return d + def testEmpty(self): # None should render as '' self.props.setProperty("empty", None, "test") @@ -873,6 +1049,12 @@ def testUpdateFromPropertiesNoRuntime(self): self.failUnlessEqual(self.props.getProperty('x'), 24) self.failUnlessEqual(self.props.getPropertySource('x'), 'old') + @compat.usesFlushWarnings + def test_setProperty_notJsonable(self): + self.props.setProperty("project", FakeRenderable('testing'), "test") + self.props.setProperty("project", object, "test") + self.assertEqual(len(self.flushWarnings([self.test_setProperty_notJsonable])), 2) + # IProperties methods def test_getProperty(self): diff --git a/master/buildbot/test/unit/test_schedulers_basic.py b/master/buildbot/test/unit/test_schedulers_basic.py index a994650a2af..a1c56114c5e 100644 --- a/master/buildbot/test/unit/test_schedulers_basic.py +++ b/master/buildbot/test/unit/test_schedulers_basic.py @@ -129,6 +129,9 @@ def check(_): onlyImportant=False) self.db.schedulers.assertClassifications(self.OBJECTID, { 20 : True }) self.assertTrue(sched.timer_started) + self.assertEqual(sched.getPendingBuildTimes(), [ 10 ]) + self.clock.advance(10) + self.assertEqual(sched.getPendingBuildTimes(), []) d.addCallback(check) d.addCallback(lambda _ : sched.stopService()) return d @@ -207,6 +210,7 @@ def test_gotChange_treeStableTimer_sequence(self): self.makeFakeChange(branch='master', number=1, when=2220), True) self.assertEqual(self.events, []) + self.assertEqual(sched.getPendingBuildTimes(), [2229]) self.db.schedulers.assertClassifications(self.OBJECTID, { 1 : True }) # but another (unimportant) change arrives before then @@ -217,6 +221,7 @@ def test_gotChange_treeStableTimer_sequence(self): self.makeFakeChange(branch='master', number=2, when=2226), False) self.assertEqual(self.events, []) + self.assertEqual(sched.getPendingBuildTimes(), [2235]) self.db.schedulers.assertClassifications(self.OBJECTID, { 1 : True, 2 : False }) self.clock.advance(3) # to 2229 @@ -230,6 +235,7 @@ def test_gotChange_treeStableTimer_sequence(self): self.makeFakeChange(branch='master', number=3, when=2232), True) self.assertEqual(self.events, []) + self.assertEqual(sched.getPendingBuildTimes(), [2241]) self.db.schedulers.assertClassifications(self.OBJECTID, { 1 : True, 2 : False, 3 : True }) self.clock.advance(3) # to 2235 @@ -238,6 +244,7 @@ def test_gotChange_treeStableTimer_sequence(self): # finally, time to start the build! self.clock.advance(6) # to 2241 self.assertEqual(self.events, [ 'B[1,2,3]@2241' ]) + self.assertEqual(sched.getPendingBuildTimes(), []) self.db.schedulers.assertClassifications(self.OBJECTID, { }) yield sched.stopService() @@ -314,16 +321,22 @@ def mkch(**kwargs): d = defer.succeed(None) d.addCallback(lambda _ : sched.gotChange(mkch(branch='master', number=13), True)) + d.addCallback(lambda _ : + self.assertEqual(sched.getPendingBuildTimes(), [10])) d.addCallback(lambda _ : self.clock.advance(1)) # time is now 1 d.addCallback(lambda _ : sched.gotChange(mkch(branch='master', number=14), False)) + d.addCallback(lambda _ : + self.assertEqual(sched.getPendingBuildTimes(), [11])) d.addCallback(lambda _ : sched.gotChange(mkch(branch='boring', number=15), False)) d.addCallback(lambda _ : self.clock.pump([1]*4)) # time is now 5 d.addCallback(lambda _ : sched.gotChange(mkch(branch='devel', number=16), True)) + d.addCallback(lambda _ : + self.assertEqual(sched.getPendingBuildTimes(), [11,15])) d.addCallback(lambda _ : self.clock.pump([1]*10)) # time is now 15 def check(_): diff --git a/master/buildbot/test/unit/test_scripts_base.py b/master/buildbot/test/unit/test_scripts_base.py index 2d88c8fea85..9ec5b3d3762 100644 --- a/master/buildbot/test/unit/test_scripts_base.py +++ b/master/buildbot/test/unit/test_scripts_base.py @@ -21,7 +21,7 @@ from twisted.trial import unittest from buildbot.scripts import base from buildbot.test.util import dirs, misc -from twisted.python import usage +from twisted.python import usage, runtime class TestIBD(dirs.DirsMixin, misc.StdoutAssertionsMixin, unittest.TestCase): @@ -115,9 +115,14 @@ def do_loadOptionsFile(self, _here, exp): # avoid breaking other parts of the test system patches = [] - def expanduser(p): - return p.replace('~', self.home + '/') - patches.append(self.patch(os.path, 'expanduser', expanduser)) + if runtime.platformType == 'win32': + from win32com.shell import shell + patches.append(self.patch(shell, 'SHGetFolderPath', + lambda *args : self.home)) + else: + def expanduser(p): + return p.replace('~/', self.home + '/') + patches.append(self.patch(os.path, 'expanduser', expanduser)) old_dirname = os.path.dirname def dirname(p): @@ -133,9 +138,9 @@ def dirname(p): for p in patches: p.restore() - def writeOptionsFile(self, dir, content): - os.makedirs(os.path.join(dir, '.buildbot')) - with open(os.path.join(dir, '.buildbot', 'options'), 'w') as f: + def writeOptionsFile(self, dir, content, bbdir='.buildbot'): + os.makedirs(os.path.join(dir, bbdir)) + with open(os.path.join(dir, bbdir, 'options'), 'w') as f: f.write(content) def test_loadOptionsFile_subdirs_not_found(self): @@ -160,7 +165,10 @@ def test_loadOptionsFile_subdirs_at_tip(self): def test_loadOptionsFile_subdirs_at_homedir(self): subdir = os.path.join(self.dir, 'a', 'b') os.makedirs(subdir) - self.writeOptionsFile(self.home, 'abc=123') + # on windows, the subdir of the home (well, appdata) dir + # is 'buildbot', not '.buildbot' + self.writeOptionsFile(self.home, 'abc=123', + 'buildbot' if runtime.platformType == 'win32' else '.buildbot') self.do_loadOptionsFile(_here=subdir, exp={'abc':123}) def test_loadOptionsFile_syntax_error(self): diff --git a/master/buildbot/test/unit/test_scripts_runner.py b/master/buildbot/test/unit/test_scripts_runner.py index 24ab9f32f5f..1f5ffb86942 100644 --- a/master/buildbot/test/unit/test_scripts_runner.py +++ b/master/buildbot/test/unit/test_scripts_runner.py @@ -21,7 +21,7 @@ import mock import cStringIO from twisted.trial import unittest -from twisted.python import usage +from twisted.python import usage, runtime from buildbot.scripts import base, runner from buildbot.test.util import misc @@ -187,8 +187,9 @@ def test_db_long(self): self.assertOptions(opts, exp) def test_db_basedir(self): - opts = self.parse('-f', '/foo/bar') - exp = self.defaults_and(force=True, basedir='/foo/bar') + path = r'c:\foo\bar' if runtime.platformType == "win32" else '/foo/bar' + opts = self.parse('-f', path) + exp = self.defaults_and(force=True, basedir=path) self.assertOptions(opts, exp) diff --git a/master/buildbot/test/unit/test_status_mail.py b/master/buildbot/test/unit/test_status_mail.py index 724f95225a3..2263a2c6ee6 100644 --- a/master/buildbot/test/unit/test_status_mail.py +++ b/master/buildbot/test/unit/test_status_mail.py @@ -17,7 +17,7 @@ from mock import Mock from buildbot import config from twisted.trial import unittest -from buildbot.status.results import SUCCESS, FAILURE +from buildbot.status.results import SUCCESS, FAILURE, WARNINGS, EXCEPTION from buildbot.status.mail import MailNotifier from twisted.internet import defer from buildbot.test.fake import fakedb @@ -440,15 +440,70 @@ def test_buildFinished_ignores_unspecified_categories(self): self.assertEqual(None, mn.buildFinished('dummyBuilder', build, SUCCESS)) - def test_buildFinished_mode_all_always_sends_email(self): + def run_simple_test_sends_email_for_mode(self, mode, result): mock_method = Mock() self.patch(MailNotifier, "buildMessage", mock_method) - mn = MailNotifier('from@example.org', mode=("failing", "passing", "warnings")) + mn = MailNotifier('from@example.org', mode=mode) build = FakeBuildStatus(name="build") - mn.buildFinished('dummyBuilder', build, FAILURE) + mn.buildFinished('dummyBuilder', build, result) - mock_method.assert_called_with('dummyBuilder', [build], FAILURE) + mock_method.assert_called_with('dummyBuilder', [build], result) + + def run_simple_test_ignores_email_for_mode(self, mode, result): + mock_method = Mock() + self.patch(MailNotifier, "buildMessage", mock_method) + mn = MailNotifier('from@example.org', mode=mode) + + build = FakeBuildStatus(name="build") + mn.buildFinished('dummyBuilder', build, result) + + self.assertFalse(mock_method.called) + + def test_buildFinished_mode_all_for_success(self): + self.run_simple_test_sends_email_for_mode("all", SUCCESS) + def test_buildFinished_mode_all_for_failure(self): + self.run_simple_test_sends_email_for_mode("all", FAILURE) + def test_buildFinished_mode_all_for_warnings(self): + self.run_simple_test_sends_email_for_mode("all", WARNINGS) + def test_buildFinished_mode_all_for_exception(self): + self.run_simple_test_sends_email_for_mode("all", EXCEPTION) + + def test_buildFinished_mode_failing_for_success(self): + self.run_simple_test_ignores_email_for_mode("failing", SUCCESS) + def test_buildFinished_mode_failing_for_failure(self): + self.run_simple_test_sends_email_for_mode("failing", FAILURE) + def test_buildFinished_mode_failing_for_warnings(self): + self.run_simple_test_ignores_email_for_mode("failing", WARNINGS) + def test_buildFinished_mode_failing_for_exception(self): + self.run_simple_test_ignores_email_for_mode("failing", EXCEPTION) + + def test_buildFinished_mode_exception_for_success(self): + self.run_simple_test_ignores_email_for_mode("exception", SUCCESS) + def test_buildFinished_mode_exception_for_failure(self): + self.run_simple_test_ignores_email_for_mode("exception", FAILURE) + def test_buildFinished_mode_exception_for_warnings(self): + self.run_simple_test_ignores_email_for_mode("exception", WARNINGS) + def test_buildFinished_mode_exception_for_exception(self): + self.run_simple_test_sends_email_for_mode("exception", EXCEPTION) + + def test_buildFinished_mode_warnings_for_success(self): + self.run_simple_test_ignores_email_for_mode("warnings", SUCCESS) + def test_buildFinished_mode_warnings_for_failure(self): + self.run_simple_test_sends_email_for_mode("warnings", FAILURE) + def test_buildFinished_mode_warnings_for_warnings(self): + self.run_simple_test_sends_email_for_mode("warnings", WARNINGS) + def test_buildFinished_mode_warnings_for_exception(self): + self.run_simple_test_ignores_email_for_mode("warnings", EXCEPTION) + + def test_buildFinished_mode_passing_for_success(self): + self.run_simple_test_sends_email_for_mode("passing", SUCCESS) + def test_buildFinished_mode_passing_for_failure(self): + self.run_simple_test_ignores_email_for_mode("passing", FAILURE) + def test_buildFinished_mode_passing_for_warnings(self): + self.run_simple_test_ignores_email_for_mode("passing", WARNINGS) + def test_buildFinished_mode_passing_for_exception(self): + self.run_simple_test_ignores_email_for_mode("passing", EXCEPTION) def test_buildFinished_mode_failing_ignores_successful_build(self): mn = MailNotifier('from@example.org', mode=("failing",)) diff --git a/master/buildbot/test/unit/test_status_web_base.py b/master/buildbot/test/unit/test_status_web_base.py index c61b4261994..aa428e60245 100644 --- a/master/buildbot/test/unit/test_status_web_base.py +++ b/master/buildbot/test/unit/test_status_web_base.py @@ -13,6 +13,7 @@ # # Copyright Buildbot Team Members +import mock from buildbot.status.web import base from twisted.internet import defer from twisted.trial import unittest @@ -58,3 +59,24 @@ def check(f): d.addErrback(check) return d +class Functions(unittest.TestCase): + + def do_test_getRequestCharset(self, hdr, exp): + req = mock.Mock() + req.getHeader.return_value = hdr + + self.assertEqual(base.getRequestCharset(req), exp) + + def test_getRequestCharset_empty(self): + return self.do_test_getRequestCharset(None, 'utf-8') + + def test_getRequestCharset_specified(self): + return self.do_test_getRequestCharset( + 'application/x-www-form-urlencoded ; charset=ISO-8859-1', + 'ISO-8859-1') + + def test_getRequestCharset_other_params(self): + return self.do_test_getRequestCharset( + 'application/x-www-form-urlencoded ; charset=UTF-16 ; foo=bar', + 'UTF-16') + diff --git a/master/buildbot/test/unit/test_status_web_change_hooks_poller.py b/master/buildbot/test/unit/test_status_web_change_hooks_poller.py new file mode 100644 index 00000000000..fca1b66e0b3 --- /dev/null +++ b/master/buildbot/test/unit/test_status_web_change_hooks_poller.py @@ -0,0 +1,107 @@ +# This file is part of Buildbot. Buildbot is free software: you can +# redistribute it and/or modify it under the terms of the GNU General Public +# License as published by the Free Software Foundation, version 2. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +# details. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, write to the Free Software Foundation, Inc., 51 +# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Copyright Buildbot Team Members + +from twisted.trial import unittest +from twisted.internet import defer +from buildbot.changes import base +import buildbot.status.web.change_hook as change_hook +from buildbot.test.fake.web import FakeRequest +from buildbot.changes.manager import ChangeManager + + +class TestPollingChangeHook(unittest.TestCase): + class Subclass(base.PollingChangeSource): + pollInterval = None + called = False + + def poll(self): + self.called = True + + def setUpRequest(self, args, options=True): + self.changeHook = change_hook.ChangeHookResource(dialects={'poller' : options}) + + self.request = FakeRequest(args=args) + self.request.uri = "/change_hook/poller" + self.request.method = "GET" + + master = self.request.site.buildbot_service.master + master.change_svc = ChangeManager(master) + + self.changesrc = self.Subclass("example", None) + self.changesrc.setServiceParent(master.change_svc) + + self.disabledChangesrc = self.Subclass("disabled", None) + self.disabledChangesrc.setServiceParent(master.change_svc) + + anotherchangesrc = base.ChangeSource() + anotherchangesrc.setName("notapoller") + anotherchangesrc.setServiceParent(master.change_svc) + + return self.request.test_render(self.changeHook) + + @defer.inlineCallbacks + def test_no_args(self): + yield self.setUpRequest({}) + self.assertEqual(self.request.written, "no changes found") + self.assertEqual(self.changesrc.called, True) + self.assertEqual(self.disabledChangesrc.called, True) + + @defer.inlineCallbacks + def test_no_poller(self): + yield self.setUpRequest({"poller": ["nosuchpoller"]}) + expected = "Could not find pollers: nosuchpoller" + self.assertEqual(self.request.written, expected) + self.request.setResponseCode.assert_called_with(400, expected) + self.assertEqual(self.changesrc.called, False) + self.assertEqual(self.disabledChangesrc.called, False) + + @defer.inlineCallbacks + def test_invalid_poller(self): + yield self.setUpRequest({"poller": ["notapoller"]}) + expected = "Could not find pollers: notapoller" + self.assertEqual(self.request.written, expected) + self.request.setResponseCode.assert_called_with(400, expected) + self.assertEqual(self.changesrc.called, False) + self.assertEqual(self.disabledChangesrc.called, False) + + @defer.inlineCallbacks + def test_trigger_poll(self): + yield self.setUpRequest({"poller": ["example"]}) + self.assertEqual(self.request.written, "no changes found") + self.assertEqual(self.changesrc.called, True) + self.assertEqual(self.disabledChangesrc.called, False) + + @defer.inlineCallbacks + def test_allowlist_deny(self): + yield self.setUpRequest({"poller": ["disabled"]}, options={"allowed": ["example"]}) + expected = "Could not find pollers: disabled" + self.assertEqual(self.request.written, expected) + self.request.setResponseCode.assert_called_with(400, expected) + self.assertEqual(self.changesrc.called, False) + self.assertEqual(self.disabledChangesrc.called, False) + + @defer.inlineCallbacks + def test_allowlist_allow(self): + yield self.setUpRequest({"poller": ["example"]}, options={"allowed": ["example"]}) + self.assertEqual(self.request.written, "no changes found") + self.assertEqual(self.changesrc.called, True) + self.assertEqual(self.disabledChangesrc.called, False) + + @defer.inlineCallbacks + def test_allowlist_all(self): + yield self.setUpRequest({}, options={"allowed": ["example"]}) + self.assertEqual(self.request.written, "no changes found") + self.assertEqual(self.changesrc.called, True) + self.assertEqual(self.disabledChangesrc.called, False) diff --git a/master/buildbot/test/unit/test_steps_master.py b/master/buildbot/test/unit/test_steps_master.py index 440d53cc02d..20903ca278c 100644 --- a/master/buildbot/test/unit/test_steps_master.py +++ b/master/buildbot/test/unit/test_steps_master.py @@ -15,12 +15,11 @@ import os import sys -import mock -from twisted.python import runtime -from twisted.internet import reactor +from twisted.python import failure, runtime +from twisted.internet import error, reactor from twisted.trial import unittest from buildbot.test.util import steps -from buildbot.status.results import SUCCESS, FAILURE +from buildbot.status.results import SUCCESS, FAILURE, EXCEPTION from buildbot.steps import master from buildbot.process.properties import WithProperties @@ -51,9 +50,11 @@ def spawnProcess(pp, cmd, argv, path, usePTY, env): elif output[0] == 'err': pp.errReceived(output[1]) elif output[0] == 'rc': - so = mock.Mock(name='status_object') - so.value.exitCode = output[1] - pp.processEnded(so) + if output[1] != 0: + so = error.ProcessTerminated(exitCode=output[1]) + else: + so = error.ProcessDone(None) + pp.processEnded(failure.Failure(so)) self.patch(reactor, 'spawnProcess', spawnProcess) def test_real_cmd(self): @@ -67,6 +68,22 @@ def test_real_cmd(self): self.expectOutcome(result=SUCCESS, status_text=["Ran"]) return self.runStep() + def test_real_cmd_interrupted(self): + cmd = [ sys.executable, '-c', 'while True: pass' ] + self.setupStep( + master.MasterShellCommand(command=cmd)) + self.expectLogfile('stdio', "") + if runtime.platformType == 'win32': + # windows doesn't have signals, so we don't get 'killed' + self.expectOutcome(result=EXCEPTION, + status_text=["failed (1)", "interrupted"]) + else: + self.expectOutcome(result=EXCEPTION, + status_text=["killed (9)", "interrupted"]) + d = self.runStep() + self.step.interrupt("KILL") + return d + def test_real_cmd_fails(self): cmd = [ sys.executable, '-c', 'import sys; sys.exit(1)' ] self.setupStep( @@ -80,6 +97,9 @@ def test_constr_args(self): master.MasterShellCommand(description='x', descriptionDone='y', env={'a':'b'}, path=['/usr/bin'], usePTY=True, command='true')) + + self.assertEqual(self.step.describe(), ['x']) + if runtime.platformType == 'win32': exp_argv = [ r'C:\WINDOWS\system32\cmd.exe', '/c', 'true' ] else: @@ -127,3 +147,29 @@ def test_prop_rendering(self): self.expectLogfile('stdio', "BUILDBOT-TEST\nBUILDBOT-TEST\n") self.expectOutcome(result=SUCCESS, status_text=["Ran"]) return self.runStep() + + def test_constr_args_descriptionSuffix(self): + self.setupStep( + master.MasterShellCommand(description='x', descriptionDone='y', + descriptionSuffix='z', + env={'a':'b'}, path=['/usr/bin'], usePTY=True, + command='true')) + + # call twice to make sure the suffix doesnt get double added + self.assertEqual(self.step.describe(), ['x', 'z']) + self.assertEqual(self.step.describe(), ['x', 'z']) + + if runtime.platformType == 'win32': + exp_argv = [ r'C:\WINDOWS\system32\cmd.exe', '/c', 'true' ] + else: + exp_argv = [ '/bin/sh', '-c', 'true' ] + self.patchSpawnProcess( + exp_cmd=exp_argv[0], exp_argv=exp_argv, + exp_path=['/usr/bin'], exp_usePTY=True, exp_env={'a':'b'}, + outputs=[ + ('out', 'hello!\n'), + ('err', 'world\n'), + ('rc', 0), + ]) + self.expectOutcome(result=SUCCESS, status_text=['y', 'z']) + return self.runStep() diff --git a/master/buildbot/test/unit/test_steps_shell.py b/master/buildbot/test/unit/test_steps_shell.py index 06e2936eafb..33273f66317 100644 --- a/master/buildbot/test/unit/test_steps_shell.py +++ b/master/buildbot/test/unit/test_steps_shell.py @@ -129,6 +129,23 @@ def test_describe_custom(self): self.assertEqual((step.describe(), step.describe(done=True)), (['echoing'], ['echoed'])) + def test_describe_with_suffix(self): + step = shell.ShellCommand(command="echo hello", descriptionSuffix="suffix") + self.assertEqual((step.describe(), step.describe(done=True)), + (["'echo", "hello'", 'suffix'],)*2) + + def test_describe_custom_with_suffix(self): + step = shell.ShellCommand(command="echo hello", + description=["echoing"], descriptionDone=["echoed"], + descriptionSuffix="suffix") + self.assertEqual((step.describe(), step.describe(done=True)), + (['echoing', 'suffix'], ['echoed', 'suffix'])) + + def test_describe_no_command_with_suffix(self): + step = shell.ShellCommand(workdir='build', descriptionSuffix="suffix") + self.assertEqual((step.describe(), step.describe(done=True)), + (['???', 'suffix'],)*2) + def test_describe_unrendered_WithProperties(self): step = shell.ShellCommand(command=properties.WithProperties('')) self.assertEqual((step.describe(), step.describe(done=True)), diff --git a/master/buildbot/test/unit/test_steps_source_base_Source.py b/master/buildbot/test/unit/test_steps_source_base_Source.py index d41c6a95b0a..3260e479369 100644 --- a/master/buildbot/test/unit/test_steps_source_base_Source.py +++ b/master/buildbot/test/unit/test_steps_source_base_Source.py @@ -73,9 +73,14 @@ def test_start_no_codebase(self): step.build.getSourceStamp = mock.Mock() step.build.getSourceStamp.return_value = None + self.assertEqual(step.describe(), ['updating']) + self.assertEqual(step.name, Source.name) + step.startStep(mock.Mock()) self.assertEqual(step.build.getSourceStamp.call_args[0], ('',)) + self.assertEqual(step.description, ['updating']) + def test_start_with_codebase(self): step = self.setupStep(Source(codebase='codebase')) step.branch = 'branch' @@ -83,9 +88,31 @@ def test_start_with_codebase(self): step.build.getSourceStamp = mock.Mock() step.build.getSourceStamp.return_value = None + self.assertEqual(step.describe(), ['updating', 'codebase']) + self.assertEqual(step.name, Source.name + " codebase") + step.startStep(mock.Mock()) self.assertEqual(step.build.getSourceStamp.call_args[0], ('codebase',)) + self.assertEqual(step.describe(True), ['update', 'codebase']) + + def test_start_with_codebase_and_descriptionSuffix(self): + step = self.setupStep(Source(codebase='my-code', + descriptionSuffix='suffix')) + step.branch = 'branch' + step.startVC = mock.Mock() + step.build.getSourceStamp = mock.Mock() + step.build.getSourceStamp.return_value = None + + self.assertEqual(step.describe(), ['updating', 'suffix']) + self.assertEqual(step.name, Source.name + " my-code") + + step.startStep(mock.Mock()) + self.assertEqual(step.build.getSourceStamp.call_args[0], ('my-code',)) + + self.assertEqual(step.describe(True), ['update', 'suffix']) + + class TestSourceDescription(steps.BuildStepMixin, unittest.TestCase): def setUp(self): @@ -107,3 +134,4 @@ def test_constructor_args_lists(self): descriptionDone=['svn', 'update']) self.assertEqual(step.description, ['svn', 'update', '(running)']) self.assertEqual(step.descriptionDone, ['svn', 'update']) + diff --git a/master/buildbot/test/unit/test_steps_source_bzr.py b/master/buildbot/test/unit/test_steps_source_bzr.py index 3d1a8c217b2..6356a2a651a 100644 --- a/master/buildbot/test/unit/test_steps_source_bzr.py +++ b/master/buildbot/test/unit/test_steps_source_bzr.py @@ -53,6 +53,36 @@ def test_mode_full(self): self.expectOutcome(result=SUCCESS, status_text=["update"]) return self.runStep() + def test_mode_full_timeout(self): + self.setupStep( + bzr.Bzr(repourl='http://bzr.squid-cache.org/bzr/squid3/trunk', + mode='full', method='fresh', timeout=1)) + self.expectCommands( + ExpectShell(workdir='wkdir', + timeout=1, + command=['bzr', '--version']) + + 0, + Expect('stat', dict(file='wkdir/.bzr', + logEnviron=True)) + + 0, + ExpectShell(workdir='wkdir', + timeout=1, + command=['bzr', 'clean-tree', '--force']) + + 0, + ExpectShell(workdir='wkdir', + timeout=1, + command=['bzr', 'update']) + + 0, + ExpectShell(workdir='wkdir', + timeout=1, + command=['bzr', 'version-info', '--custom', "--template='{revno}"]) + + ExpectShell.log('stdio', + stdout='100') + + 0, + ) + self.expectOutcome(result=SUCCESS, status_text=["update"]) + return self.runStep() + def test_mode_full_revision(self): self.setupStep( bzr.Bzr(repourl='http://bzr.squid-cache.org/bzr/squid3/trunk', diff --git a/master/buildbot/test/unit/test_steps_source_cvs.py b/master/buildbot/test/unit/test_steps_source_cvs.py index 1eefc396c54..42f0aa0008f 100644 --- a/master/buildbot/test/unit/test_steps_source_cvs.py +++ b/master/buildbot/test/unit/test_steps_source_cvs.py @@ -14,10 +14,18 @@ # Copyright Buildbot Team Members from twisted.trial import unittest +from buildbot.steps import shell from buildbot.steps.source import cvs from buildbot.status.results import SUCCESS, FAILURE from buildbot.test.util import sourcesteps -from buildbot.test.fake.remotecommand import ExpectShell, Expect +from buildbot.test.fake.remotecommand import ExpectShell, Expect, ExpectRemoteRef + +def uploadString(cvsroot): + def behavior(command): + writer = command.args['writer'] + writer.remote_write(cvsroot + "\n") + writer.remote_close() + return behavior class TestCVS(sourcesteps.SourceStepMixin, unittest.TestCase): @@ -36,13 +44,53 @@ def test_mode_full_clean(self): ExpectShell(workdir='wkdir', command=['cvs', '--version']) + 0, - Expect('stat', dict(file='wkdir/CVS', - logEnviron=True)) + Expect('uploadFile', dict(blocksize=32768, maxsize=None, + slavesrc='Root', workdir='wkdir/CVS', + writer=ExpectRemoteRef(shell.StringFileWriter))) + + Expect.behavior(uploadString(':pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot')) + + 0, + Expect('uploadFile', dict(blocksize=32768, maxsize=None, + slavesrc='Repository', workdir='wkdir/CVS', + writer=ExpectRemoteRef(shell.StringFileWriter))) + + Expect.behavior(uploadString('mozilla/browser/')) + + 0, + ExpectShell(workdir='wkdir', + command=['cvsdiscard']) + + 0, + ExpectShell(workdir='wkdir', + command=['cvs', '-z3', 'update', '-dP']) + + 0, + ) + + self.expectOutcome(result=SUCCESS, status_text=["update"]) + return self.runStep() + + def test_mode_full_clean_timeout(self): + self.setupStep( + cvs.CVS(cvsroot=":pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot", + cvsmodule="mozilla/browser/", mode='full', method='clean', + login=True, timeout=1)) + self.expectCommands( + ExpectShell(workdir='wkdir', + timeout=1, + command=['cvs', '--version']) + + 0, + Expect('uploadFile', dict(blocksize=32768, maxsize=None, + slavesrc='Root', workdir='wkdir/CVS', + writer=ExpectRemoteRef(shell.StringFileWriter))) + + Expect.behavior(uploadString(':pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot')) + + 0, + Expect('uploadFile', dict(blocksize=32768, maxsize=None, + slavesrc='Repository', workdir='wkdir/CVS', + writer=ExpectRemoteRef(shell.StringFileWriter))) + + Expect.behavior(uploadString('mozilla/browser/')) + 0, ExpectShell(workdir='wkdir', + timeout=1, command=['cvsdiscard']) + 0, ExpectShell(workdir='wkdir', + timeout=1, command=['cvs', '-z3', 'update', '-dP']) + 0, ) @@ -50,6 +98,66 @@ def test_mode_full_clean(self): self.expectOutcome(result=SUCCESS, status_text=["update"]) return self.runStep() + def test_mode_full_clean_branch(self): + self.setupStep( + cvs.CVS(cvsroot=":pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot", + cvsmodule="mozilla/browser/", mode='full', method='clean', + branch='branch', login=True)) + self.expectCommands( + ExpectShell(workdir='wkdir', + command=['cvs', '--version']) + + 0, + Expect('uploadFile', dict(blocksize=32768, maxsize=None, + slavesrc='Root', workdir='wkdir/CVS', + writer=ExpectRemoteRef(shell.StringFileWriter))) + + Expect.behavior(uploadString(':pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot')) + + 0, + Expect('uploadFile', dict(blocksize=32768, maxsize=None, + slavesrc='Repository', workdir='wkdir/CVS', + writer=ExpectRemoteRef(shell.StringFileWriter))) + + Expect.behavior(uploadString('mozilla/browser/')) + + 0, + ExpectShell(workdir='wkdir', + command=['cvsdiscard']) + + 0, + ExpectShell(workdir='wkdir', + command=['cvs', '-z3', 'update', '-dP', '-r', 'branch']) + + 0, + ) + + self.expectOutcome(result=SUCCESS, status_text=["update"]) + return self.runStep() + + def test_mode_full_clean_branch_sourcestamp(self): + self.setupStep( + cvs.CVS(cvsroot=":pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot", + cvsmodule="mozilla/browser/", mode='full', method='clean', + login=True), args={'branch':'my_branch'}) + self.expectCommands( + ExpectShell(workdir='wkdir', + command=['cvs', '--version']) + + 0, + Expect('uploadFile', dict(blocksize=32768, maxsize=None, + slavesrc='Root', workdir='wkdir/CVS', + writer=ExpectRemoteRef(shell.StringFileWriter))) + + Expect.behavior(uploadString(':pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot')) + + 0, + Expect('uploadFile', dict(blocksize=32768, maxsize=None, + slavesrc='Repository', workdir='wkdir/CVS', + writer=ExpectRemoteRef(shell.StringFileWriter))) + + Expect.behavior(uploadString('mozilla/browser/')) + + 0, + ExpectShell(workdir='wkdir', + command=['cvsdiscard']) + + 0, + ExpectShell(workdir='wkdir', + command=['cvs', '-z3', 'update', '-dP', '-r', 'my_branch']) + + 0, + ) + + self.expectOutcome(result=SUCCESS, status_text=["update"]) + return self.runStep() + def test_mode_full_fresh(self): self.setupStep( cvs.CVS(cvsroot=":pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot", @@ -59,8 +167,15 @@ def test_mode_full_fresh(self): ExpectShell(workdir='wkdir', command=['cvs', '--version']) + 0, - Expect('stat', dict(file='wkdir/CVS', - logEnviron=True)) + Expect('uploadFile', dict(blocksize=32768, maxsize=None, + slavesrc='Root', workdir='wkdir/CVS', + writer=ExpectRemoteRef(shell.StringFileWriter))) + + Expect.behavior(uploadString(':pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot')) + + 0, + Expect('uploadFile', dict(blocksize=32768, maxsize=None, + slavesrc='Repository', workdir='wkdir/CVS', + writer=ExpectRemoteRef(shell.StringFileWriter))) + + Expect.behavior(uploadString('mozilla/browser/')) + 0, ExpectShell(workdir='wkdir', command=['cvsdiscard', '--ignore']) @@ -108,9 +223,16 @@ def test_mode_full_copy(self): Expect('rmdir', dict(dir='wkdir', logEnviron=True)) + 0, - Expect('stat', dict(file='source/CVS', - logEnviron=True)) - + 0, + Expect('uploadFile', dict(blocksize=32768, maxsize=None, + slavesrc='Root', workdir='source/CVS', + writer=ExpectRemoteRef(shell.StringFileWriter))) + + Expect.behavior(uploadString(':pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot')) + + 0, + Expect('uploadFile', dict(blocksize=32768, maxsize=None, + slavesrc='Repository', workdir='source/CVS', + writer=ExpectRemoteRef(shell.StringFileWriter))) + + Expect.behavior(uploadString('mozilla/browser/')) + + 0, ExpectShell(workdir='source', command=['cvs', '-z3', 'update', '-dP']) + 0, @@ -122,6 +244,41 @@ def test_mode_full_copy(self): self.expectOutcome(result=SUCCESS, status_text=["update"]) return self.runStep() + + def test_mode_full_copy_wrong_repo(self): + self.setupStep( + cvs.CVS(cvsroot=":pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot", + cvsmodule="mozilla/browser/", mode='full', method='copy', + login=True)) + self.expectCommands( + ExpectShell(workdir='wkdir', + command=['cvs', '--version']) + + 0, + Expect('rmdir', dict(dir='wkdir', + logEnviron=True)) + + 0, + Expect('uploadFile', dict(blocksize=32768, maxsize=None, + slavesrc='Root', workdir='source/CVS', + writer=ExpectRemoteRef(shell.StringFileWriter))) + + Expect.behavior(uploadString('the-end-of-the-universe')) + + 0, + Expect('rmdir', dict(dir='source', + logEnviron=True)) + + 0, + ExpectShell(workdir='', + command=['cvs', + '-d', + ':pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot', + '-z3', 'checkout', '-d', 'source', 'mozilla/browser/']) + + 0, + Expect('cpdir', {'fromdir': 'source', 'todir': 'build', + 'logEnviron': True}) + + 0, + ) + + self.expectOutcome(result=SUCCESS, status_text=["update"]) + return self.runStep() + def test_mode_incremental(self): self.setupStep( cvs.CVS(cvsroot=":pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot", @@ -131,9 +288,16 @@ def test_mode_incremental(self): ExpectShell(workdir='wkdir', command=['cvs', '--version']) + 0, - Expect('stat', dict(file='wkdir/CVS', - logEnviron=True)) - + 0, + Expect('uploadFile', dict(blocksize=32768, maxsize=None, + slavesrc='Root', workdir='wkdir/CVS', + writer=ExpectRemoteRef(shell.StringFileWriter))) + + Expect.behavior(uploadString(':pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot')) + + 0, + Expect('uploadFile', dict(blocksize=32768, maxsize=None, + slavesrc='Repository', workdir='wkdir/CVS', + writer=ExpectRemoteRef(shell.StringFileWriter))) + + Expect.behavior(uploadString('mozilla/browser/')) + + 0, ExpectShell(workdir='wkdir', command=['cvs', '-z3', 'update', '-dP']) + 0, @@ -142,6 +306,61 @@ def test_mode_incremental(self): self.expectOutcome(result=SUCCESS, status_text=["update"]) return self.runStep() + def test_mode_incremental_branch(self): + self.setupStep( + cvs.CVS(cvsroot=":pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot", + cvsmodule="mozilla/browser/", mode='incremental', + branch='my_branch', login=True)) + self.expectCommands( + ExpectShell(workdir='wkdir', + command=['cvs', '--version']) + + 0, + Expect('uploadFile', dict(blocksize=32768, maxsize=None, + slavesrc='Root', workdir='wkdir/CVS', + writer=ExpectRemoteRef(shell.StringFileWriter))) + + Expect.behavior(uploadString(':pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot')) + + 0, + Expect('uploadFile', dict(blocksize=32768, maxsize=None, + slavesrc='Repository', workdir='wkdir/CVS', + writer=ExpectRemoteRef(shell.StringFileWriter))) + + Expect.behavior(uploadString('mozilla/browser/')) + + 0, + ExpectShell(workdir='wkdir', + command=['cvs', '-z3', 'update', '-dP', '-r', 'my_branch']) + + 0, + ) + + self.expectOutcome(result=SUCCESS, status_text=["update"]) + return self.runStep() + + def test_mode_incremental_branch_sourcestamp(self): + self.setupStep( + cvs.CVS(cvsroot=":pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot", + cvsmodule="mozilla/browser/", mode='incremental', + login=True), args={'branch':'my_branch'}) + self.expectCommands( + ExpectShell(workdir='wkdir', + command=['cvs', '--version']) + + 0, + Expect('uploadFile', dict(blocksize=32768, maxsize=None, + slavesrc='Root', workdir='wkdir/CVS', + writer=ExpectRemoteRef(shell.StringFileWriter))) + + Expect.behavior(uploadString(':pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot')) + + 0, + Expect('uploadFile', dict(blocksize=32768, maxsize=None, + slavesrc='Repository', workdir='wkdir/CVS', + writer=ExpectRemoteRef(shell.StringFileWriter))) + + Expect.behavior(uploadString('mozilla/browser/')) + + 0, + ExpectShell(workdir='wkdir', + command=['cvs', '-z3', 'update', '-dP', '-r', 'my_branch']) + + 0, + ) + + self.expectOutcome(result=SUCCESS, status_text=["update"]) + return self.runStep() + + def test_mode_incremental_not_loggedin(self): self.setupStep( cvs.CVS(cvsroot=":pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot", @@ -156,9 +375,16 @@ def test_mode_incremental_not_loggedin(self): ':pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot', 'login']) + 0, - Expect('stat', dict(file='wkdir/CVS', - logEnviron=True)) - + 0, + Expect('uploadFile', dict(blocksize=32768, maxsize=None, + slavesrc='Root', workdir='wkdir/CVS', + writer=ExpectRemoteRef(shell.StringFileWriter))) + + Expect.behavior(uploadString(':pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot')) + + 0, + Expect('uploadFile', dict(blocksize=32768, maxsize=None, + slavesrc='Repository', workdir='wkdir/CVS', + writer=ExpectRemoteRef(shell.StringFileWriter))) + + Expect.behavior(uploadString('mozilla/browser/')) + + 0, ExpectShell(workdir='wkdir', command=['cvs', '-z3', 'update', '-dP']) + 0, @@ -177,9 +403,74 @@ def test_mode_incremental_no_existing_repo(self): ExpectShell(workdir='wkdir', command=['cvs', '--version']) + 0, - Expect('stat', dict(file='wkdir/CVS', - logEnviron=True)) + Expect('uploadFile', dict(blocksize=32768, maxsize=None, + slavesrc='Root', workdir='wkdir/CVS', + writer=ExpectRemoteRef(shell.StringFileWriter))) + 1, + Expect('rmdir', dict(dir='wkdir', + logEnviron=True)) + + 0, + ExpectShell(workdir='', + command=['cvs', + '-d', + ':pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot', + '-z3', 'checkout', '-d', 'wkdir', 'mozilla/browser/']) + + 0, + ) + self.expectOutcome(result=SUCCESS, status_text=["update"]) + return self.runStep() + + + def test_mode_incremental_wrong_repo(self): + self.setupStep( + cvs.CVS(cvsroot=":pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot", + cvsmodule="mozilla/browser/", mode='incremental', + login=True)) + self.expectCommands( + ExpectShell(workdir='wkdir', + command=['cvs', '--version']) + + 0, + Expect('uploadFile', dict(blocksize=32768, maxsize=None, + slavesrc='Root', workdir='wkdir/CVS', + writer=ExpectRemoteRef(shell.StringFileWriter))) + + Expect.behavior(uploadString('the-end-of-the-universe')) + + 0, + Expect('rmdir', dict(dir='wkdir', + logEnviron=True)) + + 0, + ExpectShell(workdir='', + command=['cvs', + '-d', + ':pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot', + '-z3', 'checkout', '-d', 'wkdir', 'mozilla/browser/']) + + 0, + ) + self.expectOutcome(result=SUCCESS, status_text=["update"]) + return self.runStep() + + + def test_mode_incremental_wrong_module(self): + self.setupStep( + cvs.CVS(cvsroot=":pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot", + cvsmodule="mozilla/browser/", mode='incremental', + login=True)) + self.expectCommands( + ExpectShell(workdir='wkdir', + command=['cvs', '--version']) + + 0, + Expect('uploadFile', dict(blocksize=32768, maxsize=None, + slavesrc='Root', workdir='wkdir/CVS', + writer=ExpectRemoteRef(shell.StringFileWriter))) + + Expect.behavior(uploadString(':pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot')) + + 0, + Expect('uploadFile', dict(blocksize=32768, maxsize=None, + slavesrc='Repository', workdir='wkdir/CVS', + writer=ExpectRemoteRef(shell.StringFileWriter))) + + Expect.behavior(uploadString('the-end-of-the-universe')) + + 0, + Expect('rmdir', dict(dir='wkdir', + logEnviron=True)) + + 0, ExpectShell(workdir='', command=['cvs', '-d', @@ -200,8 +491,9 @@ def test_mode_full_clean_no_existing_repo(self): ExpectShell(workdir='wkdir', command=['cvs', '--version']) + 0, - Expect('stat', dict(file='wkdir/CVS', - logEnviron=True)) + Expect('uploadFile', dict(blocksize=32768, maxsize=None, + slavesrc='Root', workdir='wkdir/CVS', + writer=ExpectRemoteRef(shell.StringFileWriter))) + 1, ExpectShell(workdir='', command=['cvs', @@ -213,6 +505,30 @@ def test_mode_full_clean_no_existing_repo(self): self.expectOutcome(result=SUCCESS, status_text=["update"]) return self.runStep() + def test_mode_full_clean_wrong_repo(self): + self.setupStep( + cvs.CVS(cvsroot=":pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot", + cvsmodule="mozilla/browser/", mode='full', method='clean', + login=True)) + self.expectCommands( + ExpectShell(workdir='wkdir', + command=['cvs', '--version']) + + 0, + Expect('uploadFile', dict(blocksize=32768, maxsize=None, + slavesrc='Root', workdir='wkdir/CVS', + writer=ExpectRemoteRef(shell.StringFileWriter))) + + Expect.behavior(uploadString('the-end-of-the-universe')) + + 0, + ExpectShell(workdir='', + command=['cvs', + '-d', + ':pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot', + '-z3', 'checkout', '-d', 'wkdir', 'mozilla/browser/']) + + 0, + ) + self.expectOutcome(result=SUCCESS, status_text=["update"]) + return self.runStep() + def test_mode_full_no_method(self): self.setupStep( cvs.CVS(cvsroot=":pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot", @@ -222,8 +538,15 @@ def test_mode_full_no_method(self): ExpectShell(workdir='wkdir', command=['cvs', '--version']) + 0, - Expect('stat', dict(file='wkdir/CVS', - logEnviron=True)) + Expect('uploadFile', dict(blocksize=32768, maxsize=None, + slavesrc='Root', workdir='wkdir/CVS', + writer=ExpectRemoteRef(shell.StringFileWriter))) + + Expect.behavior(uploadString(':pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot')) + + 0, + Expect('uploadFile', dict(blocksize=32768, maxsize=None, + slavesrc='Repository', workdir='wkdir/CVS', + writer=ExpectRemoteRef(shell.StringFileWriter))) + + Expect.behavior(uploadString('mozilla/browser/')) + 0, ExpectShell(workdir='wkdir', command=['cvsdiscard', '--ignore']) @@ -245,13 +568,17 @@ def test_mode_incremental_with_options(self): ExpectShell(workdir='wkdir', command=['cvs', '--version']) + 0, - Expect('stat', dict(file='wkdir/CVS', - logEnviron=True)) + Expect('uploadFile', dict(blocksize=32768, maxsize=None, + slavesrc='Root', workdir='wkdir/CVS', + writer=ExpectRemoteRef(shell.StringFileWriter))) + 1, + Expect('rmdir', dict(dir='wkdir', + logEnviron=True)) + + 0, ExpectShell(workdir='', command=['cvs', '-q', '-d', ':pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot', - '-z3', 'checkout', '-d', 'wkdir', 'mozilla/browser/', '-l']) + '-z3', 'checkout', '-d', 'wkdir', '-l', 'mozilla/browser/']) + 0, ) self.expectOutcome(result=SUCCESS, status_text=["update"]) @@ -268,8 +595,15 @@ def test_mode_incremental_with_env_logEnviron(self): env={'abc': '123'}, logEnviron=False) + 0, - Expect('stat', dict(file='wkdir/CVS', - logEnviron=False)) + Expect('uploadFile', dict(blocksize=32768, maxsize=None, + slavesrc='Root', workdir='wkdir/CVS', + writer=ExpectRemoteRef(shell.StringFileWriter))) + + Expect.behavior(uploadString(':pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot')) + + 0, + Expect('uploadFile', dict(blocksize=32768, maxsize=None, + slavesrc='Repository', workdir='wkdir/CVS', + writer=ExpectRemoteRef(shell.StringFileWriter))) + + Expect.behavior(uploadString('mozilla/browser/')) + 0, ExpectShell(workdir='wkdir', command=['cvs', '-z3', 'update', '-dP'], @@ -304,8 +638,15 @@ def test_cvsdiscard_fails(self): ExpectShell(workdir='wkdir', command=['cvs', '--version']) + 0, - Expect('stat', dict(file='wkdir/CVS', - logEnviron=True)) + Expect('uploadFile', dict(blocksize=32768, maxsize=None, + slavesrc='Root', workdir='wkdir/CVS', + writer=ExpectRemoteRef(shell.StringFileWriter))) + + Expect.behavior(uploadString(':pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot')) + + 0, + Expect('uploadFile', dict(blocksize=32768, maxsize=None, + slavesrc='Repository', workdir='wkdir/CVS', + writer=ExpectRemoteRef(shell.StringFileWriter))) + + Expect.behavior(uploadString('mozilla/browser/')) + 0, ExpectShell(workdir='wkdir', command=['cvsdiscard', '--ignore']) diff --git a/master/buildbot/test/unit/test_steps_source_git.py b/master/buildbot/test/unit/test_steps_source_git.py index ba512bd1080..d70545ef850 100644 --- a/master/buildbot/test/unit/test_steps_source_git.py +++ b/master/buildbot/test/unit/test_steps_source_git.py @@ -56,6 +56,45 @@ def test_mode_full_clean(self): + 0, ) self.expectOutcome(result=SUCCESS, status_text=["update"]) + self.expectProperty('got_revision', 'f6ad368298bd941e934a41f3babc827b2aa95a1d', 'Source') + return self.runStep() + + def test_mode_full_clean_timeout(self): + self.setupStep( + git.Git(repourl='http://github.com/buildbot/buildbot.git', + timeout=1, + mode='full', method='clean')) + self.expectCommands( + ExpectShell(workdir='wkdir', + timeout=1, + command=['git', '--version']) + + 0, + Expect('stat', dict(file='wkdir/.git', + logEnviron=True)) + + 0, + ExpectShell(workdir='wkdir', + timeout=1, + command=['git', 'clean', '-f', '-d']) + + 0, + ExpectShell(workdir='wkdir', + timeout=1, + command=['git', 'fetch', '-t', + 'http://github.com/buildbot/buildbot.git', + 'HEAD']) + + 0, + ExpectShell(workdir='wkdir', + timeout=1, + command=['git', 'reset', '--hard', 'FETCH_HEAD']) + + 0, + ExpectShell(workdir='wkdir', + timeout=1, + command=['git', 'rev-parse', 'HEAD']) + + ExpectShell.log('stdio', + stdout='f6ad368298bd941e934a41f3babc827b2aa95a1d') + + 0, + ) + self.expectOutcome(result=SUCCESS, status_text=["update"]) + self.expectProperty('got_revision', 'f6ad368298bd941e934a41f3babc827b2aa95a1d', 'Source') return self.runStep() def test_mode_full_clean_patch(self): @@ -92,6 +131,7 @@ def test_mode_full_clean_patch(self): + 0, ) self.expectOutcome(result=SUCCESS, status_text=["update"]) + self.expectProperty('got_revision', 'f6ad368298bd941e934a41f3babc827b2aa95a1d', 'Source') return self.runStep() def test_mode_full_clean_patch_fail(self): @@ -123,6 +163,7 @@ def test_mode_full_clean_patch_fail(self): + 1, ) self.expectOutcome(result=FAILURE, status_text=["updating"]) + self.expectNoProperty('got_revision') return self.runStep() def test_mode_full_clean_branch(self): @@ -157,6 +198,7 @@ def test_mode_full_clean_branch(self): + 0, ) self.expectOutcome(result=SUCCESS, status_text=["update"]) + self.expectProperty('got_revision', 'f6ad368298bd941e934a41f3babc827b2aa95a1d', 'Source') return self.runStep() def test_mode_full_clean_parsefail(self): @@ -186,6 +228,7 @@ def test_mode_full_clean_parsefail(self): + 128, ) self.expectOutcome(result=FAILURE, status_text=["updating"]) + self.expectNoProperty('got_revision') return self.runStep() def test_mode_full_clean_no_existing_repo(self): @@ -265,6 +308,7 @@ def test_mode_full_clobber(self): + 0, ) self.expectOutcome(result=SUCCESS, status_text=["update"]) + self.expectProperty('got_revision', 'f6ad368298bd941e934a41f3babc827b2aa95a1d', 'Source') return self.runStep() def test_mode_full_clobber_branch(self): @@ -292,6 +336,7 @@ def test_mode_full_clobber_branch(self): + 0, ) self.expectOutcome(result=SUCCESS, status_text=["update"]) + self.expectProperty('got_revision', 'f6ad368298bd941e934a41f3babc827b2aa95a1d', 'Source') return self.runStep() def test_mode_incremental(self): @@ -321,6 +366,7 @@ def test_mode_incremental(self): ) self.expectOutcome(result=SUCCESS, status_text=["update"]) + self.expectProperty('got_revision', 'f6ad368298bd941e934a41f3babc827b2aa95a1d', 'Source') return self.runStep() def test_mode_incremental_branch(self): @@ -353,6 +399,7 @@ def test_mode_incremental_branch(self): ) self.expectOutcome(result=SUCCESS, status_text=["update"]) + self.expectProperty('got_revision', 'f6ad368298bd941e934a41f3babc827b2aa95a1d', 'Source') return self.runStep() def test_mode_full_fresh(self): @@ -384,6 +431,7 @@ def test_mode_full_fresh(self): + 0, ) self.expectOutcome(result=SUCCESS, status_text=["update"]) + self.expectProperty('got_revision', 'f6ad368298bd941e934a41f3babc827b2aa95a1d', 'Source') return self.runStep() def test_mode_incremental_given_revision(self): @@ -412,6 +460,7 @@ def test_mode_incremental_given_revision(self): + 0, ) self.expectOutcome(result=SUCCESS, status_text=["update"]) + self.expectProperty('got_revision', 'f6ad368298bd941e934a41f3babc827b2aa95a1d', 'Source') return self.runStep() def test_mode_full_fresh_submodule(self): @@ -450,6 +499,7 @@ def test_mode_full_fresh_submodule(self): + 0, ) self.expectOutcome(result=SUCCESS, status_text=["update"]) + self.expectProperty('got_revision', 'f6ad368298bd941e934a41f3babc827b2aa95a1d', 'Source') return self.runStep() def test_mode_full_clobber_shallow(self): @@ -477,6 +527,7 @@ def test_mode_full_clobber_shallow(self): + 0, ) self.expectOutcome(result=SUCCESS, status_text=["update"]) + self.expectProperty('got_revision', 'f6ad368298bd941e934a41f3babc827b2aa95a1d', 'Source') return self.runStep() def test_mode_incremental_retryFetch(self): @@ -514,6 +565,7 @@ def test_mode_incremental_retryFetch(self): + 0, ) self.expectOutcome(result=SUCCESS, status_text=["update"]) + self.expectProperty('got_revision', 'f6ad368298bd941e934a41f3babc827b2aa95a1d', 'Source') return self.runStep() def test_mode_incremental_retryFetch_branch(self): @@ -554,6 +606,7 @@ def test_mode_incremental_retryFetch_branch(self): + 0, ) self.expectOutcome(result=SUCCESS, status_text=["update"]) + self.expectProperty('got_revision', 'f6ad368298bd941e934a41f3babc827b2aa95a1d', 'Source') return self.runStep() def test_mode_incremental_clobberOnFailure(self): @@ -592,6 +645,7 @@ def test_mode_incremental_clobberOnFailure(self): + 0, ) self.expectOutcome(result=SUCCESS, status_text=["update"]) + self.expectProperty('got_revision', 'f6ad368298bd941e934a41f3babc827b2aa95a1d', 'Source') return self.runStep() def test_mode_incremental_clobberOnFailure_branch(self): @@ -630,6 +684,7 @@ def test_mode_incremental_clobberOnFailure_branch(self): + 0, ) self.expectOutcome(result=SUCCESS, status_text=["update"]) + self.expectProperty('got_revision', 'f6ad368298bd941e934a41f3babc827b2aa95a1d', 'Source') return self.runStep() def test_mode_full_copy(self): @@ -664,6 +719,7 @@ def test_mode_full_copy(self): + 0, ) self.expectOutcome(result=SUCCESS, status_text=["update"]) + self.expectProperty('got_revision', 'f6ad368298bd941e934a41f3babc827b2aa95a1d', 'Source') return self.runStep() @@ -691,6 +747,7 @@ def test_mode_incremental_no_existing_repo(self): ) self.expectOutcome(result=SUCCESS, status_text=["update"]) + self.expectProperty('got_revision', 'f6ad368298bd941e934a41f3babc827b2aa95a1d', 'Source') return self.runStep() def test_mode_full_clobber_given_revision(self): @@ -722,6 +779,7 @@ def test_mode_full_clobber_given_revision(self): + 0, ) self.expectOutcome(result=SUCCESS, status_text=["update"]) + self.expectProperty('got_revision', 'f6ad368298bd941e934a41f3babc827b2aa95a1d', 'Source') return self.runStep() def test_revparse_failure(self): @@ -753,6 +811,7 @@ def test_revparse_failure(self): + 0, ) self.expectOutcome(result=FAILURE, status_text=["updating"]) + self.expectNoProperty('got_revision') return self.runStep() def test_mode_full_clobber_submodule(self): @@ -783,6 +842,7 @@ def test_mode_full_clobber_submodule(self): + 0, ) self.expectOutcome(result=SUCCESS, status_text=["update"]) + self.expectProperty('got_revision', 'f6ad368298bd941e934a41f3babc827b2aa95a1d', 'Source') return self.runStep() def test_repourl(self): @@ -818,6 +878,7 @@ def test_mode_full_fresh_revision(self): + 0, ) self.expectOutcome(result=SUCCESS, status_text=["update"]) + self.expectProperty('got_revision', 'f6ad368298bd941e934a41f3babc827b2aa95a1d', 'Source') return self.runStep() def test_mode_full_fresh_clobberOnFailure(self): @@ -853,6 +914,7 @@ def test_mode_full_fresh_clobberOnFailure(self): + 0, ) self.expectOutcome(result=SUCCESS, status_text=["update"]) + self.expectProperty('got_revision', 'f6ad368298bd941e934a41f3babc827b2aa95a1d', 'Source') return self.runStep() def test_mode_full_no_method(self): @@ -884,6 +946,7 @@ def test_mode_full_no_method(self): + 0, ) self.expectOutcome(result=SUCCESS, status_text=["update"]) + self.expectProperty('got_revision', 'f6ad368298bd941e934a41f3babc827b2aa95a1d', 'Source') return self.runStep() def test_mode_full_with_env(self): @@ -920,6 +983,7 @@ def test_mode_full_with_env(self): + 0, ) self.expectOutcome(result=SUCCESS, status_text=["update"]) + self.expectProperty('got_revision', 'f6ad368298bd941e934a41f3babc827b2aa95a1d', 'Source') return self.runStep() def test_mode_full_logEnviron(self): @@ -956,4 +1020,337 @@ def test_mode_full_logEnviron(self): + 0, ) self.expectOutcome(result=SUCCESS, status_text=["update"]) + self.expectProperty('got_revision', 'f6ad368298bd941e934a41f3babc827b2aa95a1d', 'Source') + return self.runStep() + + def test_getDescription(self): + # clone of: test_mode_incremental + # only difference is to set the getDescription property + + self.setupStep( + git.Git(repourl='http://github.com/buildbot/buildbot.git', + mode='incremental', + getDescription=True)) + self.expectCommands( + ## copied from test_mode_incremental: + ExpectShell(workdir='wkdir', + command=['git', '--version']) + + 0, + Expect('stat', dict(file='wkdir/.git', + logEnviron=True)) + + 0, + ExpectShell(workdir='wkdir', + command=['git', 'fetch', '-t', + 'http://github.com/buildbot/buildbot.git', + 'HEAD']) + + 0, + ExpectShell(workdir='wkdir', + command=['git', 'reset', '--hard', 'FETCH_HEAD']) + + 0, + ExpectShell(workdir='wkdir', + command=['git', 'rev-parse', 'HEAD']) + + ExpectShell.log('stdio', + stdout='f6ad368298bd941e934a41f3babc827b2aa95a1d') + + 0, + + ## plus this to test describe: + ExpectShell(workdir='wkdir', + command=['git', 'describe', 'HEAD']) + + ExpectShell.log('stdio', + stdout='Tag-1234') + + 0, + ) + self.expectOutcome(result=SUCCESS, status_text=["update"]) + self.expectProperty('got_revision', 'f6ad368298bd941e934a41f3babc827b2aa95a1d', 'Source') + self.expectProperty('commit-description', 'Tag-1234', 'Source') + return self.runStep() + + def test_getDescription_failed(self): + # clone of: test_mode_incremental + # only difference is to set the getDescription property + + # this tests when 'git describe' fails; for example, there are no + # tags in the repository + + self.setupStep( + git.Git(repourl='http://github.com/buildbot/buildbot.git', + mode='incremental', + getDescription=True)) + self.expectCommands( + ## copied from test_mode_incremental: + ExpectShell(workdir='wkdir', + command=['git', '--version']) + + 0, + Expect('stat', dict(file='wkdir/.git', + logEnviron=True)) + + 0, + ExpectShell(workdir='wkdir', + command=['git', 'fetch', '-t', + 'http://github.com/buildbot/buildbot.git', + 'HEAD']) + + 0, + ExpectShell(workdir='wkdir', + command=['git', 'reset', '--hard', 'FETCH_HEAD']) + + 0, + ExpectShell(workdir='wkdir', + command=['git', 'rev-parse', 'HEAD']) + + ExpectShell.log('stdio', + stdout='f6ad368298bd941e934a41f3babc827b2aa95a1d') + + 0, + + ## plus this to test describe: + ExpectShell(workdir='wkdir', + command=['git', 'describe', 'HEAD']) + + ExpectShell.log('stdio', + stdout='') + + 128, # error, but it's suppressed + ) + self.expectOutcome(result=SUCCESS, status_text=["update"]) + self.expectProperty('got_revision', 'f6ad368298bd941e934a41f3babc827b2aa95a1d', 'Source') + self.expectNoProperty('commit-description') + return self.runStep() + + def setup_getDescription_test(self, setup_args, output_args, codebase=None): + # clone of: test_mode_full_clobber + # only difference is to set the getDescription property + + kwargs = {} + if codebase is not None: + kwargs.update(codebase=codebase) + + self.setupStep( + git.Git(repourl='http://github.com/buildbot/buildbot.git', + mode='full', method='clobber', progress=True, + getDescription=setup_args, + **kwargs)) + + self.expectCommands( + ## copied from test_mode_full_clobber: + ExpectShell(workdir='wkdir', + command=['git', '--version']) + + 0, + Expect('rmdir', dict(dir='wkdir', + logEnviron=True)) + + 0, + ExpectShell(workdir='wkdir', + command=['git', 'clone', + '--branch', 'HEAD', + 'http://github.com/buildbot/buildbot.git', + '.', '--progress']) + + 0, + ExpectShell(workdir='wkdir', + command=['git', 'rev-parse', 'HEAD']) + + ExpectShell.log('stdio', + stdout='f6ad368298bd941e934a41f3babc827b2aa95a1d') + + 0, + + ## plus this to test describe: + ExpectShell(workdir='wkdir', + command=['git', 'describe'] + + output_args + + ['HEAD']) + + ExpectShell.log('stdio', + stdout='Tag-1234') + + 0, + ) + + if codebase: + self.expectOutcome(result=SUCCESS, status_text=["update", codebase]) + self.expectProperty('got_revision', {codebase:'f6ad368298bd941e934a41f3babc827b2aa95a1d'}, 'Source') + self.expectProperty('commit-description', {codebase:'Tag-1234'}, 'Source') + else: + self.expectOutcome(result=SUCCESS, status_text=["update"]) + self.expectProperty('got_revision', 'f6ad368298bd941e934a41f3babc827b2aa95a1d', 'Source') + self.expectProperty('commit-description', 'Tag-1234', 'Source') + + def test_getDescription_empty_dict(self): + self.setup_getDescription_test( + setup_args = {}, + output_args = [] + ) + return self.runStep() + + def test_getDescription_empty_dict_with_codebase(self): + self.setup_getDescription_test( + setup_args = {}, + output_args = [], + codebase = 'baz' + ) + return self.runStep() + + def test_getDescription_match(self): + self.setup_getDescription_test( + setup_args = { 'match': 'stuff-*' }, + output_args = ['--match', 'stuff-*'] + ) + return self.runStep() + def test_getDescription_match_false(self): + self.setup_getDescription_test( + setup_args = { 'match': None }, + output_args = [] + ) + return self.runStep() + + def test_getDescription_tags(self): + self.setup_getDescription_test( + setup_args = { 'tags': True }, + output_args = ['--tags'] + ) + return self.runStep() + def test_getDescription_tags_false(self): + self.setup_getDescription_test( + setup_args = { 'tags': False }, + output_args = [] + ) + return self.runStep() + + def test_getDescription_all(self): + self.setup_getDescription_test( + setup_args = { 'all': True }, + output_args = ['--all'] + ) + return self.runStep() + def test_getDescription_all_false(self): + self.setup_getDescription_test( + setup_args = { 'all': False }, + output_args = [] + ) + return self.runStep() + + def test_getDescription_abbrev(self): + self.setup_getDescription_test( + setup_args = { 'abbrev': 7 }, + output_args = ['--abbrev=7'] + ) + return self.runStep() + def test_getDescription_abbrev_zero(self): + self.setup_getDescription_test( + setup_args = { 'abbrev': 0 }, + output_args = ['--abbrev=0'] + ) + return self.runStep() + def test_getDescription_abbrev_false(self): + self.setup_getDescription_test( + setup_args = { 'abbrev': False }, + output_args = [] + ) + return self.runStep() + + def test_getDescription_dirty(self): + self.setup_getDescription_test( + setup_args = { 'dirty': True }, + output_args = ['--dirty'] + ) + return self.runStep() + def test_getDescription_dirty_empty_str(self): + self.setup_getDescription_test( + setup_args = { 'dirty': '' }, + output_args = ['--dirty'] + ) + return self.runStep() + def test_getDescription_dirty_str(self): + self.setup_getDescription_test( + setup_args = { 'dirty': 'foo' }, + output_args = ['--dirty=foo'] + ) + return self.runStep() + def test_getDescription_dirty_false(self): + self.setup_getDescription_test( + setup_args = { 'dirty': False }, + output_args = [] + ) + return self.runStep() + + def test_getDescription_contains(self): + self.setup_getDescription_test( + setup_args = { 'contains': True }, + output_args = ['--contains'] + ) + return self.runStep() + def test_getDescription_contains_false(self): + self.setup_getDescription_test( + setup_args = { 'contains': False }, + output_args = [] + ) + return self.runStep() + + def test_getDescription_candidates(self): + self.setup_getDescription_test( + setup_args = { 'candidates': 7 }, + output_args = ['--candidates=7'] + ) + return self.runStep() + def test_getDescription_candidates_zero(self): + self.setup_getDescription_test( + setup_args = { 'candidates': 0 }, + output_args = ['--candidates=0'] + ) + return self.runStep() + def test_getDescription_candidates_false(self): + self.setup_getDescription_test( + setup_args = { 'candidates': False }, + output_args = [] + ) + return self.runStep() + + def test_getDescription_exact_match(self): + self.setup_getDescription_test( + setup_args = { 'exact-match': True }, + output_args = ['--exact-match'] + ) + return self.runStep() + def test_getDescription_exact_match_false(self): + self.setup_getDescription_test( + setup_args = { 'exact-match': False }, + output_args = [] + ) + return self.runStep() + + def test_getDescription_debug(self): + self.setup_getDescription_test( + setup_args = { 'debug': True }, + output_args = ['--debug'] + ) + return self.runStep() + def test_getDescription_debug_false(self): + self.setup_getDescription_test( + setup_args = { 'debug': False }, + output_args = [] + ) + return self.runStep() + + def test_getDescription_long(self): + self.setup_getDescription_test( + setup_args = { 'long': True }, + output_args = ['--long'] + ) + def test_getDescription_long_false(self): + self.setup_getDescription_test( + setup_args = { 'long': False }, + output_args = [] + ) + return self.runStep() + + def test_getDescription_always(self): + self.setup_getDescription_test( + setup_args = { 'always': True }, + output_args = ['--always'] + ) + def test_getDescription_always_false(self): + self.setup_getDescription_test( + setup_args = { 'always': False }, + output_args = [] + ) + return self.runStep() + + def test_getDescription_lotsa_stuff(self): + self.setup_getDescription_test( + setup_args = { 'match': 'stuff-*', + 'abbrev': 6, + 'exact-match': True}, + output_args = ['--exact-match', + '--match', 'stuff-*', + '--abbrev=6'], + codebase='baz' + ) return self.runStep() diff --git a/master/buildbot/test/unit/test_steps_source_mercurial.py b/master/buildbot/test/unit/test_steps_source_mercurial.py index 104bce514a2..db2e03928c8 100644 --- a/master/buildbot/test/unit/test_steps_source_mercurial.py +++ b/master/buildbot/test/unit/test_steps_source_mercurial.py @@ -86,6 +86,52 @@ def test_mode_full_clean(self): self.expectOutcome(result=SUCCESS, status_text=["update"]) return self.runStep() + def test_mode_full_clean_timeout(self): + self.setupStep( + mercurial.Mercurial(repourl='http://hg.mozilla.org', + timeout=1, + mode='full', method='clean', branchType='inrepo')) + self.expectCommands( + ExpectShell(workdir='wkdir', + timeout=1, + command=['hg', '--verbose', '--version']) + + 0, + Expect('stat', dict(file='wkdir/.hg', + logEnviron=True)) + + 0, + ExpectShell(workdir='wkdir', + timeout=1, + command=['hg', '--verbose', '--config', + 'extensions.purge=', 'purge']) + + 0, + ExpectShell(workdir='wkdir', + timeout=1, + command=['hg', '--verbose', 'pull', + 'http://hg.mozilla.org']) + + 0, + ExpectShell(workdir='wkdir', + timeout=1, + command=['hg', '--verbose', 'identify', '--branch']) + + ExpectShell.log('stdio', + stdout='default') + + 0, + ExpectShell(workdir='wkdir', + timeout=1, + command=['hg', '--verbose', 'update', + '--clean']) + + 0, + ExpectShell(workdir='wkdir', + timeout=1, + command=['hg', '--verbose', 'identify', + '--id', '--debug']) + + ExpectShell.log('stdio', stdout='\n') + + ExpectShell.log('stdio', + stdout='f6ad368298bd941e934a41f3babc827b2aa95a1d') + + 0, + ) + self.expectOutcome(result=SUCCESS, status_text=["update"]) + return self.runStep() + def test_mode_full_clean_no_existing_repo(self): self.setupStep( mercurial.Mercurial(repourl='http://hg.mozilla.org', diff --git a/master/buildbot/test/unit/test_steps_source_oldsource_ComputeRepositoryURL.py b/master/buildbot/test/unit/test_steps_source_oldsource_ComputeRepositoryURL.py index 5dd48a51d5b..65c8739d634 100644 --- a/master/buildbot/test/unit/test_steps_source_oldsource_ComputeRepositoryURL.py +++ b/master/buildbot/test/unit/test_steps_source_oldsource_ComputeRepositoryURL.py @@ -37,12 +37,15 @@ def render(self, value): self.props.build = self return defer.maybeDeferred(IRenderable(value).getRenderingFor, self.props) +class FakeStep(object): + codebase = '' + class RepoURL(unittest.TestCase): def setUp(self): self.build = Build() def test_backward_compatibility(self): - url = _ComputeRepositoryURL("repourl") + url = _ComputeRepositoryURL(FakeStep(), "repourl") d = self.build.render(url) @d.addCallback def callback(res): @@ -50,7 +53,7 @@ def callback(res): return d def test_format_string(self): - url = _ComputeRepositoryURL("http://server/%s") + url = _ComputeRepositoryURL(FakeStep(), "http://server/%s") d = self.build.render(url) @d.addCallback def callback(res): @@ -60,7 +63,7 @@ def callback(res): def test_dict(self): dict = {} dict['test'] = "ssh://server/testrepository" - url = _ComputeRepositoryURL(dict) + url = _ComputeRepositoryURL(FakeStep(), dict) d = self.build.render(url) @d.addCallback def callback(res): @@ -69,7 +72,7 @@ def callback(res): def test_callable(self): func = lambda x: x[::-1] - url = _ComputeRepositoryURL(func) + url = _ComputeRepositoryURL(FakeStep(), func) d = self.build.render(url) @d.addCallback def callback(res): @@ -77,7 +80,7 @@ def callback(res): return d def test_backward_compatibility_render(self): - url = _ComputeRepositoryURL(WithProperties("repourl%(foo)s")) + url = _ComputeRepositoryURL(FakeStep(), WithProperties("repourl%(foo)s")) d = self.build.render(url) @d.addCallback def callback(res): @@ -86,7 +89,7 @@ def callback(res): def test_dict_render(self): d = dict(test=WithProperties("repourl%(foo)s")) - url = _ComputeRepositoryURL(d) + url = _ComputeRepositoryURL(FakeStep(), d) d = self.build.render(url) @d.addCallback def callback(res): @@ -95,7 +98,7 @@ def callback(res): def test_callable_render(self): func = lambda x: WithProperties(x+"%(foo)s") - url = _ComputeRepositoryURL(func) + url = _ComputeRepositoryURL(FakeStep(), func) d = self.build.render(url) @d.addCallback def callback(res): diff --git a/master/buildbot/test/unit/test_steps_source_svn.py b/master/buildbot/test/unit/test_steps_source_svn.py index 22eb9ddb6fe..d79709b0529 100644 --- a/master/buildbot/test/unit/test_steps_source_svn.py +++ b/master/buildbot/test/unit/test_steps_source_svn.py @@ -104,6 +104,44 @@ def test_mode_incremental(self): self.expectOutcome(result=SUCCESS, status_text=["update"]) return self.runStep() + def test_mode_incremental_timeout(self): + self.setupStep( + svn.SVN(repourl='http://svn.local/app/trunk', + mode='incremental',username='user', + timeout=1, + password='pass', extra_args=['--random'])) + self.expectCommands( + ExpectShell(workdir='wkdir', + timeout=1, + command=['svn', '--version']) + + 0, + Expect('stat', dict(file='wkdir/.svn', + logEnviron=True)) + + 0, + ExpectShell(workdir='wkdir', + timeout=1, + command=['svn', 'info', '--non-interactive', + '--no-auth-cache', '--username', 'user', + '--password', 'pass', '--random']) + + ExpectShell.log('stdio', + stdout="URL: http://svn.local/app/trunk") + + 0, + ExpectShell(workdir='wkdir', + timeout=1, + command=['svn', 'update', '--non-interactive', + '--no-auth-cache', '--username', 'user', + '--password', 'pass', '--random']) + + 0, + ExpectShell(workdir='wkdir', + timeout=1, + command=['svnversion']) + + ExpectShell.log('stdio', + stdout='100') + + 0, + ) + self.expectOutcome(result=SUCCESS, status_text=["update"]) + return self.runStep() + def test_mode_incremental_repourl_renderable(self): self.setupStep( svn.SVN(repourl=FakeRenderable('http://svn.local/trunk'), @@ -724,6 +762,48 @@ def test_mode_full_export(self): self.expectOutcome(result=SUCCESS, status_text=["update"]) return self.runStep() + def test_mode_full_export_timeout(self): + self.setupStep( + svn.SVN(repourl='http://svn.local/app/trunk', + timeout=1, + mode='full', method='export')) + self.expectCommands( + ExpectShell(workdir='wkdir', + timeout=1, + command=['svn', '--version']) + + 0, + Expect('rmdir', dict(dir='wkdir', + logEnviron=True)) + + 0, + Expect('stat', dict(file='source/.svn', + logEnviron=True)) + + 0, + ExpectShell(workdir='source', + timeout=1, + command=['svn', 'info', '--non-interactive', + '--no-auth-cache' ]) + + ExpectShell.log('stdio', + stdout="URL: http://svn.local/app/trunk") + + 0, + ExpectShell(workdir='source', + timeout=1, + command=['svn', 'update', '--non-interactive', + '--no-auth-cache']) + + 0, + ExpectShell(workdir='', + timeout=1, + command=['svn', 'export', 'source', 'wkdir']) + + 0, + ExpectShell(workdir='source', + timeout=1, + command=['svnversion']) + + ExpectShell.log('stdio', + stdout='100') + + 0, + ) + self.expectOutcome(result=SUCCESS, status_text=["update"]) + return self.runStep() + def test_mode_full_export_given_revision(self): self.setupStep( svn.SVN(repourl='http://svn.local/app/trunk', diff --git a/master/buildbot/test/unit/test_steps_trigger.py b/master/buildbot/test/unit/test_steps_trigger.py index 463edc3c3f5..7578c6e3252 100644 --- a/master/buildbot/test/unit/test_steps_trigger.py +++ b/master/buildbot/test/unit/test_steps_trigger.py @@ -78,7 +78,8 @@ def setupStep(self, *args, **kwargs): # should be fixed! # set up a buildmaster that knows about two fake schedulers, a and b - self.build.builder.botmaster.parent = m = fakemaster.make_master() + m = fakemaster.make_master() + self.build.builder.botmaster = m.botmaster m.db = fakedb.FakeDBConnector(self) m.status = master.Status(m) m.config.buildbotURL = "baseurl/" diff --git a/master/buildbot/test/util/compat.py b/master/buildbot/test/util/compat.py index 59799c8d539..cf6305dd99b 100644 --- a/master/buildbot/test/util/compat.py +++ b/master/buildbot/test/util/compat.py @@ -25,6 +25,14 @@ def usesFlushLoggedErrors(test): "flushLoggedErrors is broken on Python==2.7 and Twisted<=9.0.0" return test +def usesFlushWarnings(test): + "Decorate a test method that uses flushWarnings with this decorator" + if (sys.version_info[:2] == (2,7) + and twisted.version <= versions.Version('twisted', 9, 0, 0)): + test.skip = \ + "flushWarnings is broken on Python==2.7 and Twisted<=9.0.0" + return test + def skipUnlessPlatformIs(platform): def closure(test): if runtime.platformType != platform: diff --git a/master/buildbot/test/util/steps.py b/master/buildbot/test/util/steps.py index 2724981749e..89ba5f3e5ff 100644 --- a/master/buildbot/test/util/steps.py +++ b/master/buildbot/test/util/steps.py @@ -71,10 +71,8 @@ def setupStep(self, step, slave_version={'*':"99.99"}, slave_env={}): @param slave_env: environment from the slave at slave startup """ - # yes, Virginia, "factory" refers both to the tuple and its first - # element TODO: fix that up - factory, args = step.getStepFactory() - step = self.step = factory(**args) + factory = interfaces.IBuildStepFactory(step) + step = self.step = factory.buildStep() # step.build @@ -252,5 +250,7 @@ def _remotecommand_run(self, command, step, remote): self.assertEqual((exp.remote_command, exp.args), got) # let the Expect object show any behaviors that are required - return exp.runBehaviors(command) + d = exp.runBehaviors(command) + d.addCallback(lambda _: command) + return d diff --git a/master/docs/developer/cls-buildsteps.rst b/master/docs/developer/cls-buildsteps.rst index ad792d332aa..2514d6cdcbe 100644 --- a/master/docs/developer/cls-buildsteps.rst +++ b/master/docs/developer/cls-buildsteps.rst @@ -3,17 +3,16 @@ BuildSteps .. py:module:: buildbot.process.buildstep -There are a few parent classes that are used as base classes for real -buildsteps. This section describes the base classes. The "leaf" classes are -described in :doc:`../manual/cfg-buildsteps`. +There are a few parent classes that are used as base classes for real buildsteps. +This section describes the base classes. The "leaf" classes are described in :doc:`../manual/cfg-buildsteps`. BuildStep --------- .. py:class:: BuildStep(name, locks, haltOnFailure, flunkOnWarnings, flunkOnFailure, warnOnWarnings, warnOnFailure, alwaysRun, progressMetrics, useProgress, doStepIf, hideStepIf) - All constructor arguments must be given as keyword arguments. Each - constructor parameter is copied to the corresponding attribute. + All constructor arguments must be given as keyword arguments. + Each constructor parameter is copied to the corresponding attribute. .. py:attribute:: name @@ -25,14 +24,13 @@ BuildStep .. py:attribute:: progressMetrics - List of names of metrics that should be used to track the progress of - this build, and build ETA's for users. This is generally set in the + List of names of metrics that should be used to track the progress of this build, and build ETA's for users. + This is generally set in the .. py:attribute:: useProgress - If true (the default), then ETAs will be calculated for this step using - progress metrics. If the step is known to have unpredictable timing - (e.g., an incremental build), then this should be set to false. + If true (the default), then ETAs will be calculated for this step using progress metrics. + If the step is known to have unpredictable timing (e.g., an incremental build), then this should be set to false. .. py:attribute:: doStepIf @@ -41,20 +39,18 @@ BuildStep .. py:attribute:: hideStepIf - A callable or bool to determine whether this step should be shown in the - waterfall and build details pages. See :ref:`Buildstep-Common-Parameters` for details. + A callable or bool to determine whether this step should be shown in the waterfall and build details pages. + See :ref:`Buildstep-Common-Parameters` for details. The following attributes affect the behavior of the containing build: .. py:attribute:: haltOnFailure - If true, the build will halt on a failure of this step, and not execute - subsequent tests (except those with ``alwaysRun``). + If true, the build will halt on a failure of this step, and not execute subsequent tests (except those with ``alwaysRun``). .. py:attribute:: flunkOnWarnings - If true, the build will be marked as a failure if this step ends with - warnings. + If true, the build will be marked as a failure if this step ends with warnings. .. py:attribute:: flunkOnFailure @@ -62,48 +58,26 @@ BuildStep .. py:attribute:: warnOnWarnings - If true, the build will be marked as warnings, or worse, if this step - ends with warnings. + If true, the build will be marked as warnings, or worse, if this step ends with warnings. .. py:attribute:: warnOnFailure - If true, the build will be marked as warnings, or worse, if this step - fails. + If true, the build will be marked as warnings, or worse, if this step fails. .. py:attribute:: alwaysRun - If true, the step will run even if a previous step halts the build with - ``haltOnFailure``. - - A step acts as a factory for more steps. See - :ref:`Writing-BuildStep-Constructors` for advice on writing subclass - constructors. The following methods handle this factory behavior. - - .. py:method:: addFactoryArguments(..) - - Add the given keyword arguments to the arguments used to create new - step instances; - - .. py:method:: getStepFactory() - - :returns: tuple of (class, keyword arguments) - - Get a factory for new instances of this step. The step can be created - by calling the class with the given keyword arguments. + If true, the step will run even if a previous step halts the build with ``haltOnFailure``. - A few important pieces of information are not available when a step is - constructed, and are added later. These are set by the following methods; - the order in which these methods are called is not defined. + A few important pieces of information are not available when a step is constructed, and are added later. + These are set by the following methods; the order in which these methods are called is not defined. .. py:method:: setBuild(build) - :param build: the :class:`~buildbot.process.build.Build` instance - controlling this step. + :param build: the :class:`~buildbot.process.build.Build` instance controlling this step. - This method is called during setup to set the build instance - controlling this slave. Subclasses can override this to get access to - the build object as soon as it is available. The default - implementation sets the :attr:`build` attribute. + This method is called during setup to set the build instance controlling this slave. + Subclasses can override this to get access to the build object as soon as it is available. + The default implementation sets the :attr:`build` attribute. .. py:attribute:: build @@ -111,12 +85,10 @@ BuildStep .. py:method:: setBuildSlave(build) - :param build: the :class:`~buildbot.buildslave.BuildSlave` instance on - which this step will run. + :param build: the :class:`~buildbot.buildslave.BuildSlave` instance on which this step will run. - Similarly, this method is called with the build slave that will run - this step. The default implementation sets the :attr:`buildslave` - attribute. + Similarly, this method is called with the build slave that will run this step. + The default implementation sets the :attr:`buildslave` attribute. .. py:attribute:: buildslave @@ -126,36 +98,30 @@ BuildStep :param workdir: the default workdir, from the build - This method is called at build startup with the default workdir for the - build. Steps which allow a workdir to be specified, but want to - override it with the build's default workdir, can use this method to - apply the default. + This method is called at build startup with the default workdir for the build. + Steps which allow a workdir to be specified, but want to override it with the build's default workdir, can use this method to apply the default. .. py:method:: setStepStatus(status) :param status: step status :type status: :class:`~buildbot.status.buildstep.BuildStepStatus` - This method is called to set the status instance to which the step - should report. The default implementation sets :attr:`step_status`. + This method is called to set the status instance to which the step should report. + The default implementation sets :attr:`step_status`. .. py:attribute:: step_status - The :class:`~buildbot.status.buildstep.BuildStepStatus` object tracking - the status of this step. + The :class:`~buildbot.status.buildstep.BuildStepStatus` object tracking the status of this step. .. py:method:: setupProgress() - This method is called during build setup to give the step a chance to - set up progress tracking. It is only called if the build has - :attr:`useProgress` set. There is rarely any reason to override this - method. + This method is called during build setup to give the step a chance to set up progress tracking. + It is only called if the build has :attr:`useProgress` set. + There is rarely any reason to override this method. .. py:attribute:: progress - If the step is tracking progress, this is a - :class:`~buildbot.status.progress.StepProgress` instance performing - that task. + If the step is tracking progress, this is a :class:`~buildbot.status.progress.StepProgress` instance performing that task. Exeuction of the step itself is governed by the following methods and attributes. @@ -168,34 +134,29 @@ BuildStep Begin the step. This is the build's interface to step execution. Subclasses should override :meth:`start` to implement custom behaviors. - The method returns a Deferred that fires when the step finishes. It - fires with a tuple of ``(result, [extra text])``, where ``result`` is - one of the constants from :mod:`buildbot.status.builder`. The extra - text is a list of short strings which should be appended to the Build's - text results. For example, a test step may add ``17 failures`` to the - Build's status by this mechanism. + The method returns a Deferred that fires when the step finishes. + It fires with a tuple of ``(result, [extra text])``, where ``result`` is one of the constants from :mod:`buildbot.status.builder`. + The extra text is a list of short strings which should be appended to the Build's text results. + For example, a test step may add ``17 failures`` to the Build's status by this mechanism. - The deferred will errback if the step encounters an exception, - including an exception on the slave side (or if the slave goes away - altogether). Normal build/test failures will *not* cause an errback. + The deferred will errback if the step encounters an exception, including an exception on the slave side (or if the slave goes away altogether). + Normal build/test failures will *not* cause an errback. .. py:method:: start() :returns: ``None`` or :data:`~buildbot.status.results.SKIPPED`, optionally via a Deferred. - Begin the step. Subclasses should override this method to do local - processing, fire off remote commands, etc. The parent method raises - :exc:`NotImplementedError`. + Begin the step. + Subclasses should override this method to do local processing, fire off remote commands, etc. + The parent method raises :exc:`NotImplementedError`. - When the step is done, it should call :meth:`finished`, with a result - -- a constant from :mod:`buildbot.status.results`. The result will be - handed off to the :class:`~buildbot.process.build.Build`. + When the step is done, it should call :meth:`finished`, with a result -- a constant from :mod:`buildbot.status.results`. + The result will be handed off to the :class:`~buildbot.process.build.Build`. - If the step encounters an exception, it should call :meth:`failed` with - a Failure object. This method automatically fails the whole build with - an exception. A common idiom is to add :meth:`failed` as an errback on - a Deferred:: + If the step encounters an exception, it should call :meth:`failed` with a Failure object. + This method automatically fails the whole build with an exception. + A common idiom is to add :meth:`failed` as an errback on a Deferred:: cmd = RemoteCommand(args) d = self.runCommand(cmd) @@ -204,78 +165,61 @@ BuildStep d.addCallback(succeed) d.addErrback(self.failed) - If the step decides it does not need to be run, :meth:`start` can - return the constant :data:`~buildbot.status.results.SKIPPED`. In this - case, it is not necessary to call :meth:`finished` directly. + If the step decides it does not need to be run, :meth:`start` can return the constant :data:`~buildbot.status.results.SKIPPED`. + In this case, it is not necessary to call :meth:`finished` directly. .. py:method:: finished(results) :param results: a constant from :mod:`~buildbot.status.results` - A call to this method indicates that the step is finished and the build - should analyze the results and perhaps proceed to the next step. The - step should not perform any additional processing after calling this - method. + A call to this method indicates that the step is finished and the build should analyze the results and perhaps proceed to the next step. + The step should not perform any additional processing after calling this method. .. py:method:: failed(failure) :param failure: a :class:`~twisted.python.failure.Failure` instance - Similar to :meth:`finished`, this method indicates that the step is - finished, but handles exceptions with appropriate logging and - diagnostics. + Similar to :meth:`finished`, this method indicates that the step is finished, but handles exceptions with appropriate logging and diagnostics. - This method handles :exc:`BuildStepFailed` specially, by calling - ``finished(FAILURE)``. This provides subclasses with a shortcut to - stop execution of a step by raising this failure in a context where - :meth:`failed` will catch it. + This method handles :exc:`BuildStepFailed` specially, by calling ``finished(FAILURE)``. + This provides subclasses with a shortcut to stop execution of a step by raising this failure in a context where :meth:`failed` will catch it. .. py:method:: interrupt(reason) :param reason: why the build was interrupted :type reason: string or :class:`~twisted.python.failure.Failure` - This method is used from various control interfaces to stop a running - step. The step should be brought to a halt as quickly as possible, by - cancelling a remote command, killing a local process, etc. The step - must still finish with either :meth:`finished` or :meth:`failed`. + This method is used from various control interfaces to stop a running step. + The step should be brought to a halt as quickly as possible, by cancelling a remote command, killing a local process, etc. + The step must still finish with either :meth:`finished` or :meth:`failed`. - The ``reason`` parameter can be a string or, when a slave is lost - during step processing, a :exc:`~twisted.internet.error.ConnectionLost` - failure. + The ``reason`` parameter can be a string or, when a slave is lost during step processing, a :exc:`~twisted.internet.error.ConnectionLost` failure. - The parent method handles any pending lock operations, and should be - called by implementations in subclasses. + The parent method handles any pending lock operations, and should be called by implementations in subclasses. .. py:attribute:: stopped - If false, then the step is running. If true, the step is not running, - or has been interrupted. + If false, then the step is running. If true, the step is not running, or has been interrupted. - This method provides a convenient way to summarize the status of the step - for status displays: + This method provides a convenient way to summarize the status of the step for status displays: .. py:method:: describe(done=False) :param done: If true, the step is finished. :returns: list of strings - Describe the step succinctly. The return value should be a sequence of - short strings suitable for display in a horizontally constrained space. + Describe the step succinctly. + The return value should be a sequence of short strings suitable for display in a horizontally constrained space. .. note:: - Be careful not to assume that the step has been started in this - method. In relatively rare circumstances, steps are described - before they have started. Ideally, unit tests should be used to - ensure that this method is resilient. + Be careful not to assume that the step has been started in this method. + In relatively rare circumstances, steps are described before they have started. + Ideally, unit tests should be used to ensure that this method is resilient. - Build steps support progress metrics - values that increase roughly - linearly during the execution of the step, and can thus be used to - calculate an expected completion time for a running step. A metric may be - a count of lines logged, tests executed, or files compiled. The build - mechanics will take care of translating this progress information into an - ETA for the user. + Build steps support progress metrics - values that increase roughly linearly during the execution of the step, and can thus be used to calculate an expected completion time for a running step. + A metric may be a count of lines logged, tests executed, or files compiled. + The build mechanics will take care of translating this progress information into an ETA for the user. .. py:method:: setProgress(metric, value) @@ -284,13 +228,13 @@ BuildStep :param value: the new value for the metric :type value: integer - Update a progress metric. This should be called by subclasses that can - provide useful progress-tracking information. + Update a progress metric. + This should be called by subclasses that can provide useful progress-tracking information. The specified metric name must be included in :attr:`progressMetrics`. - The following methods are provided as utilities to subclasses. These - methods should only be invoked after the step is started. + The following methods are provided as utilities to subclasses. + These methods should only be invoked after the step is started. .. py:method:: slaveVersion(command, oldVersion=None) @@ -299,19 +243,14 @@ BuildStep :param oldVersion: return value if the slave does not specify a version :returns: string - Fetch the version of the named command, as specified on the slave. In - practice, all commands on a slave have the same version, but passing - ``command`` is still useful to ensure that the command is implemented - on the slave. If the command is not implemented on the slave, - :meth:`slaveVersion` will return ``None``. + Fetch the version of the named command, as specified on the slave. + In practice, all commands on a slave have the same version, but passing ``command`` is still useful to ensure that the command is implemented on the slave. + If the command is not implemented on the slave, :meth:`slaveVersion` will return ``None``. - Versions take the form ``x.y`` where ``x`` and ``y`` are integers, and - are compared as expected for version numbers. + Versions take the form ``x.y`` where ``x`` and ``y`` are integers, and are compared as expected for version numbers. - Buildbot versions older than 0.5.0 did not support version queries; in - this case, :meth:`slaveVersion` will return ``oldVersion``. Since such - ancient versions of Buildbot are no longer in use, this functionality - is largely vestigial. + Buildbot versions older than 0.5.0 did not support version queries; in this case, :meth:`slaveVersion` will return ``oldVersion``. + Since such ancient versions of Buildbot are no longer in use, this functionality is largely vestigial. .. py:method:: slaveVersionIsOlderThan(command, minversion) @@ -320,8 +259,7 @@ BuildStep :param minversion: minimum version :returns: boolean - This method returns true if ``command`` is not implemented on the - slave, or if it is older than ``minversion``. + This method returns true if ``command`` is not implemented on the slave, or if it is older than ``minversion``. .. py:method:: getSlaveName() @@ -333,31 +271,26 @@ BuildStep :returns: Deferred - This method connects the given command to the step's buildslave and - runs it, returning the Deferred from - :meth:`~buildbot.process.buildstep.RemoteCommand.run`. + This method connects the given command to the step's buildslave and runs it, returning the Deferred from :meth:`~buildbot.process.buildstep.RemoteCommand.run`. .. py:method:: addURL(name, url) :param name: URL name :param url: the URL - Add a link to the given ``url``, with the given ``name`` to displays of - this step. This allows a step to provide links to data that is not - available in the log files. + Add a link to the given ``url``, with the given ``name`` to displays of this step. + This allows a step to provide links to data that is not available in the log files. - The :class:`BuildStep` class provides minimal support for log handling, - that is extended by the :class:`LoggingBuildStep` class. The following - methods provide some useful behaviors. These methods can be called while - the step is running, but not before. + The :class:`BuildStep` class provides minimal support for log handling, that is extended by the :class:`LoggingBuildStep` class. + The following methods provide some useful behaviors. + These methods can be called while the step is running, but not before. .. py:method:: addLog(name) :param name: log name :returns: :class:`~buildbot.status.logfile.LogFile` instance - Add a new logfile with the given name to the step, and return the log - file instance. + Add a new logfile with the given name to the step, and return the log file instance. .. py:method:: getLog(name) @@ -372,27 +305,24 @@ BuildStep :param name: log name :param text: content of the logfile - This method adds a new log and sets ``text`` as its content. This is - often useful to add a short logfile describing activities performed on - the master. The logfile is immediately closed, and no further data can - be added. + This method adds a new log and sets ``text`` as its content. + This is often useful to add a short logfile describing activities performed on the master. + The logfile is immediately closed, and no further data can be added. .. py:method:: addHTMLLog(name, html) :param name: log name :param html: content of the logfile - Similar to :meth:`addCompleteLog`, this adds a logfile containing - pre-formatted HTML, allowing more expressiveness than the text format - supported by :meth:`addCompleteLog`. + Similar to :meth:`addCompleteLog`, this adds a logfile containing pre-formatted HTML, allowing more expressiveness than the text format supported by :meth:`addCompleteLog`. .. py:method:: addLogObserver(logname, observer) :param logname: log name :param observer: log observer instance - Add a log observer for the named log. The named log need not have been - added already: the observer will be connected when the log is added. + Add a log observer for the named log. + The named log need not have been added already: the observer will be connected when the log is added. See :ref:`Adding-LogObservers` for more information on log observers. @@ -407,29 +337,25 @@ LoggingBuildStep The remaining arguments are passed to the :class:`BuildStep` constructor. - This subclass of :class:`BuildStep` is designed to help its subclasses run - remote commands that produce standard I/O logfiles. It: + This subclass of :class:`BuildStep` is designed to help its subclasses run remote commands that produce standard I/O logfiles. + It: * tracks progress using the length of the stdout logfile * provides hooks for summarizing and evaluating the command's result * supports lazy logfiles - * handles the mechanics of starting, interrupting, and finishing remote - commands + * handles the mechanics of starting, interrupting, and finishing remote commands * detects lost slaves and finishes with a status of :data:`~buildbot.status.results.RETRY` .. py:attribute:: logfiles - The logfiles to track, as described for :bb:step:`ShellCommand`. The - contents of the class-level ``logfiles`` attribute are combined with - those passed to the constructor, so subclasses may add log files with a - class attribute:: + The logfiles to track, as described for :bb:step:`ShellCommand`. + The contents of the class-level ``logfiles`` attribute are combined with those passed to the constructor, so subclasses may add log files with a class attribute:: class MyStep(LoggingBuildStep): logfiles = dict(debug='debug.log') - Note that lazy logfiles cannot be specified using this method; they - must be provided as constructor arguments. + Note that lazy logfiles cannot be specified using this method; they must be provided as constructor arguments. .. py:method:: startCommand(command) @@ -438,60 +364,47 @@ LoggingBuildStep .. note:: - This method permits an optional ``errorMessages`` parameter, - allowing errors detected early in the command process to be logged. + This method permits an optional ``errorMessages`` parameter, allowing errors detected early in the command process to be logged. It will be removed, and its use is deprecated. - Handle all of the mechanics of running the given command. This sets - up all required logfiles, keeps status text up to date, and calls the - utility hooks described below. When the command is finished, the step - is finished as well, making this class is unsuitable for steps that - run more than one command in sequence. + Handle all of the mechanics of running the given command. + This sets up all required logfiles, keeps status text up to date, and calls the utility hooks described below. + When the command is finished, the step is finished as well, making this class is unsuitable for steps that run more than one command in sequence. - Subclasses should override - :meth:`~buildbot.process.buildstep.BuildStep.start` and, after setting - up an appropriate command, call this method. :: + Subclasses should override :meth:`~buildbot.process.buildstep.BuildStep.start` and, after setting up an appropriate command, call this method. :: def start(self): cmd = RemoteShellCommand(..) self.startCommand(cmd, warnings) To refine the status output, override one or more of the following methods. - The :class:`LoggingBuildStep` implementations are stubs, so there is no - need to call the parent method. + The :class:`LoggingBuildStep` implementations are stubs, so there is no need to call the parent method. .. py:method:: commandComplete(command) :param command: the just-completed remote command - This is a general-purpose hook method for subclasses. It will be called - after the remote command has finished, but before any of the other hook - functions are called. + This is a general-purpose hook method for subclasses. + It will be called after the remote command has finished, but before any of the other hook functions are called. .. py:method:: createSummary(stdio) :param stdio: stdio :class:`~buildbot.status.logfile.LogFile` - This hook is designed to perform any summarization of the step, based - either on the contents of the stdio logfile, or on instance attributes - set earlier in the step processing. Implementations of this method - often call e.g., :meth:`~BuildStep.addURL`. + This hook is designed to perform any summarization of the step, based either on the contents of the stdio logfile, or on instance attributes set earlier in the step processing. + Implementations of this method often call e.g., :meth:`~BuildStep.addURL`. .. py:method:: evaluateCommand(command) :param command: the just-completed remote command :returns: step result from :mod:`buildbot.status.results` - This hook should decide what result the step should have. The default - implementation invokes ``log_eval_func`` if it exists, and looks at - :attr:`~buildbot.process.buildstep.RemoteCommand.rc` to distinguish - :data:`~buildbot.status.results.SUCCESS` from - :data:`~buildbot.status.results.FAILURE`. + This hook should decide what result the step should have. + The default implementation invokes ``log_eval_func`` if it exists, and looks at :attr:`~buildbot.process.buildstep.RemoteCommand.rc` to distinguish :data:`~buildbot.status.results.SUCCESS` from :data:`~buildbot.status.results.FAILURE`. - The remaining methods provide an embarassment of ways to set the summary of - the step that appears in the various status interfaces. The easiest way to - affect this output is to override :meth:`~BuildStep.describe`. If that is - not flexible enough, override :meth:`getText` and/or :meth:`getText2`. + The remaining methods provide an embarassment of ways to set the summary of the step that appears in the various status interfaces. + The easiest way to affect this output is to override :meth:`~BuildStep.describe`. + If that is not flexible enough, override :meth:`getText` and/or :meth:`getText2`. .. py:method:: getText(command, results) @@ -499,10 +412,8 @@ LoggingBuildStep :param results: step result from :meth:`evaluateCommand` :returns: a list of short strings - This method is the primary means of describing the step. The default - implementation calls :meth:`~BuildStep.describe`, which is usally the - easiest method to override, and then appends a string describing the - step status if it was not successful. + This method is the primary means of describing the step. + The default implementation calls :meth:`~BuildStep.describe`, which is usally the easiest method to override, and then appends a string describing the step status if it was not successful. .. py:method:: getText2(command, results) @@ -510,15 +421,13 @@ LoggingBuildStep :param results: step result from :meth:`evaluateCommand` :returns: a list of short strings - Like :meth:`getText`, this method summarizes the step's result, but it - is only called when that result affects the build, either by making it - halt, flunk, or end with warnings. + Like :meth:`getText`, this method summarizes the step's result, but it is only called when that result affects the build, either by making it halt, flunk, or end with warnings. Exceptions ---------- .. py:exception:: BuildStepFailed - This exception indicates that the buildstep has failed. It is useful as a - way to skip all subsequent processing when a step goes wrong. It is - handled by :meth:`BuildStep.failed`. + This exception indicates that the buildstep has failed. + It is useful as a way to skip all subsequent processing when a step goes wrong. + It is handled by :meth:`BuildStep.failed`. diff --git a/master/docs/developer/cls-remotecommands.rst b/master/docs/developer/cls-remotecommands.rst index 00565279b31..c89bb195515 100644 --- a/master/docs/developer/cls-remotecommands.rst +++ b/master/docs/developer/cls-remotecommands.rst @@ -13,7 +13,7 @@ detail in :ref:`master-slave-updates`. RemoteCommand ~~~~~~~~~~~~~ -.. py:class:: RemoteCommand(remote_command, args, collectStdout=False, ignore_updates=False) +.. py:class:: RemoteCommand(remote_command, args, collectStdout=False, ignore_updates=False, successfulRC=tuple(0)) :param remote_command: command to run on the slave :type remote_command: string @@ -21,6 +21,7 @@ RemoteCommand :type args: dictionary :param collectStdout: if True, collect the command's stdout :param ignore_updates: true to ignore remote updates + :param successfulRC: list or tuple of ``rc`` values to treat as successes This class handles running commands, consisting of a command name and a dictionary of arguments. If true, ``ignore_updates`` will suppress any @@ -61,6 +62,12 @@ RemoteCommand slave; this may be a long time before the command itself completes, at which time the Deferred returned from :meth:`run` will fire. + .. py:method:: didFail() + + :returns: bool + + This method checks the ``rc`` against the list of successful exit statuses, and returns ``True`` if it is not in the list. + The following methods are invoked from the slave. They should not be called directly. diff --git a/master/docs/developer/style.rst b/master/docs/developer/style.rst index 7850833fc3c..bccd450b17d 100644 --- a/master/docs/developer/style.rst +++ b/master/docs/developer/style.rst @@ -9,7 +9,7 @@ the formatting of symbol names. The single exception in naming of functions and methods. Because Buildbot uses Twisted so heavily, and Twisted uses interCaps, Buildbot methods should do the -same. That is, methods and functions should be spelled with the first character +same. That is, you should spell methods and functions with the first character in lower-case, and the first letter of subsequent words capitalized, e.g., ``compareToOther`` or ``getChangesGreaterThan``. This point is not applied very consistently in Buildbot, but let's try to be consistent in new code. @@ -30,8 +30,8 @@ Just about anything might block - even getters and setters! Helpful Twisted Classes ~~~~~~~~~~~~~~~~~~~~~~~ -Twisted has some useful, but little-known classes. They are listed here with -brief descriptions, but you should consult the API documentation or source code +Twisted has some useful, but little-known classes. +Brief descriptions follow, but you should consult the API documentation or source code for the full details. :class:`twisted.internet.task.LoopingCall` @@ -39,8 +39,8 @@ for the full details. :class:`twisted.application.internet.TimerService` Similar to ``t.i.t.LoopingCall``, but implemented as a service that will - automatically start and stop the function calls when the service is started and - stopped. + automatically start and stop the function calls when the service starts and + stops. Sequences of Operations ~~~~~~~~~~~~~~~~~~~~~~~ @@ -134,11 +134,11 @@ The key points to notice here: * Use the decorator form of ``inlineCallbacks`` * In most cases, the result of a ``yield`` expression should be assigned to a variable. It can be used in a larger expression, but remember that Python - requires that it be enclosed in its own set of parentheses. + requires that you enclose the expression in its own set of parentheses. * Python does not permit returning a value from a generator, so statements like ``return xval + y`` are invalid. Instead, yield the result of ``defer.returnValue``. Although this function does cause an immediate - function exit, for clarity it should be followed by a bare ``return``, as in + function exit, for clarity follow it with a bare ``return``, as in the example, unless it is the last statement in a function. The great advantage of ``inlineCallbacks`` is that it allows you to use all @@ -157,7 +157,7 @@ operations, each time you wait for a Deferred, arbitrary other actions can take place. In general, you should try to perform actions atomically, but for the rare -times synchronization is required, the following might be useful: +situations that require synchronization, the following might be useful: * :py:class:`twisted.internet.defer.DeferredLock` * :py:func:`buildbot.util.misc.deferredLocked` @@ -168,8 +168,8 @@ Joining Sequences It's often the case that you'll want to perform multiple operations in parallel, and re-join the results at the end. For this purpose, you'll want to -use a `DeferredList -`_:: +use a `DeferredList `_ +:: def getRevInfo(revname): results = {} @@ -192,5 +192,5 @@ use a `DeferredList return d Here the deferred list will wait for both ``rev_parse_d`` and ``log_d`` to -fire, or for one of them to fail. Callbacks and errbacks can be attached to a +fire, or for one of them to fail. You may attach Callbacks and errbacks to a ``DeferredList`` just as for a deferred. diff --git a/master/docs/manual/cfg-buildslaves.rst b/master/docs/manual/cfg-buildslaves.rst index 969801398ab..37db52bd7b6 100644 --- a/master/docs/manual/cfg-buildslaves.rst +++ b/master/docs/manual/cfg-buildslaves.rst @@ -154,7 +154,19 @@ very useful for some situations. The buildslaves that are started on-demand are called "latent" buildslaves. As of this writing, buildbot ships with an abstract base class for building -latent buildslaves, and a concrete implementation for AWS EC2. +latent buildslaves, and a concrete implementation for AWS EC2 and for libvirt. + +Common Options +++++++++++++++ + +The following options are available for all latent buildslaves. + +``build_wait_timeout`` + This option allows you to specify how long a latent slave should wait after + a build for another build before it shuts down. It defaults to 10 minutes. + If this is set to 0 then the slave will be shut down immediately. If it is + less than 0 it will never automatically shutdown. + .. index:: AWS EC2 @@ -344,10 +356,6 @@ to wait for an EC2 instance to attach before considering the attempt to have failed, and email addresses to alert, respectively. ``missing_timeout`` defaults to 20 minutes. -The ``build_wait_timeout`` allows you to specify how long an :class:`EC2LatentBuildSlave` -should wait after a build for another build before it shuts down the EC2 -instance. It defaults to 10 minutes. - ``keypair_name`` and ``security_name`` allow you to specify different names for these AWS EC2 values. They both default to ``latent_buildbot_slave``. diff --git a/master/docs/manual/cfg-buildsteps.rst b/master/docs/manual/cfg-buildsteps.rst index ba82d827f7d..e8f2f4a14d3 100644 --- a/master/docs/manual/cfg-buildsteps.rst +++ b/master/docs/manual/cfg-buildsteps.rst @@ -402,6 +402,38 @@ The Git step takes the following arguments: performs all the incremental checkout behavior in ``source`` directory. +``getDescription`` + + (optional) After checkout, invoke a `git describe` on the revision and save + the result in a property; the property's name is either ``commit-description`` + or ``commit-description-foo``, depending on whether the ``codebase`` + argument was also provided. The argument should either be a ``bool`` or ``dict``, + and will change how `git describe` is called: + + * ``getDescription=False``: disables this feature explicitly + * ``getDescription=True`` or empty ``dict()``: Run `git describe` with no args + * ``getDescription={...}``: a dict with keys named the same as the git option. + Each key's value can be ``False`` or ``None`` to explicitly skip that argument. + + For the following keys, a value of ``True`` appends the same-named git argument: + + * ``all`` : `--all` + * ``always``: `--always` + * ``contains``: `--contains` + * ``debug``: `--debug` + * ``long``: `--long`` + * ``exact-match``: `--exact-match` + * ``tags``: `--tags` + * ``dirty``: `--dirty` + + For the following keys, an integer or string value (depending on what git expects) + will set the argument's parameter appropriately. Examples show the key-value pair: + + * ``match=foo``: `--match foo` + * ``abbrev=7``: `--abbrev=7` + * ``candidates=7``: `--candidates=7` + * ``dirty=foo``: `--dirty=foo` + .. bb:step:: SVN .. _Step-SVN: @@ -1177,6 +1209,10 @@ The Repo step takes the following arguments: directory which contains all the git objects. This feature helps to minimize network usage on very big projects. +``jobs`` + (optional, defaults to ``None``): Number of projects to fetch + simultaneously while syncing. Passed to repo sync subcommand with "-j". + This Source step integrates with :bb:chsrc:`GerritChangeSource`, and will automatically use the :command:`repo download` command of repo to download the additionnal changes introduced by a pending changeset. @@ -1436,6 +1472,17 @@ The :bb:step:`ShellCommand` arguments are: description=["testing"], descriptionDone=["tests"])) +``descriptionSuffix`` + This is an optional suffix appended to the end of the description (ie, + after ``description`` and ``descriptionDone``). This can be used to distinguish + between build steps that would display the same descriptions in the waterfall. + This parameter may be set to list of short strings, a single string, or ``None``. + + For example, a builder might use the ``Compile`` step to build two different + codebases. The ``descriptionSuffix`` could be set to `projectFoo` and `projectBar`, + respectively for each step, which will result in the full descriptions + `compiling projectFoo` and `compiling projectBar` to be shown in the waterfall. + ``logEnviron`` If this option is ``True`` (the default), then the step's logfile will describe the environment variables on the slave. In situations where the environment is not @@ -1453,6 +1500,10 @@ The :bb:step:`ShellCommand` arguments are: handled as a single string throughout Buildbot -- for example, do not pass the contents of a tarball with this parameter. +``successfulRC`` + This is a list or tuple of the exit codes that should be treated as successful. + The default is to treat just 0 as successful. + .. bb:step:: Configure Configure @@ -2300,6 +2351,9 @@ Note that environment values must be strings (or lists that are turned into strings). In particular, numeric properties such as ``buildnumber`` must be substituted using :ref:`WithProperties`. +``interruptSignal`` + (optional) Signal to use to end the process, if the step is interrupted. + .. index:: Properties; from steps .. _Setting-Properties: diff --git a/master/docs/manual/cfg-changesources.rst b/master/docs/manual/cfg-changesources.rst index ec0b134a355..836f80dad43 100644 --- a/master/docs/manual/cfg-changesources.rst +++ b/master/docs/manual/cfg-changesources.rst @@ -511,6 +511,16 @@ hostname/portnumber as appropriate for your buildbot: master = buildmaster.example.org:9987 # .. other hgbuildbot parameters .. +The ``master`` configuration key allows to have more than one buildmaster +specification. The buildmasters have to be separated by a whitspace +or comma (see also 'hg help config'): + +.. code-block:: ini + + master = + buildmaster.example.org:9987 + buildmaster2.example.org:9989 + .. note:: Mercurial lets you define multiple ``changegroup`` hooks by giving them distinct names, like ``changegroup.foo`` and ``changegroup.bar``, which is why we use ``changegroup.buildbot`` diff --git a/master/docs/manual/cfg-global.rst b/master/docs/manual/cfg-global.rst index 48c80a1e21f..ab2d2b3c261 100644 --- a/master/docs/manual/cfg-global.rst +++ b/master/docs/manual/cfg-global.rst @@ -704,9 +704,9 @@ an URL to the revision. Note that the revision id may not always be in the form you expect, so code defensively. In particular, a revision of "??" may be supplied when no other information is available. -Note that :class:`SourceStamp`\s that are not created from version-control changes (e.g., -those created by a Nightly or Periodic scheduler) will have an empty repository -string, as the respository is not known. +Note that :class:`SourceStamp`\s that are not created from version-control +changes (e.g., those created by a Nightly or Periodic scheduler) may have an +empty repository string, if the respository is not known to the scheduler. Revision Link Helpers +++++++++++++++++++++ @@ -737,20 +737,18 @@ Codebase Generator :: + all_repositories = { + r'https://hg/hg/mailsuite/mailclient': 'mailexe', + r'https://hg/hg/mailsuite/mapilib': 'mapilib', + r'https://hg/hg/mailsuite/imaplib': 'imaplib', + r'https://github.com/mailinc/mailsuite/mailclient': 'mailexe', + r'https://github.com/mailinc/mailsuite/mapilib': 'mapilib', + r'https://github.com/mailinc/mailsuite/imaplib': 'imaplib', + } + def codebaseGenerator(chdict): - all_repositories = { - r'https://hg/hg/mailsuite/mailclient': 'mailexe', - r'https://hg/hg/mailsuite/mapilib': 'mapilib', - r'https://hg/hg/mailsuite/imaplib': 'imaplib', - r'https://github.com/mailinc/mailsuite/mailclient': 'mailexe', - r'https://github.com/mailinc/mailsuite/mapilib': 'mapilib', - r'https://github.com/mailinc/mailsuite/imaplib': 'imaplib', - } - if chdict['repository'] in all_repositories: - return all_repositories[chdict['repository']] - else: - return '' - + return all_repositories[chdict['repository']] + c['codebaseGenerator'] = codebaseGenerator For any incomming change a :ref:`codebase` is set to ''. This diff --git a/master/docs/manual/cfg-properties.rst b/master/docs/manual/cfg-properties.rst index 4c97f23316a..1219bf52476 100644 --- a/master/docs/manual/cfg-properties.rst +++ b/master/docs/manual/cfg-properties.rst @@ -248,6 +248,17 @@ syntaxes in the parentheses. If ``propname`` exists, substitute ``replacement``; otherwise, substitute an empty string. +``propname:?:sub_if_true:sub_if_false`` + +``propname:#?:sub_if_exists:sub_if_missing`` + Ternary substitution, depending on either ``propname`` being ``True`` (with + ``:?``, similar to ``:~``) or being present (with ``:#?``, like ``:+``). + Notice that there is a colon immediately following the question mark *and* + between the two substitution alternatives. The character that follows the + question mark is used as the delimeter between the two alternatives. In the + above examples, it is a colon, but any single character can be used. + + Although these are similar to shell substitutions, no other substitutions are currently supported, and ``replacement`` in the above cannot contain more substitutions. diff --git a/master/docs/manual/cfg-schedulers.rst b/master/docs/manual/cfg-schedulers.rst index 7c3a9c6f828..fe9a685a6c6 100644 --- a/master/docs/manual/cfg-schedulers.rst +++ b/master/docs/manual/cfg-schedulers.rst @@ -73,14 +73,21 @@ available with all schedulers. When the scheduler processes data from more than 1 repository at the same time then a corresponding codebase definition should be passed for each repository. A codebase definition is a dictionary with one or more of the - following keys: repository, branch, revision. The codebase definitions are - also to be passed as dictionary:: - - codebases = {'codebase1': {'repository':'....', ...}, 'codebase2': {} } + following keys: repository, branch, revision. The codebase definitions have + also to be passed as dictionary. + + .. code-block:: python + + codebases = {'codebase1': {'repository':'....', + 'branch':'default', + 'revision': None}, + 'codebase2': {'repository':'....'} } .. IMPORTANT:: ``codebases`` behaves also like a change_filter on codebase. - If ``codebases`` is set then the scheduler will only process changes when their - codebases are found in ``codebases`` + The scheduler will only process changes when their codebases are found + in ``codebases``. By default ``codebases`` is set to ``{'':{}}`` which + means that only changes with codebase '' (default value for codebase) + will be accepted by the scheduler. Buildsteps can have a reference to one of the codebases. The step will only get information (revision, branch etc.) that is related to that codebase. diff --git a/master/docs/manual/cfg-statustargets.rst b/master/docs/manual/cfg-statustargets.rst index b78a4720074..95f01b6536e 100644 --- a/master/docs/manual/cfg-statustargets.rst +++ b/master/docs/manual/cfg-statustargets.rst @@ -57,11 +57,8 @@ server and retrieve information about every build the buildbot knows about, as well as find out what the buildbot is currently working on. The first page you will see is the *Welcome Page*, which contains -links to all the other useful pages. By default, this page is served from -the :file:`status/web/templates/root.html` file in buildbot's library area. -If you'd like to override this page or the other templates found there, -copy the files you're interested in into a :file:`templates/` directory in -the buildmaster's base directory. +links to all the other useful pages. By default, this page is served from the +:file:`status/web/templates/root.html` file in buildbot's library area. One of the most complex resource provided by :class:`WebStatus` is the *Waterfall Display*, which shows a time-based chart of events. This @@ -86,25 +83,39 @@ in great detail below. Configuration +++++++++++++ -Buildbot now uses a templating system for the web interface. The source +The simplest possible configuration for WebStatus is:: + + from buildbot.status.html import WebStatus + c['status'].append(WebStatus(8080)) + +Buildbot uses a templating system for the web interface. The source of these templates can be found in the :file:`status/web/templates/` directory in buildbot's library area. You can override these templates by creating alternate versions in a :file:`templates/` directory within the buildmaster's base directory. +If that isn't enough you can also provide additional Jinja2 template loaders:: + + import jinja2 + myloaders = [ + jinja2.FileSystemLoader("/tmp/mypath"), + ] + + c['status'].append(html.WebStatus( + …, + jinja_loaders = myloaders, + )) + The first time a buildmaster is created, the :file:`public_html/` directory is populated with some sample files, which you will probably want to customize for your own project. These files are all static: the buildbot does not modify them in any way as it serves them to HTTP clients. -Note that templates in :file:`templates/` take precedence over static files in -:file:`public_html/`. :: +Templates in :file:`templates/` take precedence over static files in +:file:`public_html/`. - from buildbot.status.html import WebStatus - c['status'].append(WebStatus(8080)) - -Note that the initial :file:`robots.txt` file has Disallow lines for all of +The initial :file:`robots.txt` file has Disallow lines for all of the dynamically-generated buildbot pages, to discourage web spiders and search engines from consuming a lot of CPU time as they crawl through the entire history of your buildbot. If you are running the @@ -709,6 +720,45 @@ that periodically poll the Google Code commit feed for changes. change_hook_dialects={'googlecode': {'secret_key': 'FSP3p-Ghdn4T0oqX', 'branch': 'master'}} +Poller hook +########### + +The poller hook allows you to use GET requests to trigger polling. One +advantage of this is your buildbot instance can (at start up) poll to get +changes that happened while it was down, but then you can still use a commit +hook to get fast notification of new changes. + +Suppose you have a poller configured like this:: + + c['change_source'] = SVNPoller( + name="amanda", + svnurl="https://amanda.svn.sourceforge.net/svnroot/amanda/amanda", + split_file=split_file_branches) + +And you configure your WebStatus to enable this hook:: + + c['status'].append(html.WebStatus( + …, + change_hook_dialects={'poller': True} + )) + +Then you will be able to trigger a poll of the SVN repository by poking the +``/change_hook/poller`` URL from a commit hook like this:: + + curl http://yourbuildbot/change_hook/poller?poller=amanda + +If no ``poller`` argument is provided then the hook will trigger polling of all +polling change sources. + +You can restrict which pollers the webhook has access to using the ``allowed`` +option:: + + c['status'].append(html.WebStatus( + …, + change_hook_dialects={'poller': {'allowed': ['amanda']}} + )) + + .. bb:status:: MailNotifier .. index:: single: email; MailNotifier @@ -949,6 +999,12 @@ MailNotifier arguments ``warnings`` Send mail about builds which generate warnings. + ``exception`` + Send mail about builds which generate exceptions. + + ``all`` + Always send mail about builds. + Defaults to (``failing``, ``passing``, ``warnings``). ``builders`` @@ -1048,7 +1104,8 @@ Name of the project :meth:`master_status.getProjectName()` MailNotifier mode - ``mode`` (a combination of ``change``, ``failing``, ``passing``, ``problem``, ``warnings``) + ``mode`` (a combination of ``change``, ``failing``, ``passing``, ``problem``, ``warnings``, + ``exception``, ``all``) Builder result as a string :: diff --git a/master/docs/manual/concepts.rst b/master/docs/manual/concepts.rst index 8aa43538ab1..f39340d1cc3 100644 --- a/master/docs/manual/concepts.rst +++ b/master/docs/manual/concepts.rst @@ -5,131 +5,61 @@ This chapter defines some of the basic concepts that the Buildbot uses. You'll need to understand how the Buildbot sees the world to configure it properly. +.. index: repository +.. index: codebase +.. index: project +.. index: revision +.. index: branch +.. index: source stamp + +.. _Source-Stamps: + +Source Stamps +------------- + +Source code comes from *respositories*, provided by version control systems. +Repositories are generally identified by URLs, e.g., ``git://github.com/buildbot/buildbot.git``. + +In these days of distribtued version control systems, the same *codebase* may appear in mutiple repositories. +For example, ``https://github.com/mozilla/mozilla-central`` and ``http://hg.mozilla.org/mozilla-release`` both contain the Firefox codebase, although not exactly the same code. + +Many *projects* are built from multiple codebases. +For example, a company may build several applications based on the same core library. +The "app" codebase and the "core" codebase are in separate repositories, but are compiled together and constitute a single project. +Changes to either codebase should cause a rebuild of the application. + +Most version control systems define some sort of *revision* that can be used (sometimes in combination with a *branch*) to uniquely specify a particular version of the source code. + +To build a project, Buildbot needs to know exactly which version of each codebase it should build. +It uses a *source stamp* to do so for each codebase; the collection of sourcestamps required for a project is called a *source stamp set*. + +.. index: change + .. _Version-Control-Systems: Version Control Systems ----------------------- -These source trees come from a Version Control System of some kind. -CVS and Subversion are two popular ones, but the Buildbot supports -others. All VC systems have some notion of an upstream -`repository` which acts as a server [#]_, from which clients -can obtain source trees according to various parameters. The VC -repository provides source trees of various projects, for different -branches, and from various points in time. The first thing we have to -do is to specify which source tree we want to get. +Buildbot supports a significant number of version control systems, so it treats them abstractly. -.. _Generalizing-VC-Systems: +For purposes of deciding when to perform builds, Buildbot's change sources monitor repositories, and represent any updates to those repositories as *changes*. +These change sources fall broadly into two categories: pollers which periodically check the repository for updates; and hooks, where the repository is configured to notify Buildbot whenever an update occurs. -Generalizing VC Systems -~~~~~~~~~~~~~~~~~~~~~~~ +This concept does not map perfectly to every version control system. +For example, for CVS Buildbot must guess that version updates made to multiple files within a short time represent a single change; for DVCS's like Git, Buildbot records a change when a commit is pushed to the monitored repository, not when it is initially committed. +We assume that the :class:`Change`\s arrive at the master in the same order in which they are committed to the repository. + +When it comes time to actually perform a build, a scheduler prepares a source stamp set, as described above, based on its configuration. +When the build begins, one or more source steps use the information in the source stamp set to actually check out the source code, using the normal VCS commands. -For the purposes of the Buildbot, we will try to generalize all VC -systems as having repositories that each provide sources for a variety -of projects. Each project is defined as a directory tree with source -files. The individual files may each have revisions, but we ignore -that and treat the project as a whole as having a set of revisions -(CVS is the only VC system still in widespread use that has -per-file revisions, as everything modern has moved to atomic tree-wide -changesets). Each time someone commits a change to the project, a new -revision becomes available. These revisions can be described by a -tuple with two items: the first is a branch tag, and the second is -some kind of revision stamp or timestamp. Complex projects may have -multiple branch tags, but there is always a default branch. The -timestamp may be an actual timestamp (such as the :option:`-D` option to CVS), -or it may be a monotonically-increasing transaction number (such as -the change number used by SVN and P4, or the revision number used by -Bazaar, or a labeled tag used in CVS. [#]_) -The SHA1 revision ID used by Mercurial, and Git is -also a kind of revision stamp, in that it specifies a unique copy of -the source tree, as does a Darcs ``context`` file. - -When we aren't intending to make any changes to the sources we check out -(at least not any that need to be committed back upstream), there are two -basic ways to use a VC system: - - * Retrieve a specific set of source revisions: some tag or key is used - to index this set, which is fixed and cannot be changed by subsequent - developers committing new changes to the tree. Releases are built from - tagged revisions like this, so that they can be rebuilt again later - (probably with controlled modifications). - - * Retrieve the latest sources along a specific branch: some tag is used - to indicate which branch is to be used, but within that constraint we want - to get the latest revisions. - -Build personnel or CM staff typically use the first approach: the -build that results is (ideally) completely specified by the two -parameters given to the VC system: repository and revision tag. This -gives QA and end-users something concrete to point at when reporting -bugs. Release engineers are also reportedly fond of shipping code that -can be traced back to a concise revision tag of some sort. - -Developers are more likely to use the second approach: each morning -the developer does an update to pull in the changes committed by the -team over the last day. These builds are not easy to fully specify: it -depends upon exactly when you did a checkout, and upon what local -changes the developer has in their tree. Developers do not normally -tag each build they produce, because there is usually significant -overhead involved in creating these tags. Recreating the trees used by -one of these builds can be a challenge. Some VC systems may provide -implicit tags (like a revision number), while others may allow the use -of timestamps to mean "the state of the tree at time X" as opposed -to a tree-state that has been explicitly marked. - -The Buildbot is designed to help developers, so it usually works in -terms of *the latest* sources as opposed to specific tagged -revisions. However, it would really prefer to build from reproducible -source trees, so implicit revisions are used whenever possible. - -.. _Source-Tree-Specifications: - -Source Tree Specifications -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -So for the Buildbot's purposes we treat each VC system as a server -which can take a list of specifications as input and produce a source -tree as output. Some of these specifications are static: they are -attributes of the builder and do not change over time. Others are more -variable: each build will have a different value. The repository is -changed over time by a sequence of Changes, each of which represents a -single developer making changes to some set of files. These Changes -are cumulative. - -For normal builds, the Buildbot wants to get well-defined source trees -that contain specific :class:`Change`\s, and exclude other :class:`Change`\s that may have -occurred after the desired ones. We assume that the :class:`Change`\s arrive at -the buildbot (through one of the mechanisms described in -:ref:`Change-Sources`) in the same order in which they are committed to the -repository. The Buildbot waits for the tree to become ``stable`` -before initiating a build, for two reasons. The first is that -developers frequently make multiple related commits in quick -succession, even when the VC system provides ways to make atomic -transactions involving multiple files at the same time. Running a -build in the middle of these sets of changes would use an inconsistent -set of source files, and is likely to fail (and is certain to be less -useful than a build which uses the full set of changes). The -tree-stable-timer is intended to avoid these useless builds that -include some of the developer's changes but not all. The second reason -is that some VC systems (i.e. CVS) do not provide repository-wide -transaction numbers, so that timestamps are the only way to refer to -a specific repository state. These timestamps may be somewhat -ambiguous, due to processing and notification delays. By waiting until -the tree has been stable for, say, 10 minutes, we can choose a -timestamp from the middle of that period to use for our source -checkout, and then be reasonably sure that any clock-skew errors will -not cause the build to be performed on an inconsistent set of source -files. - -The :class:`Scheduler`\s always use the tree-stable-timer, with a timeout that -is configured to reflect a reasonable tradeoff between build latency -and change frequency. When the VC system provides coherent -repository-wide revision markers (such as Subversion's revision -numbers, or in fact anything other than CVS's timestamps), the -resulting :class:`Build` is simply performed against a source tree defined by -that revision marker. When the VC system does not provide this, a -timestamp from the middle of the tree-stable period is used to -generate the source tree [#]_. +Tree Stability +~~~~~~~~~~~~~~ + +Changes tend to arrive at a buildmaster in bursts. +In many cases, these bursts of changes are meant to be taken together. +For example, a developer may have pushed multiple commits to a DVCS that comprise the same new feature or bugfix. +To avoid trying to build every change, Buildbot supports the notion of *tree stability*, by waiting for a burst of changes to finish before starting to schedule builds. +This is implemented as a timer, with builds not scheduled until no changes have occurred for the duration of the timer. .. _How-Different-VC-Systems-Specify-Sources: @@ -236,15 +166,17 @@ SHA1 hash as returned by e.g. ``mtn automate select w:``. No attempt is made to ensure that the specified revision is actually a subset of the specified branch. +.. index: change + .. _Attributes-of-Changes: -Attributes of Changes -~~~~~~~~~~~~~~~~~~~~~ +Changes +------- .. _Attr-Who: Who -+++ +~~~ Each :class:`Change` has a :attr:`who` attribute, which specifies which developer is responsible for the change. This is a string which comes from a namespace @@ -261,7 +193,7 @@ incoming Changes will have their ``who`` parsed and stored. .. _Attr-Files: Files -+++++ +~~~~~ It also has a list of :attr:`files`, which are just the tree-relative filenames of any files that were added, deleted, or modified for this @@ -285,47 +217,42 @@ full test suite. .. _Attr-Comments: Comments -++++++++ +~~~~~~~~ -The Change also has a :attr:`comments` attribute, which is a string -containing any checkin comments. +The Change also has a :attr:`comments` attribute, which is a string containing any checkin comments. .. _Attr-Project: Project -+++++++ +~~~~~~~ -The :attr:`project` attribute of a change or source stamp describes the project -to which it corresponds, as a short human-readable string. This is useful in -cases where multiple independent projects are built on the same buildmaster. -In such cases, it can be used to control which builds are scheduled for a given -commit, and to limit status displays to only one project. +The :attr:`project` attribute of a change or source stamp describes the project to which it corresponds, as a short human-readable string. +This is useful in cases where multiple independent projects are built on the same buildmaster. +In such cases, it can be used to control which builds are scheduled for a given commit, and to limit status displays to only one project. .. _Attr-Repository: Repository -++++++++++ - -A change occurs within the context of a specific repository. This is a string, -and for most version-control systems, it takes the form of a URL. It uniquely -identifies the repository in which the change occurred. This is particularly -helpful for DVCS's, where a change may occur in a repository other than the -"main" repository for the project. +~~~~~~~~~~ -:class:`Change`\s can be filtered on repository, but more often this field is used as a -hint for the build steps to figure out which code to check out. +This attibute specifies the repository in which this change occurred. +In the case of DVCS's, this information may be required to check out the committed source code. +However, using the repository from a change has security risks: if Buildbot is configured to blidly trust this information, then it may easily be tricked into building arbitrary source code, potentially compromising the buildslaves and the integrity of subsequent builds. .. _Attr-Codebase: Codebase -++++++++ +~~~~~~~~ -The codebase is derived from a change. A complete software product may be composed of more than one repository. Each repository has its own unique position inside the product design (i.e. main module, shared library, resources, documentation). To be able to start builds from different VCS's and still distinquish the different repositories `codebase`'s are used. By default the codebase is ''. The `master.cfg` may contain a callable that determines the codebase from an incomming change and replaces the default value(see. :bb:cfg:`codebaseGenerator`). A codebase is not allowed to contain ':'. +This attribute specifies the codebase to which this change was made. +As described :ref:`above `, multiple repositories may contain the same codebase. +A change's codebase is usually determined by the bb:cfg:`codebaseGenerator` configuration. +By default the codebase is ''; this value is used automatically for single-codebase configurations. .. _Attr-Revision: Revision -++++++++ +~~~~~~~~ Each Change can have a :attr:`revision` attribute, which describes how to get a tree with a specific state: a tree which includes this Change @@ -356,7 +283,7 @@ Revisions are always strings. Branches -++++++++ +~~~~~~~~ The Change might also have a :attr:`branch` attribute. This indicates that all of the Change's files are in the same named branch. The @@ -390,8 +317,8 @@ same as Darcs. `Monotone` branch='warner-newfeature', files=['src/foo.c'] -Build Properties -++++++++++++++++ +Change Properties +~~~~~~~~~~~~~~~~~ A Change may have one or more properties attached to it, usually specified through the Force Build form or :bb:cmdline:`sendchange`. Properties are discussed @@ -454,8 +381,8 @@ individual :class:`BuildRequests` are delivered to the target .. _BuildSet: -BuildSet --------- +BuildSets +--------- A :class:`BuildSet` is the name given to a set of :class:`Build`\s that all compile/test the same version of the tree on multiple :class:`Builder`\s. In @@ -513,8 +440,8 @@ appropriate :class:`Builder`\s. .. _BuildRequest: -BuildRequest ------------- +BuildRequests +------------- A :class:`BuildRequest` is a request to build a specific set of source code (specified by one ore more source stamps) on a single :class:`Builder`. @@ -538,8 +465,8 @@ A merge of buildrequests is performed per codebase, thus on changes having the s .. _Builder: -Builder -------- +Builders +-------- The Buildmaster runs a collection of :class:`Builder`\s, each of which handles a single type of build (e.g. full versus quick), on one or more build slaves. :class:`Builder`\s @@ -562,7 +489,7 @@ checkout/compile/test commands are executed). .. _Concepts-Build-Factories: Build Factories -~~~~~~~~~~~~~~~ +--------------- A builder also has a :class:`BuildFactory`, which is responsible for creating new :class:`Build` instances: because the :class:`Build` instance is what actually performs each build, @@ -572,7 +499,7 @@ is done (:ref:`Concepts-Build`). .. _Concepts-Build-Slaves: Build Slaves -~~~~~~~~~~~~ +------------ Each builder is associated with one of more :class:`BuildSlave`\s. A builder which is used to perform Mac OS X builds (as opposed to Linux or Solaris builds) should @@ -595,8 +522,8 @@ all these things mean you should use separate Builders. .. _Concepts-Build: -Build ------ +Builds +------ A build is a single compile or test run of a particular version of the source code, and is comprised of a series of steps. It is ultimately up to you what @@ -845,16 +772,42 @@ Rather than create a build factory for each slave, the steps can use buildslave properties to identify the unique aspects of each slave and adapt the build process dynamically. -.. rubric:: Footnotes +.. _Multiple-Codebase-Builds: + +Multiple-Codebase Builds +------------------------ + +What if an end-product is composed of code from several codebases? +Changes may arrive from different repositories within the tree-stable-timer period. +Buildbot will not only use the source-trees that contain changes but also needs the remaining source-trees to build the complete product. + +For this reason a :ref:`Scheduler` can be configured to base a build on a set of several source-trees that can (partly) be overidden by the information from incoming :class:`Change`\s. + +As descibed :ref:`above `, the source for each codebase is identified by a source stamp, containing its repository, branch and revision. +A full build set will specify a source stamp set describing the source to use for each codebase. + +Configuring all of this takes a coordinated approach. A complete multiple repository configuration consists of: + + - a *codebase generator* + + Every relevant change arriving from a VC must contain a codebase. + This is done by a :bb:cfg:`codebaseGenerator` that is defined in the configuration. + Most generators examine the repository of a change to determine its codebase, using project-specific rules. + + - some *schedulers* + + Each :bb:cfg:`scheduler` has to be configured with a set of all required ``codebases`` to build a product. + These codebases indicate the set of required source-trees. + In order for the scheduler to be able to produce a complete set for each build, the configuration can give a default repository, branch, and revision for each codebase. + When a scheduler must generate a source stamp for a codebase that has received no changes, it applies these default values. + + - multiple *source steps* - one for each codebase + + A :ref:`Builder`'s build factory must include a :ref:`source step` for each codebase. + Each of the source steps has a ``codebase`` attribute which is used to select an appropriate source stamp from the source stamp set for a build. + This information comes from the arrived changes or from the scheduler's configured default values. + +.. warning:: + + Defining a :bb:cfg:`codebaseGenerator` that returns non-empty (not ``''``) codebases will change the behavior of all the schedulers. -.. [#] Except Darcs, but since the Buildbot never modifies its local source tree we can ignore - the fact that Darcs uses a less centralized model - -.. [#] Many VC systems provide more complexity than this: in particular the local - views that P4 and ClearCase can assemble out of various source - directories are more complex than we're prepared to take advantage of - here - -.. [#] This ``checkoutDelay`` defaults - to half the tree-stable timer, but it can be overridden with an - argument to the :class:`Source` Step diff --git a/master/docs/manual/customization.rst b/master/docs/manual/customization.rst index ed1ee5073b9..ffc1022a512 100644 --- a/master/docs/manual/customization.rst +++ b/master/docs/manual/customization.rst @@ -525,69 +525,37 @@ Properties Objects Writing New BuildSteps ---------------------- -While it is a good idea to keep your build process self-contained in -the source code tree, sometimes it is convenient to put more -intelligence into your Buildbot configuration. One way to do this is -to write a custom :class:`BuildStep`. Once written, this Step can be used in -the :file:`master.cfg` file. - -The best reason for writing a custom :class:`BuildStep` is to better parse the -results of the command being run. For example, a :class:`BuildStep` that knows -about JUnit could look at the logfiles to determine which tests had -been run, how many passed and how many failed, and then report more -detailed information than a simple ``rc==0`` -based `good/bad` -decision. - -Buildbot has acquired a large fleet of build steps, and sports a number of -knobs and hooks to make steps easier to write. This section may seem a bit -overwhelming, but most custom steps will only need to apply one or two of the -techniques outlined here. - -For complete documentation of the build step interfaces, see -:doc:`../developer/cls-buildsteps`. +While it is a good idea to keep your build process self-contained in the source code tree, sometimes it is convenient to put more intelligence into your Buildbot configuration. +One way to do this is to write a custom :class:`BuildStep`. +Once written, this Step can be used in the :file:`master.cfg` file. + +The best reason for writing a custom :class:`BuildStep` is to better parse the results of the command being run. +For example, a :class:`BuildStep` that knows about JUnit could look at the logfiles to determine which tests had been run, how many passed and how many failed, and then report more detailed information than a simple ``rc==0`` -based `good/bad` decision. + +Buildbot has acquired a large fleet of build steps, and sports a number of knobs and hooks to make steps easier to write. +This section may seem a bit overwhelming, but most custom steps will only need to apply one or two of the techniques outlined here. + +For complete documentation of the build step interfaces, see :doc:`../developer/cls-buildsteps`. .. _Writing-BuildStep-Constructors: Writing BuildStep Constructors ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Build steps act as their own factories, so their constructors are a bit more -complex than necessary. In the configuration file, a -:class:`~buildbot.process.buildstep.BuildStep` object is instantiated, but -because steps store state locally while executing, this object cannot be used -during builds. Instead, the build machinery calls the step's -:meth:`~buildbot.process.buildstep.BuildStep.getStepFactory` method to get a -tuple of a callable and keyword arguments that should be used to create a new -instance. +Build steps act as their own factories, so their constructors are a bit more complex than necessary. +In the configuration file, a :class:`~buildbot.process.buildstep.BuildStep` object is instantiated, but because steps store state locally while executing, this object cannot be used during builds. Consider the use of a :class:`BuildStep` in :file:`master.cfg`:: f.addStep(MyStep(someopt="stuff", anotheropt=1)) -This creates a single instance of class ``MyStep``. However, Buildbot needs a -new object each time the step is executed. this is accomplished by storing the -information required to instantiate a new object in the -:attr:`~buildbot.process.buildstep.BuildStep.factory` attribute. When the time -comes to construct a new :class:`~buildbot.process.build.Build`, -:class:`~buildbot.process.factory.BuildFactory` consults this attribute (via -:meth:`~buildbot.process.buildstep.BuildStep.getStepFactory`) and instantiates -a new step object. - -When writing a new step class, then, keep in mind are that you cannot do -anything "interesting" in the constructor -- limit yourself to checking and -storing arguments. Each constructor in a sequence of :class:`BuildStep` -subclasses must ensure the following: +This creates a single instance of class ``MyStep``. +However, Buildbot needs a new object each time the step is executed. +An instance of :class:`~buildbot.process.buildstep.BuildStep` rembers how it was constructed, and can create copies of itself. +When writing a new step class, then, keep in mind are that you cannot do anything "interesting" in the constructor -- limit yourself to checking and storing arguments. -* the parent class's constructor is called with all otherwise-unspecified - keyword arguments. - -* all keyword arguments for the class itself are passed to - :meth:`addFactoryArguments`. - -Keep a ``**kwargs`` argument on the end of your options, and pass that up to -the parent class's constructor. If the class overrides constructor arguments -for the parent class, those should be updated in ``kwargs``, rather than passed -directly (which will cause errors during instantiation). +It is customary to call the parent class's constructor with all otherwise-unspecified keyword arguments. +Keep a ``**kwargs`` argument on the end of your options, and pass that up to the parent class's constructor. The whole thing looks like this:: @@ -613,36 +581,24 @@ The whole thing looks like this:: self.frob_how_many = how_many self.frob_how = frob_how - # and record arguments for later - self.addFactoryArguments( - frob_what=frob_what, - frob_how_many=frob_how_many, - frob_how=frob_how) - class FastFrobnify(Frobnify): def __init__(self, speed=5, **kwargs) Frobnify.__init__(self, **kwargs) self.speed = speed - self.addFactoryArguments( - speed=speed) Running Commands ~~~~~~~~~~~~~~~~ -To spawn a command in the buildslave, create a -:class:`~buildbot.process.buildstep.RemoteCommand` instance in your step's -``start`` method and run it with -:meth:`~buildbot.process.buildstep.BuildStep.runCommand`:: +To spawn a command in the buildslave, create a :class:`~buildbot.process.buildstep.RemoteCommand` instance in your step's ``start`` method and run it with :meth:`~buildbot.process.buildstep.BuildStep.runCommand`:: cmd = RemoteCommand(args) d = self.runCommand(cmd) To add a LogFile, use :meth:`~buildbot.process.buildstep.BuildStep.addLog`. -Make sure the log gets closed when it finishes. When giving a Logfile to a -:class:`~buildbot.process.buildstep.RemoteShellCommand`, just ask it to close -the log when the command completes:: +Make sure the log gets closed when it finishes. +When giving a Logfile to a :class:`~buildbot.process.buildstep.RemoteShellCommand`, just ask it to close the log when the command completes:: log = self.addLog('output') cmd.useLog(log, closeWhenFinished=True) diff --git a/master/docs/release-notes.rst b/master/docs/release-notes.rst index 4e5ee09b830..01cd4d4db36 100644 --- a/master/docs/release-notes.rst +++ b/master/docs/release-notes.rst @@ -66,18 +66,37 @@ Deprecations, Removals, and Non-Compatible Changes from buildbot.steps.source.svn import SVN factory.append(SVN(repourl=Interpolate("svn://svn.example.org/svn/%(src::branch:-branches/test)s"))) +* The ``P4Sync`` step, deprecated since 0.8.5, has been removed. The ``P4`` step remains. Changes for Developers ~~~~~~~~~~~~~~~~~~~~~~ - * ``BuildStep.start`` can now optionally return a deferred and any errback - will be handled gracefully. If you use inlineCallbacks, this means that - unexpected exceptions and failures raised will be captured and logged and - the build shut down normally. +* ``BuildStep.start`` can now optionally return a deferred and any errback will + be handled gracefully. If you use inlineCallbacks, this means that unexpected + exceptions and failures raised will be captured and logged and the build shut + down normally. Features ~~~~~~~~ +* Buildbot now supports building projects composed of multiple codebases. New + schedulers can aggregate changes to multiple codebases into source stamp sets + (with one source stamp for each codebase). Source steps then check out each + codebase as required, and the remainder of the build process proceeds + normally. See the :ref:`Multiple-Codebase-Builds` for details. + +* ``Source`` and ``ShellCommand`` steps now have an optional ``descriptionSuffix``, a suffix to the + ``description``/``descriptionDone`` values. For example this can help distinguish between + multiple ``Compile`` steps that are applied to different codebases. + +* ``Git`` has a new ``getDescription`` option, which will run `git describe` after checkout + normally. See the documentation for details. + +* A new ternary substitution operator ``:?:`` and ``:#?:`` to use with the ``Interpolate`` + and ``WithProperties`` classes. + +* The mercurial hook now supports multple masters. See :bb:pull:`436`. + Slave ----- diff --git a/master/docs/tutorial/tour.rst b/master/docs/tutorial/tour.rst index 1ceeeaa9845..b3a0dcb5a62 100644 --- a/master/docs/tutorial/tour.rst +++ b/master/docs/tutorial/tour.rst @@ -48,6 +48,7 @@ Now, look for the section marked *PROJECT IDENTITY* which reads:: c['titleURL'] = "http://divmod.org/trac/wiki/DivmodPyflakes" If you want, you can change either of these links to anything you want to see what happens when you change them. + After making a change go into the terminal and type:: buildbot reconfig master @@ -240,6 +241,34 @@ You can also see the new builds in the web interface. .. image:: _images/irc-testrun.png :alt: a successful test run from IRC happened. +Setting Authorized Web Users +---------------------------- + +Further down, look for the WebStatus configuration:: + + c['status'] = [] + + from buildbot.status import html + from buildbot.status.web import authz, auth + + authz_cfg=authz.Authz( + # change any of these to True to enable; see the manual for more + # options + auth=auth.BasicAuth([("pyflakes","pyflakes")]), + gracefulShutdown = False, + forceBuild = 'auth', # use this to test your slave once it is set up + forceAllBuilds = False, + pingBuilder = False, + stopBuild = False, + stopAllBuilds = False, + cancelPendingBuild = False, + ) + c['status'].append(html.WebStatus(http_port=8010, authz=authz_cfg)) + +The ``auth.BasicAuth()`` define authorized users and their passwords. You can +change these or add new ones. See :bb:status:`WebStatus` for more about the +WebStatus configuration. + Debugging with Manhole ---------------------- @@ -280,7 +309,7 @@ After restarting the master, you can ssh into the master and get an interactive If you see this, the temporary solution is to install the previous version of pyasn1:: - pip instasll pyasn1-0.0.13b + pip install pyasn1-0.0.13b If you wanted to check which slaves are connected and what builders those slaves are assigned to you could do:: diff --git a/master/setup.cfg b/master/setup.cfg new file mode 100644 index 00000000000..55c57e2da41 --- /dev/null +++ b/master/setup.cfg @@ -0,0 +1,2 @@ +[aliases] +test = trial -m buildbot diff --git a/master/setup.py b/master/setup.py index fa32bf9c021..7896d6788c3 100755 --- a/master/setup.py +++ b/master/setup.py @@ -37,107 +37,6 @@ def include(d, e): return (d, [f for f in glob.glob('%s/%s'%(d, e)) if os.path.isfile(f)]) -class _SetupBuildCommand(Command): - """ - Master setup build command to subclass from. - """ - - user_options = [] - - def initialize_options(self): - """ - Setup the current dir. - """ - self._dir = os.getcwd() - - def finalize_options(self): - """ - Required. - """ - pass - - -class TestCommand(_SetupBuildCommand): - """ - Executes tests from setup. - """ - - description = "Run unittests inline" - - def run(self): - """ - Public run method. - """ - self._run(os.path.normpath(os.path.abspath( - os.path.join('buildbot', 'test')))) - - def _run(self, test_loc): - """ - Executes the test step. - - @param test_loc: location of test module - @type test_loc: str - """ - from twisted.scripts.trial import run - - # remove the 'test' option from argv - sys.argv.remove('test') - - # Mimick the trial script by adding the path as the last arg - sys.argv.append(test_loc) - - # Add the current dir to path and pull it all together - sys.path.insert(0, os.path.curdir) - sys.path[:] = map(os.path.abspath, sys.path) - # GO! - run() - - -class SdistTestCommand(TestCommand): - """ - Runs unittests from the sdist output. - """ - - description = "Run unittests from inside an sdist distribution" - - def run(self): - """ - Interesting magic to get a source dist and running trial on it. - - NOTE: there is magic going on here! If you know a better way feel - free to update it. - """ - # Clean out dist/ - if os.path.exists('dist'): - for root, dirs, files in os.walk('dist', topdown=False): - for name in files: - os.remove(os.path.join(root, name)) - for name in dirs: - os.rmdir(os.path.join(root, name)) - # Import setup making it as if we ran setup.py with the sdist arg - sys.argv.append('sdist') - import setup #@Reimport @UnresolvedImport @UnusedImport - try: - # attempt to extract the sdist data - from gzip import GzipFile - from tarfile import TarFile - # We open up the gzip as well as using the first item as the sdist - gz = GzipFile(os.path.join('dist', os.listdir('dist')[0])) - tf = TarFile(fileobj=gz) - # Make the output dir and generate the extract path - os.mkdir(os.path.join('dist', 'sdist_test')) - ex_path = os.path.join('dist', 'sdist_test', - tf.getmembers()[0].name, 'buildbot', 'test') - # Extract the data and run tests - print "Extracting to %s" % ex_path - tf.extractall(os.path.join('dist', 'sdist_test')) - print "Executing tests ..." - self._run(os.path.normpath(os.path.abspath(ex_path))) - except IndexError, ie: - # We get called twice and the IndexError is OK - pass - - class install_data_twisted(install_data): """make sure data files are installed in package. this is evil. @@ -255,18 +154,18 @@ def make_release_tree(self, base_dir, files): "buildbot/status/web/files/default.css", "buildbot/status/web/files/bg_gradient.jpg", "buildbot/status/web/files/robots.txt", + "buildbot/status/web/files/templates_readme.txt", "buildbot/status/web/files/favicon.ico", ]), include("buildbot/status/web/templates", '*.html'), include("buildbot/status/web/templates", '*.xml'), ("buildbot/scripts", [ "buildbot/scripts/sample.cfg", + "buildbot/scripts/buildbot_tac.tmpl", ]), ], 'scripts': scripts, 'cmdclass': {'install_data': install_data_twisted, - 'test': TestCommand, - 'sdist_test': SdistTestCommand, 'sdist': our_sdist}, } @@ -295,6 +194,12 @@ def make_release_tree(self, base_dir, files): 'sqlalchemy-migrate ==0.6.1, ==0.7.0, ==0.7.1, ==0.7.2', 'python-dateutil==1.5', ] + setup_args['setup_requires'] = [ + 'setuptools_trial', + ] + setup_args['tests_require'] = [ + 'mock', + ] # Python-2.6 and up includes json if not py_26: setup_args['install_requires'].append('simplejson') diff --git a/master/tox.ini b/master/tox.ini new file mode 100644 index 00000000000..e9e6fe099d0 --- /dev/null +++ b/master/tox.ini @@ -0,0 +1,10 @@ +# Tox (http://tox.testrun.org/) is a tool for running tests +# in multiple virtualenvs. This configuration file will run the +# test suite on all supported python versions. To use it, "pip install tox" +# and then run "tox" from this directory. + +[tox] +envlist = py25, py26, py27 + +[testenv] +commands = python setup.py test diff --git a/slave/buildslave/commands/p4.py b/slave/buildslave/commands/p4.py index d1f92b9fc22..7aa17d8e5df 100644 --- a/slave/buildslave/commands/p4.py +++ b/slave/buildslave/commands/p4.py @@ -219,58 +219,3 @@ def parseGotRevision(self): else: return P4Base.parseGotRevision(self) - -class P4Sync(P4Base): - """A partial P4 source-updater. Requires manual setup of a per-slave P4 - environment. The only thing which comes from the master is P4PORT. - 'mode' is required to be 'copy'. - - ['p4port'] (required): host:port for server to access - ['p4user'] (optional): user to use for access - ['p4passwd'] (optional): passwd to try for the user - ['p4client'] (optional): client spec to use - """ - - header = "p4 sync" - - def setup(self, args): - P4Base.setup(self, args) - - def sourcedirIsUpdateable(self): - return True - - def _doVC(self, force): - d = os.path.join(self.builder.basedir, self.srcdir) - command = [self.getCommand('p4')] - if self.p4port: - command.extend(['-p', self.p4port]) - if self.p4user: - command.extend(['-u', self.p4user]) - if self.p4passwd: - command.extend(['-P', Obfuscated(self.p4passwd, "XXXXXXXX")]) - if self.p4client: - command.extend(['-c', self.p4client]) - command.extend(['sync']) - if force: - command.extend(['-f']) - if self.revision: - command.extend(['@' + self.revision]) - env = {} - c = runprocess.RunProcess(self.builder, command, d, environ=env, - sendRC=False, timeout=self.timeout, - maxTime=self.maxTime, usePTY=False, - logEnviron=self.logEnviron) - self.command = c - return c.start() - - def doVCUpdate(self): - return self._doVC(force=False) - - def doVCFull(self): - return self._doVC(force=True) - - def parseGotRevision(self): - if self.revision: - return str(self.revision) - else: - return P4Base.parseGotRevision(self) diff --git a/slave/buildslave/commands/registry.py b/slave/buildslave/commands/registry.py index 19c2060d8dc..2c2b5564c9c 100644 --- a/slave/buildslave/commands/registry.py +++ b/slave/buildslave/commands/registry.py @@ -29,7 +29,6 @@ "bzr" : "buildslave.commands.bzr.Bzr", "hg" : "buildslave.commands.hg.Mercurial", "p4" : "buildslave.commands.p4.P4", - "p4sync" : "buildslave.commands.p4.P4Sync", "mtn" : "buildslave.commands.mtn.Monotone", "mkdir" : "buildslave.commands.fs.MakeDirectory", "rmdir" : "buildslave.commands.fs.RemoveDirectory", diff --git a/slave/buildslave/commands/repo.py b/slave/buildslave/commands/repo.py index abc94a3a598..a2fb6a87f76 100644 --- a/slave/buildslave/commands/repo.py +++ b/slave/buildslave/commands/repo.py @@ -37,6 +37,8 @@ class Repo(SourceBaseCommand): ['tarball'] (optional): The tarball base to accelerate the fetch. ['repo_downloads'] (optional): Repo downloads to do. Computer from GerritChangeSource and forced build properties. + ['jobs'] (optional): number of connections to run in parallel + repo tool will use while syncing """ header = "repo operation" @@ -52,6 +54,7 @@ def setup(self, args): # we're using string instead of an array here, because it will be transferred back # to the master as string anyway and using eval() could have security implications. self.repo_downloaded = "" + self.jobs = args.get('jobs') self.sourcedata = "%s %s" % (self.manifest_url, self.manifest_file) self.re_change = re.compile(".* refs/changes/\d\d/(\d+)/(\d+) -> FETCH_HEAD$") @@ -165,6 +168,8 @@ def _doSync(self, dummy): if self.manifest_override_url: os.system("cd %s/.repo; ln -sf ../manifest_override.xml manifest.xml"%(self._fullSrcdir())) command = ['sync'] + if self.jobs: + command.append('-j' + str(self.jobs)) self.sendStatus({"header": "synching manifest %s from branch %s from %s\n" % (self.manifest_file, self.manifest_branch, self.manifest_url)}) return self._repoCmd(command, self._didSync) diff --git a/slave/setup.cfg b/slave/setup.cfg new file mode 100644 index 00000000000..c8be2270745 --- /dev/null +++ b/slave/setup.cfg @@ -0,0 +1,2 @@ +[aliases] +test = trial -m buildslave diff --git a/slave/setup.py b/slave/setup.py index 53a4aea3e4a..dcfeda50f7f 100755 --- a/slave/setup.py +++ b/slave/setup.py @@ -121,6 +121,12 @@ def make_release_tree(self, base_dir, files): setup_args['install_requires'] = [ 'twisted >= 8.0.0', ] + setup_args['setup_requires'] = [ + 'setuptools_trial', + ] + setup_args['tests_require'] = [ + 'mock', + ] if os.getenv('NO_INSTALL_REQS'): setup_args['install_requires'] = None diff --git a/slave/tox.ini b/slave/tox.ini new file mode 100644 index 00000000000..aea725bfd75 --- /dev/null +++ b/slave/tox.ini @@ -0,0 +1,10 @@ +# Tox (http://tox.testrun.org/) is a tool for running tests +# in multiple virtualenvs. This configuration file will run the +# test suite on all supported python versions. To use it, "pip install tox" +# and then run "tox" from this directory. + +[tox] +envlist = py24, py25, py26, py27 + +[testenv] +commands = python setup.py test