From a8f88771f93ac92e0a9043bd2e5c359a4395a1e6 Mon Sep 17 00:00:00 2001 From: Tom Prince Date: Wed, 22 Feb 2012 11:52:04 -0500 Subject: [PATCH 001/136] Use magic __new__ for step factories. --- master/buildbot/config.py | 2 +- master/buildbot/process/build.py | 11 ++++++----- master/buildbot/process/buildstep.py | 12 +++++++++--- master/buildbot/test/unit/test_config.py | 4 ++-- master/buildbot/test/unit/test_process_build.py | 16 ++++++++-------- master/buildbot/test/util/steps.py | 4 ++-- 6 files changed, 28 insertions(+), 21 deletions(-) diff --git a/master/buildbot/config.py b/master/buildbot/config.py index 8411de6bb55..12a9d809fe8 100644 --- a/master/buildbot/config.py +++ b/master/buildbot/config.py @@ -552,7 +552,7 @@ def check_lock(l): if not hasattr(b.factory, 'steps'): continue for s in b.factory.steps: - for l in s[1].get('locks', []): + for l in s[2].get('locks', []): check_lock(l) diff --git a/master/buildbot/process/build.py b/master/buildbot/process/build.py index c915cbc911e..ec0f0299558 100644 --- a/master/buildbot/process/build.py +++ b/master/buildbot/process/build.py @@ -297,13 +297,14 @@ def setupBuild(self, expectations): stepnames = {} sps = [] - for factory, args in self.stepFactories: - args = args.copy() + for factory, args, kwargs in self.stepFactories: + args = tuple(args) + kwargs = kwargs.copy() try: - step = factory(**args) + step = factory(*args, **kwargs) except: - log.msg("error while creating step, factory=%s, args=%s" - % (factory, args)) + log.msg("error while creating step, factory=%s, args=%s, kwargs=%s" + % (factory, args, kwargs)) raise step.setBuild(self) diff --git a/master/buildbot/process/buildstep.py b/master/buildbot/process/buildstep.py index c8c37a12671..7f25fb481fa 100644 --- a/master/buildbot/process/buildstep.py +++ b/master/buildbot/process/buildstep.py @@ -381,7 +381,7 @@ def _start(self): def __repr__(self): return "" % repr(self.command) -class BuildStep(properties.PropertiesMixin): +class BuildStep(object, properties.PropertiesMixin): haltOnFailure = False flunkOnWarnings = False @@ -424,7 +424,6 @@ class BuildStep(properties.PropertiesMixin): progress = None def __init__(self, **kwargs): - self.factory = (self.__class__, dict(kwargs)) for p in self.__class__.parms: if kwargs.has_key(p): setattr(self, p, kwargs[p]) @@ -438,6 +437,12 @@ def __init__(self, **kwargs): self._acquiringLock = None self.stopped = False + def __new__(klass, *args, **kwargs): + self = object.__new__(klass) + klass.__init__(self, *args, **kwargs) + self.factory = (klass, args, kwargs) + return self + def describe(self, done=False): return [self.name] @@ -451,7 +456,8 @@ def setDefaultWorkdir(self, workdir): pass def addFactoryArguments(self, **kwargs): - self.factory[1].update(kwargs) + # this is here for backwards compatability + pass def getStepFactory(self): return self.factory diff --git a/master/buildbot/test/unit/test_config.py b/master/buildbot/test/unit/test_config.py index 426b6d98690..0da9333b65e 100644 --- a/master/buildbot/test/unit/test_config.py +++ b/master/buildbot/test/unit/test_config.py @@ -706,7 +706,7 @@ def bldr(name): b = mock.Mock() b.name = name b.locks = [] - b.factory.steps = [ ('cls', dict(locks=[])) ] + b.factory.steps = [ ('cls', (), dict(locks=[])) ] return b def lock(name): @@ -721,7 +721,7 @@ def lock(name): if dup_builder_lock: b2.locks.append(lock(builder_lock)) if step_lock: - s1, s2 = b1.factory.steps[0][1], b2.factory.steps[0][1] + s1, s2 = b1.factory.steps[0][2], b2.factory.steps[0][2] s1['locks'].append(lock(step_lock)) if dup_step_lock: s2['locks'].append(lock(step_lock)) diff --git a/master/buildbot/test/unit/test_process_build.py b/master/buildbot/test/unit/test_process_build.py index 3d1cea46a74..697dbf5f70e 100644 --- a/master/buildbot/test/unit/test_process_build.py +++ b/master/buildbot/test/unit/test_process_build.py @@ -102,7 +102,7 @@ def testRunSuccessfulBuild(self): step = Mock() step.return_value = step step.startStep.return_value = SUCCESS - b.setStepFactories([(step, {})]) + b.setStepFactories([(step, (), {})]) slavebuilder = Mock() @@ -117,7 +117,7 @@ def testStopBuild(self): step = Mock() step.return_value = step - b.setStepFactories([(step, {})]) + b.setStepFactories([(step, (), {})]) slavebuilder = Mock() @@ -148,8 +148,8 @@ def testAlwaysRunStepStopBuild(self): step2.return_value = step2 step2.alwaysRun = True b.setStepFactories([ - (step1, {}), - (step2, {}), + (step1, (), {}), + (step2, (), {}), ]) slavebuilder = Mock() @@ -196,7 +196,7 @@ def claim(owner, access): step = Mock() step.return_value = step step.startStep.return_value = SUCCESS - b.setStepFactories([(step, {})]) + b.setStepFactories([(step, (), {})]) b.startBuild(FakeBuildStatus(), None, slavebuilder) @@ -225,7 +225,7 @@ def claim(owner, access): step = Mock() step.return_value = step step.startStep.return_value = SUCCESS - b.setStepFactories([(step, {})]) + b.setStepFactories([(step, (), {})]) real_lock.claim(Mock(), l.access('counting')) @@ -252,7 +252,7 @@ def testStopBuildWaitingForLocks(self): step.return_value = step step.startStep.return_value = SUCCESS step.alwaysRun = False - b.setStepFactories([(step, {})]) + b.setStepFactories([(step, (), {})]) real_lock.claim(Mock(), l.access('counting')) @@ -318,7 +318,7 @@ def testStopBuildWaitingForStepLocks(self): step = LoggingBuildStep(locks=[lock_access]) def factory(*args): return step - b.setStepFactories([(factory, {})]) + b.setStepFactories([(factory, (), {})]) real_lock.claim(Mock(), l.access('counting')) diff --git a/master/buildbot/test/util/steps.py b/master/buildbot/test/util/steps.py index 2724981749e..526cb709514 100644 --- a/master/buildbot/test/util/steps.py +++ b/master/buildbot/test/util/steps.py @@ -73,8 +73,8 @@ def setupStep(self, step, slave_version={'*':"99.99"}, slave_env={}): """ # yes, Virginia, "factory" refers both to the tuple and its first # element TODO: fix that up - factory, args = step.getStepFactory() - step = self.step = factory(**args) + factory, args, kwargs = step.getStepFactory() + step = self.step = factory(*args, **kwargs) # step.build From a77e511860b1be57feaa79ed4cf6a66d08eafca8 Mon Sep 17 00:00:00 2001 From: Tom Prince Date: Wed, 22 Feb 2012 19:42:55 -0500 Subject: [PATCH 002/136] Remove calls to BuildStep.addFactoryArguments. This call does nothing anymore, so remove calls to it. The function is still around for backwards compatibility. --- master/buildbot/process/buildstep.py | 3 - master/buildbot/process/mtrlogobserver.py | 9 --- master/buildbot/steps/master.py | 4 -- master/buildbot/steps/maxq.py | 1 - master/buildbot/steps/package/rpm/rpmbuild.py | 10 --- master/buildbot/steps/python.py | 11 --- master/buildbot/steps/python_twisted.py | 12 ---- master/buildbot/steps/shell.py | 16 ----- master/buildbot/steps/slave.py | 5 -- master/buildbot/steps/source/base.py | 10 --- master/buildbot/steps/source/bzr.py | 6 -- master/buildbot/steps/source/cvs.py | 8 --- master/buildbot/steps/source/git.py | 11 --- master/buildbot/steps/source/mercurial.py | 8 --- master/buildbot/steps/source/oldsource.py | 69 ------------------- master/buildbot/steps/source/svn.py | 11 --- master/buildbot/steps/subunit.py | 1 - master/buildbot/steps/transfer.py | 32 --------- master/buildbot/steps/trigger.py | 7 -- master/buildbot/steps/vstudio.py | 12 ---- 20 files changed, 246 deletions(-) diff --git a/master/buildbot/process/buildstep.py b/master/buildbot/process/buildstep.py index 7f25fb481fa..aacbf2d167d 100644 --- a/master/buildbot/process/buildstep.py +++ b/master/buildbot/process/buildstep.py @@ -761,9 +761,6 @@ class LoggingBuildStep(BuildStep): def __init__(self, logfiles={}, lazylogfiles=False, log_eval_func=None, *args, **kwargs): BuildStep.__init__(self, *args, **kwargs) - self.addFactoryArguments(logfiles=logfiles, - lazylogfiles=lazylogfiles, - log_eval_func=log_eval_func) if logfiles and not isinstance(logfiles, dict): config.error( diff --git a/master/buildbot/process/mtrlogobserver.py b/master/buildbot/process/mtrlogobserver.py index 43168a6bb60..a49b90e2129 100644 --- a/master/buildbot/process/mtrlogobserver.py +++ b/master/buildbot/process/mtrlogobserver.py @@ -296,15 +296,6 @@ def __init__(self, dbpool=None, test_type=None, test_info="", self.mtr_subdir = mtr_subdir self.progressMetrics += ('tests',) - self.addFactoryArguments(dbpool=self.dbpool, - test_type=self.test_type, - test_info=self.test_info, - autoCreateTables=self.autoCreateTables, - textLimit=self.textLimit, - testNameLimit=self.testNameLimit, - parallel=self.parallel, - mtr_subdir=self.mtr_subdir) - def start(self): # Add mysql server logfiles. for mtr in range(0, self.parallel+1): diff --git a/master/buildbot/steps/master.py b/master/buildbot/steps/master.py index 42680f0767f..ce8fa62bb63 100644 --- a/master/buildbot/steps/master.py +++ b/master/buildbot/steps/master.py @@ -38,10 +38,6 @@ def __init__(self, command, env=None, path=None, usePTY=0, **kwargs): BuildStep.__init__(self, **kwargs) - self.addFactoryArguments(description=description, - descriptionDone=descriptionDone, - env=env, path=path, usePTY=usePTY, - command=command) self.command=command if description: diff --git a/master/buildbot/steps/maxq.py b/master/buildbot/steps/maxq.py index 1d7a1409ea2..c165e0c1300 100644 --- a/master/buildbot/steps/maxq.py +++ b/master/buildbot/steps/maxq.py @@ -26,7 +26,6 @@ def __init__(self, testdir=None, **kwargs): config.error("please pass testdir") kwargs['command'] = 'run_maxq.py %s' % (testdir,) ShellCommand.__init__(self, **kwargs) - self.addFactoryArguments(testdir=testdir) def commandComplete(self, cmd): output = cmd.logs['stdio'].getText() diff --git a/master/buildbot/steps/package/rpm/rpmbuild.py b/master/buildbot/steps/package/rpm/rpmbuild.py index fa54956507f..b43583584ac 100644 --- a/master/buildbot/steps/package/rpm/rpmbuild.py +++ b/master/buildbot/steps/package/rpm/rpmbuild.py @@ -41,16 +41,6 @@ def __init__(self, vcsRevision=False, **kwargs): ShellCommand.__init__(self, **kwargs) - self.addFactoryArguments(topdir=topdir, - builddir=builddir, - rpmdir=rpmdir, - sourcedir=sourcedir, - specdir=specdir, - srcrpmdir=srcrpmdir, - specfile=specfile, - dist=dist, - autoRelease=autoRelease, - vcsRevision=vcsRevision) self.rpmbuild = ( 'rpmbuild --define "_topdir %s" --define "_builddir %s"' ' --define "_rpmdir %s" --define "_sourcedir %s"' diff --git a/master/buildbot/steps/python.py b/master/buildbot/steps/python.py index 81f4538a88e..6b0882263d9 100644 --- a/master/buildbot/steps/python.py +++ b/master/buildbot/steps/python.py @@ -260,17 +260,6 @@ def __init__(self, sphinx_sourcedir='.', sphinx_builddir=None, command.extend([sphinx_sourcedir, sphinx_builddir]) self.setCommand(command) - self.addFactoryArguments( - sphinx = sphinx, - sphinx_sourcedir = sphinx_sourcedir, - sphinx_builddir = sphinx_builddir, - sphinx_builder = sphinx_builder, - tags = tags, - defines = defines, - mode = mode, - ) - - def createSummary(self, log): msgs = ['WARNING', 'ERROR', 'SEVERE'] diff --git a/master/buildbot/steps/python_twisted.py b/master/buildbot/steps/python_twisted.py index d49da432462..f7fa03d308f 100644 --- a/master/buildbot/steps/python_twisted.py +++ b/master/buildbot/steps/python_twisted.py @@ -47,7 +47,6 @@ class HLint(ShellCommand): def __init__(self, python=None, **kwargs): ShellCommand.__init__(self, **kwargs) - self.addFactoryArguments(python=python) self.python = python def start(self): @@ -287,17 +286,6 @@ def __init__(self, reactor=UNSPECIFIED, python=None, trial=None, timeout. """ ShellCommand.__init__(self, **kwargs) - self.addFactoryArguments(reactor=reactor, - python=python, - trial=trial, - testpath=testpath, - tests=tests, - testChanges=testChanges, - recurse=recurse, - randomly=randomly, - trialMode=trialMode, - trialArgs=trialArgs, - ) if python: self.python = python diff --git a/master/buildbot/steps/shell.py b/master/buildbot/steps/shell.py index 71b4bc20ec5..bf2f970e6dc 100644 --- a/master/buildbot/steps/shell.py +++ b/master/buildbot/steps/shell.py @@ -105,17 +105,11 @@ def __init__(self, workdir=None, buildstep_kwargs[k] = kwargs[k] del kwargs[k] buildstep.LoggingBuildStep.__init__(self, **buildstep_kwargs) - self.addFactoryArguments(workdir=workdir, - description=description, - descriptionDone=descriptionDone, - command=command) # everything left over goes to the RemoteShellCommand kwargs['workdir'] = workdir # including a copy of 'workdir' kwargs['usePTY'] = usePTY self.remote_kwargs = kwargs - # we need to stash the RemoteShellCommand's args too - self.addFactoryArguments(**kwargs) def setBuild(self, build): buildstep.LoggingBuildStep.setBuild(self, build) @@ -303,10 +297,6 @@ def __init__(self, property=None, extract_fn=None, strip=True, **kwargs): ShellCommand.__init__(self, **kwargs) - self.addFactoryArguments(property=self.property) - self.addFactoryArguments(extract_fn=self.extract_fn) - self.addFactoryArguments(strip=self.strip) - self.property_changes = {} def commandComplete(self, cmd): @@ -405,12 +395,6 @@ def __init__(self, # And upcall to let the base class do its work ShellCommand.__init__(self, **kwargs) - self.addFactoryArguments(warningPattern=warningPattern, - directoryEnterPattern=directoryEnterPattern, - directoryLeavePattern=directoryLeavePattern, - warningExtractor=warningExtractor, - maxWarnCount=maxWarnCount, - suppressionFile=suppressionFile) self.suppressions = [] self.directoryStack = [] diff --git a/master/buildbot/steps/slave.py b/master/buildbot/steps/slave.py index 7e1eedaacf1..dfe346fc46d 100644 --- a/master/buildbot/steps/slave.py +++ b/master/buildbot/steps/slave.py @@ -30,8 +30,6 @@ class SetPropertiesFromEnv(buildstep.BuildStep): def __init__(self, variables, source="SlaveEnvironment", **kwargs): buildstep.BuildStep.__init__(self, **kwargs) - self.addFactoryArguments(variables = variables, - source = source) self.variables = variables self.source = source @@ -74,7 +72,6 @@ class FileExists(buildstep.BuildStep): def __init__(self, file, **kwargs): buildstep.BuildStep.__init__(self, **kwargs) - self.addFactoryArguments(file = file) self.file = file def start(self): @@ -115,7 +112,6 @@ class RemoveDirectory(buildstep.BuildStep): def __init__(self, dir, **kwargs): buildstep.BuildStep.__init__(self, **kwargs) - self.addFactoryArguments(dir = dir) self.dir = dir def start(self): @@ -150,7 +146,6 @@ class MakeDirectory(buildstep.BuildStep): def __init__(self, dir, **kwargs): buildstep.BuildStep.__init__(self, **kwargs) - self.addFactoryArguments(dir = dir) self.dir = dir def start(self): diff --git a/master/buildbot/steps/source/base.py b/master/buildbot/steps/source/base.py index e96576146e1..7c0e41dcbdd 100644 --- a/master/buildbot/steps/source/base.py +++ b/master/buildbot/steps/source/base.py @@ -130,16 +130,6 @@ def __init__(self, workdir=None, mode='update', alwaysUseLatest=False, """ LoggingBuildStep.__init__(self, **kwargs) - self.addFactoryArguments(workdir=workdir, - mode=mode, - alwaysUseLatest=alwaysUseLatest, - timeout=timeout, - retry=retry, - logEnviron=logEnviron, - env=env, - description=description, - descriptionDone=descriptionDone - ) assert mode in ("update", "copy", "clobber", "export") if retry: diff --git a/master/buildbot/steps/source/bzr.py b/master/buildbot/steps/source/bzr.py index 2f9f1983290..afa0add92fe 100644 --- a/master/buildbot/steps/source/bzr.py +++ b/master/buildbot/steps/source/bzr.py @@ -34,12 +34,6 @@ def __init__(self, repourl=None, baseURL=None, mode='incremental', self.mode = mode self.method = method Source.__init__(self, **kwargs) - self.addFactoryArguments(repourl=repourl, - mode=mode, - method=method, - baseURL=baseURL, - defaultBranch=defaultBranch, - ) if repourl and baseURL: raise ValueError("you must provide exactly one of repourl and" " baseURL") diff --git a/master/buildbot/steps/source/cvs.py b/master/buildbot/steps/source/cvs.py index 66585c08fe2..0c0b2feb28b 100644 --- a/master/buildbot/steps/source/cvs.py +++ b/master/buildbot/steps/source/cvs.py @@ -43,14 +43,6 @@ def __init__(self, cvsroot=None, cvsmodule='', mode='incremental', self.method = method self.srcdir = 'source' Source.__init__(self, **kwargs) - self.addFactoryArguments(cvsroot=cvsroot, - cvsmodule=cvsmodule, - mode=mode, - method=method, - global_options=global_options, - extra_options=extra_options, - login=login, - ) def startVC(self, branch, revision, patch): self.revision = revision diff --git a/master/buildbot/steps/source/git.py b/master/buildbot/steps/source/git.py index 0e686e869ff..9f2f2b03beb 100644 --- a/master/buildbot/steps/source/git.py +++ b/master/buildbot/steps/source/git.py @@ -70,17 +70,6 @@ def __init__(self, repourl=None, branch='HEAD', mode='incremental', self.clobberOnFailure = clobberOnFailure self.mode = mode Source.__init__(self, **kwargs) - self.addFactoryArguments(branch=branch, - mode=mode, - method=method, - progress=progress, - repourl=repourl, - submodules=submodules, - shallow=shallow, - retryFetch=retryFetch, - clobberOnFailure= - clobberOnFailure, - ) assert self.mode in ['incremental', 'full'] assert self.repourl is not None diff --git a/master/buildbot/steps/source/mercurial.py b/master/buildbot/steps/source/mercurial.py index 44fc5a43fec..aa57a67ed0a 100644 --- a/master/buildbot/steps/source/mercurial.py +++ b/master/buildbot/steps/source/mercurial.py @@ -71,14 +71,6 @@ def __init__(self, repourl=None, mode='incremental', self.clobberOnBranchChange = clobberOnBranchChange self.mode = mode Source.__init__(self, **kwargs) - self.addFactoryArguments(repourl=repourl, - mode=mode, - method=method, - defaultBranch=defaultBranch, - branchType=branchType, - clobberOnBranchChange= - clobberOnBranchChange, - ) errors = [] if self.mode not in self.possible_modes: diff --git a/master/buildbot/steps/source/oldsource.py b/master/buildbot/steps/source/oldsource.py index 6f8c1d004df..4f0da58e26e 100644 --- a/master/buildbot/steps/source/oldsource.py +++ b/master/buildbot/steps/source/oldsource.py @@ -62,7 +62,6 @@ def getRenderingFor(self, props): - class CVS(Source): """I do CVS checkout/update operations. @@ -164,16 +163,6 @@ def __init__(self, cvsroot=None, cvsmodule="", self.cvsroot = _ComputeRepositoryURL(cvsroot) Source.__init__(self, **kwargs) - self.addFactoryArguments(cvsroot=cvsroot, - cvsmodule=cvsmodule, - global_options=global_options, - checkout_options=checkout_options, - export_options=export_options, - extra_options=extra_options, - branch=branch, - checkoutDelay=checkoutDelay, - login=login, - ) self.args.update({'cvsmodule': cvsmodule, 'global_options': global_options, @@ -310,18 +299,6 @@ def __init__(self, svnurl=None, baseURL=None, defaultBranch=None, self.depth = depth Source.__init__(self, **kwargs) - self.addFactoryArguments(svnurl=svnurl, - baseURL=baseURL, - defaultBranch=defaultBranch, - directory=directory, - username=username, - password=password, - extra_args=extra_args, - keep_on_purge=keep_on_purge, - ignore_ignores=ignore_ignores, - always_purge=always_purge, - depth=depth, - ) if svnurl and baseURL: raise ValueError("you must use either svnurl OR baseURL") @@ -466,10 +443,6 @@ def __init__(self, repourl=None, baseURL=None, defaultBranch=None, self.baseURL = _ComputeRepositoryURL(baseURL) self.branch = defaultBranch Source.__init__(self, **kwargs) - self.addFactoryArguments(repourl=repourl, - baseURL=baseURL, - defaultBranch=defaultBranch, - ) assert self.args['mode'] != "export", \ "Darcs does not have an 'export' mode" if repourl and baseURL: @@ -564,14 +537,6 @@ def __init__(self, repourl=None, Source.__init__(self, **kwargs) self.repourl = _ComputeRepositoryURL(repourl) self.branch = branch - self.addFactoryArguments(repourl=repourl, - branch=branch, - submodules=submodules, - ignore_ignores=ignore_ignores, - reference=reference, - shallow=shallow, - progress=progress, - ) self.args.update({'submodules': submodules, 'ignore_ignores': ignore_ignores, 'reference': reference, @@ -640,11 +605,6 @@ def __init__(self, """ Source.__init__(self, **kwargs) self.manifest_url = _ComputeRepositoryURL(manifest_url) - self.addFactoryArguments(manifest_url=manifest_url, - manifest_branch=manifest_branch, - manifest_file=manifest_file, - tarball=tarball, - ) self.args.update({'manifest_branch': manifest_branch, 'manifest_file': manifest_file, 'tarball': tarball, @@ -788,11 +748,6 @@ def __init__(self, repourl=None, baseURL=None, defaultBranch=None, self.baseURL = _ComputeRepositoryURL(baseURL) self.branch = defaultBranch Source.__init__(self, **kwargs) - self.addFactoryArguments(repourl=repourl, - baseURL=baseURL, - defaultBranch=defaultBranch, - forceSharedRepo=forceSharedRepo - ) self.args.update({'forceSharedRepo': forceSharedRepo}) if repourl and baseURL: raise ValueError("you must provide exactly one of repourl and" @@ -878,12 +833,6 @@ def __init__(self, repourl=None, baseURL=None, defaultBranch=None, self.branchType = branchType self.clobberOnBranchChange = clobberOnBranchChange Source.__init__(self, **kwargs) - self.addFactoryArguments(repourl=repourl, - baseURL=baseURL, - defaultBranch=defaultBranch, - branchType=branchType, - clobberOnBranchChange=clobberOnBranchChange, - ) if repourl and baseURL: raise ValueError("you must provide exactly one of repourl and" " baseURL") @@ -975,15 +924,6 @@ def __init__(self, p4base=None, defaultBranch=None, p4port=None, p4user=None, self.p4base = _ComputeRepositoryURL(p4base) self.branch = defaultBranch Source.__init__(self, **kwargs) - self.addFactoryArguments(p4base=p4base, - defaultBranch=defaultBranch, - p4port=p4port, - p4user=p4user, - p4passwd=p4passwd, - p4extra_views=p4extra_views, - p4line_end=p4line_end, - p4client=p4client, - ) self.args['p4port'] = p4port self.args['p4user'] = p4user self.args['p4passwd'] = p4passwd @@ -1042,11 +982,6 @@ def __init__(self, p4port, p4user, p4passwd, p4client, **kwargs): assert kwargs['mode'] == "copy", "P4Sync can only be used in mode=copy" self.branch = None Source.__init__(self, **kwargs) - self.addFactoryArguments(p4port=p4port, - p4user=p4user, - p4passwd=p4passwd, - p4client=p4client, - ) self.args['p4port'] = p4port self.args['p4user'] = p4user self.args['p4passwd'] = p4passwd @@ -1093,10 +1028,6 @@ def __init__(self, repourl=None, branch=None, progress=False, **kwargs): raise ValueError("you must provide a repository uri in 'repourl'") if (not branch): raise ValueError("you must provide a default branch in 'branch'") - self.addFactoryArguments(repourl=repourl, - branch=branch, - progress=progress, - ) self.args.update({'branch': branch, 'progress': progress, }) diff --git a/master/buildbot/steps/source/svn.py b/master/buildbot/steps/source/svn.py index 17797a0e7c6..8c34a455925 100644 --- a/master/buildbot/steps/source/svn.py +++ b/master/buildbot/steps/source/svn.py @@ -51,17 +51,6 @@ def __init__(self, repourl=None, baseURL=None, mode='incremental', self.method=method self.mode = mode Source.__init__(self, **kwargs) - self.addFactoryArguments(repourl=repourl, - baseURL=baseURL, - mode=mode, - method=method, - defaultBranch=defaultBranch, - password=password, - username=username, - extra_args=extra_args, - keep_on_purge=keep_on_purge, - depth=depth, - ) errors = [] if self.mode not in self.possible_modes: errors.append("mode %s is not one of %s" % (self.mode, self.possible_modes)) diff --git a/master/buildbot/steps/subunit.py b/master/buildbot/steps/subunit.py index f4f0ca19c4c..2de89c7de7a 100644 --- a/master/buildbot/steps/subunit.py +++ b/master/buildbot/steps/subunit.py @@ -24,7 +24,6 @@ class SubunitShellCommand(ShellCommand): def __init__(self, failureOnNoTests=False, *args, **kwargs): ShellCommand.__init__(self, *args, **kwargs) self.failureOnNoTests = failureOnNoTests - self.addFactoryArguments(failureOnNoTests=failureOnNoTests) # importing here gets around an import loop from buildbot.process import subunitlogobserver diff --git a/master/buildbot/steps/transfer.py b/master/buildbot/steps/transfer.py index b460fa8bacb..cf9b620c038 100644 --- a/master/buildbot/steps/transfer.py +++ b/master/buildbot/steps/transfer.py @@ -229,15 +229,6 @@ def __init__(self, slavesrc, masterdest, keepstamp=False, url=None, **buildstep_kwargs): BuildStep.__init__(self, **buildstep_kwargs) - self.addFactoryArguments(slavesrc=slavesrc, - masterdest=masterdest, - workdir=workdir, - maxsize=maxsize, - blocksize=blocksize, - mode=mode, - keepstamp=keepstamp, - url=url, - ) self.slavesrc = slavesrc self.masterdest = masterdest @@ -309,14 +300,6 @@ def __init__(self, slavesrc, masterdest, workdir=None, maxsize=None, blocksize=16*1024, compress=None, url=None, **buildstep_kwargs): BuildStep.__init__(self, **buildstep_kwargs) - self.addFactoryArguments(slavesrc=slavesrc, - masterdest=masterdest, - workdir=workdir, - maxsize=maxsize, - blocksize=blocksize, - compress=compress, - url=url, - ) self.slavesrc = slavesrc self.masterdest = masterdest @@ -428,13 +411,6 @@ def __init__(self, mastersrc, slavedest, workdir=None, maxsize=None, blocksize=16*1024, mode=None, **buildstep_kwargs): BuildStep.__init__(self, **buildstep_kwargs) - self.addFactoryArguments(mastersrc=mastersrc, - slavedest=slavedest, - workdir=workdir, - maxsize=maxsize, - blocksize=blocksize, - mode=mode, - ) self.mastersrc = mastersrc self.slavedest = slavedest @@ -499,13 +475,6 @@ def __init__(self, s, slavedest, workdir=None, maxsize=None, blocksize=16*1024, mode=None, **buildstep_kwargs): BuildStep.__init__(self, **buildstep_kwargs) - self.addFactoryArguments(s=s, - slavedest=slavedest, - workdir=workdir, - maxsize=maxsize, - blocksize=blocksize, - mode=mode, - ) self.s = s self.slavedest = slavedest @@ -558,7 +527,6 @@ def __init__(self, o, slavedest, **buildstep_kwargs): del buildstep_kwargs['s'] s = json.dumps(o) StringDownload.__init__(self, s=s, slavedest=slavedest, **buildstep_kwargs) - self.addFactoryArguments(o=o) class JSONPropertiesDownload(StringDownload): diff --git a/master/buildbot/steps/trigger.py b/master/buildbot/steps/trigger.py index ee93cc4d5e0..2cb85e3fa8b 100644 --- a/master/buildbot/steps/trigger.py +++ b/master/buildbot/steps/trigger.py @@ -56,13 +56,6 @@ def __init__(self, schedulerNames=[], sourceStamp=None, updateSourceStamp=None, self.running = False self.ended = False LoggingBuildStep.__init__(self, **kwargs) - self.addFactoryArguments(schedulerNames=schedulerNames, - sourceStamp=sourceStamp, - updateSourceStamp=updateSourceStamp, - alwaysUseLatest=alwaysUseLatest, - waitForFinish=waitForFinish, - set_properties=set_properties, - copy_properties=copy_properties) def interrupt(self, reason): if self.running and not self.ended: diff --git a/master/buildbot/steps/vstudio.py b/master/buildbot/steps/vstudio.py index 65cecbad206..eea97e46f3d 100644 --- a/master/buildbot/steps/vstudio.py +++ b/master/buildbot/steps/vstudio.py @@ -131,17 +131,6 @@ def __init__(self, self.PATH = PATH # always upcall ! ShellCommand.__init__(self, **kwargs) - self.addFactoryArguments( - installdir = installdir, - mode = mode, - projectfile = projectfile, - config = config, - useenv = useenv, - project = project, - INCLUDE = INCLUDE, - LIB = LIB, - PATH = PATH - ) def setupLogfiles(self, cmd, logfiles): logwarnings = self.addLog("warnings") @@ -305,7 +294,6 @@ def __init__(self, arch = "x86", **kwargs): # always upcall ! VisualStudio.__init__(self, **kwargs) - self.addFactoryArguments(arch = arch) def setupEnvironment(self, cmd): VisualStudio.setupEnvironment(self, cmd) From fe0de9359599f78dacc2896abe5b706f640783e2 Mon Sep 17 00:00:00 2001 From: Tom Prince Date: Thu, 23 Feb 2012 01:07:27 -0500 Subject: [PATCH 003/136] Don't bother copying build step arguments. *args and **kwargs automatically make copies of what get passed to them. --- master/buildbot/process/build.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/master/buildbot/process/build.py b/master/buildbot/process/build.py index ec0f0299558..d8dee254589 100644 --- a/master/buildbot/process/build.py +++ b/master/buildbot/process/build.py @@ -298,8 +298,6 @@ def setupBuild(self, expectations): sps = [] for factory, args, kwargs in self.stepFactories: - args = tuple(args) - kwargs = kwargs.copy() try: step = factory(*args, **kwargs) except: From 87ba7381dd04cfc81b14d773e81c6169ca5f7c2c Mon Sep 17 00:00:00 2001 From: Tom Prince Date: Thu, 23 Feb 2012 01:44:47 -0500 Subject: [PATCH 004/136] BuildStep: Make getStepFactory return a factory function. Previously, it returned a factory, and arguments for the factory. This was used to print an error when the factory failed. This is moved into the factory function itself. The config also checks step locks. This functionality will be removed. --- master/buildbot/process/build.py | 10 ++-------- master/buildbot/process/buildstep.py | 9 ++++++++- .../buildbot/test/unit/test_process_build.py | 20 +++++++++---------- master/buildbot/test/util/steps.py | 6 ++---- 4 files changed, 21 insertions(+), 24 deletions(-) diff --git a/master/buildbot/process/build.py b/master/buildbot/process/build.py index d8dee254589..2c3ffe5e6f4 100644 --- a/master/buildbot/process/build.py +++ b/master/buildbot/process/build.py @@ -297,14 +297,8 @@ def setupBuild(self, expectations): stepnames = {} sps = [] - for factory, args, kwargs in self.stepFactories: - try: - step = factory(*args, **kwargs) - except: - log.msg("error while creating step, factory=%s, args=%s, kwargs=%s" - % (factory, args, kwargs)) - raise - + for factory in self.stepFactories: + step = factory() step.setBuild(self) step.setBuildSlave(self.slavebuilder.slave) if callable (self.workdir): diff --git a/master/buildbot/process/buildstep.py b/master/buildbot/process/buildstep.py index aacbf2d167d..a575ac80b7e 100644 --- a/master/buildbot/process/buildstep.py +++ b/master/buildbot/process/buildstep.py @@ -440,7 +440,14 @@ def __init__(self, **kwargs): def __new__(klass, *args, **kwargs): self = object.__new__(klass) klass.__init__(self, *args, **kwargs) - self.factory = (klass, args, kwargs) + def factory(): + try: + return klass(*args, **kwargs) + except: + log.msg("error while creating step, factory=%s, args=%s, kwargs=%s" + % (klass, args, kwargs)) + raise + self.factory = factory return self def describe(self, done=False): diff --git a/master/buildbot/test/unit/test_process_build.py b/master/buildbot/test/unit/test_process_build.py index 697dbf5f70e..3ef7f888d6e 100644 --- a/master/buildbot/test/unit/test_process_build.py +++ b/master/buildbot/test/unit/test_process_build.py @@ -102,7 +102,7 @@ def testRunSuccessfulBuild(self): step = Mock() step.return_value = step step.startStep.return_value = SUCCESS - b.setStepFactories([(step, (), {})]) + b.setStepFactories([lambda: step]) slavebuilder = Mock() @@ -117,7 +117,7 @@ def testStopBuild(self): step = Mock() step.return_value = step - b.setStepFactories([(step, (), {})]) + b.setStepFactories([lambda: step]) slavebuilder = Mock() @@ -148,8 +148,8 @@ def testAlwaysRunStepStopBuild(self): step2.return_value = step2 step2.alwaysRun = True b.setStepFactories([ - (step1, (), {}), - (step2, (), {}), + lambda: step1, + lambda: step2, ]) slavebuilder = Mock() @@ -196,7 +196,7 @@ def claim(owner, access): step = Mock() step.return_value = step step.startStep.return_value = SUCCESS - b.setStepFactories([(step, (), {})]) + b.setStepFactories([lambda: step]) b.startBuild(FakeBuildStatus(), None, slavebuilder) @@ -225,7 +225,7 @@ def claim(owner, access): step = Mock() step.return_value = step step.startStep.return_value = SUCCESS - b.setStepFactories([(step, (), {})]) + b.setStepFactories([lambda: step]) real_lock.claim(Mock(), l.access('counting')) @@ -252,7 +252,7 @@ def testStopBuildWaitingForLocks(self): step.return_value = step step.startStep.return_value = SUCCESS step.alwaysRun = False - b.setStepFactories([(step, (), {})]) + b.setStepFactories([lambda: step]) real_lock.claim(Mock(), l.access('counting')) @@ -285,7 +285,7 @@ def testStopBuildWaitingForLocks_lostRemote(self): step.return_value = step step.startStep.return_value = SUCCESS step.alwaysRun = False - b.setStepFactories([(step, {})]) + b.setStepFactories([lambda: step]) real_lock.claim(Mock(), l.access('counting')) @@ -316,9 +316,7 @@ def testStopBuildWaitingForStepLocks(self): real_lock = b.builder.botmaster.getLockByID(l).getLock(slavebuilder) step = LoggingBuildStep(locks=[lock_access]) - def factory(*args): - return step - b.setStepFactories([(factory, (), {})]) + b.setStepFactories([lambda: step]) real_lock.claim(Mock(), l.access('counting')) diff --git a/master/buildbot/test/util/steps.py b/master/buildbot/test/util/steps.py index 526cb709514..b15d87ca09e 100644 --- a/master/buildbot/test/util/steps.py +++ b/master/buildbot/test/util/steps.py @@ -71,10 +71,8 @@ def setupStep(self, step, slave_version={'*':"99.99"}, slave_env={}): @param slave_env: environment from the slave at slave startup """ - # yes, Virginia, "factory" refers both to the tuple and its first - # element TODO: fix that up - factory, args, kwargs = step.getStepFactory() - step = self.step = factory(*args, **kwargs) + factory = step.getStepFactory() + step = self.step = factory() # step.build From f114f217e971c59978b077a56446dfb4a6326267 Mon Sep 17 00:00:00 2001 From: Tom Prince Date: Thu, 23 Feb 2012 02:06:04 -0500 Subject: [PATCH 005/136] Remove deprecated support for passing step factories a tuples. Instead, we support either BuildStep objects, or 0-argument callables. --- master/buildbot/process/factory.py | 37 ++++++------------------------ 1 file changed, 7 insertions(+), 30 deletions(-) diff --git a/master/buildbot/process/factory.py b/master/buildbot/process/factory.py index e44e46ca430..f30fabd46ed 100644 --- a/master/buildbot/process/factory.py +++ b/master/buildbot/process/factory.py @@ -13,13 +13,10 @@ # # Copyright Buildbot Team Members -import warnings - from twisted.python import deprecate, versions from buildbot import util from buildbot.process.build import Build -from buildbot.process.buildstep import BuildStep from buildbot.steps.source import CVS, SVN from buildbot.steps.shell import Configure, Compile, Test, PerlModuleTest @@ -54,18 +51,9 @@ class BuildFactory(util.ComparableMixin): compare_attrs = ['buildClass', 'steps', 'useProgress', 'workdir'] def __init__(self, steps=None): - if steps is None: - steps = [] - self.steps = [self._makeStepFactory(s) for s in steps] - - def _makeStepFactory(self, step_or_factory): - if isinstance(step_or_factory, BuildStep): - return step_or_factory.getStepFactory() - warnings.warn( - "Passing a BuildStep subclass to factory.addStep is deprecated. " + - "Please pass a BuildStep instance instead. Support will be dropped in v0.8.7.", - DeprecationWarning, stacklevel=3) - return step_or_factory + self.steps = [] + if steps: + self.addSteps(steps) def newBuild(self, requests): """Create a new Build instance. @@ -79,22 +67,11 @@ def newBuild(self, requests): b.setStepFactories(self.steps) return b - def addStep(self, step_or_factory, **kwargs): - if isinstance(step_or_factory, BuildStep): - if kwargs: - raise ArgumentsInTheWrongPlace() - s = step_or_factory.getStepFactory() - elif type(step_or_factory) == type(BuildStep) and \ - issubclass(step_or_factory, BuildStep): - s = (step_or_factory, dict(kwargs)) - warnings.warn( - "Passing a BuildStep subclass to factory.addStep is deprecated. " + - "Please pass a BuildStep instance instead. Support will be dropped in v0.8.7.", - DeprecationWarning, stacklevel=2) - + def addStep(self, step): + if callable(step): + self.steps.append(step) else: - raise ValueError('%r is not a BuildStep nor BuildStep subclass' % step_or_factory) - self.steps.append(s) + self.steps.append(step.getStepFactory()) def addSteps(self, steps): for s in steps: From af7b6149b82a6ee648d22dbc1f18678cfbd872b9 Mon Sep 17 00:00:00 2001 From: Tom Prince Date: Thu, 23 Feb 2012 02:10:17 -0500 Subject: [PATCH 006/136] MasterConfig: Don't check for slave lock conflicts. With the change to representing step factories as callables, we no longer are able to examine the step arguments to check for a locks argument, so remove the check. --- master/buildbot/config.py | 9 --------- master/buildbot/test/unit/test_config.py | 20 ++------------------ 2 files changed, 2 insertions(+), 27 deletions(-) diff --git a/master/buildbot/config.py b/master/buildbot/config.py index 12a9d809fe8..fe2e1299446 100644 --- a/master/buildbot/config.py +++ b/master/buildbot/config.py @@ -547,15 +547,6 @@ def check_lock(l): for l in b.locks: check_lock(l) - # factories don't necessarily need to implement a .steps attribute - # but in practice most do, so we'll check that if it exists - if not hasattr(b.factory, 'steps'): - continue - for s in b.factory.steps: - for l in s[2].get('locks', []): - check_lock(l) - - def check_builders(self, errors): # look both for duplicate builder names, and for builders pointing # to unknown slaves diff --git a/master/buildbot/test/unit/test_config.py b/master/buildbot/test/unit/test_config.py index 0da9333b65e..115eac07e32 100644 --- a/master/buildbot/test/unit/test_config.py +++ b/master/buildbot/test/unit/test_config.py @@ -700,8 +700,7 @@ def setup_basic_attrs(self): self.cfg.slaves = [ mock.Mock() ] self.cfg.builders = [ b1, b2 ] - def setup_builder_locks(self, builder_lock=None, dup_builder_lock=False, - step_lock=None, dup_step_lock=False): + def setup_builder_locks(self, builder_lock=None, dup_builder_lock=False): def bldr(name): b = mock.Mock() b.name = name @@ -720,11 +719,6 @@ def lock(name): b1.locks.append(lock(builder_lock)) if dup_builder_lock: b2.locks.append(lock(builder_lock)) - if step_lock: - s1, s2 = b1.factory.steps[0][2], b2.factory.steps[0][2] - s1['locks'].append(lock(step_lock)) - if dup_step_lock: - s2['locks'].append(lock(step_lock)) # tests @@ -767,23 +761,13 @@ def test_check_schedulers(self): self.assertNoConfigErrors(self.errors) - def test_check_locks_step_and_builder(self): - self.setup_builder_locks(builder_lock='l', step_lock='l') - self.cfg.check_locks(self.errors) - self.assertConfigError(self.errors, "Two locks share") - def test_check_locks_dup_builder_lock(self): self.setup_builder_locks(builder_lock='l', dup_builder_lock=True) self.cfg.check_locks(self.errors) self.assertConfigError(self.errors, "Two locks share") - def test_check_locks_dup_step_lock(self): - self.setup_builder_locks(step_lock='l', dup_step_lock=True) - self.cfg.check_locks(self.errors) - self.assertConfigError(self.errors, "Two locks share") - def test_check_locks(self): - self.setup_builder_locks(builder_lock='bl', step_lock='sl') + self.setup_builder_locks(builder_lock='bl') self.cfg.check_locks(self.errors) self.assertNoConfigErrors(self.errors) From 88cc9ab41fb4d2eb2b07276b03a8da4369127230 Mon Sep 17 00:00:00 2001 From: Tom Prince Date: Thu, 23 Feb 2012 10:04:25 -0500 Subject: [PATCH 007/136] Remove silly call of __init__ from __new__ in BuildStep. --- master/buildbot/process/buildstep.py | 1 - 1 file changed, 1 deletion(-) diff --git a/master/buildbot/process/buildstep.py b/master/buildbot/process/buildstep.py index a575ac80b7e..9204e9045bb 100644 --- a/master/buildbot/process/buildstep.py +++ b/master/buildbot/process/buildstep.py @@ -439,7 +439,6 @@ def __init__(self, **kwargs): def __new__(klass, *args, **kwargs): self = object.__new__(klass) - klass.__init__(self, *args, **kwargs) def factory(): try: return klass(*args, **kwargs) From 92feb5062728c5e8b555031c52836c02692e7245 Mon Sep 17 00:00:00 2001 From: Tom Prince Date: Wed, 14 Mar 2012 18:28:59 -0400 Subject: [PATCH 008/136] Remove s(). --- master/buildbot/process/factory.py | 15 +++------------ master/buildbot/test/unit/test_process_factory.py | 6 +----- 2 files changed, 4 insertions(+), 17 deletions(-) diff --git a/master/buildbot/process/factory.py b/master/buildbot/process/factory.py index f30fabd46ed..478c4070d23 100644 --- a/master/buildbot/process/factory.py +++ b/master/buildbot/process/factory.py @@ -13,20 +13,11 @@ # # Copyright Buildbot Team Members -from twisted.python import deprecate, versions - from buildbot import util from buildbot.process.build import Build from buildbot.steps.source import CVS, SVN from buildbot.steps.shell import Configure, Compile, Test, PerlModuleTest -# deprecated, use BuildFactory.addStep -@deprecate.deprecated(versions.Version("buildbot", 0, 8, 6)) -def s(steptype, **kwargs): - # convenience function for master.cfg files, to create step - # specification tuples - return (steptype, kwargs) - class ArgumentsInTheWrongPlace(Exception): """When calling BuildFactory.addStep(stepinstance), addStep() only takes one argument. You passed extra arguments to addStep(), which you probably @@ -176,7 +167,7 @@ def __init__(self, cvsroot, cvsmodule, mode = "clobber" if cvsCopy: mode = "copy" - source = s(CVS, cvsroot=cvsroot, cvsmodule=cvsmodule, mode=mode) + source = CVS(cvsroot=cvsroot, cvsmodule=cvsmodule, mode=mode) GNUAutoconf.__init__(self, source, configure=configure, configureEnv=configureEnv, compile=compile, @@ -190,7 +181,7 @@ def __init__(self, cvsroot, cvsmodule, compile="make all", test="make check", cvsCopy=False): mode = "update" - source = s(CVS, cvsroot=cvsroot, cvsmodule=cvsmodule, mode=mode) + source = CVS(cvsroot=cvsroot, cvsmodule=cvsmodule, mode=mode) GNUAutoconf.__init__(self, source, configure=configure, configureEnv=configureEnv, compile=compile, @@ -202,7 +193,7 @@ def __init__(self, svnurl, configure=None, configureEnv={}, compile="make all", test="make check"): - source = s(SVN, svnurl=svnurl, mode="update") + source = SVN(svnurl=svnurl, mode="update") GNUAutoconf.__init__(self, source, configure=configure, configureEnv=configureEnv, compile=compile, diff --git a/master/buildbot/test/unit/test_process_factory.py b/master/buildbot/test/unit/test_process_factory.py index 6d57a38d589..aded9b0bec6 100644 --- a/master/buildbot/test/unit/test_process_factory.py +++ b/master/buildbot/test/unit/test_process_factory.py @@ -16,7 +16,7 @@ from twisted.trial import unittest from mock import Mock -from buildbot.process.factory import BuildFactory, ArgumentsInTheWrongPlace, s +from buildbot.process.factory import BuildFactory, ArgumentsInTheWrongPlace from buildbot.process.buildstep import BuildStep class TestBuildFactory(unittest.TestCase): @@ -26,10 +26,6 @@ def test_init(self): factory = BuildFactory([step]) self.assertEqual(factory.steps, [(BuildStep, {})]) - def test_init_deprecated(self): - factory = BuildFactory([s(BuildStep)]) - self.assertEqual(factory.steps, [(BuildStep, {})]) - def test_addStep(self): step = BuildStep() factory = BuildFactory() From 798120c7ceabf005a4eb6c236d11be4f671d6796 Mon Sep 17 00:00:00 2001 From: Tom Prince Date: Sat, 31 Mar 2012 17:03:52 -0400 Subject: [PATCH 009/136] Create a _BuildStepFactory to capture the arguments to BuildStep subclasses. We use an instance of this class, rather than a closure mostly to make it easier to test that the right factories are getting created. --- master/buildbot/process/buildstep.py | 29 ++++++++++++++----- .../test/unit/test_process_factory.py | 21 +++++--------- 2 files changed, 29 insertions(+), 21 deletions(-) diff --git a/master/buildbot/process/buildstep.py b/master/buildbot/process/buildstep.py index 9204e9045bb..5fb1f909ad3 100644 --- a/master/buildbot/process/buildstep.py +++ b/master/buildbot/process/buildstep.py @@ -381,6 +381,26 @@ def _start(self): def __repr__(self): return "" % repr(self.command) +class _BuildStepFactory(util.ComparableMixin): + """ + This is a wrapper to record the arguments passed to as BuildStep subclass. + We use an instance of this class, rather than a closure mostly to make it + easier to test that the right factories are getting created. + """ + compare_attrs = ['factory', 'args', 'kwargs' ] + def __init__(self, factory, *args, **kwargs): + self.factory = factory + self.args = args + self.kwargs = kwargs + + def __call__(self): + try: + return self.factory(*self.args, **self.kwargs) + except: + log.msg("error while creating step, factory=%s, args=%s, kwargs=%s" + % (self.factory, self.args, self.kwargs)) + raise + class BuildStep(object, properties.PropertiesMixin): haltOnFailure = False @@ -439,14 +459,7 @@ def __init__(self, **kwargs): def __new__(klass, *args, **kwargs): self = object.__new__(klass) - def factory(): - try: - return klass(*args, **kwargs) - except: - log.msg("error while creating step, factory=%s, args=%s, kwargs=%s" - % (klass, args, kwargs)) - raise - self.factory = factory + self.factory = _BuildStepFactory(klass, *args, **kwargs) return self def describe(self, done=False): diff --git a/master/buildbot/test/unit/test_process_factory.py b/master/buildbot/test/unit/test_process_factory.py index aded9b0bec6..c8ffb7100cf 100644 --- a/master/buildbot/test/unit/test_process_factory.py +++ b/master/buildbot/test/unit/test_process_factory.py @@ -14,38 +14,33 @@ # Copyright Buildbot Team Members from twisted.trial import unittest -from mock import Mock -from buildbot.process.factory import BuildFactory, ArgumentsInTheWrongPlace -from buildbot.process.buildstep import BuildStep +from buildbot.process.factory import BuildFactory +from buildbot.process.buildstep import BuildStep, _BuildStepFactory class TestBuildFactory(unittest.TestCase): def test_init(self): step = BuildStep() factory = BuildFactory([step]) - self.assertEqual(factory.steps, [(BuildStep, {})]) + self.assertEqual(factory.steps, [_BuildStepFactory(BuildStep)]) def test_addStep(self): step = BuildStep() factory = BuildFactory() factory.addStep(step) - self.assertEqual(factory.steps, [(BuildStep, {})]) - - def test_addStep_deprecated(self): - factory = BuildFactory() - factory.addStep(BuildStep) - self.assertEqual(factory.steps, [(BuildStep, {})]) + self.assertEqual(factory.steps, [_BuildStepFactory(BuildStep)]) def test_addStep_notAStep(self): factory = BuildFactory() - self.assertRaises(ValueError, factory.addStep, Mock()) + # This fails because object doesn't have .getStepFactory + self.assertRaises(AttributeError, factory.addStep, object()) def test_addStep_ArgumentsInTheWrongPlace(self): factory = BuildFactory() - self.assertRaises(ArgumentsInTheWrongPlace, factory.addStep, BuildStep(), name="name") + self.assertRaises(TypeError, factory.addStep, BuildStep(), name="name") def test_addSteps(self): factory = BuildFactory() factory.addSteps([BuildStep(), BuildStep()]) - self.assertEqual(factory.steps, [(BuildStep, {}), (BuildStep, {})]) + self.assertEqual(factory.steps, [_BuildStepFactory(BuildStep), _BuildStepFactory(BuildStep)]) From 060c98297f759de8b9e6a678d9c472d29bc10579 Mon Sep 17 00:00:00 2001 From: Tom Prince Date: Sun, 1 Apr 2012 17:18:48 -0400 Subject: [PATCH 010/136] Add a IBuildStepFactory, and make BuildFactory require instances. Rather than having a class implementing __call__ representing step factories, use a class with a real method. --- master/buildbot/interfaces.py | 5 +++++ master/buildbot/process/build.py | 2 +- master/buildbot/process/buildstep.py | 7 ++++++- master/buildbot/process/factory.py | 7 ++----- master/buildbot/test/util/steps.py | 2 +- 5 files changed, 15 insertions(+), 8 deletions(-) diff --git a/master/buildbot/interfaces.py b/master/buildbot/interfaces.py index 1e91a69f98d..0fca488fe3a 100644 --- a/master/buildbot/interfaces.py +++ b/master/buildbot/interfaces.py @@ -1208,3 +1208,8 @@ def render(value): class IScheduler(Interface): pass + +class IBuildStepFactory(Interface): + def buildStep(): + """ + """ diff --git a/master/buildbot/process/build.py b/master/buildbot/process/build.py index 2c3ffe5e6f4..97ba78cf80b 100644 --- a/master/buildbot/process/build.py +++ b/master/buildbot/process/build.py @@ -298,7 +298,7 @@ def setupBuild(self, expectations): sps = [] for factory in self.stepFactories: - step = factory() + step = factory.buildStep() step.setBuild(self) step.setBuildSlave(self.slavebuilder.slave) if callable (self.workdir): diff --git a/master/buildbot/process/buildstep.py b/master/buildbot/process/buildstep.py index 5fb1f909ad3..62067d5f287 100644 --- a/master/buildbot/process/buildstep.py +++ b/master/buildbot/process/buildstep.py @@ -388,12 +388,14 @@ class _BuildStepFactory(util.ComparableMixin): easier to test that the right factories are getting created. """ compare_attrs = ['factory', 'args', 'kwargs' ] + implements(interfaces.IBuildStepFactory) + def __init__(self, factory, *args, **kwargs): self.factory = factory self.args = args self.kwargs = kwargs - def __call__(self): + def buildStep(self): try: return self.factory(*self.args, **self.kwargs) except: @@ -752,6 +754,9 @@ def _maybeEvaluate(value, *args, **kwargs): value = value(*args, **kwargs) return value +components.registerAdapter( + BuildStep.getStepFactory, + BuildStep, interfaces.IBuildStepFactory) components.registerAdapter( lambda step : interfaces.IProperties(step.build), BuildStep, interfaces.IProperties) diff --git a/master/buildbot/process/factory.py b/master/buildbot/process/factory.py index 478c4070d23..7516386b38c 100644 --- a/master/buildbot/process/factory.py +++ b/master/buildbot/process/factory.py @@ -13,7 +13,7 @@ # # Copyright Buildbot Team Members -from buildbot import util +from buildbot import interfaces, util from buildbot.process.build import Build from buildbot.steps.source import CVS, SVN from buildbot.steps.shell import Configure, Compile, Test, PerlModuleTest @@ -59,10 +59,7 @@ def newBuild(self, requests): return b def addStep(self, step): - if callable(step): - self.steps.append(step) - else: - self.steps.append(step.getStepFactory()) + self.steps.append(interfaces.IBuildStepFactory(step)) def addSteps(self, steps): for s in steps: diff --git a/master/buildbot/test/util/steps.py b/master/buildbot/test/util/steps.py index b15d87ca09e..bee83d0e02e 100644 --- a/master/buildbot/test/util/steps.py +++ b/master/buildbot/test/util/steps.py @@ -72,7 +72,7 @@ def setupStep(self, step, slave_version={'*':"99.99"}, slave_env={}): @param slave_env: environment from the slave at slave startup """ factory = step.getStepFactory() - step = self.step = factory() + step = self.step = factory.buildStep() # step.build From 2049177c1ad0a6db15fab2ae6b379fb69fc5ace5 Mon Sep 17 00:00:00 2001 From: John Carr Date: Mon, 16 Apr 2012 21:21:31 +0100 Subject: [PATCH 011/136] Allow Domain class to be specified by Connection class This allows subclasses of Connection to use a subclass of Domain without any fuss. --- master/buildbot/libvirtbuildslave.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/master/buildbot/libvirtbuildslave.py b/master/buildbot/libvirtbuildslave.py index 52f716fcd06..099d144cde7 100644 --- a/master/buildbot/libvirtbuildslave.py +++ b/master/buildbot/libvirtbuildslave.py @@ -109,6 +109,8 @@ class Connection(object): I am a wrapper around a libvirt Connection object. """ + DomainClass = Domain + def __init__(self, uri): self.uri = uri self.connection = libvirt.open(uri) @@ -117,7 +119,7 @@ def lookupByName(self, name): """ I lookup an existing prefined domain """ d = queue.executeInThread(self.connection.lookupByName, name) def _(res): - return Domain(self, res) + return self.DomainClass(self, res) d.addCallback(_) return d @@ -125,7 +127,7 @@ def create(self, xml): """ I take libvirt XML and start a new VM """ d = queue.executeInThread(self.connection.createXML, xml, 0) def _(res): - return Domain(self, res) + return self.DomainClass(self, res) d.addCallback(_) return d From c764c6f026834ecb101d40911616092770b21a22 Mon Sep 17 00:00:00 2001 From: John Carr Date: Mon, 16 Apr 2012 21:24:16 +0100 Subject: [PATCH 012/136] More wrappers for libvirt API Allows enumeration of libvirt VM's that are already running when buildbot starts. --- master/buildbot/libvirtbuildslave.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/master/buildbot/libvirtbuildslave.py b/master/buildbot/libvirtbuildslave.py index 099d144cde7..6353dba2a0a 100644 --- a/master/buildbot/libvirtbuildslave.py +++ b/master/buildbot/libvirtbuildslave.py @@ -93,6 +93,9 @@ def __init__(self, connection, domain): self.connection = connection self.domain = domain + def name(self): + return queue.executeInThread(self.domain.name) + def create(self): return queue.executeInThread(self.domain.create) @@ -131,6 +134,17 @@ def _(res): d.addCallback(_) return d + @defer.inlineCallbacks + def all(self): + domains = [] + domain_ids = yield queue.executeInThread(self.connection.listDomainsID) + + for did in domain_ids: + domain = yield queue.executeInThread(self.connection.lookupByID, did) + domains.append(self.DomainClass(self, domain)) + + defer.returnValue(domains) + class LibVirtSlave(AbstractLatentBuildSlave): From 6bb96d2912d948b994d7bf06929469c299701d98 Mon Sep 17 00:00:00 2001 From: John Carr Date: Mon, 16 Apr 2012 21:26:59 +0100 Subject: [PATCH 013/136] Can use inlineCallbacks now --- master/buildbot/libvirtbuildslave.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/master/buildbot/libvirtbuildslave.py b/master/buildbot/libvirtbuildslave.py index 6353dba2a0a..ca3b33d10cb 100644 --- a/master/buildbot/libvirtbuildslave.py +++ b/master/buildbot/libvirtbuildslave.py @@ -118,21 +118,17 @@ def __init__(self, uri): self.uri = uri self.connection = libvirt.open(uri) + @defer.inlineCallbacks def lookupByName(self, name): """ I lookup an existing prefined domain """ - d = queue.executeInThread(self.connection.lookupByName, name) - def _(res): - return self.DomainClass(self, res) - d.addCallback(_) - return d + d = yield queue.executeInThread(self.connection.lookupByName, name) + defer.returnValue(self.DomainClass(self, res)) + @defer.inlineCallbacks def create(self, xml): """ I take libvirt XML and start a new VM """ - d = queue.executeInThread(self.connection.createXML, xml, 0) - def _(res): - return self.DomainClass(self, res) - d.addCallback(_) - return d + d = yield queue.executeInThread(self.connection.createXML, xml, 0) + defer.returnVlalue(self.DomainClass(self, res)) @defer.inlineCallbacks def all(self): From 11edf46ea1a6a65b7bfe1ad611f7920229969169 Mon Sep 17 00:00:00 2001 From: John Carr Date: Mon, 16 Apr 2012 21:36:14 +0100 Subject: [PATCH 014/136] Cope with orphaned build slaves When insubstantiate_after_build is False you can get orphan slaves that are duplicated when you restart buildbot. This prevents that by enumerating existing VM's and attempting to match them to defined slaves. --- master/buildbot/libvirtbuildslave.py | 32 ++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/master/buildbot/libvirtbuildslave.py b/master/buildbot/libvirtbuildslave.py index ca3b33d10cb..8e535d4d2c5 100644 --- a/master/buildbot/libvirtbuildslave.py +++ b/master/buildbot/libvirtbuildslave.py @@ -160,6 +160,38 @@ def __init__(self, name, password, connection, hd_image, base_image = None, xml= self.domain = None + self.ready = False + self._find_existing_instance() + + @defer.inlineCallbacks + def _find_existing_instance(self): + """ + I find existing VMs that are already running that might be orphaned instances of this slave. + """ + if not self.connection: + defer.returnValue(None) + + domains = yield self.connection.all() + for d in domains: + name = yield d.name() + if name.startswith(self.name): + self.domain = d + self.substantiated = True + break + + self.ready = True + + def canStartBuild(self): + if not self.ready: + log.msg("Not accepting builds as existing domains not iterated") + return False + + if self.domain and not self.isConnected(): + log.msg("Not accepting builds as existing domain but slave not connected") + return False + + return AbstractLatentBuildSlave.canStartBuild(self) + def _prepare_base_image(self): """ I am a private method for creating (possibly cheap) copies of a From 1617ec53bb65dcede60677eafddad6d46cdc1b6b Mon Sep 17 00:00:00 2001 From: John Carr Date: Mon, 16 Apr 2012 21:40:06 +0100 Subject: [PATCH 015/136] Fix typo in log message --- master/buildbot/libvirtbuildslave.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/master/buildbot/libvirtbuildslave.py b/master/buildbot/libvirtbuildslave.py index 8e535d4d2c5..422b97b55b3 100644 --- a/master/buildbot/libvirtbuildslave.py +++ b/master/buildbot/libvirtbuildslave.py @@ -275,7 +275,7 @@ def stop_instance(self, fast=False): """ log.msg("Attempting to stop '%s'" % self.name) if self.domain is None: - log.msg("I don't think that domain is evening running, aborting") + log.msg("I don't think that domain is even running, aborting") return defer.succeed(None) domain = self.domain From 68de3502e61a8310fdd4b66f9d9148c204b54598 Mon Sep 17 00:00:00 2001 From: John Carr Date: Mon, 16 Apr 2012 21:49:43 +0100 Subject: [PATCH 016/136] Make sure we can't start new builds when a latent slave is shutting down --- master/buildbot/buildslave.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/master/buildbot/buildslave.py b/master/buildbot/buildslave.py index f7c39047532..51ce220a68f 100644 --- a/master/buildbot/buildslave.py +++ b/master/buildbot/buildslave.py @@ -754,6 +754,7 @@ class AbstractLatentBuildSlave(AbstractBuildSlave): substantiated = False substantiation_deferred = None substantiation_build = None + insubstantiating = False build_wait_timer = None _shutdown_callback_handle = None @@ -866,6 +867,11 @@ def _substantiation_failed(self, failure): subject = "Buildbot: buildslave %s never substantiated" % self.slavename return self._mail_missing_message(subject, text) + def canStartBuild(self): + if self.insubstantiating: + return False + return AbstractBuildSlave.canStartBuild(self) + def buildStarted(self, sb): assert self.substantiated self._clearBuildWaitTimer() @@ -889,7 +895,9 @@ def _setBuildWaitTimer(self): self.build_wait_timer = reactor.callLater( self.build_wait_timeout, self._soft_disconnect) + @defer.inlineCallbacks def insubstantiate(self, fast=False): + self.insubstantiating = True self._clearBuildWaitTimer() d = self.stop_instance(fast) if self._shutdown_callback_handle is not None: @@ -898,7 +906,8 @@ def insubstantiate(self, fast=False): reactor.removeSystemEventTrigger(handle) self.substantiated = False self.building.clear() # just to be sure - return d + yield d + self.insubstantiating = False def _soft_disconnect(self, fast=False): d = AbstractBuildSlave.disconnect(self) From 895ef265d07396fecba99cee3006a35bb1049eaa Mon Sep 17 00:00:00 2001 From: John Carr Date: Mon, 16 Apr 2012 22:14:56 +0100 Subject: [PATCH 017/136] Try and unify build_wait_timeout and insub_after_build When i added insubtantiate_after_build i wasn't aware of the build_wait_timeout option. They are both "how quickly should the slave go away" options, so unify them and support a "never turn the slave off" option. --- master/buildbot/buildslave.py | 12 ++++++++++-- master/buildbot/libvirtbuildslave.py | 12 ------------ 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/master/buildbot/buildslave.py b/master/buildbot/buildslave.py index 51ce220a68f..7c605713887 100644 --- a/master/buildbot/buildslave.py +++ b/master/buildbot/buildslave.py @@ -825,7 +825,7 @@ def clean_up(failure): return d def attached(self, bot): - if self.substantiation_deferred is None: + if self.substantiation_deferred is None and self.build_wait_timeout >= 0: msg = 'Slave %s received connection while not trying to ' \ 'substantiate. Disconnecting.' % (self.slavename,) log.msg(msg) @@ -882,7 +882,10 @@ def buildFinished(self, sb): self.building.remove(sb.builder_name) if not self.building: - self._setBuildWaitTimer() + if self.build_wait_timeout == 0: + self.insubstantiate() + else: + self._setBuildWaitTimer() def _clearBuildWaitTimer(self): if self.build_wait_timer is not None: @@ -892,6 +895,8 @@ def _clearBuildWaitTimer(self): def _setBuildWaitTimer(self): self._clearBuildWaitTimer() + if self.build_wait_timeout < 0: + return self.build_wait_timer = reactor.callLater( self.build_wait_timeout, self._soft_disconnect) @@ -910,6 +915,9 @@ def insubstantiate(self, fast=False): self.insubstantiating = False def _soft_disconnect(self, fast=False): + if not self.build_wait_timeout < 0: + return AbstractBuildSlave.disconnect(self) + d = AbstractBuildSlave.disconnect(self) if self.slave is not None: # this could be called when the slave needs to shut down, such as diff --git a/master/buildbot/libvirtbuildslave.py b/master/buildbot/libvirtbuildslave.py index 422b97b55b3..3b7fd1c0d6d 100644 --- a/master/buildbot/libvirtbuildslave.py +++ b/master/buildbot/libvirtbuildslave.py @@ -154,7 +154,6 @@ def __init__(self, name, password, connection, hd_image, base_image = None, xml= self.base_image = base_image self.xml = xml - self.insubstantiate_after_build = True self.cheap_copy = True self.graceful_shutdown = False @@ -302,14 +301,3 @@ def _disconnected(res): return d - def buildFinished(self, *args, **kwargs): - """ - I insubstantiate a slave after it has done a build, if that is - desired behaviour. - """ - AbstractLatentBuildSlave.buildFinished(self, *args, **kwargs) - if self.insubstantiate_after_build: - log.msg("Got buildFinished notification - attempting to insubstantiate") - self.insubstantiate() - - From 91c96e63a8228930eba5799d5a7d3121b434c941 Mon Sep 17 00:00:00 2001 From: John Carr Date: Mon, 16 Apr 2012 22:36:43 +0100 Subject: [PATCH 018/136] start_instance much more readable with inlineCallbacks --- master/buildbot/libvirtbuildslave.py | 34 ++++++++-------------------- 1 file changed, 10 insertions(+), 24 deletions(-) diff --git a/master/buildbot/libvirtbuildslave.py b/master/buildbot/libvirtbuildslave.py index 3b7fd1c0d6d..af06fa61371 100644 --- a/master/buildbot/libvirtbuildslave.py +++ b/master/buildbot/libvirtbuildslave.py @@ -221,6 +221,7 @@ def _log_result(res): d.addBoth(_log_result) return d + @defer.inlineCallbacks def start_instance(self, build): """ I start a new instance of a VM. @@ -234,36 +235,21 @@ def start_instance(self, build): if self.domain is not None: raise ValueError('domain active') - d = self._prepare_base_image() + yield self._prepare_base_image() - def _start(res): + try: if self.xml: - d = self.connection.create(self.xml) - def _xml_start(res): - self.domain = res - return - d.addCallback(_xml_start) - return d - d = self.connection.lookupByName(self.name) - def _really_start(res): - self.domain = res - return self.domain.create() - d.addCallback(_really_start) - return d - d.addCallback(_start) - - def _started(res): - return True - d.addCallback(_started) - - def _start_failed(failure): + self.domain = yield self.connection.create(self.xml) + else: + self.domain = yield self.connection.lookupByName(self.name) + yield self.domain.create() + except Exception, f: log.msg("Cannot start a VM (%s), failing gracefully and triggering a new build check" % self.name) log.err(failure) self.domain = None return False - d.addErrback(_start_failed) - - return d + + return True def stop_instance(self, fast=False): """ From 2da11349143e81e50841c04eba6914d9049f1570 Mon Sep 17 00:00:00 2001 From: John Carr Date: Sun, 29 Apr 2012 00:46:46 +0100 Subject: [PATCH 019/136] Complain nicely via the configuration loader if libvirt isn't available --- master/buildbot/libvirtbuildslave.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/master/buildbot/libvirtbuildslave.py b/master/buildbot/libvirtbuildslave.py index af06fa61371..eb27a05a818 100644 --- a/master/buildbot/libvirtbuildslave.py +++ b/master/buildbot/libvirtbuildslave.py @@ -19,8 +19,12 @@ from twisted.internet import defer, utils, reactor, threads from twisted.python import log from buildbot.buildslave import AbstractBuildSlave, AbstractLatentBuildSlave +from buildbot import config -import libvirt +try: + import libvirt +except ImportError: + libvirt = None class WorkQueue(object): @@ -148,6 +152,10 @@ def __init__(self, name, password, connection, hd_image, base_image = None, xml= missing_timeout=60*20, build_wait_timeout=60*10, properties={}, locks=None): AbstractLatentBuildSlave.__init__(self, name, password, max_builds, notify_on_missing, missing_timeout, build_wait_timeout, properties, locks) + + if not libvirt: + config.error("The python module 'libvirt' is needed to use a LibVirtSlave") + self.name = name self.connection = connection self.image = hd_image From 322ec7cc32557b4d9f9de8e237ed57c6980c41bc Mon Sep 17 00:00:00 2001 From: John Carr Date: Sun, 29 Apr 2012 01:43:17 +0100 Subject: [PATCH 020/136] Add some basic tests --- master/buildbot/libvirtbuildslave.py | 8 +- .../buildbot/test/unit/test_libvirtslave.py | 148 ++++++++++++++++++ 2 files changed, 152 insertions(+), 4 deletions(-) create mode 100644 master/buildbot/test/unit/test_libvirtslave.py diff --git a/master/buildbot/libvirtbuildslave.py b/master/buildbot/libvirtbuildslave.py index eb27a05a818..c39d7e0a9de 100644 --- a/master/buildbot/libvirtbuildslave.py +++ b/master/buildbot/libvirtbuildslave.py @@ -197,7 +197,7 @@ def canStartBuild(self): log.msg("Not accepting builds as existing domain but slave not connected") return False - return AbstractLatentBuildSlave.canStartBuild(self) + return AbstractLatentBuildSlave.canStartBuild(self) def _prepare_base_image(self): """ @@ -255,9 +255,9 @@ def start_instance(self, build): log.msg("Cannot start a VM (%s), failing gracefully and triggering a new build check" % self.name) log.err(failure) self.domain = None - return False - - return True + defer.returnValue(False) + + defer.returnValue(True) def stop_instance(self, fast=False): """ diff --git a/master/buildbot/test/unit/test_libvirtslave.py b/master/buildbot/test/unit/test_libvirtslave.py new file mode 100644 index 00000000000..8532ea3ff08 --- /dev/null +++ b/master/buildbot/test/unit/test_libvirtslave.py @@ -0,0 +1,148 @@ +# This file is part of Buildbot. Buildbot is free software: you can +# redistribute it and/or modify it under the terms of the GNU General Public +# License as published by the Free Software Foundation, version 2. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +# details. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, write to the Free Software Foundation, Inc., 51 +# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Copyright Buildbot Team Members + +import mock +from twisted.trial import unittest +from twisted.internet import defer, reactor +from buildbot import libvirtbuildslave, config +from buildbot.test.fake import fakemaster + +class TestLibVirtSlave(unittest.TestCase): + + class ConcreteBuildSlave(libvirtbuildslave.LibVirtSlave): + pass + + def test_constructor_nolibvirt(self): + self.patch(libvirtbuildslave, "libvirt", None) + self.assertRaises(config.ConfigErrors, self.ConcreteBuildSlave, + 'bot', 'pass', None, 'path', 'path') + + def test_constructor_minimal(self): + self.patch(libvirtbuildslave, "libvirt", mock.Mock()) + + connection = mock.Mock() + connection.all.return_value = [] + + bs = self.ConcreteBuildSlave('bot', 'pass', connection, 'path', 'otherpath') + self.assertEqual(bs.slavename, 'bot') + self.assertEqual(bs.password, 'pass') + self.assertEqual(bs.connection, connection) + self.assertEqual(bs.image, 'path') + self.assertEqual(bs.base_image, 'otherpath') + self.assertEqual(bs.keepalive_interval, 3600) + + +class TestWorkQueue(unittest.TestCase): + + def setUp(self): + self.queue = libvirtbuildslave.WorkQueue() + + def delayed_success(self): + def work(): + d = defer.Deferred() + reactor.callLater(0, d.callback, True) + return d + return work + + def delayed_errback(self): + def work(): + d = defer.Deferred() + reactor.callLater(0, d.errback, Failure("Test failure")) + return d + return work + + def expect_errback(self, d): + def shouldnt_get_called(f): + self.failUnlessEqual(True, False) + d.addCallback(shouldnt_get_called) + def errback(f): + #log.msg("errback called?") + pass + d.addErrback(errback) + return d + + def test_handle_exceptions(self): + def work(): + raise ValueError + return self.expect_errback(self.queue.execute(work)) + + def test_handle_immediate_errback(self): + def work(): + return defer.fail("Sad times") + return self.expect_errback(self.queue.execute(work)) + + def test_handle_delayed_errback(self): + work = self.delayed_errback() + return self.expect_errback(self.queue.execute(work)) + + def test_handle_immediate_success(self): + def work(): + return defer.succeed(True) + return self.queue.execute(work) + + def test_handle_delayed_success(self): + work = self.delayed_success() + return self.queue.execute(work) + + def test_single_pow_fires(self): + return self.queue.execute(self.delayed_success()) + + def test_single_pow_errors_gracefully(self): + d = self.queue.execute(self.delayed_errback()) + return self.expect_errback(d) + + def test_fail_doesnt_break_further_work(self): + self.expect_errback(self.queue.execute(self.delayed_errback())) + return self.queue.execute(self.delayed_success()) + + def test_second_pow_fires(self): + self.queue.execute(self.delayed_success()) + return self.queue.execute(self.delayed_success()) + + def test_work(self): + # We want these deferreds to fire in order + flags = {1: False, 2: False, 3: False } + + # When first deferred fires, flags[2] and flags[3] should still be false + # flags[1] shouldnt already be set, either + d1 = self.queue.execute(self.delayed_success()) + def cb1(res): + self.failUnlessEqual(flags[1], False) + flags[1] = True + self.failUnlessEqual(flags[2], False) + self.failUnlessEqual(flags[3], False) + d1.addCallback(cb1) + + # When second deferred fires, only flags[3] should be set + # flags[2] should definitely be False + d2 = self.queue.execute(self.delayed_success()) + def cb2(res): + assert flags[2] == False + flags[2] = True + assert flags[1] == True + assert flags[3] == False + d2.addCallback(cb2) + + # When third deferred fires, only flags[3] should be unset + d3 = self.queue.execute(self.delayed_success()) + def cb3(res): + assert flags[3] == False + flags[3] = True + assert flags[1] == True + assert flags[2] == True + d3.addCallback(cb3) + + return defer.DeferredList([d1, d2, d3], fireOnOneErrback=True) + From b47b07aac45ed0c51ab4bee5837792f4a7e2e65a Mon Sep 17 00:00:00 2001 From: John Carr Date: Sun, 29 Apr 2012 15:23:43 +0100 Subject: [PATCH 021/136] Add more tests --- master/buildbot/libvirtbuildslave.py | 15 +- .../buildbot/test/unit/test_libvirtslave.py | 145 +++++++++++++++++- 2 files changed, 145 insertions(+), 15 deletions(-) diff --git a/master/buildbot/libvirtbuildslave.py b/master/buildbot/libvirtbuildslave.py index c39d7e0a9de..2aa2e19184b 100644 --- a/master/buildbot/libvirtbuildslave.py +++ b/master/buildbot/libvirtbuildslave.py @@ -17,7 +17,7 @@ import os from twisted.internet import defer, utils, reactor, threads -from twisted.python import log +from twisted.python import log, failure from buildbot.buildslave import AbstractBuildSlave, AbstractLatentBuildSlave from buildbot import config @@ -125,14 +125,14 @@ def __init__(self, uri): @defer.inlineCallbacks def lookupByName(self, name): """ I lookup an existing prefined domain """ - d = yield queue.executeInThread(self.connection.lookupByName, name) + res = yield queue.executeInThread(self.connection.lookupByName, name) defer.returnValue(self.DomainClass(self, res)) @defer.inlineCallbacks def create(self, xml): """ I take libvirt XML and start a new VM """ - d = yield queue.executeInThread(self.connection.createXML, xml, 0) - defer.returnVlalue(self.DomainClass(self, res)) + res = yield queue.executeInThread(self.connection.createXML, xml, 0) + defer.returnValue(self.DomainClass(self, res)) @defer.inlineCallbacks def all(self): @@ -168,7 +168,7 @@ def __init__(self, name, password, connection, hd_image, base_image = None, xml= self.domain = None self.ready = False - self._find_existing_instance() + self._find_existing_deferred = self._find_existing_instance() @defer.inlineCallbacks def _find_existing_instance(self): @@ -241,7 +241,8 @@ def start_instance(self, build): in the list of defined virtual machines and start that. """ if self.domain is not None: - raise ValueError('domain active') + log.msg("Cannot start_instance '%s' as already active" % self.name) + defer.returnValue(False) yield self._prepare_base_image() @@ -253,7 +254,7 @@ def start_instance(self, build): yield self.domain.create() except Exception, f: log.msg("Cannot start a VM (%s), failing gracefully and triggering a new build check" % self.name) - log.err(failure) + log.err(failure.Failure()) self.domain = None defer.returnValue(False) diff --git a/master/buildbot/test/unit/test_libvirtslave.py b/master/buildbot/test/unit/test_libvirtslave.py index 8532ea3ff08..8d8b3a4a5a8 100644 --- a/master/buildbot/test/unit/test_libvirtslave.py +++ b/master/buildbot/test/unit/test_libvirtslave.py @@ -15,34 +15,163 @@ import mock from twisted.trial import unittest -from twisted.internet import defer, reactor +from twisted.internet import defer, reactor, utils from buildbot import libvirtbuildslave, config from buildbot.test.fake import fakemaster +class FakeLibVirt(object): + + def __init__(self, patch): + self.patch = patch + self.domains = {} + + self.libvirt = mock.Mock() + self.patch(libvirtbuildslave, "libvirt", self.libvirt) + + conn = self.libvirt_conn = self.libvirt.open.return_value = mock.Mock() + conn.listDomainsID.side_effect = self.domains.keys + conn.lookupByName.side_effect = lambda name: self.domains[name] + conn.lookupByID.side_effect = lambda name: self.domains[name] + + self.conn = libvirtbuildslave.Connection("test:///") + + def add_domain(self, name): + domain = mock.Mock() + domain.name.return_value = name + self.domains[name] = domain + return domain + + class TestLibVirtSlave(unittest.TestCase): class ConcreteBuildSlave(libvirtbuildslave.LibVirtSlave): pass + def setUp(self): + self.libvirt = FakeLibVirt(patch=self.patch) + def test_constructor_nolibvirt(self): self.patch(libvirtbuildslave, "libvirt", None) self.assertRaises(config.ConfigErrors, self.ConcreteBuildSlave, 'bot', 'pass', None, 'path', 'path') def test_constructor_minimal(self): - self.patch(libvirtbuildslave, "libvirt", mock.Mock()) - - connection = mock.Mock() - connection.all.return_value = [] - - bs = self.ConcreteBuildSlave('bot', 'pass', connection, 'path', 'otherpath') + conn = self.libvirt.conn + bs = self.ConcreteBuildSlave('bot', 'pass', conn, 'path', 'otherpath') + yield bs._find_existing_deferred self.assertEqual(bs.slavename, 'bot') self.assertEqual(bs.password, 'pass') - self.assertEqual(bs.connection, connection) + self.assertEqual(bs.connection, conn) self.assertEqual(bs.image, 'path') self.assertEqual(bs.base_image, 'otherpath') self.assertEqual(bs.keepalive_interval, 3600) + @defer.inlineCallbacks + def test_find_existing(self): + d = self.libvirt.add_domain("bot") + + bs = self.ConcreteBuildSlave('bot', 'pass', self.libvirt.conn, 'p', 'o') + yield bs._find_existing_deferred + + self.assertEqual(bs.domain.domain, d) + self.assertEqual(bs.substantiated, True) + + @defer.inlineCallbacks + def test_prepare_base_image_none(self): + self.patch(utils, "getProcessValue", mock.Mock()) + utils.getProcessValue.side_effect = lambda x,y: defer.succeed(0) + + bs = self.ConcreteBuildSlave('bot', 'pass', self.libvirt.conn, 'p', None) + yield bs._find_existing_deferred + yield bs._prepare_base_image() + + self.assertEqual(utils.getProcessValue.call_count, 0) + + @defer.inlineCallbacks + def test_prepare_base_image_cheap(self): + self.patch(utils, "getProcessValue", mock.Mock()) + utils.getProcessValue.side_effect = lambda x,y: defer.succeed(0) + + bs = self.ConcreteBuildSlave('bot', 'pass', self.libvirt.conn, 'p', 'o') + yield bs._find_existing_deferred + yield bs._prepare_base_image() + + utils.getProcessValue.assert_called_with( + "qemu-img", ["create", "-b", "o", "-f", "qcow2", "p"]) + + @defer.inlineCallbacks + def test_prepare_base_image_full(self): + pass + self.patch(utils, "getProcessValue", mock.Mock()) + utils.getProcessValue.side_effect = lambda x,y: defer.succeed(0) + + bs = self.ConcreteBuildSlave('bot', 'pass', self.libvirt.conn, 'p', 'o') + yield bs._find_existing_deferred + bs.cheap_copy = False + yield bs._prepare_base_image() + + utils.getProcessValue.assert_called_with( + "cp", ["o", "p"]) + + @defer.inlineCallbacks + def test_start_instance(self): + bs = self.ConcreteBuildSlave('b', 'p', self.libvirt.conn, 'p', 'o', + xml='') + + prep = mock.Mock() + prep.side_effect = lambda: defer.succeed(0) + self.patch(bs, "_prepare_base_image", prep) + + yield bs._find_existing_deferred + started = yield bs.start_instance(mock.Mock()) + + self.assertEqual(started, True) + + @defer.inlineCallbacks + def setup_canStartBuild(self): + bs = self.ConcreteBuildSlave('b', 'p', self.libvirt.conn, 'p', 'o') + yield bs._find_existing_deferred + bs.updateLocks() + defer.returnValue(bs) + + @defer.inlineCallbacks + def test_canStartBuild(self): + bs = yield self.setup_canStartBuild() + self.assertEqual(bs.canStartBuild(), True) + + @defer.inlineCallbacks + def test_canStartBuild_notready(self): + """ + If a LibVirtSlave hasnt finished scanning for existing VMs then we shouldn't + start builds on it as it might create a 2nd VM when we want to reuse the existing + one. + """ + bs = yield self.setup_canStartBuild() + bs.ready = False + self.assertEqual(bs.canStartBuild(), False) + + @defer.inlineCallbacks + def test_canStartBuild_domain_and_not_connected(self): + """ + If we've found that the VM this slave would instance already exists but hasnt + connected then we shouldn't start builds or we'll end up with a dupe. + """ + bs = yield self.setup_canStartBuild() + bs.domain = mock.Mock() + self.assertEqual(bs.canStartBuild(), False) + + @defer.inlineCallbacks + def test_canStartBuild_domain_and_connected(self): + """ + If we've found an existing VM and it is connected then we should start builds + """ + bs = yield self.setup_canStartBuild() + bs.domain = mock.Mock() + isconnected = mock.Mock() + isconnected.return_value = True + self.patch(bs, "isConnected", isconnected) + self.assertEqual(bs.canStartBuild(), True) + class TestWorkQueue(unittest.TestCase): From 0c8c56ef81ef3e416b933e922f40a87abbc26503 Mon Sep 17 00:00:00 2001 From: John Carr Date: Sun, 29 Apr 2012 19:54:58 +0100 Subject: [PATCH 022/136] Move libvirt fake to buildbot.test.fake --- master/buildbot/test/fake/libvirt.py | 67 +++++++++++++++++++ .../buildbot/test/unit/test_libvirtslave.py | 49 ++++---------- 2 files changed, 81 insertions(+), 35 deletions(-) create mode 100644 master/buildbot/test/fake/libvirt.py diff --git a/master/buildbot/test/fake/libvirt.py b/master/buildbot/test/fake/libvirt.py new file mode 100644 index 00000000000..2a32f6e71c8 --- /dev/null +++ b/master/buildbot/test/fake/libvirt.py @@ -0,0 +1,67 @@ +# This file is part of Buildbot. Buildbot is free software: you can +# redistribute it and/or modify it under the terms of the GNU General Public +# License as published by the Free Software Foundation, version 2. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +# details. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, write to the Free Software Foundation, Inc., 51 +# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Copyright Buildbot Team Members + +class Domain(object): + + def __init__(self, name, conn): + self.conn = conn + self._name = name + self.running = False + + def name(self): + return self._name + + def create(self): + self.running = True + + def shutdown(self): + self.running = False + + def destroy(self): + self.running = False + del self.conn[self._name] + + +class Connection(object): + + def __init__(self, uri): + self.uri = uri + self.domains = {} + + def createXML(self, xml, flags): + #FIXME: This should really parse the name out of the xml, i guess + d = self.fake_add("instance") + d.running = True + return d + + def listDomainsID(self): + return self.domains.keys() + + def lookupByName(self, name): + return self.domains[name] + + def lookupByID(self, ID): + return self.domains[ID] + + def fake_add(self, name): + d = Domain(name, self) + self.domains[name] = d + return d + + +def open(uri): + return Connection(uri) + + diff --git a/master/buildbot/test/unit/test_libvirtslave.py b/master/buildbot/test/unit/test_libvirtslave.py index 8d8b3a4a5a8..c4cc6e8fd15 100644 --- a/master/buildbot/test/unit/test_libvirtslave.py +++ b/master/buildbot/test/unit/test_libvirtslave.py @@ -17,29 +17,7 @@ from twisted.trial import unittest from twisted.internet import defer, reactor, utils from buildbot import libvirtbuildslave, config -from buildbot.test.fake import fakemaster - -class FakeLibVirt(object): - - def __init__(self, patch): - self.patch = patch - self.domains = {} - - self.libvirt = mock.Mock() - self.patch(libvirtbuildslave, "libvirt", self.libvirt) - - conn = self.libvirt_conn = self.libvirt.open.return_value = mock.Mock() - conn.listDomainsID.side_effect = self.domains.keys - conn.lookupByName.side_effect = lambda name: self.domains[name] - conn.lookupByID.side_effect = lambda name: self.domains[name] - - self.conn = libvirtbuildslave.Connection("test:///") - - def add_domain(self, name): - domain = mock.Mock() - domain.name.return_value = name - self.domains[name] = domain - return domain +from buildbot.test.fake import fakemaster, libvirt class TestLibVirtSlave(unittest.TestCase): @@ -48,7 +26,9 @@ class ConcreteBuildSlave(libvirtbuildslave.LibVirtSlave): pass def setUp(self): - self.libvirt = FakeLibVirt(patch=self.patch) + self.patch(libvirtbuildslave, "libvirt", libvirt) + self.conn = libvirtbuildslave.Connection("test://") + self.lvconn = self.conn.connection def test_constructor_nolibvirt(self): self.patch(libvirtbuildslave, "libvirt", None) @@ -56,22 +36,21 @@ def test_constructor_nolibvirt(self): 'bot', 'pass', None, 'path', 'path') def test_constructor_minimal(self): - conn = self.libvirt.conn - bs = self.ConcreteBuildSlave('bot', 'pass', conn, 'path', 'otherpath') + bs = self.ConcreteBuildSlave('bot', 'pass', self.conn, 'path', 'otherpath') yield bs._find_existing_deferred self.assertEqual(bs.slavename, 'bot') self.assertEqual(bs.password, 'pass') - self.assertEqual(bs.connection, conn) + self.assertEqual(bs.connection, self.conn) self.assertEqual(bs.image, 'path') self.assertEqual(bs.base_image, 'otherpath') self.assertEqual(bs.keepalive_interval, 3600) @defer.inlineCallbacks def test_find_existing(self): - d = self.libvirt.add_domain("bot") + d = self.lvconn.fake_add("bot") - bs = self.ConcreteBuildSlave('bot', 'pass', self.libvirt.conn, 'p', 'o') - yield bs._find_existing_deferred + bs = self.ConcreteBuildSlave('bot', 'pass', self.conn, 'p', 'o') + yield bs._find_existing_deferred self.assertEqual(bs.domain.domain, d) self.assertEqual(bs.substantiated, True) @@ -81,7 +60,7 @@ def test_prepare_base_image_none(self): self.patch(utils, "getProcessValue", mock.Mock()) utils.getProcessValue.side_effect = lambda x,y: defer.succeed(0) - bs = self.ConcreteBuildSlave('bot', 'pass', self.libvirt.conn, 'p', None) + bs = self.ConcreteBuildSlave('bot', 'pass', self.conn, 'p', None) yield bs._find_existing_deferred yield bs._prepare_base_image() @@ -92,7 +71,7 @@ def test_prepare_base_image_cheap(self): self.patch(utils, "getProcessValue", mock.Mock()) utils.getProcessValue.side_effect = lambda x,y: defer.succeed(0) - bs = self.ConcreteBuildSlave('bot', 'pass', self.libvirt.conn, 'p', 'o') + bs = self.ConcreteBuildSlave('bot', 'pass', self.conn, 'p', 'o') yield bs._find_existing_deferred yield bs._prepare_base_image() @@ -105,7 +84,7 @@ def test_prepare_base_image_full(self): self.patch(utils, "getProcessValue", mock.Mock()) utils.getProcessValue.side_effect = lambda x,y: defer.succeed(0) - bs = self.ConcreteBuildSlave('bot', 'pass', self.libvirt.conn, 'p', 'o') + bs = self.ConcreteBuildSlave('bot', 'pass', self.conn, 'p', 'o') yield bs._find_existing_deferred bs.cheap_copy = False yield bs._prepare_base_image() @@ -115,7 +94,7 @@ def test_prepare_base_image_full(self): @defer.inlineCallbacks def test_start_instance(self): - bs = self.ConcreteBuildSlave('b', 'p', self.libvirt.conn, 'p', 'o', + bs = self.ConcreteBuildSlave('b', 'p', self.conn, 'p', 'o', xml='') prep = mock.Mock() @@ -129,7 +108,7 @@ def test_start_instance(self): @defer.inlineCallbacks def setup_canStartBuild(self): - bs = self.ConcreteBuildSlave('b', 'p', self.libvirt.conn, 'p', 'o') + bs = self.ConcreteBuildSlave('b', 'p', self.conn, 'p', 'o') yield bs._find_existing_deferred bs.updateLocks() defer.returnValue(bs) From 7d1eeba9958dddfa2fde48aaea8c47bd63db72b9 Mon Sep 17 00:00:00 2001 From: John Carr Date: Sun, 29 Apr 2012 20:02:35 +0100 Subject: [PATCH 023/136] Update documentation for build_wait_timeout --- master/docs/manual/cfg-buildslaves.rst | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/master/docs/manual/cfg-buildslaves.rst b/master/docs/manual/cfg-buildslaves.rst index 969801398ab..37db52bd7b6 100644 --- a/master/docs/manual/cfg-buildslaves.rst +++ b/master/docs/manual/cfg-buildslaves.rst @@ -154,7 +154,19 @@ very useful for some situations. The buildslaves that are started on-demand are called "latent" buildslaves. As of this writing, buildbot ships with an abstract base class for building -latent buildslaves, and a concrete implementation for AWS EC2. +latent buildslaves, and a concrete implementation for AWS EC2 and for libvirt. + +Common Options +++++++++++++++ + +The following options are available for all latent buildslaves. + +``build_wait_timeout`` + This option allows you to specify how long a latent slave should wait after + a build for another build before it shuts down. It defaults to 10 minutes. + If this is set to 0 then the slave will be shut down immediately. If it is + less than 0 it will never automatically shutdown. + .. index:: AWS EC2 @@ -344,10 +356,6 @@ to wait for an EC2 instance to attach before considering the attempt to have failed, and email addresses to alert, respectively. ``missing_timeout`` defaults to 20 minutes. -The ``build_wait_timeout`` allows you to specify how long an :class:`EC2LatentBuildSlave` -should wait after a build for another build before it shuts down the EC2 -instance. It defaults to 10 minutes. - ``keypair_name`` and ``security_name`` allow you to specify different names for these AWS EC2 values. They both default to ``latent_buildbot_slave``. From 024182e4ef4c2ecdc110b26eb18f56fd3abf7576 Mon Sep 17 00:00:00 2001 From: Tom Prince Date: Tue, 8 May 2012 16:00:22 -0600 Subject: [PATCH 024/136] Rewrap some buildstep documentation. --- master/docs/developer/cls-buildsteps.rst | 323 +++++++++-------------- master/docs/manual/customization.rst | 79 ++---- 2 files changed, 150 insertions(+), 252 deletions(-) diff --git a/master/docs/developer/cls-buildsteps.rst b/master/docs/developer/cls-buildsteps.rst index 3b517eaf5cd..81b41d880c1 100644 --- a/master/docs/developer/cls-buildsteps.rst +++ b/master/docs/developer/cls-buildsteps.rst @@ -3,17 +3,16 @@ BuildSteps .. py:module:: buildbot.process.buildstep -There are a few parent classes that are used as base classes for real -buildsteps. This section describes the base classes. The "leaf" classes are -described in :doc:`../manual/cfg-buildsteps`. +There are a few parent classes that are used as base classes for real buildsteps. +This section describes the base classes. The "leaf" classes are described in :doc:`../manual/cfg-buildsteps`. BuildStep --------- .. py:class:: BuildStep(name, locks, haltOnFailure, flunkOnWarnings, flunkOnFailure, warnOnWarnings, warnOnFailure, alwaysRun, progressMetrics, useProgress, doStepIf, hideStepIf) - All constructor arguments must be given as keyword arguments. Each - constructor parameter is copied to the corresponding attribute. + All constructor arguments must be given as keyword arguments. + Each constructor parameter is copied to the corresponding attribute. .. py:attribute:: name @@ -25,14 +24,13 @@ BuildStep .. py:attribute:: progressMetrics - List of names of metrics that should be used to track the progress of - this build, and build ETA's for users. This is generally set in the + List of names of metrics that should be used to track the progress of this build, and build ETA's for users. + This is generally set in the .. py:attribute:: useProgress - If true (the default), then ETAs will be calculated for this step using - progress metrics. If the step is known to have unpredictable timing - (e.g., an incremental build), then this should be set to false. + If true (the default), then ETAs will be calculated for this step using progress metrics. + If the step is known to have unpredictable timing (e.g., an incremental build), then this should be set to false. .. py:attribute:: doStepIf @@ -41,20 +39,18 @@ BuildStep .. py:attribute:: hideStepIf - A callable or bool to determine whether this step should be shown in the - waterfall and build details pages. See :ref:`Buildstep-Common-Parameters` for details. + A callable or bool to determine whether this step should be shown in the waterfall and build details pages. + See :ref:`Buildstep-Common-Parameters` for details. The following attributes affect the behavior of the containing build: .. py:attribute:: haltOnFailure - If true, the build will halt on a failure of this step, and not execute - subsequent tests (except those with ``alwaysRun``). + If true, the build will halt on a failure of this step, and not execute subsequent tests (except those with ``alwaysRun``). .. py:attribute:: flunkOnWarnings - If true, the build will be marked as a failure if this step ends with - warnings. + If true, the build will be marked as a failure if this step ends with warnings. .. py:attribute:: flunkOnFailure @@ -62,22 +58,19 @@ BuildStep .. py:attribute:: warnOnWarnings - If true, the build will be marked as warnings, or worse, if this step - ends with warnings. + If true, the build will be marked as warnings, or worse, if this step ends with warnings. .. py:attribute:: warnOnFailure - If true, the build will be marked as warnings, or worse, if this step - fails. + If true, the build will be marked as warnings, or worse, if this step fails. .. py:attribute:: alwaysRun - If true, the step will run even if a previous step halts the build with - ``haltOnFailure``. + If true, the step will run even if a previous step halts the build with ``haltOnFailure``. - A step acts as a factory for more steps. See - :ref:`Writing-BuildStep-Constructors` for advice on writing subclass - constructors. The following methods handle this factory behavior. + A step acts as a factory for more steps. + See :ref:`Writing-BuildStep-Constructors` for advice on writing subclass constructors. + The following methods handle this factory behavior. .. py:method:: addFactoryArguments(..) @@ -91,19 +84,16 @@ BuildStep Get a factory for new instances of this step. The step can be created by calling the class with the given keyword arguments. - A few important pieces of information are not available when a step is - constructed, and are added later. These are set by the following methods; - the order in which these methods are called is not defined. + A few important pieces of information are not available when a step is constructed, and are added later. + These are set by the following methods; the order in which these methods are called is not defined. .. py:method:: setBuild(build) - :param build: the :class:`~buildbot.process.build.Build` instance - controlling this step. + :param build: the :class:`~buildbot.process.build.Build` instance controlling this step. - This method is called during setup to set the build instance - controlling this slave. Subclasses can override this to get access to - the build object as soon as it is available. The default - implementation sets the :attr:`build` attribute. + This method is called during setup to set the build instance controlling this slave. + Subclasses can override this to get access to the build object as soon as it is available. + The default implementation sets the :attr:`build` attribute. .. py:attribute:: build @@ -111,12 +101,10 @@ BuildStep .. py:method:: setBuildSlave(build) - :param build: the :class:`~buildbot.buildslave.BuildSlave` instance on - which this step will run. + :param build: the :class:`~buildbot.buildslave.BuildSlave` instance on which this step will run. - Similarly, this method is called with the build slave that will run - this step. The default implementation sets the :attr:`buildslave` - attribute. + Similarly, this method is called with the build slave that will run this step. + The default implementation sets the :attr:`buildslave` attribute. .. py:attribute:: buildslave @@ -126,36 +114,30 @@ BuildStep :param workdir: the default workdir, from the build - This method is called at build startup with the default workdir for the - build. Steps which allow a workdir to be specified, but want to - override it with the build's default workdir, can use this method to - apply the default. + This method is called at build startup with the default workdir for the build. + Steps which allow a workdir to be specified, but want to override it with the build's default workdir, can use this method to apply the default. .. py:method:: setStepStatus(status) :param status: step status :type status: :class:`~buildbot.status.buildstep.BuildStepStatus` - This method is called to set the status instance to which the step - should report. The default implementation sets :attr:`step_status`. + This method is called to set the status instance to which the step should report. + The default implementation sets :attr:`step_status`. .. py:attribute:: step_status - The :class:`~buildbot.status.buildstep.BuildStepStatus` object tracking - the status of this step. + The :class:`~buildbot.status.buildstep.BuildStepStatus` object tracking the status of this step. .. py:method:: setupProgress() - This method is called during build setup to give the step a chance to - set up progress tracking. It is only called if the build has - :attr:`useProgress` set. There is rarely any reason to override this - method. + This method is called during build setup to give the step a chance to set up progress tracking. + It is only called if the build has :attr:`useProgress` set. + There is rarely any reason to override this method. .. py:attribute:: progress - If the step is tracking progress, this is a - :class:`~buildbot.status.progress.StepProgress` instance performing - that task. + If the step is tracking progress, this is a :class:`~buildbot.status.progress.StepProgress` instance performing that task. Exeuction of the step itself is governed by the following methods and attributes. @@ -168,34 +150,29 @@ BuildStep Begin the step. This is the build's interface to step execution. Subclasses should override :meth:`start` to implement custom behaviors. - The method returns a Deferred that fires when the step finishes. It - fires with a tuple of ``(result, [extra text])``, where ``result`` is - one of the constants from :mod:`buildbot.status.builder`. The extra - text is a list of short strings which should be appended to the Build's - text results. For example, a test step may add ``17 failures`` to the - Build's status by this mechanism. + The method returns a Deferred that fires when the step finishes. + It fires with a tuple of ``(result, [extra text])``, where ``result`` is one of the constants from :mod:`buildbot.status.builder`. + The extra text is a list of short strings which should be appended to the Build's text results. + For example, a test step may add ``17 failures`` to the Build's status by this mechanism. - The deferred will errback if the step encounters an exception, - including an exception on the slave side (or if the slave goes away - altogether). Normal build/test failures will *not* cause an errback. + The deferred will errback if the step encounters an exception, including an exception on the slave side (or if the slave goes away altogether). + Normal build/test failures will *not* cause an errback. .. py:method:: start() :returns: ``None`` or :data:`~buildbot.status.results.SKIPPED` - Begin the step. Subclasses should override this method to do local - processing, fire off remote commands, etc. The parent method raises - :exc:`NotImplementedError`. + Begin the step. + Subclasses should override this method to do local processing, fire off remote commands, etc. + The parent method raises :exc:`NotImplementedError`. - Note that this method does *not* return a Deferred. When the step is - done, it should call :meth:`finished`, with a result -- a constant from - :mod:`buildbot.status.results`. The result will be handed off to - the :class:`~buildbot.process.build.Build`. + Note that this method does *not* return a Deferred. + When the step is done, it should call :meth:`finished`, with a result -- a constant from :mod:`buildbot.status.results`. + The result will be handed off to the :class:`~buildbot.process.build.Build`. - If the step encounters an exception, it should call :meth:`failed` with - a Failure object. This method automatically fails the whole build with - an exception. A common idiom is to add :meth:`failed` as an errback on - a Deferred:: + If the step encounters an exception, it should call :meth:`failed` with a Failure object. + This method automatically fails the whole build with an exception. + A common idiom is to add :meth:`failed` as an errback on a Deferred:: cmd = RemoteCommand(args) d = self.runCommand(cmd) @@ -204,78 +181,61 @@ BuildStep d.addCallback(succeed) d.addErrback(self.failed) - If the step decides it does not need to be run, :meth:`start` can - return the constant :data:`~buildbot.status.results.SKIPPED`. In this - case, it is not necessary to call :meth:`finished` directly. + If the step decides it does not need to be run, :meth:`start` can return the constant :data:`~buildbot.status.results.SKIPPED`. + In this case, it is not necessary to call :meth:`finished` directly. .. py:method:: finished(results) :param results: a constant from :mod:`~buildbot.status.results` - A call to this method indicates that the step is finished and the build - should analyze the results and perhaps proceed to the next step. The - step should not perform any additional processing after calling this - method. + A call to this method indicates that the step is finished and the build should analyze the results and perhaps proceed to the next step. + The step should not perform any additional processing after calling this method. .. py:method:: failed(failure) :param failure: a :class:`~twisted.python.failure.Failure` instance - Similar to :meth:`finished`, this method indicates that the step is - finished, but handles exceptions with appropriate logging and - diagnostics. + Similar to :meth:`finished`, this method indicates that the step is finished, but handles exceptions with appropriate logging and diagnostics. - This method handles :exc:`BuildStepFailed` specially, by calling - ``finished(FAILURE)``. This provides subclasses with a shortcut to - stop execution of a step by raising this failure in a context where - :meth:`failed` will catch it. + This method handles :exc:`BuildStepFailed` specially, by calling ``finished(FAILURE)``. + This provides subclasses with a shortcut to stop execution of a step by raising this failure in a context where :meth:`failed` will catch it. .. py:method:: interrupt(reason) :param reason: why the build was interrupted :type reason: string or :class:`~twisted.python.failure.Failure` - This method is used from various control interfaces to stop a running - step. The step should be brought to a halt as quickly as possible, by - cancelling a remote command, killing a local process, etc. The step - must still finish with either :meth:`finished` or :meth:`failed`. + This method is used from various control interfaces to stop a running step. + The step should be brought to a halt as quickly as possible, by cancelling a remote command, killing a local process, etc. + The step must still finish with either :meth:`finished` or :meth:`failed`. - The ``reason`` parameter can be a string or, when a slave is lost - during step processing, a :exc:`~twisted.internet.error.ConnectionLost` - failure. + The ``reason`` parameter can be a string or, when a slave is lost during step processing, a :exc:`~twisted.internet.error.ConnectionLost` failure. - The parent method handles any pending lock operations, and should be - called by implementations in subclasses. + The parent method handles any pending lock operations, and should be called by implementations in subclasses. .. py:attribute:: stopped - If false, then the step is running. If true, the step is not running, - or has been interrupted. + If false, then the step is running. If true, the step is not running, or has been interrupted. - This method provides a convenient way to summarize the status of the step - for status displays: + This method provides a convenient way to summarize the status of the step for status displays: .. py:method:: describe(done=False) :param done: If true, the step is finished. :returns: list of strings - Describe the step succinctly. The return value should be a sequence of - short strings suitable for display in a horizontally constrained space. + Describe the step succinctly. + The return value should be a sequence of short strings suitable for display in a horizontally constrained space. .. note:: - Be careful not to assume that the step has been started in this - method. In relatively rare circumstances, steps are described - before they have started. Ideally, unit tests should be used to - ensure that this method is resilient. + Be careful not to assume that the step has been started in this method. + In relatively rare circumstances, steps are described before they have started. + Ideally, unit tests should be used to ensure that this method is resilient. - Build steps support progress metrics - values that increase roughly - linearly during the execution of the step, and can thus be used to - calculate an expected completion time for a running step. A metric may be - a count of lines logged, tests executed, or files compiled. The build - mechanics will take care of translating this progress information into an - ETA for the user. + Build steps support progress metrics - values that increase roughly linearly during the execution of the step, and can thus be used to calculate an expected completion time for a running step. + A metric may be a count of lines logged, tests executed, or files compiled. + The build mechanics will take care of translating this progress information into an ETA for the user. .. py:method:: setProgress(metric, value) @@ -284,13 +244,13 @@ BuildStep :param value: the new value for the metric :type value: integer - Update a progress metric. This should be called by subclasses that can - provide useful progress-tracking information. + Update a progress metric. + This should be called by subclasses that can provide useful progress-tracking information. The specified metric name must be included in :attr:`progressMetrics`. - The following methods are provided as utilities to subclasses. These - methods should only be invoked after the step is started. + The following methods are provided as utilities to subclasses. + These methods should only be invoked after the step is started. .. py:method:: slaveVersion(command, oldVersion=None) @@ -299,19 +259,14 @@ BuildStep :param oldVersion: return value if the slave does not specify a version :returns: string - Fetch the version of the named command, as specified on the slave. In - practice, all commands on a slave have the same version, but passing - ``command`` is still useful to ensure that the command is implemented - on the slave. If the command is not implemented on the slave, - :meth:`slaveVersion` will return ``None``. + Fetch the version of the named command, as specified on the slave. + In practice, all commands on a slave have the same version, but passing ``command`` is still useful to ensure that the command is implemented on the slave. + If the command is not implemented on the slave, :meth:`slaveVersion` will return ``None``. - Versions take the form ``x.y`` where ``x`` and ``y`` are integers, and - are compared as expected for version numbers. + Versions take the form ``x.y`` where ``x`` and ``y`` are integers, and are compared as expected for version numbers. - Buildbot versions older than 0.5.0 did not support version queries; in - this case, :meth:`slaveVersion` will return ``oldVersion``. Since such - ancient versions of Buildbot are no longer in use, this functionality - is largely vestigial. + Buildbot versions older than 0.5.0 did not support version queries; in this case, :meth:`slaveVersion` will return ``oldVersion``. + Since such ancient versions of Buildbot are no longer in use, this functionality is largely vestigial. .. py:method:: slaveVersionIsOlderThan(command, minversion) @@ -320,8 +275,7 @@ BuildStep :param minversion: minimum version :returns: boolean - This method returns true if ``command`` is not implemented on the - slave, or if it is older than ``minversion``. + This method returns true if ``command`` is not implemented on the slave, or if it is older than ``minversion``. .. py:method:: getSlaveName() @@ -333,31 +287,26 @@ BuildStep :returns: Deferred - This method connects the given command to the step's buildslave and - runs it, returning the Deferred from - :meth:`~buildbot.process.buildstep.RemoteCommand.run`. + This method connects the given command to the step's buildslave and runs it, returning the Deferred from :meth:`~buildbot.process.buildstep.RemoteCommand.run`. .. py:method:: addURL(name, url) :param name: URL name :param url: the URL - Add a link to the given ``url``, with the given ``name`` to displays of - this step. This allows a step to provide links to data that is not - available in the log files. + Add a link to the given ``url``, with the given ``name`` to displays of this step. + This allows a step to provide links to data that is not available in the log files. - The :class:`BuildStep` class provides minimal support for log handling, - that is extended by the :class:`LoggingBuildStep` class. The following - methods provide some useful behaviors. These methods can be called while - the step is running, but not before. + The :class:`BuildStep` class provides minimal support for log handling, that is extended by the :class:`LoggingBuildStep` class. + The following methods provide some useful behaviors. + These methods can be called while the step is running, but not before. .. py:method:: addLog(name) :param name: log name :returns: :class:`~buildbot.status.logfile.LogFile` instance - Add a new logfile with the given name to the step, and return the log - file instance. + Add a new logfile with the given name to the step, and return the log file instance. .. py:method:: getLog(name) @@ -372,27 +321,24 @@ BuildStep :param name: log name :param text: content of the logfile - This method adds a new log and sets ``text`` as its content. This is - often useful to add a short logfile describing activities performed on - the master. The logfile is immediately closed, and no further data can - be added. + This method adds a new log and sets ``text`` as its content. + This is often useful to add a short logfile describing activities performed on the master. + The logfile is immediately closed, and no further data can be added. .. py:method:: addHTMLLog(name, html) :param name: log name :param html: content of the logfile - Similar to :meth:`addCompleteLog`, this adds a logfile containing - pre-formatted HTML, allowing more expressiveness than the text format - supported by :meth:`addCompleteLog`. + Similar to :meth:`addCompleteLog`, this adds a logfile containing pre-formatted HTML, allowing more expressiveness than the text format supported by :meth:`addCompleteLog`. .. py:method:: addLogObserver(logname, observer) :param logname: log name :param observer: log observer instance - Add a log observer for the named log. The named log need not have been - added already: the observer will be connected when the log is added. + Add a log observer for the named log. + The named log need not have been added already: the observer will be connected when the log is added. See :ref:`Adding-LogObservers` for more information on log observers. @@ -407,29 +353,25 @@ LoggingBuildStep The remaining arguments are passed to the :class:`BuildStep` constructor. - This subclass of :class:`BuildStep` is designed to help its subclasses run - remote commands that produce standard I/O logfiles. It: + This subclass of :class:`BuildStep` is designed to help its subclasses run remote commands that produce standard I/O logfiles. + It: * tracks progress using the length of the stdout logfile * provides hooks for summarizing and evaluating the command's result * supports lazy logfiles - * handles the mechanics of starting, interrupting, and finishing remote - commands + * handles the mechanics of starting, interrupting, and finishing remote commands * detects lost slaves and finishes with a status of :data:`~buildbot.status.results.RETRY` .. py:attribute:: logfiles - The logfiles to track, as described for :bb:step:`ShellCommand`. The - contents of the class-level ``logfiles`` attribute are combined with - those passed to the constructor, so subclasses may add log files with a - class attribute:: + The logfiles to track, as described for :bb:step:`ShellCommand`. + The contents of the class-level ``logfiles`` attribute are combined with those passed to the constructor, so subclasses may add log files with a class attribute:: class MyStep(LoggingBuildStep): logfiles = dict(debug='debug.log') - Note that lazy logfiles cannot be specified using this method; they - must be provided as constructor arguments. + Note that lazy logfiles cannot be specified using this method; they must be provided as constructor arguments. .. py:method:: startCommand(command) @@ -438,60 +380,47 @@ LoggingBuildStep .. note:: - This method permits an optional ``errorMessages`` parameter, - allowing errors detected early in the command process to be logged. + This method permits an optional ``errorMessages`` parameter, allowing errors detected early in the command process to be logged. It will be removed, and its use is deprecated. - Handle all of the mechanics of running the given command. This sets - up all required logfiles, keeps status text up to date, and calls the - utility hooks described below. When the command is finished, the step - is finished as well, making this class is unsuitable for steps that - run more than one command in sequence. + Handle all of the mechanics of running the given command. + This sets up all required logfiles, keeps status text up to date, and calls the utility hooks described below. + When the command is finished, the step is finished as well, making this class is unsuitable for steps that run more than one command in sequence. - Subclasses should override - :meth:`~buildbot.process.buildstep.BuildStep.start` and, after setting - up an appropriate command, call this method. :: + Subclasses should override :meth:`~buildbot.process.buildstep.BuildStep.start` and, after setting up an appropriate command, call this method. :: def start(self): cmd = RemoteShellCommand(..) self.startCommand(cmd, warnings) To refine the status output, override one or more of the following methods. - The :class:`LoggingBuildStep` implementations are stubs, so there is no - need to call the parent method. + The :class:`LoggingBuildStep` implementations are stubs, so there is no need to call the parent method. .. py:method:: commandComplete(command) :param command: the just-completed remote command - This is a general-purpose hook method for subclasses. It will be called - after the remote command has finished, but before any of the other hook - functions are called. + This is a general-purpose hook method for subclasses. + It will be called after the remote command has finished, but before any of the other hook functions are called. .. py:method:: createSummary(stdio) :param stdio: stdio :class:`~buildbot.status.logfile.LogFile` - This hook is designed to perform any summarization of the step, based - either on the contents of the stdio logfile, or on instance attributes - set earlier in the step processing. Implementations of this method - often call e.g., :meth:`~BuildStep.addURL`. + This hook is designed to perform any summarization of the step, based either on the contents of the stdio logfile, or on instance attributes set earlier in the step processing. + Implementations of this method often call e.g., :meth:`~BuildStep.addURL`. .. py:method:: evaluateCommand(command) :param command: the just-completed remote command :returns: step result from :mod:`buildbot.status.results` - This hook should decide what result the step should have. The default - implementation invokes ``log_eval_func`` if it exists, and looks at - :attr:`~buildbot.process.buildstep.RemoteCommand.rc` to distinguish - :data:`~buildbot.status.results.SUCCESS` from - :data:`~buildbot.status.results.FAILURE`. + This hook should decide what result the step should have. + The default implementation invokes ``log_eval_func`` if it exists, and looks at :attr:`~buildbot.process.buildstep.RemoteCommand.rc` to distinguish :data:`~buildbot.status.results.SUCCESS` from :data:`~buildbot.status.results.FAILURE`. - The remaining methods provide an embarassment of ways to set the summary of - the step that appears in the various status interfaces. The easiest way to - affect this output is to override :meth:`~BuildStep.describe`. If that is - not flexible enough, override :meth:`getText` and/or :meth:`getText2`. + The remaining methods provide an embarassment of ways to set the summary of the step that appears in the various status interfaces. + The easiest way to affect this output is to override :meth:`~BuildStep.describe`. + If that is not flexible enough, override :meth:`getText` and/or :meth:`getText2`. .. py:method:: getText(command, results) @@ -499,10 +428,8 @@ LoggingBuildStep :param results: step result from :meth:`evaluateCommand` :returns: a list of short strings - This method is the primary means of describing the step. The default - implementation calls :meth:`~BuildStep.describe`, which is usally the - easiest method to override, and then appends a string describing the - step status if it was not successful. + This method is the primary means of describing the step. + The default implementation calls :meth:`~BuildStep.describe`, which is usally the easiest method to override, and then appends a string describing the step status if it was not successful. .. py:method:: getText2(command, results) @@ -510,15 +437,13 @@ LoggingBuildStep :param results: step result from :meth:`evaluateCommand` :returns: a list of short strings - Like :meth:`getText`, this method summarizes the step's result, but it - is only called when that result affects the build, either by making it - halt, flunk, or end with warnings. + Like :meth:`getText`, this method summarizes the step's result, but it is only called when that result affects the build, either by making it halt, flunk, or end with warnings. Exceptions ---------- .. py:exception:: BuildStepFailed - This exception indicates that the buildstep has failed. It is useful as a - way to skip all subsequent processing when a step goes wrong. It is - handled by :meth:`BuildStep.failed`. + This exception indicates that the buildstep has failed. + It is useful as a way to skip all subsequent processing when a step goes wrong. + It is handled by :meth:`BuildStep.failed`. diff --git a/master/docs/manual/customization.rst b/master/docs/manual/customization.rst index ed1ee5073b9..0c7e0ee66e1 100644 --- a/master/docs/manual/customization.rst +++ b/master/docs/manual/customization.rst @@ -525,69 +525,46 @@ Properties Objects Writing New BuildSteps ---------------------- -While it is a good idea to keep your build process self-contained in -the source code tree, sometimes it is convenient to put more -intelligence into your Buildbot configuration. One way to do this is -to write a custom :class:`BuildStep`. Once written, this Step can be used in -the :file:`master.cfg` file. - -The best reason for writing a custom :class:`BuildStep` is to better parse the -results of the command being run. For example, a :class:`BuildStep` that knows -about JUnit could look at the logfiles to determine which tests had -been run, how many passed and how many failed, and then report more -detailed information than a simple ``rc==0`` -based `good/bad` -decision. - -Buildbot has acquired a large fleet of build steps, and sports a number of -knobs and hooks to make steps easier to write. This section may seem a bit -overwhelming, but most custom steps will only need to apply one or two of the -techniques outlined here. - -For complete documentation of the build step interfaces, see -:doc:`../developer/cls-buildsteps`. +While it is a good idea to keep your build process self-contained in the source code tree, sometimes it is convenient to put more intelligence into your Buildbot configuration. +One way to do this is to write a custom :class:`BuildStep`. +Once written, this Step can be used in the :file:`master.cfg` file. + +The best reason for writing a custom :class:`BuildStep` is to better parse the results of the command being run. +For example, a :class:`BuildStep` that knows about JUnit could look at the logfiles to determine which tests had been run, how many passed and how many failed, and then report more detailed information than a simple ``rc==0`` -based `good/bad` decision. + +Buildbot has acquired a large fleet of build steps, and sports a number of knobs and hooks to make steps easier to write. +This section may seem a bit overwhelming, but most custom steps will only need to apply one or two of the techniques outlined here. + +For complete documentation of the build step interfaces, see :doc:`../developer/cls-buildsteps`. .. _Writing-BuildStep-Constructors: Writing BuildStep Constructors ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Build steps act as their own factories, so their constructors are a bit more -complex than necessary. In the configuration file, a -:class:`~buildbot.process.buildstep.BuildStep` object is instantiated, but -because steps store state locally while executing, this object cannot be used -during builds. Instead, the build machinery calls the step's -:meth:`~buildbot.process.buildstep.BuildStep.getStepFactory` method to get a -tuple of a callable and keyword arguments that should be used to create a new -instance. +Build steps act as their own factories, so their constructors are a bit more complex than necessary. +In the configuration file, a :class:`~buildbot.process.buildstep.BuildStep` object is instantiated, but because steps store state locally while executing, this object cannot be used during builds. +Instead, the build machinery calls the step's :meth:`~buildbot.process.buildstep.BuildStep.getStepFactory` method to get a tuple of a callable and keyword arguments that should be used to create a new instance. Consider the use of a :class:`BuildStep` in :file:`master.cfg`:: f.addStep(MyStep(someopt="stuff", anotheropt=1)) -This creates a single instance of class ``MyStep``. However, Buildbot needs a -new object each time the step is executed. this is accomplished by storing the -information required to instantiate a new object in the -:attr:`~buildbot.process.buildstep.BuildStep.factory` attribute. When the time -comes to construct a new :class:`~buildbot.process.build.Build`, -:class:`~buildbot.process.factory.BuildFactory` consults this attribute (via -:meth:`~buildbot.process.buildstep.BuildStep.getStepFactory`) and instantiates -a new step object. +This creates a single instance of class ``MyStep``. +However, Buildbot needs a new object each time the step is executed. +this is accomplished by storing the information required to instantiate a new object in the :attr:`~buildbot.process.buildstep.BuildStep.factory` attribute. +When the time comes to construct a new :class:`~buildbot.process.build.Build`, :class:`~buildbot.process.factory.BuildFactory` consults this attribute (via :meth:`~buildbot.process.buildstep.BuildStep.getStepFactory`) and instantiates a new step object. -When writing a new step class, then, keep in mind are that you cannot do -anything "interesting" in the constructor -- limit yourself to checking and -storing arguments. Each constructor in a sequence of :class:`BuildStep` -subclasses must ensure the following: +When writing a new step class, then, keep in mind are that you cannot do anything "interesting" in the constructor -- limit yourself to checking and storing arguments. +Each constructor in a sequence of :class:`BuildStep` subclasses must ensure the following: * the parent class's constructor is called with all otherwise-unspecified keyword arguments. -* all keyword arguments for the class itself are passed to - :meth:`addFactoryArguments`. +* all keyword arguments for the class itself are passed to :meth:`addFactoryArguments`. -Keep a ``**kwargs`` argument on the end of your options, and pass that up to -the parent class's constructor. If the class overrides constructor arguments -for the parent class, those should be updated in ``kwargs``, rather than passed -directly (which will cause errors during instantiation). +Keep a ``**kwargs`` argument on the end of your options, and pass that up to the parent class's constructor. +If the class overrides constructor arguments for the parent class, those should be updated in ``kwargs``, rather than passed directly (which will cause errors during instantiation). The whole thing looks like this:: @@ -631,18 +608,14 @@ The whole thing looks like this:: Running Commands ~~~~~~~~~~~~~~~~ -To spawn a command in the buildslave, create a -:class:`~buildbot.process.buildstep.RemoteCommand` instance in your step's -``start`` method and run it with -:meth:`~buildbot.process.buildstep.BuildStep.runCommand`:: +To spawn a command in the buildslave, create a :class:`~buildbot.process.buildstep.RemoteCommand` instance in your step's ``start`` method and run it with :meth:`~buildbot.process.buildstep.BuildStep.runCommand`:: cmd = RemoteCommand(args) d = self.runCommand(cmd) To add a LogFile, use :meth:`~buildbot.process.buildstep.BuildStep.addLog`. -Make sure the log gets closed when it finishes. When giving a Logfile to a -:class:`~buildbot.process.buildstep.RemoteShellCommand`, just ask it to close -the log when the command completes:: +Make sure the log gets closed when it finishes. +When giving a Logfile to a :class:`~buildbot.process.buildstep.RemoteShellCommand`, just ask it to close the log when the command completes:: log = self.addLog('output') cmd.useLog(log, closeWhenFinished=True) From 369e4185bf1b8b50fe116d9d9eb0bd7624710582 Mon Sep 17 00:00:00 2001 From: Tom Prince Date: Tue, 8 May 2012 16:12:17 -0600 Subject: [PATCH 025/136] Update documentation to reflect new BuildStep constructors. Since addFactoryArguments no longer needs to be called, remove the documentation about it --- master/docs/developer/cls-buildsteps.rst | 16 ---------------- master/docs/manual/customization.rst | 21 ++------------------- 2 files changed, 2 insertions(+), 35 deletions(-) diff --git a/master/docs/developer/cls-buildsteps.rst b/master/docs/developer/cls-buildsteps.rst index 81b41d880c1..027dff70630 100644 --- a/master/docs/developer/cls-buildsteps.rst +++ b/master/docs/developer/cls-buildsteps.rst @@ -68,22 +68,6 @@ BuildStep If true, the step will run even if a previous step halts the build with ``haltOnFailure``. - A step acts as a factory for more steps. - See :ref:`Writing-BuildStep-Constructors` for advice on writing subclass constructors. - The following methods handle this factory behavior. - - .. py:method:: addFactoryArguments(..) - - Add the given keyword arguments to the arguments used to create new - step instances; - - .. py:method:: getStepFactory() - - :returns: tuple of (class, keyword arguments) - - Get a factory for new instances of this step. The step can be created - by calling the class with the given keyword arguments. - A few important pieces of information are not available when a step is constructed, and are added later. These are set by the following methods; the order in which these methods are called is not defined. diff --git a/master/docs/manual/customization.rst b/master/docs/manual/customization.rst index 0c7e0ee66e1..ffc1022a512 100644 --- a/master/docs/manual/customization.rst +++ b/master/docs/manual/customization.rst @@ -544,7 +544,6 @@ Writing BuildStep Constructors Build steps act as their own factories, so their constructors are a bit more complex than necessary. In the configuration file, a :class:`~buildbot.process.buildstep.BuildStep` object is instantiated, but because steps store state locally while executing, this object cannot be used during builds. -Instead, the build machinery calls the step's :meth:`~buildbot.process.buildstep.BuildStep.getStepFactory` method to get a tuple of a callable and keyword arguments that should be used to create a new instance. Consider the use of a :class:`BuildStep` in :file:`master.cfg`:: @@ -552,19 +551,11 @@ Consider the use of a :class:`BuildStep` in :file:`master.cfg`:: This creates a single instance of class ``MyStep``. However, Buildbot needs a new object each time the step is executed. -this is accomplished by storing the information required to instantiate a new object in the :attr:`~buildbot.process.buildstep.BuildStep.factory` attribute. -When the time comes to construct a new :class:`~buildbot.process.build.Build`, :class:`~buildbot.process.factory.BuildFactory` consults this attribute (via :meth:`~buildbot.process.buildstep.BuildStep.getStepFactory`) and instantiates a new step object. - +An instance of :class:`~buildbot.process.buildstep.BuildStep` rembers how it was constructed, and can create copies of itself. When writing a new step class, then, keep in mind are that you cannot do anything "interesting" in the constructor -- limit yourself to checking and storing arguments. -Each constructor in a sequence of :class:`BuildStep` subclasses must ensure the following: - -* the parent class's constructor is called with all otherwise-unspecified - keyword arguments. - -* all keyword arguments for the class itself are passed to :meth:`addFactoryArguments`. +It is customary to call the parent class's constructor with all otherwise-unspecified keyword arguments. Keep a ``**kwargs`` argument on the end of your options, and pass that up to the parent class's constructor. -If the class overrides constructor arguments for the parent class, those should be updated in ``kwargs``, rather than passed directly (which will cause errors during instantiation). The whole thing looks like this:: @@ -590,20 +581,12 @@ The whole thing looks like this:: self.frob_how_many = how_many self.frob_how = frob_how - # and record arguments for later - self.addFactoryArguments( - frob_what=frob_what, - frob_how_many=frob_how_many, - frob_how=frob_how) - class FastFrobnify(Frobnify): def __init__(self, speed=5, **kwargs) Frobnify.__init__(self, **kwargs) self.speed = speed - self.addFactoryArguments( - speed=speed) Running Commands ~~~~~~~~~~~~~~~~ From d73d939aa70ff388d83ce09d4179fb11fbef6e4c Mon Sep 17 00:00:00 2001 From: Tom Prince Date: Tue, 1 May 2012 20:02:29 -0600 Subject: [PATCH 026/136] Mark _getStepFactory and _factory as private. --- master/buildbot/process/buildstep.py | 8 ++++---- master/buildbot/test/unit/test_process_factory.py | 4 ++-- master/buildbot/test/util/steps.py | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/master/buildbot/process/buildstep.py b/master/buildbot/process/buildstep.py index 62067d5f287..5f2e4034514 100644 --- a/master/buildbot/process/buildstep.py +++ b/master/buildbot/process/buildstep.py @@ -461,7 +461,7 @@ def __init__(self, **kwargs): def __new__(klass, *args, **kwargs): self = object.__new__(klass) - self.factory = _BuildStepFactory(klass, *args, **kwargs) + self._factory = _BuildStepFactory(klass, *args, **kwargs) return self def describe(self, done=False): @@ -480,8 +480,8 @@ def addFactoryArguments(self, **kwargs): # this is here for backwards compatability pass - def getStepFactory(self): - return self.factory + def _getStepFactory(self): + return self._factory def setStepStatus(self, step_status): self.step_status = step_status @@ -755,7 +755,7 @@ def _maybeEvaluate(value, *args, **kwargs): return value components.registerAdapter( - BuildStep.getStepFactory, + BuildStep._getStepFactory, BuildStep, interfaces.IBuildStepFactory) components.registerAdapter( lambda step : interfaces.IProperties(step.build), diff --git a/master/buildbot/test/unit/test_process_factory.py b/master/buildbot/test/unit/test_process_factory.py index c8ffb7100cf..c60d7636da7 100644 --- a/master/buildbot/test/unit/test_process_factory.py +++ b/master/buildbot/test/unit/test_process_factory.py @@ -33,8 +33,8 @@ def test_addStep(self): def test_addStep_notAStep(self): factory = BuildFactory() - # This fails because object doesn't have .getStepFactory - self.assertRaises(AttributeError, factory.addStep, object()) + # This fails because object isn't adaptable to IBuildStepFactory + self.assertRaises(TypeError, factory.addStep, object()) def test_addStep_ArgumentsInTheWrongPlace(self): factory = BuildFactory() diff --git a/master/buildbot/test/util/steps.py b/master/buildbot/test/util/steps.py index bee83d0e02e..404c648444c 100644 --- a/master/buildbot/test/util/steps.py +++ b/master/buildbot/test/util/steps.py @@ -71,7 +71,7 @@ def setupStep(self, step, slave_version={'*':"99.99"}, slave_env={}): @param slave_env: environment from the slave at slave startup """ - factory = step.getStepFactory() + factory = interfaces.IBuildStepFactory(step) step = self.step = factory.buildStep() # step.build From 51ae9da33893ba8205be9cd871ad24584cfcbd09 Mon Sep 17 00:00:00 2001 From: Tom Prince Date: Tue, 8 May 2012 16:27:50 -0600 Subject: [PATCH 027/136] Fix test_process_build to use IBuildStepFactory. These tests should be fixed to not use Mock, but until then, use a custom FakeStepFactory to "build" the mock. --- .../buildbot/test/unit/test_process_build.py | 27 ++++++++++++------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/master/buildbot/test/unit/test_process_build.py b/master/buildbot/test/unit/test_process_build.py index 3ef7f888d6e..baa79005ffa 100644 --- a/master/buildbot/test/unit/test_process_build.py +++ b/master/buildbot/test/unit/test_process_build.py @@ -83,6 +83,15 @@ class FakeBuildStatus(Mock): class FakeBuilderStatus: implements(interfaces.IBuilderStatus) +class FakeStepFactory(object): + """Fake step factory that just returns a fixed step object.""" + implements(interfaces.IBuildStepFactory) + def __init__(self, step): + self.step = step + + def buildStep(self): + return self.step + class TestBuild(unittest.TestCase): def setUp(self): @@ -102,7 +111,7 @@ def testRunSuccessfulBuild(self): step = Mock() step.return_value = step step.startStep.return_value = SUCCESS - b.setStepFactories([lambda: step]) + b.setStepFactories([FakeStepFactory(step)]) slavebuilder = Mock() @@ -117,7 +126,7 @@ def testStopBuild(self): step = Mock() step.return_value = step - b.setStepFactories([lambda: step]) + b.setStepFactories([FakeStepFactory(step)]) slavebuilder = Mock() @@ -148,8 +157,8 @@ def testAlwaysRunStepStopBuild(self): step2.return_value = step2 step2.alwaysRun = True b.setStepFactories([ - lambda: step1, - lambda: step2, + FakeStepFactory(step1), + FakeStepFactory(step2), ]) slavebuilder = Mock() @@ -196,7 +205,7 @@ def claim(owner, access): step = Mock() step.return_value = step step.startStep.return_value = SUCCESS - b.setStepFactories([lambda: step]) + b.setStepFactories([FakeStepFactory(step)]) b.startBuild(FakeBuildStatus(), None, slavebuilder) @@ -225,7 +234,7 @@ def claim(owner, access): step = Mock() step.return_value = step step.startStep.return_value = SUCCESS - b.setStepFactories([lambda: step]) + b.setStepFactories([FakeStepFactory(step)]) real_lock.claim(Mock(), l.access('counting')) @@ -252,7 +261,7 @@ def testStopBuildWaitingForLocks(self): step.return_value = step step.startStep.return_value = SUCCESS step.alwaysRun = False - b.setStepFactories([lambda: step]) + b.setStepFactories([FakeStepFactory(step)]) real_lock.claim(Mock(), l.access('counting')) @@ -285,7 +294,7 @@ def testStopBuildWaitingForLocks_lostRemote(self): step.return_value = step step.startStep.return_value = SUCCESS step.alwaysRun = False - b.setStepFactories([lambda: step]) + b.setStepFactories([FakeStepFactory(step)]) real_lock.claim(Mock(), l.access('counting')) @@ -316,7 +325,7 @@ def testStopBuildWaitingForStepLocks(self): real_lock = b.builder.botmaster.getLockByID(l).getLock(slavebuilder) step = LoggingBuildStep(locks=[lock_access]) - b.setStepFactories([lambda: step]) + b.setStepFactories([FakeStepFactory(step)]) real_lock.claim(Mock(), l.access('counting')) From 7ea9f954c77dbb9ae48ba49ed07d94c453f659c4 Mon Sep 17 00:00:00 2001 From: Jared Grubb Date: Mon, 12 Mar 2012 17:52:46 -0700 Subject: [PATCH 028/136] Fixes #2214: Add 'codebase' to Source base; add 'descriptionSuffix' to Source and Shell --- master/buildbot/steps/master.py | 24 +++++++++--- master/buildbot/steps/shell.py | 22 ++++++++++- master/buildbot/steps/source/base.py | 38 ++++++++++++------- .../buildbot/test/unit/test_steps_master.py | 29 ++++++++++++++ master/buildbot/test/unit/test_steps_shell.py | 17 +++++++++ .../unit/test_steps_source_base_Source.py | 28 ++++++++++++++ master/docs/manual/cfg-buildsteps.rst | 11 ++++++ master/docs/release-notes.rst | 3 ++ 8 files changed, 152 insertions(+), 20 deletions(-) diff --git a/master/buildbot/steps/master.py b/master/buildbot/steps/master.py index 42680f0767f..bd1ef9ff786 100644 --- a/master/buildbot/steps/master.py +++ b/master/buildbot/steps/master.py @@ -29,17 +29,19 @@ class MasterShellCommand(BuildStep): name='MasterShellCommand' description='Running' descriptionDone='Ran' - renderables = [ 'command', 'env' ] + descriptionSuffix = None + renderables = [ 'command', 'env', 'description', 'descriptionDone', 'descriptionSuffix' ] haltOnFailure = True flunkOnFailure = True def __init__(self, command, - description=None, descriptionDone=None, + description=None, descriptionDone=None, descriptionSuffix=None, env=None, path=None, usePTY=0, **kwargs): BuildStep.__init__(self, **kwargs) self.addFactoryArguments(description=description, descriptionDone=descriptionDone, + descriptionSuffix=descriptionSuffix, env=env, path=path, usePTY=usePTY, command=command) @@ -52,6 +54,10 @@ def __init__(self, command, self.descriptionDone = descriptionDone if isinstance(self.descriptionDone, str): self.descriptionDone = [self.descriptionDone] + if descriptionSuffix: + self.descriptionSuffix = descriptionSuffix + if isinstance(self.descriptionSuffix, str): + self.descriptionSuffix = [self.descriptionSuffix] self.env=env self.path=path self.usePTY=usePTY @@ -100,7 +106,7 @@ def start(self): stdio_log.addHeader("** RUNNING ON BUILDMASTER **\n") stdio_log.addHeader(" in dir %s\n" % os.getcwd()) stdio_log.addHeader(" argv: %s\n" % (argv,)) - self.step_status.setText(list(self.description)) + self.step_status.setText(self.describe()) if self.env is None: env = os.environ @@ -126,8 +132,16 @@ def subst(match): def processEnded(self, status_object): if status_object.value.exitCode != 0: - self.step_status.setText(["failed (%d)" % status_object.value.exitCode]) + self.descriptionDone = ["failed (%d)" % status_object.value.exitCode] + self.step_status.setText(self.describe(done=True)) self.finished(FAILURE) else: - self.step_status.setText(list(self.descriptionDone)) + self.step_status.setText(self.describe(done=True)) self.finished(SUCCESS) + + def describe(self, done=False): + desc = self.descriptionDone if done else self.description + if self.descriptionSuffix: + desc = desc[:] + desc.extend(self.descriptionSuffix) + return desc diff --git a/master/buildbot/steps/shell.py b/master/buildbot/steps/shell.py index 71b4bc20ec5..2f526d6ff9a 100644 --- a/master/buildbot/steps/shell.py +++ b/master/buildbot/steps/shell.py @@ -65,9 +65,13 @@ class ShellCommand(buildstep.LoggingBuildStep): """ name = "shell" - renderables = [ 'description', 'descriptionDone', 'slaveEnvironment', 'remote_kwargs', 'command', 'logfiles' ] + renderables = buildstep.LoggingBuildStep.renderables + [ + 'slaveEnvironment', 'remote_kwargs', 'command'] + description = None # set this to a list of short strings to override descriptionDone = None # alternate description when the step is complete + descriptionSuffix = None # extra information to append to suffix + command = None # set this to a command, or set in kwargs # logfiles={} # you can also set 'logfiles' to a dictionary, and it # will be merged with any logfiles= argument passed in @@ -78,7 +82,7 @@ class ShellCommand(buildstep.LoggingBuildStep): flunkOnFailure = True def __init__(self, workdir=None, - description=None, descriptionDone=None, + description=None, descriptionDone=None, descriptionSuffix=None, command=None, usePTY="slave-config", **kwargs): @@ -95,6 +99,12 @@ def __init__(self, workdir=None, self.descriptionDone = descriptionDone if isinstance(self.descriptionDone, str): self.descriptionDone = [self.descriptionDone] + + if descriptionSuffix: + self.descriptionSuffix = descriptionSuffix + if isinstance(self.descriptionSuffix, str): + self.descriptionSuffix = [self.descriptionSuffix] + if command: self.setCommand(command) @@ -108,6 +118,7 @@ def __init__(self, workdir=None, self.addFactoryArguments(workdir=workdir, description=description, descriptionDone=descriptionDone, + descriptionSuffix=descriptionSuffix, command=command) # everything left over goes to the RemoteShellCommand @@ -148,6 +159,13 @@ def _flattenList(self, mainlist, commands): self._flattenList(mainlist, x) def describe(self, done=False): + desc = self._describe(done) + if self.descriptionSuffix: + desc = desc[:] + desc.extend(self.descriptionSuffix) + return desc + + def _describe(self, done=False): """Return a list of short strings to describe this step, for the status display. This uses the first few words of the shell command. You can replace this by setting .description in your subclass, or by diff --git a/master/buildbot/steps/source/base.py b/master/buildbot/steps/source/base.py index 9dc89fa2522..4db97175693 100644 --- a/master/buildbot/steps/source/base.py +++ b/master/buildbot/steps/source/base.py @@ -26,9 +26,13 @@ class Source(LoggingBuildStep): starts a RemoteCommand with those arguments. """ - renderables = [ 'workdir', 'description', 'descriptionDone' ] + renderables = LoggingBuildStep.renderables + [ + 'description', 'descriptionDone', 'descriptionSuffix', + 'workdir' ] + description = None # set this to a list of short strings to override descriptionDone = None # alternate description when the step is complete + descriptionSuffix = None # extra information to append to suffix # if the checkout fails, there's no point in doing anything else haltOnFailure = True @@ -39,8 +43,8 @@ class Source(LoggingBuildStep): def __init__(self, workdir=None, mode='update', alwaysUseLatest=False, timeout=20*60, retry=None, env=None, logEnviron=True, - description=None, descriptionDone=None, codebase='', - **kwargs): + description=None, descriptionDone=None, descriptionSuffix=None, + codebase='', **kwargs): """ @type workdir: string @param workdir: local directory (relative to the Builder's root) @@ -128,7 +132,7 @@ def __init__(self, workdir=None, mode='update', alwaysUseLatest=False, variables on the slave. In situations where the environment is not relevant and is long, it may be easier to set logEnviron=False. -+ + @type codebase: string @param codebase: Specifies which changes in a build are processed by the step. The default codebase value is ''. The codebase must correspond @@ -147,7 +151,8 @@ def __init__(self, workdir=None, mode='update', alwaysUseLatest=False, env=env, description=description, descriptionDone=descriptionDone, - codebase=codebase, + descriptionSuffix=descriptionSuffix, + codebase=codebase ) assert mode in ("update", "copy", "clobber", "export") @@ -164,8 +169,10 @@ def __init__(self, workdir=None, mode='update', alwaysUseLatest=False, self.workdir = workdir self.sourcestamp = None - # Codebase cannot be set yet + self.codebase = codebase + if self.codebase: + self.name = ' '.join((self.name, self.codebase)) self.alwaysUseLatest = alwaysUseLatest @@ -183,8 +190,6 @@ def __init__(self, workdir=None, mode='update', alwaysUseLatest=False, else: self.description = [ descriptions_for_mode.get(mode, "updating")] - if self.codebase: - self.description.append(self.codebase) if isinstance(self.description, str): self.description = [self.description] @@ -193,11 +198,16 @@ def __init__(self, workdir=None, mode='update', alwaysUseLatest=False, else: self.descriptionDone = [ descriptionDones_for_mode.get(mode, "update")] - if self.codebase: - self.descriptionDone.append(self.codebase) if isinstance(self.descriptionDone, str): self.descriptionDone = [self.descriptionDone] + if descriptionSuffix: + self.descriptionSuffix = descriptionSuffix + else: + self.descriptionSuffix = self.codebase or None # want None in lieu of '' + if isinstance(self.descriptionSuffix, str): + self.descriptionSuffix = [self.descriptionSuffix] + def setProperty(self, name, value , source): if self.codebase != '': assert not isinstance(self.getProperty(name, None), str), \ @@ -219,9 +229,11 @@ def setDefaultWorkdir(self, workdir): self.workdir = self.workdir or workdir def describe(self, done=False): - if done: - return self.descriptionDone - return self.description + desc = self.descriptionDone if done else self.description + if self.descriptionSuffix: + desc = desc[:] + desc.extend(self.descriptionSuffix) + return desc def computeSourceRevision(self, changes): """Each subclass must implement this method to do something more diff --git a/master/buildbot/test/unit/test_steps_master.py b/master/buildbot/test/unit/test_steps_master.py index 440d53cc02d..a99d4905901 100644 --- a/master/buildbot/test/unit/test_steps_master.py +++ b/master/buildbot/test/unit/test_steps_master.py @@ -80,6 +80,9 @@ def test_constr_args(self): master.MasterShellCommand(description='x', descriptionDone='y', env={'a':'b'}, path=['/usr/bin'], usePTY=True, command='true')) + + self.assertEqual(self.step.describe(), ['x']) + if runtime.platformType == 'win32': exp_argv = [ r'C:\WINDOWS\system32\cmd.exe', '/c', 'true' ] else: @@ -127,3 +130,29 @@ def test_prop_rendering(self): self.expectLogfile('stdio', "BUILDBOT-TEST\nBUILDBOT-TEST\n") self.expectOutcome(result=SUCCESS, status_text=["Ran"]) return self.runStep() + + def test_constr_args_descriptionSuffix(self): + self.setupStep( + master.MasterShellCommand(description='x', descriptionDone='y', + descriptionSuffix='z', + env={'a':'b'}, path=['/usr/bin'], usePTY=True, + command='true')) + + # call twice to make sure the suffix doesnt get double added + self.assertEqual(self.step.describe(), ['x', 'z']) + self.assertEqual(self.step.describe(), ['x', 'z']) + + if runtime.platformType == 'win32': + exp_argv = [ r'C:\WINDOWS\system32\cmd.exe', '/c', 'true' ] + else: + exp_argv = [ '/bin/sh', '-c', 'true' ] + self.patchSpawnProcess( + exp_cmd=exp_argv[0], exp_argv=exp_argv, + exp_path=['/usr/bin'], exp_usePTY=True, exp_env={'a':'b'}, + outputs=[ + ('out', 'hello!\n'), + ('err', 'world\n'), + ('rc', 0), + ]) + self.expectOutcome(result=SUCCESS, status_text=['y', 'z']) + return self.runStep() diff --git a/master/buildbot/test/unit/test_steps_shell.py b/master/buildbot/test/unit/test_steps_shell.py index 06e2936eafb..33273f66317 100644 --- a/master/buildbot/test/unit/test_steps_shell.py +++ b/master/buildbot/test/unit/test_steps_shell.py @@ -129,6 +129,23 @@ def test_describe_custom(self): self.assertEqual((step.describe(), step.describe(done=True)), (['echoing'], ['echoed'])) + def test_describe_with_suffix(self): + step = shell.ShellCommand(command="echo hello", descriptionSuffix="suffix") + self.assertEqual((step.describe(), step.describe(done=True)), + (["'echo", "hello'", 'suffix'],)*2) + + def test_describe_custom_with_suffix(self): + step = shell.ShellCommand(command="echo hello", + description=["echoing"], descriptionDone=["echoed"], + descriptionSuffix="suffix") + self.assertEqual((step.describe(), step.describe(done=True)), + (['echoing', 'suffix'], ['echoed', 'suffix'])) + + def test_describe_no_command_with_suffix(self): + step = shell.ShellCommand(workdir='build', descriptionSuffix="suffix") + self.assertEqual((step.describe(), step.describe(done=True)), + (['???', 'suffix'],)*2) + def test_describe_unrendered_WithProperties(self): step = shell.ShellCommand(command=properties.WithProperties('')) self.assertEqual((step.describe(), step.describe(done=True)), diff --git a/master/buildbot/test/unit/test_steps_source_base_Source.py b/master/buildbot/test/unit/test_steps_source_base_Source.py index d41c6a95b0a..3260e479369 100644 --- a/master/buildbot/test/unit/test_steps_source_base_Source.py +++ b/master/buildbot/test/unit/test_steps_source_base_Source.py @@ -73,9 +73,14 @@ def test_start_no_codebase(self): step.build.getSourceStamp = mock.Mock() step.build.getSourceStamp.return_value = None + self.assertEqual(step.describe(), ['updating']) + self.assertEqual(step.name, Source.name) + step.startStep(mock.Mock()) self.assertEqual(step.build.getSourceStamp.call_args[0], ('',)) + self.assertEqual(step.description, ['updating']) + def test_start_with_codebase(self): step = self.setupStep(Source(codebase='codebase')) step.branch = 'branch' @@ -83,9 +88,31 @@ def test_start_with_codebase(self): step.build.getSourceStamp = mock.Mock() step.build.getSourceStamp.return_value = None + self.assertEqual(step.describe(), ['updating', 'codebase']) + self.assertEqual(step.name, Source.name + " codebase") + step.startStep(mock.Mock()) self.assertEqual(step.build.getSourceStamp.call_args[0], ('codebase',)) + self.assertEqual(step.describe(True), ['update', 'codebase']) + + def test_start_with_codebase_and_descriptionSuffix(self): + step = self.setupStep(Source(codebase='my-code', + descriptionSuffix='suffix')) + step.branch = 'branch' + step.startVC = mock.Mock() + step.build.getSourceStamp = mock.Mock() + step.build.getSourceStamp.return_value = None + + self.assertEqual(step.describe(), ['updating', 'suffix']) + self.assertEqual(step.name, Source.name + " my-code") + + step.startStep(mock.Mock()) + self.assertEqual(step.build.getSourceStamp.call_args[0], ('my-code',)) + + self.assertEqual(step.describe(True), ['update', 'suffix']) + + class TestSourceDescription(steps.BuildStepMixin, unittest.TestCase): def setUp(self): @@ -107,3 +134,4 @@ def test_constructor_args_lists(self): descriptionDone=['svn', 'update']) self.assertEqual(step.description, ['svn', 'update', '(running)']) self.assertEqual(step.descriptionDone, ['svn', 'update']) + diff --git a/master/docs/manual/cfg-buildsteps.rst b/master/docs/manual/cfg-buildsteps.rst index d85cb21ea48..aac6d62152a 100644 --- a/master/docs/manual/cfg-buildsteps.rst +++ b/master/docs/manual/cfg-buildsteps.rst @@ -1436,6 +1436,17 @@ The :bb:step:`ShellCommand` arguments are: description=["testing"], descriptionDone=["tests"])) +``descriptionSuffix`` + This is an optional suffix appended to the end of the description (ie, + after ``description`` and ``descriptionDone``). This can be used to distinguish + between build steps that would display the same descriptions in the waterfall. + This parameter may be set to list of short strings, a single string, or ``None``. + + For example, a builder might use the ``Compile`` step to build two different + codebases. The ``descriptionSuffix`` could be set to `projectFoo` and `projectBar`, + respectively for each step, which will result in the full descriptions + `compiling projectFoo` and `compiling projectBar` to be shown in the waterfall. + ``logEnviron`` If this option is ``True`` (the default), then the step's logfile will describe the environment variables on the slave. In situations where the environment is not diff --git a/master/docs/release-notes.rst b/master/docs/release-notes.rst index 4e5ee09b830..e6b227c3b49 100644 --- a/master/docs/release-notes.rst +++ b/master/docs/release-notes.rst @@ -66,6 +66,9 @@ Deprecations, Removals, and Non-Compatible Changes from buildbot.steps.source.svn import SVN factory.append(SVN(repourl=Interpolate("svn://svn.example.org/svn/%(src::branch:-branches/test)s"))) +* ``Source`` and ``ShellCommand`` steps now have an optional ``descriptionSuffix``, a suffix to the + ``description``/``descriptionDone`` values. For example this can help distinguish between + multiple ``Compile`` steps that are applied to different codebases. Changes for Developers ~~~~~~~~~~~~~~~~~~~~~~ From c977f04d92f3cd986ed98dc5b229b1b351db5507 Mon Sep 17 00:00:00 2001 From: Harry Borkhuis Date: Wed, 9 May 2012 15:47:29 +0200 Subject: [PATCH 029/136] Support for multiple sourcestamps in build status --- master/buildbot/status/build.py | 1 - master/buildbot/status/web/base.py | 6 +- master/buildbot/status/web/build.py | 32 ++--- .../buildbot/status/web/templates/build.html | 111 +++++++++++------- 4 files changed, 87 insertions(+), 63 deletions(-) diff --git a/master/buildbot/status/build.py b/master/buildbot/status/build.py index 912bf75122d..b9569606320 100644 --- a/master/buildbot/status/build.py +++ b/master/buildbot/status/build.py @@ -456,7 +456,6 @@ def asDict(self): # Constant result['builderName'] = self.builder.name result['number'] = self.getNumber() - # TODO: enable multiple sourcestamps to outside the buildstatus result['sourceStamps'] = [ss.asDict() for ss in self.getSourceStamps()] result['reason'] = self.getReason() result['blame'] = self.getResponsibleUsers() diff --git a/master/buildbot/status/web/base.py b/master/buildbot/status/web/base.py index ab36161593b..5e8902cee80 100644 --- a/master/buildbot/status/web/base.py +++ b/master/buildbot/status/web/base.py @@ -436,10 +436,12 @@ def get_line_values(self, req, build, include_builder=True): css_class = css_classes.get(results, "") ss_list = build.getSourceStamps() if ss_list: - # TODO: support multiple sourcestamps in web interface repo = ss_list[0].repository if all_got_revision: - rev = all_got_revision[ss_list[0].codebase] + if len(ss_list) == 1: + rev = all_got_revision.get(ss_list[0].codebase, "??") + else: + rev = "multiple rev." else: rev = "??" else: diff --git a/master/buildbot/status/web/build.py b/master/buildbot/status/web/build.py index 8ebb2075968..561ae3a2815 100644 --- a/master/buildbot/status/web/build.py +++ b/master/buildbot/status/web/build.py @@ -160,16 +160,10 @@ def content(self, req, cxt): cxt['tests_link'] = req.childLink("tests") ssList = b.getSourceStamps() - # TODO: support multiple sourcestamps - ss = cxt['ss'] = ssList[0] + sourcestamps = cxt['sourcestamps'] = ssList - if ss.branch is None and ss.revision is None and ss.patch is None and not ss.changes: - cxt['most_recent_rev_build'] = True - - all_got_revisions = b.getAllGotRevisions() - if all_got_revisions: - got_revision = all_got_revisions.get(ss.codebase, "??") - cxt['got_revision'] = str(got_revision) + all_got_revisions = b.getAllGotRevisions() or {} + cxt['got_revisions'] = all_got_revisions try: cxt['slave_url'] = path_to_slave(req, status.getSlave(b.getSlavename())) @@ -224,10 +218,13 @@ def content(self, req, cxt): ps = cxt['properties'] = [] for name, value, source in b.getProperties().asList(): - uvalue = unicode(value) - p = { 'name': name, 'value': uvalue, 'source': source} - if len(uvalue) > 500: - p['short_value'] = uvalue[:500] + if not isinstance(value, dict): + cxt_value = unicode(value) + else: + cxt_value = value + p = { 'name': name, 'value': cxt_value, 'source': source} + if len(cxt_value) > 500: + p['short_value'] = cxt_value[:500] if name in parameters: param = parameters[name] if isinstance(param, TextParameter): @@ -249,8 +246,13 @@ def content(self, req, cxt): now = util.now() cxt['elapsed'] = util.formatInterval(now - start) - cxt['exactly'] = (ss.revision is not None) or b.getChanges() - + exactly = True + has_changes = False + for ss in sourcestamps: + exactly = exactly and (ss.revision is not None) + has_changes = has_changes or ss.changes + cxt['exactly'] = (exactly) or b.getChanges() + cxt['has_changes'] = has_changes cxt['build_url'] = path_to_build(req, b) cxt['authz'] = self.getAuthz(req) diff --git a/master/buildbot/status/web/templates/build.html b/master/buildbot/status/web/templates/build.html index 9a669f7fc21..2148486d66a 100644 --- a/master/buildbot/status/web/templates/build.html +++ b/master/buildbot/status/web/templates/build.html @@ -36,44 +36,52 @@

{% endif %} {% endif %} -

SourceStamp:

- - -{% set ss_class = cycler('alt','') %} - -{% if ss.project %} - +

+{% if sourcestamps|count == 1 %} +SourceStamp: +{% else %} +SourceStamps: {% endif %} +

-{% if ss.repository %} - -{% endif %} +{% for ss in sourcestamps %} +

{{ ss.codebase }}

+
Project{{ ss.project|projectlink }}
Repository{{ ss.repository|repolink }}
+ {% set ss_class = cycler('alt','') %} -{% if ss.branch %} - -{% endif %} + {% if ss.project %} + + {% endif %} -{% if ss.revision %} - -{% endif %} + {% if ss.repository %} + + {% endif %} -{% if got_revision %} - -{% endif %} + {% if ss.branch %} + + {% endif %} -{% if ss.patch %} - -{% endif %} + {% if ss.revision %} + + {% endif %} -{% if ss.changes %} - -{% endif %} + {% if got_revisions[ss.codebase] %} + + {% endif %} -{% if most_recent_rev_build %} - -{% endif %} + {% if ss.patch %} + + {% endif %} -
Branch{{ ss.branch|e }}
Project{{ ss.project|projectlink }}
Revision{{ ss.revision|revlink(ss.repository) }}
Repository{{ ss.repository|repolink }}
Got Revision{{ got_revision|revlink(ss.repository) }}
Branch{{ ss.branch|e }}
PatchYES
Revision{{ ss.revision|revlink(ss.repository) }}
Changessee below
Got Revision{{ got_revisions[ss.codebase]|revlink(ss.repository) }}
Build of most recent revision
PatchYES
+ {% if ss.changes %} + Changes{{ ss.changes|count }} change{{ 's' if ss.changes|count > 1 else '' }} + {% endif %} + + {% if not ss.branch and not ss.revision and not ss.patch and not ss.changes %} + Build of most recent revision + {% endif %} + +{% endfor %} {# # TODO: turn this into a table, or some other sort of definition-list @@ -141,9 +149,19 @@

Build Properties:

{{ p.name|e }} {% if p.short_value %} - {{ p.short_value|e }} .. [property value too long] + {{ p.short_value|e }} .. [property value too long] {% else %} - {{ p.value|e }} + {% if p.value is not mapping %} + {{ p.value|e }} + {% else %} + + + {%- for key, value in p.value.items() recursive %} + + {% endfor %} +
{{ key|e }}{{ value|e }}
+ + {% endif %} {% endif %} {{ p.source|e }} @@ -197,26 +215,29 @@

Timing:

{% if authz.advertiseAction('forceBuild', request) %}

Resubmit Build:

- {{ forms.rebuild_build(build_url+"/rebuild", authz, exactly, ss) }} + {{ forms.rebuild_build(build_url+"/rebuild", authz, exactly, sourcestamps[0]) }} {% endif %}
-{% if ss.changes %} -
-

All Changes:

-
    - {% for c in ss.changes %} -

  1. Change #{{ c.number }}

    - {{ change(c.asDict()) }} -
    2. - {% else %} -
  2. no changes
    3. - {% endfor %} -
-
+{% if has_changes %} +
+

All Changes:

+ {% for ss in sourcestamps %} + {% if ss.changes %} +

{{ ss.codebase }}:

+
    + {% for c in ss.changes %} +

  1. Change #{{ c.number }}

    + {{ change(c.asDict()) }} +
    2. + {% endfor %} +
+ {% endif %} + {% endfor %} +
{% endif %} {% endblock %} From 7b8a89107788ed2e115939ced6be1f8dbaaa5c56 Mon Sep 17 00:00:00 2001 From: Harry Borkhuis Date: Wed, 9 May 2012 15:47:58 +0200 Subject: [PATCH 030/136] Filter out all builds with more than one sourcestamp --- master/buildbot/status/web/console.py | 51 ++++++++++++++------------- master/buildbot/status/web/grid.py | 11 +++--- 2 files changed, 33 insertions(+), 29 deletions(-) diff --git a/master/buildbot/status/web/console.py b/master/buildbot/status/web/console.py index 01490831451..368941693f7 100644 --- a/master/buildbot/status/web/console.py +++ b/master/buildbot/status/web/console.py @@ -234,30 +234,33 @@ def getBuildsForRevision(self, request, builder, builderName, lastRevision, number = 0 while build and number < numBuilds: debugInfo["builds_scanned"] += 1 - number += 1 - - # Get the last revision in this build. - # We first try "got_revision", but if it does not work, then - # we try "revision". - got_rev = build.getProperty("got_revision", build.getProperty("revision", -1)) - if got_rev != -1 and not self.comparator.isValidRevision(got_rev): - got_rev = -1 - - # We ignore all builds that don't have last revisions. - # TODO(nsylvain): If the build is over, maybe it was a problem - # with the update source step. We need to find a way to tell the - # user that his change might have broken the source update. - if got_rev != -1: - details = self.getBuildDetails(request, builderName, build) - devBuild = DevBuild(got_rev, build, details) - builds.append(devBuild) - - # Now break if we have enough builds. - current_revision = self.getChangeForBuild( - build, revision) - if self.comparator.isRevisionEarlier( - devBuild, current_revision): - break + + # The console page cannot handle builds that have more than 1 revision + if len(build.getSourceStamps()) == 1: + number += 1 + # Get the last revision in this build. + # We first try "got_revision", but if it does not work, then + # we try "revision". + got_rev = build.getProperty("got_revision", build.getProperty("revision", -1)) + if got_rev != -1 and not self.comparator.isValidRevision(got_rev): + got_rev = -1 + + + # We ignore all builds that don't have last revisions. + # TODO(nsylvain): If the build is over, maybe it was a problem + # with the update source step. We need to find a way to tell the + # user that his change might have broken the source update. + if got_rev != -1: + details = self.getBuildDetails(request, builderName, build) + devBuild = DevBuild(got_rev, build, details) + builds.append(devBuild) + + # Now break if we have enough builds. + current_revision = self.getChangeForBuild( + build, revision) + if self.comparator.isRevisionEarlier( + devBuild, current_revision): + break build = build.getPreviousBuild() diff --git a/master/buildbot/status/web/grid.py b/master/buildbot/status/web/grid.py index 49c7193a792..0deb018268b 100644 --- a/master/buildbot/status/web/grid.py +++ b/master/buildbot/status/web/grid.py @@ -193,11 +193,12 @@ def content(self, request, cxt): for build in self.getRecentBuilds(builder, numBuilds, branch): #TODO: support multiple sourcestamps - ss = build.getSourceStamps(absolute=True)[0] - key= self.getSourceStampKey(ss) - for i in range(len(stamps)): - if key == self.getSourceStampKey(stamps[i]) and builds[i] is None: - builds[i] = build + if build.getSourceStamps() == 1: + ss = build.getSourceStamps(absolute=True)[0] + key= self.getSourceStampKey(ss) + for i in range(len(stamps)): + if key == self.getSourceStampKey(stamps[i]) and builds[i] is None: + builds[i] = build b = yield self.builder_cxt(request, builder) From e5cac61ecff59bd65701eba425aa5e9c04d048d3 Mon Sep 17 00:00:00 2001 From: "Dustin J. Mitchell" Date: Wed, 9 May 2012 21:11:32 -0500 Subject: [PATCH 031/136] release notes for multirepo --- master/docs/release-notes.rst | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/master/docs/release-notes.rst b/master/docs/release-notes.rst index e6b227c3b49..a7bd36c81bd 100644 --- a/master/docs/release-notes.rst +++ b/master/docs/release-notes.rst @@ -73,14 +73,20 @@ Deprecations, Removals, and Non-Compatible Changes Changes for Developers ~~~~~~~~~~~~~~~~~~~~~~ - * ``BuildStep.start`` can now optionally return a deferred and any errback - will be handled gracefully. If you use inlineCallbacks, this means that - unexpected exceptions and failures raised will be captured and logged and - the build shut down normally. +* ``BuildStep.start`` can now optionally return a deferred and any errback will + be handled gracefully. If you use inlineCallbacks, this means that unexpected + exceptions and failures raised will be captured and logged and the build shut + down normally. Features ~~~~~~~~ +* Buildbot now supports building projects composed of multiple codebases. New + schedulers can aggregate changes to multipl codebases into source stamp sets + (with one source stamp for each codebase). Source steps then check out each + codebase as required, and the remainder of the build process proceeds + normally. See the documentation for details. + Slave ----- From dc91bfee77b8adf9ef9957fa38bd0209887c635c Mon Sep 17 00:00:00 2001 From: "Dustin J. Mitchell" Date: Wed, 9 May 2012 21:20:25 -0500 Subject: [PATCH 032/136] fix now-incorrect warning for an un-upgraded master --- master/buildbot/db/connector.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/master/buildbot/db/connector.py b/master/buildbot/db/connector.py index dff7652f9b1..bc5716495e0 100644 --- a/master/buildbot/db/connector.py +++ b/master/buildbot/db/connector.py @@ -33,12 +33,7 @@ class DatabaseNotReadyError(Exception): buildbot upgrade-master path/to/master to upgrade the database, and try starting the buildmaster again. You may - want to make a backup of your buildmaster before doing so. If you are - using MySQL, you must specify the connector string on the upgrade-master - command line: - - buildbot upgrade-master --db= path/to/master - + want to make a backup of your buildmaster before doing so. """).strip() class DBConnector(config.ReconfigurableServiceMixin, service.MultiService): From ffd8e16b647498a614cbca2faf554dbff1605590 Mon Sep 17 00:00:00 2001 From: "Dustin J. Mitchell" Date: Wed, 9 May 2012 21:29:05 -0500 Subject: [PATCH 033/136] remove remaining deferredGenerator uses from buildmaster --- master/buildbot/process/buildrequest.py | 43 +++-------- master/buildbot/schedulers/base.py | 71 +++++++------------ master/buildbot/test/unit/test_buildslave.py | 18 ++--- .../test/unit/test_process_builder.py | 14 ++-- 4 files changed, 45 insertions(+), 101 deletions(-) diff --git a/master/buildbot/process/buildrequest.py b/master/buildbot/process/buildrequest.py index a6d26a7d36e..b65160b30ff 100644 --- a/master/buildbot/process/buildrequest.py +++ b/master/buildbot/process/buildrequest.py @@ -91,7 +91,7 @@ def fromBrdict(cls, master, brdict): return cache.get(brdict['brid'], brdict=brdict, master=master) @classmethod - @defer.deferredGenerator + @defer.inlineCallbacks def _make_br(cls, brid, brdict, master): buildrequest = cls() buildrequest.id = brid @@ -103,18 +103,12 @@ def _make_br(cls, brid, brdict, master): buildrequest.master = master # fetch the buildset to get the reason - wfd = defer.waitForDeferred( - master.db.buildsets.getBuildset(brdict['buildsetid'])) - yield wfd - buildset = wfd.getResult() + buildset = yield master.db.buildsets.getBuildset(brdict['buildsetid']) assert buildset # schema should guarantee this buildrequest.reason = buildset['reason'] # fetch the buildset properties, and convert to Properties - wfd = defer.waitForDeferred( - master.db.buildsets.getBuildsetProperties(brdict['buildsetid'])) - yield wfd - buildset_properties = wfd.getResult() + buildset_properties = yield master.db.buildsets.getBuildsetProperties(brdict['buildsetid']) pr = properties.Properties() for name, (value, source) in buildset_properties.iteritems(): @@ -122,10 +116,7 @@ def _make_br(cls, brid, brdict, master): buildrequest.properties = pr # fetch the sourcestamp dictionary - wfd = defer.waitForDeferred( - master.db.sourcestamps.getSourceStamps(buildset['sourcestampsetid'])) - yield wfd - sslist = wfd.getResult() + sslist = yield master.db.sourcestamps.getSourceStamps(buildset['sourcestampsetid']) assert len(sslist) > 0, "Empty sourcestampset: db schema enforces set to exist but cannot enforce a non empty set" # and turn it into a SourceStamps @@ -139,15 +130,12 @@ def store_source(source): d.addCallback(store_source) dlist.append(d) - dl = defer.gatherResults(dlist) - wfd = defer.waitForDeferred(dl) - yield wfd - wfd.getResult() + yield defer.gatherResults(dlist) if buildrequest.sources: buildrequest.source = buildrequest.sources.values()[0] - yield buildrequest # return value + defer.returnValue(buildrequest) def requestsHaveSameCodebases(self, other): self_codebases = set(self.sources.iterkeys()) @@ -230,15 +218,12 @@ def mergeReasons(self, others): def getSubmitTime(self): return self.submittedAt - @defer.deferredGenerator + @defer.inlineCallbacks def cancelBuildRequest(self): # first, try to claim the request; if this fails, then it's too late to # cancel the build anyway try: - wfd = defer.waitForDeferred( - self.master.db.buildrequests.claimBuildRequests([self.id])) - yield wfd - wfd.getResult() + yield self.master.db.buildrequests.claimBuildRequests([self.id]) except buildrequests.AlreadyClaimedError: log.msg("build request already claimed; cannot cancel") return @@ -246,17 +231,11 @@ def cancelBuildRequest(self): # then complete it with 'FAILURE'; this is the closest we can get to # cancelling a request without running into trouble with dangling # references. - wfd = defer.waitForDeferred( - self.master.db.buildrequests.completeBuildRequests([self.id], - FAILURE)) - yield wfd - wfd.getResult() + yield self.master.db.buildrequests.completeBuildRequests([self.id], + FAILURE) # and let the master know that the enclosing buildset may be complete - wfd = defer.waitForDeferred( - self.master.maybeBuildsetComplete(self.bsid)) - yield wfd - wfd.getResult() + yield self.master.maybeBuildsetComplete(self.bsid) class BuildRequestControl: implements(interfaces.IBuildRequestControl) diff --git a/master/buildbot/schedulers/base.py b/master/buildbot/schedulers/base.py index f7180e06f70..4c9089f48f1 100644 --- a/master/buildbot/schedulers/base.py +++ b/master/buildbot/schedulers/base.py @@ -127,7 +127,7 @@ def stopService(self): ## state management - @defer.deferredGenerator + @defer.inlineCallbacks def getState(self, *args, **kwargs): """ For use by subclasses; get a named state value from the scheduler's @@ -141,18 +141,14 @@ def getState(self, *args, **kwargs): """ # get the objectid, if not known if self._objectid is None: - wfd = defer.waitForDeferred( - self.master.db.state.getObjectId(self.name, - self.__class__.__name__)) - yield wfd - self._objectid = wfd.getResult() - - wfd = defer.waitForDeferred( - self.master.db.state.getState(self._objectid, *args, **kwargs)) - yield wfd - yield wfd.getResult() - - @defer.deferredGenerator + self._objectid = yield self.master.db.state.getObjectId(self.name, + self.__class__.__name__) + + rv = yield self.master.db.state.getState(self._objectid, *args, + **kwargs) + defer.returnValue(rv) + + @defer.inlineCallbacks def setState(self, key, value): """ For use by subclasses; set a named state value in the scheduler's @@ -165,16 +161,10 @@ def setState(self, key, value): """ # get the objectid, if not known if self._objectid is None: - wfd = defer.waitForDeferred( - self.master.db.state.getObjectId(self.name, - self.__class__.__name__)) - yield wfd - self._objectid = wfd.getResult() + self._objectid = yield self.master.db.state.getObjectId(self.name, + self.__class__.__name__) - wfd = defer.waitForDeferred( - self.master.db.state.setState(self._objectid, key, value)) - yield wfd - wfd.getResult() + yield self.master.db.state.setState(self._objectid, key, value) ## status queries @@ -334,7 +324,7 @@ def addBuildsetForLatest(self, reason='', external_idstring=None, defer.returnValue((bsid,brids)) - @defer.deferredGenerator + @defer.inlineCallbacks def addBuildSetForSourceStampDetails(self, reason='', external_idstring=None, branch=None, repository='', project='', revision=None, builderNames=None, properties=None): @@ -357,23 +347,18 @@ def addBuildSetForSourceStampDetails(self, reason='', external_idstring=None, @returns: (buildset ID, buildrequest IDs) via Deferred """ # Define setid for this set of changed repositories - wfd = defer.waitForDeferred(self.master.db.sourcestampsets.addSourceStampSet()) - yield wfd - setid = wfd.getResult() + setid = yield self.master.db.sourcestampsets.addSourceStampSet() - wfd = defer.waitForDeferred(self.master.db.sourcestamps.addSourceStamp( + yield self.master.db.sourcestamps.addSourceStamp( branch=branch, revision=revision, repository=repository, - project=project, sourcestampsetid=setid)) - yield wfd - wfd.getResult() + project=project, sourcestampsetid=setid) - wfd = defer.waitForDeferred(self.addBuildsetForSourceStamp( + rv = yield self.addBuildsetForSourceStamp( setid=setid, reason=reason, external_idstring=external_idstring, builderNames=builderNames, - properties=properties)) - yield wfd - yield wfd.getResult() + properties=properties) + defer.returnValue(rv) @defer.inlineCallbacks @@ -419,7 +404,7 @@ def get_last_change_for_codebase(codebase): defer.returnValue((bsid,brids)) - @defer.deferredGenerator + @defer.inlineCallbacks def addBuildsetForSourceStamp(self, ssid=None, setid=None, reason='', external_idstring=None, properties=None, builderNames=None): """ @@ -460,19 +445,15 @@ def addBuildsetForSourceStamp(self, ssid=None, setid=None, reason='', external_i if setid == None: if ssid is not None: - wfd = defer.waitForDeferred(self.master.db.sourcestamps.getSourceStamp(ssid)) - yield wfd - ssdict = wfd.getResult() + ssdict = yield self.master.db.sourcestamps.getSourceStamp(ssid) setid = ssdict['sourcestampsetid'] else: # no sourcestamp and no sets yield None - wfd = defer.waitForDeferred(self.master.addBuildset( - sourcestampsetid=setid, reason=reason, - properties=properties_dict, - builderNames=builderNames, - external_idstring=external_idstring)) - yield wfd - yield wfd.getResult() + rv = yield self.master.addBuildset(sourcestampsetid=setid, + reason=reason, properties=properties_dict, + builderNames=builderNames, + external_idstring=external_idstring) + defer.returnValue(rv) diff --git a/master/buildbot/test/unit/test_buildslave.py b/master/buildbot/test/unit/test_buildslave.py index 043bdefae39..078c729ff9f 100644 --- a/master/buildbot/test/unit/test_buildslave.py +++ b/master/buildbot/test/unit/test_buildslave.py @@ -107,26 +107,16 @@ def test_reconfigService_attrs(self): self.assertEqual(self.master.pbmanager._registrations, []) self.assertTrue(old.updateSlave.called) - @defer.deferredGenerator + @defer.inlineCallbacks def test_reconfigService_has_properties(self): old = self.ConcreteBuildSlave('bot', 'pass') - - wfd = defer.waitForDeferred( - self.do_test_reconfigService(old, 'tcp:1234', old, 'tcp:1234')) - yield wfd - wfd.getResult() - + yield self.do_test_reconfigService(old, 'tcp:1234', old, 'tcp:1234') self.assertTrue(old.properties.getProperty('slavename'), 'bot') - @defer.deferredGenerator + @defer.inlineCallbacks def test_reconfigService_initial_registration(self): old = self.ConcreteBuildSlave('bot', 'pass') - - wfd = defer.waitForDeferred( - self.do_test_reconfigService(old, None, old, 'tcp:1234')) - yield wfd - wfd.getResult() - + yield self.do_test_reconfigService(old, None, old, 'tcp:1234') self.assertEqual(self.master.pbmanager._registrations, [('tcp:1234', 'bot', 'pass')]) @defer.inlineCallbacks diff --git a/master/buildbot/test/unit/test_process_builder.py b/master/buildbot/test/unit/test_process_builder.py index e6d379a9b93..552007aa451 100644 --- a/master/buildbot/test/unit/test_process_builder.py +++ b/master/buildbot/test/unit/test_process_builder.py @@ -322,12 +322,9 @@ def test_maybeStartBuild_builder_stopped(self): yield self.bldr.stopService() yield self.bldr.maybeStartBuild() - @defer.deferredGenerator + @defer.inlineCallbacks def test_maybeStartBuild_merge_ordering(self): - wfd = defer.waitForDeferred( - self.makeBuilder(patch_random=True)) - yield wfd - wfd.getResult() + yield self.makeBuilder(patch_random=True) self.setSlaveBuilders({'bldr':1}) @@ -347,12 +344,9 @@ def test_maybeStartBuild_merge_ordering(self): fakedb.BuildRequest(id=42922, buildsetid=1981, buildername="bldr", submitted_at=1332025495.19141), ] - wfd = defer.waitForDeferred( - self.do_test_maybeStartBuild(rows=rows, + yield self.do_test_maybeStartBuild(rows=rows, exp_claims=[42880, 42922], - exp_builds=[('bldr', [42880, 42922])])) - yield wfd - wfd.getResult() + exp_builds=[('bldr', [42880, 42922])]) # _chooseSlave From 700825df7f1efbac3c44e9d7e5328b722214b980 Mon Sep 17 00:00:00 2001 From: Anders Waldenborg Date: Thu, 10 May 2012 09:13:38 +0200 Subject: [PATCH 034/136] Make sure category is properly updated when loading buildstatus from pickle. There even was a comment about doing it, but the actual code to do it seems to have got lost. --- master/buildbot/status/master.py | 1 + 1 file changed, 1 insertion(+) diff --git a/master/buildbot/status/master.py b/master/buildbot/status/master.py index 862ed8b63aa..e26138a9600 100644 --- a/master/buildbot/status/master.py +++ b/master/buildbot/status/master.py @@ -336,6 +336,7 @@ def builderAdded(self, name, basedir, category=None): log.msg("added builder %s in category %s" % (name, category)) # an unpickled object might not have category set from before, # so set it here to make sure + builder_status.category = category builder_status.master = self.master builder_status.basedir = os.path.join(self.basedir, basedir) builder_status.name = name # it might have been updated From dac8887e7763a2627be4eb4c907f737ace6d13b6 Mon Sep 17 00:00:00 2001 From: Harry Borkhuis Date: Thu, 10 May 2012 09:31:13 +0200 Subject: [PATCH 035/136] in gridview filter out builds with more than 1 sourcestamp --- master/buildbot/status/web/grid.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/master/buildbot/status/web/grid.py b/master/buildbot/status/web/grid.py index 0deb018268b..368488c4a50 100644 --- a/master/buildbot/status/web/grid.py +++ b/master/buildbot/status/web/grid.py @@ -193,7 +193,7 @@ def content(self, request, cxt): for build in self.getRecentBuilds(builder, numBuilds, branch): #TODO: support multiple sourcestamps - if build.getSourceStamps() == 1: + if len(build.getSourceStamps()) == 1: ss = build.getSourceStamps(absolute=True)[0] key= self.getSourceStampKey(ss) for i in range(len(stamps)): @@ -205,6 +205,7 @@ def content(self, request, cxt): b['builds'] = [] for build in builds: b['builds'].append(self.build_cxt(request, build)) + cxt['builders'].append(b) template = request.site.buildbot_service.templates.get_template("grid.html") From 407c04675e5fe24631d6622b2e414dc064707de9 Mon Sep 17 00:00:00 2001 From: Harry Borkhuis Date: Thu, 10 May 2012 09:55:45 +0200 Subject: [PATCH 036/136] in transposed gridview filter out builds with more than 1 sourcestamp. Fixed transposed grid --- master/buildbot/status/web/grid.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/master/buildbot/status/web/grid.py b/master/buildbot/status/web/grid.py index 368488c4a50..81ad44b9411 100644 --- a/master/buildbot/status/web/grid.py +++ b/master/buildbot/status/web/grid.py @@ -265,11 +265,12 @@ def content(self, request, cxt): for build in self.getRecentBuilds(builder, numBuilds, branch): #TODO: support multiple sourcestamps - ss = build.getSourceStamps(absolute=True)[0] - key = self.getSourceStampKey(ss) - for i in range(len(stamps)): - if key == self.getSourceStampKey(stamps[i]) and builds[i] is None: - builds[i] = build + if len(build.getSourceStamps()) == 1: + ss = build.getSourceStamps(absolute=True)[0] + key = self.getSourceStampKey(ss) + for i in range(len(stamps)): + if key == self.getSourceStampKey(stamps[i]) and builds[i] is None: + builds[i] = build b = yield self.builder_cxt(request, builder) builders.append(b) @@ -277,5 +278,4 @@ def content(self, request, cxt): builder_builds.append(map(lambda b: self.build_cxt(request, b), builds)) template = request.site.buildbot_service.templates.get_template('grid_transposed.html') - yield template.render(**cxt) - + defer.returnValue(template.render(**cxt)) \ No newline at end of file From 90b3b66da8caa2a4cc8897fb655852b8d18bf9b2 Mon Sep 17 00:00:00 2001 From: Harry Borkhuis Date: Thu, 10 May 2012 15:28:38 +0200 Subject: [PATCH 037/136] use empty dictionairy if no got_revisions found --- master/buildbot/status/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/master/buildbot/status/build.py b/master/buildbot/status/build.py index b9569606320..8b5eda4a2fe 100644 --- a/master/buildbot/status/build.py +++ b/master/buildbot/status/build.py @@ -101,7 +101,7 @@ def getSourceStamps(self, absolute=False): if not absolute: sourcestamps.extend(self.sources) else: - all_got_revisions = self.getAllGotRevisions() + all_got_revisions = self.getAllGotRevisions() or {} # always make a new instance for ss in self.sources: if ss.codebase in all_got_revisions: From ad8edd85304a672f0947dcfc0c9bceae0e11496c Mon Sep 17 00:00:00 2001 From: "Dustin J. Mitchell" Date: Thu, 10 May 2012 09:47:39 -0500 Subject: [PATCH 038/136] typo --- master/docs/release-notes.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/master/docs/release-notes.rst b/master/docs/release-notes.rst index a7bd36c81bd..5714ae44c9f 100644 --- a/master/docs/release-notes.rst +++ b/master/docs/release-notes.rst @@ -82,7 +82,7 @@ Features ~~~~~~~~ * Buildbot now supports building projects composed of multiple codebases. New - schedulers can aggregate changes to multipl codebases into source stamp sets + schedulers can aggregate changes to multiple codebases into source stamp sets (with one source stamp for each codebase). Source steps then check out each codebase as required, and the remainder of the build process proceeds normally. See the documentation for details. From f22ed6a5237e6baf413f410b506e019c3757477f Mon Sep 17 00:00:00 2001 From: Benoit Allard Date: Thu, 10 May 2012 21:31:04 +0200 Subject: [PATCH 039/136] gotAllRevisions now returns an empty dict when the property is not set --- master/buildbot/status/build.py | 2 +- master/buildbot/status/web/base.py | 5 +---- master/buildbot/status/web/build.py | 5 ++--- master/buildbot/status/web/builder.py | 2 +- master/buildbot/status/web/feeds.py | 2 +- 5 files changed, 6 insertions(+), 10 deletions(-) diff --git a/master/buildbot/status/build.py b/master/buildbot/status/build.py index 912bf75122d..0b0c0b13d76 100644 --- a/master/buildbot/status/build.py +++ b/master/buildbot/status/build.py @@ -89,7 +89,7 @@ def getPreviousBuild(self): return self.builder.getBuild(self.number-1) def getAllGotRevisions(self): - all_got_revisions = self.properties.getProperty('got_revision', None) + all_got_revisions = self.properties.getProperty('got_revision', {}) # For backwards compatibility all_got_revisions is a string if codebases # are not used. Convert to the default internal type (dict) if isinstance(all_got_revisions, str): diff --git a/master/buildbot/status/web/base.py b/master/buildbot/status/web/base.py index ab36161593b..02127894269 100644 --- a/master/buildbot/status/web/base.py +++ b/master/buildbot/status/web/base.py @@ -438,10 +438,7 @@ def get_line_values(self, req, build, include_builder=True): if ss_list: # TODO: support multiple sourcestamps in web interface repo = ss_list[0].repository - if all_got_revision: - rev = all_got_revision[ss_list[0].codebase] - else: - rev = "??" + rev = all_got_revision.get(ss_list[0].codebase, "??") else: repo = 'unknown, no information in build' rev = 'unknown' diff --git a/master/buildbot/status/web/build.py b/master/buildbot/status/web/build.py index 8ebb2075968..f35e4b0f5e5 100644 --- a/master/buildbot/status/web/build.py +++ b/master/buildbot/status/web/build.py @@ -167,9 +167,8 @@ def content(self, req, cxt): cxt['most_recent_rev_build'] = True all_got_revisions = b.getAllGotRevisions() - if all_got_revisions: - got_revision = all_got_revisions.get(ss.codebase, "??") - cxt['got_revision'] = str(got_revision) + got_revision = all_got_revisions.get(ss.codebase, "??") + cxt['got_revision'] = str(got_revision) try: cxt['slave_url'] = path_to_slave(req, status.getSlave(b.getSlavename())) diff --git a/master/buildbot/status/web/builder.py b/master/buildbot/status/web/builder.py index 8052eb18855..5a9955c6a9e 100644 --- a/master/buildbot/status/web/builder.py +++ b/master/buildbot/status/web/builder.py @@ -515,7 +515,7 @@ def keep_count(statuses, builderName): b = builds[0] bld['build_url'] = (bld['link'] + "/builds/%d" % b.getNumber()) label = None - all_got_revisions = b.getAllGotRevisions() or {} + all_got_revisions = b.getAllGotRevisions() # If len = 1 then try if revision can be used as label. if len(all_got_revisions) == 1: label = all_got_revisions[all_got_revisions.keys()[0]] diff --git a/master/buildbot/status/web/feeds.py b/master/buildbot/status/web/feeds.py index 39f49a33618..701339a8af0 100644 --- a/master/buildbot/status/web/feeds.py +++ b/master/buildbot/status/web/feeds.py @@ -179,7 +179,7 @@ def content(self, request): # title: trunk r22191 (plus patch) failed on # 'i686-debian-sarge1 shared gcc-3.3.5' ss_list = build.getSourceStamps() - all_got_revisions = build.getAllGotRevisions() or {} + all_got_revisions = build.getAllGotRevisions() src_cxts = [] for ss in ss_list: sc = {} From 588c516c099ce1addcdfb752380cba35fb39cae2 Mon Sep 17 00:00:00 2001 From: "Dustin J. Mitchell" Date: Thu, 10 May 2012 22:54:25 -0500 Subject: [PATCH 040/136] add some tests for Change --- .../test/unit/test_changes_changes.py | 112 ++++++++++++++++++ 1 file changed, 112 insertions(+) create mode 100644 master/buildbot/test/unit/test_changes_changes.py diff --git a/master/buildbot/test/unit/test_changes_changes.py b/master/buildbot/test/unit/test_changes_changes.py new file mode 100644 index 00000000000..946adc4151f --- /dev/null +++ b/master/buildbot/test/unit/test_changes_changes.py @@ -0,0 +1,112 @@ +# This file is part of Buildbot. Buildbot is free software: you can +# redistribute it and/or modify it under the terms of the GNU General Public +# License as published by the Free Software Foundation, version 2. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +# details. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, write to the Free Software Foundation, Inc., 51 +# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Copyright Buildbot Team Members + +import textwrap +import re +from twisted.trial import unittest +from buildbot.test.fake import fakedb +from buildbot.changes import changes + +class Change(unittest.TestCase): + + change23_rows = [ + fakedb.Change(changeid=23, author="dustin", comments="fix whitespace", + is_dir=0, branch="warnerdb", revision="deadbeef", + when_timestamp=266738404, revlink='http://warner/0e92a098b', + category='devel', repository='git://warner', codebase='mainapp', + project='Buildbot'), + + fakedb.ChangeFile(changeid=23, filename='master/README.txt'), + fakedb.ChangeFile(changeid=23, filename='slave/README.txt'), + + fakedb.ChangeProperty(changeid=23, property_name='notest', + property_value='["no","Change"]'), + + fakedb.ChangeUser(changeid=23, uid=27), + ] + + def setUp(self): + self.change23 = changes.Change(**dict( # using **dict(..) forces kwargs + category='devel', + isdir=0, + repository=u'git://warner', + codebase=u'mainapp', + who=u'dustin', + when=266738404, + comments=u'fix whitespace', + project=u'Buildbot', + branch=u'warnerdb', + revlink=u'http://warner/0e92a098b', + properties={'notest':"no"}, + files=[u'master/README.txt', u'slave/README.txt'], + revision=u'deadbeef')) + self.change23.number = 23 + + def test_str(self): + string = str(self.change23) + self.assertTrue(re.match(r"Change\(.*\)", string), string) + + def test_asText(self): + text = self.change23.asText() + self.assertEqual(text, textwrap.dedent(u'''\ + Files: + master/README.txt + slave/README.txt + On: git://warner + For: Buildbot + At: Thu 15 Jun 1978 01:00:04 + Changed By: dustin + Comments: fix whitespaceProperties: + notest: no + + ''')) + + def test_asDict(self): + dict = self.change23.asDict() + self.assertEqual(dict, { + 'at': 'Thu 15 Jun 1978 01:00:04', + 'branch': u'warnerdb', + 'category': u'devel', + 'codebase': u'mainapp', + 'comments': u'fix whitespace', + 'files': [{'name': u'master/README.txt'}, + {'name': u'slave/README.txt'}], + 'number': 23, + 'project': u'Buildbot', + 'properties': [('notest', 'no', 'Change')], + 'repository': u'git://warner', + 'rev': u'deadbeef', + 'revision': u'deadbeef', + 'revlink': u'http://warner/0e92a098b', + 'when': 266738404, + 'who': u'dustin'}) + + def test_getShortAuthor(self): + self.assertEqual(self.change23.getShortAuthor(), 'dustin') + + def test_getTime(self): + # careful, or timezones will hurt here + self.assertIn('Jun 1978', self.change23.getTime()) + + def test_getTimes(self): + self.assertEqual(self.change23.getTimes(), (266738404, None)) + + def test_getText(self): + self.change23.who = 'nasty < nasty' # test the html escaping (ugh!) + self.assertEqual(self.change23.getText(), ['nasty < nasty']) + + def test_getLogs(self): + self.assertEqual(self.change23.getLogs(), {}) + From 53d9dd7abc024deead611af459668a25b58be3c8 Mon Sep 17 00:00:00 2001 From: Harry Borkhuis Date: Fri, 11 May 2012 11:12:36 +0200 Subject: [PATCH 041/136] added the concept of multiple repositories to documentation --- master/docs/manual/cfg-global.rst | 24 +++++++------- master/docs/manual/cfg-schedulers.rst | 13 +++++--- master/docs/manual/concepts.rst | 45 +++++++++++++++++++++++++++ master/docs/release-notes.rst | 6 ++-- 4 files changed, 68 insertions(+), 20 deletions(-) diff --git a/master/docs/manual/cfg-global.rst b/master/docs/manual/cfg-global.rst index 48c80a1e21f..5a05ccbcc9c 100644 --- a/master/docs/manual/cfg-global.rst +++ b/master/docs/manual/cfg-global.rst @@ -737,20 +737,18 @@ Codebase Generator :: + all_repositories = { + r'https://hg/hg/mailsuite/mailclient': 'mailexe', + r'https://hg/hg/mailsuite/mapilib': 'mapilib', + r'https://hg/hg/mailsuite/imaplib': 'imaplib', + r'https://github.com/mailinc/mailsuite/mailclient': 'mailexe', + r'https://github.com/mailinc/mailsuite/mapilib': 'mapilib', + r'https://github.com/mailinc/mailsuite/imaplib': 'imaplib', + } + def codebaseGenerator(chdict): - all_repositories = { - r'https://hg/hg/mailsuite/mailclient': 'mailexe', - r'https://hg/hg/mailsuite/mapilib': 'mapilib', - r'https://hg/hg/mailsuite/imaplib': 'imaplib', - r'https://github.com/mailinc/mailsuite/mailclient': 'mailexe', - r'https://github.com/mailinc/mailsuite/mapilib': 'mapilib', - r'https://github.com/mailinc/mailsuite/imaplib': 'imaplib', - } - if chdict['repository'] in all_repositories: - return all_repositories[chdict['repository']] - else: - return '' - + return all_repositories[chdict['repository']] + c['codebaseGenerator'] = codebaseGenerator For any incomming change a :ref:`codebase` is set to ''. This diff --git a/master/docs/manual/cfg-schedulers.rst b/master/docs/manual/cfg-schedulers.rst index 8dd6116c7ce..45fb736d065 100644 --- a/master/docs/manual/cfg-schedulers.rst +++ b/master/docs/manual/cfg-schedulers.rst @@ -73,10 +73,15 @@ available with all schedulers. When the scheduler processes data from more than 1 repository at the same time then a corresponding codebase definition should be passed for each repository. A codebase definition is a dictionary with one or more of the - following keys: repository, branch, revision. The codebase definitions are - also to be passed as dictionary:: - - codebases = {'codebase1': {'repository':'....', ...}, 'codebase2': {} } + following keys: repository, branch, revision. The codebase definitions have + also to be passed as dictionary. + + .. code-block:: python + + codebases = {'codebase1': {'repository':'....', + 'branch':'default', + 'revision': None}, + 'codebase2': {'repository':'....'} } .. IMPORTANT:: ``codebases`` behaves also like a change_filter on codebase. If ``codebases`` is set then the scheduler will only process changes when their diff --git a/master/docs/manual/concepts.rst b/master/docs/manual/concepts.rst index 8aa43538ab1..953ca196014 100644 --- a/master/docs/manual/concepts.rst +++ b/master/docs/manual/concepts.rst @@ -131,6 +131,51 @@ that revision marker. When the VC system does not provide this, a timestamp from the middle of the tree-stable period is used to generate the source tree [#]_. +.. _Multiple-source-trees: + +Multiple source-trees ++++++++++++++++++++++ + +What if an end-product is composed of several components and all these components +have different repositories? Changes may arrive from different repositories within +the tree-stable-timer period. Buildbot will not only use the source-trees that +contain changes but also needs the remaining source-trees to build the complete +product. The remaining source-trees could be in the 'stable' branches but for +nightly tests the latest revision of the 'default' branch could also be used. + +For this reason a :class:`Scheduler` can be configured to base a build on a set +of several source-trees that can (partly) be overidden by the specific revisions +inside the arrived :class:`Change`\s. + +A single source-tree is identified by its repository, branch and revision. The +repository is a reference to a place where the sources reside. In projects it is +possible to have the same sources at different places (for example a local +repository and one on Github). For this reason an extra identifier 'codebase' is +used to distinguish different repositories and to treat equal repositories as +the same. + +A complete multiple repository configuration consists of: + + - a *codebase generator* + + Every relevant change arriving from a VC must contain a codebase. Because VC's + do not supply a codebase this has to be done by a so called + :bb:cfg:`codebaseGenerator` that is defined in the configuration. + + - some *schedulers* + + The :bb:cfg:`scheduler` is configured with a set of codebases. + These codebases represent the set of required repositories. For each + codebase a default repository, a default branch, and a default revision are supplied to retrieve the source-trees that are not in the arrived changes. + + - multiple *source steps* + + A :ref:`Builder` contains a :ref:`source step` for + each codebase. Each of the sourcesteps has a codebase attribute. With this + attribute the source step can ask the :class:`Build` object for the + correct repository, branch and revision data. The information comes from + the arrived changes or from the scheduler's default values. + .. _How-Different-VC-Systems-Specify-Sources: How Different VC Systems Specify Sources diff --git a/master/docs/release-notes.rst b/master/docs/release-notes.rst index a7bd36c81bd..590e010bfdc 100644 --- a/master/docs/release-notes.rst +++ b/master/docs/release-notes.rst @@ -82,11 +82,11 @@ Features ~~~~~~~~ * Buildbot now supports building projects composed of multiple codebases. New - schedulers can aggregate changes to multipl codebases into source stamp sets + schedulers can aggregate changes to multiple codebases into source stamp sets (with one source stamp for each codebase). Source steps then check out each codebase as required, and the remainder of the build process proceeds - normally. See the documentation for details. - + normally. See the :ref:`Multiple-source-trees` for details. + Slave ----- From c3d29da2528902f9a51c30a7ad8c3278f629bf79 Mon Sep 17 00:00:00 2001 From: Harry Borkhuis Date: Fri, 11 May 2012 12:17:03 +0200 Subject: [PATCH 042/136] document that by default schedulers only accept changes with an empty sourcestamp --- master/docs/manual/cfg-schedulers.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/master/docs/manual/cfg-schedulers.rst b/master/docs/manual/cfg-schedulers.rst index 45fb736d065..45cd7603ecc 100644 --- a/master/docs/manual/cfg-schedulers.rst +++ b/master/docs/manual/cfg-schedulers.rst @@ -84,8 +84,10 @@ available with all schedulers. 'codebase2': {'repository':'....'} } .. IMPORTANT:: ``codebases`` behaves also like a change_filter on codebase. - If ``codebases`` is set then the scheduler will only process changes when their - codebases are found in ``codebases`` + The scheduler will only process changes when their codebases are found + in ``codebases``. By default ``codebases`` is set to ``{'':{}}`` which + means that only changes with codebase '' (default value for codebase) + will be accepted by the scheduler. Buildsteps can have a reference to one of the codebases. The step will only get information (revision, branch etc.) that is related to that codebase. From d4a043900b818f373adada76b647f597f935b672 Mon Sep 17 00:00:00 2001 From: Harry Borkhuis Date: Fri, 11 May 2012 14:32:33 +0200 Subject: [PATCH 043/136] rewriting of some sentences multiple-tree concept (by Benoit) --- master/docs/manual/concepts.rst | 46 ++++++++++++++++++++------------- 1 file changed, 28 insertions(+), 18 deletions(-) diff --git a/master/docs/manual/concepts.rst b/master/docs/manual/concepts.rst index 953ca196014..159f922a466 100644 --- a/master/docs/manual/concepts.rst +++ b/master/docs/manual/concepts.rst @@ -140,41 +140,51 @@ What if an end-product is composed of several components and all these component have different repositories? Changes may arrive from different repositories within the tree-stable-timer period. Buildbot will not only use the source-trees that contain changes but also needs the remaining source-trees to build the complete -product. The remaining source-trees could be in the 'stable' branches but for -nightly tests the latest revision of the 'default' branch could also be used. +product. -For this reason a :class:`Scheduler` can be configured to base a build on a set +For this reason a :ref:`Scheduler` can be configured to base a build on a set of several source-trees that can (partly) be overidden by the specific revisions inside the arrived :class:`Change`\s. + A single source-tree is identified by its repository, branch and revision. The repository is a reference to a place where the sources reside. In projects it is -possible to have the same sources at different places (for example a local -repository and one on Github). For this reason an extra identifier 'codebase' is -used to distinguish different repositories and to treat equal repositories as -the same. +possible to have the same sources at different places (for example a collection +of fork of the same repository). For this reason an extra identifier 'codebase' +has been introduced to make buildbot categorize all copies of the same source-tree +as the same. Furthermore, the codebase_ allow you to uniquely identify a *part* of +your project. A complete multiple repository configuration consists of: - a *codebase generator* - Every relevant change arriving from a VC must contain a codebase. Because VC's - do not supply a codebase this has to be done by a so called - :bb:cfg:`codebaseGenerator` that is defined in the configuration. + Every relevant change arriving from a VC must contain a codebase. + Because VC's do not supply a codebase this has to be done by a so + called :bb:cfg:`codebaseGenerator` that is defined in the + configuration. - some *schedulers* - The :bb:cfg:`scheduler` is configured with a set of codebases. - These codebases represent the set of required repositories. For each - codebase a default repository, a default branch, and a default revision are supplied to retrieve the source-trees that are not in the arrived changes. + The :bb:cfg:`scheduler` has to be configured with a set of all + required ``codebases`` to build a product. These codebases indicate the set of + required source-trees. In order for the scheduler to be able to produce a + complete set for each build, each given codebase will be supplied a default + repository, branch, and revision. Those ones will be used for the codebases + where no changes arrived. - multiple *source steps* - A :ref:`Builder` contains a :ref:`source step` for - each codebase. Each of the sourcesteps has a codebase attribute. With this - attribute the source step can ask the :class:`Build` object for the - correct repository, branch and revision data. The information comes from - the arrived changes or from the scheduler's default values. + A :ref:`Builder` has to contains a :ref:`source step` for + each needed source-tree. Each of the source steps has a ``codebase`` attribute + which is used to retrieve the corresponding repository, branch and revision + data. This information comes from the arrived changes or from the + scheduler's configured default values. + +.. warning:: + + Defining a :bb:cfg:`codebaseGenerator` that returns non empty (not + ``''``) codebases, change the behavior of all the schedulers. .. _How-Different-VC-Systems-Specify-Sources: From e844003796cebc697cfd31bf37ee856df68cd02a Mon Sep 17 00:00:00 2001 From: Jared Grubb Date: Sat, 31 Mar 2012 15:23:23 -0700 Subject: [PATCH 044/136] Fixes #2266: Add new 'exception' mode for MailNotifier --- master/buildbot/status/mail.py | 12 +++- master/buildbot/test/unit/test_status_mail.py | 65 +++++++++++++++++-- 2 files changed, 69 insertions(+), 8 deletions(-) diff --git a/master/buildbot/status/mail.py b/master/buildbot/status/mail.py index 1114ad000e9..781db39ea6f 100644 --- a/master/buildbot/status/mail.py +++ b/master/buildbot/status/mail.py @@ -49,7 +49,7 @@ from buildbot import interfaces, util, config from buildbot.process.users import users from buildbot.status import base -from buildbot.status.results import FAILURE, SUCCESS, WARNINGS, Results +from buildbot.status.results import FAILURE, SUCCESS, WARNINGS, EXCEPTION, Results VALID_EMAIL = re.compile("[a-zA-Z0-9\.\_\%\-\+]+@[a-zA-Z0-9\.\_\%\-]+.[a-zA-Z]{2,6}") @@ -92,6 +92,8 @@ def defaultMessage(mode, name, build, results, master_status): text += "The Buildbot has detected a restored build" else: text += "The Buildbot has detected a passing build" + elif results == EXCEPTION: + text += "The Buildbot has detected a build exception" projects = [] if ss_list: @@ -177,7 +179,7 @@ class MailNotifier(base.StatusReceiverMultiService): "subject", "sendToInterestedUsers", "customMesg", "messageFormatter", "extraHeaders"] - possible_modes = ("change", "failing", "passing", "problem", "warnings") + possible_modes = ("change", "failing", "passing", "problem", "warnings", "exception") def __init__(self, fromaddr, mode=("failing", "passing", "warnings"), categories=None, builders=None, addLogs=False, @@ -220,6 +222,8 @@ def __init__(self, fromaddr, mode=("failing", "passing", "warnings"), - "problem": send mail about a build which failed when the previous build passed - "warnings": send mail if a build contain warnings + - "exception": send mail if a build fails due to an exception + - "all": always send mail Defaults to ("failing", "passing", "warnings"). @type builders: list of strings @@ -316,7 +320,7 @@ def __init__(self, fromaddr, mode=("failing", "passing", "warnings"), self.fromaddr = fromaddr if isinstance(mode, basestring): if mode == "all": - mode = ("failing", "passing", "warnings") + mode = ("failing", "passing", "warnings", "exception") elif mode == "warnings": mode = ("failing", "warnings") else: @@ -434,6 +438,8 @@ def isMailNeeded(self, build, results): return True if "warnings" in self.mode and results == WARNINGS: return True + if "exception" in self.mode and results == EXCEPTION: + return True return False diff --git a/master/buildbot/test/unit/test_status_mail.py b/master/buildbot/test/unit/test_status_mail.py index 724f95225a3..2263a2c6ee6 100644 --- a/master/buildbot/test/unit/test_status_mail.py +++ b/master/buildbot/test/unit/test_status_mail.py @@ -17,7 +17,7 @@ from mock import Mock from buildbot import config from twisted.trial import unittest -from buildbot.status.results import SUCCESS, FAILURE +from buildbot.status.results import SUCCESS, FAILURE, WARNINGS, EXCEPTION from buildbot.status.mail import MailNotifier from twisted.internet import defer from buildbot.test.fake import fakedb @@ -440,15 +440,70 @@ def test_buildFinished_ignores_unspecified_categories(self): self.assertEqual(None, mn.buildFinished('dummyBuilder', build, SUCCESS)) - def test_buildFinished_mode_all_always_sends_email(self): + def run_simple_test_sends_email_for_mode(self, mode, result): mock_method = Mock() self.patch(MailNotifier, "buildMessage", mock_method) - mn = MailNotifier('from@example.org', mode=("failing", "passing", "warnings")) + mn = MailNotifier('from@example.org', mode=mode) build = FakeBuildStatus(name="build") - mn.buildFinished('dummyBuilder', build, FAILURE) + mn.buildFinished('dummyBuilder', build, result) - mock_method.assert_called_with('dummyBuilder', [build], FAILURE) + mock_method.assert_called_with('dummyBuilder', [build], result) + + def run_simple_test_ignores_email_for_mode(self, mode, result): + mock_method = Mock() + self.patch(MailNotifier, "buildMessage", mock_method) + mn = MailNotifier('from@example.org', mode=mode) + + build = FakeBuildStatus(name="build") + mn.buildFinished('dummyBuilder', build, result) + + self.assertFalse(mock_method.called) + + def test_buildFinished_mode_all_for_success(self): + self.run_simple_test_sends_email_for_mode("all", SUCCESS) + def test_buildFinished_mode_all_for_failure(self): + self.run_simple_test_sends_email_for_mode("all", FAILURE) + def test_buildFinished_mode_all_for_warnings(self): + self.run_simple_test_sends_email_for_mode("all", WARNINGS) + def test_buildFinished_mode_all_for_exception(self): + self.run_simple_test_sends_email_for_mode("all", EXCEPTION) + + def test_buildFinished_mode_failing_for_success(self): + self.run_simple_test_ignores_email_for_mode("failing", SUCCESS) + def test_buildFinished_mode_failing_for_failure(self): + self.run_simple_test_sends_email_for_mode("failing", FAILURE) + def test_buildFinished_mode_failing_for_warnings(self): + self.run_simple_test_ignores_email_for_mode("failing", WARNINGS) + def test_buildFinished_mode_failing_for_exception(self): + self.run_simple_test_ignores_email_for_mode("failing", EXCEPTION) + + def test_buildFinished_mode_exception_for_success(self): + self.run_simple_test_ignores_email_for_mode("exception", SUCCESS) + def test_buildFinished_mode_exception_for_failure(self): + self.run_simple_test_ignores_email_for_mode("exception", FAILURE) + def test_buildFinished_mode_exception_for_warnings(self): + self.run_simple_test_ignores_email_for_mode("exception", WARNINGS) + def test_buildFinished_mode_exception_for_exception(self): + self.run_simple_test_sends_email_for_mode("exception", EXCEPTION) + + def test_buildFinished_mode_warnings_for_success(self): + self.run_simple_test_ignores_email_for_mode("warnings", SUCCESS) + def test_buildFinished_mode_warnings_for_failure(self): + self.run_simple_test_sends_email_for_mode("warnings", FAILURE) + def test_buildFinished_mode_warnings_for_warnings(self): + self.run_simple_test_sends_email_for_mode("warnings", WARNINGS) + def test_buildFinished_mode_warnings_for_exception(self): + self.run_simple_test_ignores_email_for_mode("warnings", EXCEPTION) + + def test_buildFinished_mode_passing_for_success(self): + self.run_simple_test_sends_email_for_mode("passing", SUCCESS) + def test_buildFinished_mode_passing_for_failure(self): + self.run_simple_test_ignores_email_for_mode("passing", FAILURE) + def test_buildFinished_mode_passing_for_warnings(self): + self.run_simple_test_ignores_email_for_mode("passing", WARNINGS) + def test_buildFinished_mode_passing_for_exception(self): + self.run_simple_test_ignores_email_for_mode("passing", EXCEPTION) def test_buildFinished_mode_failing_ignores_successful_build(self): mn = MailNotifier('from@example.org', mode=("failing",)) From 88995bdf9bbb3bafa66c64dbb9c868011c6f2f24 Mon Sep 17 00:00:00 2001 From: Jared Grubb Date: Fri, 11 May 2012 07:23:12 -0700 Subject: [PATCH 045/136] Add docs for the 'exception' and 'all' mail modes --- master/docs/manual/cfg-statustargets.rst | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/master/docs/manual/cfg-statustargets.rst b/master/docs/manual/cfg-statustargets.rst index b78a4720074..b13ade92d83 100644 --- a/master/docs/manual/cfg-statustargets.rst +++ b/master/docs/manual/cfg-statustargets.rst @@ -949,6 +949,12 @@ MailNotifier arguments ``warnings`` Send mail about builds which generate warnings. + ``exception`` + Send mail about builds which generate exceptions. + + ``all`` + Always send mail about builds. + Defaults to (``failing``, ``passing``, ``warnings``). ``builders`` @@ -1048,7 +1054,8 @@ Name of the project :meth:`master_status.getProjectName()` MailNotifier mode - ``mode`` (a combination of ``change``, ``failing``, ``passing``, ``problem``, ``warnings``) + ``mode`` (a combination of ``change``, ``failing``, ``passing``, ``problem``, ``warnings``, + ``exception``, ``all``) Builder result as a string :: From e45c5af912b452bae14a0191062e5621e013a6af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Leroy?= Date: Fri, 11 May 2012 16:40:30 +0200 Subject: [PATCH 046/136] add 'project' support to svnpoller When using SVNPoller( ... , project='myproject' ), project is always empty. This patch fixes it. --- master/buildbot/changes/svnpoller.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/master/buildbot/changes/svnpoller.py b/master/buildbot/changes/svnpoller.py index d59849fab2c..12bfe053727 100644 --- a/master/buildbot/changes/svnpoller.py +++ b/master/buildbot/changes/svnpoller.py @@ -53,7 +53,7 @@ class SVNPoller(base.PollingChangeSource, util.ComparableMixin): """ compare_attrs = ["svnurl", "split_file", - "svnuser", "svnpasswd", + "svnuser", "svnpasswd", "project", "pollInterval", "histmax", "svnbin", "category", "cachepath"] From 5dffe8d99046a1552517a866145b772125c927d7 Mon Sep 17 00:00:00 2001 From: Marc Abramowitz Date: Fri, 11 May 2012 11:16:22 -0700 Subject: [PATCH 047/136] Make `python setup.py test` do something useful --- slave/setup.cfg | 2 ++ slave/setup.py | 6 ++++++ 2 files changed, 8 insertions(+) create mode 100644 slave/setup.cfg diff --git a/slave/setup.cfg b/slave/setup.cfg new file mode 100644 index 00000000000..c8be2270745 --- /dev/null +++ b/slave/setup.cfg @@ -0,0 +1,2 @@ +[aliases] +test = trial -m buildslave diff --git a/slave/setup.py b/slave/setup.py index 53a4aea3e4a..dcfeda50f7f 100755 --- a/slave/setup.py +++ b/slave/setup.py @@ -121,6 +121,12 @@ def make_release_tree(self, base_dir, files): setup_args['install_requires'] = [ 'twisted >= 8.0.0', ] + setup_args['setup_requires'] = [ + 'setuptools_trial', + ] + setup_args['tests_require'] = [ + 'mock', + ] if os.getenv('NO_INSTALL_REQS'): setup_args['install_requires'] = None From 18a64779cca54aedba6091ee40c2dda5a3ced70c Mon Sep 17 00:00:00 2001 From: Marc Abramowitz Date: Fri, 11 May 2012 11:29:16 -0700 Subject: [PATCH 048/136] Make `python setup.py test` work (in a fresh virtualenv!) for buildbot/master --- master/setup.cfg | 2 + master/setup.py | 109 +++-------------------------------------------- 2 files changed, 8 insertions(+), 103 deletions(-) create mode 100644 master/setup.cfg diff --git a/master/setup.cfg b/master/setup.cfg new file mode 100644 index 00000000000..55c57e2da41 --- /dev/null +++ b/master/setup.cfg @@ -0,0 +1,2 @@ +[aliases] +test = trial -m buildbot diff --git a/master/setup.py b/master/setup.py index fa32bf9c021..0cbc75bc1a7 100755 --- a/master/setup.py +++ b/master/setup.py @@ -37,107 +37,6 @@ def include(d, e): return (d, [f for f in glob.glob('%s/%s'%(d, e)) if os.path.isfile(f)]) -class _SetupBuildCommand(Command): - """ - Master setup build command to subclass from. - """ - - user_options = [] - - def initialize_options(self): - """ - Setup the current dir. - """ - self._dir = os.getcwd() - - def finalize_options(self): - """ - Required. - """ - pass - - -class TestCommand(_SetupBuildCommand): - """ - Executes tests from setup. - """ - - description = "Run unittests inline" - - def run(self): - """ - Public run method. - """ - self._run(os.path.normpath(os.path.abspath( - os.path.join('buildbot', 'test')))) - - def _run(self, test_loc): - """ - Executes the test step. - - @param test_loc: location of test module - @type test_loc: str - """ - from twisted.scripts.trial import run - - # remove the 'test' option from argv - sys.argv.remove('test') - - # Mimick the trial script by adding the path as the last arg - sys.argv.append(test_loc) - - # Add the current dir to path and pull it all together - sys.path.insert(0, os.path.curdir) - sys.path[:] = map(os.path.abspath, sys.path) - # GO! - run() - - -class SdistTestCommand(TestCommand): - """ - Runs unittests from the sdist output. - """ - - description = "Run unittests from inside an sdist distribution" - - def run(self): - """ - Interesting magic to get a source dist and running trial on it. - - NOTE: there is magic going on here! If you know a better way feel - free to update it. - """ - # Clean out dist/ - if os.path.exists('dist'): - for root, dirs, files in os.walk('dist', topdown=False): - for name in files: - os.remove(os.path.join(root, name)) - for name in dirs: - os.rmdir(os.path.join(root, name)) - # Import setup making it as if we ran setup.py with the sdist arg - sys.argv.append('sdist') - import setup #@Reimport @UnresolvedImport @UnusedImport - try: - # attempt to extract the sdist data - from gzip import GzipFile - from tarfile import TarFile - # We open up the gzip as well as using the first item as the sdist - gz = GzipFile(os.path.join('dist', os.listdir('dist')[0])) - tf = TarFile(fileobj=gz) - # Make the output dir and generate the extract path - os.mkdir(os.path.join('dist', 'sdist_test')) - ex_path = os.path.join('dist', 'sdist_test', - tf.getmembers()[0].name, 'buildbot', 'test') - # Extract the data and run tests - print "Extracting to %s" % ex_path - tf.extractall(os.path.join('dist', 'sdist_test')) - print "Executing tests ..." - self._run(os.path.normpath(os.path.abspath(ex_path))) - except IndexError, ie: - # We get called twice and the IndexError is OK - pass - - class install_data_twisted(install_data): """make sure data files are installed in package. this is evil. @@ -265,8 +164,6 @@ def make_release_tree(self, base_dir, files): ], 'scripts': scripts, 'cmdclass': {'install_data': install_data_twisted, - 'test': TestCommand, - 'sdist_test': SdistTestCommand, 'sdist': our_sdist}, } @@ -295,6 +192,12 @@ def make_release_tree(self, base_dir, files): 'sqlalchemy-migrate ==0.6.1, ==0.7.0, ==0.7.1, ==0.7.2', 'python-dateutil==1.5', ] + setup_args['setup_requires'] = [ + 'setuptools_trial', + ] + setup_args['tests_require'] = [ + 'mock', + ] # Python-2.6 and up includes json if not py_26: setup_args['install_requires'].append('simplejson') From 2949eb9b2ac76d4e18e3d849f1e2fb93188af655 Mon Sep 17 00:00:00 2001 From: Marc Abramowitz Date: Fri, 11 May 2012 12:50:51 -0700 Subject: [PATCH 049/136] Add support for tox (http://tox.testrun.org/) for quick local testing across multiple Pythons. --- .gitignore | 1 + master/tox.ini | 10 ++++++++++ slave/tox.ini | 10 ++++++++++ 3 files changed, 21 insertions(+) create mode 100644 master/tox.ini create mode 100644 slave/tox.ini diff --git a/.gitignore b/.gitignore index 80390732195..f439f079354 100644 --- a/.gitignore +++ b/.gitignore @@ -39,3 +39,4 @@ apidocs/reference.tgz common/googlecode_upload.py master/docs/tutorial/_build _build +.tox diff --git a/master/tox.ini b/master/tox.ini new file mode 100644 index 00000000000..e9e6fe099d0 --- /dev/null +++ b/master/tox.ini @@ -0,0 +1,10 @@ +# Tox (http://tox.testrun.org/) is a tool for running tests +# in multiple virtualenvs. This configuration file will run the +# test suite on all supported python versions. To use it, "pip install tox" +# and then run "tox" from this directory. + +[tox] +envlist = py25, py26, py27 + +[testenv] +commands = python setup.py test diff --git a/slave/tox.ini b/slave/tox.ini new file mode 100644 index 00000000000..aea725bfd75 --- /dev/null +++ b/slave/tox.ini @@ -0,0 +1,10 @@ +# Tox (http://tox.testrun.org/) is a tool for running tests +# in multiple virtualenvs. This configuration file will run the +# test suite on all supported python versions. To use it, "pip install tox" +# and then run "tox" from this directory. + +[tox] +envlist = py24, py25, py26, py27 + +[testenv] +commands = python setup.py test From cf4309e4ae34f37d2a11b473dc5886335abea896 Mon Sep 17 00:00:00 2001 From: "Dustin J. Mitchell" Date: Sat, 12 May 2012 10:07:34 -0500 Subject: [PATCH 050/136] add a 'mapping' test for Jinja This makes the template changes in c977f04d work. --- master/buildbot/status/web/base.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/master/buildbot/status/web/base.py b/master/buildbot/status/web/base.py index 5e8902cee80..87050b3e161 100644 --- a/master/buildbot/status/web/base.py +++ b/master/buildbot/status/web/base.py @@ -514,6 +514,8 @@ def createJinjaEnv(revlink=None, changecommentlink=None, env.install_null_translations() # needed until we have a proper i18n backend + env.tests['mapping'] = lambda obj : isinstance(obj, dict) + env.filters.update(dict( urlencode = urllib.quote, email = emailfilter, From 696a762b6d8d06814035875e3cf4aa88e986dd91 Mon Sep 17 00:00:00 2001 From: "Dustin J. Mitchell" Date: Sat, 12 May 2012 10:14:50 -0500 Subject: [PATCH 051/136] fix timezone-sensitive test --- master/buildbot/test/unit/test_changes_changes.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/master/buildbot/test/unit/test_changes_changes.py b/master/buildbot/test/unit/test_changes_changes.py index 946adc4151f..af29b7e59a2 100644 --- a/master/buildbot/test/unit/test_changes_changes.py +++ b/master/buildbot/test/unit/test_changes_changes.py @@ -60,18 +60,18 @@ def test_str(self): def test_asText(self): text = self.change23.asText() - self.assertEqual(text, textwrap.dedent(u'''\ + self.assertTrue(re.match(textwrap.dedent(u'''\ Files: master/README.txt slave/README.txt On: git://warner For: Buildbot - At: Thu 15 Jun 1978 01:00:04 + At: .* Changed By: dustin Comments: fix whitespaceProperties: notest: no - ''')) + '''), text), text) def test_asDict(self): dict = self.change23.asDict() From c9c60178c9237b11148a8e28952d0f450e606f05 Mon Sep 17 00:00:00 2001 From: "Dustin J. Mitchell" Date: Sat, 12 May 2012 10:17:47 -0500 Subject: [PATCH 052/136] another timezeone-sensitive test --- master/buildbot/test/unit/test_changes_changes.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/master/buildbot/test/unit/test_changes_changes.py b/master/buildbot/test/unit/test_changes_changes.py index af29b7e59a2..6dcee925731 100644 --- a/master/buildbot/test/unit/test_changes_changes.py +++ b/master/buildbot/test/unit/test_changes_changes.py @@ -75,8 +75,9 @@ def test_asText(self): def test_asDict(self): dict = self.change23.asDict() + self.assertIn('1978', dict['at']) # timezone-sensitive + del dict['at'] self.assertEqual(dict, { - 'at': 'Thu 15 Jun 1978 01:00:04', 'branch': u'warnerdb', 'category': u'devel', 'codebase': u'mainapp', From 31736e2ae006471bfd392807d7b4ad3385345472 Mon Sep 17 00:00:00 2001 From: Jared Grubb Date: Fri, 16 Mar 2012 14:38:16 -0700 Subject: [PATCH 053/136] Fixes #2180: Allow for "git describe" in addition to got_revision --- master/buildbot/steps/source/git.py | 94 ++++- .../test/unit/test_steps_source_git.py | 344 ++++++++++++++++++ 2 files changed, 426 insertions(+), 12 deletions(-) diff --git a/master/buildbot/steps/source/git.py b/master/buildbot/steps/source/git.py index e2d150e3677..0d1819d1520 100644 --- a/master/buildbot/steps/source/git.py +++ b/master/buildbot/steps/source/git.py @@ -20,6 +20,39 @@ from buildbot.steps.source.base import Source from buildbot.interfaces import BuildSlaveTooOldError +def isTrueOrIsExactlyZero(v): + # nonzero values are true... + if v: + return True + + # ... and True for the number zero, but we have to + # explicitly guard against v==False, since + # isinstance(False, int) is surprisingly True + if isinstance(v, int) and v is not False: + return True + + # all other false-ish values are false + return False + +git_describe_flags = [ + # on or off + ('all', lambda v: ['--all'] if v else None), + ('always', lambda v: ['--always'] if v else None), + ('contains', lambda v: ['--contains'] if v else None), + ('debug', lambda v: ['--debug'] if v else None), + ('long', lambda v: ['--long'] if v else None), + ('exact-match', lambda v: ['--exact-match'] if v else None), + ('tags', lambda v: ['--tags'] if v else None), + # string parameter + ('match', lambda v: ['--match', v] if v else None), + # numeric parameter + ('abbrev', lambda v: ['--abbrev=%s' % v] if isTrueOrIsExactlyZero(v) else None), + ('candidates', lambda v: ['--candidates=%s' % v] if isTrueOrIsExactlyZero(v) else None), + # optional string parameter + ('dirty', lambda v: ['--dirty'] if (v is True or v=='') else None), + ('dirty', lambda v: ['--dirty=%s' % v] if (v and v is not True) else None), +] + class Git(Source): """ Class for Git with all the smarts """ name='git' @@ -27,7 +60,8 @@ class Git(Source): def __init__(self, repourl=None, branch='HEAD', mode='incremental', method=None, submodules=False, shallow=False, progress=False, - retryFetch=False, clobberOnFailure=False, **kwargs): + retryFetch=False, clobberOnFailure=False, getDescription=False, + **kwargs): """ @type repourl: string @param repourl: the URL which points at the git repository @@ -57,7 +91,12 @@ def __init__(self, repourl=None, branch='HEAD', mode='incremental', @type retryFetch: boolean @param retryFetch: Retry fetching before failing source checkout. + + @type getDescription: boolean or dict + @param getDescription: Use 'git describe' to describe the fetched revision """ + if not getDescription: + getDescription = False self.branch = branch self.method = method @@ -69,6 +108,7 @@ def __init__(self, repourl=None, branch='HEAD', mode='incremental', self.fetchcount = 0 self.clobberOnFailure = clobberOnFailure self.mode = mode + self.getDescription = getDescription Source.__init__(self, **kwargs) self.addFactoryArguments(branch=branch, mode=mode, @@ -80,12 +120,15 @@ def __init__(self, repourl=None, branch='HEAD', mode='incremental', retryFetch=retryFetch, clobberOnFailure= clobberOnFailure, + getDescription= + getDescription ) assert self.mode in ['incremental', 'full'] assert self.repourl is not None if self.mode == 'full': assert self.method in ['clean', 'fresh', 'clobber', 'copy', None] + assert isinstance(self.getDescription, (bool, dict)) def startVC(self, branch, revision, patch): self.branch = branch or 'HEAD' @@ -107,6 +150,7 @@ def checkInstall(gitInstalled): if patch: d.addCallback(self.patch, patch) d.addCallback(self.parseGotRevision) + d.addCallback(self.parseCommitDescription) d.addCallback(self.finish) d.addErrback(self.failed) return d @@ -224,17 +268,43 @@ def _gotResults(results): d.addCallbacks(self.finished, self.checkDisconnect) return d - def parseGotRevision(self, _): - d = self._dovccmd(['rev-parse', 'HEAD'], collectStdout=True) - def setrev(stdout): - revision = stdout.strip() - if len(revision) != 40: - raise buildstep.BuildStepFailed() - log.msg("Got Git revision %s" % (revision, )) - self.setProperty('got_revision', revision, 'Source') - return 0 - d.addCallback(setrev) - return d + @defer.inlineCallbacks + def parseGotRevision(self, _=None): + stdout = yield self._dovccmd(['rev-parse', 'HEAD'], collectStdout=True) + revision = stdout.strip() + if len(revision) != 40: + raise buildstep.BuildStepFailed() + log.msg("Got Git revision %s" % (revision, )) + self.setProperty('got_revision', revision, 'Source') + + defer.returnValue(0) + + @defer.inlineCallbacks + def parseCommitDescription(self, _=None): + if not self.getDescription: + defer.returnValue(0) + return + + cmd = ['describe'] + if isinstance(self.getDescription, dict): + for opt, arg in git_describe_flags: + opt = self.getDescription.get(opt, None) + arg = arg(opt) + if arg: + cmd.extend(arg) + cmd.append('HEAD') + + try: + stdout = yield self._dovccmd(cmd, collectStdout=True) + desc = stdout.strip() + if self.codebase: + self.setProperty('commit-description-%s'%self.codebase, desc, 'Source') + else: + self.setProperty('commit-description', desc, 'Source') + except: + pass + + defer.returnValue(0) def _dovccmd(self, command, abandonOnFailure=True, collectStdout=False, initialStdin=None): cmd = buildstep.RemoteShellCommand(self.workdir, ['git'] + command, diff --git a/master/buildbot/test/unit/test_steps_source_git.py b/master/buildbot/test/unit/test_steps_source_git.py index ba512bd1080..2b8a35bccda 100644 --- a/master/buildbot/test/unit/test_steps_source_git.py +++ b/master/buildbot/test/unit/test_steps_source_git.py @@ -56,6 +56,7 @@ def test_mode_full_clean(self): + 0, ) self.expectOutcome(result=SUCCESS, status_text=["update"]) + self.expectProperty('got_revision', 'f6ad368298bd941e934a41f3babc827b2aa95a1d', 'Source') return self.runStep() def test_mode_full_clean_patch(self): @@ -92,6 +93,7 @@ def test_mode_full_clean_patch(self): + 0, ) self.expectOutcome(result=SUCCESS, status_text=["update"]) + self.expectProperty('got_revision', 'f6ad368298bd941e934a41f3babc827b2aa95a1d', 'Source') return self.runStep() def test_mode_full_clean_patch_fail(self): @@ -123,6 +125,7 @@ def test_mode_full_clean_patch_fail(self): + 1, ) self.expectOutcome(result=FAILURE, status_text=["updating"]) + self.expectNoProperty('got_revision') return self.runStep() def test_mode_full_clean_branch(self): @@ -157,6 +160,7 @@ def test_mode_full_clean_branch(self): + 0, ) self.expectOutcome(result=SUCCESS, status_text=["update"]) + self.expectProperty('got_revision', 'f6ad368298bd941e934a41f3babc827b2aa95a1d', 'Source') return self.runStep() def test_mode_full_clean_parsefail(self): @@ -186,6 +190,7 @@ def test_mode_full_clean_parsefail(self): + 128, ) self.expectOutcome(result=FAILURE, status_text=["updating"]) + self.expectNoProperty('got_revision') return self.runStep() def test_mode_full_clean_no_existing_repo(self): @@ -265,6 +270,7 @@ def test_mode_full_clobber(self): + 0, ) self.expectOutcome(result=SUCCESS, status_text=["update"]) + self.expectProperty('got_revision', 'f6ad368298bd941e934a41f3babc827b2aa95a1d', 'Source') return self.runStep() def test_mode_full_clobber_branch(self): @@ -292,6 +298,7 @@ def test_mode_full_clobber_branch(self): + 0, ) self.expectOutcome(result=SUCCESS, status_text=["update"]) + self.expectProperty('got_revision', 'f6ad368298bd941e934a41f3babc827b2aa95a1d', 'Source') return self.runStep() def test_mode_incremental(self): @@ -321,6 +328,7 @@ def test_mode_incremental(self): ) self.expectOutcome(result=SUCCESS, status_text=["update"]) + self.expectProperty('got_revision', 'f6ad368298bd941e934a41f3babc827b2aa95a1d', 'Source') return self.runStep() def test_mode_incremental_branch(self): @@ -353,6 +361,7 @@ def test_mode_incremental_branch(self): ) self.expectOutcome(result=SUCCESS, status_text=["update"]) + self.expectProperty('got_revision', 'f6ad368298bd941e934a41f3babc827b2aa95a1d', 'Source') return self.runStep() def test_mode_full_fresh(self): @@ -384,6 +393,7 @@ def test_mode_full_fresh(self): + 0, ) self.expectOutcome(result=SUCCESS, status_text=["update"]) + self.expectProperty('got_revision', 'f6ad368298bd941e934a41f3babc827b2aa95a1d', 'Source') return self.runStep() def test_mode_incremental_given_revision(self): @@ -412,6 +422,7 @@ def test_mode_incremental_given_revision(self): + 0, ) self.expectOutcome(result=SUCCESS, status_text=["update"]) + self.expectProperty('got_revision', 'f6ad368298bd941e934a41f3babc827b2aa95a1d', 'Source') return self.runStep() def test_mode_full_fresh_submodule(self): @@ -450,6 +461,7 @@ def test_mode_full_fresh_submodule(self): + 0, ) self.expectOutcome(result=SUCCESS, status_text=["update"]) + self.expectProperty('got_revision', 'f6ad368298bd941e934a41f3babc827b2aa95a1d', 'Source') return self.runStep() def test_mode_full_clobber_shallow(self): @@ -477,6 +489,7 @@ def test_mode_full_clobber_shallow(self): + 0, ) self.expectOutcome(result=SUCCESS, status_text=["update"]) + self.expectProperty('got_revision', 'f6ad368298bd941e934a41f3babc827b2aa95a1d', 'Source') return self.runStep() def test_mode_incremental_retryFetch(self): @@ -514,6 +527,7 @@ def test_mode_incremental_retryFetch(self): + 0, ) self.expectOutcome(result=SUCCESS, status_text=["update"]) + self.expectProperty('got_revision', 'f6ad368298bd941e934a41f3babc827b2aa95a1d', 'Source') return self.runStep() def test_mode_incremental_retryFetch_branch(self): @@ -554,6 +568,7 @@ def test_mode_incremental_retryFetch_branch(self): + 0, ) self.expectOutcome(result=SUCCESS, status_text=["update"]) + self.expectProperty('got_revision', 'f6ad368298bd941e934a41f3babc827b2aa95a1d', 'Source') return self.runStep() def test_mode_incremental_clobberOnFailure(self): @@ -592,6 +607,7 @@ def test_mode_incremental_clobberOnFailure(self): + 0, ) self.expectOutcome(result=SUCCESS, status_text=["update"]) + self.expectProperty('got_revision', 'f6ad368298bd941e934a41f3babc827b2aa95a1d', 'Source') return self.runStep() def test_mode_incremental_clobberOnFailure_branch(self): @@ -630,6 +646,7 @@ def test_mode_incremental_clobberOnFailure_branch(self): + 0, ) self.expectOutcome(result=SUCCESS, status_text=["update"]) + self.expectProperty('got_revision', 'f6ad368298bd941e934a41f3babc827b2aa95a1d', 'Source') return self.runStep() def test_mode_full_copy(self): @@ -664,6 +681,7 @@ def test_mode_full_copy(self): + 0, ) self.expectOutcome(result=SUCCESS, status_text=["update"]) + self.expectProperty('got_revision', 'f6ad368298bd941e934a41f3babc827b2aa95a1d', 'Source') return self.runStep() @@ -691,6 +709,7 @@ def test_mode_incremental_no_existing_repo(self): ) self.expectOutcome(result=SUCCESS, status_text=["update"]) + self.expectProperty('got_revision', 'f6ad368298bd941e934a41f3babc827b2aa95a1d', 'Source') return self.runStep() def test_mode_full_clobber_given_revision(self): @@ -722,6 +741,7 @@ def test_mode_full_clobber_given_revision(self): + 0, ) self.expectOutcome(result=SUCCESS, status_text=["update"]) + self.expectProperty('got_revision', 'f6ad368298bd941e934a41f3babc827b2aa95a1d', 'Source') return self.runStep() def test_revparse_failure(self): @@ -753,6 +773,7 @@ def test_revparse_failure(self): + 0, ) self.expectOutcome(result=FAILURE, status_text=["updating"]) + self.expectNoProperty('got_revision') return self.runStep() def test_mode_full_clobber_submodule(self): @@ -783,6 +804,7 @@ def test_mode_full_clobber_submodule(self): + 0, ) self.expectOutcome(result=SUCCESS, status_text=["update"]) + self.expectProperty('got_revision', 'f6ad368298bd941e934a41f3babc827b2aa95a1d', 'Source') return self.runStep() def test_repourl(self): @@ -818,6 +840,7 @@ def test_mode_full_fresh_revision(self): + 0, ) self.expectOutcome(result=SUCCESS, status_text=["update"]) + self.expectProperty('got_revision', 'f6ad368298bd941e934a41f3babc827b2aa95a1d', 'Source') return self.runStep() def test_mode_full_fresh_clobberOnFailure(self): @@ -853,6 +876,7 @@ def test_mode_full_fresh_clobberOnFailure(self): + 0, ) self.expectOutcome(result=SUCCESS, status_text=["update"]) + self.expectProperty('got_revision', 'f6ad368298bd941e934a41f3babc827b2aa95a1d', 'Source') return self.runStep() def test_mode_full_no_method(self): @@ -884,6 +908,7 @@ def test_mode_full_no_method(self): + 0, ) self.expectOutcome(result=SUCCESS, status_text=["update"]) + self.expectProperty('got_revision', 'f6ad368298bd941e934a41f3babc827b2aa95a1d', 'Source') return self.runStep() def test_mode_full_with_env(self): @@ -920,6 +945,7 @@ def test_mode_full_with_env(self): + 0, ) self.expectOutcome(result=SUCCESS, status_text=["update"]) + self.expectProperty('got_revision', 'f6ad368298bd941e934a41f3babc827b2aa95a1d', 'Source') return self.runStep() def test_mode_full_logEnviron(self): @@ -956,4 +982,322 @@ def test_mode_full_logEnviron(self): + 0, ) self.expectOutcome(result=SUCCESS, status_text=["update"]) + self.expectProperty('got_revision', 'f6ad368298bd941e934a41f3babc827b2aa95a1d', 'Source') + return self.runStep() + + def test_getDescription(self): + # clone of: test_mode_incremental + # only difference is to set the getDescription property + + self.setupStep( + git.Git(repourl='http://github.com/buildbot/buildbot.git', + mode='incremental', + getDescription=True)) + self.expectCommands( + ## copied from test_mode_incremental: + ExpectShell(workdir='wkdir', + command=['git', '--version']) + + 0, + Expect('stat', dict(file='wkdir/.git', + logEnviron=True)) + + 0, + ExpectShell(workdir='wkdir', + command=['git', 'fetch', '-t', + 'http://github.com/buildbot/buildbot.git', + 'HEAD']) + + 0, + ExpectShell(workdir='wkdir', + command=['git', 'reset', '--hard', 'FETCH_HEAD']) + + 0, + ExpectShell(workdir='wkdir', + command=['git', 'rev-parse', 'HEAD']) + + ExpectShell.log('stdio', + stdout='f6ad368298bd941e934a41f3babc827b2aa95a1d') + + 0, + + ## plus this to test describe: + ExpectShell(workdir='wkdir', + command=['git', 'describe', 'HEAD']) + + ExpectShell.log('stdio', + stdout='Tag-1234') + + 0, + ) + self.expectOutcome(result=SUCCESS, status_text=["update"]) + self.expectProperty('got_revision', 'f6ad368298bd941e934a41f3babc827b2aa95a1d', 'Source') + self.expectProperty('commit-description', 'Tag-1234', 'Source') + return self.runStep() + + def test_getDescription_failed(self): + # clone of: test_mode_incremental + # only difference is to set the getDescription property + + # this tests when 'git describe' fails; for example, there are no + # tags in the repository + + self.setupStep( + git.Git(repourl='http://github.com/buildbot/buildbot.git', + mode='incremental', + getDescription=True)) + self.expectCommands( + ## copied from test_mode_incremental: + ExpectShell(workdir='wkdir', + command=['git', '--version']) + + 0, + Expect('stat', dict(file='wkdir/.git', + logEnviron=True)) + + 0, + ExpectShell(workdir='wkdir', + command=['git', 'fetch', '-t', + 'http://github.com/buildbot/buildbot.git', + 'HEAD']) + + 0, + ExpectShell(workdir='wkdir', + command=['git', 'reset', '--hard', 'FETCH_HEAD']) + + 0, + ExpectShell(workdir='wkdir', + command=['git', 'rev-parse', 'HEAD']) + + ExpectShell.log('stdio', + stdout='f6ad368298bd941e934a41f3babc827b2aa95a1d') + + 0, + + ## plus this to test describe: + ExpectShell(workdir='wkdir', + command=['git', 'describe', 'HEAD']) + + ExpectShell.log('stdio', + stdout='') + + 128, # error, but it's suppressed + ) + self.expectOutcome(result=SUCCESS, status_text=["update"]) + self.expectProperty('got_revision', 'f6ad368298bd941e934a41f3babc827b2aa95a1d', 'Source') + self.expectNoProperty('commit-description') + return self.runStep() + + def setup_getDescription_test(self, setup_args, output_args, codebase=None): + # clone of: test_mode_full_clobber + # only difference is to set the getDescription property + + kwargs = {} + if codebase is not None: + kwargs.update(codebase=codebase) + + self.setupStep( + git.Git(repourl='http://github.com/buildbot/buildbot.git', + mode='full', method='clobber', progress=True, + getDescription=setup_args, + **kwargs)) + + self.expectCommands( + ## copied from test_mode_full_clobber: + ExpectShell(workdir='wkdir', + command=['git', '--version']) + + 0, + Expect('rmdir', dict(dir='wkdir', + logEnviron=True)) + + 0, + ExpectShell(workdir='wkdir', + command=['git', 'clone', + '--branch', 'HEAD', + 'http://github.com/buildbot/buildbot.git', + '.', '--progress']) + + 0, + ExpectShell(workdir='wkdir', + command=['git', 'rev-parse', 'HEAD']) + + ExpectShell.log('stdio', + stdout='f6ad368298bd941e934a41f3babc827b2aa95a1d') + + 0, + + ## plus this to test describe: + ExpectShell(workdir='wkdir', + command=['git', 'describe'] + + output_args + + ['HEAD']) + + ExpectShell.log('stdio', + stdout='Tag-1234') + + 0, + ) + + self.expectProperty('got_revision', 'f6ad368298bd941e934a41f3babc827b2aa95a1d', 'Source') + + if codebase: + self.expectOutcome(result=SUCCESS, status_text=["update", codebase]) + self.expectProperty('commit-description-%s' % codebase, 'Tag-1234', 'Source') + else: + self.expectOutcome(result=SUCCESS, status_text=["update"]) + self.expectProperty('commit-description', 'Tag-1234', 'Source') + + def test_getDescription_match(self): + self.setup_getDescription_test( + setup_args = { 'match': 'stuff-*' }, + output_args = ['--match', 'stuff-*'] + ) + return self.runStep() + def test_getDescription_match_false(self): + self.setup_getDescription_test( + setup_args = { 'match': None }, + output_args = [] + ) + return self.runStep() + + def test_getDescription_tags(self): + self.setup_getDescription_test( + setup_args = { 'tags': True }, + output_args = ['--tags'] + ) + return self.runStep() + def test_getDescription_tags_false(self): + self.setup_getDescription_test( + setup_args = { 'tags': False }, + output_args = [] + ) + return self.runStep() + + def test_getDescription_all(self): + self.setup_getDescription_test( + setup_args = { 'all': True }, + output_args = ['--all'] + ) + return self.runStep() + def test_getDescription_all_false(self): + self.setup_getDescription_test( + setup_args = { 'all': False }, + output_args = [] + ) + return self.runStep() + + def test_getDescription_abbrev(self): + self.setup_getDescription_test( + setup_args = { 'abbrev': 7 }, + output_args = ['--abbrev=7'] + ) + return self.runStep() + def test_getDescription_abbrev_zero(self): + self.setup_getDescription_test( + setup_args = { 'abbrev': 0 }, + output_args = ['--abbrev=0'] + ) + return self.runStep() + def test_getDescription_abbrev_false(self): + self.setup_getDescription_test( + setup_args = { 'abbrev': False }, + output_args = [] + ) + return self.runStep() + + def test_getDescription_dirty(self): + self.setup_getDescription_test( + setup_args = { 'dirty': True }, + output_args = ['--dirty'] + ) + return self.runStep() + def test_getDescription_dirty_empty_str(self): + self.setup_getDescription_test( + setup_args = { 'dirty': '' }, + output_args = ['--dirty'] + ) + return self.runStep() + def test_getDescription_dirty_str(self): + self.setup_getDescription_test( + setup_args = { 'dirty': 'foo' }, + output_args = ['--dirty=foo'] + ) + return self.runStep() + def test_getDescription_dirty_false(self): + self.setup_getDescription_test( + setup_args = { 'dirty': False }, + output_args = [] + ) + return self.runStep() + + def test_getDescription_contains(self): + self.setup_getDescription_test( + setup_args = { 'contains': True }, + output_args = ['--contains'] + ) + return self.runStep() + def test_getDescription_contains_false(self): + self.setup_getDescription_test( + setup_args = { 'contains': False }, + output_args = [] + ) + return self.runStep() + + def test_getDescription_candidates(self): + self.setup_getDescription_test( + setup_args = { 'candidates': 7 }, + output_args = ['--candidates=7'] + ) + return self.runStep() + def test_getDescription_candidates_zero(self): + self.setup_getDescription_test( + setup_args = { 'candidates': 0 }, + output_args = ['--candidates=0'] + ) + return self.runStep() + def test_getDescription_candidates_false(self): + self.setup_getDescription_test( + setup_args = { 'candidates': False }, + output_args = [] + ) + return self.runStep() + + def test_getDescription_exact_match(self): + self.setup_getDescription_test( + setup_args = { 'exact-match': True }, + output_args = ['--exact-match'] + ) + return self.runStep() + def test_getDescription_exact_match_false(self): + self.setup_getDescription_test( + setup_args = { 'exact-match': False }, + output_args = [] + ) + return self.runStep() + + def test_getDescription_debug(self): + self.setup_getDescription_test( + setup_args = { 'debug': True }, + output_args = ['--debug'] + ) + return self.runStep() + def test_getDescription_debug_false(self): + self.setup_getDescription_test( + setup_args = { 'debug': False }, + output_args = [] + ) + return self.runStep() + + def test_getDescription_long(self): + self.setup_getDescription_test( + setup_args = { 'long': True }, + output_args = ['--long'] + ) + def test_getDescription_long_false(self): + self.setup_getDescription_test( + setup_args = { 'long': False }, + output_args = [] + ) + return self.runStep() + + def test_getDescription_always(self): + self.setup_getDescription_test( + setup_args = { 'always': True }, + output_args = ['--always'] + ) + def test_getDescription_always_false(self): + self.setup_getDescription_test( + setup_args = { 'always': False }, + output_args = [] + ) + return self.runStep() + + def test_getDescription_lotsa_stuff(self): + self.setup_getDescription_test( + setup_args = { 'match': 'stuff-*', + 'abbrev': 6, + 'exact-match': True}, + output_args = ['--exact-match', + '--match', 'stuff-*', + '--abbrev=6'], + codebase='baz' + ) return self.runStep() From 593b67ebcafd1b3b21e06dd1da208170682185ae Mon Sep 17 00:00:00 2001 From: "Eric W. Anderson" Date: Sat, 12 May 2012 14:14:20 -0500 Subject: [PATCH 054/136] Add tutorial tour section on adjusting the authorized users in the webstatus --- master/docs/tutorial/tour.rst | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/master/docs/tutorial/tour.rst b/master/docs/tutorial/tour.rst index 1ceeeaa9845..33028080f89 100644 --- a/master/docs/tutorial/tour.rst +++ b/master/docs/tutorial/tour.rst @@ -48,6 +48,33 @@ Now, look for the section marked *PROJECT IDENTITY* which reads:: c['titleURL'] = "http://divmod.org/trac/wiki/DivmodPyflakes" If you want, you can change either of these links to anything you want to see what happens when you change them. + +Setting Authorized Web Users +---------------------------- + +Further down, look for the WebStatus configuration:: + + c['status'] = [] + + from buildbot.status import html + from buildbot.status.web import authz, auth + + authz_cfg=authz.Authz( + # change any of these to True to enable; see the manual for more + # options + auth=auth.BasicAuth([("pyflakes","pyflakes")]), + gracefulShutdown = False, + forceBuild = 'auth', # use this to test your slave once it is set up + forceAllBuilds = False, + pingBuilder = False, + stopBuild = False, + stopAllBuilds = False, + cancelPendingBuild = False, + ) + c['status'].append(html.WebStatus(http_port=8010, authz=authz_cfg)) + +The auth.BasicAuth() define authorized users and their passwords. You can change these or add new ones. + After making a change go into the terminal and type:: buildbot reconfig master From 1e563c3d390b68b03c44741a97ea1da0ec7c76ef Mon Sep 17 00:00:00 2001 From: Jared Grubb Date: Sat, 12 May 2012 11:37:16 -0700 Subject: [PATCH 055/136] git-describe: update docs; adjust behavior of 'dict()' --- master/buildbot/steps/source/git.py | 9 ++---- .../test/unit/test_steps_source_git.py | 21 ++++++++++-- master/docs/manual/cfg-buildsteps.rst | 32 +++++++++++++++++++ 3 files changed, 53 insertions(+), 9 deletions(-) diff --git a/master/buildbot/steps/source/git.py b/master/buildbot/steps/source/git.py index 0d1819d1520..a9a75effb69 100644 --- a/master/buildbot/steps/source/git.py +++ b/master/buildbot/steps/source/git.py @@ -95,7 +95,7 @@ def __init__(self, repourl=None, branch='HEAD', mode='incremental', @type getDescription: boolean or dict @param getDescription: Use 'git describe' to describe the fetched revision """ - if not getDescription: + if not getDescription and not isinstance(getDescription, dict): getDescription = False self.branch = branch @@ -281,7 +281,7 @@ def parseGotRevision(self, _=None): @defer.inlineCallbacks def parseCommitDescription(self, _=None): - if not self.getDescription: + if self.getDescription==False: # dict() should not return here defer.returnValue(0) return @@ -297,10 +297,7 @@ def parseCommitDescription(self, _=None): try: stdout = yield self._dovccmd(cmd, collectStdout=True) desc = stdout.strip() - if self.codebase: - self.setProperty('commit-description-%s'%self.codebase, desc, 'Source') - else: - self.setProperty('commit-description', desc, 'Source') + self.setProperty('commit-description', desc, 'Source') except: pass diff --git a/master/buildbot/test/unit/test_steps_source_git.py b/master/buildbot/test/unit/test_steps_source_git.py index 2b8a35bccda..0d7f73e2a5e 100644 --- a/master/buildbot/test/unit/test_steps_source_git.py +++ b/master/buildbot/test/unit/test_steps_source_git.py @@ -1116,15 +1116,30 @@ def setup_getDescription_test(self, setup_args, output_args, codebase=None): + 0, ) - self.expectProperty('got_revision', 'f6ad368298bd941e934a41f3babc827b2aa95a1d', 'Source') - if codebase: self.expectOutcome(result=SUCCESS, status_text=["update", codebase]) - self.expectProperty('commit-description-%s' % codebase, 'Tag-1234', 'Source') + self.expectProperty('got_revision', {codebase:'f6ad368298bd941e934a41f3babc827b2aa95a1d'}, 'Source') + self.expectProperty('commit-description', {codebase:'Tag-1234'}, 'Source') else: self.expectOutcome(result=SUCCESS, status_text=["update"]) + self.expectProperty('got_revision', 'f6ad368298bd941e934a41f3babc827b2aa95a1d', 'Source') self.expectProperty('commit-description', 'Tag-1234', 'Source') + def test_getDescription_empty_dict(self): + self.setup_getDescription_test( + setup_args = {}, + output_args = [] + ) + return self.runStep() + + def test_getDescription_empty_dict_with_codebase(self): + self.setup_getDescription_test( + setup_args = {}, + output_args = [], + codebase = 'baz' + ) + return self.runStep() + def test_getDescription_match(self): self.setup_getDescription_test( setup_args = { 'match': 'stuff-*' }, diff --git a/master/docs/manual/cfg-buildsteps.rst b/master/docs/manual/cfg-buildsteps.rst index aac6d62152a..e197bf8dec5 100644 --- a/master/docs/manual/cfg-buildsteps.rst +++ b/master/docs/manual/cfg-buildsteps.rst @@ -402,6 +402,38 @@ The Git step takes the following arguments: performs all the incremental checkout behavior in ``source`` directory. +``getDescription`` + + (optional) After checkout, invoke a `git describe` on the revision and save + the result in a property; the property's name is either ``commit-description`` + or ``commit-description-foo``, depending on whether the ``codebase`` + argument was also provided. The argument should either be a ``bool`` or ``dict``, + and will change how `git describe` is called: + + * ``getDescription=False``: disables this feature explicitly + * ``getDescription=True`` or empty ``dict()``: Run `git describe` with no args + * ``getDescription={...}``: a dict with keys named the same as the git option. + Each key's value can be ``False`` or ``None`` to explicitly skip that argument. + + For the following keys, a value of ``True`` appends the same-named git argument: + + * ``all`` : `--all` + * ``always``: `--always` + * ``contains``: `--contains` + * ``debug``: `--debug` + * ``long``: `--long`` + * ``exact-match``: `--exact-match` + * ``tags``: `--tags` + * ``dirty``: `--dirty` + + For the following keys, an integer or string value (depending on what git expects) + will set the argument's parameter appropriately. Examples show the key-value pair: + + * ``match=foo``: `--match foo` + * ``abbrev=7``: `--abbrev=7` + * ``candidates=7``: `--candidates=7` + * ``dirty=foo``: `--dirty=foo` + .. bb:step:: SVN .. _Step-SVN: From cecd0c9a6db88f6485506074ce9041b981ce7801 Mon Sep 17 00:00:00 2001 From: "Dustin J. Mitchell" Date: Sat, 12 May 2012 14:16:40 -0500 Subject: [PATCH 056/136] move 'Setting Authorized Web Users' down a bit, add a link --- master/docs/tutorial/tour.rst | 54 ++++++++++++++++++----------------- 1 file changed, 28 insertions(+), 26 deletions(-) diff --git a/master/docs/tutorial/tour.rst b/master/docs/tutorial/tour.rst index 33028080f89..26c735dd677 100644 --- a/master/docs/tutorial/tour.rst +++ b/master/docs/tutorial/tour.rst @@ -49,32 +49,6 @@ Now, look for the section marked *PROJECT IDENTITY* which reads:: If you want, you can change either of these links to anything you want to see what happens when you change them. -Setting Authorized Web Users ----------------------------- - -Further down, look for the WebStatus configuration:: - - c['status'] = [] - - from buildbot.status import html - from buildbot.status.web import authz, auth - - authz_cfg=authz.Authz( - # change any of these to True to enable; see the manual for more - # options - auth=auth.BasicAuth([("pyflakes","pyflakes")]), - gracefulShutdown = False, - forceBuild = 'auth', # use this to test your slave once it is set up - forceAllBuilds = False, - pingBuilder = False, - stopBuild = False, - stopAllBuilds = False, - cancelPendingBuild = False, - ) - c['status'].append(html.WebStatus(http_port=8010, authz=authz_cfg)) - -The auth.BasicAuth() define authorized users and their passwords. You can change these or add new ones. - After making a change go into the terminal and type:: buildbot reconfig master @@ -267,6 +241,34 @@ You can also see the new builds in the web interface. .. image:: _images/irc-testrun.png :alt: a successful test run from IRC happened. +Setting Authorized Web Users +---------------------------- + +Further down, look for the WebStatus configuration:: + + c['status'] = [] + + from buildbot.status import html + from buildbot.status.web import authz, auth + + authz_cfg=authz.Authz( + # change any of these to True to enable; see the manual for more + # options + auth=auth.BasicAuth([("pyflakes","pyflakes")]), + gracefulShutdown = False, + forceBuild = 'auth', # use this to test your slave once it is set up + forceAllBuilds = False, + pingBuilder = False, + stopBuild = False, + stopAllBuilds = False, + cancelPendingBuild = False, + ) + c['status'].append(html.WebStatus(http_port=8010, authz=authz_cfg)) + +The ``auth.BasicAuth()`` define authorized users and their passwords. You can +change these or add new ones. See :bb:status:`WebStatus` for more about the +WebStatus configuration. + Debugging with Manhole ---------------------- From db9865f57c3c60dc57dd404e06c0d7ea7f94a72c Mon Sep 17 00:00:00 2001 From: Jared Grubb Date: Sat, 12 May 2012 12:20:40 -0700 Subject: [PATCH 057/136] git-describe: update release notes; also fix a misplaced note for the 'descriptionSuffix' change --- master/docs/release-notes.rst | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/master/docs/release-notes.rst b/master/docs/release-notes.rst index 5714ae44c9f..ba375eb8c83 100644 --- a/master/docs/release-notes.rst +++ b/master/docs/release-notes.rst @@ -66,10 +66,6 @@ Deprecations, Removals, and Non-Compatible Changes from buildbot.steps.source.svn import SVN factory.append(SVN(repourl=Interpolate("svn://svn.example.org/svn/%(src::branch:-branches/test)s"))) -* ``Source`` and ``ShellCommand`` steps now have an optional ``descriptionSuffix``, a suffix to the - ``description``/``descriptionDone`` values. For example this can help distinguish between - multiple ``Compile`` steps that are applied to different codebases. - Changes for Developers ~~~~~~~~~~~~~~~~~~~~~~ @@ -87,6 +83,12 @@ Features codebase as required, and the remainder of the build process proceeds normally. See the documentation for details. +* ``Source`` and ``ShellCommand`` steps now have an optional ``descriptionSuffix``, a suffix to the + ``description``/``descriptionDone`` values. For example this can help distinguish between + multiple ``Compile`` steps that are applied to different codebases. + +* ``Git`` has a new ``getDescription`` option, which will run `git describe` after checkout + Slave ----- From d2bc38106ddfcc9e1aa0a892a5632d4d1fe2f417 Mon Sep 17 00:00:00 2001 From: Jared Grubb Date: Sat, 17 Mar 2012 13:06:09 -0700 Subject: [PATCH 058/136] Fixes #2165: Add 'ternary' interpolation for WithProperties --- master/buildbot/process/properties.py | 70 +++++- .../test/unit/test_process_properties.py | 213 ++++++++++++++++++ 2 files changed, 280 insertions(+), 3 deletions(-) diff --git a/master/buildbot/process/properties.py b/master/buildbot/process/properties.py index 99796a6dd70..a9cbdd553fc 100644 --- a/master/buildbot/process/properties.py +++ b/master/buildbot/process/properties.py @@ -185,6 +185,17 @@ class _PropertyMap(object): colon_minus_re = re.compile(r"(.*):-(.*)") colon_tilde_re = re.compile(r"(.*):~(.*)") colon_plus_re = re.compile(r"(.*):\+(.*)") + + colon_ternary_re = re.compile(r"""(?P.*) # the property to match + : # colon + (?P\#)? # might have the alt marker '#' + \? # question mark + (?P.) # the delimiter + (?P.*) # sub-if-true + (?P=delim) # the delimiter again + (?P.*)# sub-if-false + """, re.VERBOSE) + def __init__(self, properties): # use weakref here to avoid a reference loop self.properties = weakref.ref(properties) @@ -225,10 +236,38 @@ def colon_plus(mo): else: return '' + def colon_ternary(mo): + # %(prop:?:T:F)s + # if prop exists, use T; otherwise, F + # %(prop:#?:T:F)s + # if prop is true, use T; otherwise, F + groups = mo.groupdict() + + prop = groups['prop'] + + if prop in self.temp_vals: + if groups['alt']: + use_true = self.temp_vals[prop] + else: + use_true = True + elif properties.has_key(prop): + if groups['alt']: + use_true = properties[prop] + else: + use_true = True + else: + use_true = False + + if use_true: + return groups['true'] + else: + return groups['false'] + for regexp, fn in [ ( self.colon_minus_re, colon_minus ), ( self.colon_tilde_re, colon_tilde ), ( self.colon_plus_re, colon_plus ), + ( self.colon_ternary_re, colon_ternary ), ]: mo = regexp.match(key) if mo: @@ -436,6 +475,27 @@ def _parseColon_plus(self, d, kw, repl): defaultWhenFalse=False, elideNoneAs='') + colon_ternary_re = re.compile(r"""(?P.) # the delimiter + (?P.*) # sub-if-true + (?P=delim) # the delimiter again + (?P.*)# sub-if-false + """, re.VERBOSE) + + def _parseColon_ternary(self, d, kw, repl, defaultWhenFalse=False): + m = self.colon_ternary_re.match(repl) + if not m: + config.error("invalid Interpolate ternary expression for selector '%s' and delim '%s'" % (kw, repl[0])) + return None + m = m.groupdict() + return _Lookup(d, kw, + hasKey=Interpolate(m['true'], **self.kwargs), + default=Interpolate(m['false'], **self.kwargs), + defaultWhenFalse=defaultWhenFalse, + elideNoneAs='') + + def _parseColon_ternary_hash(self, d, kw, repl): + return self._parseColon_ternary(d, kw, repl, defaultWhenFalse=True) + def _parse(self, fmtstring): keys = _getInterpolationList(fmtstring) for key in keys: @@ -443,13 +503,17 @@ def _parse(self, fmtstring): d, kw, repl = self._parseSubstitution(key) if repl is None: repl = '-' - for char, fn in [ + for pattern, fn in [ ( "-", self._parseColon_minus ), ( "~", self._parseColon_tilde ), ( "+", self._parseColon_plus ), + ( "?", self._parseColon_ternary ), + ( "#?", self._parseColon_ternary_hash ) ]: - if repl[0] == char: - self.interpolations[key] = fn(d, kw, repl[1:]) + junk, matches, tail = repl.partition(pattern) + if not junk and matches: + self.interpolations[key] = fn(d, kw, tail) + break if not self.interpolations.has_key(key): config.error("invalid Interpolate default type '%s'" % repl[0]) diff --git a/master/buildbot/test/unit/test_process_properties.py b/master/buildbot/test/unit/test_process_properties.py index d86a1395512..8238e7741e3 100644 --- a/master/buildbot/test/unit/test_process_properties.py +++ b/master/buildbot/test/unit/test_process_properties.py @@ -197,6 +197,58 @@ def testColonPlusEmpty(self): def testColonPlusUnset(self): return self.doTestSimpleWithProperties('%(prop_nosuch:+present)s', '') + + def testColonTernarySet(self): + return self.doTestSimpleWithProperties('%(prop_str:?:present:missing)s', 'present') + + def testColonTernaryNone(self): + return self.doTestSimpleWithProperties('%(prop_none:?:present:missing)s', 'present') + + def testColonTernaryZero(self): + return self.doTestSimpleWithProperties('%(prop_zero:?|present|missing)s', 'present') + + def testColonTernaryOne(self): + return self.doTestSimpleWithProperties('%(prop_one:?:present:missing)s', 'present') + + def testColonTernaryFalse(self): + return self.doTestSimpleWithProperties('%(prop_false:?|present|missing)s', 'present') + + def testColonTernaryTrue(self): + return self.doTestSimpleWithProperties('%(prop_true:?:present:missing)s', 'present') + + def testColonTernaryEmpty(self): + return self.doTestSimpleWithProperties('%(prop_empty:?ApresentAmissing)s', 'present') + + def testColonTernaryUnset(self): + return self.doTestSimpleWithProperties('%(prop_nosuch:?#present#missing)s', 'missing') + + + def testColonTernaryHashSet(self): + return self.doTestSimpleWithProperties('%(prop_str:#?:truish:falsish)s', 'truish') + + def testColonTernaryHashNone(self): + # None is special-cased *differently* for '#?' + return self.doTestSimpleWithProperties('%(prop_none:#?|truish|falsish)s', 'falsish') + + def testColonTernaryHashZero(self): + return self.doTestSimpleWithProperties('%(prop_zero:#?:truish:falsish)s', 'falsish') + + def testColonTernaryHashOne(self): + return self.doTestSimpleWithProperties('%(prop_one:#?:truish:falsish)s', 'truish') + + def testColonTernaryHashFalse(self): + return self.doTestSimpleWithProperties('%(prop_false:#?:truish:falsish)s', 'falsish') + + def testColonTernaryHashTrue(self): + return self.doTestSimpleWithProperties('%(prop_true:#?|truish|falsish)s', 'truish') + + def testColonTernaryHashEmpty(self): + return self.doTestSimpleWithProperties('%(prop_empty:#?:truish:falsish)s', 'falsish') + + def testColonTernaryHashUnset(self): + return self.doTestSimpleWithProperties('%(prop_nosuch:#?.truish.falsish)s', 'falsish') + + def testClearTempValues(self): d = self.doTestSimpleWithProperties('', '', prop_temp=lambda b: 'present') @@ -274,6 +326,23 @@ def testTempValuePlusUnsetSet(self): prop_nosuch=lambda b: 1) + def testTempValueColonTernaryTrue(self): + return self.doTestSimpleWithProperties('%(prop_temp:?:present:missing)s', 'present', + prop_temp=lambda b: True) + + def testTempValueColonTernaryFalse(self): + return self.doTestSimpleWithProperties('%(prop_temp:?|present|missing)s', 'present', + prop_temp=lambda b: False) + + def testTempValueColonTernaryHashTrue(self): + return self.doTestSimpleWithProperties('%(prop_temp:#?|truish|falsish)s', 'truish', + prop_temp=lambda b: 1) + + def testTempValueColonTernaryHashFalse(self): + return self.doTestSimpleWithProperties('%(prop_temp:#?|truish|falsish)s', 'falsish', + prop_nosuch=lambda b: 0) + + class TestInterpolateConfigure(unittest.TestCase, ConfigErrorsMixin): """ Test that Interpolate reports erros in the interpolation string @@ -314,6 +383,14 @@ def test_nested_invalid_selector(self): lambda: Interpolate("%(prop:some_prop:~%(garbage:test)s)s")) + def test_colon_ternary_bad_delimeter(self): + self.assertRaisesConfigError("invalid Interpolate ternary expression for selector 'P' and delim ':'", + lambda: Interpolate("echo '%(prop:P:?:one)s'")) + + def test_colon_ternary_hash_bad_delimeter(self): + self.assertRaisesConfigError("invalid Interpolate ternary expression for selector 'P' and delim '|'", + lambda: Interpolate("echo '%(prop:P:#?|one)s'")) + class TestInterpolatePositional(unittest.TestCase): def setUp(self): @@ -417,6 +494,133 @@ def test_nested_property(self): "echo 'so long!'") return d + def test_nested_property_deferred(self): + renderable = DeferredRenderable() + self.props.setProperty("missing", renderable, "test") + self.props.setProperty("project", "so long!", "test") + command = Interpolate("echo '%(prop:missing:~%(prop:project)s)s'") + d = self.build.render(command) + d.addCallback(self.failUnlessEqual, + "echo 'so long!'") + renderable.callback(False) + return d + + def test_property_substitute_recursively(self): + self.props.setProperty("project", "proj1", "test") + command = Interpolate("echo '%(prop:no_such:-%(prop:project)s)s'") + d = self.build.render(command) + d.addCallback(self.failUnlessEqual, + "echo 'proj1'") + return d + + def test_property_colon_ternary_present(self): + self.props.setProperty("project", "proj1", "test") + command = Interpolate("echo %(prop:project:?:defined:missing)s") + d = self.build.render(command) + d.addCallback(self.failUnlessEqual, + "echo defined") + return d + + def test_property_colon_ternary_missing(self): + command = Interpolate("echo %(prop:project:?|defined|missing)s") + d = self.build.render(command) + d.addCallback(self.failUnlessEqual, + "echo missing") + return d + + def test_property_colon_ternary_hash_true(self): + self.props.setProperty("project", "winbld", "test") + command = Interpolate("echo buildby-%(prop:project:#?:T:F)s") + d = self.build.render(command) + d.addCallback(self.failUnlessEqual, + "echo buildby-T") + return d + + def test_property_colon_ternary_hash_false(self): + self.props.setProperty("project", "", "test") + command = Interpolate("echo buildby-%(prop:project:#?|T|F)s") + d = self.build.render(command) + d.addCallback(self.failUnlessEqual, + "echo buildby-F") + return d + + def test_property_colon_ternary_substitute_recursively_true(self): + self.props.setProperty("P", "present", "test") + self.props.setProperty("one", "proj1", "test") + self.props.setProperty("two", "proj2", "test") + command = Interpolate("echo '%(prop:P:?|%(prop:one)s|%(prop:two)s)s'") + d = self.build.render(command) + d.addCallback(self.failUnlessEqual, + "echo 'proj1'") + return d + + def test_property_colon_ternary_substitute_recursively_false(self): + self.props.setProperty("one", "proj1", "test") + self.props.setProperty("two", "proj2", "test") + command = Interpolate("echo '%(prop:P:?|%(prop:one)s|%(prop:two)s)s'") + d = self.build.render(command) + d.addCallback(self.failUnlessEqual, + "echo 'proj2'") + return d + + def test_property_substitute_recursively(self): + self.props.setProperty("project", "proj1", "test") + command = Interpolate("echo '%(prop:no_such:-%(prop:project)s)s'") + d = self.build.render(command) + d.addCallback(self.failUnlessEqual, + "echo 'proj1'") + return d + + def test_property_colon_ternary_present(self): + self.props.setProperty("project", "proj1", "test") + command = Interpolate("echo %(prop:project:?:defined:missing)s") + d = self.build.render(command) + d.addCallback(self.failUnlessEqual, + "echo defined") + return d + + def test_property_colon_ternary_missing(self): + command = Interpolate("echo %(prop:project:?|defined|missing)s") + d = self.build.render(command) + d.addCallback(self.failUnlessEqual, + "echo missing") + return d + + def test_property_colon_ternary_hash_true(self): + self.props.setProperty("project", "winbld", "test") + command = Interpolate("echo buildby-%(prop:project:#?:T:F)s") + d = self.build.render(command) + d.addCallback(self.failUnlessEqual, + "echo buildby-T") + return d + + def test_property_colon_ternary_hash_false(self): + self.props.setProperty("project", "", "test") + command = Interpolate("echo buildby-%(prop:project:#?|T|F)s") + d = self.build.render(command) + d.addCallback(self.failUnlessEqual, + "echo buildby-F") + return d + + def test_property_colon_ternary_substitute_recursively_true(self): + self.props.setProperty("P", "present", "test") + self.props.setProperty("one", "proj1", "test") + self.props.setProperty("two", "proj2", "test") + command = Interpolate("echo '%(prop:P:?|%(prop:one)s|%(prop:two)s)s'") + d = self.build.render(command) + d.addCallback(self.failUnlessEqual, + "echo 'proj1'") + return d + + def test_property_colon_ternary_substitute_recursively_false(self): + self.props.setProperty("one", "proj1", "test") + self.props.setProperty("two", "proj2", "test") + command = Interpolate("echo '%(prop:P:?|%(prop:one)s|%(prop:two)s)s'") + d = self.build.render(command) + d.addCallback(self.failUnlessEqual, + "echo 'proj2'") + return d + class TestInterpolateSrc(unittest.TestCase): def setUp(self): self.props = Properties() @@ -716,6 +920,15 @@ def testDictColonPlus(self): "build-exists-.tar.gz") return d + def testDictColonTernary(self): + # test dict-style substitution with WithProperties + self.props.setProperty("prop1", "foo", "test") + command = WithProperties("build-%(prop1:?:exists:missing)s-%(prop2:?:exists:missing)s.tar.gz") + d = self.build.render(command) + d.addCallback(self.failUnlessEqual, + "build-exists-missing.tar.gz") + return d + def testEmpty(self): # None should render as '' self.props.setProperty("empty", None, "test") From 805ef880f17943f1570ad0aa5bc1ab5a508f3357 Mon Sep 17 00:00:00 2001 From: Jared Grubb Date: Sat, 12 May 2012 12:35:49 -0700 Subject: [PATCH 059/136] ternary-sub: add docs --- master/docs/manual/cfg-properties.rst | 11 +++++++++++ master/docs/release-notes.rst | 3 +++ 2 files changed, 14 insertions(+) diff --git a/master/docs/manual/cfg-properties.rst b/master/docs/manual/cfg-properties.rst index 4c97f23316a..09520aaba49 100644 --- a/master/docs/manual/cfg-properties.rst +++ b/master/docs/manual/cfg-properties.rst @@ -248,6 +248,17 @@ syntaxes in the parentheses. If ``propname`` exists, substitute ``replacement``; otherwise, substitute an empty string. +``propname:?:sub_if_true:sub_if_false`` + +``propname:#?:sub_if_exists:sub_if_missing`` + Ternary substitution, depending on either ``propname`` being + ``True`` (like ``:~``) or being present (like ``:+``). Notice that + there is a colon immediately following the question mark *and* between + the two substitution alternatives. The character that follows the question + mark is used as the delimeter between the two alternatives. In the above + examples, it is a colon, but any single character can be used. + + Although these are similar to shell substitutions, no other substitutions are currently supported, and ``replacement`` in the above cannot contain more substitutions. diff --git a/master/docs/release-notes.rst b/master/docs/release-notes.rst index ba375eb8c83..4dce7abdcf6 100644 --- a/master/docs/release-notes.rst +++ b/master/docs/release-notes.rst @@ -89,6 +89,9 @@ Features * ``Git`` has a new ``getDescription`` option, which will run `git describe` after checkout +* A new ternary substitution operator ``:?:`` and ``:#?:`` to use with the ``Interpolate`` + and ``WithProperties`` classes. + Slave ----- From bbfc8d57d6f490f281c9fe5040a4e278b53ee696 Mon Sep 17 00:00:00 2001 From: "Dustin J. Mitchell" Date: Sat, 12 May 2012 17:27:01 -0500 Subject: [PATCH 060/136] ignore detritus from 'python setup.py test' --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index f439f079354..8e9dffb034c 100644 --- a/.gitignore +++ b/.gitignore @@ -40,3 +40,4 @@ common/googlecode_upload.py master/docs/tutorial/_build _build .tox +master/setuptools_trial* From f518b0bfeec17f946d7b728bd5adc2a263d66250 Mon Sep 17 00:00:00 2001 From: "Dustin J. Mitchell" Date: Sat, 12 May 2012 17:27:15 -0500 Subject: [PATCH 061/136] Apply some editing to the concepts documentation --- master/docs/manual/concepts.rst | 332 +++++++++++--------------------- master/docs/release-notes.rst | 2 +- 2 files changed, 116 insertions(+), 218 deletions(-) diff --git a/master/docs/manual/concepts.rst b/master/docs/manual/concepts.rst index 159f922a466..f39340d1cc3 100644 --- a/master/docs/manual/concepts.rst +++ b/master/docs/manual/concepts.rst @@ -5,186 +5,61 @@ This chapter defines some of the basic concepts that the Buildbot uses. You'll need to understand how the Buildbot sees the world to configure it properly. -.. _Version-Control-Systems: +.. index: repository +.. index: codebase +.. index: project +.. index: revision +.. index: branch +.. index: source stamp -Version Control Systems ------------------------ +.. _Source-Stamps: -These source trees come from a Version Control System of some kind. -CVS and Subversion are two popular ones, but the Buildbot supports -others. All VC systems have some notion of an upstream -`repository` which acts as a server [#]_, from which clients -can obtain source trees according to various parameters. The VC -repository provides source trees of various projects, for different -branches, and from various points in time. The first thing we have to -do is to specify which source tree we want to get. +Source Stamps +------------- -.. _Generalizing-VC-Systems: +Source code comes from *respositories*, provided by version control systems. +Repositories are generally identified by URLs, e.g., ``git://github.com/buildbot/buildbot.git``. -Generalizing VC Systems -~~~~~~~~~~~~~~~~~~~~~~~ +In these days of distribtued version control systems, the same *codebase* may appear in mutiple repositories. +For example, ``https://github.com/mozilla/mozilla-central`` and ``http://hg.mozilla.org/mozilla-release`` both contain the Firefox codebase, although not exactly the same code. -For the purposes of the Buildbot, we will try to generalize all VC -systems as having repositories that each provide sources for a variety -of projects. Each project is defined as a directory tree with source -files. The individual files may each have revisions, but we ignore -that and treat the project as a whole as having a set of revisions -(CVS is the only VC system still in widespread use that has -per-file revisions, as everything modern has moved to atomic tree-wide -changesets). Each time someone commits a change to the project, a new -revision becomes available. These revisions can be described by a -tuple with two items: the first is a branch tag, and the second is -some kind of revision stamp or timestamp. Complex projects may have -multiple branch tags, but there is always a default branch. The -timestamp may be an actual timestamp (such as the :option:`-D` option to CVS), -or it may be a monotonically-increasing transaction number (such as -the change number used by SVN and P4, or the revision number used by -Bazaar, or a labeled tag used in CVS. [#]_) -The SHA1 revision ID used by Mercurial, and Git is -also a kind of revision stamp, in that it specifies a unique copy of -the source tree, as does a Darcs ``context`` file. - -When we aren't intending to make any changes to the sources we check out -(at least not any that need to be committed back upstream), there are two -basic ways to use a VC system: - - * Retrieve a specific set of source revisions: some tag or key is used - to index this set, which is fixed and cannot be changed by subsequent - developers committing new changes to the tree. Releases are built from - tagged revisions like this, so that they can be rebuilt again later - (probably with controlled modifications). - - * Retrieve the latest sources along a specific branch: some tag is used - to indicate which branch is to be used, but within that constraint we want - to get the latest revisions. - -Build personnel or CM staff typically use the first approach: the -build that results is (ideally) completely specified by the two -parameters given to the VC system: repository and revision tag. This -gives QA and end-users something concrete to point at when reporting -bugs. Release engineers are also reportedly fond of shipping code that -can be traced back to a concise revision tag of some sort. - -Developers are more likely to use the second approach: each morning -the developer does an update to pull in the changes committed by the -team over the last day. These builds are not easy to fully specify: it -depends upon exactly when you did a checkout, and upon what local -changes the developer has in their tree. Developers do not normally -tag each build they produce, because there is usually significant -overhead involved in creating these tags. Recreating the trees used by -one of these builds can be a challenge. Some VC systems may provide -implicit tags (like a revision number), while others may allow the use -of timestamps to mean "the state of the tree at time X" as opposed -to a tree-state that has been explicitly marked. - -The Buildbot is designed to help developers, so it usually works in -terms of *the latest* sources as opposed to specific tagged -revisions. However, it would really prefer to build from reproducible -source trees, so implicit revisions are used whenever possible. - -.. _Source-Tree-Specifications: - -Source Tree Specifications -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -So for the Buildbot's purposes we treat each VC system as a server -which can take a list of specifications as input and produce a source -tree as output. Some of these specifications are static: they are -attributes of the builder and do not change over time. Others are more -variable: each build will have a different value. The repository is -changed over time by a sequence of Changes, each of which represents a -single developer making changes to some set of files. These Changes -are cumulative. - -For normal builds, the Buildbot wants to get well-defined source trees -that contain specific :class:`Change`\s, and exclude other :class:`Change`\s that may have -occurred after the desired ones. We assume that the :class:`Change`\s arrive at -the buildbot (through one of the mechanisms described in -:ref:`Change-Sources`) in the same order in which they are committed to the -repository. The Buildbot waits for the tree to become ``stable`` -before initiating a build, for two reasons. The first is that -developers frequently make multiple related commits in quick -succession, even when the VC system provides ways to make atomic -transactions involving multiple files at the same time. Running a -build in the middle of these sets of changes would use an inconsistent -set of source files, and is likely to fail (and is certain to be less -useful than a build which uses the full set of changes). The -tree-stable-timer is intended to avoid these useless builds that -include some of the developer's changes but not all. The second reason -is that some VC systems (i.e. CVS) do not provide repository-wide -transaction numbers, so that timestamps are the only way to refer to -a specific repository state. These timestamps may be somewhat -ambiguous, due to processing and notification delays. By waiting until -the tree has been stable for, say, 10 minutes, we can choose a -timestamp from the middle of that period to use for our source -checkout, and then be reasonably sure that any clock-skew errors will -not cause the build to be performed on an inconsistent set of source -files. - -The :class:`Scheduler`\s always use the tree-stable-timer, with a timeout that -is configured to reflect a reasonable tradeoff between build latency -and change frequency. When the VC system provides coherent -repository-wide revision markers (such as Subversion's revision -numbers, or in fact anything other than CVS's timestamps), the -resulting :class:`Build` is simply performed against a source tree defined by -that revision marker. When the VC system does not provide this, a -timestamp from the middle of the tree-stable period is used to -generate the source tree [#]_. - -.. _Multiple-source-trees: - -Multiple source-trees -+++++++++++++++++++++ - -What if an end-product is composed of several components and all these components -have different repositories? Changes may arrive from different repositories within -the tree-stable-timer period. Buildbot will not only use the source-trees that -contain changes but also needs the remaining source-trees to build the complete -product. - -For this reason a :ref:`Scheduler` can be configured to base a build on a set -of several source-trees that can (partly) be overidden by the specific revisions -inside the arrived :class:`Change`\s. - - -A single source-tree is identified by its repository, branch and revision. The -repository is a reference to a place where the sources reside. In projects it is -possible to have the same sources at different places (for example a collection -of fork of the same repository). For this reason an extra identifier 'codebase' -has been introduced to make buildbot categorize all copies of the same source-tree -as the same. Furthermore, the codebase_ allow you to uniquely identify a *part* of -your project. - -A complete multiple repository configuration consists of: +Many *projects* are built from multiple codebases. +For example, a company may build several applications based on the same core library. +The "app" codebase and the "core" codebase are in separate repositories, but are compiled together and constitute a single project. +Changes to either codebase should cause a rebuild of the application. - - a *codebase generator* +Most version control systems define some sort of *revision* that can be used (sometimes in combination with a *branch*) to uniquely specify a particular version of the source code. - Every relevant change arriving from a VC must contain a codebase. - Because VC's do not supply a codebase this has to be done by a so - called :bb:cfg:`codebaseGenerator` that is defined in the - configuration. +To build a project, Buildbot needs to know exactly which version of each codebase it should build. +It uses a *source stamp* to do so for each codebase; the collection of sourcestamps required for a project is called a *source stamp set*. - - some *schedulers* +.. index: change + +.. _Version-Control-Systems: - The :bb:cfg:`scheduler` has to be configured with a set of all - required ``codebases`` to build a product. These codebases indicate the set of - required source-trees. In order for the scheduler to be able to produce a - complete set for each build, each given codebase will be supplied a default - repository, branch, and revision. Those ones will be used for the codebases - where no changes arrived. +Version Control Systems +----------------------- - - multiple *source steps* +Buildbot supports a significant number of version control systems, so it treats them abstractly. - A :ref:`Builder` has to contains a :ref:`source step` for - each needed source-tree. Each of the source steps has a ``codebase`` attribute - which is used to retrieve the corresponding repository, branch and revision - data. This information comes from the arrived changes or from the - scheduler's configured default values. +For purposes of deciding when to perform builds, Buildbot's change sources monitor repositories, and represent any updates to those repositories as *changes*. +These change sources fall broadly into two categories: pollers which periodically check the repository for updates; and hooks, where the repository is configured to notify Buildbot whenever an update occurs. -.. warning:: +This concept does not map perfectly to every version control system. +For example, for CVS Buildbot must guess that version updates made to multiple files within a short time represent a single change; for DVCS's like Git, Buildbot records a change when a commit is pushed to the monitored repository, not when it is initially committed. +We assume that the :class:`Change`\s arrive at the master in the same order in which they are committed to the repository. - Defining a :bb:cfg:`codebaseGenerator` that returns non empty (not - ``''``) codebases, change the behavior of all the schedulers. +When it comes time to actually perform a build, a scheduler prepares a source stamp set, as described above, based on its configuration. +When the build begins, one or more source steps use the information in the source stamp set to actually check out the source code, using the normal VCS commands. + +Tree Stability +~~~~~~~~~~~~~~ + +Changes tend to arrive at a buildmaster in bursts. +In many cases, these bursts of changes are meant to be taken together. +For example, a developer may have pushed multiple commits to a DVCS that comprise the same new feature or bugfix. +To avoid trying to build every change, Buildbot supports the notion of *tree stability*, by waiting for a burst of changes to finish before starting to schedule builds. +This is implemented as a timer, with builds not scheduled until no changes have occurred for the duration of the timer. .. _How-Different-VC-Systems-Specify-Sources: @@ -291,15 +166,17 @@ SHA1 hash as returned by e.g. ``mtn automate select w:``. No attempt is made to ensure that the specified revision is actually a subset of the specified branch. +.. index: change + .. _Attributes-of-Changes: -Attributes of Changes -~~~~~~~~~~~~~~~~~~~~~ +Changes +------- .. _Attr-Who: Who -+++ +~~~ Each :class:`Change` has a :attr:`who` attribute, which specifies which developer is responsible for the change. This is a string which comes from a namespace @@ -316,7 +193,7 @@ incoming Changes will have their ``who`` parsed and stored. .. _Attr-Files: Files -+++++ +~~~~~ It also has a list of :attr:`files`, which are just the tree-relative filenames of any files that were added, deleted, or modified for this @@ -340,47 +217,42 @@ full test suite. .. _Attr-Comments: Comments -++++++++ +~~~~~~~~ -The Change also has a :attr:`comments` attribute, which is a string -containing any checkin comments. +The Change also has a :attr:`comments` attribute, which is a string containing any checkin comments. .. _Attr-Project: Project -+++++++ +~~~~~~~ -The :attr:`project` attribute of a change or source stamp describes the project -to which it corresponds, as a short human-readable string. This is useful in -cases where multiple independent projects are built on the same buildmaster. -In such cases, it can be used to control which builds are scheduled for a given -commit, and to limit status displays to only one project. +The :attr:`project` attribute of a change or source stamp describes the project to which it corresponds, as a short human-readable string. +This is useful in cases where multiple independent projects are built on the same buildmaster. +In such cases, it can be used to control which builds are scheduled for a given commit, and to limit status displays to only one project. .. _Attr-Repository: Repository -++++++++++ - -A change occurs within the context of a specific repository. This is a string, -and for most version-control systems, it takes the form of a URL. It uniquely -identifies the repository in which the change occurred. This is particularly -helpful for DVCS's, where a change may occur in a repository other than the -"main" repository for the project. +~~~~~~~~~~ -:class:`Change`\s can be filtered on repository, but more often this field is used as a -hint for the build steps to figure out which code to check out. +This attibute specifies the repository in which this change occurred. +In the case of DVCS's, this information may be required to check out the committed source code. +However, using the repository from a change has security risks: if Buildbot is configured to blidly trust this information, then it may easily be tricked into building arbitrary source code, potentially compromising the buildslaves and the integrity of subsequent builds. .. _Attr-Codebase: Codebase -++++++++ +~~~~~~~~ -The codebase is derived from a change. A complete software product may be composed of more than one repository. Each repository has its own unique position inside the product design (i.e. main module, shared library, resources, documentation). To be able to start builds from different VCS's and still distinquish the different repositories `codebase`'s are used. By default the codebase is ''. The `master.cfg` may contain a callable that determines the codebase from an incomming change and replaces the default value(see. :bb:cfg:`codebaseGenerator`). A codebase is not allowed to contain ':'. +This attribute specifies the codebase to which this change was made. +As described :ref:`above `, multiple repositories may contain the same codebase. +A change's codebase is usually determined by the bb:cfg:`codebaseGenerator` configuration. +By default the codebase is ''; this value is used automatically for single-codebase configurations. .. _Attr-Revision: Revision -++++++++ +~~~~~~~~ Each Change can have a :attr:`revision` attribute, which describes how to get a tree with a specific state: a tree which includes this Change @@ -411,7 +283,7 @@ Revisions are always strings. Branches -++++++++ +~~~~~~~~ The Change might also have a :attr:`branch` attribute. This indicates that all of the Change's files are in the same named branch. The @@ -445,8 +317,8 @@ same as Darcs. `Monotone` branch='warner-newfeature', files=['src/foo.c'] -Build Properties -++++++++++++++++ +Change Properties +~~~~~~~~~~~~~~~~~ A Change may have one or more properties attached to it, usually specified through the Force Build form or :bb:cmdline:`sendchange`. Properties are discussed @@ -509,8 +381,8 @@ individual :class:`BuildRequests` are delivered to the target .. _BuildSet: -BuildSet --------- +BuildSets +--------- A :class:`BuildSet` is the name given to a set of :class:`Build`\s that all compile/test the same version of the tree on multiple :class:`Builder`\s. In @@ -568,8 +440,8 @@ appropriate :class:`Builder`\s. .. _BuildRequest: -BuildRequest ------------- +BuildRequests +------------- A :class:`BuildRequest` is a request to build a specific set of source code (specified by one ore more source stamps) on a single :class:`Builder`. @@ -593,8 +465,8 @@ A merge of buildrequests is performed per codebase, thus on changes having the s .. _Builder: -Builder -------- +Builders +-------- The Buildmaster runs a collection of :class:`Builder`\s, each of which handles a single type of build (e.g. full versus quick), on one or more build slaves. :class:`Builder`\s @@ -617,7 +489,7 @@ checkout/compile/test commands are executed). .. _Concepts-Build-Factories: Build Factories -~~~~~~~~~~~~~~~ +--------------- A builder also has a :class:`BuildFactory`, which is responsible for creating new :class:`Build` instances: because the :class:`Build` instance is what actually performs each build, @@ -627,7 +499,7 @@ is done (:ref:`Concepts-Build`). .. _Concepts-Build-Slaves: Build Slaves -~~~~~~~~~~~~ +------------ Each builder is associated with one of more :class:`BuildSlave`\s. A builder which is used to perform Mac OS X builds (as opposed to Linux or Solaris builds) should @@ -650,8 +522,8 @@ all these things mean you should use separate Builders. .. _Concepts-Build: -Build ------ +Builds +------ A build is a single compile or test run of a particular version of the source code, and is comprised of a series of steps. It is ultimately up to you what @@ -900,16 +772,42 @@ Rather than create a build factory for each slave, the steps can use buildslave properties to identify the unique aspects of each slave and adapt the build process dynamically. -.. rubric:: Footnotes +.. _Multiple-Codebase-Builds: + +Multiple-Codebase Builds +------------------------ + +What if an end-product is composed of code from several codebases? +Changes may arrive from different repositories within the tree-stable-timer period. +Buildbot will not only use the source-trees that contain changes but also needs the remaining source-trees to build the complete product. + +For this reason a :ref:`Scheduler` can be configured to base a build on a set of several source-trees that can (partly) be overidden by the information from incoming :class:`Change`\s. + +As descibed :ref:`above `, the source for each codebase is identified by a source stamp, containing its repository, branch and revision. +A full build set will specify a source stamp set describing the source to use for each codebase. + +Configuring all of this takes a coordinated approach. A complete multiple repository configuration consists of: + + - a *codebase generator* + + Every relevant change arriving from a VC must contain a codebase. + This is done by a :bb:cfg:`codebaseGenerator` that is defined in the configuration. + Most generators examine the repository of a change to determine its codebase, using project-specific rules. + + - some *schedulers* + + Each :bb:cfg:`scheduler` has to be configured with a set of all required ``codebases`` to build a product. + These codebases indicate the set of required source-trees. + In order for the scheduler to be able to produce a complete set for each build, the configuration can give a default repository, branch, and revision for each codebase. + When a scheduler must generate a source stamp for a codebase that has received no changes, it applies these default values. + + - multiple *source steps* - one for each codebase + + A :ref:`Builder`'s build factory must include a :ref:`source step` for each codebase. + Each of the source steps has a ``codebase`` attribute which is used to select an appropriate source stamp from the source stamp set for a build. + This information comes from the arrived changes or from the scheduler's configured default values. + +.. warning:: + + Defining a :bb:cfg:`codebaseGenerator` that returns non-empty (not ``''``) codebases will change the behavior of all the schedulers. -.. [#] Except Darcs, but since the Buildbot never modifies its local source tree we can ignore - the fact that Darcs uses a less centralized model - -.. [#] Many VC systems provide more complexity than this: in particular the local - views that P4 and ClearCase can assemble out of various source - directories are more complex than we're prepared to take advantage of - here - -.. [#] This ``checkoutDelay`` defaults - to half the tree-stable timer, but it can be overridden with an - argument to the :class:`Source` Step diff --git a/master/docs/release-notes.rst b/master/docs/release-notes.rst index 86de70cd32b..7ebaa399263 100644 --- a/master/docs/release-notes.rst +++ b/master/docs/release-notes.rst @@ -81,7 +81,7 @@ Features schedulers can aggregate changes to multiple codebases into source stamp sets (with one source stamp for each codebase). Source steps then check out each codebase as required, and the remainder of the build process proceeds - normally. See the :ref:`Multiple-source-trees` for details. + normally. See the :ref:`Multiple-Codebase-Builds` for details. * ``Source`` and ``ShellCommand`` steps now have an optional ``descriptionSuffix``, a suffix to the ``description``/``descriptionDone`` values. For example this can help distinguish between From dabf0587f8b34d6fd3bd9042c19a58280441cb63 Mon Sep 17 00:00:00 2001 From: "Dustin J. Mitchell" Date: Sat, 12 May 2012 17:58:56 -0500 Subject: [PATCH 062/136] clarify docs as to how :#? and :? differ --- master/docs/manual/cfg-properties.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/master/docs/manual/cfg-properties.rst b/master/docs/manual/cfg-properties.rst index 09520aaba49..1219bf52476 100644 --- a/master/docs/manual/cfg-properties.rst +++ b/master/docs/manual/cfg-properties.rst @@ -251,12 +251,12 @@ syntaxes in the parentheses. ``propname:?:sub_if_true:sub_if_false`` ``propname:#?:sub_if_exists:sub_if_missing`` - Ternary substitution, depending on either ``propname`` being - ``True`` (like ``:~``) or being present (like ``:+``). Notice that - there is a colon immediately following the question mark *and* between - the two substitution alternatives. The character that follows the question - mark is used as the delimeter between the two alternatives. In the above - examples, it is a colon, but any single character can be used. + Ternary substitution, depending on either ``propname`` being ``True`` (with + ``:?``, similar to ``:~``) or being present (with ``:#?``, like ``:+``). + Notice that there is a colon immediately following the question mark *and* + between the two substitution alternatives. The character that follows the + question mark is used as the delimeter between the two alternatives. In the + above examples, it is a colon, but any single character can be used. Although these are similar to shell substitutions, no other From e8656b70bf99219a45569341bc86e770dace6075 Mon Sep 17 00:00:00 2001 From: "Dustin J. Mitchell" Date: Sat, 12 May 2012 18:26:55 -0500 Subject: [PATCH 063/136] remove duplicate tests --- .../test/unit/test_process_properties.py | 57 ------------------- 1 file changed, 57 deletions(-) diff --git a/master/buildbot/test/unit/test_process_properties.py b/master/buildbot/test/unit/test_process_properties.py index 8238e7741e3..a9faae2af4e 100644 --- a/master/buildbot/test/unit/test_process_properties.py +++ b/master/buildbot/test/unit/test_process_properties.py @@ -563,63 +563,6 @@ def test_property_colon_ternary_substitute_recursively_false(self): "echo 'proj2'") return d - def test_property_substitute_recursively(self): - self.props.setProperty("project", "proj1", "test") - command = Interpolate("echo '%(prop:no_such:-%(prop:project)s)s'") - d = self.build.render(command) - d.addCallback(self.failUnlessEqual, - "echo 'proj1'") - return d - - def test_property_colon_ternary_present(self): - self.props.setProperty("project", "proj1", "test") - command = Interpolate("echo %(prop:project:?:defined:missing)s") - d = self.build.render(command) - d.addCallback(self.failUnlessEqual, - "echo defined") - return d - - def test_property_colon_ternary_missing(self): - command = Interpolate("echo %(prop:project:?|defined|missing)s") - d = self.build.render(command) - d.addCallback(self.failUnlessEqual, - "echo missing") - return d - - def test_property_colon_ternary_hash_true(self): - self.props.setProperty("project", "winbld", "test") - command = Interpolate("echo buildby-%(prop:project:#?:T:F)s") - d = self.build.render(command) - d.addCallback(self.failUnlessEqual, - "echo buildby-T") - return d - - def test_property_colon_ternary_hash_false(self): - self.props.setProperty("project", "", "test") - command = Interpolate("echo buildby-%(prop:project:#?|T|F)s") - d = self.build.render(command) - d.addCallback(self.failUnlessEqual, - "echo buildby-F") - return d - - def test_property_colon_ternary_substitute_recursively_true(self): - self.props.setProperty("P", "present", "test") - self.props.setProperty("one", "proj1", "test") - self.props.setProperty("two", "proj2", "test") - command = Interpolate("echo '%(prop:P:?|%(prop:one)s|%(prop:two)s)s'") - d = self.build.render(command) - d.addCallback(self.failUnlessEqual, - "echo 'proj1'") - return d - - def test_property_colon_ternary_substitute_recursively_false(self): - self.props.setProperty("one", "proj1", "test") - self.props.setProperty("two", "proj2", "test") - command = Interpolate("echo '%(prop:P:?|%(prop:one)s|%(prop:two)s)s'") - d = self.build.render(command) - d.addCallback(self.failUnlessEqual, - "echo 'proj2'") - return d class TestInterpolateSrc(unittest.TestCase): def setUp(self): From 56adff35ecc758772aa6880be524fd794077c16f Mon Sep 17 00:00:00 2001 From: "Dustin J. Mitchell" Date: Sat, 12 May 2012 18:45:01 -0500 Subject: [PATCH 064/136] update docs regarding empty repository string - refs #913. --- master/docs/manual/cfg-global.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/master/docs/manual/cfg-global.rst b/master/docs/manual/cfg-global.rst index 5a05ccbcc9c..ab2d2b3c261 100644 --- a/master/docs/manual/cfg-global.rst +++ b/master/docs/manual/cfg-global.rst @@ -704,9 +704,9 @@ an URL to the revision. Note that the revision id may not always be in the form you expect, so code defensively. In particular, a revision of "??" may be supplied when no other information is available. -Note that :class:`SourceStamp`\s that are not created from version-control changes (e.g., -those created by a Nightly or Periodic scheduler) will have an empty repository -string, as the respository is not known. +Note that :class:`SourceStamp`\s that are not created from version-control +changes (e.g., those created by a Nightly or Periodic scheduler) may have an +empty repository string, if the respository is not known to the scheduler. Revision Link Helpers +++++++++++++++++++++ From 56523b37eb51f9c5adedcc8af54b001f7bde79d0 Mon Sep 17 00:00:00 2001 From: Tom Prince Date: Mon, 14 May 2012 18:49:37 -0600 Subject: [PATCH 065/136] Fix test handling exception from BuildStep.start. --- master/buildbot/test/unit/test_process_buildstep.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/master/buildbot/test/unit/test_process_buildstep.py b/master/buildbot/test/unit/test_process_buildstep.py index b5fe86f3308..a7e66464dea 100644 --- a/master/buildbot/test/unit/test_process_buildstep.py +++ b/master/buildbot/test/unit/test_process_buildstep.py @@ -223,6 +223,10 @@ def test_step_raining_buildstepfailed_in_start(self): def test_step_raising_exception_in_start(self): self.setupStep(FailingCustomStep(exception=ValueError)) - self.expectOutcome(result=FAILURE, status_text=["generic"]) - return self.runStep() + self.expectOutcome(result=EXCEPTION, status_text=["generic", "exception"]) + d = self.runStep() + @d.addCallback + def cb(_): + self.assertEqual(len(self.flushLoggedErrors(ValueError)), 1) + return d From 5d202962b2e01e3bf07e9a6bc85f51a9995e426c Mon Sep 17 00:00:00 2001 From: Tom Prince Date: Wed, 16 May 2012 10:26:13 -0600 Subject: [PATCH 066/136] Fix upgrading old builds? --- master/buildbot/status/logfile.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/master/buildbot/status/logfile.py b/master/buildbot/status/logfile.py index b8a8294c300..5f3c69bd77f 100644 --- a/master/buildbot/status/logfile.py +++ b/master/buildbot/status/logfile.py @@ -680,6 +680,8 @@ def finish(self): def __getstate__(self): d = self.__dict__.copy() del d['step'] + if d.has_key('master'): + del d['master'] return d From 5d61618979fbdfcd3e87c37c246a82d0a111db16 Mon Sep 17 00:00:00 2001 From: "Dustin J. Mitchell" Date: Wed, 16 May 2012 23:08:40 -0500 Subject: [PATCH 067/136] Prune changes in batches of 100. Fixes #2299. --- master/buildbot/db/changes.py | 9 ++++--- master/buildbot/test/unit/test_db_changes.py | 27 ++++++++++++++++++++ 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/master/buildbot/db/changes.py b/master/buildbot/db/changes.py index 236b140c1ed..4288be532cc 100644 --- a/master/buildbot/db/changes.py +++ b/master/buildbot/db/changes.py @@ -196,9 +196,12 @@ def thd(conn): for table_name in ('scheduler_changes', 'sourcestamp_changes', 'change_files', 'change_properties', 'changes', 'change_users'): - table = self.db.model.metadata.tables[table_name] - conn.execute( - table.delete(table.c.changeid.in_(ids_to_delete))) + remaining = ids_to_delete[:] + while remaining: + batch, remaining = remaining[:100], remaining[100:] + table = self.db.model.metadata.tables[table_name] + conn.execute( + table.delete(table.c.changeid.in_(batch))) return self.db.pool.do(thd) def _chdict_from_change_row_thd(self, conn, ch_row): diff --git a/master/buildbot/test/unit/test_db_changes.py b/master/buildbot/test/unit/test_db_changes.py index 4ab8472c8d9..227c3842cf4 100644 --- a/master/buildbot/test/unit/test_db_changes.py +++ b/master/buildbot/test/unit/test_db_changes.py @@ -438,6 +438,33 @@ def thd(conn): d.addCallback(check) return d + def test_pruneChanges_lots(self): + d = self.insertTestData([ + fakedb.Change(changeid=n) + for n in xrange(150) + ]) + + d.addCallback(lambda _ : self.db.changes.pruneChanges(1)) + def check(_): + def thd(conn): + results = {} + for tbl_name in ('scheduler_changes', 'sourcestamp_changes', + 'change_files', 'change_properties', + 'changes'): + tbl = self.db.model.metadata.tables[tbl_name] + r = conn.execute(sa.select([tbl.c.changeid])) + results[tbl_name] = len([ r for r in r.fetchall() ]) + self.assertEqual(results, { + 'scheduler_changes': 0, + 'sourcestamp_changes': 0, + 'change_files': 0, + 'change_properties': 0, + 'changes': 1, + }) + return self.db.pool.do(thd) + d.addCallback(check) + return d + def test_pruneChanges_None(self): d = self.insertTestData(self.change13_rows) From 1c515be63e78cf87bb1ccab6738252893dbd3810 Mon Sep 17 00:00:00 2001 From: "Dustin J. Mitchell" Date: Wed, 16 May 2012 23:28:41 -0500 Subject: [PATCH 068/136] don't try to insert an id=0, as mysql will autoincrement it --- master/buildbot/test/unit/test_db_changes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/master/buildbot/test/unit/test_db_changes.py b/master/buildbot/test/unit/test_db_changes.py index 227c3842cf4..f7427875814 100644 --- a/master/buildbot/test/unit/test_db_changes.py +++ b/master/buildbot/test/unit/test_db_changes.py @@ -441,7 +441,7 @@ def thd(conn): def test_pruneChanges_lots(self): d = self.insertTestData([ fakedb.Change(changeid=n) - for n in xrange(150) + for n in xrange(1, 151) ]) d.addCallback(lambda _ : self.db.changes.pruneChanges(1)) From 1f124cd81de74c1e3adc5e34085c349dea65141e Mon Sep 17 00:00:00 2001 From: "Dustin J. Mitchell" Date: Wed, 16 May 2012 23:38:32 -0500 Subject: [PATCH 069/136] remove NEWS from manifest --- master/MANIFEST.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/master/MANIFEST.in b/master/MANIFEST.in index d0cd257f8f6..6e30f3be76d 100644 --- a/master/MANIFEST.in +++ b/master/MANIFEST.in @@ -1,4 +1,4 @@ -include MANIFEST.in README NEWS CREDITS COPYING UPGRADING +include MANIFEST.in README CREDITS COPYING UPGRADING include docs/examples/*.cfg include docs/conf.py From c8079379bc130997e97ab7e4b96aebaae1bf3912 Mon Sep 17 00:00:00 2001 From: "Dustin J. Mitchell" Date: Thu, 17 May 2012 00:23:35 -0500 Subject: [PATCH 070/136] Decode form values from HTTP POSTs Fixes #1054. One hopes. --- master/buildbot/status/web/base.py | 11 ++++++++++ master/buildbot/status/web/build.py | 6 ++++- master/buildbot/status/web/builder.py | 21 +++++++++++++----- .../test/unit/test_status_web_base.py | 22 +++++++++++++++++++ 4 files changed, 53 insertions(+), 7 deletions(-) diff --git a/master/buildbot/status/web/base.py b/master/buildbot/status/web/base.py index 87050b3e161..522cf0a671d 100644 --- a/master/buildbot/status/web/base.py +++ b/master/buildbot/status/web/base.py @@ -790,3 +790,14 @@ class AlmostStrictUndefined(jinja2.StrictUndefined): fully as strict as StrictUndefined ''' def __nonzero__(self): return False + +_charsetRe = re.compile('charset=([^;]*)', re.I) +def getRequestCharset(req): + """Get the charset for an x-www-form-urlencoded request""" + # per http://stackoverflow.com/questions/708915/detecting-the-character-encoding-of-an-http-post-request + hdr = req.getHeader('Content-Type') + if hdr: + mo = _charsetRe.search(hdr) + if mo: + return mo.group(1).strip() + return 'utf-8' # reasonable guess, works for ascii diff --git a/master/buildbot/status/web/build.py b/master/buildbot/status/web/build.py index 1c97f864d96..13f43eba7cb 100644 --- a/master/buildbot/status/web/build.py +++ b/master/buildbot/status/web/build.py @@ -22,7 +22,8 @@ from twisted.python import log from buildbot.status.web.base import HtmlResource, \ css_classes, path_to_build, path_to_builder, path_to_slave, \ - getAndCheckProperties, ActionResource, path_to_authzfail + getAndCheckProperties, ActionResource, path_to_authzfail, \ + getRequestCharset from buildbot.schedulers.forcesched import ForceScheduler, TextParameter from buildbot.status.web.step import StepsResource from buildbot.status.web.tests import TestsResource @@ -53,6 +54,7 @@ def performAction(self, req): log.msg("web rebuild of build %s:%s" % (builder_name, b.getNumber())) name =authz.getUsernameFull(req) comments = req.args.get("comments", [""])[0] + comments.decode(getRequestCharset(req)) reason = ("The web-page 'rebuild' button was pressed by " "'%s': %s\n" % (name, comments)) msg = "" @@ -105,6 +107,7 @@ def performAction(self, req): (b.getBuilder().getName(), b.getNumber())) name = authz.getUsernameFull(req) comments = req.args.get("comments", [""])[0] + comments.decode(getRequestCharset(req)) # html-quote both the username and comments, just to be safe reason = ("The web-page 'stop build' button was pressed by " "'%s': %s\n" % (html.escape(name), html.escape(comments))) @@ -270,6 +273,7 @@ def stop(self, req, auth_ok=False): name = self.getAuthz(req).getUsernameFull(req) comments = req.args.get("comments", [""])[0] + comments.decode(getRequestCharset(req)) # html-quote both the username and comments, just to be safe reason = ("The web-page 'stop build' button was pressed by " "'%s': %s\n" % (html.escape(name), html.escape(comments))) diff --git a/master/buildbot/status/web/builder.py b/master/buildbot/status/web/builder.py index 5a9955c6a9e..21b0278c21e 100644 --- a/master/buildbot/status/web/builder.py +++ b/master/buildbot/status/web/builder.py @@ -22,7 +22,8 @@ from buildbot.status.web.base import HtmlResource, BuildLineMixin, \ path_to_build, path_to_slave, path_to_builder, path_to_change, \ path_to_root, ICurrentBox, build_get_class, \ - map_branches, path_to_authzfail, ActionResource + map_branches, path_to_authzfail, ActionResource, \ + getRequestCharset from buildbot.schedulers.forcesched import ForceScheduler from buildbot.schedulers.forcesched import InheritBuildParameter from buildbot.schedulers.forcesched import ValidationError @@ -141,11 +142,16 @@ def performAction(self, req): "forcescheduler arg not found")) return - args = {} + args = req.args.copy() + + # decode all of the args + encoding = getRequestCharset(req) + for name, argl in args.iteritems(): + args[name] = [ arg.decode(encoding) for arg in argl ] + # damn html's ungeneric checkbox implementation... - for cb in req.args.get("checkbox", []): + for cb in args.get("checkbox", []): args[cb] = True - args.update(req.args) builder_name = self.builder_status.getName() @@ -183,7 +189,7 @@ def buildForceContext(cxt, req, master, buildername=None): else: # filter out unicode chars, and html stuff if type(default)==unicode: - default = html.escape(default.encode('ascii','ignore')) + default = html.escape(default.encode('utf-8','ignore')) default_props[pname] = default cxt['force_schedulers'] = force_schedulers cxt['default_props'] = default_props @@ -481,9 +487,12 @@ class BuildersResource(HtmlResource): @defer.inlineCallbacks def content(self, req, cxt): status = self.getStatus(req) + encoding = getRequestCharset(req) builders = req.args.get("builder", status.getBuilderNames()) - branches = [b for b in req.args.get("branch", []) if b] + branches = [ b.decode(encoding) + for b in req.args.get("branch", []) + if b ] # get counts of pending builds for each builder brstatus_ds = [] diff --git a/master/buildbot/test/unit/test_status_web_base.py b/master/buildbot/test/unit/test_status_web_base.py index c61b4261994..aa428e60245 100644 --- a/master/buildbot/test/unit/test_status_web_base.py +++ b/master/buildbot/test/unit/test_status_web_base.py @@ -13,6 +13,7 @@ # # Copyright Buildbot Team Members +import mock from buildbot.status.web import base from twisted.internet import defer from twisted.trial import unittest @@ -58,3 +59,24 @@ def check(f): d.addErrback(check) return d +class Functions(unittest.TestCase): + + def do_test_getRequestCharset(self, hdr, exp): + req = mock.Mock() + req.getHeader.return_value = hdr + + self.assertEqual(base.getRequestCharset(req), exp) + + def test_getRequestCharset_empty(self): + return self.do_test_getRequestCharset(None, 'utf-8') + + def test_getRequestCharset_specified(self): + return self.do_test_getRequestCharset( + 'application/x-www-form-urlencoded ; charset=ISO-8859-1', + 'ISO-8859-1') + + def test_getRequestCharset_other_params(self): + return self.do_test_getRequestCharset( + 'application/x-www-form-urlencoded ; charset=UTF-16 ; foo=bar', + 'UTF-16') + From 74ab736bd4a0d348af7358194787685ea57fab37 Mon Sep 17 00:00:00 2001 From: Tom Prince Date: Tue, 15 May 2012 17:09:01 -0600 Subject: [PATCH 071/136] Skip balanced parentheses in ternary Interpolations. Normal python % substitution skips over nested balanced parentheses. We should do the same thing when parsing Interpolate. --- master/buildbot/process/properties.py | 36 ++++++++++++------- .../test/unit/test_process_properties.py | 30 +++++++++++++--- 2 files changed, 50 insertions(+), 16 deletions(-) diff --git a/master/buildbot/process/properties.py b/master/buildbot/process/properties.py index a9cbdd553fc..c6c72cbff9b 100644 --- a/master/buildbot/process/properties.py +++ b/master/buildbot/process/properties.py @@ -456,6 +456,20 @@ def _parseSubstitution(self, fmt): else: return fn(arg) + @staticmethod + def _splitBalancedParen(delim, arg): + parenCount = 0 + for i in range(0, len(arg)): + if arg[i] == "(": + parenCount += 1 + if arg[i] == ")": + parenCount -= 1 + if parenCount < 0: + raise ValueError + if parenCount == 0 and arg[i] == delim: + return arg[0:i], arg[i+1:] + return arg + def _parseColon_minus(self, d, kw, repl): return _Lookup(d, kw, default=Interpolate(repl, **self.kwargs), @@ -475,21 +489,19 @@ def _parseColon_plus(self, d, kw, repl): defaultWhenFalse=False, elideNoneAs='') - colon_ternary_re = re.compile(r"""(?P.) # the delimiter - (?P.*) # sub-if-true - (?P=delim) # the delimiter again - (?P.*)# sub-if-false - """, re.VERBOSE) - def _parseColon_ternary(self, d, kw, repl, defaultWhenFalse=False): - m = self.colon_ternary_re.match(repl) - if not m: - config.error("invalid Interpolate ternary expression for selector '%s' and delim '%s'" % (kw, repl[0])) + delim = repl[0] + if delim == '(': + config.error("invalid Interpolate ternary delimiter '('") + return None + try: + truePart, falsePart = self._splitBalancedParen(delim, repl[1:]) + except ValueError: + config.error("invalid Interpolate ternary expression '%s' with delimiter '%s'" % (repl[1:], repl[0])) return None - m = m.groupdict() return _Lookup(d, kw, - hasKey=Interpolate(m['true'], **self.kwargs), - default=Interpolate(m['false'], **self.kwargs), + hasKey=Interpolate(truePart, **self.kwargs), + default=Interpolate(falsePart, **self.kwargs), defaultWhenFalse=defaultWhenFalse, elideNoneAs='') diff --git a/master/buildbot/test/unit/test_process_properties.py b/master/buildbot/test/unit/test_process_properties.py index a9faae2af4e..b08eb0d521a 100644 --- a/master/buildbot/test/unit/test_process_properties.py +++ b/master/buildbot/test/unit/test_process_properties.py @@ -382,13 +382,16 @@ def test_nested_invalid_selector(self): self.assertRaisesConfigError("invalid Interpolate selector 'garbage'", lambda: Interpolate("%(prop:some_prop:~%(garbage:test)s)s")) - - def test_colon_ternary_bad_delimeter(self): - self.assertRaisesConfigError("invalid Interpolate ternary expression for selector 'P' and delim ':'", + def test_colon_ternary_missing_delimeter(self): + self.assertRaisesConfigError("invalid Interpolate ternary expression 'one' with delimiter ':'", lambda: Interpolate("echo '%(prop:P:?:one)s'")) + def test_colon_ternary_paren_delimiter(self): + self.assertRaisesConfigError("invalid Interpolate ternary expression 'one(:)' with delimiter ':'", + lambda: Interpolate("echo '%(prop:P:?:one(:))s'")) + def test_colon_ternary_hash_bad_delimeter(self): - self.assertRaisesConfigError("invalid Interpolate ternary expression for selector 'P' and delim '|'", + self.assertRaisesConfigError("invalid Interpolate ternary expression 'one' with delimiter '|'", lambda: Interpolate("echo '%(prop:P:#?|one)s'")) @@ -563,6 +566,25 @@ def test_property_colon_ternary_substitute_recursively_false(self): "echo 'proj2'") return d + def test_property_colon_ternary_substitute_recursively_delimited_true(self): + self.props.setProperty("P", "present", "test") + self.props.setProperty("one", "proj1", "test") + self.props.setProperty("two", "proj2", "test") + command = Interpolate("echo '%(prop:P:?|%(prop:one:?|true|false)s|%(prop:two:?|false|true)s)s'") + d = self.build.render(command) + d.addCallback(self.failUnlessEqual, + "echo 'true'") + return d + + def test_property_colon_ternary_substitute_recursively_delimited_false(self): + self.props.setProperty("one", "proj1", "test") + self.props.setProperty("two", "proj2", "test") + command = Interpolate("echo '%(prop:P:?|%(prop:one:?|true|false)s|%(prop:two:?|false|true)s)s'") + d = self.build.render(command) + d.addCallback(self.failUnlessEqual, + "echo 'false'") + return d + class TestInterpolateSrc(unittest.TestCase): def setUp(self): From a878a0a7e51b8aeafd0fdd6410a8198c6d17a871 Mon Sep 17 00:00:00 2001 From: Tom Prince Date: Thu, 17 May 2012 11:22:19 -0600 Subject: [PATCH 072/136] Be strict about identifiers accepted by Interpolate. --- master/buildbot/process/properties.py | 14 ++++++++++++++ .../test/unit/test_process_properties.py | 16 ++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/master/buildbot/process/properties.py b/master/buildbot/process/properties.py index c6c72cbff9b..bfd8afe2601 100644 --- a/master/buildbot/process/properties.py +++ b/master/buildbot/process/properties.py @@ -402,6 +402,8 @@ class Interpolate(util.ComparableMixin): implements(IRenderable) compare_attrs = ('fmtstring', 'args', 'kwargs') + + identifier_re = re.compile('^[\w-]*$') def __init__(self, fmtstring, *args, **kwargs): self.fmtstring = fmtstring @@ -419,6 +421,9 @@ def _parse_prop(arg): prop, repl = arg.split(":", 1) except ValueError: prop, repl = arg, None + if not Interpolate.identifier_re.match(prop): + config.error("Property name must be alphanumeric for prop Interpolation '%s'" % arg) + prop = repl = None return _thePropertyDict, prop, repl @staticmethod @@ -433,6 +438,12 @@ def _parse_src(arg): except ValueError: config.error("Must specify both codebase and attribute for src Interpolation '%s'" % arg) codebase = attr = repl = None + if not Interpolate.identifier_re.match(codebase): + config.error("Codebase must be alphanumeric for src Interpolation '%s'" % arg) + codebase = attr = repl = None + if not Interpolate.identifier_re.match(attr): + config.error("Attribute must be alphanumeric for src Interpolation '%s'" % arg) + codebase = attr = repl = None return _SourceStampDict(codebase), attr, repl def _parse_kw(self, arg): @@ -440,6 +451,9 @@ def _parse_kw(self, arg): kw, repl = arg.split(":", 1) except ValueError: kw, repl = arg, None + if not Interpolate.identifier_re.match(kw): + config.error("Keyword must be alphanumeric for kw Interpolation '%s'" % arg) + kw = repl = None return _Lazy(self.kwargs), kw, repl def _parseSubstitution(self, fmt): diff --git a/master/buildbot/test/unit/test_process_properties.py b/master/buildbot/test/unit/test_process_properties.py index b08eb0d521a..632ea94b6fc 100644 --- a/master/buildbot/test/unit/test_process_properties.py +++ b/master/buildbot/test/unit/test_process_properties.py @@ -394,6 +394,22 @@ def test_colon_ternary_hash_bad_delimeter(self): self.assertRaisesConfigError("invalid Interpolate ternary expression 'one' with delimiter '|'", lambda: Interpolate("echo '%(prop:P:#?|one)s'")) + def test_prop_invalid_character(self): + self.assertRaisesConfigError("Property name must be alphanumeric for prop Interpolation 'a+a'", + lambda: Interpolate("echo '%(prop:a+a)s'")) + + def test_kw_invalid_character(self): + self.assertRaisesConfigError("Keyword must be alphanumeric for kw Interpolation 'a+a'", + lambda: Interpolate("echo '%(kw:a+a)s'")) + + def test_src_codebase_invalid_character(self): + self.assertRaisesConfigError("Codebase must be alphanumeric for src Interpolation 'a+a:a'", + lambda: Interpolate("echo '%(src:a+a:a)s'")) + + def test_src_attr_invalid_character(self): + self.assertRaisesConfigError("Attribute must be alphanumeric for src Interpolation 'a:a+a'", + lambda: Interpolate("echo '%(src:a:a+a)s'")) + class TestInterpolatePositional(unittest.TestCase): def setUp(self): From 34a3f6d9a6121b860d36f7c5b80f5bb96a251119 Mon Sep 17 00:00:00 2001 From: Tom Prince Date: Fri, 18 May 2012 09:19:12 -0600 Subject: [PATCH 073/136] Fix race condition during web status reconfig. Service.setServiceParent starts the current service, if the parent is already started. Thus, the current service needs to be ready to run when we call it. Fixes #2301. --- master/buildbot/status/web/baseweb.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/master/buildbot/status/web/baseweb.py b/master/buildbot/status/web/baseweb.py index 2d9fa42e60d..6f1989a1436 100644 --- a/master/buildbot/status/web/baseweb.py +++ b/master/buildbot/status/web/baseweb.py @@ -377,8 +377,6 @@ def __repr__(self): (self.http_port, self.distrib_port, hex(id(self)))) def setServiceParent(self, parent): - service.MultiService.setServiceParent(self, parent) - # this class keeps a *separate* link to the buildmaster, rather than # just using self.parent, so that when we are "disowned" (and thus # parent=None), any remaining HTTP clients of this WebStatus will still @@ -444,6 +442,8 @@ def _openLogFile(self, path): self.setupSite() + service.MultiService.setServiceParent(self, parent) + def setupSite(self): # this is responsible for creating the root resource. It isn't done # at __init__ time because we need to reference the parent's basedir. From 32459822263e23f1d4baeac77faa020614301a99 Mon Sep 17 00:00:00 2001 From: Tom Prince Date: Fri, 18 May 2012 14:18:56 -0400 Subject: [PATCH 074/136] Refactor Properties.update to use Properties.setProperty). --- master/buildbot/process/properties.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/master/buildbot/process/properties.py b/master/buildbot/process/properties.py index bfd8afe2601..08d09e562d2 100644 --- a/master/buildbot/process/properties.py +++ b/master/buildbot/process/properties.py @@ -94,9 +94,7 @@ def __repr__(self): def update(self, dict, source, runtime=False): """Update this object from a dictionary, with an explicit source specified.""" for k, v in dict.items(): - self.properties[k] = (v, source) - if runtime: - self.runtime.add(k) + self.setProperty(k, v, source, runtime=runtime) def updateFromProperties(self, other): """Update this object based on another object; the other object's """ From 834fb7b3af372846dbcd4b1245dee080ae2bb6a1 Mon Sep 17 00:00:00 2001 From: Tom Prince Date: Fri, 18 May 2012 14:31:16 -0400 Subject: [PATCH 075/136] Warn about non-jsonable properties. Once builds are stored in the database, properties that are not jsonable won't be supported. As it is, there are many places that don't like non-jsonable properties. Provide a warning for now, to allow people notice of the change. Refs #2265. --- master/buildbot/process/properties.py | 10 ++++++++ .../test/unit/test_process_properties.py | 24 ++++--------------- 2 files changed, 15 insertions(+), 19 deletions(-) diff --git a/master/buildbot/process/properties.py b/master/buildbot/process/properties.py index 08d09e562d2..b9d3a9c6a99 100644 --- a/master/buildbot/process/properties.py +++ b/master/buildbot/process/properties.py @@ -15,8 +15,10 @@ import collections import re +import warnings import weakref from buildbot import config, util +from buildbot.util import json from buildbot.interfaces import IRenderable, IProperties from twisted.internet import defer from twisted.python.components import registerAdapter @@ -119,6 +121,14 @@ def hasProperty(self, name): has_key = hasProperty def setProperty(self, name, value, source, runtime=False): + try: + json.dumps(value) + except TypeError: + warnings.warn( + "Non jsonable properties are not explicitly supported and" + + "will be explicitly disallowed in a future version.", + DeprecationWarning, stacklevel=2) + self.properties[name] = (value, source) if runtime: self.runtime.add(name) diff --git a/master/buildbot/test/unit/test_process_properties.py b/master/buildbot/test/unit/test_process_properties.py index 632ea94b6fc..4a99a17f891 100644 --- a/master/buildbot/test/unit/test_process_properties.py +++ b/master/buildbot/test/unit/test_process_properties.py @@ -497,14 +497,6 @@ def test_property_colon_plus(self): "echo projectdefined") return d - def test_property_renderable(self): - self.props.setProperty("project", FakeRenderable('testing'), "test") - command = Interpolate("echo '%(prop:project)s'") - d = self.build.render(command) - d.addCallback(self.failUnlessEqual, - "echo 'testing'") - return d - def test_nested_property(self): self.props.setProperty("project", "so long!", "test") command = Interpolate("echo '%(prop:missing:~%(prop:project)s)s'") @@ -513,17 +505,6 @@ def test_nested_property(self): "echo 'so long!'") return d - def test_nested_property_deferred(self): - renderable = DeferredRenderable() - self.props.setProperty("missing", renderable, "test") - self.props.setProperty("project", "so long!", "test") - command = Interpolate("echo '%(prop:missing:~%(prop:project)s)s'") - d = self.build.render(command) - d.addCallback(self.failUnlessEqual, - "echo 'so long!'") - renderable.callback(False) - return d - def test_property_substitute_recursively(self): self.props.setProperty("project", "proj1", "test") command = Interpolate("echo '%(prop:no_such:-%(prop:project)s)s'") @@ -1067,6 +1048,11 @@ def testUpdateFromPropertiesNoRuntime(self): self.failUnlessEqual(self.props.getProperty('x'), 24) self.failUnlessEqual(self.props.getPropertySource('x'), 'old') + def test_setProperty_notJsonable(self): + self.props.setProperty("project", FakeRenderable('testing'), "test") + self.props.setProperty("project", object, "test") + self.assertEqual(len(self.flushWarnings([self.test_setProperty_notJsonable])), 2) + # IProperties methods def test_getProperty(self): From 5b6e0e51560366bdf3931b31538d4e4d353fa8d7 Mon Sep 17 00:00:00 2001 From: Jay Soffian Date: Fri, 18 May 2012 14:33:15 -0400 Subject: [PATCH 076/136] Fix AttributeError in BuilderStatus.prune Fix bug introduced by b4fb9fbf22 (Refactor configuration handling, 2011-11-12), which removed the BuilderStatus.buildHorizon attribute. --- master/buildbot/status/builder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/master/buildbot/status/builder.py b/master/buildbot/status/builder.py index c2d003c07a0..8c38b5f6675 100644 --- a/master/buildbot/status/builder.py +++ b/master/buildbot/status/builder.py @@ -219,7 +219,7 @@ def prune(self, events_only=False): # get the horizons straight buildHorizon = self.master.config.buildHorizon if buildHorizon is not None: - earliest_build = self.nextBuildNumber - self.buildHorizon + earliest_build = self.nextBuildNumber - buildHorizon else: earliest_build = 0 From 78c9e45291f8af81b7116e66ec98b1408cee625c Mon Sep 17 00:00:00 2001 From: Tom Prince Date: Fri, 18 May 2012 17:40:58 -0600 Subject: [PATCH 077/136] Store a copy of the slave's path module on Build. This is saves navigating to through the slavebuilder and slave each time this is needed. --- master/buildbot/process/build.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/master/buildbot/process/build.py b/master/buildbot/process/build.py index 4d8ff0bdfed..26f35772f8e 100644 --- a/master/buildbot/process/build.py +++ b/master/buildbot/process/build.py @@ -184,13 +184,15 @@ def setupProperties(self): def setupSlaveBuilder(self, slavebuilder): self.slavebuilder = slavebuilder + self.path_module = slavebuilder.slave.path_module + # navigate our way back to the L{buildbot.buildslave.BuildSlave} # object that came from the config, and get its properties buildslave_properties = slavebuilder.slave.properties self.getProperties().updateFromProperties(buildslave_properties) if slavebuilder.slave.slave_basedir: self.setProperty("workdir", - slavebuilder.slave.path_module.join( + self.path_module.join( slavebuilder.slave.slave_basedir, self.builder.config.slavebuilddir), "slave") From 1690dd92709ae0ae8285db612bbc3df9c3ac08ed Mon Sep 17 00:00:00 2001 From: Jay Soffian Date: Fri, 18 May 2012 20:13:36 -0400 Subject: [PATCH 078/136] Remove waitUntilSuccess from IBuildSetStatus 7109604105 (remove waitUntilSuccess, rewrite waitUntilFinished, 2011-04-24) removed the method from the implementation. --- master/buildbot/interfaces.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/master/buildbot/interfaces.py b/master/buildbot/interfaces.py index 0ac15266fda..1cdbc0873ae 100644 --- a/master/buildbot/interfaces.py +++ b/master/buildbot/interfaces.py @@ -220,10 +220,6 @@ def getBuilderNames(): @returns: list of names via Deferred""" def isFinished(): pass - def waitUntilSuccess(): - """Return a Deferred that fires (with this IBuildSetStatus object) - when the outcome of the BuildSet is known, i.e., upon the first - failure, or after all builds complete successfully.""" def waitUntilFinished(): """Return a Deferred that fires (with this IBuildSetStatus object) when all builds have finished.""" From f27d85d348acfbd0c0599258cebbe9413a8d3cfa Mon Sep 17 00:00:00 2001 From: Tom Prince Date: Fri, 18 May 2012 17:42:27 -0600 Subject: [PATCH 079/136] Check that cvs checkout is pointing at the correct location. This checks 'CVS/Root' and 'CVS/Repository'. Fixes #2287. --- master/buildbot/steps/source/cvs.py | 40 ++- master/buildbot/test/fake/fakebuild.py | 2 + .../test/unit/test_steps_source_cvs.py | 230 ++++++++++++++++-- 3 files changed, 236 insertions(+), 36 deletions(-) diff --git a/master/buildbot/steps/source/cvs.py b/master/buildbot/steps/source/cvs.py index 0c0b2feb28b..80411f8c7ce 100644 --- a/master/buildbot/steps/source/cvs.py +++ b/master/buildbot/steps/source/cvs.py @@ -20,6 +20,7 @@ from twisted.internet import defer from buildbot.process import buildstep +from buildbot.steps.shell import StringFileWriter from buildbot.steps.source.base import Source from buildbot.interfaces import BuildSlaveTooOldError @@ -221,17 +222,36 @@ def evaluateCommand(cmd): d.addCallback(lambda _: evaluateCommand(cmd)) return d + @defer.inlineCallbacks def _sourcedirIsUpdatable(self): - cmd = buildstep.RemoteCommand('stat', {'file': self.workdir + '/CVS', - 'logEnviron': self.logEnviron}) - cmd.useLog(self.stdio_log, False) - d = self.runCommand(cmd) - def _fail(tmp): - if cmd.rc != 0: - return False - return True - d.addCallback(_fail) - return d + myFileWriter = StringFileWriter() + args = { + 'workdir': self.build.path_module.join(self.workdir, 'CVS'), + 'writer': myFileWriter, + 'maxSize': None, + 'blocksize': 32*1024, + } + + cmd = buildstep.RemoteCommand('uploadFile', + dict(slavesrc='Root', **args), + ignore_updates=True) + yield self.runCommand(cmd) + if cmd.rc != 0: + defer.returnValue(False) + if myFileWriter.buffer.strip() != self.cvsroot: + defer.returnValue(False) + + myFileWriter.buffer = "" + cmd = buildstep.RemoteCommand('uploadFile', + dict(slavesrc='Repository', **args), + ignore_updates=True) + yield self.runCommand(cmd) + if cmd.rc != 0: + defer.returnValue(False) + if myFileWriter.buffer.strip() != self.cvsmodule: + defer.returnValue(False) + + defer.returnValue(True) def parseGotRevision(self, res): revision = time.strftime("%Y-%m-%d %H:%M:%S +0000", time.gmtime()) diff --git a/master/buildbot/test/fake/fakebuild.py b/master/buildbot/test/fake/fakebuild.py index 02ee1b14e7d..50251a28dbf 100644 --- a/master/buildbot/test/fake/fakebuild.py +++ b/master/buildbot/test/fake/fakebuild.py @@ -14,6 +14,7 @@ # Copyright Buildbot Team Members import mock +import posixpath from twisted.python import components from buildbot.process import properties from buildbot import interfaces @@ -34,6 +35,7 @@ class FakeBuild(mock.Mock, properties.PropertiesMixin): def __init__(self, *args, **kwargs): mock.Mock.__init__(self, *args, **kwargs) self.build_status = FakeBuildStatus() + self.path_module = posixpath pr = self.build_status.properties = properties.Properties() pr.build = self diff --git a/master/buildbot/test/unit/test_steps_source_cvs.py b/master/buildbot/test/unit/test_steps_source_cvs.py index 1eefc396c54..b3ad24124a2 100644 --- a/master/buildbot/test/unit/test_steps_source_cvs.py +++ b/master/buildbot/test/unit/test_steps_source_cvs.py @@ -14,10 +14,18 @@ # Copyright Buildbot Team Members from twisted.trial import unittest +from buildbot.steps import shell from buildbot.steps.source import cvs from buildbot.status.results import SUCCESS, FAILURE from buildbot.test.util import sourcesteps -from buildbot.test.fake.remotecommand import ExpectShell, Expect +from buildbot.test.fake.remotecommand import ExpectShell, Expect, ExpectRemoteRef + +def uploadString(cvsroot): + def behavior(command): + writer = command.args['writer'] + writer.remote_write(cvsroot + "\n") + writer.remote_close() + return behavior class TestCVS(sourcesteps.SourceStepMixin, unittest.TestCase): @@ -36,8 +44,15 @@ def test_mode_full_clean(self): ExpectShell(workdir='wkdir', command=['cvs', '--version']) + 0, - Expect('stat', dict(file='wkdir/CVS', - logEnviron=True)) + Expect('uploadFile', dict(blocksize=32768, maxSize=None, + slavesrc='Root', workdir='wkdir/CVS', + writer=ExpectRemoteRef(shell.StringFileWriter))) + + Expect.behavior(uploadString(':pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot')) + + 0, + Expect('uploadFile', dict(blocksize=32768, maxSize=None, + slavesrc='Repository', workdir='wkdir/CVS', + writer=ExpectRemoteRef(shell.StringFileWriter))) + + Expect.behavior(uploadString('mozilla/browser/')) + 0, ExpectShell(workdir='wkdir', command=['cvsdiscard']) @@ -59,8 +74,15 @@ def test_mode_full_fresh(self): ExpectShell(workdir='wkdir', command=['cvs', '--version']) + 0, - Expect('stat', dict(file='wkdir/CVS', - logEnviron=True)) + Expect('uploadFile', dict(blocksize=32768, maxSize=None, + slavesrc='Root', workdir='wkdir/CVS', + writer=ExpectRemoteRef(shell.StringFileWriter))) + + Expect.behavior(uploadString(':pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot')) + + 0, + Expect('uploadFile', dict(blocksize=32768, maxSize=None, + slavesrc='Repository', workdir='wkdir/CVS', + writer=ExpectRemoteRef(shell.StringFileWriter))) + + Expect.behavior(uploadString('mozilla/browser/')) + 0, ExpectShell(workdir='wkdir', command=['cvsdiscard', '--ignore']) @@ -108,9 +130,16 @@ def test_mode_full_copy(self): Expect('rmdir', dict(dir='wkdir', logEnviron=True)) + 0, - Expect('stat', dict(file='source/CVS', - logEnviron=True)) - + 0, + Expect('uploadFile', dict(blocksize=32768, maxSize=None, + slavesrc='Root', workdir='source/CVS', + writer=ExpectRemoteRef(shell.StringFileWriter))) + + Expect.behavior(uploadString(':pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot')) + + 0, + Expect('uploadFile', dict(blocksize=32768, maxSize=None, + slavesrc='Repository', workdir='source/CVS', + writer=ExpectRemoteRef(shell.StringFileWriter))) + + Expect.behavior(uploadString('mozilla/browser/')) + + 0, ExpectShell(workdir='source', command=['cvs', '-z3', 'update', '-dP']) + 0, @@ -122,6 +151,38 @@ def test_mode_full_copy(self): self.expectOutcome(result=SUCCESS, status_text=["update"]) return self.runStep() + + def test_mode_full_copy_wrong_repo(self): + self.setupStep( + cvs.CVS(cvsroot=":pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot", + cvsmodule="mozilla/browser/", mode='full', method='copy', + login=True)) + self.expectCommands( + ExpectShell(workdir='wkdir', + command=['cvs', '--version']) + + 0, + Expect('rmdir', dict(dir='wkdir', + logEnviron=True)) + + 0, + Expect('uploadFile', dict(blocksize=32768, maxSize=None, + slavesrc='Root', workdir='source/CVS', + writer=ExpectRemoteRef(shell.StringFileWriter))) + + Expect.behavior(uploadString('the-end-of-the-universe')) + + 0, + ExpectShell(workdir='', + command=['cvs', + '-d', + ':pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot', + '-z3', 'checkout', '-d', 'source', 'mozilla/browser/']) + + 0, + Expect('cpdir', {'fromdir': 'source', 'todir': 'build', + 'logEnviron': True}) + + 0, + ) + + self.expectOutcome(result=SUCCESS, status_text=["update"]) + return self.runStep() + def test_mode_incremental(self): self.setupStep( cvs.CVS(cvsroot=":pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot", @@ -131,9 +192,16 @@ def test_mode_incremental(self): ExpectShell(workdir='wkdir', command=['cvs', '--version']) + 0, - Expect('stat', dict(file='wkdir/CVS', - logEnviron=True)) - + 0, + Expect('uploadFile', dict(blocksize=32768, maxSize=None, + slavesrc='Root', workdir='wkdir/CVS', + writer=ExpectRemoteRef(shell.StringFileWriter))) + + Expect.behavior(uploadString(':pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot')) + + 0, + Expect('uploadFile', dict(blocksize=32768, maxSize=None, + slavesrc='Repository', workdir='wkdir/CVS', + writer=ExpectRemoteRef(shell.StringFileWriter))) + + Expect.behavior(uploadString('mozilla/browser/')) + + 0, ExpectShell(workdir='wkdir', command=['cvs', '-z3', 'update', '-dP']) + 0, @@ -156,9 +224,16 @@ def test_mode_incremental_not_loggedin(self): ':pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot', 'login']) + 0, - Expect('stat', dict(file='wkdir/CVS', - logEnviron=True)) - + 0, + Expect('uploadFile', dict(blocksize=32768, maxSize=None, + slavesrc='Root', workdir='wkdir/CVS', + writer=ExpectRemoteRef(shell.StringFileWriter))) + + Expect.behavior(uploadString(':pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot')) + + 0, + Expect('uploadFile', dict(blocksize=32768, maxSize=None, + slavesrc='Repository', workdir='wkdir/CVS', + writer=ExpectRemoteRef(shell.StringFileWriter))) + + Expect.behavior(uploadString('mozilla/browser/')) + + 0, ExpectShell(workdir='wkdir', command=['cvs', '-z3', 'update', '-dP']) + 0, @@ -177,8 +252,9 @@ def test_mode_incremental_no_existing_repo(self): ExpectShell(workdir='wkdir', command=['cvs', '--version']) + 0, - Expect('stat', dict(file='wkdir/CVS', - logEnviron=True)) + Expect('uploadFile', dict(blocksize=32768, maxSize=None, + slavesrc='Root', workdir='wkdir/CVS', + writer=ExpectRemoteRef(shell.StringFileWriter))) + 1, ExpectShell(workdir='', command=['cvs', @@ -191,6 +267,61 @@ def test_mode_incremental_no_existing_repo(self): return self.runStep() + def test_mode_incremental_wrong_repo(self): + self.setupStep( + cvs.CVS(cvsroot=":pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot", + cvsmodule="mozilla/browser/", mode='incremental', + login=True)) + self.expectCommands( + ExpectShell(workdir='wkdir', + command=['cvs', '--version']) + + 0, + Expect('uploadFile', dict(blocksize=32768, maxSize=None, + slavesrc='Root', workdir='wkdir/CVS', + writer=ExpectRemoteRef(shell.StringFileWriter))) + + Expect.behavior(uploadString('the-end-of-the-universe')) + + 0, + ExpectShell(workdir='', + command=['cvs', + '-d', + ':pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot', + '-z3', 'checkout', '-d', 'wkdir', 'mozilla/browser/']) + + 0, + ) + self.expectOutcome(result=SUCCESS, status_text=["update"]) + return self.runStep() + + + def test_mode_incremental_wrong_module(self): + self.setupStep( + cvs.CVS(cvsroot=":pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot", + cvsmodule="mozilla/browser/", mode='incremental', + login=True)) + self.expectCommands( + ExpectShell(workdir='wkdir', + command=['cvs', '--version']) + + 0, + Expect('uploadFile', dict(blocksize=32768, maxSize=None, + slavesrc='Root', workdir='wkdir/CVS', + writer=ExpectRemoteRef(shell.StringFileWriter))) + + Expect.behavior(uploadString(':pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot')) + + 0, + Expect('uploadFile', dict(blocksize=32768, maxSize=None, + slavesrc='Repository', workdir='wkdir/CVS', + writer=ExpectRemoteRef(shell.StringFileWriter))) + + Expect.behavior(uploadString('the-end-of-the-universe')) + + 0, + ExpectShell(workdir='', + command=['cvs', + '-d', + ':pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot', + '-z3', 'checkout', '-d', 'wkdir', 'mozilla/browser/']) + + 0, + ) + self.expectOutcome(result=SUCCESS, status_text=["update"]) + return self.runStep() + + def test_mode_full_clean_no_existing_repo(self): self.setupStep( cvs.CVS(cvsroot=":pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot", @@ -200,8 +331,9 @@ def test_mode_full_clean_no_existing_repo(self): ExpectShell(workdir='wkdir', command=['cvs', '--version']) + 0, - Expect('stat', dict(file='wkdir/CVS', - logEnviron=True)) + Expect('uploadFile', dict(blocksize=32768, maxSize=None, + slavesrc='Root', workdir='wkdir/CVS', + writer=ExpectRemoteRef(shell.StringFileWriter))) + 1, ExpectShell(workdir='', command=['cvs', @@ -213,6 +345,30 @@ def test_mode_full_clean_no_existing_repo(self): self.expectOutcome(result=SUCCESS, status_text=["update"]) return self.runStep() + def test_mode_full_clean_wrong_repo(self): + self.setupStep( + cvs.CVS(cvsroot=":pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot", + cvsmodule="mozilla/browser/", mode='full', method='clean', + login=True)) + self.expectCommands( + ExpectShell(workdir='wkdir', + command=['cvs', '--version']) + + 0, + Expect('uploadFile', dict(blocksize=32768, maxSize=None, + slavesrc='Root', workdir='wkdir/CVS', + writer=ExpectRemoteRef(shell.StringFileWriter))) + + Expect.behavior(uploadString('the-end-of-the-universe')) + + 0, + ExpectShell(workdir='', + command=['cvs', + '-d', + ':pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot', + '-z3', 'checkout', '-d', 'wkdir', 'mozilla/browser/']) + + 0, + ) + self.expectOutcome(result=SUCCESS, status_text=["update"]) + return self.runStep() + def test_mode_full_no_method(self): self.setupStep( cvs.CVS(cvsroot=":pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot", @@ -222,8 +378,15 @@ def test_mode_full_no_method(self): ExpectShell(workdir='wkdir', command=['cvs', '--version']) + 0, - Expect('stat', dict(file='wkdir/CVS', - logEnviron=True)) + Expect('uploadFile', dict(blocksize=32768, maxSize=None, + slavesrc='Root', workdir='wkdir/CVS', + writer=ExpectRemoteRef(shell.StringFileWriter))) + + Expect.behavior(uploadString(':pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot')) + + 0, + Expect('uploadFile', dict(blocksize=32768, maxSize=None, + slavesrc='Repository', workdir='wkdir/CVS', + writer=ExpectRemoteRef(shell.StringFileWriter))) + + Expect.behavior(uploadString('mozilla/browser/')) + 0, ExpectShell(workdir='wkdir', command=['cvsdiscard', '--ignore']) @@ -245,8 +408,9 @@ def test_mode_incremental_with_options(self): ExpectShell(workdir='wkdir', command=['cvs', '--version']) + 0, - Expect('stat', dict(file='wkdir/CVS', - logEnviron=True)) + Expect('uploadFile', dict(blocksize=32768, maxSize=None, + slavesrc='Root', workdir='wkdir/CVS', + writer=ExpectRemoteRef(shell.StringFileWriter))) + 1, ExpectShell(workdir='', command=['cvs', '-q', '-d', @@ -268,8 +432,15 @@ def test_mode_incremental_with_env_logEnviron(self): env={'abc': '123'}, logEnviron=False) + 0, - Expect('stat', dict(file='wkdir/CVS', - logEnviron=False)) + Expect('uploadFile', dict(blocksize=32768, maxSize=None, + slavesrc='Root', workdir='wkdir/CVS', + writer=ExpectRemoteRef(shell.StringFileWriter))) + + Expect.behavior(uploadString(':pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot')) + + 0, + Expect('uploadFile', dict(blocksize=32768, maxSize=None, + slavesrc='Repository', workdir='wkdir/CVS', + writer=ExpectRemoteRef(shell.StringFileWriter))) + + Expect.behavior(uploadString('mozilla/browser/')) + 0, ExpectShell(workdir='wkdir', command=['cvs', '-z3', 'update', '-dP'], @@ -304,8 +475,15 @@ def test_cvsdiscard_fails(self): ExpectShell(workdir='wkdir', command=['cvs', '--version']) + 0, - Expect('stat', dict(file='wkdir/CVS', - logEnviron=True)) + Expect('uploadFile', dict(blocksize=32768, maxSize=None, + slavesrc='Root', workdir='wkdir/CVS', + writer=ExpectRemoteRef(shell.StringFileWriter))) + + Expect.behavior(uploadString(':pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot')) + + 0, + Expect('uploadFile', dict(blocksize=32768, maxSize=None, + slavesrc='Repository', workdir='wkdir/CVS', + writer=ExpectRemoteRef(shell.StringFileWriter))) + + Expect.behavior(uploadString('mozilla/browser/')) + 0, ExpectShell(workdir='wkdir', command=['cvsdiscard', '--ignore']) From 0464a04c48887d03d830ee4579dcaba5eb918734 Mon Sep 17 00:00:00 2001 From: Tom Prince Date: Fri, 18 May 2012 18:42:21 -0600 Subject: [PATCH 080/136] Fix _ComputeRepositoryURL for multirepo support. Fixes #2241, #2230. --- master/buildbot/steps/source/oldsource.py | 31 ++++++++++--------- ...s_source_oldsource_ComputeRepositoryURL.py | 17 +++++----- 2 files changed, 26 insertions(+), 22 deletions(-) diff --git a/master/buildbot/steps/source/oldsource.py b/master/buildbot/steps/source/oldsource.py index 4f0da58e26e..22544cf809e 100644 --- a/master/buildbot/steps/source/oldsource.py +++ b/master/buildbot/steps/source/oldsource.py @@ -26,7 +26,8 @@ class _ComputeRepositoryURL(object): implements(IRenderable) - def __init__(self, repository): + def __init__(self, step, repository): + self.step = step self.repository = repository def getRenderingFor(self, props): @@ -37,7 +38,7 @@ def getRenderingFor(self, props): build = props.getBuild() assert build is not None, "Build should be available *during* a build?" - s = build.getSourceStamp('') # TODO: use correct codebase + s = build.getSourceStamp(self.step.codebase) repository = self.repository @@ -160,7 +161,7 @@ def __init__(self, cvsroot=None, cvsmodule="", self.checkoutDelay = checkoutDelay self.branch = branch - self.cvsroot = _ComputeRepositoryURL(cvsroot) + self.cvsroot = _ComputeRepositoryURL(self, cvsroot) Source.__init__(self, **kwargs) @@ -287,8 +288,8 @@ def __init__(self, svnurl=None, baseURL=None, defaultBranch=None, warn("Please use workdir=, not directory=", DeprecationWarning) kwargs['workdir'] = directory - self.svnurl = svnurl and _ComputeRepositoryURL(svnurl) - self.baseURL = _ComputeRepositoryURL(baseURL) + self.svnurl = svnurl and _ComputeRepositoryURL(self, svnurl) + self.baseURL = _ComputeRepositoryURL(self, baseURL) self.branch = defaultBranch self.username = username self.password = password @@ -439,8 +440,8 @@ def __init__(self, repourl=None, baseURL=None, defaultBranch=None, C{baseURL} and the result handed to the 'darcs pull' command. """ - self.repourl = _ComputeRepositoryURL(repourl) - self.baseURL = _ComputeRepositoryURL(baseURL) + self.repourl = _ComputeRepositoryURL(self, repourl) + self.baseURL = _ComputeRepositoryURL(self, baseURL) self.branch = defaultBranch Source.__init__(self, **kwargs) assert self.args['mode'] != "export", \ @@ -535,7 +536,7 @@ def __init__(self, repourl=None, lack of output, but requires Git 1.7.2+. """ Source.__init__(self, **kwargs) - self.repourl = _ComputeRepositoryURL(repourl) + self.repourl = _ComputeRepositoryURL(self, repourl) self.branch = branch self.args.update({'submodules': submodules, 'ignore_ignores': ignore_ignores, @@ -604,7 +605,7 @@ def __init__(self, """ Source.__init__(self, **kwargs) - self.manifest_url = _ComputeRepositoryURL(manifest_url) + self.manifest_url = _ComputeRepositoryURL(self, manifest_url) self.args.update({'manifest_branch': manifest_branch, 'manifest_file': manifest_file, 'tarball': tarball, @@ -744,8 +745,8 @@ def __init__(self, repourl=None, baseURL=None, defaultBranch=None, if not using update/copy mode, or if using update/copy mode with multiple branches. """ - self.repourl = _ComputeRepositoryURL(repourl) - self.baseURL = _ComputeRepositoryURL(baseURL) + self.repourl = _ComputeRepositoryURL(self, repourl) + self.baseURL = _ComputeRepositoryURL(self, baseURL) self.branch = defaultBranch Source.__init__(self, **kwargs) self.args.update({'forceSharedRepo': forceSharedRepo}) @@ -827,8 +828,8 @@ def __init__(self, repourl=None, baseURL=None, defaultBranch=None, at each branch change. Otherwise, just update to the branch. """ - self.repourl = _ComputeRepositoryURL(repourl) - self.baseURL = _ComputeRepositoryURL(baseURL) + self.repourl = _ComputeRepositoryURL(self, repourl) + self.baseURL = _ComputeRepositoryURL(self, baseURL) self.branch = defaultBranch self.branchType = branchType self.clobberOnBranchChange = clobberOnBranchChange @@ -921,7 +922,7 @@ def __init__(self, p4base=None, defaultBranch=None, p4port=None, p4user=None, @param p4client: The perforce client to use for this buildslave. """ - self.p4base = _ComputeRepositoryURL(p4base) + self.p4base = _ComputeRepositoryURL(self, p4base) self.branch = defaultBranch Source.__init__(self, **kwargs) self.args['p4port'] = p4port @@ -1023,7 +1024,7 @@ def __init__(self, repourl=None, branch=None, progress=False, **kwargs): lack of output. """ Source.__init__(self, **kwargs) - self.repourl = _ComputeRepositoryURL(repourl) + self.repourl = _ComputeRepositoryURL(self, repourl) if (not repourl): raise ValueError("you must provide a repository uri in 'repourl'") if (not branch): diff --git a/master/buildbot/test/unit/test_steps_source_oldsource_ComputeRepositoryURL.py b/master/buildbot/test/unit/test_steps_source_oldsource_ComputeRepositoryURL.py index 5dd48a51d5b..65c8739d634 100644 --- a/master/buildbot/test/unit/test_steps_source_oldsource_ComputeRepositoryURL.py +++ b/master/buildbot/test/unit/test_steps_source_oldsource_ComputeRepositoryURL.py @@ -37,12 +37,15 @@ def render(self, value): self.props.build = self return defer.maybeDeferred(IRenderable(value).getRenderingFor, self.props) +class FakeStep(object): + codebase = '' + class RepoURL(unittest.TestCase): def setUp(self): self.build = Build() def test_backward_compatibility(self): - url = _ComputeRepositoryURL("repourl") + url = _ComputeRepositoryURL(FakeStep(), "repourl") d = self.build.render(url) @d.addCallback def callback(res): @@ -50,7 +53,7 @@ def callback(res): return d def test_format_string(self): - url = _ComputeRepositoryURL("http://server/%s") + url = _ComputeRepositoryURL(FakeStep(), "http://server/%s") d = self.build.render(url) @d.addCallback def callback(res): @@ -60,7 +63,7 @@ def callback(res): def test_dict(self): dict = {} dict['test'] = "ssh://server/testrepository" - url = _ComputeRepositoryURL(dict) + url = _ComputeRepositoryURL(FakeStep(), dict) d = self.build.render(url) @d.addCallback def callback(res): @@ -69,7 +72,7 @@ def callback(res): def test_callable(self): func = lambda x: x[::-1] - url = _ComputeRepositoryURL(func) + url = _ComputeRepositoryURL(FakeStep(), func) d = self.build.render(url) @d.addCallback def callback(res): @@ -77,7 +80,7 @@ def callback(res): return d def test_backward_compatibility_render(self): - url = _ComputeRepositoryURL(WithProperties("repourl%(foo)s")) + url = _ComputeRepositoryURL(FakeStep(), WithProperties("repourl%(foo)s")) d = self.build.render(url) @d.addCallback def callback(res): @@ -86,7 +89,7 @@ def callback(res): def test_dict_render(self): d = dict(test=WithProperties("repourl%(foo)s")) - url = _ComputeRepositoryURL(d) + url = _ComputeRepositoryURL(FakeStep(), d) d = self.build.render(url) @d.addCallback def callback(res): @@ -95,7 +98,7 @@ def callback(res): def test_callable_render(self): func = lambda x: WithProperties(x+"%(foo)s") - url = _ComputeRepositoryURL(func) + url = _ComputeRepositoryURL(FakeStep(), func) d = self.build.render(url) @d.addCallback def callback(res): From cdd3271731dadbd06b53f01fb69a79ae481bcaa3 Mon Sep 17 00:00:00 2001 From: Tom Prince Date: Fri, 18 May 2012 23:04:23 -0400 Subject: [PATCH 081/136] Use deferredLock for nested function in BaseBasicScheduler. --- master/buildbot/schedulers/basic.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/master/buildbot/schedulers/basic.py b/master/buildbot/schedulers/basic.py index 19823123918..17339b88299 100644 --- a/master/buildbot/schedulers/basic.py +++ b/master/buildbot/schedulers/basic.py @@ -96,14 +96,12 @@ def startService(self, _returnDeferred=False): def stopService(self): # the base stopService will unsubscribe from new changes d = base.BaseScheduler.stopService(self) - d.addCallback(lambda _ : - self._stable_timers_lock.acquire()) + @util.deferredLocked(self._stable_timers_lock) def cancel_timers(_): for timer in self._stable_timers.values(): if timer: timer.cancel() self._stable_timers = {} - self._stable_timers_lock.release() d.addCallback(cancel_timers) return d From 76e2165c1347bafff54ae151d83be594651ffc41 Mon Sep 17 00:00:00 2001 From: Tom Prince Date: Fri, 18 May 2012 23:05:35 -0400 Subject: [PATCH 082/136] _stable_timers is a defaultdict, so clear it, instead of changing it to a normal dict. --- master/buildbot/schedulers/basic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/master/buildbot/schedulers/basic.py b/master/buildbot/schedulers/basic.py index 17339b88299..ff3db0944c2 100644 --- a/master/buildbot/schedulers/basic.py +++ b/master/buildbot/schedulers/basic.py @@ -101,7 +101,7 @@ def cancel_timers(_): for timer in self._stable_timers.values(): if timer: timer.cancel() - self._stable_timers = {} + self._stable_timers.clear() d.addCallback(cancel_timers) return d From 6864057675e2cd7f6c91dde28837934e28fb9df7 Mon Sep 17 00:00:00 2001 From: Tom Prince Date: Fri, 18 May 2012 23:21:36 -0400 Subject: [PATCH 083/136] Make basic schedulers report pending build times. Fixes #2251. --- master/buildbot/schedulers/basic.py | 5 +++++ master/buildbot/test/unit/test_schedulers_basic.py | 13 +++++++++++++ 2 files changed, 18 insertions(+) diff --git a/master/buildbot/schedulers/basic.py b/master/buildbot/schedulers/basic.py index ff3db0944c2..594f9dd6a15 100644 --- a/master/buildbot/schedulers/basic.py +++ b/master/buildbot/schedulers/basic.py @@ -193,6 +193,11 @@ def stableTimerFired(self, timer_name): yield self.master.db.schedulers.flushChangeClassifications( self.objectid, less_than=max_changeid+1) + def getPendingBuildTimes(self): + # This isn't locked, since the caller expects and immediate value, + # and in any case, this is only an estimate. + return [timer.getTime() for timer in self._stable_timers.values() if timer and timer.active()] + class SingleBranchScheduler(BaseBasicScheduler): def getChangeFilter(self, branch, branches, change_filter, categories): if branch is NotABranch and not change_filter and self.codebases is not None: diff --git a/master/buildbot/test/unit/test_schedulers_basic.py b/master/buildbot/test/unit/test_schedulers_basic.py index a994650a2af..a1c56114c5e 100644 --- a/master/buildbot/test/unit/test_schedulers_basic.py +++ b/master/buildbot/test/unit/test_schedulers_basic.py @@ -129,6 +129,9 @@ def check(_): onlyImportant=False) self.db.schedulers.assertClassifications(self.OBJECTID, { 20 : True }) self.assertTrue(sched.timer_started) + self.assertEqual(sched.getPendingBuildTimes(), [ 10 ]) + self.clock.advance(10) + self.assertEqual(sched.getPendingBuildTimes(), []) d.addCallback(check) d.addCallback(lambda _ : sched.stopService()) return d @@ -207,6 +210,7 @@ def test_gotChange_treeStableTimer_sequence(self): self.makeFakeChange(branch='master', number=1, when=2220), True) self.assertEqual(self.events, []) + self.assertEqual(sched.getPendingBuildTimes(), [2229]) self.db.schedulers.assertClassifications(self.OBJECTID, { 1 : True }) # but another (unimportant) change arrives before then @@ -217,6 +221,7 @@ def test_gotChange_treeStableTimer_sequence(self): self.makeFakeChange(branch='master', number=2, when=2226), False) self.assertEqual(self.events, []) + self.assertEqual(sched.getPendingBuildTimes(), [2235]) self.db.schedulers.assertClassifications(self.OBJECTID, { 1 : True, 2 : False }) self.clock.advance(3) # to 2229 @@ -230,6 +235,7 @@ def test_gotChange_treeStableTimer_sequence(self): self.makeFakeChange(branch='master', number=3, when=2232), True) self.assertEqual(self.events, []) + self.assertEqual(sched.getPendingBuildTimes(), [2241]) self.db.schedulers.assertClassifications(self.OBJECTID, { 1 : True, 2 : False, 3 : True }) self.clock.advance(3) # to 2235 @@ -238,6 +244,7 @@ def test_gotChange_treeStableTimer_sequence(self): # finally, time to start the build! self.clock.advance(6) # to 2241 self.assertEqual(self.events, [ 'B[1,2,3]@2241' ]) + self.assertEqual(sched.getPendingBuildTimes(), []) self.db.schedulers.assertClassifications(self.OBJECTID, { }) yield sched.stopService() @@ -314,16 +321,22 @@ def mkch(**kwargs): d = defer.succeed(None) d.addCallback(lambda _ : sched.gotChange(mkch(branch='master', number=13), True)) + d.addCallback(lambda _ : + self.assertEqual(sched.getPendingBuildTimes(), [10])) d.addCallback(lambda _ : self.clock.advance(1)) # time is now 1 d.addCallback(lambda _ : sched.gotChange(mkch(branch='master', number=14), False)) + d.addCallback(lambda _ : + self.assertEqual(sched.getPendingBuildTimes(), [11])) d.addCallback(lambda _ : sched.gotChange(mkch(branch='boring', number=15), False)) d.addCallback(lambda _ : self.clock.pump([1]*4)) # time is now 5 d.addCallback(lambda _ : sched.gotChange(mkch(branch='devel', number=16), True)) + d.addCallback(lambda _ : + self.assertEqual(sched.getPendingBuildTimes(), [11,15])) d.addCallback(lambda _ : self.clock.pump([1]*10)) # time is now 15 def check(_): From 516d41c436acf2e9fb08a7549c7beb499c2e70d6 Mon Sep 17 00:00:00 2001 From: Tom Prince Date: Sat, 19 May 2012 10:37:44 -0400 Subject: [PATCH 084/136] Add warning about absolute builddir. Refs #2260. --- master/buildbot/config.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/master/buildbot/config.py b/master/buildbot/config.py index fe2e1299446..6bd3c273ae3 100644 --- a/master/buildbot/config.py +++ b/master/buildbot/config.py @@ -15,9 +15,10 @@ from __future__ import with_statement -import re import os +import re import sys +import warnings from buildbot.util import safeTranslate from buildbot import interfaces from buildbot import locks @@ -446,6 +447,10 @@ def load_slaves(self, filename, config_dict, errors): msg = "slave name '%s' is reserved" % sl.slavename errors.addError(msg) + if os.path.isabs(sl.builder): + warnings.warn("Absolute path '%s' for builder may cause mayhem. " + + "Perhaps you meant to specify slavebuilddir instead.") + self.slaves = config_dict['slaves'] From efcea6932de4e33eddb96bf0ea6047911dd6c93e Mon Sep 17 00:00:00 2001 From: "Dustin J. Mitchell" Date: Sat, 19 May 2012 15:49:07 -0500 Subject: [PATCH 085/136] fix pyflakes, with one significant change (undefined Failure) --- master/buildbot/libvirtbuildslave.py | 3 ++- master/buildbot/test/unit/test_libvirtslave.py | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/master/buildbot/libvirtbuildslave.py b/master/buildbot/libvirtbuildslave.py index 2aa2e19184b..26fbc6c5584 100644 --- a/master/buildbot/libvirtbuildslave.py +++ b/master/buildbot/libvirtbuildslave.py @@ -23,6 +23,7 @@ try: import libvirt + libvirt = libvirt except ImportError: libvirt = None @@ -252,7 +253,7 @@ def start_instance(self, build): else: self.domain = yield self.connection.lookupByName(self.name) yield self.domain.create() - except Exception, f: + except Exception: log.msg("Cannot start a VM (%s), failing gracefully and triggering a new build check" % self.name) log.err(failure.Failure()) self.domain = None diff --git a/master/buildbot/test/unit/test_libvirtslave.py b/master/buildbot/test/unit/test_libvirtslave.py index c4cc6e8fd15..b8cedfccde1 100644 --- a/master/buildbot/test/unit/test_libvirtslave.py +++ b/master/buildbot/test/unit/test_libvirtslave.py @@ -16,8 +16,9 @@ import mock from twisted.trial import unittest from twisted.internet import defer, reactor, utils +from twisted.python import failure from buildbot import libvirtbuildslave, config -from buildbot.test.fake import fakemaster, libvirt +from buildbot.test.fake import libvirt class TestLibVirtSlave(unittest.TestCase): @@ -167,7 +168,8 @@ def work(): def delayed_errback(self): def work(): d = defer.Deferred() - reactor.callLater(0, d.errback, Failure("Test failure")) + reactor.callLater(0, d.errback, + failure.Failure(RuntimeError("Test failure"))) return d return work From 508b3b61b466532f7997a3beeef91c32d1148eda Mon Sep 17 00:00:00 2001 From: "Dustin J. Mitchell" Date: Sat, 19 May 2012 15:50:00 -0500 Subject: [PATCH 086/136] fix error handling --- master/buildbot/libvirtbuildslave.py | 7 +++--- .../buildbot/test/unit/test_libvirtslave.py | 23 +++++++++++++++++++ 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/master/buildbot/libvirtbuildslave.py b/master/buildbot/libvirtbuildslave.py index 26fbc6c5584..285ad23eac2 100644 --- a/master/buildbot/libvirtbuildslave.py +++ b/master/buildbot/libvirtbuildslave.py @@ -253,9 +253,10 @@ def start_instance(self, build): else: self.domain = yield self.connection.lookupByName(self.name) yield self.domain.create() - except Exception: - log.msg("Cannot start a VM (%s), failing gracefully and triggering a new build check" % self.name) - log.err(failure.Failure()) + except: + log.err(failure.Failure(), + "Cannot start a VM (%s), failing gracefully and triggering" + "a new build check" % self.name) self.domain = None defer.returnValue(False) diff --git a/master/buildbot/test/unit/test_libvirtslave.py b/master/buildbot/test/unit/test_libvirtslave.py index b8cedfccde1..274e7b2b650 100644 --- a/master/buildbot/test/unit/test_libvirtslave.py +++ b/master/buildbot/test/unit/test_libvirtslave.py @@ -19,6 +19,7 @@ from twisted.python import failure from buildbot import libvirtbuildslave, config from buildbot.test.fake import libvirt +from buildbot.test.util import compat class TestLibVirtSlave(unittest.TestCase): @@ -107,6 +108,28 @@ def test_start_instance(self): self.assertEqual(started, True) + @compat.usesFlushLoggedErrors + @defer.inlineCallbacks + def test_start_instance_create_fails(self): + bs = self.ConcreteBuildSlave('b', 'p', self.conn, 'p', 'o', + xml='') + + prep = mock.Mock() + prep.side_effect = lambda: defer.succeed(0) + self.patch(bs, "_prepare_base_image", prep) + + create = mock.Mock() + create.side_effect = lambda self : defer.fail( + failure.Failure(RuntimeError('oh noes'))) + self.patch(libvirtbuildslave.Connection, 'create', create) + + yield bs._find_existing_deferred + started = yield bs.start_instance(mock.Mock()) + + self.assertEqual(bs.domain, None) + self.assertEqual(started, False) + self.assertEqual(len(self.flushLoggedErrors(RuntimeError)), 1) + @defer.inlineCallbacks def setup_canStartBuild(self): bs = self.ConcreteBuildSlave('b', 'p', self.conn, 'p', 'o') From 2033200de441e924951eaa5a675967a0fd3d21a3 Mon Sep 17 00:00:00 2001 From: "Dustin J. Mitchell" Date: Sat, 19 May 2012 15:50:17 -0500 Subject: [PATCH 087/136] rename unit test file to match the module it tests --- .../test/unit/{test_libvirtslave.py => test_libvirtbuildslave.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename master/buildbot/test/unit/{test_libvirtslave.py => test_libvirtbuildslave.py} (100%) diff --git a/master/buildbot/test/unit/test_libvirtslave.py b/master/buildbot/test/unit/test_libvirtbuildslave.py similarity index 100% rename from master/buildbot/test/unit/test_libvirtslave.py rename to master/buildbot/test/unit/test_libvirtbuildslave.py From b92efa494faf77913beec1f79e4b1e3abe495ed5 Mon Sep 17 00:00:00 2001 From: "Dustin J. Mitchell" Date: Sat, 19 May 2012 16:27:40 -0500 Subject: [PATCH 088/136] don't pass strings to defer.fail --- master/buildbot/test/unit/test_libvirtbuildslave.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/master/buildbot/test/unit/test_libvirtbuildslave.py b/master/buildbot/test/unit/test_libvirtbuildslave.py index 274e7b2b650..fa04413e32a 100644 --- a/master/buildbot/test/unit/test_libvirtbuildslave.py +++ b/master/buildbot/test/unit/test_libvirtbuildslave.py @@ -213,7 +213,7 @@ def work(): def test_handle_immediate_errback(self): def work(): - return defer.fail("Sad times") + return defer.fail(RuntimeError("Sad times")) return self.expect_errback(self.queue.execute(work)) def test_handle_delayed_errback(self): From dffbdc25a726a7d4803dd5dea5161116cc6648e6 Mon Sep 17 00:00:00 2001 From: "Dustin J. Mitchell" Date: Sat, 19 May 2012 16:37:11 -0500 Subject: [PATCH 089/136] support for using fakeWarnings, which is not supported on py27+tw0900 --- master/buildbot/test/unit/test_process_properties.py | 2 ++ master/buildbot/test/util/compat.py | 8 ++++++++ 2 files changed, 10 insertions(+) diff --git a/master/buildbot/test/unit/test_process_properties.py b/master/buildbot/test/unit/test_process_properties.py index 4a99a17f891..837ba451c71 100644 --- a/master/buildbot/test/unit/test_process_properties.py +++ b/master/buildbot/test/unit/test_process_properties.py @@ -24,6 +24,7 @@ from buildbot.interfaces import IRenderable, IProperties from buildbot.test.util.config import ConfigErrorsMixin from buildbot.test.util.properties import FakeRenderable +from buildbot.test.util import compat class FakeSource: def __init__(self): @@ -1048,6 +1049,7 @@ def testUpdateFromPropertiesNoRuntime(self): self.failUnlessEqual(self.props.getProperty('x'), 24) self.failUnlessEqual(self.props.getPropertySource('x'), 'old') + @compat.usesFlushWarnings def test_setProperty_notJsonable(self): self.props.setProperty("project", FakeRenderable('testing'), "test") self.props.setProperty("project", object, "test") diff --git a/master/buildbot/test/util/compat.py b/master/buildbot/test/util/compat.py index 59799c8d539..cf6305dd99b 100644 --- a/master/buildbot/test/util/compat.py +++ b/master/buildbot/test/util/compat.py @@ -25,6 +25,14 @@ def usesFlushLoggedErrors(test): "flushLoggedErrors is broken on Python==2.7 and Twisted<=9.0.0" return test +def usesFlushWarnings(test): + "Decorate a test method that uses flushWarnings with this decorator" + if (sys.version_info[:2] == (2,7) + and twisted.version <= versions.Version('twisted', 9, 0, 0)): + test.skip = \ + "flushWarnings is broken on Python==2.7 and Twisted<=9.0.0" + return test + def skipUnlessPlatformIs(platform): def closure(test): if runtime.platformType != platform: From c6e3e2d20da60fd6da8985153bb66e24d89e38f1 Mon Sep 17 00:00:00 2001 From: Tom Prince Date: Sat, 19 May 2012 18:04:03 -0400 Subject: [PATCH 090/136] Allow MasterShellCommand to be interrupted. Fixes #2149. --- master/buildbot/steps/master.py | 26 ++++++++++++++++--- .../buildbot/test/unit/test_steps_master.py | 25 +++++++++++++----- master/docs/manual/cfg-buildsteps.rst | 3 +++ 3 files changed, 43 insertions(+), 11 deletions(-) diff --git a/master/buildbot/steps/master.py b/master/buildbot/steps/master.py index 881a6ca9bc8..508980460c7 100644 --- a/master/buildbot/steps/master.py +++ b/master/buildbot/steps/master.py @@ -18,6 +18,7 @@ from twisted.internet import reactor from buildbot.process.buildstep import BuildStep from buildbot.process.buildstep import SUCCESS, FAILURE +from twisted.internet import error from twisted.internet.protocol import ProcessProtocol class MasterShellCommand(BuildStep): @@ -36,7 +37,7 @@ class MasterShellCommand(BuildStep): def __init__(self, command, description=None, descriptionDone=None, descriptionSuffix=None, - env=None, path=None, usePTY=0, + env=None, path=None, usePTY=0, interruptSignal="KILL", **kwargs): BuildStep.__init__(self, **kwargs) @@ -56,6 +57,7 @@ def __init__(self, command, self.env=env self.path=path self.usePTY=usePTY + self.interruptSignal = interruptSignal class LocalPP(ProcessProtocol): def __init__(self, step): @@ -68,7 +70,10 @@ def errReceived(self, data): self.step.stdio_log.addStderr(data) def processEnded(self, status_object): - self.step.stdio_log.addHeader("exit status %d\n" % status_object.value.exitCode) + if status_object.value.exitCode is not None: + self.step.stdio_log.addHeader("exit status %d\n" % status_object.value.exitCode) + if status_object.value.signal is not None: + self.step.stdio_log.addHeader("signal %s\n" % status_object.value.signal) self.step.processEnded(status_object) def start(self): @@ -121,12 +126,16 @@ def subst(match): stdio_log.addHeader(" env: %r\n" % (env,)) # TODO add a timeout? - reactor.spawnProcess(self.LocalPP(self), argv[0], argv, + self.process = reactor.spawnProcess(self.LocalPP(self), argv[0], argv, path=self.path, usePTY=self.usePTY, env=env ) # (the LocalPP object will call processEnded for us) def processEnded(self, status_object): - if status_object.value.exitCode != 0: + if status_object.value.signal is not None: + self.descriptionDone = ["killed (%s)" % status_object.value.signal] + self.step_status.setText(self.describe(done=True)) + self.finished(FAILURE) + elif status_object.value.exitCode != 0: self.descriptionDone = ["failed (%d)" % status_object.value.exitCode] self.step_status.setText(self.describe(done=True)) self.finished(FAILURE) @@ -140,3 +149,12 @@ def describe(self, done=False): desc = desc[:] desc.extend(self.descriptionSuffix) return desc + + def interrupt(self, reason): + try: + self.process.signalProcess(self.interruptSignal) + except KeyError: # Process not started yet + pass + except error.ProcessExitedAlready: + pass + BuildStep.interrupt(self, reason) diff --git a/master/buildbot/test/unit/test_steps_master.py b/master/buildbot/test/unit/test_steps_master.py index a99d4905901..a6b60d31d51 100644 --- a/master/buildbot/test/unit/test_steps_master.py +++ b/master/buildbot/test/unit/test_steps_master.py @@ -15,12 +15,11 @@ import os import sys -import mock -from twisted.python import runtime -from twisted.internet import reactor +from twisted.python import failure, runtime +from twisted.internet import error, reactor from twisted.trial import unittest from buildbot.test.util import steps -from buildbot.status.results import SUCCESS, FAILURE +from buildbot.status.results import SUCCESS, FAILURE, EXCEPTION from buildbot.steps import master from buildbot.process.properties import WithProperties @@ -51,9 +50,11 @@ def spawnProcess(pp, cmd, argv, path, usePTY, env): elif output[0] == 'err': pp.errReceived(output[1]) elif output[0] == 'rc': - so = mock.Mock(name='status_object') - so.value.exitCode = output[1] - pp.processEnded(so) + if output[1] != 0: + so = error.ProcessTerminated(exitCode=output[1]) + else: + so = error.ProcessDone(None) + pp.processEnded(failure.Failure(so)) self.patch(reactor, 'spawnProcess', spawnProcess) def test_real_cmd(self): @@ -67,6 +68,16 @@ def test_real_cmd(self): self.expectOutcome(result=SUCCESS, status_text=["Ran"]) return self.runStep() + def test_real_cmd_interrupted(self): + cmd = [ sys.executable, '-c', 'while True: pass' ] + self.setupStep( + master.MasterShellCommand(command=cmd)) + self.expectLogfile('stdio', "") + self.expectOutcome(result=EXCEPTION, status_text=["killed (9)", "interrupted"]) + d = self.runStep() + self.step.interrupt("KILL") + return d + def test_real_cmd_fails(self): cmd = [ sys.executable, '-c', 'import sys; sys.exit(1)' ] self.setupStep( diff --git a/master/docs/manual/cfg-buildsteps.rst b/master/docs/manual/cfg-buildsteps.rst index e197bf8dec5..c00c4cab150 100644 --- a/master/docs/manual/cfg-buildsteps.rst +++ b/master/docs/manual/cfg-buildsteps.rst @@ -2343,6 +2343,9 @@ Note that environment values must be strings (or lists that are turned into strings). In particular, numeric properties such as ``buildnumber`` must be substituted using :ref:`WithProperties`. +``interruptSignal`` + (optional) Signal to use to end the process, if the step is interrupted. + .. index:: Properties; from steps .. _Setting-Properties: From 7226de53321c549c600410241b1be4f9d3815e1c Mon Sep 17 00:00:00 2001 From: "Dustin J. Mitchell" Date: Sat, 19 May 2012 17:36:13 -0500 Subject: [PATCH 091/136] Spell maxsize correctly, and check for None or 0 in cmd.rc --- master/buildbot/steps/source/cvs.py | 10 ++-- .../test/unit/test_steps_source_cvs.py | 48 +++++++++---------- 2 files changed, 31 insertions(+), 27 deletions(-) diff --git a/master/buildbot/steps/source/cvs.py b/master/buildbot/steps/source/cvs.py index 80411f8c7ce..fc7dbdc3daf 100644 --- a/master/buildbot/steps/source/cvs.py +++ b/master/buildbot/steps/source/cvs.py @@ -228,7 +228,7 @@ def _sourcedirIsUpdatable(self): args = { 'workdir': self.build.path_module.join(self.workdir, 'CVS'), 'writer': myFileWriter, - 'maxSize': None, + 'maxsize': None, 'blocksize': 32*1024, } @@ -236,20 +236,24 @@ def _sourcedirIsUpdatable(self): dict(slavesrc='Root', **args), ignore_updates=True) yield self.runCommand(cmd) - if cmd.rc != 0: + if cmd.rc is not None and cmd.rc != 0: defer.returnValue(False) + return if myFileWriter.buffer.strip() != self.cvsroot: defer.returnValue(False) + return myFileWriter.buffer = "" cmd = buildstep.RemoteCommand('uploadFile', dict(slavesrc='Repository', **args), ignore_updates=True) yield self.runCommand(cmd) - if cmd.rc != 0: + if cmd.rc is not None and cmd.rc != 0: defer.returnValue(False) + return if myFileWriter.buffer.strip() != self.cvsmodule: defer.returnValue(False) + return defer.returnValue(True) diff --git a/master/buildbot/test/unit/test_steps_source_cvs.py b/master/buildbot/test/unit/test_steps_source_cvs.py index b3ad24124a2..9c0afe13849 100644 --- a/master/buildbot/test/unit/test_steps_source_cvs.py +++ b/master/buildbot/test/unit/test_steps_source_cvs.py @@ -44,12 +44,12 @@ def test_mode_full_clean(self): ExpectShell(workdir='wkdir', command=['cvs', '--version']) + 0, - Expect('uploadFile', dict(blocksize=32768, maxSize=None, + Expect('uploadFile', dict(blocksize=32768, maxsize=None, slavesrc='Root', workdir='wkdir/CVS', writer=ExpectRemoteRef(shell.StringFileWriter))) + Expect.behavior(uploadString(':pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot')) + 0, - Expect('uploadFile', dict(blocksize=32768, maxSize=None, + Expect('uploadFile', dict(blocksize=32768, maxsize=None, slavesrc='Repository', workdir='wkdir/CVS', writer=ExpectRemoteRef(shell.StringFileWriter))) + Expect.behavior(uploadString('mozilla/browser/')) @@ -74,12 +74,12 @@ def test_mode_full_fresh(self): ExpectShell(workdir='wkdir', command=['cvs', '--version']) + 0, - Expect('uploadFile', dict(blocksize=32768, maxSize=None, + Expect('uploadFile', dict(blocksize=32768, maxsize=None, slavesrc='Root', workdir='wkdir/CVS', writer=ExpectRemoteRef(shell.StringFileWriter))) + Expect.behavior(uploadString(':pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot')) + 0, - Expect('uploadFile', dict(blocksize=32768, maxSize=None, + Expect('uploadFile', dict(blocksize=32768, maxsize=None, slavesrc='Repository', workdir='wkdir/CVS', writer=ExpectRemoteRef(shell.StringFileWriter))) + Expect.behavior(uploadString('mozilla/browser/')) @@ -130,12 +130,12 @@ def test_mode_full_copy(self): Expect('rmdir', dict(dir='wkdir', logEnviron=True)) + 0, - Expect('uploadFile', dict(blocksize=32768, maxSize=None, + Expect('uploadFile', dict(blocksize=32768, maxsize=None, slavesrc='Root', workdir='source/CVS', writer=ExpectRemoteRef(shell.StringFileWriter))) + Expect.behavior(uploadString(':pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot')) + 0, - Expect('uploadFile', dict(blocksize=32768, maxSize=None, + Expect('uploadFile', dict(blocksize=32768, maxsize=None, slavesrc='Repository', workdir='source/CVS', writer=ExpectRemoteRef(shell.StringFileWriter))) + Expect.behavior(uploadString('mozilla/browser/')) @@ -164,7 +164,7 @@ def test_mode_full_copy_wrong_repo(self): Expect('rmdir', dict(dir='wkdir', logEnviron=True)) + 0, - Expect('uploadFile', dict(blocksize=32768, maxSize=None, + Expect('uploadFile', dict(blocksize=32768, maxsize=None, slavesrc='Root', workdir='source/CVS', writer=ExpectRemoteRef(shell.StringFileWriter))) + Expect.behavior(uploadString('the-end-of-the-universe')) @@ -192,12 +192,12 @@ def test_mode_incremental(self): ExpectShell(workdir='wkdir', command=['cvs', '--version']) + 0, - Expect('uploadFile', dict(blocksize=32768, maxSize=None, + Expect('uploadFile', dict(blocksize=32768, maxsize=None, slavesrc='Root', workdir='wkdir/CVS', writer=ExpectRemoteRef(shell.StringFileWriter))) + Expect.behavior(uploadString(':pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot')) + 0, - Expect('uploadFile', dict(blocksize=32768, maxSize=None, + Expect('uploadFile', dict(blocksize=32768, maxsize=None, slavesrc='Repository', workdir='wkdir/CVS', writer=ExpectRemoteRef(shell.StringFileWriter))) + Expect.behavior(uploadString('mozilla/browser/')) @@ -224,12 +224,12 @@ def test_mode_incremental_not_loggedin(self): ':pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot', 'login']) + 0, - Expect('uploadFile', dict(blocksize=32768, maxSize=None, + Expect('uploadFile', dict(blocksize=32768, maxsize=None, slavesrc='Root', workdir='wkdir/CVS', writer=ExpectRemoteRef(shell.StringFileWriter))) + Expect.behavior(uploadString(':pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot')) + 0, - Expect('uploadFile', dict(blocksize=32768, maxSize=None, + Expect('uploadFile', dict(blocksize=32768, maxsize=None, slavesrc='Repository', workdir='wkdir/CVS', writer=ExpectRemoteRef(shell.StringFileWriter))) + Expect.behavior(uploadString('mozilla/browser/')) @@ -252,7 +252,7 @@ def test_mode_incremental_no_existing_repo(self): ExpectShell(workdir='wkdir', command=['cvs', '--version']) + 0, - Expect('uploadFile', dict(blocksize=32768, maxSize=None, + Expect('uploadFile', dict(blocksize=32768, maxsize=None, slavesrc='Root', workdir='wkdir/CVS', writer=ExpectRemoteRef(shell.StringFileWriter))) + 1, @@ -276,7 +276,7 @@ def test_mode_incremental_wrong_repo(self): ExpectShell(workdir='wkdir', command=['cvs', '--version']) + 0, - Expect('uploadFile', dict(blocksize=32768, maxSize=None, + Expect('uploadFile', dict(blocksize=32768, maxsize=None, slavesrc='Root', workdir='wkdir/CVS', writer=ExpectRemoteRef(shell.StringFileWriter))) + Expect.behavior(uploadString('the-end-of-the-universe')) @@ -301,12 +301,12 @@ def test_mode_incremental_wrong_module(self): ExpectShell(workdir='wkdir', command=['cvs', '--version']) + 0, - Expect('uploadFile', dict(blocksize=32768, maxSize=None, + Expect('uploadFile', dict(blocksize=32768, maxsize=None, slavesrc='Root', workdir='wkdir/CVS', writer=ExpectRemoteRef(shell.StringFileWriter))) + Expect.behavior(uploadString(':pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot')) + 0, - Expect('uploadFile', dict(blocksize=32768, maxSize=None, + Expect('uploadFile', dict(blocksize=32768, maxsize=None, slavesrc='Repository', workdir='wkdir/CVS', writer=ExpectRemoteRef(shell.StringFileWriter))) + Expect.behavior(uploadString('the-end-of-the-universe')) @@ -331,7 +331,7 @@ def test_mode_full_clean_no_existing_repo(self): ExpectShell(workdir='wkdir', command=['cvs', '--version']) + 0, - Expect('uploadFile', dict(blocksize=32768, maxSize=None, + Expect('uploadFile', dict(blocksize=32768, maxsize=None, slavesrc='Root', workdir='wkdir/CVS', writer=ExpectRemoteRef(shell.StringFileWriter))) + 1, @@ -354,7 +354,7 @@ def test_mode_full_clean_wrong_repo(self): ExpectShell(workdir='wkdir', command=['cvs', '--version']) + 0, - Expect('uploadFile', dict(blocksize=32768, maxSize=None, + Expect('uploadFile', dict(blocksize=32768, maxsize=None, slavesrc='Root', workdir='wkdir/CVS', writer=ExpectRemoteRef(shell.StringFileWriter))) + Expect.behavior(uploadString('the-end-of-the-universe')) @@ -378,12 +378,12 @@ def test_mode_full_no_method(self): ExpectShell(workdir='wkdir', command=['cvs', '--version']) + 0, - Expect('uploadFile', dict(blocksize=32768, maxSize=None, + Expect('uploadFile', dict(blocksize=32768, maxsize=None, slavesrc='Root', workdir='wkdir/CVS', writer=ExpectRemoteRef(shell.StringFileWriter))) + Expect.behavior(uploadString(':pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot')) + 0, - Expect('uploadFile', dict(blocksize=32768, maxSize=None, + Expect('uploadFile', dict(blocksize=32768, maxsize=None, slavesrc='Repository', workdir='wkdir/CVS', writer=ExpectRemoteRef(shell.StringFileWriter))) + Expect.behavior(uploadString('mozilla/browser/')) @@ -408,7 +408,7 @@ def test_mode_incremental_with_options(self): ExpectShell(workdir='wkdir', command=['cvs', '--version']) + 0, - Expect('uploadFile', dict(blocksize=32768, maxSize=None, + Expect('uploadFile', dict(blocksize=32768, maxsize=None, slavesrc='Root', workdir='wkdir/CVS', writer=ExpectRemoteRef(shell.StringFileWriter))) + 1, @@ -432,12 +432,12 @@ def test_mode_incremental_with_env_logEnviron(self): env={'abc': '123'}, logEnviron=False) + 0, - Expect('uploadFile', dict(blocksize=32768, maxSize=None, + Expect('uploadFile', dict(blocksize=32768, maxsize=None, slavesrc='Root', workdir='wkdir/CVS', writer=ExpectRemoteRef(shell.StringFileWriter))) + Expect.behavior(uploadString(':pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot')) + 0, - Expect('uploadFile', dict(blocksize=32768, maxSize=None, + Expect('uploadFile', dict(blocksize=32768, maxsize=None, slavesrc='Repository', workdir='wkdir/CVS', writer=ExpectRemoteRef(shell.StringFileWriter))) + Expect.behavior(uploadString('mozilla/browser/')) @@ -475,12 +475,12 @@ def test_cvsdiscard_fails(self): ExpectShell(workdir='wkdir', command=['cvs', '--version']) + 0, - Expect('uploadFile', dict(blocksize=32768, maxSize=None, + Expect('uploadFile', dict(blocksize=32768, maxsize=None, slavesrc='Root', workdir='wkdir/CVS', writer=ExpectRemoteRef(shell.StringFileWriter))) + Expect.behavior(uploadString(':pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot')) + 0, - Expect('uploadFile', dict(blocksize=32768, maxSize=None, + Expect('uploadFile', dict(blocksize=32768, maxsize=None, slavesrc='Repository', workdir='wkdir/CVS', writer=ExpectRemoteRef(shell.StringFileWriter))) + Expect.behavior(uploadString('mozilla/browser/')) From 0375655ecfc5f4dfcec21188b94102fd17ca097e Mon Sep 17 00:00:00 2001 From: "Dustin J. Mitchell" Date: Sat, 19 May 2012 17:38:12 -0500 Subject: [PATCH 092/136] give the module last on the cvs commandline --- master/buildbot/steps/source/cvs.py | 4 ++-- master/buildbot/test/unit/test_steps_source_cvs.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/master/buildbot/steps/source/cvs.py b/master/buildbot/steps/source/cvs.py index fc7dbdc3daf..e6c52d6268b 100644 --- a/master/buildbot/steps/source/cvs.py +++ b/master/buildbot/steps/source/cvs.py @@ -162,13 +162,13 @@ def evaluate(rc): return d def doCheckout(self, dir): - command = ['-d', self.cvsroot, '-z3', 'checkout', '-d', dir, - self.cvsmodule] + command = ['-d', self.cvsroot, '-z3', 'checkout', '-d', dir ] command = self.global_options + command + self.extra_options if self.branch: command += ['-r', self.branch] if self.revision: command += ['-D', self.revision] + command += [ self.cvsmodule ] d = self._dovccmd(command, '') return d diff --git a/master/buildbot/test/unit/test_steps_source_cvs.py b/master/buildbot/test/unit/test_steps_source_cvs.py index 9c0afe13849..7f09b79793e 100644 --- a/master/buildbot/test/unit/test_steps_source_cvs.py +++ b/master/buildbot/test/unit/test_steps_source_cvs.py @@ -415,7 +415,7 @@ def test_mode_incremental_with_options(self): ExpectShell(workdir='', command=['cvs', '-q', '-d', ':pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot', - '-z3', 'checkout', '-d', 'wkdir', 'mozilla/browser/', '-l']) + '-z3', 'checkout', '-d', 'wkdir', '-l', 'mozilla/browser/']) + 0, ) self.expectOutcome(result=SUCCESS, status_text=["update"]) From 2b2eba96ace2a477baeb960eb6ab7615859268d0 Mon Sep 17 00:00:00 2001 From: "Dustin J. Mitchell" Date: Sat, 19 May 2012 17:46:58 -0500 Subject: [PATCH 093/136] correctly rmdir when a CVS dir is not updatable --- master/buildbot/steps/source/cvs.py | 2 +- .../buildbot/test/unit/test_steps_source_cvs.py | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/master/buildbot/steps/source/cvs.py b/master/buildbot/steps/source/cvs.py index e6c52d6268b..b89da7021b8 100644 --- a/master/buildbot/steps/source/cvs.py +++ b/master/buildbot/steps/source/cvs.py @@ -73,7 +73,7 @@ def incremental(self): if updatable: rv = yield self.doUpdate() else: - rv = yield self.doCheckout(self.workdir) + rv = yield self.clobber() defer.returnValue(rv) @defer.inlineCallbacks diff --git a/master/buildbot/test/unit/test_steps_source_cvs.py b/master/buildbot/test/unit/test_steps_source_cvs.py index 7f09b79793e..e6977e907ae 100644 --- a/master/buildbot/test/unit/test_steps_source_cvs.py +++ b/master/buildbot/test/unit/test_steps_source_cvs.py @@ -169,6 +169,9 @@ def test_mode_full_copy_wrong_repo(self): writer=ExpectRemoteRef(shell.StringFileWriter))) + Expect.behavior(uploadString('the-end-of-the-universe')) + 0, + Expect('rmdir', dict(dir='source', + logEnviron=True)) + + 0, ExpectShell(workdir='', command=['cvs', '-d', @@ -256,6 +259,9 @@ def test_mode_incremental_no_existing_repo(self): slavesrc='Root', workdir='wkdir/CVS', writer=ExpectRemoteRef(shell.StringFileWriter))) + 1, + Expect('rmdir', dict(dir='wkdir', + logEnviron=True)) + + 0, ExpectShell(workdir='', command=['cvs', '-d', @@ -281,6 +287,9 @@ def test_mode_incremental_wrong_repo(self): writer=ExpectRemoteRef(shell.StringFileWriter))) + Expect.behavior(uploadString('the-end-of-the-universe')) + 0, + Expect('rmdir', dict(dir='wkdir', + logEnviron=True)) + + 0, ExpectShell(workdir='', command=['cvs', '-d', @@ -311,6 +320,9 @@ def test_mode_incremental_wrong_module(self): writer=ExpectRemoteRef(shell.StringFileWriter))) + Expect.behavior(uploadString('the-end-of-the-universe')) + 0, + Expect('rmdir', dict(dir='wkdir', + logEnviron=True)) + + 0, ExpectShell(workdir='', command=['cvs', '-d', @@ -412,6 +424,9 @@ def test_mode_incremental_with_options(self): slavesrc='Root', workdir='wkdir/CVS', writer=ExpectRemoteRef(shell.StringFileWriter))) + 1, + Expect('rmdir', dict(dir='wkdir', + logEnviron=True)) + + 0, ExpectShell(workdir='', command=['cvs', '-q', '-d', ':pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot', From 13378f4fc88bf7b7a83fcf3c59140c48a50c714b Mon Sep 17 00:00:00 2001 From: "Dustin J. Mitchell" Date: Sat, 19 May 2012 18:02:52 -0500 Subject: [PATCH 094/136] Remove P4Sync (but not P4) as promised --- master/buildbot/steps/source/__init__.py | 4 +- master/buildbot/steps/source/oldsource.py | 45 --------------- .../test/regressions/test_oldpaths.py | 4 -- master/docs/release-notes.rst | 2 + slave/buildslave/commands/p4.py | 55 ------------------- slave/buildslave/commands/registry.py | 1 - 6 files changed, 4 insertions(+), 107 deletions(-) diff --git a/master/buildbot/steps/source/__init__.py b/master/buildbot/steps/source/__init__.py index 9dcb31ef310..fa414452231 100644 --- a/master/buildbot/steps/source/__init__.py +++ b/master/buildbot/steps/source/__init__.py @@ -15,7 +15,7 @@ from buildbot.steps.source.base import Source from buildbot.steps.source.oldsource import CVS, \ - SVN, Git, Darcs, Repo, Bzr, Mercurial, P4, P4Sync, Monotone + SVN, Git, Darcs, Repo, Bzr, Mercurial, P4, Monotone _hush_pyflakes = [ Source, CVS, SVN, \ - Git, Darcs, Repo, Bzr, Mercurial, P4, P4Sync, Monotone ] + Git, Darcs, Repo, Bzr, Mercurial, P4, Monotone ] diff --git a/master/buildbot/steps/source/oldsource.py b/master/buildbot/steps/source/oldsource.py index 22544cf809e..46499ccbd33 100644 --- a/master/buildbot/steps/source/oldsource.py +++ b/master/buildbot/steps/source/oldsource.py @@ -956,51 +956,6 @@ def startVC(self, branch, revision, patch): cmd = RemoteCommand("p4", args) self.startCommand(cmd) -class P4Sync(Source): - """ - DEPRECATED - will be removed in 0.8.5. - - This is a partial solution for using a P4 source repository. You are - required to manually set up each build slave with a useful P4 - environment, which means setting various per-slave environment variables, - and creating a P4 client specification which maps the right files into - the slave's working directory. Once you have done that, this step merely - performs a 'p4 sync' to update that workspace with the newest files. - - Each slave needs the following environment: - - - PATH: the 'p4' binary must be on the slave's PATH - - P4USER: each slave needs a distinct user account - - P4CLIENT: each slave needs a distinct client specification - - You should use 'p4 client' (?) to set up a client view spec which maps - the desired files into $SLAVEBASE/$BUILDERBASE/source . - """ - - name = "p4sync" - - def __init__(self, p4port, p4user, p4passwd, p4client, **kwargs): - assert kwargs['mode'] == "copy", "P4Sync can only be used in mode=copy" - self.branch = None - Source.__init__(self, **kwargs) - self.args['p4port'] = p4port - self.args['p4user'] = p4user - self.args['p4passwd'] = p4passwd - self.args['p4client'] = p4client - - def computeSourceRevision(self, changes): - if not changes: - return None - lastChange = max([int(c.revision) for c in changes]) - return lastChange - - def startVC(self, branch, revision, patch): - slavever = self.slaveVersion("p4sync") - assert slavever, "slave is too old, does not know about p4" - cmd = RemoteCommand("p4sync", self.args) - self.startCommand(cmd) - - class Monotone(Source): """Check out a source tree from a monotone repository 'repourl'.""" diff --git a/master/buildbot/test/regressions/test_oldpaths.py b/master/buildbot/test/regressions/test_oldpaths.py index c42fc2a39ed..54f4ae0992b 100644 --- a/master/buildbot/test/regressions/test_oldpaths.py +++ b/master/buildbot/test/regressions/test_oldpaths.py @@ -181,10 +181,6 @@ def test_steps_source_P4(self): from buildbot.steps.source import P4 assert P4 - def test_steps_source_P4Sync(self): - from buildbot.steps.source import P4Sync - assert P4Sync - def test_steps_source_Monotone(self): from buildbot.steps.source import Monotone assert Monotone diff --git a/master/docs/release-notes.rst b/master/docs/release-notes.rst index f9b73e43b89..de451fb1be4 100644 --- a/master/docs/release-notes.rst +++ b/master/docs/release-notes.rst @@ -66,6 +66,8 @@ Deprecations, Removals, and Non-Compatible Changes from buildbot.steps.source.svn import SVN factory.append(SVN(repourl=Interpolate("svn://svn.example.org/svn/%(src::branch:-branches/test)s"))) +* The ``P4Sync`` step, deprecated since 0.8.5, has been removed. The ``P4`` step remains. + Changes for Developers ~~~~~~~~~~~~~~~~~~~~~~ diff --git a/slave/buildslave/commands/p4.py b/slave/buildslave/commands/p4.py index d1f92b9fc22..7aa17d8e5df 100644 --- a/slave/buildslave/commands/p4.py +++ b/slave/buildslave/commands/p4.py @@ -219,58 +219,3 @@ def parseGotRevision(self): else: return P4Base.parseGotRevision(self) - -class P4Sync(P4Base): - """A partial P4 source-updater. Requires manual setup of a per-slave P4 - environment. The only thing which comes from the master is P4PORT. - 'mode' is required to be 'copy'. - - ['p4port'] (required): host:port for server to access - ['p4user'] (optional): user to use for access - ['p4passwd'] (optional): passwd to try for the user - ['p4client'] (optional): client spec to use - """ - - header = "p4 sync" - - def setup(self, args): - P4Base.setup(self, args) - - def sourcedirIsUpdateable(self): - return True - - def _doVC(self, force): - d = os.path.join(self.builder.basedir, self.srcdir) - command = [self.getCommand('p4')] - if self.p4port: - command.extend(['-p', self.p4port]) - if self.p4user: - command.extend(['-u', self.p4user]) - if self.p4passwd: - command.extend(['-P', Obfuscated(self.p4passwd, "XXXXXXXX")]) - if self.p4client: - command.extend(['-c', self.p4client]) - command.extend(['sync']) - if force: - command.extend(['-f']) - if self.revision: - command.extend(['@' + self.revision]) - env = {} - c = runprocess.RunProcess(self.builder, command, d, environ=env, - sendRC=False, timeout=self.timeout, - maxTime=self.maxTime, usePTY=False, - logEnviron=self.logEnviron) - self.command = c - return c.start() - - def doVCUpdate(self): - return self._doVC(force=False) - - def doVCFull(self): - return self._doVC(force=True) - - def parseGotRevision(self): - if self.revision: - return str(self.revision) - else: - return P4Base.parseGotRevision(self) diff --git a/slave/buildslave/commands/registry.py b/slave/buildslave/commands/registry.py index 19c2060d8dc..2c2b5564c9c 100644 --- a/slave/buildslave/commands/registry.py +++ b/slave/buildslave/commands/registry.py @@ -29,7 +29,6 @@ "bzr" : "buildslave.commands.bzr.Bzr", "hg" : "buildslave.commands.hg.Mercurial", "p4" : "buildslave.commands.p4.P4", - "p4sync" : "buildslave.commands.p4.P4Sync", "mtn" : "buildslave.commands.mtn.Monotone", "mkdir" : "buildslave.commands.fs.MakeDirectory", "rmdir" : "buildslave.commands.fs.RemoveDirectory", From 0ba689bf655d5b1982bd026cb2002daf01e60ccd Mon Sep 17 00:00:00 2001 From: "Dustin J. Mitchell" Date: Sat, 19 May 2012 18:21:22 -0500 Subject: [PATCH 095/136] Fixes to builddir warning --- master/buildbot/config.py | 10 ++++++---- master/buildbot/test/unit/test_config.py | 9 +++++++++ 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/master/buildbot/config.py b/master/buildbot/config.py index 6bd3c273ae3..b73d331028c 100644 --- a/master/buildbot/config.py +++ b/master/buildbot/config.py @@ -425,6 +425,12 @@ def mapper(b): errors.addError("c['builders'] must be a list of builder configs") return + for builder in builders: + if os.path.isabs(builder.builddir): + warnings.warn("Absolute path '%s' for builder may cause " + "mayhem. Perhaps you meant to specify slavebuilddir " + "instead.") + self.builders = builders @@ -447,10 +453,6 @@ def load_slaves(self, filename, config_dict, errors): msg = "slave name '%s' is reserved" % sl.slavename errors.addError(msg) - if os.path.isabs(sl.builder): - warnings.warn("Absolute path '%s' for builder may cause mayhem. " + - "Perhaps you meant to specify slavebuilddir instead.") - self.slaves = config_dict['slaves'] diff --git a/master/buildbot/test/unit/test_config.py b/master/buildbot/test/unit/test_config.py index 115eac07e32..cfbc1f90882 100644 --- a/master/buildbot/test/unit/test_config.py +++ b/master/buildbot/test/unit/test_config.py @@ -607,6 +607,15 @@ def test_load_builders_dict(self): self.assertIsInstance(self.cfg.builders[0], config.BuilderConfig) self.assertEqual(self.cfg.builders[0].name, 'x') + @compat.usesFlushWarnings + def test_load_builders_abs_builddir(self): + bldr = dict(name='x', factory=mock.Mock(), slavename='x', + builddir=os.path.abspath('.')) + self.cfg.load_builders(self.filename, + dict(builders=[bldr]), self.errors) + self.assertEqual( + len(self.flushWarnings([self.cfg.load_builders])), + 1) def test_load_slaves_defaults(self): self.cfg.load_slaves(self.filename, {}, self.errors) From eaf13be615110d18c7c1aec7ea474c5ac34b0273 Mon Sep 17 00:00:00 2001 From: Tom Prince Date: Sat, 19 May 2012 11:11:08 -0400 Subject: [PATCH 096/136] Factor checking of cmd.rc to allow other successful results. This provides an argument successfulRC to RemoteCommand, to specify some non-zero RCs as successes. Fixes #2225. --- master/buildbot/process/buildstep.py | 16 +++++++++++----- master/buildbot/steps/maxq.py | 2 +- master/buildbot/steps/python.py | 4 ++-- master/buildbot/steps/python_twisted.py | 6 +++--- master/buildbot/steps/shell.py | 8 ++++---- master/buildbot/steps/slave.py | 6 +++--- master/buildbot/steps/source/bzr.py | 4 ++-- master/buildbot/steps/source/cvs.py | 8 ++++---- master/buildbot/steps/source/git.py | 4 ++-- master/buildbot/steps/source/mercurial.py | 4 ++-- master/buildbot/steps/source/svn.py | 12 ++++++------ master/buildbot/steps/subunit.py | 2 +- master/buildbot/steps/transfer.py | 16 +++++++--------- master/buildbot/steps/vstudio.py | 2 +- master/buildbot/test/fake/remotecommand.py | 11 ++++++++--- master/buildbot/test/util/steps.py | 4 +++- master/docs/developer/cls-remotecommands.rst | 9 ++++++++- master/docs/manual/cfg-buildsteps.rst | 4 ++++ 18 files changed, 72 insertions(+), 50 deletions(-) diff --git a/master/buildbot/process/buildstep.py b/master/buildbot/process/buildstep.py index 23733093a6a..997b238a8df 100644 --- a/master/buildbot/process/buildstep.py +++ b/master/buildbot/process/buildstep.py @@ -42,7 +42,8 @@ class RemoteCommand(pb.Referenceable): rc = None debug = False - def __init__(self, remote_command, args, ignore_updates=False, collectStdout=False): + def __init__(self, remote_command, args, ignore_updates=False, + collectStdout=False, successfulRC=(0,)): self.logs = {} self.delayedLogs = {} self._closeWhenFinished = {} @@ -54,6 +55,7 @@ def __init__(self, remote_command, args, ignore_updates=False, collectStdout=Fal self.remote_command = remote_command self.args = args self.ignore_updates = ignore_updates + self.successfulRC = successfulRC def __repr__(self): return "" % (self.remote_command, id(self)) @@ -268,6 +270,9 @@ def remoteComplete(self, maybeFailure): log.msg("closing log %s" % loog) loog.finish() return maybeFailure + + def didFail(self): + return self.rc not in self.successfulRC LoggedRemoteCommand = RemoteCommand @@ -345,7 +350,7 @@ def __init__(self, workdir, command, env=None, timeout=20*60, maxTime=None, logfiles={}, usePTY="slave-config", logEnviron=True, collectStdout=False, interruptSignal=None, - initialStdin=None): + initialStdin=None, successfulRC=(0,)): self.command = command # stash .command, set it later if env is not None: @@ -366,7 +371,8 @@ def __init__(self, workdir, command, env=None, } if interruptSignal is not None: args['interruptSignal'] = interruptSignal - RemoteCommand.__init__(self, "shell", args, collectStdout=collectStdout) + RemoteCommand.__init__(self, "shell", args, collectStdout=collectStdout, + sucessfulRC=successfulRC) def _start(self): self.args['command'] = self.command @@ -893,7 +899,7 @@ def createSummary(self, stdio): def evaluateCommand(self, cmd): if self.log_eval_func: return self.log_eval_func(cmd, self.step_status) - if cmd.rc != 0: + if cmd.didFail(): return FAILURE return SUCCESS @@ -941,7 +947,7 @@ def setStatus(self, cmd, results): # ) def regex_log_evaluator(cmd, step_status, regexes): worst = SUCCESS - if cmd.rc != 0: + if cmd.didFail(): worst = FAILURE for err, possible_status in regexes: # worst_status returns the worse of the two status' passed to it. diff --git a/master/buildbot/steps/maxq.py b/master/buildbot/steps/maxq.py index c165e0c1300..bb40677af4b 100644 --- a/master/buildbot/steps/maxq.py +++ b/master/buildbot/steps/maxq.py @@ -34,7 +34,7 @@ def commandComplete(self, cmd): def evaluateCommand(self, cmd): # treat a nonzero exit status as a failure, if no other failures are # detected - if not self.failures and cmd.rc != 0: + if not self.failures and cmd.didFail(): self.failures = 1 if self.failures: return FAILURE diff --git a/master/buildbot/steps/python.py b/master/buildbot/steps/python.py index 6b0882263d9..e94d4209ff3 100644 --- a/master/buildbot/steps/python.py +++ b/master/buildbot/steps/python.py @@ -58,7 +58,7 @@ def createSummary(self, log): self.errors = errors def evaluateCommand(self, cmd): - if cmd.rc != 0: + if cmd.didFail(): return FAILURE if self.warnings or self.errors: return WARNINGS @@ -118,7 +118,7 @@ def createSummary(self, log): def evaluateCommand(self, cmd): - if cmd.rc != 0: + if cmd.didFail(): return FAILURE for m in self.flunkingIssues: if self.getProperty("pyflakes-%s" % m): diff --git a/master/buildbot/steps/python_twisted.py b/master/buildbot/steps/python_twisted.py index f7fa03d308f..0123e54277a 100644 --- a/master/buildbot/steps/python_twisted.py +++ b/master/buildbot/steps/python_twisted.py @@ -86,14 +86,14 @@ def commandComplete(self, cmd): def evaluateCommand(self, cmd): # warnings are in stdout, rc is always 0, unless the tools break - if cmd.rc != 0: + if cmd.didFail(): return FAILURE if self.warnings: return WARNINGS return SUCCESS def getText2(self, cmd, results): - if cmd.rc != 0: + if cmd.didFail(): return ["hlint"] return ["%d hlin%s" % (self.warnings, self.warnings == 1 and 't' or 'ts')] @@ -409,7 +409,7 @@ def commandComplete(self, cmd): text = [] text2 = "" - if cmd.rc == 0: + if not cmd.didFail(): if parsed: results = SUCCESS if total: diff --git a/master/buildbot/steps/shell.py b/master/buildbot/steps/shell.py index 5011c05b362..ee5f15143e2 100644 --- a/master/buildbot/steps/shell.py +++ b/master/buildbot/steps/shell.py @@ -287,7 +287,7 @@ def commandComplete(self, cmd): self.setProperty("tree-size-KiB", self.kib, "treesize") def evaluateCommand(self, cmd): - if cmd.rc != 0: + if cmd.didFail(): return FAILURE if self.kib is None: return WARNINGS # not sure how 'du' could fail, but whatever @@ -318,7 +318,7 @@ def __init__(self, property=None, extract_fn=None, strip=True, **kwargs): def commandComplete(self, cmd): if self.property: - if cmd.rc != 0: + if cmd.didFail(): return result = cmd.logs['stdio'].getText() if self.strip: result = result.strip() @@ -585,7 +585,7 @@ def createSummary(self, log): def evaluateCommand(self, cmd): - if ( cmd.rc != 0 or + if ( cmd.didFail() or ( self.maxWarnCount != None and self.warnCount > self.maxWarnCount ) ): return FAILURE if self.warnCount: @@ -661,7 +661,7 @@ def evaluateCommand(self, cmd): passed = 0 failed = 0 rc = SUCCESS - if cmd.rc > 0: + if cmd.didFail(): rc = FAILURE # New version of Test::Harness? diff --git a/master/buildbot/steps/slave.py b/master/buildbot/steps/slave.py index dfe346fc46d..4d9b969e64c 100644 --- a/master/buildbot/steps/slave.py +++ b/master/buildbot/steps/slave.py @@ -85,7 +85,7 @@ def start(self): d.addErrback(self.failed) def commandComplete(self, cmd): - if cmd.rc != 0: + if cmd.didFail(): self.step_status.setText(["File not found."]) self.finished(FAILURE) return @@ -125,7 +125,7 @@ def start(self): d.addErrback(self.failed) def commandComplete(self, cmd): - if cmd.rc != 0: + if cmd.didFail(): self.step_status.setText(["Delete failed."]) self.finished(FAILURE) return @@ -159,7 +159,7 @@ def start(self): d.addErrback(self.failed) def commandComplete(self, cmd): - if cmd.rc != 0: + if cmd.didFail(): self.step_status.setText(["Create failed."]) self.finished(FAILURE) return diff --git a/master/buildbot/steps/source/bzr.py b/master/buildbot/steps/source/bzr.py index afa0add92fe..245069cf4cf 100644 --- a/master/buildbot/steps/source/bzr.py +++ b/master/buildbot/steps/source/bzr.py @@ -180,7 +180,7 @@ def _sourcedirIsUpdatable(self): cmd.useLog(self.stdio_log, False) d = self.runCommand(cmd) def _fail(tmp): - if cmd.rc != 0: + if cmd.didFail(): return False return True d.addCallback(_fail) @@ -200,7 +200,7 @@ def _dovccmd(self, command, abandonOnFailure=True, collectStdout=False): cmd.useLog(self.stdio_log, False) d = self.runCommand(cmd) def evaluateCommand(cmd): - if abandonOnFailure and cmd.rc != 0: + if abandonOnFailure and cmd.didFail(): log.msg("Source step failed while running command %s" % cmd) raise buildstep.BuildStepFailed() if collectStdout: diff --git a/master/buildbot/steps/source/cvs.py b/master/buildbot/steps/source/cvs.py index 0c0b2feb28b..a189a4589aa 100644 --- a/master/buildbot/steps/source/cvs.py +++ b/master/buildbot/steps/source/cvs.py @@ -153,11 +153,11 @@ def purge(self, ignore_ignores): logEnviron=self.logEnviron) cmd.useLog(self.stdio_log, False) d = self.runCommand(cmd) - def evaluate(rc): - if rc != 0: + def evaluate(cmd): + if cmd.didFail(): raise buildstep.BuildStepFailed() - return rc - d.addCallback(lambda _: evaluate(cmd.rc)) + return cmd.rc + d.addCallback(evaluate) return d def doCheckout(self, dir): diff --git a/master/buildbot/steps/source/git.py b/master/buildbot/steps/source/git.py index 4ed690b6115..fc19f165507 100644 --- a/master/buildbot/steps/source/git.py +++ b/master/buildbot/steps/source/git.py @@ -300,7 +300,7 @@ def _dovccmd(self, command, abandonOnFailure=True, collectStdout=False, initialS log.msg("Starting git command : git %s" % (" ".join(command), )) d = self.runCommand(cmd) def evaluateCommand(cmd): - if abandonOnFailure and cmd.rc != 0: + if abandonOnFailure and cmd.didFail(): log.msg("Source step failed while running command %s" % cmd) raise buildstep.BuildStepFailed() if collectStdout: @@ -410,7 +410,7 @@ def _sourcedirIsUpdatable(self): cmd.useLog(self.stdio_log, False) d = self.runCommand(cmd) def _fail(tmp): - if cmd.rc != 0: + if cmd.didFail(): return False return True d.addCallback(_fail) diff --git a/master/buildbot/steps/source/mercurial.py b/master/buildbot/steps/source/mercurial.py index 83a392da79f..2ae76f371cc 100644 --- a/master/buildbot/steps/source/mercurial.py +++ b/master/buildbot/steps/source/mercurial.py @@ -229,7 +229,7 @@ def _dovccmd(self, command, collectStdout=False): log.msg("Starting mercurial command : hg %s" % (" ".join(command), )) d = self.runCommand(cmd) def evaluateCommand(cmd): - if cmd.rc != 0: + if cmd.didFail(): log.msg("Source step failed while running command %s" % cmd) raise buildstep.BuildStepFailed() if collectStdout: @@ -276,7 +276,7 @@ def _sourcedirIsUpdatable(self): cmd.useLog(self.stdio_log, False) d = self.runCommand(cmd) def _fail(tmp): - if cmd.rc != 0: + if cmd.didFail(): return False return True d.addCallback(_fail) diff --git a/master/buildbot/steps/source/svn.py b/master/buildbot/steps/source/svn.py index a4719b6b720..a71ca470b6e 100644 --- a/master/buildbot/steps/source/svn.py +++ b/master/buildbot/steps/source/svn.py @@ -130,7 +130,7 @@ def clobber(self): 'logEnviron': self.logEnviron,}) cmd.useLog(self.stdio_log, False) yield self.runCommand(cmd) - if cmd.rc != 0: + if cmd.didFail(): raise buildstep.BuildStepFailed() checkout_cmd = ['checkout', self.repourl, '.'] @@ -162,7 +162,7 @@ def copy(self): cmd.useLog(self.stdio_log, False) yield self.runCommand(cmd) - if cmd.rc != 0: + if cmd.didFail(): raise buildstep.BuildStepFailed() # temporarily set workdir = 'source' and do an incremental checkout @@ -192,7 +192,7 @@ def copy(self): yield self.runCommand(cmd) - if cmd.rc != 0: + if cmd.didFail(): raise buildstep.BuildStepFailed() def finish(self, res): @@ -210,7 +210,7 @@ def _rmdir(self, dir): {'dir': dir, 'logEnviron': self.logEnviron }) cmd.useLog(self.stdio_log, False) yield self.runCommand(cmd) - if cmd.rc != 0: + if cmd.didFail(): raise buildstep.BuildStepFailed() def _dovccmd(self, command, collectStdout=False): @@ -233,7 +233,7 @@ def _dovccmd(self, command, collectStdout=False): log.msg("Starting SVN command : svn %s" % (" ".join(command), )) d = self.runCommand(cmd) def evaluateCommand(cmd): - if cmd.rc != 0: + if cmd.didFail(): log.msg("Source step failed while running command %s" % cmd) raise buildstep.BuildStepFailed() if collectStdout: @@ -259,7 +259,7 @@ def _sourcedirIsUpdatable(self): cmd.useLog(self.stdio_log, False) yield self.runCommand(cmd) - if cmd.rc != 0: + if cmd.didFail(): defer.returnValue(False) return diff --git a/master/buildbot/steps/subunit.py b/master/buildbot/steps/subunit.py index 2de89c7de7a..f1e5749ca9d 100644 --- a/master/buildbot/steps/subunit.py +++ b/master/buildbot/steps/subunit.py @@ -81,7 +81,7 @@ def commandComplete(self, cmd): self.text2 = [text2] def evaluateCommand(self, cmd): - if cmd.rc != 0: + if cmd.didFail(): return FAILURE return self.results diff --git a/master/buildbot/steps/transfer.py b/master/buildbot/steps/transfer.py index cf9b620c038..01e49dcecd2 100644 --- a/master/buildbot/steps/transfer.py +++ b/master/buildbot/steps/transfer.py @@ -172,7 +172,7 @@ def remote_unpack(self): def makeStatusRemoteCommand(step, remote_command, args): - self = buildstep.RemoteCommand(remote_command, args) + self = buildstep.RemoteCommand(remote_command, args, successfulRC=(None, 0)) callback = lambda arg: step.step_status.addLog('stdio') self.useLogDelayed('stdio', callback, True) return self @@ -213,9 +213,9 @@ def finished(self, result): if result == SKIPPED: return BuildStep.finished(self, SKIPPED) - if self.cmd.rc is None or self.cmd.rc == 0: - return BuildStep.finished(self, SUCCESS) - return BuildStep.finished(self, FAILURE) + if self.cmd.didFail(): + return BuildStep.finished(self, FAILURE) + return BuildStep.finished(self, SUCCESS) class FileUpload(_TransferBuildStep): @@ -361,11 +361,9 @@ def finished(self, result): if result == SKIPPED: return BuildStep.finished(self, SKIPPED) - if self.cmd.rc is None or self.cmd.rc == 0: - return BuildStep.finished(self, SUCCESS) - return BuildStep.finished(self, FAILURE) - - + if self.cmd.didFail(): + return BuildStep.finished(self, FAILURE) + return BuildStep.finished(self, SUCCESS) class _FileReader(pb.Referenceable): diff --git a/master/buildbot/steps/vstudio.py b/master/buildbot/steps/vstudio.py index eea97e46f3d..592a5574195 100644 --- a/master/buildbot/steps/vstudio.py +++ b/master/buildbot/steps/vstudio.py @@ -178,7 +178,7 @@ def createSummary(self, log): self.step_status.setStatistic('errors', self.logobserver.nbErrors) def evaluateCommand(self, cmd): - if cmd.rc != 0: + if cmd.didFail(): return FAILURE if self.logobserver.nbErrors > 0: return FAILURE diff --git a/master/buildbot/test/fake/remotecommand.py b/master/buildbot/test/fake/remotecommand.py index b3d176e621d..e70f6ff6ddf 100644 --- a/master/buildbot/test/fake/remotecommand.py +++ b/master/buildbot/test/fake/remotecommand.py @@ -25,7 +25,7 @@ class FakeRemoteCommand: - def __init__(self, remote_command, args, collectStdout=False, ignore_updates=False): + def __init__(self, remote_command, args, collectStdout=False, ignore_updates=False, successfulRC=(0,)): # copy the args and set a few defaults self.remote_command = remote_command self.args = args.copy() @@ -34,6 +34,7 @@ def __init__(self, remote_command, args, collectStdout=False, ignore_updates=Fal self.rc = -999 self.collectStdout = collectStdout self.updates = {} + self.successfulRC = successfulRC if collectStdout: self.stdout = '' @@ -49,6 +50,9 @@ def run(self, step, remote): # delegate back to the test case return self.testcase._remotecommand_run(self, step, remote) + def didFail(self): + return self.rc not in self.successfulRC + class FakeRemoteShellCommand(FakeRemoteCommand): @@ -56,14 +60,15 @@ def __init__(self, workdir, command, env=None, want_stdout=1, want_stderr=1, timeout=DEFAULT_TIMEOUT, maxTime=DEFAULT_MAXTIME, logfiles={}, initialStdin=None, - usePTY=DEFAULT_USEPTY, logEnviron=True, collectStdout=False): + usePTY=DEFAULT_USEPTY, logEnviron=True, collectStdout=False, + successfulRC=(0,)): args = dict(workdir=workdir, command=command, env=env or {}, want_stdout=want_stdout, want_stderr=want_stderr, initial_stdin=initialStdin, timeout=timeout, maxTime=maxTime, logfiles=logfiles, usePTY=usePTY, logEnviron=logEnviron) FakeRemoteCommand.__init__(self, "shell", args, - collectStdout=collectStdout) + collectStdout=collectStdout, successfulRC=successfulRC) class FakeLogFile(object): diff --git a/master/buildbot/test/util/steps.py b/master/buildbot/test/util/steps.py index 404c648444c..89ba5f3e5ff 100644 --- a/master/buildbot/test/util/steps.py +++ b/master/buildbot/test/util/steps.py @@ -250,5 +250,7 @@ def _remotecommand_run(self, command, step, remote): self.assertEqual((exp.remote_command, exp.args), got) # let the Expect object show any behaviors that are required - return exp.runBehaviors(command) + d = exp.runBehaviors(command) + d.addCallback(lambda _: command) + return d diff --git a/master/docs/developer/cls-remotecommands.rst b/master/docs/developer/cls-remotecommands.rst index 00565279b31..c89bb195515 100644 --- a/master/docs/developer/cls-remotecommands.rst +++ b/master/docs/developer/cls-remotecommands.rst @@ -13,7 +13,7 @@ detail in :ref:`master-slave-updates`. RemoteCommand ~~~~~~~~~~~~~ -.. py:class:: RemoteCommand(remote_command, args, collectStdout=False, ignore_updates=False) +.. py:class:: RemoteCommand(remote_command, args, collectStdout=False, ignore_updates=False, successfulRC=tuple(0)) :param remote_command: command to run on the slave :type remote_command: string @@ -21,6 +21,7 @@ RemoteCommand :type args: dictionary :param collectStdout: if True, collect the command's stdout :param ignore_updates: true to ignore remote updates + :param successfulRC: list or tuple of ``rc`` values to treat as successes This class handles running commands, consisting of a command name and a dictionary of arguments. If true, ``ignore_updates`` will suppress any @@ -61,6 +62,12 @@ RemoteCommand slave; this may be a long time before the command itself completes, at which time the Deferred returned from :meth:`run` will fire. + .. py:method:: didFail() + + :returns: bool + + This method checks the ``rc`` against the list of successful exit statuses, and returns ``True`` if it is not in the list. + The following methods are invoked from the slave. They should not be called directly. diff --git a/master/docs/manual/cfg-buildsteps.rst b/master/docs/manual/cfg-buildsteps.rst index e197bf8dec5..ffabee89523 100644 --- a/master/docs/manual/cfg-buildsteps.rst +++ b/master/docs/manual/cfg-buildsteps.rst @@ -1496,6 +1496,10 @@ The :bb:step:`ShellCommand` arguments are: handled as a single string throughout Buildbot -- for example, do not pass the contents of a tarball with this parameter. +``successfulRC`` + This is a list or tuple of the exit codes that should be treated as successful. + The default is to treat just 0 as successful. + .. bb:step:: Configure Configure From cd3ffbf731bd94e81ebf374fe8720d5573ef59e2 Mon Sep 17 00:00:00 2001 From: David Alfonso Date: Tue, 22 May 2012 19:31:35 +0200 Subject: [PATCH 097/136] Fixes #2304 (self.branch was always empty) --- master/buildbot/steps/source/cvs.py | 1 + 1 file changed, 1 insertion(+) diff --git a/master/buildbot/steps/source/cvs.py b/master/buildbot/steps/source/cvs.py index b89da7021b8..2613607d249 100644 --- a/master/buildbot/steps/source/cvs.py +++ b/master/buildbot/steps/source/cvs.py @@ -46,6 +46,7 @@ def __init__(self, cvsroot=None, cvsmodule='', mode='incremental', Source.__init__(self, **kwargs) def startVC(self, branch, revision, patch): + self.branch = branch self.revision = revision self.stdio_log = self.addLog("stdio") self.method = self._getMethod() From dc9c28b1f356248e99386caac0cd0a8b9bb9f8f8 Mon Sep 17 00:00:00 2001 From: ewongbb Date: Thu, 24 May 2012 22:49:15 +0800 Subject: [PATCH 098/136] Update master/docs/tutorial/tour.rst --- master/docs/tutorial/tour.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/master/docs/tutorial/tour.rst b/master/docs/tutorial/tour.rst index 672406ec95f..d715253c379 100644 --- a/master/docs/tutorial/tour.rst +++ b/master/docs/tutorial/tour.rst @@ -278,7 +278,7 @@ After restarting the master, you can ssh into the master and get an interactive If you see this, the temporary solution is to install the previous version of pyasn1:: - pip instasll pyasn1-0.0.13b + pip install pyasn1-0.0.13b If you wanted to check which slaves are connected and what builders those slaves are assigned to you could do:: From 975126d9e3ef71d9f0a7f086fbe68c8059b1c9af Mon Sep 17 00:00:00 2001 From: Jorge Gonzalez Date: Tue, 17 Apr 2012 12:25:16 -0700 Subject: [PATCH 099/136] Add -j option tree syncs using Repo source step Change-Id: I478dbdcbae9b7277d920ab7abc28328f18535ff2 --- master/buildbot/steps/source/oldsource.py | 4 +++- master/docs/manual/cfg-buildsteps.rst | 4 ++++ slave/buildslave/commands/repo.py | 5 +++++ 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/master/buildbot/steps/source/oldsource.py b/master/buildbot/steps/source/oldsource.py index 46499ccbd33..1de0a713497 100644 --- a/master/buildbot/steps/source/oldsource.py +++ b/master/buildbot/steps/source/oldsource.py @@ -592,6 +592,7 @@ def __init__(self, manifest_branch="master", manifest_file="default.xml", tarball=None, + jobs=None, **kwargs): """ @type manifest_url: string @@ -609,7 +610,8 @@ def __init__(self, self.args.update({'manifest_branch': manifest_branch, 'manifest_file': manifest_file, 'tarball': tarball, - 'manifest_override_url': None + 'manifest_override_url': None, + 'jobs': jobs }) def computeSourceRevision(self, changes): diff --git a/master/docs/manual/cfg-buildsteps.rst b/master/docs/manual/cfg-buildsteps.rst index c00c4cab150..b9b5f0aa509 100644 --- a/master/docs/manual/cfg-buildsteps.rst +++ b/master/docs/manual/cfg-buildsteps.rst @@ -1209,6 +1209,10 @@ The Repo step takes the following arguments: directory which contains all the git objects. This feature helps to minimize network usage on very big projects. +``jobs`` + (optional, defaults to ``None``): Number of projects to fetch + simultaneously while syncing. Passed to repo sync subcommand with "-j". + This Source step integrates with :bb:chsrc:`GerritChangeSource`, and will automatically use the :command:`repo download` command of repo to download the additionnal changes introduced by a pending changeset. diff --git a/slave/buildslave/commands/repo.py b/slave/buildslave/commands/repo.py index abc94a3a598..a2fb6a87f76 100644 --- a/slave/buildslave/commands/repo.py +++ b/slave/buildslave/commands/repo.py @@ -37,6 +37,8 @@ class Repo(SourceBaseCommand): ['tarball'] (optional): The tarball base to accelerate the fetch. ['repo_downloads'] (optional): Repo downloads to do. Computer from GerritChangeSource and forced build properties. + ['jobs'] (optional): number of connections to run in parallel + repo tool will use while syncing """ header = "repo operation" @@ -52,6 +54,7 @@ def setup(self, args): # we're using string instead of an array here, because it will be transferred back # to the master as string anyway and using eval() could have security implications. self.repo_downloaded = "" + self.jobs = args.get('jobs') self.sourcedata = "%s %s" % (self.manifest_url, self.manifest_file) self.re_change = re.compile(".* refs/changes/\d\d/(\d+)/(\d+) -> FETCH_HEAD$") @@ -165,6 +168,8 @@ def _doSync(self, dummy): if self.manifest_override_url: os.system("cd %s/.repo; ln -sf ../manifest_override.xml manifest.xml"%(self._fullSrcdir())) command = ['sync'] + if self.jobs: + command.append('-j' + str(self.jobs)) self.sendStatus({"header": "synching manifest %s from branch %s from %s\n" % (self.manifest_file, self.manifest_branch, self.manifest_url)}) return self._repoCmd(command, self._didSync) From 98649f7cc2de7fedb772210f484c3e6c33d64618 Mon Sep 17 00:00:00 2001 From: David Alfonso Date: Fri, 25 May 2012 00:05:08 +0200 Subject: [PATCH 100/136] Added four tests for branches. Two for when the branch is specified on the CVS step and two for when the branch comes from a sourcestamp (e.g. using a timed.Nightly with a specific branch). --- .../test/unit/test_steps_source_cvs.py | 115 ++++++++++++++++++ 1 file changed, 115 insertions(+) diff --git a/master/buildbot/test/unit/test_steps_source_cvs.py b/master/buildbot/test/unit/test_steps_source_cvs.py index e6977e907ae..29f3ae3c536 100644 --- a/master/buildbot/test/unit/test_steps_source_cvs.py +++ b/master/buildbot/test/unit/test_steps_source_cvs.py @@ -65,6 +65,66 @@ def test_mode_full_clean(self): self.expectOutcome(result=SUCCESS, status_text=["update"]) return self.runStep() + def test_mode_full_clean_branch(self): + self.setupStep( + cvs.CVS(cvsroot=":pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot", + cvsmodule="mozilla/browser/", mode='full', method='clean', + branch='branch', login=True)) + self.expectCommands( + ExpectShell(workdir='wkdir', + command=['cvs', '--version']) + + 0, + Expect('uploadFile', dict(blocksize=32768, maxsize=None, + slavesrc='Root', workdir='wkdir/CVS', + writer=ExpectRemoteRef(shell.StringFileWriter))) + + Expect.behavior(uploadString(':pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot')) + + 0, + Expect('uploadFile', dict(blocksize=32768, maxsize=None, + slavesrc='Repository', workdir='wkdir/CVS', + writer=ExpectRemoteRef(shell.StringFileWriter))) + + Expect.behavior(uploadString('mozilla/browser/')) + + 0, + ExpectShell(workdir='wkdir', + command=['cvsdiscard']) + + 0, + ExpectShell(workdir='wkdir', + command=['cvs', '-z3', 'update', '-dP', '-r', 'branch']) + + 0, + ) + + self.expectOutcome(result=SUCCESS, status_text=["update"]) + return self.runStep() + + def test_mode_full_clean_branch_sourcestamp(self): + self.setupStep( + cvs.CVS(cvsroot=":pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot", + cvsmodule="mozilla/browser/", mode='full', method='clean', + login=True), args={'branch':'my_branch'}) + self.expectCommands( + ExpectShell(workdir='wkdir', + command=['cvs', '--version']) + + 0, + Expect('uploadFile', dict(blocksize=32768, maxsize=None, + slavesrc='Root', workdir='wkdir/CVS', + writer=ExpectRemoteRef(shell.StringFileWriter))) + + Expect.behavior(uploadString(':pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot')) + + 0, + Expect('uploadFile', dict(blocksize=32768, maxsize=None, + slavesrc='Repository', workdir='wkdir/CVS', + writer=ExpectRemoteRef(shell.StringFileWriter))) + + Expect.behavior(uploadString('mozilla/browser/')) + + 0, + ExpectShell(workdir='wkdir', + command=['cvsdiscard']) + + 0, + ExpectShell(workdir='wkdir', + command=['cvs', '-z3', 'update', '-dP', '-r', 'my_branch']) + + 0, + ) + + self.expectOutcome(result=SUCCESS, status_text=["update"]) + return self.runStep() + def test_mode_full_fresh(self): self.setupStep( cvs.CVS(cvsroot=":pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot", @@ -213,6 +273,61 @@ def test_mode_incremental(self): self.expectOutcome(result=SUCCESS, status_text=["update"]) return self.runStep() + def test_mode_incremental_branch(self): + self.setupStep( + cvs.CVS(cvsroot=":pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot", + cvsmodule="mozilla/browser/", mode='incremental', + branch='my_branch', login=True)) + self.expectCommands( + ExpectShell(workdir='wkdir', + command=['cvs', '--version']) + + 0, + Expect('uploadFile', dict(blocksize=32768, maxsize=None, + slavesrc='Root', workdir='wkdir/CVS', + writer=ExpectRemoteRef(shell.StringFileWriter))) + + Expect.behavior(uploadString(':pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot')) + + 0, + Expect('uploadFile', dict(blocksize=32768, maxsize=None, + slavesrc='Repository', workdir='wkdir/CVS', + writer=ExpectRemoteRef(shell.StringFileWriter))) + + Expect.behavior(uploadString('mozilla/browser/')) + + 0, + ExpectShell(workdir='wkdir', + command=['cvs', '-z3', 'update', '-dP', '-r', 'my_branch']) + + 0, + ) + + self.expectOutcome(result=SUCCESS, status_text=["update"]) + return self.runStep() + + def test_mode_incremental_branch_sourcestamp(self): + self.setupStep( + cvs.CVS(cvsroot=":pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot", + cvsmodule="mozilla/browser/", mode='incremental', + login=True), args={'branch':'my_branch'}) + self.expectCommands( + ExpectShell(workdir='wkdir', + command=['cvs', '--version']) + + 0, + Expect('uploadFile', dict(blocksize=32768, maxsize=None, + slavesrc='Root', workdir='wkdir/CVS', + writer=ExpectRemoteRef(shell.StringFileWriter))) + + Expect.behavior(uploadString(':pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot')) + + 0, + Expect('uploadFile', dict(blocksize=32768, maxsize=None, + slavesrc='Repository', workdir='wkdir/CVS', + writer=ExpectRemoteRef(shell.StringFileWriter))) + + Expect.behavior(uploadString('mozilla/browser/')) + + 0, + ExpectShell(workdir='wkdir', + command=['cvs', '-z3', 'update', '-dP', '-r', 'my_branch']) + + 0, + ) + + self.expectOutcome(result=SUCCESS, status_text=["update"]) + return self.runStep() + + def test_mode_incremental_not_loggedin(self): self.setupStep( cvs.CVS(cvsroot=":pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot", From 9fc970bf20a2146810286cb042d5bdd66e788f8d Mon Sep 17 00:00:00 2001 From: Bryce Lelbach Date: Fri, 25 May 2012 02:10:40 +0000 Subject: [PATCH 101/136] Add missing files to setup.py so that they get installed. --- master/setup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/master/setup.py b/master/setup.py index 0cbc75bc1a7..7896d6788c3 100755 --- a/master/setup.py +++ b/master/setup.py @@ -154,12 +154,14 @@ def make_release_tree(self, base_dir, files): "buildbot/status/web/files/default.css", "buildbot/status/web/files/bg_gradient.jpg", "buildbot/status/web/files/robots.txt", + "buildbot/status/web/files/templates_readme.txt", "buildbot/status/web/files/favicon.ico", ]), include("buildbot/status/web/templates", '*.html'), include("buildbot/status/web/templates", '*.xml'), ("buildbot/scripts", [ "buildbot/scripts/sample.cfg", + "buildbot/scripts/buildbot_tac.tmpl", ]), ], 'scripts': scripts, From 27bf15c64006b271a1f21fda64628239abfae58e Mon Sep 17 00:00:00 2001 From: "Dustin J. Mitchell" Date: Sat, 26 May 2012 23:55:31 -0500 Subject: [PATCH 102/136] Handle exceptions in hideStepIf Fixes #2305. --- master/buildbot/process/buildstep.py | 14 +++++++++++--- .../buildbot/test/unit/test_process_buildstep.py | 9 +++++++-- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/master/buildbot/process/buildstep.py b/master/buildbot/process/buildstep.py index 23733093a6a..04c5b35b4a9 100644 --- a/master/buildbot/process/buildstep.py +++ b/master/buildbot/process/buildstep.py @@ -640,11 +640,19 @@ def _finishFinished(self, results): # from finished() so that subclasses can override finished() if self.progress: self.progress.finish() + + try: + hidden = self._maybeEvaluate(self.hideStepIf, results, self) + except Exception: + why = Failure() + self.addHTMLLog("err.html", formatFailure(why)) + self.addCompleteLog("err.text", why.getTraceback()) + results = EXCEPTION + hidden = False + self.step_status.stepFinished(results) - - hidden = self._maybeEvaluate(self.hideStepIf, results, self) self.step_status.setHidden(hidden) - + self.releaseLocks() self.deferred.callback(results) diff --git a/master/buildbot/test/unit/test_process_buildstep.py b/master/buildbot/test/unit/test_process_buildstep.py index a7e66464dea..18eec13c42e 100644 --- a/master/buildbot/test/unit/test_process_buildstep.py +++ b/master/buildbot/test/unit/test_process_buildstep.py @@ -85,9 +85,9 @@ def tearDown(self): # support - def _setupWaterfallTest(self, hideStepIf, expect): + def _setupWaterfallTest(self, hideStepIf, expect, expectedResult=SUCCESS): self.setupStep(TestBuildStep.FakeBuildStep(hideStepIf=hideStepIf)) - self.expectOutcome(result=SUCCESS, status_text=["generic"]) + self.expectOutcome(result=expectedResult, status_text=["generic"]) self.expectHidden(expect) # tests @@ -146,6 +146,11 @@ def shouldHide(result, step): d.addCallback(lambda _ : self.assertTrue(called[0])) return d + def test_hideStepIf_fails(self): + # 0/0 causes DivideByZeroError, which should be flagged as an exception + self._setupWaterfallTest(lambda : 0/0, False, expectedResult=EXCEPTION) + return self.runStep() + @compat.usesFlushLoggedErrors def test_hideStepIf_Callable_Exception(self): called = [False] From cc5b2ed5a952250846d98b06617dbf346bbc9998 Mon Sep 17 00:00:00 2001 From: "Dustin J. Mitchell" Date: Sun, 27 May 2012 11:53:05 -0500 Subject: [PATCH 103/136] Don't reset webstatus.master to None when it is replaced This is an awful hack, but re-creating WebStatus on every reconfig is also a bad idea. Fixes #2301. --- master/buildbot/status/master.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/master/buildbot/status/master.py b/master/buildbot/status/master.py index e26138a9600..a7c62c11a25 100644 --- a/master/buildbot/status/master.py +++ b/master/buildbot/status/master.py @@ -69,7 +69,15 @@ def reconfigService(self, new_config): for sr in list(self): yield defer.maybeDeferred(lambda : sr.disownServiceParent()) - sr.master = None + + # WebStatus instances tend to "hang around" longer than we'd like - + # if there's an ongoing HTTP request, or even a connection held + # open by keepalive, then users may still be talking to an old + # WebStatus. So WebStatus objects get to keep their `master` + # attribute, but all other status objects lose theirs. And we want + # to test this without importing WebStatus, so we use name + if not sr.__class__.__name__.endswith('WebStatus'): + sr.master = None for sr in new_config.status: sr.master = self.master From 374e8c7dd9d248d6e596e4a7db8145a5f82459bc Mon Sep 17 00:00:00 2001 From: "Dustin J. Mitchell" Date: Sun, 27 May 2012 12:50:01 -0500 Subject: [PATCH 104/136] Make FakeMaster a full class, not a Mock This is a compromise between adding the required methods (and other classes) to fakemaster.py, and mocking out the required methods/class in the tests, or using a real method/class where appropriate. Fixes #2302 --- master/buildbot/process/builder.py | 18 ++++---- master/buildbot/test/fake/fakemaster.py | 46 +++++++++++++++++-- .../test/integration/test_slave_comm.py | 3 ++ .../test/unit/test_process_builder.py | 13 ++++-- .../buildbot/test/unit/test_steps_trigger.py | 3 +- 5 files changed, 64 insertions(+), 19 deletions(-) diff --git a/master/buildbot/process/builder.py b/master/buildbot/process/builder.py index 066e88378c4..1a48327a05f 100644 --- a/master/buildbot/process/builder.py +++ b/master/buildbot/process/builder.py @@ -692,21 +692,21 @@ def _breakBrdictRefloops(self, requests): class BuilderControl: implements(interfaces.IBuilderControl) - def __init__(self, builder, master): + def __init__(self, builder, control): self.original = builder - self.master = master + self.control = control def submitBuildRequest(self, ss, reason, props=None): - d = ss.getSourceStampSetId(self.master.master) + d = ss.getSourceStampSetId(self.control.master) def add_buildset(sourcestampsetid): - return self.master.master.addBuildset( + return self.control.master.addBuildset( builderNames=[self.original.name], sourcestampsetid=sourcestampsetid, reason=reason, properties=props) d.addCallback(add_buildset) def get_brs((bsid,brids)): brs = BuildRequestStatus(self.original.name, brids[self.original.name], - self.master.master.status) + self.control.master.status) return brs d.addCallback(get_brs) return d @@ -727,14 +727,14 @@ def rebuildBuild(self, bs, reason="", extraProperties= ssList = bs.getSourceStamps(absolute=True) if ssList: - sourcestampsetid = yield ssList[0].getSourceStampSetId(self.master.master) + sourcestampsetid = yield ssList[0].getSourceStampSetId(self.control.master) dl = [] for ss in ssList[1:]: # add defered to the list - dl.append(ss.addSourceStampToDatabase(self.master.master, sourcestampsetid)) + dl.append(ss.addSourceStampToDatabase(self.control.master, sourcestampsetid)) yield defer.gatherResults(dl) - bsid, brids = yield self.master.master.addBuildset( + bsid, brids = yield self.control.master.addBuildset( builderNames=[self.original.name], sourcestampsetid=sourcestampsetid, reason=reason, @@ -755,7 +755,7 @@ def getPendingBuildRequestControls(self): buildrequests = [ ] for brdict in brdicts: br = yield buildrequest.BuildRequest.fromBrdict( - self.master.master, brdict) + self.control.master, brdict) buildrequests.append(br) # and return the corresponding control objects diff --git a/master/buildbot/test/fake/fakemaster.py b/master/buildbot/test/fake/fakemaster.py index 4494513912c..2c0a7be5c51 100644 --- a/master/buildbot/test/fake/fakemaster.py +++ b/master/buildbot/test/fake/fakemaster.py @@ -16,7 +16,7 @@ import weakref from twisted.internet import defer from buildbot.test.fake import fakedb -from buildbot.test.fake.pbmanager import FakePBManager +from buildbot.test.fake import pbmanager from buildbot import config import mock @@ -37,7 +37,36 @@ def mkref(x): return d -class FakeMaster(mock.Mock): +class FakeCaches(object): + + def get_cache(self, name, miss_fn): + return FakeCache(name, miss_fn) + + +class FakeBotMaster(object): + + pass + + +class FakeStatus(object): + + def builderAdded(self, name, basedir, category=None): + return FakeBuilderStatus() + + +class FakeBuilderStatus(object): + + def setSlavenames(self, names): + pass + + def setCacheSize(self, size): + pass + + def setBigState(self, state): + pass + + +class FakeMaster(object): """ Create a fake Master instance: a Mock with some convenience implementations: @@ -46,15 +75,22 @@ class FakeMaster(mock.Mock): """ def __init__(self, master_id=fakedb.FakeBuildRequestsComponent.MASTER_ID): - mock.Mock.__init__(self, name="fakemaster") self._master_id = master_id self.config = config.MasterConfig() - self.caches.get_cache = FakeCache - self.pbmanager = FakePBManager() + self.caches = FakeCaches() + self.pbmanager = pbmanager.FakePBManager() + self.basedir = 'basedir' + self.botmaster = FakeBotMaster() + self.botmaster.parent = self + self.status = FakeStatus() + self.status.master = self def getObjectId(self): return defer.succeed(self._master_id) + def subscribeToBuildRequests(self, callback): + pass + # work around http://code.google.com/p/mock/issues/detail?id=105 def _get_child_mock(self, **kw): return mock.Mock(**kw) diff --git a/master/buildbot/test/integration/test_slave_comm.py b/master/buildbot/test/integration/test_slave_comm.py index 4ebc451190b..9de9035bfb6 100644 --- a/master/buildbot/test/integration/test_slave_comm.py +++ b/master/buildbot/test/integration/test_slave_comm.py @@ -23,6 +23,7 @@ from buildbot.test.util import compat from buildbot.process import botmaster, builder from buildbot import pbmanager, buildslave, config +from buildbot.status import master from buildbot.test.fake import fakemaster class FakeSlaveBuilder(pb.Referenceable): @@ -115,6 +116,8 @@ def setUp(self): self.botmaster = botmaster.BotMaster(self.master) self.botmaster.startService() + self.master.status = master.Status(self.master) + self.buildslave = None self.port = None self.slavebuildslave = None diff --git a/master/buildbot/test/unit/test_process_builder.py b/master/buildbot/test/unit/test_process_builder.py index 552007aa451..00823b741a8 100644 --- a/master/buildbot/test/unit/test_process_builder.py +++ b/master/buildbot/test/unit/test_process_builder.py @@ -19,6 +19,7 @@ from twisted.python import failure from twisted.internet import defer from buildbot import config +from buildbot.status import master from buildbot.test.fake import fakedb, fakemaster from buildbot.process import builder from buildbot.db import buildrequests @@ -705,6 +706,7 @@ def makeBuilder(self, name): self.bstatus = mock.Mock() self.factory = mock.Mock() self.master = fakemaster.make_master() + self.master.status = master.Status(self.master) # only include the necessary required config builder_config = config.BuilderConfig( name=name, slavename="slv", builddir="bdir", @@ -754,7 +756,8 @@ def makeBuilder(self, name, sourcestamps): self.bldr = builder.Builder(builder_config.name) self.master.db = self.db = fakedb.FakeDBConnector(self) self.bldr.master = self.master - self.bldr.master.master.addBuildset.return_value = (1, [100]) + self.master.addBuildset = addBuildset = mock.Mock() + addBuildset.return_value = (1, [100]) def do_test_rebuild(self, sourcestampsetid, @@ -778,7 +781,9 @@ def getSourceStampSetId(master): sslist.append(ssx) self.makeBuilder(name='bldr1', sourcestamps = sslist) - self.bldrctrl = builder.BuilderControl(self.bldr, self.master) + control = mock.Mock(spec=['master']) + control.master = self.master + self.bldrctrl = builder.BuilderControl(self.bldr, control) d = self.bldrctrl.rebuildBuild(self.bstatus, reason = 'unit test', extraProperties = {}) @@ -793,7 +798,7 @@ def test_rebuild_with_no_sourcestamps(self): def test_rebuild_with_single_sourcestamp(self): yield self.do_test_rebuild(101, 1) self.assertEqual(self.sslist, {1:101}) - self.master.master.addBuildset.assert_called_with(builderNames=['bldr1'], + self.master.addBuildset.assert_called_with(builderNames=['bldr1'], sourcestampsetid=101, reason = 'unit test', properties = {}) @@ -803,7 +808,7 @@ def test_rebuild_with_single_sourcestamp(self): def test_rebuild_with_multiple_sourcestamp(self): yield self.do_test_rebuild(101, 3) self.assertEqual(self.sslist, {1:101, 2:101, 3:101}) - self.master.master.addBuildset.assert_called_with(builderNames=['bldr1'], + self.master.addBuildset.assert_called_with(builderNames=['bldr1'], sourcestampsetid=101, reason = 'unit test', properties = {}) diff --git a/master/buildbot/test/unit/test_steps_trigger.py b/master/buildbot/test/unit/test_steps_trigger.py index 7cd08a3abb5..810c8887ed7 100644 --- a/master/buildbot/test/unit/test_steps_trigger.py +++ b/master/buildbot/test/unit/test_steps_trigger.py @@ -81,7 +81,8 @@ def setupStep(self, *args, **kwargs): # should be fixed! # set up a buildmaster that knows about two fake schedulers, a and b - self.build.builder.botmaster.parent = m = fakemaster.make_master() + m = fakemaster.make_master() + self.build.builder.botmaster = m.botmaster m.db = fakedb.FakeDBConnector(self) m.status = master.Status(m) m.config.buildbotURL = "baseurl/" From 94ccf27b379a68a247ded9090fbb807879442a61 Mon Sep 17 00:00:00 2001 From: "Dustin J. Mitchell" Date: Sun, 27 May 2012 13:44:07 -0500 Subject: [PATCH 105/136] Remove local 'FakeCmd' fake, add more interface tests for FakeRemoteCommand Refs #2225 --- master/buildbot/test/fake/remotecommand.py | 44 ++++++++----- .../test/interfaces/test_remotecommand.py | 63 +++++++++++++------ .../test/unit/test_process_buildstep.py | 55 ++++++++++------ 3 files changed, 111 insertions(+), 51 deletions(-) diff --git a/master/buildbot/test/fake/remotecommand.py b/master/buildbot/test/fake/remotecommand.py index e70f6ff6ddf..51861047f78 100644 --- a/master/buildbot/test/fake/remotecommand.py +++ b/master/buildbot/test/fake/remotecommand.py @@ -19,13 +19,13 @@ from cStringIO import StringIO -DEFAULT_TIMEOUT="DEFAULT_TIMEOUT" -DEFAULT_MAXTIME="DEFAULT_MAXTIME" -DEFAULT_USEPTY="DEFAULT_USEPTY" +class FakeRemoteCommand(object): -class FakeRemoteCommand: + # callers should set this to the running TestCase instance + testcase = None - def __init__(self, remote_command, args, collectStdout=False, ignore_updates=False, successfulRC=(0,)): + def __init__(self, remote_command, args, + ignore_updates=False, collectStdout=False, successfulRC=(0,)): # copy the args and set a few defaults self.remote_command = remote_command self.args = args.copy() @@ -38,6 +38,10 @@ def __init__(self, remote_command, args, collectStdout=False, ignore_updates=Fal if collectStdout: self.stdout = '' + def run(self, step, remote): + # delegate back to the test case + return self.testcase._remotecommand_run(self, step, remote) + def useLog(self, log, closeWhenFinished=False, logfileName=None): if not logfileName: logfileName = log.getName() @@ -46,22 +50,24 @@ def useLog(self, log, closeWhenFinished=False, logfileName=None): def useLogDelayed(self, logfileName, activateCallBack, closeWhenFinished=False): self.delayedLogs[logfileName] = (activateCallBack, closeWhenFinished) - def run(self, step, remote): - # delegate back to the test case - return self.testcase._remotecommand_run(self, step, remote) + def interrupt(self, why): + raise NotImplementedError def didFail(self): return self.rc not in self.successfulRC + def fakeLogData(self, step, log, header='', stdout='', stderr=''): + self.logs[log] = l = FakeLogFile(log, step) + l.fakeData(header=header, stdout=stdout, stderr=stderr) + class FakeRemoteShellCommand(FakeRemoteCommand): def __init__(self, workdir, command, env=None, want_stdout=1, want_stderr=1, - timeout=DEFAULT_TIMEOUT, maxTime=DEFAULT_MAXTIME, logfiles={}, - initialStdin=None, - usePTY=DEFAULT_USEPTY, logEnviron=True, collectStdout=False, - successfulRC=(0,)): + timeout=20*60, maxTime=None, logfiles={}, + usePTY="slave-config", logEnviron=True, collectStdout=False, + interruptSignal=None, initialStdin=None, successfulRC=(0,)): args = dict(workdir=workdir, command=command, env=env or {}, want_stdout=want_stdout, want_stderr=want_stderr, initial_stdin=initialStdin, @@ -123,6 +129,16 @@ def getChunks(self, channels=[], onlyText=False): def finish(self): pass + def fakeData(self, header='', stdout='', stderr=''): + if header: + self.header += header + self.chunks.append((HEADER, header)) + if stdout: + self.stdout += stdout + self.chunks.append((STDOUT, stdout)) + if stderr: + self.stderr += stderr + self.chunks.append((STDERR, stderr)) class ExpectRemoteRef(object): """ @@ -245,8 +261,8 @@ class ExpectShell(Expect): """ def __init__(self, workdir, command, env={}, want_stdout=1, want_stderr=1, initialStdin=None, - timeout=DEFAULT_TIMEOUT, maxTime=DEFAULT_MAXTIME, logfiles={}, - usePTY=DEFAULT_USEPTY, logEnviron=True): + timeout=20*60, maxTime=None, logfiles={}, + usePTY="slave-config", logEnviron=True): args = dict(workdir=workdir, command=command, env=env, want_stdout=want_stdout, want_stderr=want_stderr, initial_stdin=initialStdin, diff --git a/master/buildbot/test/interfaces/test_remotecommand.py b/master/buildbot/test/interfaces/test_remotecommand.py index 72b863f036b..de2d8e998d7 100644 --- a/master/buildbot/test/interfaces/test_remotecommand.py +++ b/master/buildbot/test/interfaces/test_remotecommand.py @@ -25,41 +25,68 @@ class Tests(interfaces.InterfaceTests): - def makeRemoteCommand(self, name, args): - raise NotImplementedError + remoteCommandClass = None + + def makeRemoteCommand(self): + return self.remoteCommandClass('ping', {'arg':'val'}) + + def test_signature_RemoteCommand_constructor(self): + @self.assertArgSpecMatches(self.remoteCommandClass.__init__) + def __init__(self, remote_command, args, ignore_updates=False, + collectStdout=False, successfulRC=(0,)): + pass + + def test_signature_RemoteShellCommand_constructor(self): + @self.assertArgSpecMatches(self.remoteShellCommandClass.__init__) + def __init__(self, workdir, command, env=None, want_stdout=1, + want_stderr=1, timeout=20*60, maxTime=None, logfiles={}, + usePTY="slave-config", logEnviron=True, collectStdout=False, + interruptSignal=None, initialStdin=None, successfulRC=(0,)): + pass + + def test_signature_run(self): + cmd = self.makeRemoteCommand() + @self.assertArgSpecMatches(cmd.run) + def run(self, step, remote): + pass def test_signature_useLog(self): - rc = self.makeRemoteCommand('ping', {'arg':'val'}) - @self.assertArgSpecMatches(rc.useLog) + cmd = self.makeRemoteCommand() + @self.assertArgSpecMatches(cmd.useLog) def useLog(self, log, closeWhenFinished=False, logfileName=None): pass def test_signature_useLogDelayed(self): - rc = self.makeRemoteCommand('ping', {'arg':'val'}) - @self.assertArgSpecMatches(rc.useLogDelayed) + cmd = self.makeRemoteCommand() + @self.assertArgSpecMatches(cmd.useLogDelayed) def useLogDelayed(self, logfileName, activateCallBack, closeWhenFinished=False): pass - def test_signature_run(self): - rc = self.makeRemoteCommand('ping', {'arg':'val'}) - @self.assertArgSpecMatches(rc.run) - def run(self, step, remote): + def test_signature_interrupt(self): + cmd = self.makeRemoteCommand() + @self.assertArgSpecMatches(cmd.interrupt) + def useLogDelayed(self, why): pass + def test_signature_didFail(self): + cmd = self.makeRemoteCommand() + @self.assertArgSpecMatches(cmd.didFail) + def useLogDelayed(self): + pass -class RealTests(Tests): - pass + def test_signature_logs(self): + cmd = self.makeRemoteCommand() + self.assertIsInstance(cmd.logs, dict) -class TestRunCommand(unittest.TestCase, RealTests): +class TestRunCommand(unittest.TestCase, Tests): - def makeRemoteCommand(self, name, args): - return buildstep.RemoteCommand(name, args) + remoteCommandClass = buildstep.RemoteCommand + remoteShellCommandClass = buildstep.RemoteShellCommand class TestFakeRunCommand(unittest.TestCase, Tests): - def makeRemoteCommand(self, name, args): - return remotecommand.FakeRemoteCommand(name, args) - + remoteCommandClass = remotecommand.FakeRemoteCommand + remoteShellCommandClass = remotecommand.FakeRemoteShellCommand diff --git a/master/buildbot/test/unit/test_process_buildstep.py b/master/buildbot/test/unit/test_process_buildstep.py index 18eec13c42e..83c946413e1 100644 --- a/master/buildbot/test/unit/test_process_buildstep.py +++ b/master/buildbot/test/unit/test_process_buildstep.py @@ -21,7 +21,7 @@ from buildbot.process import buildstep from buildbot.process.buildstep import regex_log_evaluator from buildbot.status.results import FAILURE, SUCCESS, WARNINGS, EXCEPTION -from buildbot.test.fake import fakebuild +from buildbot.test.fake import fakebuild, remotecommand from buildbot.test.util import steps, compat class FakeLogFile: @@ -31,44 +31,54 @@ def __init__(self, text): def getText(self): return self.text -class FakeCmd: - def __init__(self, stdout, stderr, rc=0): - self.logs = {'stdout': FakeLogFile(stdout), - 'stderr': FakeLogFile(stderr)} - self.rc = rc - class FakeStepStatus: pass class TestRegexLogEvaluator(unittest.TestCase): + + def makeRemoteCommand(self, rc, stdout, stderr=''): + cmd = remotecommand.FakeRemoteCommand('cmd', {}) + cmd.fakeLogData(self, 'stdio', stdout=stdout, stderr=stderr) + cmd.rc = rc + return cmd + def test_find_worse_status(self): - cmd = FakeCmd("This is log text", "") + cmd = self.makeRemoteCommand(0, 'This is a big step') step_status = FakeStepStatus() - r = [(re.compile("This is"), FAILURE)] + r = [(re.compile("This is"), WARNINGS)] new_status = regex_log_evaluator(cmd, step_status, r) - self.assertEqual(new_status, FAILURE, "regex_log_evaluator returned %d, should've returned %d" % (new_status, FAILURE)) + self.assertEqual(new_status, WARNINGS, + "regex_log_evaluator returned %d, expected %d" + % (new_status, WARNINGS)) def test_multiple_regexes(self): - cmd = FakeCmd("Normal stdout text\nan error", "") + cmd = self.makeRemoteCommand(0, "Normal stdout text\nan error") step_status = FakeStepStatus() r = [(re.compile("Normal stdout"), SUCCESS), (re.compile("error"), FAILURE)] new_status = regex_log_evaluator(cmd, step_status, r) - self.assertEqual(new_status, FAILURE, "regex_log_evaluator returned %d, should've returned %d" % (new_status, FAILURE)) + self.assertEqual(new_status, FAILURE, + "regex_log_evaluator returned %d, expected %d" + % (new_status, FAILURE)) def test_exception_not_in_stdout(self): - cmd = FakeCmd("Completely normal output", "exception output") + cmd = self.makeRemoteCommand(0, + "Completely normal output", "exception output") step_status = FakeStepStatus() r = [(re.compile("exception"), EXCEPTION)] new_status = regex_log_evaluator(cmd, step_status, r) - self.assertEqual(new_status, EXCEPTION, "regex_log_evaluator returned %d, should've returned %d" % (new_status, EXCEPTION)) + self.assertEqual(new_status, EXCEPTION, + "regex_log_evaluator returned %d, expected %d" + % (new_status, EXCEPTION)) def test_pass_a_string(self): - cmd = FakeCmd("Output", "Some weird stuff on stderr") + cmd = self.makeRemoteCommand(0, "Output", "Some weird stuff on stderr") step_status = FakeStepStatus() r = [("weird stuff", WARNINGS)] new_status = regex_log_evaluator(cmd, step_status, r) - self.assertEqual(new_status, WARNINGS, "regex_log_evaluator returned %d, should've returned %d" % (new_status, WARNINGS)) + self.assertEqual(new_status, WARNINGS, + "regex_log_evaluator returned %d, expected %d" + % (new_status, WARNINGS)) class TestBuildStep(steps.BuildStepMixin, unittest.TestCase): @@ -180,20 +190,27 @@ def createException(*args, **kwargs): class TestLoggingBuildStep(unittest.TestCase): + + def makeRemoteCommand(self, rc, stdout, stderr=''): + cmd = remotecommand.FakeRemoteCommand('cmd', {}) + cmd.fakeLogData(self, 'stdio', stdout=stdout, stderr=stderr) + cmd.rc = rc + return cmd + def test_evaluateCommand_success(self): - cmd = FakeCmd("Log text", "Log text") + cmd = self.makeRemoteCommand(0, "Log text", "Log text") lbs = buildstep.LoggingBuildStep() status = lbs.evaluateCommand(cmd) self.assertEqual(status, SUCCESS, "evaluateCommand returned %d, should've returned %d" % (status, SUCCESS)) def test_evaluateCommand_failed(self): - cmd = FakeCmd("Log text", "", 23) + cmd = self.makeRemoteCommand(23, "Log text", "") lbs = buildstep.LoggingBuildStep() status = lbs.evaluateCommand(cmd) self.assertEqual(status, FAILURE, "evaluateCommand returned %d, should've returned %d" % (status, FAILURE)) def test_evaluateCommand_log_eval_func(self): - cmd = FakeCmd("Log text", "") + cmd = self.makeRemoteCommand(0, "Log text") def eval(cmd, step_status): return WARNINGS lbs = buildstep.LoggingBuildStep(log_eval_func=eval) From 7eb6257b501ff8c246ac8c5eaa30f15a45e270d8 Mon Sep 17 00:00:00 2001 From: "Dustin J. Mitchell" Date: Sun, 27 May 2012 13:47:39 -0500 Subject: [PATCH 106/136] add and test for an active attribute --- master/buildbot/test/fake/remotecommand.py | 2 ++ master/buildbot/test/interfaces/test_remotecommand.py | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/master/buildbot/test/fake/remotecommand.py b/master/buildbot/test/fake/remotecommand.py index 51861047f78..aee6594fc33 100644 --- a/master/buildbot/test/fake/remotecommand.py +++ b/master/buildbot/test/fake/remotecommand.py @@ -24,6 +24,8 @@ class FakeRemoteCommand(object): # callers should set this to the running TestCase instance testcase = None + active = False + def __init__(self, remote_command, args, ignore_updates=False, collectStdout=False, successfulRC=(0,)): # copy the args and set a few defaults diff --git a/master/buildbot/test/interfaces/test_remotecommand.py b/master/buildbot/test/interfaces/test_remotecommand.py index de2d8e998d7..f7bda5640e7 100644 --- a/master/buildbot/test/interfaces/test_remotecommand.py +++ b/master/buildbot/test/interfaces/test_remotecommand.py @@ -79,6 +79,10 @@ def test_signature_logs(self): cmd = self.makeRemoteCommand() self.assertIsInstance(cmd.logs, dict) + def test_signature_active(self): + cmd = self.makeRemoteCommand() + self.assertIsInstance(cmd.active, bool) + class TestRunCommand(unittest.TestCase, Tests): From 935399f5a1b6a8f826fdcaca9058081b7af89ae7 Mon Sep 17 00:00:00 2001 From: "Dustin J. Mitchell" Date: Mon, 28 May 2012 11:19:09 -0500 Subject: [PATCH 107/136] add a comment about useLog and fakeLogData --- master/buildbot/test/fake/remotecommand.py | 1 + 1 file changed, 1 insertion(+) diff --git a/master/buildbot/test/fake/remotecommand.py b/master/buildbot/test/fake/remotecommand.py index aee6594fc33..97ddedc505b 100644 --- a/master/buildbot/test/fake/remotecommand.py +++ b/master/buildbot/test/fake/remotecommand.py @@ -59,6 +59,7 @@ def didFail(self): return self.rc not in self.successfulRC def fakeLogData(self, step, log, header='', stdout='', stderr=''): + # note that this should not be used in the same test as useLog(Delayed) self.logs[log] = l = FakeLogFile(log, step) l.fakeData(header=header, stdout=stdout, stderr=stderr) From 2717407450b0710a5fc1e3a92b9cad55f965bf9d Mon Sep 17 00:00:00 2001 From: "Dustin J. Mitchell" Date: Mon, 28 May 2012 12:57:09 -0500 Subject: [PATCH 108/136] add upgrade test from v0.8.5 --- master/buildbot/status/build.py | 3 -- .../buildbot/test/integration/test_upgrade.py | 51 +++++++++++++++++- .../buildbot/test/integration/v085-README.txt | 4 ++ master/buildbot/test/integration/v085.tgz | Bin 0 -> 24365 bytes 4 files changed, 54 insertions(+), 4 deletions(-) create mode 100644 master/buildbot/test/integration/v085-README.txt create mode 100644 master/buildbot/test/integration/v085.tgz diff --git a/master/buildbot/status/build.py b/master/buildbot/status/build.py index 6091c3c85f2..9f55922a5fc 100644 --- a/master/buildbot/status/build.py +++ b/master/buildbot/status/build.py @@ -211,9 +211,6 @@ def getTestResults(self): return self.testResults def getLogs(self): - # TODO: steps should contribute significant logs instead of this - # hack, which returns every log from every step. The logs should get - # names like "compile" and "test" instead of "compile.output" logs = [] for s in self.steps: for loog in s.getLogs(): diff --git a/master/buildbot/test/integration/test_upgrade.py b/master/buildbot/test/integration/test_upgrade.py index e41b14053f0..0d7f46a6a17 100644 --- a/master/buildbot/test/integration/test_upgrade.py +++ b/master/buildbot/test/integration/test_upgrade.py @@ -107,7 +107,7 @@ def setUpUpgradeTest(self): os.makedirs("basedir") self.basedir = os.path.abspath("basedir") - master = fakemaster.make_master() + self.master = master = fakemaster.make_master() master.config.db['db_url'] = self.db_url self.db = connector.DBConnector(master, self.basedir) yield self.db.setup(check_version=False) @@ -490,6 +490,55 @@ def test_upgrade(self): return self.do_test_upgrade() +class UpgradeTestV085(UpgradeTestMixin, unittest.TestCase): + + source_tarball = "v085.tgz" + + def verify_thd(self, conn): + "partially verify the contents of the db - run in a thread" + model = self.db.model + + tbl = model.buildrequests + r = conn.execute(tbl.select(order_by=tbl.c.id)) + buildreqs = [ (br.id, br.buildsetid, + br.complete, br.results) + for br in r.fetchall() ] + self.assertEqual(buildreqs, [(1, 1, 1, 0), (2, 2, 1, 0)]) + + br_claims = model.buildrequest_claims + objects = model.objects + r = conn.execute(sa.select([ br_claims.outerjoin(objects, + br_claims.c.objectid == objects.c.id)])) + buildreqs = [ (brc.brid, int(brc.claimed_at), brc.name, brc.class_name) + for brc in r.fetchall() ] + self.assertEqual(buildreqs, [ + (1, 1338226540, u'euclid.r.igoro.us:/A/bbrun', + u'buildbot.master.BuildMaster'), + (2, 1338226574, u'euclid.r.igoro.us:/A/bbrun', + u'buildbot.master.BuildMaster') + ]) + + def test_upgrade(self): + d = self.do_test_upgrade() + @d.addCallback + def check_pickles(_): + # try to unpickle things down to the level of a logfile + filename = os.path.join(self.basedir, 'builder', 'builder') + with open(filename, "rb") as f: + builder_status = cPickle.load(f) + builder_status.master = self.master + builder_status.basedir = os.path.join(self.basedir, 'builder') + b1 = builder_status.loadBuildFromFile(1) + logs = b1.getLogs() + log = logs[0] + text = log.getText() + self.assertIn('HEAD is now at', text) + b2 = builder_status.loadBuildFromFile(1) + self.assertEqual(b2.getReason(), + "The web-page 'rebuild' button was pressed by '': \n") + return d + + class TestWeirdChanges(change_import.ChangeImportMixin, unittest.TestCase): def setUp(self): d = self.setUpChangeImport() diff --git a/master/buildbot/test/integration/v085-README.txt b/master/buildbot/test/integration/v085-README.txt new file mode 100644 index 00000000000..1e2747ed069 --- /dev/null +++ b/master/buildbot/test/integration/v085-README.txt @@ -0,0 +1,4 @@ +-- Basic v0.8.5 tarball -- + +This tarball is the result of a few runs from a single incarnation of a master +that was running Buildbot-0.8.5. diff --git a/master/buildbot/test/integration/v085.tgz b/master/buildbot/test/integration/v085.tgz new file mode 100644 index 0000000000000000000000000000000000000000..a2ef15d6bec62f93a06d014fe0db08e04dade376 GIT binary patch literal 24365 zcmYiMQ;aTL(>08?-K%@GZFjG>ZQHhO+qP}nwr$(Cb^Z7A?(F0{n8`euNnz9&Rh5Y! z1qFoJJ|qVOdaVQNjJ=Z4%S3ENLoNy8A&qp#(0UYRZ!MK-&4EbX5@Wc?E^DOa!JGw0 zY+)_;Pil2~JXT4Nh^shjr=BK&2XZ|zNoUbnLtR&AGWC{Yz7aMgHzb(G27$&#Xwu0* zgCba_!1!+ZoPjvTIBYmX_^rvs?DQ9~{k8XYmFE?9BLnZEZ)p%_&&J$u)iboBzowl8 z^e3!q;ntONCnkOlI4|4ZHr=oTMOY*P)f@|KA0B^lmhJBk(nx*wFFg!Nt!gxwM?@9~ z-!Yi2b4T>j-oa{FU(estzNUg*c=Iom@x%abIlyfUF8#vAZaw|DWHWL)P;5}$z&lVs zs2?>;HPLql^V~W~_M3m^nk4N04D+GJ#|PG-(xJ?u&LPVo=%L4<+OWi-%3-u*=Qm;% z6+y+Jp>?&Qp^t!Vc*^%rHb!6iRwwsA9=y)7l9nN%v40npM6mezYp$>En>;$sXQ6nV zOIr&K*|d!mI0Clmu%_fpysF1oI?i!sQgN|VqCUX7^4BM0cFr|Ady6e8x@Q}bY7f9# z2{vI9(i_(6c6ZF%+Ml6v~d{`s=umaCDK`T^%Fxl*aY zlJ;~=i4rHgQcWh?QdU(Xe+2Mj&o9f4VcLqo&>8R;RL8Dcbzpc)US=9q^XXjJn2@Oa z16}Z#LBS0qVmy$0d|==M>R8!oeTV7D7SRv=e)QDK&!HggoK3chr!Y|M96Esn#kQb& znB(AOr+zi`x^7bx;Er!Uuzfxd5J5bBtcLZ(a{lF404WgQFCro>2lm)w!+iHu;q@fbY zcH^9O4ofbrtZ8N`N4>pBX_*K+`n^f10|IKN?}@qtqFhIGtHM%%-dY)kdQN(i5&uZM ziGcQkRJh}ZEdUKjlb-Dr?{~r@Z6)2Du%eUG+Pjvu4*~jsP&WSExKvIONP(U^uU6oE?Z)pq`+CFydBsZ*c*b@`7-Iy%bh@ z5zn*Y+cSW1qB6_#z?UZM}vWbNI@LX=T)YF3(d{QjKCBm2rEJL1NitC zSMdYzs}gzwKtX-LD*ygPd_*t*CMGzhTYdnT@^eGMgMkG{1<1Jn1!4dp{%33;$0rtN z-kR87Iyu^k>W ztL)N)MnR}>DlRO$c#eamKC=)q=`Zj{WAIaIURm0sM(IIPJ<7r6xZzQQYF}Ih1g#4pj)10@IL}_rZZAgN5iewaWhe-E&buQX@hN+N-=PlwI5GBmP zkRfpVs284>&4W~CCO>viW^V71hm`nk0ho-I_Heh=EH%oUR z&I}=R7`ixVGu^7V-d_wG{`vpFV)9q-{U1;tI93{5z1nd_#LoeADcLyPd&oW5k*AE` zKL$-X?D;2yfHcu0I=w56rNf)^0_vNT3nz4YX>@({iEx#vgC#kl;)M73ypY3jfhC9F zlh}%36DRHcVROV-uofdbhD+>YQVF_b+0*tStj}VIqNMCPs2TE#5~+r)N*5=V2}~@9 z(1UL2T40VgrNkww@q1teq+`0^7A6Ku?s`M2)5KL|zX`UBaY<2!*J^#78c_*m1?9Xk zv!Kg&{FeXu0v+=!r}C|4ai7$7&laOmN0LGsP?PD3&N_NGp1Ri{JC8Bf6&~~OP9~XP z4B)TK5SO;T#2cSJC1L@N-1J-iJ@fgJn+9Wd$M$b)SQ_Sick^eDuc>uc;Fu<|v`-&N zN#dOb@nqZ7ju@dZaYEGzFjn*sR^uRP&C=NNs4;`hxk2?5GBYhHoN#GK0G~9#Im(4i z1Q^;87B@C3RPa@6mN`ew~)yxfJXi6W#D5`KV6l zDp5HdLy$`5sLv5l9L&=Jm72CVg;MkO_~;tQ(P295xtI`TtVi&~^Ts!wU1x77nelyG zXRLmcM9eQ?s|v|QHF;FFvUE=KSP|(R-6*Xp7A)F`1JHp9{Q@TcSC1h7SBxPk>7X$6 zNd9~=d*O5gc%zcbeRNv16sy3MxPKxm;UPpUnhZN1W)5*ci=H=_iv|?1;{KWHh$>p; za0)zU_FsSs@wCt=sd79RgHR@J} z8gD6Bmf}b?*p+>xq*qIBCL2GHa{x--hz)GClnhHJ(e$H7eMD8YVyrf>+AHIgW)~_` ztsremsnGR(G~JY4xLyn@5~LhtbYiqBW4NL3ZCg1imy>H~k7zQ5Yn6&PtwMtq;Cay0 z_~Xp5JaFQehB#b=>{uM~E-Z+^k6-c0d4Y$`hCl9pqW)7k%0Xdu`fLK2f0iG7#A$xT zPjUY7=oAF>aA7BVh64YitX;z0U-C&nmj64{|IsXQDKbgwx)frbXng80ufO959f>T^ ze=U{Kq0gbsA@c*(gOvII^z%aCViTQ;tRmfXghpOE$q>o%ZBJMqO?wCM_Mbur|4*S$ zAgFBzX(EsV+ULrQ$f87eAj3Hy;jjSzu`=%BD8T#||Gzxt=K;h%{r-pO=b!&$P+9;u z1`tL+-Vo+D;L+xk@3b*#*%blpO4XUx$;M{Nyfu1e#B7byM0Qm~Lx$;Xv~v^J%IM@z z;$^5|U9P&dOW2M!%}Y`md#}!2@L;8P>w33m;^zLap{>g7vf7CBH?ybcTSq5wkP_Xt zFfB!)3FCrA-pCj$$I9FQ^hwcqz1(lRdbFN&r(D5NYRBvMOh?emFqbm6U&nT9K~q;* zmB=m&KWe}MsJ3__Tk5@Q?aukjyh>xy)I~C6E0I>rlynP<5r52`czoI$O1PxKNk&9i zh{Ld*b~)g`*!S_D{LktC7o6Z!{|`6)|0Jt=n2}#!3#b)*FA^W*?;)5w%&!bCZJtxe zlB&(ppX8nffemLb?-o;^3}?ud&nn9p`ss2GZzAQjBovQ@40^z>`L=qieVF*mFDh1r~p z8Z@lD1RCFyXG+Vz|I=Oh4>A66z3~%#Vqf6@HGhL9z*!X~(}N;E0=)9n%a1+=gxc7s zb+uL%PanbN z4V@C1=+RR7!<6?lwVLYT=dsd1rX6WK_Y49S_DdUNTyG9c-5p^&$KTFzv@Qcu z367lg_r>U^#FZmTVbCfQjXBv^rtydiWEC#5G^R?aDDOGZ0_N$#LJ-Z4J8a>|-Fd#tW{jgcavzbxNp!jKCLCc81U;<--Q>hUXVC7mX} zR5~qI-+>1YqMm45{VF6Uv}1n3_FU@}%*@;3V!+Y#gU z(hb$DkJ59s57LC@+*?iA#hRDT3Y9!sG&5r;CLIZF5OkK`t9at6CefTUS&YT9 z&Ql|*Y#%Y0&t>B$yV(Lx+1aRlASnJEK-{3G`)&Py`+b0huWLv|6xP!#yO2;h1eBve zbmHV262JTxXnZ_ex&n1`r#AE6J}@>?;BLS}7>6Sd#nqvT6&5`^6kejgUeOd+sW#hq zi-M}IHY1H!Wo>4SZj}3P zJ>VGYi8-LO8%~f;2GlBL)n$p3Nkzh-i^bt@om6s|VG%=OtW1&ub4t9;8djnu?SP>P zM8Y)55|>FIdzV;79p*5=kyD}tz9mT+{@Kf-bB{49-Lp2kUoM1ar>>aI0sn;CBFd&$ zRzVRb%KX$Wc^t9jO$CC9L%I=SoFgZx=G>ggr}1a`^m}^srF&A6JL{awz|ym40<&ARXw(h-CvxnJi1wos`9^(@fF&?+m9<3l)GK9HfLr48xmRAHilWg~X&vX*Vzc_C`K+er-d-F6wb%O965nK} z!Aza3>CK;G<4`^PY^2PaRWg9@WaUHuw{Up)wc0@$3B(=TnFp$S`J)!3<0J#Zj&m#8 zwRO?Rh0}jqZr8j9+G!AZyg*1LSPyi$p$xOur^R|S8GpU zsH=9@65O7Xc<35@{fsuChO|5=YiD4%L1XxE$hDI|WOo2XJ+LW>NW9eQCF3YY&bT8r zY7(v9BVrgIqRa2az-Z)UT)H-SV|+4T8zUG#ZP+8DA*fyjYjA`39E?s%dwj+^(RMJ; zInAYWAS3QBQsx%R`+UVw!G{yBeL?EobxGXR!(`wm4WDyqX)If|Ry`bmkAcodD71omdeTUBboJoS=l5lvFAfoc4N_TUbCU#!w`|<+luDj~`xgS9~ujj*oaqy*7+~kY8l50r2 ziKUUM?{m=_Ib-zlNLk}9r-&4x>JBmVQ-%o^6HJ(V_sUN6rXNXJuDy)@wCuZvi?P?8 zUjf8orDw9xqk{)U7l|f=c(6!RZb0F2nwYUFK= zmp3eff8N)Ot~}s{=Zu%f?RFbK&wum<4G0N*H%3Nw;p88+I*L7owupTKyIz|D zNr$~^-D-GUe*xCLuB`6=(j=3zZ*;o)T^)4&*=s(|ZB5#%XWf&kcI$XI@@gPvLZh|E z<V9j7<<@Y-$(pV9 zyf5JM_745}Zp!oZT)R4&wQ9BFRrm>N19yD6BF@>X0PM#M=KAf38i>$;d(+#EZ((k$fF(Pj7t+@he5r-=h_Jqo!>-fEfYlW2*jYM{XT3}_T+g@R zuU;LaYb8IG?(?VHpk1v!t8&jX=^dr-rfQ466&F95{mQpfz`;YT$r>AY>}T5DxkrcN z!Lv{Rz#M-a@CKWvKIh11>3J14_UdzQK0@{sa5wAGdN7GU;;Z1u_S@L)Q5ZklQuT)~IgxSC5gq*Igyg7+#D>DBvLZgw|n7VoXkV8^fvZl{#AF#!B)lSi8yRZZgG6?tYW}cyL#rbT0Ivz<97A_~SJeXi- zhaiZ6Qjv%tpCE?WaA)9sG*udUFlu~ zqeqku3Q{cF^mS${G~>OY2S@lea(3jf!4S(%5-tO`GCp&EIgpZi8(*>D=M1zbq=~p|QGh1D*Z!FKBCLpj)s}K$>8cJJl5JUOxg7xS4Rt z-@d+d&pbTh_gsl^GcW()v)4p=+!S_j4ePCKTS;~CCJNa|QSAZ>^0P*?_F&cq5(T=z zws3i|wC{$ZO(SVnat^Y7C5;w@z_+cL`p z0+=VYG81G}rnek3no`ka*Y?~wCJS}uUgUk!vY1-rWT)=#3tTz0?&z;|$W^D7DR zN%zoE1nm1a;6kP93Yb`uasv;}k_%GqE;nL`PNx*PZ|xrv1w6zlRi+aQ%U+BY3uGpp zz%8lCc3M_BGOs>{+?uOS;2N`7biV?0~|nK(D3+MSp7mV3~#Xsxx^lR_=Ff z*e5E~9A|ni40Mx&Fb^(G2&~O&ggG%M^_vF*95Fm-N9if;@UTr7^N*;+u-3+50N0&V z29=#p;&wpM_15Ik=i>1akO2%sIJ#|3gdpYboQu?vMAyy6#ia z8&Cv1RMnH#5XKvQh6@U-Y*|`>R=>i_^Qh`S@C1CAb|{Z6Hh*-K$MfNN8kuah)>?1}2GDuYSmxRGdDBU;l*!&6F=*A*b+i zRK^kC>0zF-m7GV9`-$)H$#uXj{d_B}BH~^#x==xfax#bY4KjtVP{)&|cs}M7l(SHU z9?Gy%qz5%vqX-aN-&KFVD1EiS)WnqWu{08JjwEuklJAmRBhld3rzO0uL}y2FyoUG2 zBao%EL{VVBWP2FgqO=wZvrNLzH6^W`992(BVD+H7`SS4#q?tkFSjv%%CrsE8v|JDc zygk8N#@bXr5o6S+&I4jlNt?+Jg~lS?A0l|xx4;-g;S*Q;f8fAoZ5uRn$|vpZ5Va~H z1%xfHRzekTFd^jagV5`2Iq&f&?ke8jK>mh)S}Cz5OI>5}lZw-b6>ACiLT9H^B~KD3}rW?h^6ksCDTVIkMi| z*;TyFD)>+3SsFOhm$zAo81w7T%vd`b3(id_PaLj%Gi(!{K2y%|M0eG8Ue#bvl2|ze zrHHQW+RxWlC9|EcY8c*{j!PC_Ge3_Nx3_P7W6}ybN$Qg6y+@1bsoYFRB5|B3;`LZ&i0FSUD7hFv5C)%lysw@T%hAjp$bH!3F8jp+=VY7M|y}d zf)yDgbHvv9o3>w+>FW3T1;|8?g}KKP@_0^0M(}y0u$*2ZDx6c3)%3FcLV@TsQlu6c zRG$KFCPQO<*%0J7Bb1D{$aW@#F=K0?Ca6N;f$V|A>=_bwh^X;tR!-%KaUS}G%BHGw zD0U8vzfxN%xG@Ve39rYK(^7Elc}Ommj=68`%h*U`zVXm{U51KREv6ed;lsmIfpAC0 zx35S}>w^niZDvK&MC+@JZQfZ4b3<#5^3e)+l+J_aXAl&mDpw^GV&;1&IQyu@xDQ|3 zm=@h>h4b8p#F074>o?IY{xCNrJQu~4$J%Yk;e$1Wfe==f^7YtZK=rU%`BO24F*&LD zM0$wXc*~5*jD|ZspbU&5W>P)R&F<+^o}G`SL)8}b9?K)?>3rgC<~4^=(hLK{Q$Hl{ zP%K@+DZj?KN z4U{a5svNH`j7mH7em|Y#3kOwU_jEU&XmJu#&6n~>Je5_A-h z-@dx1#3fH8XNQf=B`4=)ht58L&x8`Z2UF|V;K={rhhyrGEI^+>%(@E|oRgJSo0bEa zt&SXdc6wuJ6Xu`9PhVgNuz)$^lik-BdO&nl&Aqj?R~(`?_j=KHo$hsA^~O{E<2s%8 zckk&TEu*HVr+a%R-KT~-qmR_}+B&1gj!`f5)z!(>>CL6fRUvAh8W~ebJTpmQa^3>St#7Ye5!kA_8LT4kl~klNB>0uU+0z9 zY!o5Vthsn}@3|JNUwZ=djp+8BN1IdvJwfs}`JZ>EUm=G+v*~nV__;W6fUhj>EXatz zPUN@F(c6wA`_-KXJ1fO+oH1554-$lb;JgkUaBCAXavM5YOKLrGB4T4&eH(BPI4Nnc zaIp0V=|CaTpem6*J0yzS+m>8YKL;`V!vf1U`~9E3?;gcLyVhUvhXDaUnLCGp`B6u| zr}4L;AA7u+CquA3n)d@ST_MEJo4y?q-ewRxjNZ~VmgMfd^4)!6_iYDQpUI0Cg4}!%PeT;hD1l1AfH8!R!r#!0x!cfe zzUr>V(K+tESs)vm{Zd*Xs^KBhI{pa_erudi`6w-_~m!YDT*QA=7zEb4&rZEkO|fj zjF|HpJREy&6SRu|M6WuZ+kHQyri^7996`3l1#1h?eP8&JybXVHIDl@8;hiHY`kJSO zvPB4_6djt+IRy$V_yR)%a|0V7pm#}!kVX!DZA6B<(J)J=O+G|`9>d3T2@qg0_{^jS zWlArAAOw0fn>*&}<9&;!hKh9CSQ`1`lVu-d2n($n+O?JA1r_QVGlsBM-Pm{JU$6rl zmJ=8BgtZTqj_L{hP4@v5+PaJUn$@5yV1Ji4`svNI$_91&Ms8%6i+vTx`g+JWQ9B6w z%jBfS0fnEPg$*^*)D+NBWF{QHBFAL+!;I`)X>WUK)12|n=R6d&5Ni0d1`q#XO7S@x zqg#tJYlgu6jBj=@*XNTrugSMJFtmMeAaD=m%&i~QJHgpcyf75^YzCG8WfRhF1Q7=^ zKw|)QNY^&nR$@fk^YtS8<~~beBT7bI>4g-|jR9DI`$?SVwZ_rooXkJUa`} zwd=$E`xZk#P<>u=qtJ)m^j%6r4dr{MopB;(vT+VOOgxvOV<)N5|E`1S zsUiPeL4!c|qrKtt+@j~PFxkC8&HNj;q;e0bEpJHoj4O5~hVn`emHT9ca0Vd%h z{c`pDjrqYp@LLA}!$Jc*f4{hn-+uB2H;uvJs_E*27{CDCfFoy_%Rqr4U5!`S8}AL; z2IH-&UUXRAL*Rr1Fm^iSHQo9PLue~MMBjY-;fp#iFxL~#spa+Y!QA{K$&PYDaSJz& zml7nA!*vX}=!107(fnvMSeHg``E>SI0=-i&;(K-Wpp9dWrgm0ZzDIAr(!ez$AP9uQng(?8eVfHLT) zNuLX&?h%(zsiHt!@9^8{u=qpJ@#nITa0b8t%BS=Pye;(Y1p5ra*Akl*WenCmSFi0E zmgPLQ7`Tyj9Pdfo4%tmI>`BL$fiiuPY&eb@<6pLVSg!>oYYjEA%ayYKM)1h3(N<3M zN{O2oC+9&uqtbxFzXM?YkriKWgi3N+P@cy-4y{nNO}Tf!G_1PYu=bRsoyM5DQq?d}!{pROWqhgz-|%CK(uYmBr~&cJU`Qi)%s9=J=9hDhT2 z_OLujXZrEu=?b}sMoN9@@}LncfuOv9=1n?PSPLT)qHu2;>*4XoW3c6@>))jSUNoN?N?#N>Ne zGi+4~X2nBg-C+AT+k6#r47Cx%$K9w(S;p;Upc?_)YtWD$-n@e@5 zkgvSZq>G>rVE?`{t0JQ{MwO8+WG31)Qo0Kn<2NEjC=S)LZk31*lZmX+Sv6NHyqU&O zxn$cvVjd8*OYFf}DG2GQLCtj>ZHU||9NA0!d9)%K2gAk?=m1C_TT)=@` zewc8k(5>g$U2Bx{?aIQlMew@o>520cQPh;m8RgUM=p!LZ+`6FU>s)QmM!bC-S}uYw zxeUb@G&Mkc6^h%@wXq3E7xH@gKdVk_I{Oji5l4LOJmt4YNU0#c< zPs4b@9PZ0jOZM`2k$zTq{W*=X?U%t$jH=^RIVj^<)Ly;IS?Gr+K%7d@BmdzpnXW>m zH6YZn~kqEkxM)vX<3><0*X8#h)fHkq>99*EbF@d44Wj1^aH0Qi0ADf zDv7HnJj<`pCwUS=WZ(0kd4+_pTA`Z!yRYTg_1_}b)vz@67E7-XW(eg@Q`^N5o@NSR zupc(L)_VAAAj!i7b610`I1%2p>2wu0>s2w9MURg$=qj=lHdRr2)1}JTTxf*XM#R#8 z*3Gv%*dfbs*?-Xr&&?{bjY={wTQFjp+TL2{^I$(?q``3d_bDfjN3Lj9)JjipIxOfa zCQLF0s8J5-;ySX-L2=cyx|e$%ik^(7I8Mf3&!;h|^@bSYPW|rr*w-IbL3bw*XxdXB zJ&OCu#2^X_X_$1n5{_YIBpy4+W4GnDgkr>divi(0#qh1cids9X4ZV#KYyDJep^wV) zOBilc1$ks6nMLE@rgAOs4AWfn^=n6yikFqo6)KMv)EZR^NPq}()%>@jxD(n7tb`e4 zQP#VEm0_K&zy|V$UAir%9X;$pSUqfGi%vwa8+ZFf2T{k?&4#V-^_KuW7_~du?dQ?j zE8Esu_5{&&!Gs|C;%M2pj*2HMbj4FasA;+oJZ6-%MW(B*=aH`uZ)#$Jqyz}XD)zGu=H<*)hAx^#JBIDO+4raG;6Y8S5c3)6VllN19GLra| zsmMp0=Hdspl^?^@RsL1zIXH;4GdDajqgHyn@;ih&Xv8VCQIUw%hW(j1fN_?k zPc74{Ilovut}=Bf%#|-;jHv8MxES*>NN2> zX?O?>P_}&1(nl)_r$ydelZ%ffOJG{b9u34IjytE8jl9D%wk-X3iJYr!G8{AdbWm!? z!S2pQvFk#L6{Ua_Urhd#E>6?qNAN9v1y6Dcp5HBwM84T-k@<^C9+Oh&i6o%nF%eQv zpAQ$GZ_#ZR1wtb1Y;w5woJG?)5Q4k!&y7%#`Q3LXW+O!|S}x z2k`kAJ848DSxFfzdB2AV_(^&Pv^}r-$vp$;j{Fz|6bAZ60fB~seTRTPA>TMa(KtyH z!~Ofe*&{&R-r@g_-{Y3wzaWIrm>z=uE-wYpAvBs`;HwagQ3xjPximGi(xzn~BeK{R z!&9Z>P#7b3s*wa32(D27vHRnbi3h_JBIr+su#5hEylKH~bjIdl5=Li1XJ+eZfHG+< zD@Pd|8H7mhBy7oeHJFa#iV5{gKvi*LT;YG8P0?jd`?|Scux#|+J&j5~LHj&ZEP(-& zvZI#1AJb)+22RTwK3f#Gr4o<4eN5BMzo*ONX0S>Af0cKnx#}DqbB}IPROT~upH3eRY`&3E`pYSGgv z*2iwywR>0svs{Clkh7Y%smH#u+^71NN3;+1=k$wrIdwm9Jcg1+&Mv0IBkSTj)ICC%XYc3S_Id4>n?MEoDr+A~Pz#$qXW1 zY%v$o6U!0h8!PM-f`-k8l^cDJ7^KwUAc;kx+SfYLEk!Ap7X1?n3ozi|vnRkCHz#Fr+#ci>+FfJZManV)K@Wkg3)n%`?vj0u9_6n7Xt1IZZ!=-G^ zj5~i<%nnV3pu>$PNyP&z->9K4Gad!Uw^rMKCSO>O_5=~?P&X|w#i%K2eR8up(+kmv z^J}WeTwivYmcGDNu%Y8*-G;rIIz>$nl^3sIC@5)#*Ze$hB<5wtOjC@5!t2Pvr0O|j zj4m3w#tO|Jr>j8GBJt_x^xkP!`?K<|9J4m>6>VO8MseJVbFW`@r6rp2Va9QV_)ARb zHSnBPR#sF#Y_~f7U8TPxb@8+qiX?Lv9<|gI7Bj(k!kMKpYu(yZkMVeTj!dk}Op=b< zb|OFXYx?WuL&S=lhty<8;Te%P?!_xc&M)Y0;+G3Da_v;(-$gws{m<=9=jyixl0NnN}vyVSV^wDZVx^@NDe!EH{XiE%Sr z!`$;cU6GL9t@U54`hCQLB^)>XGqFq@p~JMLf&ZFh%EGk>=Ei2Sdg(G$$?8Izz(G}( zp$+AVR?O(2=;q7YM@SVwCyO=|l^&%*W}sEs=o__nm}0GE*%{Z9K%;5HN%VXlW=J62Wh;%s`~Y1x->rWHQxK+Zl6zRnrkEy4g?_ zBC$AORr%RW#u9oxpO{6xz0j+m-Ka)84<4q0Wixu!1PJ{S z1laaKf_nmAE7>UgmT*?dUpX3< z9{|>2e7$8@bc!y&2@D=Jp5JuP>9-q*Z)B~EInZP z;bJxFDm}W{Rup~0?$*4^$^WG&9qTXv+H3xt62dHCNb7XlDVPE2ftG>AhXnZ`M}`1_ zms-oo(GSI^ZGnj;Zn>ac=n=}ND6{}S0romED_rbyFSf;+lk5Z)+|w+K{9s0kOc9q@ zHC57K&u)2K&J-4>?h^YPw%o$UcRvkvs7^vEKUH2iE8fzX1Z5870Q*+>{HJ_vCzYFN zxpnQyZcbf-o(Q*I6&hWgsTwfKCuq3Renfmu@Yy~5L$YAoc;ij)<|fvYQTK8}s^T5m zq_Xm8xykJWx;toVo$&(gl_wRccvY3(FGgH$YK$?M|Bq@vcOy~*C$F)oR-9Jj(kBq* zpn`37!vl3{^9DkVJ3)=R4#@j504^C;WvozGgq8@H81 z_w(=;j^~f()teaHY2v+BtmDgSph=TmRCEKU60LX*H_;o zw)mIMlU5fz&r4`*87KA|y0oS>6^ynZ$=#V2;?gWO&~lLJJbY^OY+bGjK9Ckj=e{0z zX|3waaWBrZzWIHdVSV{W!m45kZZ&>Nk(y)9+gr^qg(DZ)c*nJV^`dQtxR94)H4dx< z(Sh5g!lepE(lAgDN?4bh&j*{j@2{Cha005Q44ZW9kHQKMuB;ZA@(voGLA5}c$i`gn z2uh#Lg(h_k^!c=1f4W84v2g&)&pbGs=1Vf-Uj;492g5p*A`A=!w;^^0t z>Wg{*%-BGnZ6yrFqu+e9?zDf&DUl|(t0$ybCWOJ1GSovq@67`oo|7_W;SL)-7aq4^ z3-^e=-sAEW<0<}MpK-TbN?OG5UMB~iN)*Is9y{8P&*E2sjE~JW@KmBHED8#ZLs}@| z#TDyb5J)Dpn=|7u3J}GnpnrF^dE}r;cvrb9Dl1Ol^5Mx5Og5E}hq+|w%zkle646=7 zToeA5^53U4oRYN|SZiRH@xDT4D~jqc!_pgP`3haU6GAdiXof+u5!L&3nP+DLkMkmI@)eQA{U+Lugt z5G{mG53Mrxxk9W)ECz<0T=K+IUU0G$|1y%06%M{h<#PNr_vZ%fbzE8>PXD9c=(;p4 zjrq5%-&CieaF0PFb99vUdJLvFQK5iFG5fjKW@^6|;$LF2g4cVa!&H>CP)i=80y!5|rM))1 z$ViMzP{Jm0*FkJ-ti4(RNYB5~U!WV0 zk>thUnm>W1i?6JMYDN{Ob%_pPY+OZA!%Q^In&an@_n167;s-7gXY}3i!T0qsP75lK zQ5$z#aP9uUa#82Ab-ZPovm~xNV+#V}|5!+#@Gq!q+x(RB_6LIuz))mL%5@pkRD9ZemWG#uPDdiG#h-qCS_#00wy1mBQ%FjZz0jE@fP6wZLsSNmB|7a~X* zBQztt#K(dE`SJwGO=ss!5?M?iu6y4OUOu8UTo$LTH3uc?Q`x(ei?EP17@dqSARs_a zG7;QKdh(xgqKxFEOz{L#sDTqzLooT@j!U1?4tB~qUn8RF2;tL82vb5$RMYxy!asWj zZ2<7b*nzsYt|=)S<51B*{9KW9no=dXl5}*>g>c0YMRAcrdo*D8n*_!e5?^-pdYi@) z%~mON{~`zTuX!-mQ(<;IN{G(%;>(c)263iwu?xkCjZN!)wcAySPfaQB;;!X0}b=?L5Zx7782SIP92hK?_mV zt&S=CbrK%EjBr;u@UUjwn$uuBsyi>K9!M3n8B+~cip3(~Jq?OwrB~TGYCuWp zY;Z|}9OmgFO@|C7tkzp!(}FfkVvn$U-s?Wpa(v40KOzd9@OgqP4pE={)B_Vc(<&E2x`e=6)7GOI*lCfo-v;$_(H|$ zj*kf@L7f?MjH@PFW}sue*tcr*9r-jfRC+cv?B9?gUuk2adJqdY<)8tktUYjDczLZF3C&2igofvVz|Phl_teU zjL$qK5c3fI5rCSb<$GZcTuN#qZZkCQU1gu*|ITPps5#e&VAjjmzAJs5lx!1J*1z-{ zl(nx`6o#8}BWmEF3oot%ND`vOwDu>{I<2I@1g?2|q(t@m!ZS)_;)iItlGsH z>Q6*LrSGwW^9DC>?%G+(+cvFt_N=ie|7+-Em6`;3V;Mk#JXG zszYyO?mxtIR6%LFlHtaHJH4k=?0kIeCs%^{oob}boH(~L&UJ-AcU9NtAEvb!8f)C8xdYIQ}sIXERG~>f9=6ukV1@1y8`)_JSyPdv5{B@ zH+`;4Uxe3QcGHclKSll)vu&X^{GC8<$RWMSh=CqnVdP%*J4Px+)@-MsA4Ods(w7(K z5-&IIe+kA#Wwl=+TTod;G&Y)de{jKp9^CVR;@_cK0F1-C#x=z$=6<4fRZ^lz>hW)x z1CU(u084~B<}<-ePl)XaF6NrPH9T!+#P$V$qN}mxys?BYkpDCwey7j!XB~|hRcQNr z-xb(dhmP;l)jfP$$rE$G}TF9X83scW(v|Loxat<^u@&75uD@-T95!cn%qrv z=Q_du2deYO2s9xs_Y;nT@pj76I>8FVxFp5mP{Q~z1?de#|7HT8dTVdn#gB|;$pJYH z#HmSoptQmFolMI6^y*N%6D;}@FGsY31V%bFF6*?0%UQWmXhIpRYHhcTc#vn$IqsF? zRc84Me^u2wJ^@*@C)a$-(}WG3b7W3o#uge51Pf_FNw+nrWi)}iJk{Z;!#xd}fe$Q; zNl-b}xsTX@e$Is{p1f>eDEp*G;Ds69cE&Z#mzU*M07%ID1%S4iHA-+%QRSF9pcHRomno@MEnJ;_e={- znpsZOB`lLpW-_Y=B-)~6vEsy&D@)Q7H4}YlFCGcmiJmB#CSH$-pdz?1U7PAVSgjkm zr+FD)P0k_1-x_PPc1S@M)*CC^fOHHVdhp12xNFL%h5<3hV^mx>jrk}QyHiwG=GkD6 z;;oX4Y7W^U@;N$|;{};EA)EKEl#f#bG&__GUnWR~F7evP+uNI1t5h79w6%Ip2AS|k z2M>`(W6>@g3n*R8b7sHA)*-J)vrEjWvLtSwx}UB;eu!KCe6hRkD(mVt`9sO9F+6z+ zdEww)<8F)4k4rJol)8$Heyp_^?kfI!;n4tQ#o&(_O2;p==I5&-x+i+f`@G+z$>->j zuiJ2^iz}D=w9uZBzmD&C^{!F+bY2t{T=)LD+DKc*Hqls2K7JuY6Vq5Oj6Qx z4QZm1m`B{RG}2u$i77c18GfgZ&GtkZTY&+H&;nPNKgVg2^+C$i=^1ohdv-%m8p+Zq zOF~P@Q}JH*9RN>mK&lnPhm1KgnUrpi!t3^wdH4`gB-GPq{sFyZIird}q*W;DRXlNN z#l?UJ87NKbhf53W$^{B#HxyRf^s2xyUWPd)23IB~^ylQ3$BPOIzfebX8>zSq57{i- zWP9!GH~RY39WtBcjgj)B{O8UN4A0y|IzDJV-1BAEjUXh%$QWDvd*Z)8$N&E`b^NRN zUljb;_+MCD=vVyzx5$5)|9VAZ759I{f6eCu2>)IF3**0q{}RLaEB|HhjXPb{Fk8DZ`{*2%Igno z6R+k!g|_(2;QTPqA4n~Me}~i(bcYKC!G%FErvGC=3x;kW%U@tD5ltAc=6^_Ff&V=M zOOz`FE*1nA|DC{+_yd6@`3C|^$_#$xe}}-5c86aLf?xXsfpz^Sf%OgM`qu>3Zw%LW z0!#5Hfu-~h1eWr50!!sj1eU59T=~xiZ)}>w?*ta?JAvi+oxsvR$Gd(k z@DKZ@U}$$Z!xVoKSS~P=|1yDvw%!%?o3U;|aQC0adVDh$9t21H$ym>GW4+MEUi>Hd zO}O`eX72pMQK@rteSRabe64<(>G#vjd{XCgGjF2J{Efi!M;rQIC9u$j27EL0j|5iW zzfWKVxx;S-!EgUYU0$1`U`;-9R!d0jle=`dj0{kiv8hKl9@kZR&hVOV+a-)1doRye~?iwAQ-yl zbD-r7W;D;HpAk6j^3{G^2xuIH~v;WB?z8+Uf%d;`LsWjN6+xY zpXF(+JI~9fqsxm&{uQRm_zO&RHwd2j|2$0fALal5UjFAF&i{c0e|`Vw-y*-~|NYM* zN56mn+h6z}=h`>(5#|r1#&#~Zm-+d&&cB!^Ke-xvOh?Y%EtqvfJq{x_^;RY+srG}^ z&$?ZY3w2>KM|e>GABGR+bHvZ*;c#h|nar!C>V^o&&SIqIPNcPJz$0`rOvLMwX zyh?^Rn27lcUVp?M)M#z%fL{h@7Via|2osYOHCgGjStDnYjww^BdmL?ghOI!0c`I=FC-1-mf_YRfgF84;cI+l`8B-B=4p=aqq+)srD*}= zfxOY>2qpX!6{gRV&noUPDY_FdF5p%%yf$@`j52o82zJ2A}NhR&hWcG(+HlMS?Yn|dY z1yxRa)BApqtCgRKu+fN^S{v>7nm0cQa-Oe0*y`h;ei{MiGEzm_IG*fWBaiGH=oA+V#`UE=i35a*)=N5;%8x!tw_(K3B=@;B zO)dE$bQdU%xxZxb{?P|jH;&CO53q!-sPEgm=@2fK!tuulKC?Yv^qyHvZEh0GKFzj! zD_>q!XqUJiEYH!5+Zn_ zI^F-O>CUa~Tq!xWTcWn|MzK*SNZSqa@GU$&P4SJIkC*KFExXooXX9^lfgja_9F*U5 zz8|soe|R++weV|}u%NLeXC_{x z<xNfV_TF2~{v;{Yu}6 zQ?*$iY*Tn*7FlB&UN9x0KR$cA8XH1zaXciZ@ML(L9Gh9*^Je1kP&fuANu&s~NmPD; zm)0H3XsAMIMWxg0TaU|vXfKK|@QF!)xdq|%@xrQU2Dusy1w4Q7#mIja-Lk`vg<_P3|<5YYACVQ;`)jooOOX>yv=2bK6crV;5n(LON- zY{ZS+y_fT{YIeVahp6yE;_ZTfX^45h0S>{SmInDHjcSVc6YSx36^GRPx=91sql;4f zt>*d-Gf55%N7tm)$Rpa>WfElGixh4%)V#P^{^*7Z!$tG$x45`?IO@)y3)9}jSlWoP zKN)amy~uIS{<#0n{&c$wbX`}}B$_LJv===IJBc8f4I$z8#?S&rS|`ylg3C8VEJs{= zdYW7WBM8l`p6k0ZzYKONBfwRag(|UtPyB@w__yjj`>8tmkB*YaW38UC_So0I2>tPe zZ#I3jd}H*tLCkrz7Z@0a#izf)KLV&)@2O$Mb(`+3-suRV)2T%3yq(nV@J}R9Q%FYV zxO7n>1$K|HZ9FUwQ9K|jPicZf1hvlQYolAi3J;g;&#xZlhIfJCoo>99RAsW%BYfE7 zyUsj-T74e;5Lu}9gJf(fY8?gH;!F{2Dc)fl-gk6HU3A^0Wb=DhU+=uC0>SRjI|SM#1a=oeXl7n|f@hO?+Dj@$Jp$m3 z;dX7pWQ2IJ)iuerc}5-g?%#{=h2rw;77{WV8tBRy=aw=&6f~CVxL+Rj&SaP~ym@P8 zf+W&9B8Kq3BhCZD;R&fKd(me;80vn>JXmX0h6;Rwj+l*{kflZ)qeK_Eo%?}zubt_M z;eZCAB_)H)pXdU8_FzDO< z@T@BosBGRN+0x6~FDs_D^t@$yWbu~!Ir{?$v26bIEHgI=Ld@^)j{p8i{Qo!e zKSF{+|H%IfivEiK{}%Z@{-3Ij@Wdcxcpc2_@hAR==2{Tt`LDtem0&7z$ztVT%3xp= zuxhIjfk0Ha=r=f$g=}n`nP6;|e3leuw4)r8a;!+gvaE%L9aCh4>4a&~g9uM4lMau( z|9|6B3T=!`oWA1i%S8hzIxbn6On1X)A0!H9ITk)Bj03}n^g^+$!2~!k$KrgYIIf21 z;`J0x4K|p@RSoGBRaF^-GFu)k)R zG3+&Cn2Zp9j(!bmOZNwI>PNTOnp|p%9-nSRfJhkAz7fSc(cf%rTE0PE>lYp6slv@5pz@`d)C0VtN6`39RkjClD< zV>X)Dd%!?wZ-@7=9~*I&CMv|KPM%mXDa6Vr9f&*Iakwc86b8dhhBi5Pp5_n{) zUs`f-Z=91bRQCUTj-L2WVHjIG+t#Z};qX z+v3i&q}0=x4H6S{$sY3`eIfp&KR=`F0EhDd{RaZ{6>C}Fwk1uvm~%uVS(-BtbF6PE zLAC%Zou2>^BLzzWANY?ZeVtF!bq~v?R$>KKTliI0cCq?OIRqS@ioQ^=?U3Xv1>1sZ z3=NI=fH=2AXIl;A&WIiT2b~7ojRFTn!yohpP$c(k&BQOS@^1lsfJ;3q!g4f%OO zSIrEdlm7D&lPyKyYKtl!(?rPxW9kqT1F3HzMTL}zd+_XY8LA( z>fMb5MAs~aZhs`n!ApLdOjhgoseVdJ*}f!UYrIIr*ND%9in*>Pz;1VowHCgp*@iNH9^&NTTLoWH zTNc7SpNaPb3ao-k_i19^Ok7)v+P=hmrtHyWM;FOFbyZ&COWG%ZGw!LrGshVK$#Dvx zn6&S-9=7c)>8~->hxnskh4vhnW^gQ+m3Gxw#Ce%c^=#A|3iQ;N8sOI3*BhEgWIeZU z=F@jpL|LIHZF)rLfQz!zmN6eNc~79)-{8<3-3r#Tw6OOKHkebAPovxzET_mY*BSTv z#AET8rzG%I-jh+3Sxez}N!@{MGHLf+UD{%J0uv$29lFM{b_sNtkmz{F}^A+FTgJC{nG- zmimNAu|d6o*!|#m%<90+lJ1~V;x!{)wTAMdpP=MMtQENghr4@)sunOIVT!epKa zUzok}BoR)A+^bw`MCOUA4g?{sPs#x*i^|l^2Dr=o=&_@?XMx-7Qz^G=gr2|ke&tbf zP5emWyd8^TJNgWcnfOwI9bP^IS=%b3O6oQR|C4W(?H``3^>G_OQ%$oF1R}4KcDhD>()>*XsRiKgwywvkj595Ud`I|%*ISu)e6j|_1!7NE|B z)$p%`TRGJGPXYT|fk{mDi!E+XkKeQ^=2TVHnxw-%@bxh-(e*tqbE8+nmYL&OWq49Y ziy!6BKYmhy^grf4E(4a+@NGg@md!RN&?{1s6%&u>x;gPM!kjpHv6+7*dBz+5fQ_#X2ee0M(I7zZ>Eon7y|&45$HL{ zGf`u%)&`LO2)B(t`Z%?ccLsz9q(8ThE5$8YtuqC3Bw1O}r=zOTMojtx+x^Osh%H`p zKT^&L;AX8h5?fDNAxsn_>OG-Z&#i7Ep>s9nD-jWMtq}hK4--`AypW`4`)O zShJ=Ep{wY=3ILv!1NO^HF)wQyD9d_XlsY9=;X>5+yqdVHRsue(7 z)PJ@6nQ#Ke!nLJE2JB!bce*=0|3a9ZdjdB1&_yV+L7gZ$HNBQIrYbY$VII48vLB-r~c{$-9?*+)z) zW!7ZyaqyHl)|k0nA!^f@r{n;=K<}CwQ0MjZ%vf;mnE(qA%8o7{rWCU#S~CzMJ1DyW ztDEJ>Eakr0iM5p`nYBzqzD^t71=mo25GS){;I-fq^eW2E58uqFzN#8p_ znmO==a0K6CTRLu5&q*1u-opq`I@S3pR-nMP##GBH5cS>yci682k~~{$UjN=~bJjw5 zz>C$FcY3JSxN@%E1Yiq?qJp@D2K)h&)l3oFrjYlQZH*Pq$seZP`cx`S&DHTcci*h+ zu!y)RFjH?{v)DmAWnRO_gHAKUMla#>3{ zvC{{ANLF`%Vz_S^o6quT`DQd7FF#GCXhgWxB|bw6TV8%h@=UEUb-QH)Qj{ShYuxB{ zBeZ1O?5uowqMhB|tdtOpzKbTGR168|e9XX|FrH06$~&xVvoId;2Zo+aIkRr%`SE5T zdb$za9!BCN^gt$Oc33me`h-`2dZE0`n(&+)RIeHEhtqX{f3mcxiZR-?&n={ub_C7Xex&Fj<64kt<%v4p~Uwb;I3wVtlp3 z3fx`ZbR@x11yZpaVOAQt*kOnAGy$w^SPWP(8iejyM7x-#ua?fX@Bst%%hF3xwyikc zC;ThxldkBg+nPb)N;f3P{hc=JOt~ zudeqti>cA-4qY&+Af9Uh7Wiji`tq+-QYTC13|p1-p>H`6ZyWdkq)4nCBR<857fyYO zi}eC>GkFfJ&?Xb0(~ijj-&4e-WmS#IG{=@{1}=)es*!ukWV$#I)$Uskq&79sx!m7a z;SFo#cdm9ZHDGZu+q8@yOB2-+lenn6c!rRXrq*tVgvqa6}jK|I^|XsAU%6g_&UV|GlD zhOHU>wfxJ&PPJx0X!2D1P4rj|AQnZT``UW8&SZL~5m?vAXOQlUel%L{*tyL*$ajkkU_DR1;tn!dZ>>HV*_@ui|w0OCX9O>|K%Wi)^xk~Cx zWAL}6ZGOow`6a*Pm;91n@=JcnFZm_E Date: Fri, 18 May 2012 15:39:15 -0400 Subject: [PATCH 109/136] Make master-side source steps respect timeout option. Fixes #2248, #2203. --- master/buildbot/steps/source/base.py | 3 ++- master/buildbot/steps/source/bzr.py | 1 + master/buildbot/steps/source/cvs.py | 1 + master/buildbot/steps/source/git.py | 1 + master/buildbot/steps/source/mercurial.py | 1 + master/buildbot/steps/source/svn.py | 1 + 6 files changed, 7 insertions(+), 1 deletion(-) diff --git a/master/buildbot/steps/source/base.py b/master/buildbot/steps/source/base.py index bad9deb8114..4084f1ae217 100644 --- a/master/buildbot/steps/source/base.py +++ b/master/buildbot/steps/source/base.py @@ -149,7 +149,6 @@ def __init__(self, workdir=None, mode='update', alwaysUseLatest=False, assert isinstance(repeats, int) assert repeats > 0 self.args = {'mode': mode, - 'timeout': timeout, 'retry': retry, 'patch': None, # set during .start } @@ -166,6 +165,7 @@ def __init__(self, workdir=None, mode='update', alwaysUseLatest=False, self.logEnviron = logEnviron self.env = env + self.timeout = timeout descriptions_for_mode = { "clobber": "checkout", @@ -285,6 +285,7 @@ def start(self): self.args['logEnviron'] = self.logEnviron self.args['env'] = self.env + self.args['timeout'] = self.timeout self.startVC(branch, revision, patch) def commandComplete(self, cmd): diff --git a/master/buildbot/steps/source/bzr.py b/master/buildbot/steps/source/bzr.py index afa0add92fe..0ce4be8f7ad 100644 --- a/master/buildbot/steps/source/bzr.py +++ b/master/buildbot/steps/source/bzr.py @@ -196,6 +196,7 @@ def _dovccmd(self, command, abandonOnFailure=True, collectStdout=False): cmd = buildstep.RemoteShellCommand(self.workdir, ['bzr'] + command, env=self.env, logEnviron=self.logEnviron, + timeout=self.timeout, collectStdout=collectStdout) cmd.useLog(self.stdio_log, False) d = self.runCommand(cmd) diff --git a/master/buildbot/steps/source/cvs.py b/master/buildbot/steps/source/cvs.py index b89da7021b8..b859959b6ac 100644 --- a/master/buildbot/steps/source/cvs.py +++ b/master/buildbot/steps/source/cvs.py @@ -211,6 +211,7 @@ def _dovccmd(self, command, workdir=None): cmd = buildstep.RemoteShellCommand(workdir, ['cvs'] + command, env=self.env, + timeout=self.timeout, logEnviron=self.logEnviron) cmd.useLog(self.stdio_log, False) d = self.runCommand(cmd) diff --git a/master/buildbot/steps/source/git.py b/master/buildbot/steps/source/git.py index 4ed690b6115..81083174790 100644 --- a/master/buildbot/steps/source/git.py +++ b/master/buildbot/steps/source/git.py @@ -294,6 +294,7 @@ def _dovccmd(self, command, abandonOnFailure=True, collectStdout=False, initialS cmd = buildstep.RemoteShellCommand(self.workdir, ['git'] + command, env=self.env, logEnviron=self.logEnviron, + timeout=self.timeout, collectStdout=collectStdout, initialStdin=initialStdin) cmd.useLog(self.stdio_log, False) diff --git a/master/buildbot/steps/source/mercurial.py b/master/buildbot/steps/source/mercurial.py index 83a392da79f..5084bda1248 100644 --- a/master/buildbot/steps/source/mercurial.py +++ b/master/buildbot/steps/source/mercurial.py @@ -224,6 +224,7 @@ def _dovccmd(self, command, collectStdout=False): cmd = buildstep.RemoteShellCommand(self.workdir, ['hg', '--verbose'] + command, env=self.env, logEnviron=self.logEnviron, + timeout=self.timeout, collectStdout=collectStdout) cmd.useLog(self.stdio_log, False) log.msg("Starting mercurial command : hg %s" % (" ".join(command), )) diff --git a/master/buildbot/steps/source/svn.py b/master/buildbot/steps/source/svn.py index a4719b6b720..9a352a03991 100644 --- a/master/buildbot/steps/source/svn.py +++ b/master/buildbot/steps/source/svn.py @@ -228,6 +228,7 @@ def _dovccmd(self, command, collectStdout=False): cmd = buildstep.RemoteShellCommand(self.workdir, ['svn'] + command, env=self.env, logEnviron=self.logEnviron, + timeout=self.timeout, collectStdout=collectStdout) cmd.useLog(self.stdio_log, False) log.msg("Starting SVN command : svn %s" % (" ".join(command), )) From 75328e15dec2b4e20590315f2a2f65b899dde521 Mon Sep 17 00:00:00 2001 From: Tom Prince Date: Sun, 20 May 2012 11:55:24 -0400 Subject: [PATCH 110/136] Factor out slave-side step specific Source code. --- master/buildbot/steps/source/base.py | 69 ------------- master/buildbot/steps/source/oldsource.py | 114 ++++++++++++++++++---- 2 files changed, 95 insertions(+), 88 deletions(-) diff --git a/master/buildbot/steps/source/base.py b/master/buildbot/steps/source/base.py index 4084f1ae217..ba15a85153c 100644 --- a/master/buildbot/steps/source/base.py +++ b/master/buildbot/steps/source/base.py @@ -50,51 +50,6 @@ def __init__(self, workdir=None, mode='update', alwaysUseLatest=False, @param workdir: local directory (relative to the Builder's root) where the tree should be placed - @type mode: string - @param mode: the kind of VC operation that is desired: - - 'update': specifies that the checkout/update should be - performed directly into the workdir. Each build is performed - in the same directory, allowing for incremental builds. This - minimizes disk space, bandwidth, and CPU time. However, it - may encounter problems if the build process does not handle - dependencies properly (if you must sometimes do a 'clean - build' to make sure everything gets compiled), or if source - files are deleted but generated files can influence test - behavior (e.g. python's .pyc files), or when source - directories are deleted but generated files prevent CVS from - removing them. When used with a patched checkout, from a - previous buildbot try for instance, it will try to "revert" - the changes first and will do a clobber if it is unable to - get a clean checkout. The behavior is SCM-dependent. - - - 'copy': specifies that the source-controlled workspace - should be maintained in a separate directory (called the - 'copydir'), using checkout or update as necessary. For each - build, a new workdir is created with a copy of the source - tree (rm -rf workdir; cp -R -P -p copydir workdir). This - doubles the disk space required, but keeps the bandwidth low - (update instead of a full checkout). A full 'clean' build - is performed each time. This avoids any generated-file - build problems, but is still occasionally vulnerable to - problems such as a CVS repository being manually rearranged - (causing CVS errors on update) which are not an issue with - a full checkout. - - - 'clobber': specifies that the working directory should be - deleted each time, necessitating a full checkout for each - build. This insures a clean build off a complete checkout, - avoiding any of the problems described above, but is - bandwidth intensive, as the whole source tree must be - pulled down for each build. - - - 'export': is like 'clobber', except that e.g. the 'cvs - export' command is used to create the working directory. - This command removes all VC metadata files (the - CVS/.svn/{arch} directories) from the tree, which is - sometimes useful for creating source tarballs (to avoid - including the metadata in the tar file). Not all VC systems - support export. - @type alwaysUseLatest: boolean @param alwaysUseLatest: whether to always update to the most recent available sources for this build. @@ -117,15 +72,6 @@ def __init__(self, workdir=None, mode='update', alwaysUseLatest=False, is can result in an incoherent set of sources (splitting a non-atomic commit) which may not build at all. - @type retry: tuple of ints (delay, repeats) (or None) - @param retry: if provided, VC update failures are re-attempted up - to REPEATS times, with DELAY seconds between each - attempt. Some users have slaves with poor connectivity - to their VC repository, and they say that up to 80% of - their build failures are due to transient network - failures that could be handled by simply retrying a - couple times. - @type logEnviron: boolean @param logEnviron: If this option is true (the default), then the step's logfile will describe the environment @@ -143,15 +89,6 @@ def __init__(self, workdir=None, mode='update', alwaysUseLatest=False, LoggingBuildStep.__init__(self, **kwargs) - assert mode in ("update", "copy", "clobber", "export") - if retry: - delay, repeats = retry - assert isinstance(repeats, int) - assert repeats > 0 - self.args = {'mode': mode, - 'retry': retry, - 'patch': None, # set during .start - } # This will get added to args later, after properties are rendered self.workdir = workdir @@ -242,9 +179,6 @@ def start(self): % self.name) return SKIPPED - # Allow workdir to be WithProperties - self.args['workdir'] = self.workdir - if not self.alwaysUseLatest: # what source stamp would this step like to use? s = self.build.getSourceStamp(self.codebase) @@ -283,9 +217,6 @@ def start(self): branch = self.branch patch = None - self.args['logEnviron'] = self.logEnviron - self.args['env'] = self.env - self.args['timeout'] = self.timeout self.startVC(branch, revision, patch) def commandComplete(self, cmd): diff --git a/master/buildbot/steps/source/oldsource.py b/master/buildbot/steps/source/oldsource.py index 46499ccbd33..bdcbd05cbe4 100644 --- a/master/buildbot/steps/source/oldsource.py +++ b/master/buildbot/steps/source/oldsource.py @@ -61,9 +61,85 @@ def getRenderingFor(self, props): d.addCallback(str) return d +class SlaveSource(Source): + def __init__(self, mode='update', retry=None, + codebase='', **kwargs): + """ + @type mode: string + @param mode: the kind of VC operation that is desired: + - 'update': specifies that the checkout/update should be + performed directly into the workdir. Each build is performed + in the same directory, allowing for incremental builds. This + minimizes disk space, bandwidth, and CPU time. However, it + may encounter problems if the build process does not handle + dependencies properly (if you must sometimes do a 'clean + build' to make sure everything gets compiled), or if source + files are deleted but generated files can influence test + behavior (e.g. python's .pyc files), or when source + directories are deleted but generated files prevent CVS from + removing them. When used with a patched checkout, from a + previous buildbot try for instance, it will try to "revert" + the changes first and will do a clobber if it is unable to + get a clean checkout. The behavior is SCM-dependent. + + - 'copy': specifies that the source-controlled workspace + should be maintained in a separate directory (called the + 'copydir'), using checkout or update as necessary. For each + build, a new workdir is created with a copy of the source + tree (rm -rf workdir; cp -R -P -p copydir workdir). This + doubles the disk space required, but keeps the bandwidth low + (update instead of a full checkout). A full 'clean' build + is performed each time. This avoids any generated-file + build problems, but is still occasionally vulnerable to + problems such as a CVS repository being manually rearranged + (causing CVS errors on update) which are not an issue with + a full checkout. + + - 'clobber': specifies that the working directory should be + deleted each time, necessitating a full checkout for each + build. This insures a clean build off a complete checkout, + avoiding any of the problems described above, but is + bandwidth intensive, as the whole source tree must be + pulled down for each build. + + - 'export': is like 'clobber', except that e.g. the 'cvs + export' command is used to create the working directory. + This command removes all VC metadata files (the + CVS/.svn/{arch} directories) from the tree, which is + sometimes useful for creating source tarballs (to avoid + including the metadata in the tar file). Not all VC systems + support export. + + @type retry: tuple of ints (delay, repeats) (or None) + @param retry: if provided, VC update failures are re-attempted up + to REPEATS times, with DELAY seconds between each + attempt. Some users have slaves with poor connectivity + to their VC repository, and they say that up to 80% of + their build failures are due to transient network + failures that could be handled by simply retrying a + couple times. + """ + Source.__init__(self, **kwargs) + + assert mode in ("update", "copy", "clobber", "export") + if retry: + delay, repeats = retry + assert isinstance(repeats, int) + assert repeats > 0 + self.args = {'mode': mode, + 'retry': retry, + } -class CVS(Source): + def start(self): + self.args['workdir'] = self.workdir + self.args['logEnviron'] = self.logEnviron + self.args['env'] = self.env + self.args['timeout'] = self.timeout + Source.start(self) + + +class CVS(SlaveSource): """I do CVS checkout/update operations. Note: if you are doing anonymous/pserver CVS operations, you will need @@ -163,7 +239,7 @@ def __init__(self, cvsroot=None, cvsmodule="", self.branch = branch self.cvsroot = _ComputeRepositoryURL(self, cvsroot) - Source.__init__(self, **kwargs) + SlaveSource.__init__(self, **kwargs) self.args.update({'cvsmodule': cvsmodule, 'global_options': global_options, @@ -245,7 +321,7 @@ def startVC(self, branch, revision, patch): self.startCommand(cmd, warnings) -class SVN(Source): +class SVN(SlaveSource): """I perform Subversion checkout/update operations.""" name = 'svn' @@ -299,7 +375,7 @@ def __init__(self, svnurl=None, baseURL=None, defaultBranch=None, self.always_purge = always_purge self.depth = depth - Source.__init__(self, **kwargs) + SlaveSource.__init__(self, **kwargs) if svnurl and baseURL: raise ValueError("you must use either svnurl OR baseURL") @@ -405,7 +481,7 @@ def startVC(self, branch, revision, patch): self.startCommand(cmd, warnings) -class Darcs(Source): +class Darcs(SlaveSource): """Check out a source tree from a Darcs repository at 'repourl'. Darcs has no concept of file modes. This means the eXecute-bit will be @@ -443,7 +519,7 @@ def __init__(self, repourl=None, baseURL=None, defaultBranch=None, self.repourl = _ComputeRepositoryURL(self, repourl) self.baseURL = _ComputeRepositoryURL(self, baseURL) self.branch = defaultBranch - Source.__init__(self, **kwargs) + SlaveSource.__init__(self, **kwargs) assert self.args['mode'] != "export", \ "Darcs does not have an 'export' mode" if repourl and baseURL: @@ -495,7 +571,7 @@ def startVC(self, branch, revision, patch): self.startCommand(cmd) -class Git(Source): +class Git(SlaveSource): """Check out a source tree from a git repository 'repourl'.""" name = "git" @@ -535,7 +611,7 @@ def __init__(self, repourl=None, can solve long fetches getting killed due to lack of output, but requires Git 1.7.2+. """ - Source.__init__(self, **kwargs) + SlaveSource.__init__(self, **kwargs) self.repourl = _ComputeRepositoryURL(self, repourl) self.branch = branch self.args.update({'submodules': submodules, @@ -580,7 +656,7 @@ def startVC(self, branch, revision, patch): self.startCommand(cmd) -class Repo(Source): +class Repo(SlaveSource): """Check out a source tree from a repo repository described by manifest.""" name = "repo" @@ -604,7 +680,7 @@ def __init__(self, @param manifest_file: The manifest to use for sync. """ - Source.__init__(self, **kwargs) + SlaveSource.__init__(self, **kwargs) self.manifest_url = _ComputeRepositoryURL(self, manifest_url) self.args.update({'manifest_branch': manifest_branch, 'manifest_file': manifest_file, @@ -705,7 +781,7 @@ def commandComplete(self, cmd): self.step_status.setText(["repo download issues"]) -class Bzr(Source): +class Bzr(SlaveSource): """Check out a source tree from a bzr (Bazaar) repository at 'repourl'. """ @@ -748,7 +824,7 @@ def __init__(self, repourl=None, baseURL=None, defaultBranch=None, self.repourl = _ComputeRepositoryURL(self, repourl) self.baseURL = _ComputeRepositoryURL(self, baseURL) self.branch = defaultBranch - Source.__init__(self, **kwargs) + SlaveSource.__init__(self, **kwargs) self.args.update({'forceSharedRepo': forceSharedRepo}) if repourl and baseURL: raise ValueError("you must provide exactly one of repourl and" @@ -786,7 +862,7 @@ def startVC(self, branch, revision, patch): self.startCommand(cmd) -class Mercurial(Source): +class Mercurial(SlaveSource): """Check out a source tree from a mercurial repository 'repourl'.""" name = "hg" @@ -833,7 +909,7 @@ def __init__(self, repourl=None, baseURL=None, defaultBranch=None, self.branch = defaultBranch self.branchType = branchType self.clobberOnBranchChange = clobberOnBranchChange - Source.__init__(self, **kwargs) + SlaveSource.__init__(self, **kwargs) if repourl and baseURL: raise ValueError("you must provide exactly one of repourl and" " baseURL") @@ -880,7 +956,7 @@ def computeSourceRevision(self, changes): return changes[-1].revision -class P4(Source): +class P4(SlaveSource): """ P4 is a class for accessing perforce revision control""" name = "p4" @@ -924,7 +1000,7 @@ def __init__(self, p4base=None, defaultBranch=None, p4port=None, p4user=None, self.p4base = _ComputeRepositoryURL(self, p4base) self.branch = defaultBranch - Source.__init__(self, **kwargs) + SlaveSource.__init__(self, **kwargs) self.args['p4port'] = p4port self.args['p4user'] = p4user self.args['p4passwd'] = p4passwd @@ -933,7 +1009,7 @@ def __init__(self, p4base=None, defaultBranch=None, p4port=None, p4user=None, self.p4client = p4client def setBuild(self, build): - Source.setBuild(self, build) + SlaveSource.setBuild(self, build) self.args['p4client'] = self.p4client % { 'slave': build.slavename, 'builder': build.builder.name, @@ -956,7 +1032,7 @@ def startVC(self, branch, revision, patch): cmd = RemoteCommand("p4", args) self.startCommand(cmd) -class Monotone(Source): +class Monotone(SlaveSource): """Check out a source tree from a monotone repository 'repourl'.""" name = "mtn" @@ -978,7 +1054,7 @@ def __init__(self, repourl=None, branch=None, progress=False, **kwargs): can solve long fetches getting killed due to lack of output. """ - Source.__init__(self, **kwargs) + SlaveSource.__init__(self, **kwargs) self.repourl = _ComputeRepositoryURL(self, repourl) if (not repourl): raise ValueError("you must provide a repository uri in 'repourl'") From 9dc97f447339206a55bac069e692f49d8167681d Mon Sep 17 00:00:00 2001 From: "Dustin J. Mitchell" Date: Mon, 28 May 2012 13:40:28 -0500 Subject: [PATCH 111/136] upgrade tests for 0.8.6p1, too --- .../buildbot/test/integration/test_upgrade.py | 44 ++++++++++++++++++ master/buildbot/test/integration/v086p1.tgz | Bin 0 -> 13460 bytes 2 files changed, 44 insertions(+) create mode 100644 master/buildbot/test/integration/v086p1.tgz diff --git a/master/buildbot/test/integration/test_upgrade.py b/master/buildbot/test/integration/test_upgrade.py index 0d7f46a6a17..90243e5feb5 100644 --- a/master/buildbot/test/integration/test_upgrade.py +++ b/master/buildbot/test/integration/test_upgrade.py @@ -539,6 +539,50 @@ def check_pickles(_): return d +class UpgradeTestV086p1(UpgradeTestMixin, unittest.TestCase): + + source_tarball = "v086p1.tgz" + + def verify_thd(self, conn): + "partially verify the contents of the db - run in a thread" + model = self.db.model + + tbl = model.buildrequests + r = conn.execute(tbl.select(order_by=tbl.c.id)) + buildreqs = [ (br.id, br.buildsetid, + br.complete, br.results) + for br in r.fetchall() ] + self.assertEqual(buildreqs, [(1, 1, 1, 4)]) # note EXCEPTION status + + br_claims = model.buildrequest_claims + objects = model.objects + r = conn.execute(sa.select([ br_claims.outerjoin(objects, + br_claims.c.objectid == objects.c.id)])) + buildreqs = [ (brc.brid, int(brc.claimed_at), brc.name, brc.class_name) + for brc in r.fetchall() ] + self.assertEqual(buildreqs, [ + (1, 1338229046, u'euclid.r.igoro.us:/A/bbrun', + u'buildbot.master.BuildMaster'), + ]) + + def test_upgrade(self): + d = self.do_test_upgrade() + @d.addCallback + def check_pickles(_): + # try to unpickle things down to the level of a logfile + filename = os.path.join(self.basedir, 'builder', 'builder') + with open(filename, "rb") as f: + builder_status = cPickle.load(f) + builder_status.master = self.master + builder_status.basedir = os.path.join(self.basedir, 'builder') + b0 = builder_status.loadBuildFromFile(0) + logs = b0.getLogs() + log = logs[0] + text = log.getText() + self.assertIn('HEAD is now at', text) + return d + + class TestWeirdChanges(change_import.ChangeImportMixin, unittest.TestCase): def setUp(self): d = self.setUpChangeImport() diff --git a/master/buildbot/test/integration/v086p1.tgz b/master/buildbot/test/integration/v086p1.tgz new file mode 100644 index 0000000000000000000000000000000000000000..89073e612553f401c6c18bfb45299c7a12944bc9 GIT binary patch literal 13460 zcmb8VV{m3c*Df5}=ESyb+qN??C$?=nnb@{%+fF7yc=7wa>-X7rH~!k&zV9d0ch{TuXn#HV z#@8?nOC;>*!PW8W1AJF-SOfa(_4Vb=8@$i3pXllRJ7`WZ_O`D^zj7i9jc$P*3mi&N zW6yx&12dYY^h<4=zC>FoKR)KEN!Hx~E?QSh^xUz}Mt^ z%JlW8c@}Uh^@!$@?U4%-&!N?$JE9YzF$L|v#`K)S1!`)8(Wb{S>h(Z|)wSQDD!LccGD~8DM5&Wic!qmkJ&pCWb?2AZ@6$ z-ofaxV{lUInJ8D&423A@Y99ee=gP656`Tm)Qn01dLQLiC@hF z5^BK*u9yQgSb1paq>Bntj@PMa#0jRjn=|ve_xh4*Qfv z3XK_=i=0Ss$+){6xK1N zYSkorB_np^mG2!Xn#&EBi(jVF#VEw=y0$YSjYem>L6*iE{BWl}5WNF@m@wWu*v&Uk z0H`zl)pI!0-YH#$n#rhKTl==-GR!BMPEC`=vYTlI&u-hIvu>LF=g-i2SE?7(WiE;a z`$kolc1uIo9RE|{3TSSyas#7U6=uwic79o*e3(sPpfu=%Qb43tIyG*QIr%k9bgsE) zn6|co;}i@$w2P$mTESpUIZ7+dcXn&C1@VpAw6x8;i?72(;;+-|Ch4Nw3YRBYmq{2Z z?PSu^v*HQ0L~?G|;@InQPZo#b40S;)NG2(*Z0n%uvw`Dc3jK5--grIB5%h@65lSNx z4k}8WxEFO~KXw!t{D3z3d@StFO z3XO(g6w>6ZPv(xQXh$N+E~QE3{{QiE^HzON(SK= z<8@D7wU@y&KlB<$3ikOKUUE7|VNs7a!;B+=6D!=$998032rQT(XzRZ|OAdg5_yn8$ zH{us`LIgVa;uK;=#@%ypA0oErO#>6J=+qKqCcvFu@kVR)u56QLnnQAP$q&scU219wdviM?$%AMdr3;w#nX-)?-)ss&u&0y zL|!mWM=nCYQ?F{2dAMls zIh5Viw9#2pNx!jZ-$X$+Yu>?r9PvuXn`UD>ro%9W7) zGk;aCv0Xn2IjxAm7k|HdTJ*X4L@MbIwyt-LdcWm4)M`Cm${a3r_uU{OR(txcvGcUW zwlPb#u|!hkb=eSQ%%?|pg(jSkjprFDYNn{?vNk|RE$OWlx^p!tO&T&bO?h#2`hw6Z?i3rR(y`(4uHvCaLb-HtsP;0Ncm&P8cp?7mn?UGg>Rh$3ebstiBZ$j@FqXkr>*xZkqkcZMY>RC-sk!UZ&@WqthVua(Pr zZTK6k+G?pu=MmhvW^ zTaw_E^n$=QxmX+iEL3|yDsze97m9fqI}k;RT|&m@R^9Zc+#TKCN#E+@Yd^urb0+A| zzb)b;S_iy3c;*rYMC}K`|A)j7sSd|r-%qGpfI(kmoBsj?igl>iuJ?`PBouN6{7F z{SR+KF8yca=v!Yz>i>QDpHu%u3eo?Nfmf_&s&Qlhm2ME~KZ_%jR`4N8Wt-gNW8@~1 z-=-Sm3|{a?oUdSDJx_`-Z1UPCp$K%*9KDJ*Xc727b>4#@M%N4N2-cS5>}RUDZ)mYM#kE&W&Vw>-IG?*B~&(U1cL4!{2Lgn&CRVk_LU)E}HuEsJjDpmVuH z@R6dDWGscq{zt6r`M>)^WFLMI`BH?}yVbbtK&^w+j)pTa80EchNG99ZhmFzM8Au2o zkWJSCCj;toe(UPqciM-KFgu+lF7B8m(T9+Z6~B0a zMW0QC<3XVW_tY$2(u%NVaSUWM-hzIO4f}-{H}zhWBJe-d103YWx|g?){>p9oRE$h& zstuiNYWH5Jd(NSmlPTg1k2`9-cjscN(FYiza`BedXiXE`2)+{hDH~n3B4sCru|WWi zZysnfx}mvgk8nZ)L`1}CEZY*PGK|9D^con~Xbe9!2z>AtOB>oJnKR?4w6ChpPxGu~ z>Io9!Vm2%j%Oz4KT$x>fB62SoI}QS}(n4)*nG>rhUiz!=PBw%HkY6tk`UV*EF9`|( z{|EN_ls|xuCU1K2MyNdZ)9TNvAb(B`NP2s|jOll3dPY&ddQ6-2Xv?81sopH$a8`6IN@;LNDCg?aQ z$BkKG))mHTaW4~)u6knvl=bH8@s_`D=}Fsw+%M}!5%Dn5U+ZZYuBhZk(@TWHT@Uo6 zpY}t^8EbfdGL%<})>eDLZNypc<}boB*W2|gL(4*^ zP3W+I5M-F(tFb1z2tKoA@JF^l%N^0-Q4iIhp#bUL(t~l}NsS@*lA&dS)h&eTS|z6R zEcE*(mr!}DVP*agpM+AC)0`Kk<(hS}dKKGIyJntG^*NKFN+Ok-r(&v}6g`mrB+(S7 zIpQvF;BN@|ot=HZ=?9sPE9lN`z|6p8@ncI%7L-l~>>umU#)67bv1po3^Re_Pxk%Kb zaiFKE$42OJkT@Pvsm_~;0HxzpC-Q%@+PMYO^LuH z<+gp9Z{v%xzn&kQ%790-(WW~BI|@#a=FJA z*`)vQ2g5Ke(TvO{a)p;|C2T?Ht7x={9g%+=86t1}A>IlhsnKGazD0@Cl$?Kr*d|Qi zjb3b?Qaj?0A_t#u1Z76E)Q7W5%vW1XyS8?r@{_iNv~|{?JxQG}?A-ZyTO`17_;VD~ z5GIVP=Y=%!3S|~;svo_6@E-5Wzc_~bz5Io&&RV)X1k51eJZ-Jfq=egxsk?aFpiH(Z zQoG1qRX1whort({wfX_wFYGLlAkx8_)*ebDDl{kkq>C3c+8`A|kmKj#yRk%!e>@MN zg=m4tXX(Og)tWD@&82p0c8z++*hg2UyV|KeIl!xrr z-)aY`f()6r&y?I$vn73$*v;MNgP{Z7Ku>7S@CKuhH1L6 z)fdIuguv0plr4f-k27pAi{=|k+f+!Xf#DIdj1J=7xn7 zN&GL(cmoVG#xDHsmwg3_{zp;zk<0@{p8g}pp`mXFQlA5sGeg{brmv+fl z7lYT&I4xL{FhdNml`8H!Z=nUwn}?;2IGe6yq}1nZu3P_%h3#F$x;%hz`@GX?(EE3<)53Ph`gXw>UPe0t7tmJ-FpKH__VVV6FYutcAF(eC z^!RHHUhhO6EfVl#acFd>;`MFp4Mv^uV%uHx`E$JSmQig^wQa3(Yq-DUUoCBiTBQ!Q zU_@rRkizpG+bDEc<*N?OZHL`WgY%BeeORspGi%|woR({Pc18He&964^$5{0}74$vX zh|vsFJJeIRnedk@cRe>o6()VaEU^h^I9t-npjEF#?{f`u=NNgid&?N)ry!Y}8?Rf% z9;cn&Rz9JJ6r*y|o7M?u80}-$!zU{S8k=ISI-PDz1V>{E3DXSP@TVoVj)-&d<1H)O zI*$CzZ#G(;4;~~@oY?bZgU$?odG+MSVCU9zXWOsjCIIet&MnTl!ghUSk03}glA7pr zLasEVnyWe1efqpDORlpW;JZ^vcI?zTG-J9y1KysY8+BK;HOw2Z5K#hyvfoNvVD4^6uSzeZPzK1b1;G9>CuF z=PiHB?Won>=X|~2>q&x;w)5<-bGhNhaNG7BZ{SO_kAK_E+Pr|b^LCxu{@WOOIrNL6 zo4`qIUfyZm&Vj&d-cSQRkpoEi+vGrX0`mOFYfX*;yS~rMvx_f)6f;(3C2{@1RD(f1Gc z?mreDj7dw_Y;V0b;rD?=i1mvvYClUykQo zOl>zl+1uZL+X0x<=Pm;1z|`;0lfUov|C)hI-1^(E3oftIM&DOV%E0>w;I^-^##z*B z$GZRHe)Y?pFVDl@?(=|CQ>r;d;5N9@=vvPkf6ytBkZ|s|=u@~>X{&3$^8s0ZK=~*C z^H#I@vX-L%?Zg%+=_}h+*jxx3{-jAs?y~1hf6i+8W2MCg$$CYt8IT*1eFe^Ofh~x0Z2texk zxV&c4kQ1wJxq(WbZ8EC$y1S*QyA+??INQ{1wZxg{O}Q?MUX;MFGaKP6*L#qRfM;hN zPv$h(Da8c3f>?$zNI#hh3X;Cf-n>i|(EnHlESne;=2 z%MSbFXm!J;VeU1YP}fsb?wy#FSk>Lzk<@N$OCYbB_t)%p z-$c#Sfc2Kii-(y-*dNQPQl(7a_9rcBMQ$oZaAw4Y@z+ zzN&z&5AQTqk@>lOQ}rrwa7QY8vf5{)Q1yAPW+%F5v-)RFK$EE2!p>aZA{2x`H5x3O zaj!!FKQEC#eR?my-FSKW*XBcgwNd()b(7tO#>xqb3s>7T|mmQzy zuhxY6hbCGKX*RbyFa&Kc+l0b(tyv@}k(E%+!hSWKGI3MYy#K9t^b?YMT+QX+Mb7Ed zGuax$UuS26;j;FK>n`p6K0ifT&CS8`IG0+DfO<)LV(3xtY8)~`49IT>R19BuG)_Ub zCJPQLp zX^5FVyBuYBdX05A2fk>Co#+uQ?v+v`aegCtQRK|W+zSoC2TMAzfh7c`hFPw+diUvf zE`$;>{OG_Tv}RhMK*|V{{Db{%>SU#?o@#gCpSOXs|K4i*tpxbnysikoqr-WQ_Qvaq zD5mIEyUPjKIsLRB%X61H+x;AMC!EdvlQ}N;CgG{#`m?QUa8T!tTq%C38oSVXbBZwN zDSsJBo5+6Og(mZxub&TU91QDEV;aotXPQ-StsTy%MOt+MqoKLnV}M=adf&0)ueCp- zr-o7+S)2f1lFnT%^5ii!BSxhRT`hbZCO!|WPOj~VYmMQK zv(j;R7MlHaJe{0w_HG9U8*J-L&G~2HKMmBQ z$fM9pwS6nHF+u_06U+K4-kGmyWc%*#WF{#IJsPg%r_Z$YW8OsIo#hAI=p%t1cs>C~<$4xw#W! zOn26dv{J2VQ%-&8$V5Aj%d{fnSU*2_W_f{eWj~>K49;0|+BUD{UfgYV9K&BWr8YC1 zedc?fwjv8{cZ`$}k?l1*=8F1tylrI8FhevFu5qv*j~qDMT8IQ3dkxs`wg4kj=*f|f z<1LJ0m2Is%o7T7nB#ggL%IUoZySygG=a0_2E6fya)#P5sirwSdH5<+vm^6EvJJOvK zT?9$}#%oq76ye!nv1V>0p?_HO{-)Z8?Tqa3q<7pVUDVUerG4aHW@N07GR|-X>b?^G zfv;uXIy*fEBo<{M-kNMe?zFkftjo|&A{gZ7w9xaa5SdQl- zuLOUtkJAxFQ#)qlv9QxI2=-pRXR?VK=im3@=<4BXGO*W8+gQQ>b5gQ&_Svv!kFy;U z`jLrog=r~_#beVOovX{2#n6RRa(J(8VESd@q_#9h>Fp%HuPvW#LMhU{t3c=B`-SfE zj1G1XcXE|RbDt^Rk4K*f-@>9rj9JKPx0K8+g_11$y)MW35^8Y!C32qQ{QO^Q=2(e| zq53_3_n`AkZU!jz8a-9{O8LP;-my)yxUGGB|4z>ct9HZdeUE zSLT<_c{O*^>po%Mu}Nh31T!~ia|kkTj08P3W6%*RtnA387(sY_2TIWaq6uD(SlSf? zwATePEt^C5=l-#x5uB!CcA^~>idqgv+b+yn0WhT&QZf+GBD zgn(Z9sSjl>Z#I{8G>5TVJD`DqGMIZ;bDJGA7FmDuVK*Qq&rB^Pwb?CVI7}CB>NQ9GQ3R^*i1K&f(b#YbmW#P z=*Am@Sghz@VO>}v9804X&~DV^o zNG&f`u8!k+Djx>%r-u_xAKD$z6Wrt+u2C}I(sdai<$kKEqQRBK}8 zptT-nANGGJXv~CVdG(=uB!5CjZkGyrDN{pm3#^#`K&})^0K+?os01ynybF{JvJ~?1 zeo>cmI#(Bwu{!f>EF=R!PiS9Tu_B1!{-_Yn<{Dlw|5!PmvHdAu7@+R}vdx@|WsU<= z>46gBaN)eL5PEhfW96A;$NqI~DS#5QjCca!sk94y`%%s7ZmsLxP(Bx;W4wpA+qev9Ou?bE?J6S2M*e%5*Lc>@|ng1CY~L_5tLMj=MSEM4&#K)O|gzXM-dg2!>z>qe-4L|Xqa$qE*PduE=^Dj_Fk&XtNl)tz& zMlQ>vK_``r0^2Qz*p25|1+^(tQG-^dmlYJ_y0VO*lT2$KSYg{7EG4nk>M9 zicAT*6i8ECwk$4CBK(1&vYa4}t0tZWEw~kobru15*R@pp?kxngQ56CP2Y+x2Sf3`U zJC(~Py;}JO^}U2*g#(&(9PbK=rW+b~(R)R^|5bn;g&_wxh)7Rau`n>sxU z3&C9=SE>k8`$T`+?OZQ_AjRAcgMvG`_JVg5?Aakfv3kxCPyCGs<5~C?@mC887U7;n z$tIc@^=r6Z#ymREW`^MU9PWf?#(>W1&&sycrcY4-MLu#Te$P$fWX>HM8Dj-^f)~sW z_{P~2uHsRkC&dzi?)qNt|7&L*xp;yv!Pi1xt!E?95${oIJdC4YRot3+*`8k%_xtH8x4O}E)+9r z9~@iTing=||1o^oAy0${>P0jez`WMDaY=ISOIs2$4r~qe4~A{LIAO;Zytt;}zjsz) zi-vxDfG>0`OQ6PAoL@xwxJDiu4Hi0bvs9c9e0ZyD0Kv1*rdUou;h#kSe&$=54npMK zmXBw>H99gY`dE%Mcax~-eIjfA>2Frfo;jXDPt=~dKiN5-`aIvn>%~p)ln;PP0TUL% zpk}vFl$lVi;tKiHX@(E!PEmXgA;9q5OTp*mz#|l}-`YfA#7ici&fE}u&KNFK>SPhP z58gy&=7H;hk-3iz0S_pK@M0Fahe3mojw`E~`Rhm*NYm}S7QiL(*UI&++kMV-jv1+3 zKmnnLeYAz=hyQ}>ONav`liC2?KRlgt2-KQ&0>LJ|{vwH#jRULyF0BFf#|5YXfhT_f zA@?qLz!sp@d&XNpURazr@Gbztmdmc?&1m-{Rqg}2zHcj`n|aNRutvEtt*Ezv3`A8WAWo2>`B5 z!{IS3&~wCCZwB_$3(C|r&eh$?Iu;fTp2+DV6P6>+P@3AI{U=k zgtP$6fD8-#2zaBG(^Wk#0lPZIuAQ^G4!meZSu>;u|3oQjrQVf6OC!M+ON7ynyq5Ga z5iX)c?5w-RTlb7-w5wRkf^o5wQ`cK@JU<+M=`YG;dSU=Sn!Q3?NRf0X|e&`6p$30`& z8fp9zgwEgZg0VI@!!q1NS#?>!l)LUrQv8Gq_alZj{n)WWOu>OQ^vo2Tml+##bUYXc zGzr%J?ZHov-C31)y|pwV3g>OZ1EH>q@2>~n9`1w)qS!ilwL}*9m$*2XD;7x7L&uad zT3Py@qWTvXHQxsUk zTdAdEK6=-{;;d{LGvyZ(^F6D&RwYf*qcF!+WkEu8%BznJYeN@1lLS}& z`+ga*6FxthO!rd;>_@cD@5Ftk0+>W^4HY(Q zD&cQzjt9d>zy0W%k7e+%6wyzpf{Y7YX>(nuUD!-i(X!j$k`FkKx%}Q@lIV`PjMLV< zDY|kc@8qwiM)>6ITWhIXawpmey7MQx*^{-ipJ;Rewc453e|L z1mv(7{g~T|HF&Ofy<(B{XUXVH5>ir>x_FB)vwed0~X(q8-hi|RH4F6HGyNZnF8TwUh-TJExx zTg;JplbsqgjLw$h0TbJM|M70)h{qYabnF`n*=zwlTmT)MVQ@5ZstcMzO^>2PGLfnF!HL8T5) zB%@0OuNhX%D4e|+I=zM^$8_GC&VTJ}dSjV;sDP5J$(U6b25x@yAwXJ8Cy!sOyD zQytXA6gFUL{#_}ig4AI7UB^pEBqM_2Z?03?exOsniP>)ieH$KD66%BxX*g{q!zS`) zTia1%eKYKL<-G0KrYJX^iIpMxaiU|C-9w0Y#j^_A@C&Mp0O;zeGcZBYG7|PD+KerV zbJs5TT{u~vf`g__@zWZ`q+e0$1H95tRK^%}HK`!{Xv>tC;BDU!IEDAgU9Xg!gf_6V zsT6DBg|h-%6kfOuQ`lKeZbTp|sJpxJXB3=cbZFf$I*}*d7|7gIG=f?+p~DgRQM5w~ z^wnogJFqNrb2AZ>s?br)eJTayNIdb4=ZLD>r{y<%a8S2Be|tvuB}=?cE!YvyEbfQ- zqV;{!B!CRBrZB|XKrpA z1m4yfryRCL$$Ee!vQ;jWJYP_Q=926R!?obUD{m~J0a7eznW@<-2NQ<8C66A^x=S(g zGf#~Rw&*2_wSSqBizsUC)&7doKTT4Rs(Ge>djm1vClPzZG{Ffu0p*7pw51aV?Gm7< z>%HBA4VuvS=EjxMKYzgMYczz~-=G=Ya7RtyzTMa&PS$%X?B;Q*B8n2HYZAc_YBZ!g z&^o=O;*xz#v>tF^S4(;`on-@)H$u&&cqKEpC%~BcGMV`Wt8%e8LODFfxIiHk9XWN> zWtTYArFW8@P8cXonzn0yv3GLIFW9J~8W>$^N7u1Bkea$t?h1>uKQ+}S+TqHOW2$w? zM1$g19jnCguGTOjf0H9Lw3Izr*Xx23#(*$zEK_%X0#(l+D=0lMV{-sEhQYqTUB;$_ zkTrtJ|L^vU%bvdQkH90E7`C;XxtUP`!C77inUBKlU1oFVv0np-pq!b&cN|l(4W+_9r1ZeUs@~uA5G$y8IBH@8TjMucwnrF#> zXo=%@z39;Hkx|lx=8M|YG1oU&e=I2`sG=HmZ}tY^=V_doVcBb<`oQSecbaeb#cN{# zVlk3e^Tc}pP`F>*pp2^bC5%oDu^g6I;fp4Hm4W+)a<_1h1nyi}3K7JHSW+z^Pi;sE zKPWAA2>YpshS9}#s?7C!qg?T)l6)nFAl1Q$K5Rv!pX_Ax329R;#{4^Ar$&VQ;kC=k zLbr{%VbD6RsaqUc+S=V5YuI7C?j33#T8PA3+Gib!4{jGmSJKW;1xF{WYHRY_R?bCF z*TgB^--)*obFtu$1VwzIlZnGzmJD0~TJDB$hUi|NV(3E?PgX~?G0RC0NUydIcGzeo zK?Nf$D)SPSffBdt$3YRA(|Vz)#D~*ghuKKYG;7d&>xK3R2wa?+GNgvL&iQ z|Frwl;Nuv9vG0jDP6irjr8;4=K^qKu4*EK>y5UD~jOLYRK!!1Mp{a>PXY#_2EwE+N zYuX{ebcWo-5ch4ZokgXeOA04l3n}9c3XLq=oO|Wa^62BprzH@J@Bcyfu9vov@JCP0 zjT`}%2S~y{)=mZ;R!iG$t74O=J-s(}<#aAUHcZF|uk5>RRr>Shu|?&#Q&E)tBl$`F zo@kbMopXnF=@u^X$_HwW_X+8}@*g@8If+0YrOmRe(9+=y;K$OJS_aGZXnwFtAyr|(c>{j@^ChE{Iq{=MS6eIoc6%H6t| zcPoKKZ+;eYg(8;9>@oVgus!--dLbF>O!Kr(T3scOlO~yodoxK)b0|=I;+*qN`7D79 zU6n-VI#ASSe^AjY&t56Y&w*xak6=+B0{>vU4AP1p1$- zW8N80QO9MM%I*mkc`}7fU1KKk0osJtt);qV=x3f)_#87l%A_B&=4h;=T z49cZoqFj0_CZ5v*L(?jPbO-Z7Mb!zTsZD{t#i~?bYCP5*vU1M_G;a5CcUM(py|rB$!=5ftSA107~qdH?_b literal 0 HcmV?d00001 From cd7b3e5e8ce7dd3071866c1587fa6d5c38186eae Mon Sep 17 00:00:00 2001 From: "Dustin J. Mitchell" Date: Mon, 28 May 2012 13:56:43 -0500 Subject: [PATCH 112/136] unpickle v0.8.2 pickles too --- master/buildbot/test/integration/test_upgrade.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/master/buildbot/test/integration/test_upgrade.py b/master/buildbot/test/integration/test_upgrade.py index 90243e5feb5..6fdc1ad7e10 100644 --- a/master/buildbot/test/integration/test_upgrade.py +++ b/master/buildbot/test/integration/test_upgrade.py @@ -382,7 +382,21 @@ def verify_thd(self, conn): ]) def test_upgrade(self): - return self.do_test_upgrade() + d = self.do_test_upgrade() + @d.addCallback + def check_pickles(_): + # try to unpickle things down to the level of a logfile + filename = os.path.join(self.basedir, 'builder', 'builder') + with open(filename, "rb") as f: + builder_status = cPickle.load(f) + builder_status.master = self.master + builder_status.basedir = os.path.join(self.basedir, 'builder') + b0 = builder_status.loadBuildFromFile(0) + logs = b0.getLogs() + log = logs[0] + text = log.getText() + self.assertIn('HEAD is now at', text) + return d class UpgradeTestV083(UpgradeTestMixin, unittest.TestCase): From 3e1071f31f98f9e5dd9f3f1ad2837bffdb6e7786 Mon Sep 17 00:00:00 2001 From: "Dustin J. Mitchell" Date: Mon, 28 May 2012 14:18:48 -0500 Subject: [PATCH 113/136] use an older version of sqlite to generate the databases, for compatibility with slaves --- master/buildbot/test/integration/v085.tgz | Bin 24365 -> 23665 bytes .../test/integration/v086p1-README.txt | 5 +++++ master/buildbot/test/integration/v086p1.tgz | Bin 13460 -> 12672 bytes 3 files changed, 5 insertions(+) create mode 100644 master/buildbot/test/integration/v086p1-README.txt diff --git a/master/buildbot/test/integration/v085.tgz b/master/buildbot/test/integration/v085.tgz index a2ef15d6bec62f93a06d014fe0db08e04dade376..90056d9e4e054240a509a2e755cf760e11536be0 100644 GIT binary patch literal 23665 zcmZU(Q;;r96Erxsea1Xvd*+O7+qP}nwr$(CZQJ%d`+ooKMr_1hbYFHyDKo302%?~X z1m}k2fIzQxV7*aQI`fhwl3L~YVS*LSm)wN(B$C)d&>2Sp`L&8|fYP)ENi*1tmWn5& zHDMfyX!EyWYXj><_QPNU348AyMs|K`s%z;ksSsa0UmB^)$2$XwBLgYkrls+jCyGS^ zq7t9VvNJf(vYdLKayTdnrZFK2W{`ASu$zWZC(*NN`=0eExHewHi1o)6H;;<39+{E{*z8LN2zP!v zC4gI=D*K{Ft3z5!l_Bfg_*g88$ZfwZ0TJV#Mfb`ddJoK@`329@)qom^^ zz~1?w`Nnf&hx&%(j>sS8?&a=D-if|-c(H+j4;BTg0s{VGLR?B?6<6u!ONI=v1{uUN z0YOMaFo;Ww2N3n=_3smafDF#zz7QHj^MNaWQIzdmnoi>0x;91w*eIJiZj3Wn(+7j) zwxBbKdJe3*TB>s_Uk2lP1tzez0CEQRi2r?aImYiTef|bof9LnU5a;iY6vqEU08Hc? z@W}@v_zm*n2NwJez6C(7kHKn1o(8`%mU(Zco=HIZec-R4q5|O&firMP1THEmNqSyP z-a>bv%vzQ=vaD2KJrT1Jkz96Tna-#93x9KE?|m3MuiQ5q`7k%136LD!7w^c79& z+|}07s>VogL5ddIQD!cSh(SDMY3Z#nm6QR+aZwst=29A(xx8RyuYwiXuu5b_HHKN4 zCiYjLY$&~8KG9i7q9B7Sc=CnmPQH`I{MU65yXk>t@D7}VFrb`hF6&J4}`lwges05Q-g zST>HyOQlvZG0AspkAFg_>k4{iQV##I7Yt9uS(V`MN$Xwx z1ncc#8Pw#aRKubDrzXCVB6ZQ0NwZRNnt21@a77uC4ZOQ|e@8f1pn4B;0$G!;w-N7} zlklVkz7Q6n5P`=?nQqmjVA{5~4iaYjizC+J*4_hmV7MIPQo5;`j}hd{AI;C|?Yc_Q*Rh=eda60~5TKe{l(C(O5PNEF5jAJ$A%^c~>*>%T9&Di4< zCj~SUrZ35~*~El3BLoqc%BIZ87bY%both@V?_q}}QtcHs1SUmqD?OY8agUED+S_WU z)@;dooLUa1A5R!uaPuTRKdwPA3!?Wep3Joo8exnLz@2PeT{+TNGH{DA$M@9+X^Uf! z$tr-WBaj`ZdNDj6C29tq;;h?M)!`VuW&^w9CJSg$*m&GN4A&hoXY357E_-q4#jk&(B1kpf@U zH)uoguCN2*S)AL~Wx%Muxy$U!(rg)4S|z45{;)^3~n^Ny@Vq zz^?x;i$zFCs7-o0p2#>I2TJz6c1*aUQ`}MYm ztYdSdim83&)Y7r7Q+ow7T9x#{+imfsSv!SKi>*I|&5si5B4xaN7+UAzvX{~1@z~=x zj8oK+B&4<5jrjo}Bpha3|D_*8&|e3>@4VLX2Be&x9Uk0amn)Z)1?^(XhFlY3m1`~q z3?U_aVufD|c^p$@8ZnM9EkJsu&g5a>ILq{9H3M(+X=);?q3qI)dbJ1`K(gw zkNqmR#3=`WmP94*$s0Xkq-q4cmw}xbpR_ibECvWwiVFse_eRqYlUF`cJ(PFPNX$2v z*kM<|$Yd()tuN~v){nazFrb*)Amf{v;>|kPvFKVo|f7_`_cQn*Shz1#>`r{fhk$-FXML0L%ds1AcZ(Ke=B~A|rqo%?7f# z5&RV)+~f2g!vy?$AiBx(Etj&q0e;v+dqKee<5&ESMpC$9(A5Gy4mT`N^?AMYw-3J$ zZ8|6gU`F&S;OQ^GonHSHfDNuGK+NRn;d`pzb}rdKQbhn*T@|y$Ai|m2Q(3yEQf{uf zcTF8xy7ty81bynDw1Kd|%se;prUDvh(2Y^rRfL$v6+0};AZ(3NPwIkD5wWm+aCgN{LsEo00#bl*-ZCm^uUp3 zZt?OjE~$Z8_s)&6J>u-*{h?e5LqHdm&7-Tf)3p@jh*V z6HmNR1?hpan>SUA?4f64@N9eSaTDfzEegYvuVgaqdmLbJo zF=8X4_ChCnGdsrjSTjl)o0=lt#&w754!&C@Fu{ zb+%C0z9frJB~s~qy5bQR9(Mv;b~l-lsf_UNsX1MPvuM-{t*8i`p(f8{+={r3W$p{eOhs{~h7~^E75PKn_HFn4egk z66bFhWS(aUXD5?-p=zbHNq-)lb0^Y!l(4&Quo6tkHa)2=+PgkfjW{Bi1CN(<^TS#*8jh1Hjn^-6B$sj@1aq2ZM5z7pFxZF zXD5^@F|7T34S)dL!BP1fV?ELmOy1BTvsJ)7yj3P83_|u==G46lYn6X9I zC}1Xrr=L>+MzopYIuwPA?~s@jDyWl%;8@ekXNBu3IwVIfz_uV=ZNu2Y$9K1WSWW79 z_;S*e%9W);3Kkr(<=_|*wA?fUJ4~dDlOlnaXTS4-APHK5!L3L4I?4#{{mqghLVo`KXRh4;-`xL4yk7x| zfcl2|H8x}#!|rLkX}kH~{}`xd2&&|RL~*Ji`?_uS-*#`0xKp|zA^*7oLRo)OsjsOk zwWt|X7#GqC|ZS?08BUY0#r{1^sHnGkS-utVd-@uhzc|oC2DXG7akzmq9>ioXBE|-Dp zVXRSL2twAlg>JPeN%jiZzl&5435=>v)btmWrc?JKo;NJ~;hX z!-Pr*)}XrFf6OtbK#-av)c?YHRI*0FZvN6aT~$rDf_7{fJuQL&*_%bkeZ>;4Hy7*w zZQz7&2sm^vnbsj{S+ERS`efB=VO#rl)uHoHa|9p}EpJLpg<%|af2B-9TJL=W;0E5$ z7yFIkwPy%$I@XO5?Y)enc-#v(S1xRGj&1Mm6IoJ*w>oDg0O`EcRoL}Hrs6qdEDo#%{C;5x?yKgd=Z!oWC3(^s;%Unse zgUulgYNHp0mK0*Ay7miFWD@DHIbwKSy2?PF2H@Jn4_RH#3=Ox`k^BkUy8UB0S_3=9 zQ*k;V8+d9pc6Rwk)lSu(DW7L3i-!cJ=wznk@0xvq*vV_O_O%2x3-tzM646brg3|iR zCgSha4E>BFMC5GJ?)4v?b8$v$8{I^G%F+1KVT?(U1qAm5M!Y8tSUiTj=@ERFf}p;i z8Zqou?)?}I2BWO46Jnca@-4<-=imt^3@^u^LT?IKgl;mxG%tkte#bBU+LIc)0vFLsBF9BzM^-X>GR`< zUHK9oxet(%;>q**0|4;9)ZraDmu5ZT*y$SLfirA;_|c3^meFzoc0QiY`T(eKr{o*> zd+Qkw^8}lZ2shuQD{lv&aw~Uw?;bnMsH!wlxp{mZE*(C4y`%1LqC=G}uxq~_N3At) zb(_i?Sy_2~vSBN=GfOh*j4nx_pjv7|JWex_#WPB6wp$C1Ysl^qF@~HR>O3wYFQYCv zX=c{B&iAFs&|42&K>?ecXx7*)R5SH=`m@EGDm|Q>Ny}V8K6!<%E5*eHLXL_nRq!}> zv*z9|2g$T4T3%rkskhAf^u{-!ptNf|zr!1Ka!GvIUr%mVytgZWN{`py!m{5B71{6h zs^8YfB(beMoj1wH1A0Vy4G+@|qtUp()87JB+21ca^xbc3CHcOuQ>mMq$vRUfO*3Yx zKQ41WLMXPqP3jBPG?(s` zaXh?_*H0}SPgU1i)+0}uyVO{$_Ax(pN}H_G)XaXP4m+?oxX8C@MFqB|_B+;6Zc<&8gPx zG|Vh=4Q|oe!+7^~Vb03VkH+n{R_RZPA31q#ryo9B+gX3;k(P;)X02A^%b&ior-LDh zt8-3B7cT_hv4e=|btU%OtCc)zzptMzUeD{1IzG<(?L}oV&mE}{#}0>MF9P4^>n^uz z?v9DC{W188tv4Hj3!F$DKf`ptwZ$$1&oBl&P9CpwPtOFun!}e*dOo~fi}nr!$6fZl zJl4G3rZ??NE4TE|;9w=W*hUhy6;A+FZv zfDYf!xTwlrz)|dOD302m{@@Mb?Vrp0yW04=8m*odVm`pQOemg(+iiQ3g~!8?1bmE8 ztHM%iz_wN;vuL+rrmJX|3}ct)n-q60E+Fj8j%k0hau!iGo8$XDcMouJU|05i$%jbR z{j=GRH$$K`=ljyO71v>>Lp&h-#5x%KCb58nrDXH-jTOTcKSXJ)pzFN!b`d;Qc zLU4SY9@jDVtp5#TRT0S2=(u0?$l?B+6|(=W)7fM6+>B7F+xA;MpP<1q@z#&~UA2O* zgtR)f$ldY*ToyekKOxBZ?bm*>DfhiNe^e|$-sWuht~Xh|J^FC&v{_Ge{3fYRHak2u zi+PH!=h6PO`n+^T9A%*9Wc77DNli6TpX`6tz0PhQ^Rb_W>v?@wOm?!i;p(nGXlAT= zT|U{J{lqQ2``vWAXeZCS-OI5W&)6X{H0vI|u$Z;T+x+hXlf1#Rt83sdR3Z~(du(ES)tGfEd#9ncBJ5u2 z_IkSNTS~(;cS)d;_X61V6&*-9Q=lZU01$siU5nzJR+3oiya0%*-}vJIGwj7x|1XG0 zr;}Na=+|IEwwBRF6L$5lW9qbqY9n2#>)v=fFDK0;FQmdZvWe*;P`GrlwPr1at~9DI@?S{#gp7u+c^ zQH%k2weVb@mSu&lR}?9JF#|4}z6v?Um8GX&%ouLfy9(}a0uugeM0Q0n&$ z?x`9%TLJ;%SG^um4LEeXL)mUecC|0X)xa@J>l^=Ss^viY!TJKV#fjvclqbdbEoUT6 z-H3`j8M@vpC_iSC3nHb})E}NFIHIZNA&-x2GUU-ZZ<42%Q=`-KSqWTp>pll1+((yA zwb^SPd8r9m`X-|Hj=w+i7`kncAtdx-Rv|sA;#GII29R3Jr?1Kux_$!v2%(U~g?H5< z=8M})kmVQ(&4-*|SxZ$&SP%PszFt(h32XWuz+vB&BTN2j-A$4(S$5uQ6*72Nx#VBT zi(Hd#51jvqy8cX`r(+37xiz+5Y^zq}Fb7O@@k=sb zv@y)MFqtFOj@PgQD_36!*DpXWb}WE;G@?@QaCUf`H!R885y9hMF?mfZ*&h(DMj2LQ zjy~?n_x^#7L?kba5M!LI_6q6FfHZP^Ezl5FIMlz@x48Yw1X3iTY)TE<>3&R?ZoIOh z;|-RL8y#M9*NqvoGLrIoyf_U@)7}STQ)yW{*FJ5!Jp-*IF<7E!#aFuS=oDq+0Y+rVL=@RXI>h+`x6_{>E1I#a^UVz&YH+IfcXnV&_ z!|FGvg$#}TA8F~M26Z|S-7{NB$-07d0IAo5s*yCezaL;Lp{KiYpN4iqT7NE z4#u-F(5tY&EHg@6)$46_uJ89xgx=HGoZ2cP;to%!Iideq^(-UuX*@5{P>%b>2L6y~ zeC&+@0Ngg;S?_w<>dN1U4}Imu^}~A)06m3Yo8L3JjXb?a6!$$Q@H(dl6~AM$d)j?4 zLWd?{UsZ>cupC&)aC`pGb`O#RQY0FRiX23cgZ+b= zyPJJ*@|92&_xJWKF2a3dBrm?r8H`)KlsdU`tu zd~Y~I2EDvX%J*w|63)IqP-V}-O5g((jIBcN?(W=O-2HP~8a|+}j7pP}kzg15G9YCQ z{!4-R#5Z+TE*9oq5Jv#Hj7Bjm zyxbVV9PS>(IX!cZ4%YHC>mlQMWWIcYp-q6yup@tzU-8lbP7(tk3Ml4hMSVy}XbCX| z2Q5lSWJ+W)N-=H*b#+ZiVd-IAMQj?nzm_Brp-uwO&wV|7AhUQD(%}}sMXMWLDEpXw z?2_LyYXp?M9OrQtJos3m8oPEF4I=CA>jV4Vn`SHFy%zUFvEQSF(7YESNTG&&LqK`qz8%$XkwpwB*?U1vFkA95I|*4tg7~3JD&vpqFkISVD9Vo^ghe_K zn*Pc0{twQ@<*TY~u4}FRA8xz8Cedr~9G-5*qpN*d^Sk}<2M|d!05R@0E2CRNKm9{+ zze&)38YAk`=pSaUoP9N#_*VIEJDI_{+zdDZk`Y+wv?l*4K#=hnL3UKonjB=&KC~KK zAaxEWmYNwoeRQ6DZ0KQ!2N%%+6^8u^pVZ*@@%i;Bf8oVNvEwVB_6%w*KG+)2&p3+E ztf%4KHS}7r;`)#H`tSIvrO(`|q4b%}(RJzmQMdxK?+i$7P(w*vSYsIbp>@cWX8+6V zcIwvK!)jRWS1zHGIyF!%nES(8S!B;FY)&yPyQZC%f+RxrO(!T(mjJgV5kDh=H5@ zGvDamNb*B#UBZLc1v3rfX)G?V0XzYQ6LT!n`D}iM>Gb}<{aHdRuu|xyK#L&;Ro4Im znk$&4FvXw02k%3LwhZNL)ABf`#=y+0#1Q?yb{YTH)XY*HAO)c5h<9)qwt^wi)P&!O zx0ep41sS3(x_#44Q=Vus;fxSCuC4L2TY$v+tr_U<`%FNwM7>C_F->}c_IdeIUVcie z^^mp!Zj-uvN^C$y*B^fIIw6q1)Glfr;RM+gUZ9fu`jAh<)!~F$d6v816@Sl#FQjL* zuvnq}#zP^E&_+J)sIebq6sBim9MojZsvz+{&s5%@+U{UA&(6^QO{3!DgT?g^gTAKA ztU}{QWcVKQ2?71s;UBhw|APT*3MJAXKT?W1JdZ0)>`?3M3bV{^!%(y!0qR4iU(P#|#6vd-?n+sXh26?&oU ziS+NNS_ulP6q7-^%dK}0NJ=&3h*b7`s2S*sD9yLnNPE+oDF}ZsJ*PG}cw81 z=L73{$~LvMF+PxGxG&CLLn(D_PyZAQp>nW>gBW$3g)mYOlM3!s=c5qE^@=Bd>p=yx z-X6e>+}8MqgGm4$OgZ_&u-~blU?dU7Cp9@CaSaj0)IVBZ^ocs5esNzYVSR3nV8P3L zPH?Q_^E_l%X#Ikcqxfm?B?*b~Mej+M9=P~|V5*$|V*M~7G zJ%p%3yED#y4g0ycVqO0G5`3fA<`mt6B~ujG&?=S(GisaUMiKnD^T9xsJwYfKL7x-u zSA^<@OxaLfNB;D<)C{ovpjGnfaZ)*sE#2@usHQg`Ms{8m8s0XAuMsWa1h=SsMa|?aU^hRFQwMB zFPly3+xB7m6m4u{87LEf!To*S1#RYG#dGg&rV~5b7~=BPLiY}A78eQ|+GYSs3x6D1On-KH=YJ+`D0(8WKcV1g`Lsxwst`D>lM#T`SMp(n_%&>hq ze-hHMyA0kJKk*!_=d4BK6RTT=s9?Xi*{qYG7o3n|o!L-TAEU@e7B&}a9x1_u&Yr9{nopl}G^E*w6LBOzkx z+$JwBjw81lG&i&|O9de<^-Nu+QjOG#P)_9JTx+XreU+QZg3ggvOEkr4@s>oYE4QTW z4*g#6GdY1ZXz>uERej<2FizlNE`LloQwY?P46H56{dQ&P;v;(9_IAYiN~kIdW&P#Q zZRjH;6caGXDq z7~Fm_?N8R}h1C5RYM#0(azz&qk$b;%)opffmD91aXk5li*->)UIg?Jr{8w|&xoaD_ z6)Lw!))IDhwMk+2wq>3*80x8o6suU&8x)M`1nabWgqz@nDa;&Kub60qqM8f`Kyf2AbuijFghvGZhZ*&~iWbhcXoi*US=S^Vdtf1# zZVBi94f;bHglv8ZdMvVC(0+v|xhh^l5`>4SF%?|4tdvA$O|i@t@mH4O78*f>H|j=6 zE=I?wDdk*pNd|Okxa)L7RF@?@%`scacAjaDQEQU$y7r`NOvz4X*Hs$B$Xog2k`B}> zSb?-mS(aBkD8|~)U5>FdB;1^GW;aUO|D0FhLF#kmixcJ-cf@N}n7SeZV^NyM7&hkJ z+}gw9TORzw;EA3_QzDuf7rNtq zs!ef?%&y8G@yQq^Z_9gDi5B~6zHpE^g)XP~T1dImRMU6kj0CAQ?rF`{&fYMysW|f-cai@j&Y zkUm#pG`|hG7ZX4HIEmYv;6&**06GMd)2KjGc(ib_WF2(F+PD%12%+7En*yo<-z8qe z^Mr$E7js(mEVtxl#)~}+amH>MTYxags1n}DPCBaw*{*si=M4K?ekBXC%&lMSo z7t|b83`&9uN!R$dqPP;qjgy5MdD2y|{}c+noggPFXI+L}hHZT$Qm8$QQ?qVla2sEH ztaifgjbN(={)Z0|!T@w`MAzqg%F#>o<5Oq{MOYdyU4yc<`7|a>y`+5hX+TnNylyV1>p_CTN^^O#V z?mQVq;H=koG>@;z`GRp06j=XXD8{c{=aLapir>a!6kEnF869k>L!vTnO$m!quP+GH zATvz-vm6JlfdlLs=a_KcvVfj~SnVYL=iw1Z*xh6G;fD9=*lsJwAiOIYeb$!vAoU|L z4MnFBVol4Oi*=5b694%$ofEEnij!rmn2v{kOb8O7Kw@{f5wr^x`6Nmk5$3O$F;E>l zg1gf{jGR~FO-2L?wwwYVL{gJsR3^vjmF}^P`+pqM!!&o1x;j~Jr0{d0+C*i0M?v6? z+0pE<@_r~>sVw!C4lRZk)TnI8E)(w*hRL8H7R{&a0ydk1j>f=O0jT-u@&m@!(FWJc&w?a~A6=xv?qtd!a_gI@X^csnfKP&H7;UaN) zKGDUOnmoR0oURV*b%8B&E<*%$zSGm$Rq&D)lwl5$hyA5K>hnVa01|gU^CEr$HsAC_ zG<4r%?|SNj$9M05I*PtgfCAd$TRjw5&FmqdZtwpbzez2>Y9Xl57#?!sb|+ofDSVnh z@T&lgF$iX!DWzp&lloNpfK zI$*rkKG>Ma#gsAkhRCDtmNo>%pj$X08K0k-NxOS zb}3T1Jnx@SSdjeqs5J?A+nnrXTI7fQF)(djwxM)ok?hbhk+93=Nw#zv!Ou`d#t~z9 z-jco1$%{NXI4n>ttF58-Cdks=8+6p#jt@x#r6o&HJKJn9Iw*bB$Vk7nE7? zIklEc4)}i6m3C<6!yd;G5-u`e(ZF!p*xAu}k-h4Sx0U{m6rdABZqlrsc*JpYc)SDy z4X2Stx3w=5y{01}c@oi1Gif?z*O?5>ufZ`Wkc<=k6H1R4hhy^B5h;F2l7UHgJ&*Li z`BR=k@BFj^i&9_w;*4FQoC~i;rki15dJlVd09P&ro;^V0wSl6iywCjFb%Vm-7k2Yq zSuTRDuwB9}Gs_D4d7asod*pMzH-5ceS zr3Xv1M4Ja*XLg&|wP)M#5({aWpCptyMl-FA;h!EQ03=S4gd)aLOlXb|lZjSAX>P#T zXXalm`P{I%UZ9)WTYHg-Seb%Q%T18-*v?x(o%rh>g3k&HOz<%7_WERIB2v{^jU`j3 zJDRd;z*@3{iVKO+ep5lZ`b|njIL!03MD;-mC1j~r5k0R~PgPJw5qS2GrmhQ7?m1@d z*Z90-A5aR7C$W|~arHG?Y=R+WKkizF%++Dhb7tJhXqZ}3Be#%c5o`HIc~Rj{WBIcY z90>lu792_P-jic4UuWPp>vD^RV_DhtvR5v&i{|V3^R9@$`ryG$sc+8*i^IHYv+#0% zs2k9AjyGqpHiAd!q1{Vjxqcu61q5{<5Fg<|mINR_qUZZ1cIxO@UOfs&)E+YX^(8{n zK+|?w-VY_8K@@KmfXO_onu{?6nw{6nI$x~$I1_`1Jel+YSH+=ebnIrJjfg%cD6 z(c5ctIyxdbZyYj>vg+@o9d+I&kwKTCuZCKu=0;E3 zAtQX7$ELW2f%7qnp+lLMq);Rx*UFM{IdWW?STRQ1|f*a&yqKQ zDq1b?@4}$QZ~&>S{WRo6t9Xl06fV>4+MS}+TeV^C3>MZA)CGB8139hImZTO zHPn`L81i<@KxI4qJjKXT+`Kg(eqfBhbo>K@Rn48rr6JmD!8o~~1)Th+EET(~I6rYJ zvN&n%FO`zceruZpF>G3+oW*Zc}n)p1UmsW>KE4BQO{YQ3vNYaT<+)mg(VQVxyD z-}F&>^6>0(xyiYR3pVTWiW_%}8w)FoJ_fph5gHZ?azQTHk#f?XeqOT&&=_2bmqqvG zFEMB+godx$@^o|Z7cP>1<8Pz!fVl9R&X-mjI@61YOlhZ%Yx*>Y)zqxkfXKbs7NXKD zHc$%?+C039(>!g?T5hlwaMzv=aA~dT>~SA1%zF7dqd{ZkZ=$Mp1tu|ZGLfQps^449 zErTHy+jvVefO^m%a$d^Ku^Kp61H?gX(qK}BB6`_dB$H}pr_0gSuE$#z+LHiFD)SZt z;-rYv6dQsTtGtrMXK)@`7@P~<35wKbGjpT33c-Aqt{lbcXg|$6>rX#CoTi9mM1)0k zw7b))xqM7>?@GF-j~+6*4k^R5+BP(O4){&pRqto5rEQ^~X&(wUKbtc>lQzDEYCnYg zdf)szMgi6-l_&+5#1&1slC~tkaMin9Pe@JzDi$q7*^Boa|Ly?Fq|6|z%flf;Rt3yJ zc5Tp^kJrXNHo;pqrSI+8X(FaUT&P}_)#Y-=XllE&#hoj4y`4uG*4NdOQu48% z3c5H-yp)T%{K^O_D#w+6A~QZZUjOHxb{U`*;YuX)o#v>jYVCuXjd+l9hr{xDzC12a zWC4+pTB(}y^TIO=M?TmcoLVMMIbz*Bk9oz*z0qM_g~Yyk?G{KTa=o&UWu4$*;p8u$ zi561e7bVkvwc4!A3?`tcwc0F=a^k~e4(lm3xGi$2MGg*;pN)Yurb(5N$i&?i+lw7pqspq#)7~3-k+%-@Vf--2|Imm8V!h6i!QwF_}t2 zO;g5?R9alJG6{Ex%)ligJJ)7O{=_KxD~S3iK5E?B@U6tNhKz3JNz|PbG)rtn8PN&P zwHr|(aJFq;(xYz+Ky;-Byq%g3$XhyJ52f|>OWvYQU3KEBBoZ@iMA3+U>LNKn@h2mnuGY`-3>^L5D=IH_yK*s^hL2BH#rWg}yRoKz;B zQ6e6cVkdfzzq2y*l~a+qvtV&Z@&$#vFIRo2T&si{joVgC;??1 zT(!xd)LU?p*ksARZhw#ku;a`8()kH|>^$||189Gxp69;P?)T;Z|C`=^@Fj4BgTS6u zt>nGaU+uK!z+;xZLy#l38I7g3>d3+b7h_n%RtIB@0&|O*IayV-x0@t7MQ`~d))O2| zk{bo(p@TVv)uZ#!dDl{eiO|6e$qp~@?9uyv7&esk{xYXT2r*y?+_xc=iztbZ$8K%P zNecT=ajoPbEg=&@DP{K$5|ffo1#*&~pp}o4lbw(*nnVoRcc`onCW&vu^DgD&DKB@k zA)gBVb<=?D0wRIoR$su9+wUje#})b7U;ENAA#-&UBDoXbo&=>OW0EsT!}OdNZ7g00 z6Dg>}1o^l@dW=5yS>vF$ZY1MsrbtB-HBfl#zS~QM-tnj?MT*ZW#}F9Bp2op3B<2FA zCy?uq4p}2qzNVSEN?sMmYb>Y8xfRP9)5aS+geHosVRx>JK<239u6Hx z^ADHluD;WLyUyFbKxs5vi$pGlmre2pm-#8CG$KRMOi%l=5*K3ypR2WVQc6mj{u@`X zuZWnQc1r1{6(CPpl;N`p$8eP<$^`EAN^WL)Szfc4y(&u~fikbi{m<3V<`h~{RZ*3# zCU3XXAFj7_>=^q&`cAS*k&172bjhC+MQKJjaPnRoq2tEPI>RBw^(={}gm_V3s_4~- zx|lNTMUmi%KMZI6GWUUFl1iujbK)m)oQ`tUR{`n^bM^(JaGlh4(aL9y{sUc)0Lf8z zn5oO|!@nCyY~?;GzJ1iu1$k^vWrZVZt56t}{$dJdV0Vn+Z`jR>i2{ zZJ{ij)oVfrC94f5=Ff3=uozNfLVR)QkNQEvOz_{=4^bXL8`oo@8BdX?h(facC=n(u zPvvaTK($za2=DCU{)y=($;r!0RGUUU=SmiRsO{Q@TGgJyv@(&3Po)rSPU7-iIzfvP zpX6*uL=$hf9FAa#!$+{}Lq-0|(rE*`jIvJy(SJ*M%X9X~sLL9*x5Of?_}gF|!xdA` zQV~_jPAIerzjDN?J9igPSfWJ*M1| z4(h8nLYC(5s|n%OJL^r$Etal7){TauN?c5pN3_N%M6n)1CIb1K*_%lUc8tBwAvgvK zmM=QS%johQVV-Kg9w64UEPoR_;-@&xC_I-&N_6YnXl@b&fbpTbHI^PeIpyZ8Xwv9; z-V687Ofy6}LYKYSA&Sf&$J5p5%*vawisvP@h$wC2`VQ%|+V=Dob3N-k_kWJXR^!{Q z>61%J2Abri>F_*znaDw}aPj?x49zR?J@9m!qvOurd@h*VUn_V+g)zX9o?s~(aQDB|ywN`YjE#C{hhMW(i@2U0 z@p$asmGj4^2u2DLmIG7;`D&!J#8e{>pnZ6*Z$>qlzsMEBvcM1$cb6P-&z5DfDQAMS z1$amg*59@T@-McRJqO__xfOPFWuLebt03xV8`yq|!WXk}VK~LnBsgS~UTH>y4=vO2 zad;hJmtbkt(Xfv?7G{i<`T4}F4TqnCQSny&Hs}^~_OOi2mhFF4aQ_hZJ3;Yp5wEmVJ@f7hptJ;>meq z3*X=%G@$&Z&kE<=%$e0>#(AEZnA>IHeWcutz}B(f0i_bX6p9eKa$Qu|(Eiyv64~W4 z1W9NdsVQYI?&ggC!*$5%652E@*PoNndhIwAF$)Bu?Q<1%EB<(^$$I-)DnY-W)e(-( zZt~WQZ{~9nOIz&?nF6H5wXlI$8BuNg7%iO5l%R0b(bAacqR$ub!!54S}j*B z^W(5Yi*VJtZY!Z+5 zZ-X8e+h4dI*k57LoR-1{;}P-2vZ1gnH5EK9oAQY|2p*9}y);RT=vCV(_{9<&1B{j- zUsSrOG($W^R)KKuq0(!2YX3o<#%yt&V}&^MNr{mZ?zeo)gZ@?13jukm%9ZEwwyAXg zysP?B(zN0O)n{F0)UAlpWS7MUX;YIvWvk?Wvf0LtA`YH~u7w0YA415G4@T%u!&S)0 zEW&4{R*F9mRnH`E&Gaai6dWdC>xb5>aFi26o`xq^3bO-3i1--@Zn7V$%tWW}(vg;B z4Fn~JuHjcx@Nllwdo;Xt2hQ}uj9PP^|M-^ZjGt~x8XYwKUArP5qQEZZ$$Ys*#Kf4j zS{{oIS22!_yApE8mI6f0w`Svmu zS`}_JuARAL8n*JDbmG}{7F%Z6G5=7WRK35Z+2I(=osKLFwA|%dL^qpSCq<7OBV+J4 zH^`t5wmWa)B#Nf5$(ve6bQJGMk?5QxtuHDi9JE<+UdNd@q@?>p;n}NfFn61J=p}2{ z1SpK91Xv9=QnW&NFnL<`KDs^H0fWK(Fwv^_qE#nM9K(4j@3xiF428)X7KrbCKt`{c zOk0X3uoaxAJrt9joPiP;=eO7p(9zmgo>HgiDrr72l!J}AvU8DXy$`01ALL50da#?t{J>fcz^3auxTT zRzyBH8}OpXtPij&(321J8}s>*#BAVi_*L4!a)VoN+k}<;?2GjLK>`?s(+l>pN_^W1 z)akXC*(I;pt^f7C@q@B~R__Nl$lr_C6a+w5{9gg!8z1B)>M&X_CF=C!2cP6CC(E9mk5GO{z_m;{f5Aj{tba8V*$VPe?nl%dcrRU!LR&= zz`FXA!1@Mr{d)rI7l!LQfu;PDz*6}Wfu;JLz*74ifu(K%*Z3WQ1+jo@{)WKPf}Rup z1A(ReCj#r*?+GlO-w;^3zYvo$;eZw!7Ru%N#qunaBWMnCwQ zmn^sQe@9>$2fVZm@vg5?EnDFqt5D zI0zQ-9}`#+p76*Z`0X>p2mF!1ih|wz1A!GC1dsWJz(Q#we}h@Y{xB-p%x^KPxSzu@ z1iKLgkB6atkWp?R7^dYjpzEwCE z2%dUY-}GnwwBOXntnkF2_33Oo&g!RQ>Ptrc5vI!c159-%2%h=>JWTb^^8bG=|MRE6 z{{svE{r=Ctr2Lxy4>*k+{r>*9Kkz@UHLvF*EFZ{>?Obdt_4jL;e?Cund^z@rfs(UJ zIP02b99C@V%}jD~-3O_k^*bLI=)>d=38L`E`cu@CuWRb`7sz4SfnhP=<;G=af$_rR z3&*gqu(L5=QnM?-`X3#LX4z)Hdt8XCs3^z4k`06XQ}E|v2w4BQ&VKgnod5uztgSc9%XSTzl05d_5=BUv4ZW*1gd>{^guxHuB53*x$%6P>W^>!2viHxE zhC>rv^7PK-P#^^oiWEG@oHT6AgK_X{%(N{~Z|$auy5;?7$$n5j%Pr)EH)ULVZx65v z4fUrTgcSCLcfNm`q3RJA)0^RShW%^=Wm`v1ExzxDG*xtjY!ctLiaVA5>_j)2v?mT) zI}z+~o+=GhB*=oM*YcQ(J%`XGg~;dIq-DCWy)JL*O?GZse-XJ(#zOkRU>x2(kbb_m zM_HBNyi`$^0F{7_6Kk!K8_WG<;5Y0?>n+U8aKz#yQ;&BiZSD($JabObp#e_rIxAjG z+?)6qwdK6k(~`5dz7<%(S=)LqD2F#o@ElHpjZKc8tZ><^V|Ivi zNb;m}_AZ@}5lk`GjZEu_)CSzKx$V{-*0oj4CAG8wc64JDkp=V0@#>L54nH3YG`zI_ z8s2F4G~4e{Z8^NcJRkBv(d1%;3So*G%jd~~^4l!Rp2W1v`c&bJW}W^hdQsPa$@Sx{ zQsHIC-WRy`PeQqJ304ZZz%#RUdOdTRd-U*z za4~^lcLOo4t=G|FNH%>)M|Xmdz^8cf)sIixWtdDClMqjMGh`|QyzcUhzj|D>acEI! zyL4W*bmD>)sXF@F^z*t{EV^a{i)^52mB&f>XANMsGJLMhiM`de?HY z*~bg}K-ws>o>@9JaLVyuYTU?`ujg29n|=P^z0KgIH#hKzHE{dA>65z;rrYYjz4Gn2 zf_`W6-iIR&-_wFC9g=qWl`ebJ`~Hy2nF(AR)2{UUL$`!g3hV8a-h!;!XgkwaX zIi4;0%q*rhH45K5x##dkv8=MdA#puek*mIQ^!?k>yFJ&hKZ7<33n~h7>n**wFwRrG z`Y9z%N^YdKU}g2Wx$^X++48_h&}jjl)kzjgEqUoD`BO!ZK;xXhFZz||UI~xvsum7` z%3OlRE`I8hPO&g*i14xcbl=Oy+c&p!WE41VirXui#73bZt=A~Sw+IZhB{!-+UU2BM z?p({6jlb3jepCl?QhnX=Zp1O*;pJrf;`l~;y#{f|gFE#^j;&itX=tc!vu@_fz2%XN zn=}kh1V9V&UgL`>x3UG3Qeo|h_lp39EqF@NgNt;yQW9(poWgvRW9m9vexsfS_Hkq4 z2`?QkU%q__2O3*^YUWK|Mmt?`TO|;D!=2XD9#>|aGEiU)KZ3Gsu$DZ|oTPD4qlA+r zBOMSOnA>MAr4ELyU+Nuksj}#WZHi9JqN>fq^QWW?$7gR<;X;VckB7t*91o9E;<73t zuO|))ghUhANemSGc^o`M5NQ{=68IfP@s7R~TLwFRCASZ^~nO z{Jw>p^WN^u(>${jprtH|D)-}UNic7)&Wc-oLGVRZI8lAdCk z8+?2MJWbcn1!=EitnI`(pY(gOpXWMbe>{I>f4V${IJOhL8#R zVCeuOZIc+7!DSm_)+27+-HmR-5yTcY&kWsJUj)0965*@MLsi(o#{r@Vf?Ku7KAQHv z!^0%XSepU%ZpXUkp+8>uX4gwEFvfTb#F|@kj){3#a{3$mBZRK;nHpAJx9i&KnT{|y znM$-1tEVJilqrTQ!#`}t|CYg|lcB4G zVxA!)|E`%9f3q1lo?7`0U+(Kr&)z z)5@p7L)ZES$s9b9n7g6Y>!lbeYe*;I%ce(+)A(-AV>8j}#-xZ6Yj=GDa)XT5U*KI~ zf85htqBY#g-aLj0qu%Wg1D&BjRm*Pa<{th&c?pfBXU)?ii#I*b*dIWMb|v6Xq`ak8=Hu(0ykbu~ypAR2tk1)g*P2M2d17?&-NErk_hDCeYX8?vx0TTxNx z6gg1_QF_cIB2dX>AfW8~Ke&}j7b_F5w`lufVSkFATUI8^o$%QQiNaaV1rG}1z%UYn zP#jw@5gyFBC{HDhr#`x9J%w9~1EzIZOEyJaUCyW!-}6Fe5bo@`#vqOj7Lk*LPI}b? z`reRKIx+%W>{;+RqK_&ZrT({PFJ;iyd0B0>7@>psTqgZWg>u|1N|I+B8L+1$2)mi{ zj^pwM#N(|I`MB7n2r|NUsi%@dL3!Gu?YZ&`bjDuPuXogJ;^{w~XFC68Myh5DkEWg| zpJS+8>4Ak^g5*8#xWIRh-OfKQ@&$?)AM+1#H9fU@XCk=&-b5f`@sY>Jph560ZFAecnuQ3yDFjSV#XmT;Y((0W!h{^nxg$Bem}iXnO-{S)gT zi0x-mk2yiyLItdYzUpP>uvg4saw3G;hSluNT^}fEAKm0=bgNE^bz`#odd~XzcDD%K zIcbFITxFcmLD`iqE#9#u^~(LzVm- zBzKD!E2Fa(jfGB3%QkVCN%gw5Rwg|y47>ZiG&5ZKJXSEgH3Uu zcxZQbXQeR*sotcw&?9rhlH%iQ?XA@xz@`~!`-GV86CsVhYAbce;VkgOnfh?pwc2VT z@Tg6|pa5jK8r~a`(UQ=1A@JtpDe$-)A}-L^tsiqW_YlnQVKK;+Jlo`0D>&gV)bknr zeA_BIP`Ap>^>om7yL-RO9)G4erH;;Gkd&xX{z&lf3+X4r`59FwI9vc2&>v{1T*LmR zHEGh#k}D#~+LDQsYkf-vvISV__ymX;$zKxsAb2?G=X#Q^e^5HL5-YUYEU31!i_=@e zCFJx}{JD~SyR<+F*dA1EY-}O`#Jedz+hU|}O6nXi=+f_La$`_D{6SA3RdV;%O#Jey z;1BcX37_r*GpYE!hd@NLH$%oV3<%`@F{AlB$ z%V;>6lt(lxP3ZNWLbl1VRbkxbje2Ec1^bRgNnL>St_A8& z8`I$Evp4&O`vL&i9QKE0R+}gbQJL5&U$`+et>dB=&*fdHKvaw)tKukswBr) z4pRJxaO=w0DWdM~e#dHl4z47}8WjM9E3~`cy4u*JG|k4wzOpi*&xBuyE4jP>!hIVX z>vm9=`^&hJ;w_5_*H}Bvs?ctkbgHo$t67T@@%Q{0AB2vrs_f11BSLOYesM+Locxrr zX%*rwfZDeDY(BkdJ{>xnFwj{39JEUw)+$PtcgLQE>hse*ro3KZ#z^?K)lOymA&zGgLitN6=d@dQ6GxqI8#@^IJczMCmfc}vLjNJwv} zjW4F(v3+kuMpfrHQvQ8BRknX*&X^Qoq9aSv5`&?u%W4bcGXR}?I=wEa+?cze0-&`v zu;Gc=6;vB)O$R^@Sdu0#KMNigzTm&fVzW+|>aVhN&yOr@jUR>h8u58hImg`$*yU-p z)+`V;TVE>3N18l*Gyf}k%SyESGwGgCzD-ccK3(kVi7QJ{+ZR|*RlPbL7$RAxE-OlX zN&5tF#Xm80!}cA;ebwfMkO0i5(C+u<8C(k%C7sn)ao*-r-5Yhr zLfzHoM)-A(b;gzvS?{Fgmcr^n(2~E$khW z9rl#;(nRHCb>_WZ$yfr`DJep=cNA2mwlajBGPhxyEV{jym$sOmz{DtWhOV%! zT>u>wH{-PKKCl3oooU>k##a*~ScMu=)nnCrRSA5PJ z<}j&c1_E5FRyXD_s#IHwrCw2TTu{#q+&*wTcGZo|;;x_)(lrx))Q(k|KmT;V=MO^5 zY-Hzy)E;*t^-N}7AmWafuXDQf-Cz%kNnEH7FrF_=o^FnmDT1` zs3ezIfU9!xP#Y>>q6IK%u}S*I4C-hGfIy3%+1S8pjfKQF-djfG?iveebg#A;MuhuT zh@*{??9EZcmuNjLP%(2IO+KdS13u>Iu8U3S#Vtlz?=1!q?=0(9E1Lx!A|i&}El6Fq zP13ucT~8kuGX(%9BQR@_Z=%{#qZOd|5pExU_;G3__Y?>ZOn>GWSAt)>T5AsEO0uzG zOh;E?l$ZRyObpX-v8_2$}d;@wHW!KTjtZB1s7X^IJ4$Pp{tm&3Iv{(0rtyEurKNw<=ogLX(!$1PX$0o+8x3l z(uQ8u(ApfY!wj^2Web3`um-Czpr zJ2&_4z{#(OnBFFO%TRp5xaVfh9S6!zX|z`?VL8VgPiDIS?xYbBg1unY9vJh|l$5G4 z@7wDKd}`tXn^btZlY6eiw z8uqY2RUs`S+H5dejH1iFWdzy=^W@AjK2fW*XF!GT6She>snzouyVi>$XG|t??;ecN z2+@61J{d7vV{YX4A?OM^=N4dm!PruH%j(?QNRiy-rizr|Kme+^3s!4V&gb?tF4j1G z@`Ze+V=FLV*bTzow;f0s$e$tJSu0SsQCVNUGRhX1en-gJVYWs2ecs8oiMJ{(CbDo< zo)cku672Ja@FLf&{38~&QdPN|JVA(P zj+X#C>R5%N57&P{wX`{%)a8RAB&#b>Ioz+5!*}_lY%`jHUy!asJR;oYf`BoVJ-;9% zd8Wpcw#~X8CC-$QHE!~%0b0Cmaay)K(Z=a$Q9=yH?4rrXTuIyX6{`?t;?k+@^mx*LCBap?F6V?Q@J?0mpT_`K{Aq(?b!_J*xL<2ao zk#XL!Pe7-DBAHSly*HKh6{x<-5@-^vUZ}IyEKnz4+GVz00ENID}~trX7 zZCM!EwOOm6dzwqrpN}2C2blPkSs3oGh7CHo3{}5f^tEg4kql#})?=|JVoTTV7!=E_YKf#p>+GOeMVVmM!%$5`J zrd|L*j>6e75m1hJ?$WEgSSO?~lk3z1Z8QVA?3le5c#4>`uB6g8fq32$IRa8m>qMJaZ5&Djo|XION|8(nmpBZ9Wz%0NyX8av9_J9HJhGk0M<4L z7^S;n9*vgU_il|JVxBdE4s%cV`GrochJzl*1`yy*T&dcNebOfut2!hW`&#ZWKIx7N yJweVRX9j|t(p%sEa+Tbb&gd^m+x%VrE`OK5%irbi@^|^Km;VoUD%z?5XaNA?MAAb5 literal 24365 zcmYiMQ;aTL(>08?-K%@GZFjG>ZQHhO+qP}nwr$(Cb^Z7A?(F0{n8`euNnz9&Rh5Y! z1qFoJJ|qVOdaVQNjJ=Z4%S3ENLoNy8A&qp#(0UYRZ!MK-&4EbX5@Wc?E^DOa!JGw0 zY+)_;Pil2~JXT4Nh^shjr=BK&2XZ|zNoUbnLtR&AGWC{Yz7aMgHzb(G27$&#Xwu0* zgCba_!1!+ZoPjvTIBYmX_^rvs?DQ9~{k8XYmFE?9BLnZEZ)p%_&&J$u)iboBzowl8 z^e3!q;ntONCnkOlI4|4ZHr=oTMOY*P)f@|KA0B^lmhJBk(nx*wFFg!Nt!gxwM?@9~ z-!Yi2b4T>j-oa{FU(estzNUg*c=Iom@x%abIlyfUF8#vAZaw|DWHWL)P;5}$z&lVs zs2?>;HPLql^V~W~_M3m^nk4N04D+GJ#|PG-(xJ?u&LPVo=%L4<+OWi-%3-u*=Qm;% z6+y+Jp>?&Qp^t!Vc*^%rHb!6iRwwsA9=y)7l9nN%v40npM6mezYp$>En>;$sXQ6nV zOIr&K*|d!mI0Clmu%_fpysF1oI?i!sQgN|VqCUX7^4BM0cFr|Ady6e8x@Q}bY7f9# z2{vI9(i_(6c6ZF%+Ml6v~d{`s=umaCDK`T^%Fxl*aY zlJ;~=i4rHgQcWh?QdU(Xe+2Mj&o9f4VcLqo&>8R;RL8Dcbzpc)US=9q^XXjJn2@Oa z16}Z#LBS0qVmy$0d|==M>R8!oeTV7D7SRv=e)QDK&!HggoK3chr!Y|M96Esn#kQb& znB(AOr+zi`x^7bx;Er!Uuzfxd5J5bBtcLZ(a{lF404WgQFCro>2lm)w!+iHu;q@fbY zcH^9O4ofbrtZ8N`N4>pBX_*K+`n^f10|IKN?}@qtqFhIGtHM%%-dY)kdQN(i5&uZM ziGcQkRJh}ZEdUKjlb-Dr?{~r@Z6)2Du%eUG+Pjvu4*~jsP&WSExKvIONP(U^uU6oE?Z)pq`+CFydBsZ*c*b@`7-Iy%bh@ z5zn*Y+cSW1qB6_#z?UZM}vWbNI@LX=T)YF3(d{QjKCBm2rEJL1NitC zSMdYzs}gzwKtX-LD*ygPd_*t*CMGzhTYdnT@^eGMgMkG{1<1Jn1!4dp{%33;$0rtN z-kR87Iyu^k>W ztL)N)MnR}>DlRO$c#eamKC=)q=`Zj{WAIaIURm0sM(IIPJ<7r6xZzQQYF}Ih1g#4pj)10@IL}_rZZAgN5iewaWhe-E&buQX@hN+N-=PlwI5GBmP zkRfpVs284>&4W~CCO>viW^V71hm`nk0ho-I_Heh=EH%oUR z&I}=R7`ixVGu^7V-d_wG{`vpFV)9q-{U1;tI93{5z1nd_#LoeADcLyPd&oW5k*AE` zKL$-X?D;2yfHcu0I=w56rNf)^0_vNT3nz4YX>@({iEx#vgC#kl;)M73ypY3jfhC9F zlh}%36DRHcVROV-uofdbhD+>YQVF_b+0*tStj}VIqNMCPs2TE#5~+r)N*5=V2}~@9 z(1UL2T40VgrNkww@q1teq+`0^7A6Ku?s`M2)5KL|zX`UBaY<2!*J^#78c_*m1?9Xk zv!Kg&{FeXu0v+=!r}C|4ai7$7&laOmN0LGsP?PD3&N_NGp1Ri{JC8Bf6&~~OP9~XP z4B)TK5SO;T#2cSJC1L@N-1J-iJ@fgJn+9Wd$M$b)SQ_Sick^eDuc>uc;Fu<|v`-&N zN#dOb@nqZ7ju@dZaYEGzFjn*sR^uRP&C=NNs4;`hxk2?5GBYhHoN#GK0G~9#Im(4i z1Q^;87B@C3RPa@6mN`ew~)yxfJXi6W#D5`KV6l zDp5HdLy$`5sLv5l9L&=Jm72CVg;MkO_~;tQ(P295xtI`TtVi&~^Ts!wU1x77nelyG zXRLmcM9eQ?s|v|QHF;FFvUE=KSP|(R-6*Xp7A)F`1JHp9{Q@TcSC1h7SBxPk>7X$6 zNd9~=d*O5gc%zcbeRNv16sy3MxPKxm;UPpUnhZN1W)5*ci=H=_iv|?1;{KWHh$>p; za0)zU_FsSs@wCt=sd79RgHR@J} z8gD6Bmf}b?*p+>xq*qIBCL2GHa{x--hz)GClnhHJ(e$H7eMD8YVyrf>+AHIgW)~_` ztsremsnGR(G~JY4xLyn@5~LhtbYiqBW4NL3ZCg1imy>H~k7zQ5Yn6&PtwMtq;Cay0 z_~Xp5JaFQehB#b=>{uM~E-Z+^k6-c0d4Y$`hCl9pqW)7k%0Xdu`fLK2f0iG7#A$xT zPjUY7=oAF>aA7BVh64YitX;z0U-C&nmj64{|IsXQDKbgwx)frbXng80ufO959f>T^ ze=U{Kq0gbsA@c*(gOvII^z%aCViTQ;tRmfXghpOE$q>o%ZBJMqO?wCM_Mbur|4*S$ zAgFBzX(EsV+ULrQ$f87eAj3Hy;jjSzu`=%BD8T#||Gzxt=K;h%{r-pO=b!&$P+9;u z1`tL+-Vo+D;L+xk@3b*#*%blpO4XUx$;M{Nyfu1e#B7byM0Qm~Lx$;Xv~v^J%IM@z z;$^5|U9P&dOW2M!%}Y`md#}!2@L;8P>w33m;^zLap{>g7vf7CBH?ybcTSq5wkP_Xt zFfB!)3FCrA-pCj$$I9FQ^hwcqz1(lRdbFN&r(D5NYRBvMOh?emFqbm6U&nT9K~q;* zmB=m&KWe}MsJ3__Tk5@Q?aukjyh>xy)I~C6E0I>rlynP<5r52`czoI$O1PxKNk&9i zh{Ld*b~)g`*!S_D{LktC7o6Z!{|`6)|0Jt=n2}#!3#b)*FA^W*?;)5w%&!bCZJtxe zlB&(ppX8nffemLb?-o;^3}?ud&nn9p`ss2GZzAQjBovQ@40^z>`L=qieVF*mFDh1r~p z8Z@lD1RCFyXG+Vz|I=Oh4>A66z3~%#Vqf6@HGhL9z*!X~(}N;E0=)9n%a1+=gxc7s zb+uL%PanbN z4V@C1=+RR7!<6?lwVLYT=dsd1rX6WK_Y49S_DdUNTyG9c-5p^&$KTFzv@Qcu z367lg_r>U^#FZmTVbCfQjXBv^rtydiWEC#5G^R?aDDOGZ0_N$#LJ-Z4J8a>|-Fd#tW{jgcavzbxNp!jKCLCc81U;<--Q>hUXVC7mX} zR5~qI-+>1YqMm45{VF6Uv}1n3_FU@}%*@;3V!+Y#gU z(hb$DkJ59s57LC@+*?iA#hRDT3Y9!sG&5r;CLIZF5OkK`t9at6CefTUS&YT9 z&Ql|*Y#%Y0&t>B$yV(Lx+1aRlASnJEK-{3G`)&Py`+b0huWLv|6xP!#yO2;h1eBve zbmHV262JTxXnZ_ex&n1`r#AE6J}@>?;BLS}7>6Sd#nqvT6&5`^6kejgUeOd+sW#hq zi-M}IHY1H!Wo>4SZj}3P zJ>VGYi8-LO8%~f;2GlBL)n$p3Nkzh-i^bt@om6s|VG%=OtW1&ub4t9;8djnu?SP>P zM8Y)55|>FIdzV;79p*5=kyD}tz9mT+{@Kf-bB{49-Lp2kUoM1ar>>aI0sn;CBFd&$ zRzVRb%KX$Wc^t9jO$CC9L%I=SoFgZx=G>ggr}1a`^m}^srF&A6JL{awz|ym40<&ARXw(h-CvxnJi1wos`9^(@fF&?+m9<3l)GK9HfLr48xmRAHilWg~X&vX*Vzc_C`K+er-d-F6wb%O965nK} z!Aza3>CK;G<4`^PY^2PaRWg9@WaUHuw{Up)wc0@$3B(=TnFp$S`J)!3<0J#Zj&m#8 zwRO?Rh0}jqZr8j9+G!AZyg*1LSPyi$p$xOur^R|S8GpU zsH=9@65O7Xc<35@{fsuChO|5=YiD4%L1XxE$hDI|WOo2XJ+LW>NW9eQCF3YY&bT8r zY7(v9BVrgIqRa2az-Z)UT)H-SV|+4T8zUG#ZP+8DA*fyjYjA`39E?s%dwj+^(RMJ; zInAYWAS3QBQsx%R`+UVw!G{yBeL?EobxGXR!(`wm4WDyqX)If|Ry`bmkAcodD71omdeTUBboJoS=l5lvFAfoc4N_TUbCU#!w`|<+luDj~`xgS9~ujj*oaqy*7+~kY8l50r2 ziKUUM?{m=_Ib-zlNLk}9r-&4x>JBmVQ-%o^6HJ(V_sUN6rXNXJuDy)@wCuZvi?P?8 zUjf8orDw9xqk{)U7l|f=c(6!RZb0F2nwYUFK= zmp3eff8N)Ot~}s{=Zu%f?RFbK&wum<4G0N*H%3Nw;p88+I*L7owupTKyIz|D zNr$~^-D-GUe*xCLuB`6=(j=3zZ*;o)T^)4&*=s(|ZB5#%XWf&kcI$XI@@gPvLZh|E z<V9j7<<@Y-$(pV9 zyf5JM_745}Zp!oZT)R4&wQ9BFRrm>N19yD6BF@>X0PM#M=KAf38i>$;d(+#EZ((k$fF(Pj7t+@he5r-=h_Jqo!>-fEfYlW2*jYM{XT3}_T+g@R zuU;LaYb8IG?(?VHpk1v!t8&jX=^dr-rfQ466&F95{mQpfz`;YT$r>AY>}T5DxkrcN z!Lv{Rz#M-a@CKWvKIh11>3J14_UdzQK0@{sa5wAGdN7GU;;Z1u_S@L)Q5ZklQuT)~IgxSC5gq*Igyg7+#D>DBvLZgw|n7VoXkV8^fvZl{#AF#!B)lSi8yRZZgG6?tYW}cyL#rbT0Ivz<97A_~SJeXi- zhaiZ6Qjv%tpCE?WaA)9sG*udUFlu~ zqeqku3Q{cF^mS${G~>OY2S@lea(3jf!4S(%5-tO`GCp&EIgpZi8(*>D=M1zbq=~p|QGh1D*Z!FKBCLpj)s}K$>8cJJl5JUOxg7xS4Rt z-@d+d&pbTh_gsl^GcW()v)4p=+!S_j4ePCKTS;~CCJNa|QSAZ>^0P*?_F&cq5(T=z zws3i|wC{$ZO(SVnat^Y7C5;w@z_+cL`p z0+=VYG81G}rnek3no`ka*Y?~wCJS}uUgUk!vY1-rWT)=#3tTz0?&z;|$W^D7DR zN%zoE1nm1a;6kP93Yb`uasv;}k_%GqE;nL`PNx*PZ|xrv1w6zlRi+aQ%U+BY3uGpp zz%8lCc3M_BGOs>{+?uOS;2N`7biV?0~|nK(D3+MSp7mV3~#Xsxx^lR_=Ff z*e5E~9A|ni40Mx&Fb^(G2&~O&ggG%M^_vF*95Fm-N9if;@UTr7^N*;+u-3+50N0&V z29=#p;&wpM_15Ik=i>1akO2%sIJ#|3gdpYboQu?vMAyy6#ia z8&Cv1RMnH#5XKvQh6@U-Y*|`>R=>i_^Qh`S@C1CAb|{Z6Hh*-K$MfNN8kuah)>?1}2GDuYSmxRGdDBU;l*!&6F=*A*b+i zRK^kC>0zF-m7GV9`-$)H$#uXj{d_B}BH~^#x==xfax#bY4KjtVP{)&|cs}M7l(SHU z9?Gy%qz5%vqX-aN-&KFVD1EiS)WnqWu{08JjwEuklJAmRBhld3rzO0uL}y2FyoUG2 zBao%EL{VVBWP2FgqO=wZvrNLzH6^W`992(BVD+H7`SS4#q?tkFSjv%%CrsE8v|JDc zygk8N#@bXr5o6S+&I4jlNt?+Jg~lS?A0l|xx4;-g;S*Q;f8fAoZ5uRn$|vpZ5Va~H z1%xfHRzekTFd^jagV5`2Iq&f&?ke8jK>mh)S}Cz5OI>5}lZw-b6>ACiLT9H^B~KD3}rW?h^6ksCDTVIkMi| z*;TyFD)>+3SsFOhm$zAo81w7T%vd`b3(id_PaLj%Gi(!{K2y%|M0eG8Ue#bvl2|ze zrHHQW+RxWlC9|EcY8c*{j!PC_Ge3_Nx3_P7W6}ybN$Qg6y+@1bsoYFRB5|B3;`LZ&i0FSUD7hFv5C)%lysw@T%hAjp$bH!3F8jp+=VY7M|y}d zf)yDgbHvv9o3>w+>FW3T1;|8?g}KKP@_0^0M(}y0u$*2ZDx6c3)%3FcLV@TsQlu6c zRG$KFCPQO<*%0J7Bb1D{$aW@#F=K0?Ca6N;f$V|A>=_bwh^X;tR!-%KaUS}G%BHGw zD0U8vzfxN%xG@Ve39rYK(^7Elc}Ommj=68`%h*U`zVXm{U51KREv6ed;lsmIfpAC0 zx35S}>w^niZDvK&MC+@JZQfZ4b3<#5^3e)+l+J_aXAl&mDpw^GV&;1&IQyu@xDQ|3 zm=@h>h4b8p#F074>o?IY{xCNrJQu~4$J%Yk;e$1Wfe==f^7YtZK=rU%`BO24F*&LD zM0$wXc*~5*jD|ZspbU&5W>P)R&F<+^o}G`SL)8}b9?K)?>3rgC<~4^=(hLK{Q$Hl{ zP%K@+DZj?KN z4U{a5svNH`j7mH7em|Y#3kOwU_jEU&XmJu#&6n~>Je5_A-h z-@dx1#3fH8XNQf=B`4=)ht58L&x8`Z2UF|V;K={rhhyrGEI^+>%(@E|oRgJSo0bEa zt&SXdc6wuJ6Xu`9PhVgNuz)$^lik-BdO&nl&Aqj?R~(`?_j=KHo$hsA^~O{E<2s%8 zckk&TEu*HVr+a%R-KT~-qmR_}+B&1gj!`f5)z!(>>CL6fRUvAh8W~ebJTpmQa^3>St#7Ye5!kA_8LT4kl~klNB>0uU+0z9 zY!o5Vthsn}@3|JNUwZ=djp+8BN1IdvJwfs}`JZ>EUm=G+v*~nV__;W6fUhj>EXatz zPUN@F(c6wA`_-KXJ1fO+oH1554-$lb;JgkUaBCAXavM5YOKLrGB4T4&eH(BPI4Nnc zaIp0V=|CaTpem6*J0yzS+m>8YKL;`V!vf1U`~9E3?;gcLyVhUvhXDaUnLCGp`B6u| zr}4L;AA7u+CquA3n)d@ST_MEJo4y?q-ewRxjNZ~VmgMfd^4)!6_iYDQpUI0Cg4}!%PeT;hD1l1AfH8!R!r#!0x!cfe zzUr>V(K+tESs)vm{Zd*Xs^KBhI{pa_erudi`6w-_~m!YDT*QA=7zEb4&rZEkO|fj zjF|HpJREy&6SRu|M6WuZ+kHQyri^7996`3l1#1h?eP8&JybXVHIDl@8;hiHY`kJSO zvPB4_6djt+IRy$V_yR)%a|0V7pm#}!kVX!DZA6B<(J)J=O+G|`9>d3T2@qg0_{^jS zWlArAAOw0fn>*&}<9&;!hKh9CSQ`1`lVu-d2n($n+O?JA1r_QVGlsBM-Pm{JU$6rl zmJ=8BgtZTqj_L{hP4@v5+PaJUn$@5yV1Ji4`svNI$_91&Ms8%6i+vTx`g+JWQ9B6w z%jBfS0fnEPg$*^*)D+NBWF{QHBFAL+!;I`)X>WUK)12|n=R6d&5Ni0d1`q#XO7S@x zqg#tJYlgu6jBj=@*XNTrugSMJFtmMeAaD=m%&i~QJHgpcyf75^YzCG8WfRhF1Q7=^ zKw|)QNY^&nR$@fk^YtS8<~~beBT7bI>4g-|jR9DI`$?SVwZ_rooXkJUa`} zwd=$E`xZk#P<>u=qtJ)m^j%6r4dr{MopB;(vT+VOOgxvOV<)N5|E`1S zsUiPeL4!c|qrKtt+@j~PFxkC8&HNj;q;e0bEpJHoj4O5~hVn`emHT9ca0Vd%h z{c`pDjrqYp@LLA}!$Jc*f4{hn-+uB2H;uvJs_E*27{CDCfFoy_%Rqr4U5!`S8}AL; z2IH-&UUXRAL*Rr1Fm^iSHQo9PLue~MMBjY-;fp#iFxL~#spa+Y!QA{K$&PYDaSJz& zml7nA!*vX}=!107(fnvMSeHg``E>SI0=-i&;(K-Wpp9dWrgm0ZzDIAr(!ez$AP9uQng(?8eVfHLT) zNuLX&?h%(zsiHt!@9^8{u=qpJ@#nITa0b8t%BS=Pye;(Y1p5ra*Akl*WenCmSFi0E zmgPLQ7`Tyj9Pdfo4%tmI>`BL$fiiuPY&eb@<6pLVSg!>oYYjEA%ayYKM)1h3(N<3M zN{O2oC+9&uqtbxFzXM?YkriKWgi3N+P@cy-4y{nNO}Tf!G_1PYu=bRsoyM5DQq?d}!{pROWqhgz-|%CK(uYmBr~&cJU`Qi)%s9=J=9hDhT2 z_OLujXZrEu=?b}sMoN9@@}LncfuOv9=1n?PSPLT)qHu2;>*4XoW3c6@>))jSUNoN?N?#N>Ne zGi+4~X2nBg-C+AT+k6#r47Cx%$K9w(S;p;Upc?_)YtWD$-n@e@5 zkgvSZq>G>rVE?`{t0JQ{MwO8+WG31)Qo0Kn<2NEjC=S)LZk31*lZmX+Sv6NHyqU&O zxn$cvVjd8*OYFf}DG2GQLCtj>ZHU||9NA0!d9)%K2gAk?=m1C_TT)=@` zewc8k(5>g$U2Bx{?aIQlMew@o>520cQPh;m8RgUM=p!LZ+`6FU>s)QmM!bC-S}uYw zxeUb@G&Mkc6^h%@wXq3E7xH@gKdVk_I{Oji5l4LOJmt4YNU0#c< zPs4b@9PZ0jOZM`2k$zTq{W*=X?U%t$jH=^RIVj^<)Ly;IS?Gr+K%7d@BmdzpnXW>m zH6YZn~kqEkxM)vX<3><0*X8#h)fHkq>99*EbF@d44Wj1^aH0Qi0ADf zDv7HnJj<`pCwUS=WZ(0kd4+_pTA`Z!yRYTg_1_}b)vz@67E7-XW(eg@Q`^N5o@NSR zupc(L)_VAAAj!i7b610`I1%2p>2wu0>s2w9MURg$=qj=lHdRr2)1}JTTxf*XM#R#8 z*3Gv%*dfbs*?-Xr&&?{bjY={wTQFjp+TL2{^I$(?q``3d_bDfjN3Lj9)JjipIxOfa zCQLF0s8J5-;ySX-L2=cyx|e$%ik^(7I8Mf3&!;h|^@bSYPW|rr*w-IbL3bw*XxdXB zJ&OCu#2^X_X_$1n5{_YIBpy4+W4GnDgkr>divi(0#qh1cids9X4ZV#KYyDJep^wV) zOBilc1$ks6nMLE@rgAOs4AWfn^=n6yikFqo6)KMv)EZR^NPq}()%>@jxD(n7tb`e4 zQP#VEm0_K&zy|V$UAir%9X;$pSUqfGi%vwa8+ZFf2T{k?&4#V-^_KuW7_~du?dQ?j zE8Esu_5{&&!Gs|C;%M2pj*2HMbj4FasA;+oJZ6-%MW(B*=aH`uZ)#$Jqyz}XD)zGu=H<*)hAx^#JBIDO+4raG;6Y8S5c3)6VllN19GLra| zsmMp0=Hdspl^?^@RsL1zIXH;4GdDajqgHyn@;ih&Xv8VCQIUw%hW(j1fN_?k zPc74{Ilovut}=Bf%#|-;jHv8MxES*>NN2> zX?O?>P_}&1(nl)_r$ydelZ%ffOJG{b9u34IjytE8jl9D%wk-X3iJYr!G8{AdbWm!? z!S2pQvFk#L6{Ua_Urhd#E>6?qNAN9v1y6Dcp5HBwM84T-k@<^C9+Oh&i6o%nF%eQv zpAQ$GZ_#ZR1wtb1Y;w5woJG?)5Q4k!&y7%#`Q3LXW+O!|S}x z2k`kAJ848DSxFfzdB2AV_(^&Pv^}r-$vp$;j{Fz|6bAZ60fB~seTRTPA>TMa(KtyH z!~Ofe*&{&R-r@g_-{Y3wzaWIrm>z=uE-wYpAvBs`;HwagQ3xjPximGi(xzn~BeK{R z!&9Z>P#7b3s*wa32(D27vHRnbi3h_JBIr+su#5hEylKH~bjIdl5=Li1XJ+eZfHG+< zD@Pd|8H7mhBy7oeHJFa#iV5{gKvi*LT;YG8P0?jd`?|Scux#|+J&j5~LHj&ZEP(-& zvZI#1AJb)+22RTwK3f#Gr4o<4eN5BMzo*ONX0S>Af0cKnx#}DqbB}IPROT~upH3eRY`&3E`pYSGgv z*2iwywR>0svs{Clkh7Y%smH#u+^71NN3;+1=k$wrIdwm9Jcg1+&Mv0IBkSTj)ICC%XYc3S_Id4>n?MEoDr+A~Pz#$qXW1 zY%v$o6U!0h8!PM-f`-k8l^cDJ7^KwUAc;kx+SfYLEk!Ap7X1?n3ozi|vnRkCHz#Fr+#ci>+FfJZManV)K@Wkg3)n%`?vj0u9_6n7Xt1IZZ!=-G^ zj5~i<%nnV3pu>$PNyP&z->9K4Gad!Uw^rMKCSO>O_5=~?P&X|w#i%K2eR8up(+kmv z^J}WeTwivYmcGDNu%Y8*-G;rIIz>$nl^3sIC@5)#*Ze$hB<5wtOjC@5!t2Pvr0O|j zj4m3w#tO|Jr>j8GBJt_x^xkP!`?K<|9J4m>6>VO8MseJVbFW`@r6rp2Va9QV_)ARb zHSnBPR#sF#Y_~f7U8TPxb@8+qiX?Lv9<|gI7Bj(k!kMKpYu(yZkMVeTj!dk}Op=b< zb|OFXYx?WuL&S=lhty<8;Te%P?!_xc&M)Y0;+G3Da_v;(-$gws{m<=9=jyixl0NnN}vyVSV^wDZVx^@NDe!EH{XiE%Sr z!`$;cU6GL9t@U54`hCQLB^)>XGqFq@p~JMLf&ZFh%EGk>=Ei2Sdg(G$$?8Izz(G}( zp$+AVR?O(2=;q7YM@SVwCyO=|l^&%*W}sEs=o__nm}0GE*%{Z9K%;5HN%VXlW=J62Wh;%s`~Y1x->rWHQxK+Zl6zRnrkEy4g?_ zBC$AORr%RW#u9oxpO{6xz0j+m-Ka)84<4q0Wixu!1PJ{S z1laaKf_nmAE7>UgmT*?dUpX3< z9{|>2e7$8@bc!y&2@D=Jp5JuP>9-q*Z)B~EInZP z;bJxFDm}W{Rup~0?$*4^$^WG&9qTXv+H3xt62dHCNb7XlDVPE2ftG>AhXnZ`M}`1_ zms-oo(GSI^ZGnj;Zn>ac=n=}ND6{}S0romED_rbyFSf;+lk5Z)+|w+K{9s0kOc9q@ zHC57K&u)2K&J-4>?h^YPw%o$UcRvkvs7^vEKUH2iE8fzX1Z5870Q*+>{HJ_vCzYFN zxpnQyZcbf-o(Q*I6&hWgsTwfKCuq3Renfmu@Yy~5L$YAoc;ij)<|fvYQTK8}s^T5m zq_Xm8xykJWx;toVo$&(gl_wRccvY3(FGgH$YK$?M|Bq@vcOy~*C$F)oR-9Jj(kBq* zpn`37!vl3{^9DkVJ3)=R4#@j504^C;WvozGgq8@H81 z_w(=;j^~f()teaHY2v+BtmDgSph=TmRCEKU60LX*H_;o zw)mIMlU5fz&r4`*87KA|y0oS>6^ynZ$=#V2;?gWO&~lLJJbY^OY+bGjK9Ckj=e{0z zX|3waaWBrZzWIHdVSV{W!m45kZZ&>Nk(y)9+gr^qg(DZ)c*nJV^`dQtxR94)H4dx< z(Sh5g!lepE(lAgDN?4bh&j*{j@2{Cha005Q44ZW9kHQKMuB;ZA@(voGLA5}c$i`gn z2uh#Lg(h_k^!c=1f4W84v2g&)&pbGs=1Vf-Uj;492g5p*A`A=!w;^^0t z>Wg{*%-BGnZ6yrFqu+e9?zDf&DUl|(t0$ybCWOJ1GSovq@67`oo|7_W;SL)-7aq4^ z3-^e=-sAEW<0<}MpK-TbN?OG5UMB~iN)*Is9y{8P&*E2sjE~JW@KmBHED8#ZLs}@| z#TDyb5J)Dpn=|7u3J}GnpnrF^dE}r;cvrb9Dl1Ol^5Mx5Og5E}hq+|w%zkle646=7 zToeA5^53U4oRYN|SZiRH@xDT4D~jqc!_pgP`3haU6GAdiXof+u5!L&3nP+DLkMkmI@)eQA{U+Lugt z5G{mG53Mrxxk9W)ECz<0T=K+IUU0G$|1y%06%M{h<#PNr_vZ%fbzE8>PXD9c=(;p4 zjrq5%-&CieaF0PFb99vUdJLvFQK5iFG5fjKW@^6|;$LF2g4cVa!&H>CP)i=80y!5|rM))1 z$ViMzP{Jm0*FkJ-ti4(RNYB5~U!WV0 zk>thUnm>W1i?6JMYDN{Ob%_pPY+OZA!%Q^In&an@_n167;s-7gXY}3i!T0qsP75lK zQ5$z#aP9uUa#82Ab-ZPovm~xNV+#V}|5!+#@Gq!q+x(RB_6LIuz))mL%5@pkRD9ZemWG#uPDdiG#h-qCS_#00wy1mBQ%FjZz0jE@fP6wZLsSNmB|7a~X* zBQztt#K(dE`SJwGO=ss!5?M?iu6y4OUOu8UTo$LTH3uc?Q`x(ei?EP17@dqSARs_a zG7;QKdh(xgqKxFEOz{L#sDTqzLooT@j!U1?4tB~qUn8RF2;tL82vb5$RMYxy!asWj zZ2<7b*nzsYt|=)S<51B*{9KW9no=dXl5}*>g>c0YMRAcrdo*D8n*_!e5?^-pdYi@) z%~mON{~`zTuX!-mQ(<;IN{G(%;>(c)263iwu?xkCjZN!)wcAySPfaQB;;!X0}b=?L5Zx7782SIP92hK?_mV zt&S=CbrK%EjBr;u@UUjwn$uuBsyi>K9!M3n8B+~cip3(~Jq?OwrB~TGYCuWp zY;Z|}9OmgFO@|C7tkzp!(}FfkVvn$U-s?Wpa(v40KOzd9@OgqP4pE={)B_Vc(<&E2x`e=6)7GOI*lCfo-v;$_(H|$ zj*kf@L7f?MjH@PFW}sue*tcr*9r-jfRC+cv?B9?gUuk2adJqdY<)8tktUYjDczLZF3C&2igofvVz|Phl_teU zjL$qK5c3fI5rCSb<$GZcTuN#qZZkCQU1gu*|ITPps5#e&VAjjmzAJs5lx!1J*1z-{ zl(nx`6o#8}BWmEF3oot%ND`vOwDu>{I<2I@1g?2|q(t@m!ZS)_;)iItlGsH z>Q6*LrSGwW^9DC>?%G+(+cvFt_N=ie|7+-Em6`;3V;Mk#JXG zszYyO?mxtIR6%LFlHtaHJH4k=?0kIeCs%^{oob}boH(~L&UJ-AcU9NtAEvb!8f)C8xdYIQ}sIXERG~>f9=6ukV1@1y8`)_JSyPdv5{B@ zH+`;4Uxe3QcGHclKSll)vu&X^{GC8<$RWMSh=CqnVdP%*J4Px+)@-MsA4Ods(w7(K z5-&IIe+kA#Wwl=+TTod;G&Y)de{jKp9^CVR;@_cK0F1-C#x=z$=6<4fRZ^lz>hW)x z1CU(u084~B<}<-ePl)XaF6NrPH9T!+#P$V$qN}mxys?BYkpDCwey7j!XB~|hRcQNr z-xb(dhmP;l)jfP$$rE$G}TF9X83scW(v|Loxat<^u@&75uD@-T95!cn%qrv z=Q_du2deYO2s9xs_Y;nT@pj76I>8FVxFp5mP{Q~z1?de#|7HT8dTVdn#gB|;$pJYH z#HmSoptQmFolMI6^y*N%6D;}@FGsY31V%bFF6*?0%UQWmXhIpRYHhcTc#vn$IqsF? zRc84Me^u2wJ^@*@C)a$-(}WG3b7W3o#uge51Pf_FNw+nrWi)}iJk{Z;!#xd}fe$Q; zNl-b}xsTX@e$Is{p1f>eDEp*G;Ds69cE&Z#mzU*M07%ID1%S4iHA-+%QRSF9pcHRomno@MEnJ;_e={- znpsZOB`lLpW-_Y=B-)~6vEsy&D@)Q7H4}YlFCGcmiJmB#CSH$-pdz?1U7PAVSgjkm zr+FD)P0k_1-x_PPc1S@M)*CC^fOHHVdhp12xNFL%h5<3hV^mx>jrk}QyHiwG=GkD6 z;;oX4Y7W^U@;N$|;{};EA)EKEl#f#bG&__GUnWR~F7evP+uNI1t5h79w6%Ip2AS|k z2M>`(W6>@g3n*R8b7sHA)*-J)vrEjWvLtSwx}UB;eu!KCe6hRkD(mVt`9sO9F+6z+ zdEww)<8F)4k4rJol)8$Heyp_^?kfI!;n4tQ#o&(_O2;p==I5&-x+i+f`@G+z$>->j zuiJ2^iz}D=w9uZBzmD&C^{!F+bY2t{T=)LD+DKc*Hqls2K7JuY6Vq5Oj6Qx z4QZm1m`B{RG}2u$i77c18GfgZ&GtkZTY&+H&;nPNKgVg2^+C$i=^1ohdv-%m8p+Zq zOF~P@Q}JH*9RN>mK&lnPhm1KgnUrpi!t3^wdH4`gB-GPq{sFyZIird}q*W;DRXlNN z#l?UJ87NKbhf53W$^{B#HxyRf^s2xyUWPd)23IB~^ylQ3$BPOIzfebX8>zSq57{i- zWP9!GH~RY39WtBcjgj)B{O8UN4A0y|IzDJV-1BAEjUXh%$QWDvd*Z)8$N&E`b^NRN zUljb;_+MCD=vVyzx5$5)|9VAZ759I{f6eCu2>)IF3**0q{}RLaEB|HhjXPb{Fk8DZ`{*2%Igno z6R+k!g|_(2;QTPqA4n~Me}~i(bcYKC!G%FErvGC=3x;kW%U@tD5ltAc=6^_Ff&V=M zOOz`FE*1nA|DC{+_yd6@`3C|^$_#$xe}}-5c86aLf?xXsfpz^Sf%OgM`qu>3Zw%LW z0!#5Hfu-~h1eWr50!!sj1eU59T=~xiZ)}>w?*ta?JAvi+oxsvR$Gd(k z@DKZ@U}$$Z!xVoKSS~P=|1yDvw%!%?o3U;|aQC0adVDh$9t21H$ym>GW4+MEUi>Hd zO}O`eX72pMQK@rteSRabe64<(>G#vjd{XCgGjF2J{Efi!M;rQIC9u$j27EL0j|5iW zzfWKVxx;S-!EgUYU0$1`U`;-9R!d0jle=`dj0{kiv8hKl9@kZR&hVOV+a-)1doRye~?iwAQ-yl zbD-r7W;D;HpAk6j^3{G^2xuIH~v;WB?z8+Uf%d;`LsWjN6+xY zpXF(+JI~9fqsxm&{uQRm_zO&RHwd2j|2$0fALal5UjFAF&i{c0e|`Vw-y*-~|NYM* zN56mn+h6z}=h`>(5#|r1#&#~Zm-+d&&cB!^Ke-xvOh?Y%EtqvfJq{x_^;RY+srG}^ z&$?ZY3w2>KM|e>GABGR+bHvZ*;c#h|nar!C>V^o&&SIqIPNcPJz$0`rOvLMwX zyh?^Rn27lcUVp?M)M#z%fL{h@7Via|2osYOHCgGjStDnYjww^BdmL?ghOI!0c`I=FC-1-mf_YRfgF84;cI+l`8B-B=4p=aqq+)srD*}= zfxOY>2qpX!6{gRV&noUPDY_FdF5p%%yf$@`j52o82zJ2A}NhR&hWcG(+HlMS?Yn|dY z1yxRa)BApqtCgRKu+fN^S{v>7nm0cQa-Oe0*y`h;ei{MiGEzm_IG*fWBaiGH=oA+V#`UE=i35a*)=N5;%8x!tw_(K3B=@;B zO)dE$bQdU%xxZxb{?P|jH;&CO53q!-sPEgm=@2fK!tuulKC?Yv^qyHvZEh0GKFzj! zD_>q!XqUJiEYH!5+Zn_ zI^F-O>CUa~Tq!xWTcWn|MzK*SNZSqa@GU$&P4SJIkC*KFExXooXX9^lfgja_9F*U5 zz8|soe|R++weV|}u%NLeXC_{x z<xNfV_TF2~{v;{Yu}6 zQ?*$iY*Tn*7FlB&UN9x0KR$cA8XH1zaXciZ@ML(L9Gh9*^Je1kP&fuANu&s~NmPD; zm)0H3XsAMIMWxg0TaU|vXfKK|@QF!)xdq|%@xrQU2Dusy1w4Q7#mIja-Lk`vg<_P3|<5YYACVQ;`)jooOOX>yv=2bK6crV;5n(LON- zY{ZS+y_fT{YIeVahp6yE;_ZTfX^45h0S>{SmInDHjcSVc6YSx36^GRPx=91sql;4f zt>*d-Gf55%N7tm)$Rpa>WfElGixh4%)V#P^{^*7Z!$tG$x45`?IO@)y3)9}jSlWoP zKN)amy~uIS{<#0n{&c$wbX`}}B$_LJv===IJBc8f4I$z8#?S&rS|`ylg3C8VEJs{= zdYW7WBM8l`p6k0ZzYKONBfwRag(|UtPyB@w__yjj`>8tmkB*YaW38UC_So0I2>tPe zZ#I3jd}H*tLCkrz7Z@0a#izf)KLV&)@2O$Mb(`+3-suRV)2T%3yq(nV@J}R9Q%FYV zxO7n>1$K|HZ9FUwQ9K|jPicZf1hvlQYolAi3J;g;&#xZlhIfJCoo>99RAsW%BYfE7 zyUsj-T74e;5Lu}9gJf(fY8?gH;!F{2Dc)fl-gk6HU3A^0Wb=DhU+=uC0>SRjI|SM#1a=oeXl7n|f@hO?+Dj@$Jp$m3 z;dX7pWQ2IJ)iuerc}5-g?%#{=h2rw;77{WV8tBRy=aw=&6f~CVxL+Rj&SaP~ym@P8 zf+W&9B8Kq3BhCZD;R&fKd(me;80vn>JXmX0h6;Rwj+l*{kflZ)qeK_Eo%?}zubt_M z;eZCAB_)H)pXdU8_FzDO< z@T@BosBGRN+0x6~FDs_D^t@$yWbu~!Ir{?$v26bIEHgI=Ld@^)j{p8i{Qo!e zKSF{+|H%IfivEiK{}%Z@{-3Ij@Wdcxcpc2_@hAR==2{Tt`LDtem0&7z$ztVT%3xp= zuxhIjfk0Ha=r=f$g=}n`nP6;|e3leuw4)r8a;!+gvaE%L9aCh4>4a&~g9uM4lMau( z|9|6B3T=!`oWA1i%S8hzIxbn6On1X)A0!H9ITk)Bj03}n^g^+$!2~!k$KrgYIIf21 z;`J0x4K|p@RSoGBRaF^-GFu)k)R zG3+&Cn2Zp9j(!bmOZNwI>PNTOnp|p%9-nSRfJhkAz7fSc(cf%rTE0PE>lYp6slv@5pz@`d)C0VtN6`39RkjClD< zV>X)Dd%!?wZ-@7=9~*I&CMv|KPM%mXDa6Vr9f&*Iakwc86b8dhhBi5Pp5_n{) zUs`f-Z=91bRQCUTj-L2WVHjIG+t#Z};qX z+v3i&q}0=x4H6S{$sY3`eIfp&KR=`F0EhDd{RaZ{6>C}Fwk1uvm~%uVS(-BtbF6PE zLAC%Zou2>^BLzzWANY?ZeVtF!bq~v?R$>KKTliI0cCq?OIRqS@ioQ^=?U3Xv1>1sZ z3=NI=fH=2AXIl;A&WIiT2b~7ojRFTn!yohpP$c(k&BQOS@^1lsfJ;3q!g4f%OO zSIrEdlm7D&lPyKyYKtl!(?rPxW9kqT1F3HzMTL}zd+_XY8LA( z>fMb5MAs~aZhs`n!ApLdOjhgoseVdJ*}f!UYrIIr*ND%9in*>Pz;1VowHCgp*@iNH9^&NTTLoWH zTNc7SpNaPb3ao-k_i19^Ok7)v+P=hmrtHyWM;FOFbyZ&COWG%ZGw!LrGshVK$#Dvx zn6&S-9=7c)>8~->hxnskh4vhnW^gQ+m3Gxw#Ce%c^=#A|3iQ;N8sOI3*BhEgWIeZU z=F@jpL|LIHZF)rLfQz!zmN6eNc~79)-{8<3-3r#Tw6OOKHkebAPovxzET_mY*BSTv z#AET8rzG%I-jh+3Sxez}N!@{MGHLf+UD{%J0uv$29lFM{b_sNtkmz{F}^A+FTgJC{nG- zmimNAu|d6o*!|#m%<90+lJ1~V;x!{)wTAMdpP=MMtQENghr4@)sunOIVT!epKa zUzok}BoR)A+^bw`MCOUA4g?{sPs#x*i^|l^2Dr=o=&_@?XMx-7Qz^G=gr2|ke&tbf zP5emWyd8^TJNgWcnfOwI9bP^IS=%b3O6oQR|C4W(?H``3^>G_OQ%$oF1R}4KcDhD>()>*XsRiKgwywvkj595Ud`I|%*ISu)e6j|_1!7NE|B z)$p%`TRGJGPXYT|fk{mDi!E+XkKeQ^=2TVHnxw-%@bxh-(e*tqbE8+nmYL&OWq49Y ziy!6BKYmhy^grf4E(4a+@NGg@md!RN&?{1s6%&u>x;gPM!kjpHv6+7*dBz+5fQ_#X2ee0M(I7zZ>Eon7y|&45$HL{ zGf`u%)&`LO2)B(t`Z%?ccLsz9q(8ThE5$8YtuqC3Bw1O}r=zOTMojtx+x^Osh%H`p zKT^&L;AX8h5?fDNAxsn_>OG-Z&#i7Ep>s9nD-jWMtq}hK4--`AypW`4`)O zShJ=Ep{wY=3ILv!1NO^HF)wQyD9d_XlsY9=;X>5+yqdVHRsue(7 z)PJ@6nQ#Ke!nLJE2JB!bce*=0|3a9ZdjdB1&_yV+L7gZ$HNBQIrYbY$VII48vLB-r~c{$-9?*+)z) zW!7ZyaqyHl)|k0nA!^f@r{n;=K<}CwQ0MjZ%vf;mnE(qA%8o7{rWCU#S~CzMJ1DyW ztDEJ>Eakr0iM5p`nYBzqzD^t71=mo25GS){;I-fq^eW2E58uqFzN#8p_ znmO==a0K6CTRLu5&q*1u-opq`I@S3pR-nMP##GBH5cS>yci682k~~{$UjN=~bJjw5 zz>C$FcY3JSxN@%E1Yiq?qJp@D2K)h&)l3oFrjYlQZH*Pq$seZP`cx`S&DHTcci*h+ zu!y)RFjH?{v)DmAWnRO_gHAKUMla#>3{ zvC{{ANLF`%Vz_S^o6quT`DQd7FF#GCXhgWxB|bw6TV8%h@=UEUb-QH)Qj{ShYuxB{ zBeZ1O?5uowqMhB|tdtOpzKbTGR168|e9XX|FrH06$~&xVvoId;2Zo+aIkRr%`SE5T zdb$za9!BCN^gt$Oc33me`h-`2dZE0`n(&+)RIeHEhtqX{f3mcxiZR-?&n={ub_C7Xex&Fj<64kt<%v4p~Uwb;I3wVtlp3 z3fx`ZbR@x11yZpaVOAQt*kOnAGy$w^SPWP(8iejyM7x-#ua?fX@Bst%%hF3xwyikc zC;ThxldkBg+nPb)N;f3P{hc=JOt~ zudeqti>cA-4qY&+Af9Uh7Wiji`tq+-QYTC13|p1-p>H`6ZyWdkq)4nCBR<857fyYO zi}eC>GkFfJ&?Xb0(~ijj-&4e-WmS#IG{=@{1}=)es*!ukWV$#I)$Uskq&79sx!m7a z;SFo#cdm9ZHDGZu+q8@yOB2-+lenn6c!rRXrq*tVgvqa6}jK|I^|XsAU%6g_&UV|GlD zhOHU>wfxJ&PPJx0X!2D1P4rj|AQnZT``UW8&SZL~5m?vAXOQlUel%L{*tyL*$ajkkU_DR1;tn!dZ>>HV*_@ui|w0OCX9O>|K%Wi)^xk~Cx zWAL}6ZGOow`6a*Pm;91n@=JcnFZm_EMfX=)i*mRMC^eReJ*?-}^1zT#r49zLfQ@|lZAmUeXHq-(>LK|s01k~yKK zZCt~lnN2^*QaoYCL_h9!1hB0XgFvz^cITUh_oPv@>l{1;$!jw$1A5>Xtc^Y#VkqZ) zi1nZ~GiN3r=T^FFzT_LeW|W|o@aKG_v7+wZ9P`%Ts`c7OWIZ>v^zqBazy`_&&x4Ig zWtOq9afnD#pGWTQB@yHJrKteTKWFMcMd zdg1#%oj;ErvV3@T2}Q>ul~XrZAY3RSMDPo4IX*=kVp+&3_~d1a?*>LjdnyBZzPt>f z0kyUX>S#mqjsn4Oirb%cYv$7uKC72s+-&&2H$fSAdT4MXuZjplOds#y9q$0Y6G>7u zXE+-WoY+UmCIAAx-{}$nZadHblLU9BuBt{o$5XjaQ9wzh0M(1BowFk!g9CH!l43zq zPkpAOzNMnMq)~2Lowch~9vicaO+~#>GB>4ElX2CiqM+|9FMC;&oj$baQf#m$P%gh_ zXKlx@I5FbKwOVRh-CYxph+4|g-C0}LR~L?$y~WmDS!Ui>SLo1Jr4DOXQ*eWIse?l;Ee&w_wKvc3K>FZB1qe8L7{DX0+ z-;l!-I_yV9PerJlqjJmES)+SO%9LuMCXTLyD=fJSm1(6m6OXHc?9RoT#nmBJ1l#Jl z&Y8t(>oVusYHN&m7&PKI)p=-|4P$Z!tdG1(ZHCfS!_Y@SNJ^dGbb*bF3LEmA%7iZ` zyiQRK(nH!UrDYHJrZ8Jg?UE(-D0T=3n;;exk2}R6tJ^mv<4Ufk5;Hn!6QeSgyEta3 zwJ+x$*x|i_>j(lUh+9N9aLv#f8?ZoTY0ClmMMT$hFXN?I+Qp-mh#?mod^MY-1q7?b zF)zWQs08j~*AVts(cO~r(DyvvScKqjXwAXxQ^TcJ&HTB4FwCXvKC2t)kg6dp)L9Lj zs*X@RY{N+Ak-lJmG9!vMrRZjgC>346stE_ZgCj!&LkSEiY&WhgXtZ7`!`pk0bQqJ9 z!Qaj(yAaRKppkh?O=y0@yKPGAuudwB^F-cj8rL#Fw`%s8`p}F~#$G?)6gv&?H=WmW z$iG3}aA9in&@0QDS%mcQ892WTK!e2ao!Q08y6|{{u6n|9^c1>GC3P>kB5GLf72WS* z(Hi(^b^JCl#SWITr|6+hEtY3$8adid5fCUW;Z0B+EvZk9VUx}S8w>tkN{9UAsqb9t z`{~vY*CCy3%>;v@cHRV=y-4>;QZj6Big!N1Cey0TU*6eTTMA=Ag99$KE+S6brfX@P zFF2=CPX7mkaZ#KN?HU9I!uk %g)G4m>a&Mi?6uLcz_4^>Nis`+G!3zo3i2e-I39 zfz%>d{80PQ4daf7^`Q>^O8A81kF@!FxNxmqo7_C9@m-?W!wmcJA>;NFk?h^tSB8>W zP=y#ZD^~Cp2Kzne^M)niEN>mx54fM%^x{iXx!QG48D1Pjd678tuTEQLemjLLvQ;f> zWe=C>@l+LumUL>OY4zO%a4MN{wwetbl;y%~{&4rQt|uFiqpPHC+&T@S&i%O$XSI(E zCjTLH3>t3_tb4o`z9ezpWs}acgtlD%z@%FdgXQ`BThCR!B<++vL&9CTM7Zyk-EWi9 zRfdzRd=DG)c?}mq1pZs?w|YkYxy+k4t->lAXI5k1U35A@we+?$@|O)9$iLdLYUIrH z=zUW9c?s(RAcABGg^{2*inTl!HL=B1D~n%W-VZHja&>^w_$H;JMrwBjf221e`@sGh zUSw|3-|!g2?W3eNqH3IsA5~>g~I+sW*`!d|OP)u6N+OzYl~Ee>a$p zrTq4k32xVmWqMQ-Q7g|GAHN-*x~1ExK_hGM($|xF2%w-B>!6)$3OLYbS{&9je64q# zBmq0X4zX|1C7YR-Jatm*tW+bJk~=N#_45bD)J&V;J$|D0g1)2O!asi`{-yVXyPPHigwH^8vG565$zMKj2Y`95Z82l)F|eR&TsLw6sO0eVjnsa`!)>If2;o$cf9XvuCE!BR+C^G(> z7;}$8jJc_%FpaXb9}^A{j!o7(ISHSu@MfUGHRaUD`PeM>9t?(9rz>q_vT$_U5F= zJIu}3Z}D8NRUBQhw@AXMHv$}H6H+@6))SnHrj|ukCZL)VS(r8z$_R;=+k)zLbh;-jt!A z3Kep=iLWDY7WC^dK+fSKEjj7n5OL*^gi~f>3pPQoh4P-Ud|!>i5pg`uOrQ9I>FK*e z*CD4ztf@_ZghhRZupFa#`Xr{obb+4QINjEmoM_-UI#s&K`29BgjF^6oUQO^=xykhX zDkL;Zp|~bj24`uo%UBZk^FiX1=cxI+{|5B?-t&J=i4hh;(A~a+aFCebpbiaTsHGwj zE3K@E~;Xj}b^|Qgm7Qa73`77vMdM9r26SBbPzc``1Lv`=Z1h^D4;t z{P~M^XD;sbFK?3QwDr@%p`>5RJCbyl7BUBfT7~V(54Q4$Y~FQ5pqAK1j*izwP4j&W zu9+mfX)1tFpI@T(F_#(nw2l!Q^uuK9zEaK$1- zNYSD?$iQL2AS*uq5eB#R1-|#?`~hbNdU5Xf4=qN&oblIIy2Y=B|Cj?hJpmZoh(o8^ zjnZtw&iZdLJ|Kn}Pp5B)V2|=*R`?B7HiUc0m!((G zf)C1BTVU|ps3jz4>Xq7IEByS!$k`Wxh(^e%Av;YAIVfZ7)%$fXt!>W!gBDIM3EO-g zs{?~pL6;I}O_#~1Aj>fja{3ygOkabL*HTM1)hnWxU!xV8OVXwy=?Q@v_zjLKVGX0T zK>!*ujoL0gBl0-b9357iutAC~*5A$9>)zisJ!0N6)Z9;)oXDpZd=j6RE~%0^<1i(| z1M06Im*!c^$WJ@8X?3ZUZ&#><@XTr|w5!$#_&BFSjXrMwKVCLK=uuu2CLqyElHe%r zq|bk=r_%fX&C^F9Mq@e41+5Z<1N6B1lp1jV@}(>dhc(TIHf zO_-3%>()v#x*;*_glQ1Dqs}nBc)yS$7yF2L1lFM94;=+=h8KMFuva)y*rqF|7&y3w zYzwAj{V{5ci7jPJxcVZtOhoS*ER|I>Ei`E+@oC7n?0C|yduy7Wj=*-+aMdDz%Rl<^ znt!aLD_L~OHQGdO&CnwT@31BB5|bM%8n_K*)rNB;Y5ov?>ygFLVf)c`;5_(tj32W& z$2gDnCbu9@^CyP#!2mZg4j~YqEcq7@>e^!j`g9ACQ_Pvl$c#3EuP=6-YLsKR_jhS! z1Vl0j6#{*{;QF8c3Xp$(1rq+}(@~HS`&otf!AbueW{`qNpvZUsd0udo;s1Mg;~! zY4_-FwowjIktQ|KU>}>J^om&|@A2^2Q5-K5X-zRHiDs0!nD`krQMfChNMTtlOj5WKg$h6345gxSixsf1a%1V} zW(x|X{thwG%%%al4LdSrgIGPO*25lBgx#2-5f)=b@vARj0bkew!R_b^u>i>xIok2} z)Mamt`GbGTmXliAEv{(VoxvGa&SQea5&~drS=iA)v`Y|rX--k9q~Zeuq~T1PMdAo!kaX%xOG!J7W5mYKA>{XhFt%NDCzE&#K9el$zbSZ$SJL_ ziufyNSe_TTB{-!2%s@0KOHD=9q66*`gzho6Q_3JNlN!b4@zHh?hUrIz%iE|NehJv(!4Vb$ zTAkbffX_G@t3m))iV>oe{IQ%EhO1;?j@(j7?cAo>h!~VBn?x`z-LR^w8Hg)mGYv;y zN&_*JMw5d%!nPt-UZ4u)vm?|@eE)}lqAk@Q7V;|*D; z#YeU{4(Bn><(f{53E!8PEZgrkM(R$dF<{Ftpq(_O<~k4R9}}S=z=%a8t%d; zkSvRTaP_7dJD}e^z;H4iDx$n^FBG3y`+mt2@cFQW3WC+lpcYJOeVH%A7@rc9M5)*7 z-~m-iCiJ+T&&D$ca9XT$JYGpyp54SUX>;1|<*t<*+*K1O%H{PQ<@;WB^?F>v&+xUn z8ZwtsSu*K&LJv#SD~8jrN!)+U#P^o*zV_0_N59wgfBeRy&ENAt)+qj%T1~%s$@(y1drgKNQ)0>Bc#j~FF#LgyXahR{~ZSJdf zL=o`|HRFw~sFZbNRDfI2&r4vy%V$-X63kh}&2U8ei4EUDC8H97Kkx`I+iq8_*f6E*F><+Xcm z{)$`?{lVx7vAljr zym)_S|BiEEEo0_78PWH$S#g$ehpZ2ZJ^d?`NYOl%cJHFvvI$oy(= ztm^XnI68K;MC|vz8^Nme0rrjDiVQw!0(tZMDvUu@Bso_*NlVR;zBeOf6YU&T+}oFo z_=Z>QjLyB7AN6xgJ8Aya&gsW5gPqS8fHr>9kKjh!osZGh{=j3i{XYH%{_B$(LXYtV z<7=j#&GvcKDWUe|hw)j-t3cbq_lmqCiuNc0k|#{H5kx?Le%dRL`Hze#;}F-m@*s zjbi-(|Fs+ZslV7)uX`v(%urK1obeX*mg&fO6);`!hPs#gpvhE z^kqy!9FV4+-WKnl%?TR-!uz~qAV<9C-s8J_V)8o^;Ppi|HhI*xW1=D$x0>3;4P^*> zwH23Py529q`A$b+e945*)ETeeZ?}EiBSGQ3ap%bIn<1}rx@7;xYe!aOU435l-3(FN zcB|cOL@ocfW8$wb*?MbkKZdzMj`}b|nw#(K*(@&l$t?L~XLM!s7P<(RsOI z^ZK)yptqG!qEUPMXXEtYrt$ReuBhmZ!xYpR9QqPKU z8=527wws+?e5i8kW82Q$Clk%1Jl>4Mz=Hrn@psA=iZ%1}c(a8aq0T-6WgQjn1Mre2 zKCRsowsFcyCLyO`uJK9zrNnT1>wSX|zGET4A8(@Z*SWaQIRhG_Swx2(%M~R4cR# zCG--ZZ>lpn-?8gZ{t9UObU2%YbO<%1!Yfn%d?76tC@u_;X)CU6<-dN=JWso73AO%? zgWC0C2+<;9>+2vppovR#b&<_iT4_9Ue&vsRfe-Mewp$2`7oxa!jW!gL$x7j~-wS)C zW}`>%jBV6L*3?kgAm587GMQu=@h8#A_7d}fb@sSYJ0IyB?1R7hF#Nkb(u0)b2-L^; zy?9whtL>|!@CBod#&rY9{XjaB+4wR!k`OozY?Q(%Kj9_7l}XZ3exD*YDZ~9Vb8Xf% z_-QlTBT~z|`c}Di=@`mO%}fFtX2DdRLXoa&H;mfOxX1EgcC1NopF_BPBn><~Gdlxa_GUfX>PhhEEAfdlJ5R$Yx) z>OYHAw0$2)?%C9)9M;$81l=)u1;-t3@3W2X?*X#|dGA$|vza#e4ehFuyp4j|X zo6%XzY_8b5cH*H2#gE8T*>!#m*=Z(Wt5-A>(6g}cksOL^*&67KIi%7@+Z6J z$?hH=sZZWZ>V|D4;?v&;&y$m?1#H}4E# zQ%S(-~LCg%t8b#ZD_8K|^Tc_xnhDRYl!`AtuB?q0PN^jcd-Vdy}UH zZEak8908}J`~?bwYHMpDC!Y-o*O?>ziLT+vlRJ_P`!>2*a~;L*PQu=zq~t({Rn^^< zUT|TBy`3S#2d#XKE#%*B6E~v8#Eo&41}~OJY{%vwho;1xL)Pv)b&o0@36V=SE_U80lL?! zz|48Z#uxL+vP8R=EnEuRwDM^kd#BDt>9*bd&yA~(X7(~%ZL$GIufVgjJQCy#!UW)t z0o<2^=LCZs!F#74ySG#H(7h=oP>J2z*oO8!nFl;T@=3W~=h@flEcm%MN9qy2E&G&! z-0s7VfY~BG{P=}b-fmO{taS#DJ|}QA+|`AaE<~=K%cH3AOO1Qm;Fb-+bLiaG?m`P6Bq>oVLr6K zf`CmRPw)GY+Y#=)nos#%yTKp^>D12TzHKG4+)UBb_3P3Zjo;pmoIa4&DM18O2qE{6 z6C=?S1DX_Q16i*MWZYOyyE09GVnh7*p>RnRNVNwl9ZmwXRzXW_orR7WZw4`Qh;Yc9 zr7x0Fmm zEGVbpf(<0A1nWp)(=_B0)y47Or_F&uP6FN$OTM;b{G`9s{~W_QS`$OQ#c@QxTRa>i zrBDhi8p4UR5x)$VOt2g?{$9J6-w+`1Lu~MdhZD7xY{Ck6Sr=h4n}eawnP&~YLSi)f zLE}WIkLv%-GO^iG38tmxKU@&rI8Tly%@(N%#hV0%nOnpC!W6+<^OeDw0U-UXu9^~C z6iX(hLDIoMLi$XCaZY->)^=&O>I)jffq>lpp znrEP?B>}M*fF5RSh+1bF>UiWc$blx|3u%>ty;PG#-;w#U&=1QB7y6DJB?Qqo>}1Ly z=EMSi0^`d4&fPiO4w1iay99N!$UWFRPz~g0^I27~YOX9Wqup;BHd6PX4n!u1)hAu! z3ONsjH-C4v$BqBI@csz$ME@Yiiny`=zHZ!B*x(cC%-bg9YlqwD@5igxFJ5iGcW#G_ zF0D}*f)=O>u$ls{+}dw-SU3jyLVJh(&^0ZzN1;GOS(E&OY9A@?b*2m1>^iM>Ve}&( zr2z6|(!ZXnlN~1)1M$)Yj2jf&Dfb(c8G{AkJu0$`vY)OMCD(+64lgP$64k^R$FXw! zDK4z$M9aVgse<@USD_#tRu_ZhDtje zUR1;rdSJ}-nSiy4gi4;6y}=aNv={Z4X0HW}BniScD7tbOPjyc#%GBiSCY^R|1Z(?q z6^A3paqL)8&+gkbY~nac#CWWj5QVMRaf2G1eAzFMpx%C>|3SPl$f|AkYg`u&DHy_} z@W*=F(BH|2L_yMgX4?y8y@s!5tyN>aLQbrapb%qDgy{X8{o_XJ@(NP#EKm<0Z8bq-@0BCZYS{&+;=9 z%wM#LFmN8vEzS-Baerfs6u&!s-kIgBUNT=QC`C%++QIT}tC2iq^d*kEeWsB1)-1(PBLT{vs2&pv#g)5jm23yY29CqJYPb&TPv@!G` z3Ig9HeH_co*kD4IhmdAeI3+9!6<78bLI+_gL36|`!!By+n24n@Jh7+Ke0v-V$qlkZ z1L=T`SreOSB2w6_JtgCNHKoa(?y%wI5)vV7f*I==5}zZ4eUl=ZA24*kZXO!OEt%tV z#*@21HdZAyWg3=Ojt^CKo{LqYruw>5fpi2vH#{RR65D(YVbdBRTzj-T%Boq|+e z&ICUV`-a@U+U-sw@B2Y;pUz*;>*%_I##P+5K>LdK^TzxBHPhq3=GpNi9~48`?zRAt zuE$E*oEmm2M;T0J^erlqZJvYWR;-ZY08b2?wTSG}X3K6h{aiu!r=RSYRGC|juVdRW z?9X2hC8@e906t~UqO*}O_*tg3Rt5O;>q{Pd9(RZr4`h!0)nXxSud~u4 zY1HF^C2kB<7n}3C<@Ii8h0kA~Vgko~DP)?RG_#hi7qe$YvZ{H{Vk-YI~zPF&?27c?Rj;BYWHdxUA8S2Ce%DtsxKu} zUI@Jr&r4dVe3OnGIJk~A=t(v7YZko78e(RM(&gi1c~3Ma36k7sQGe3J62OOUl#oHA zXVQo;zp@5EWqQH%QoLjEkJag+XyGS^VvYW>KTCq)x~5Ea>b-DdV(Y)fEO4&#{8>lU zW}Jwdy!qu1m;Igh}FA9!hsWRAB{Nk&UPaVZnYYerD4;ujtlsa>^}!Fkd$GOM-HP^drmLM=184P>Qa z+vtf}&GIsVI?5HY*MNf`jHRb$zR*BIgx@bw)n9w{OBB-Fmq*k3bw{03T4haVHj;(hN>|^79wVNugrgWmHRzNCoB-$(^7d_I^v^a~Y zbK|3<6qIY>mYgL%q_-y44lCvS#40S|D?nE0Vzx+;Ys$G~p4fB@vlRmy&`yMR|K}=8 zL#JG6moNkK(47h_F?IyoX^KgU*-pmxwK;Gub-W$hK;fYIVly_GsC1h>*sO~XOu)Sh z0VOMr3k|%R4p6$SP+o03ClSO)O__B8qqTCOAXZ!|z@NtsJ?Lue>N0Ud8H7=;;Z24n zrln#C=H=|qlx4D4rAC)k?qM6^aB}^Jo{j(mGNxRr#%KOX@?#PKVIjV?v*u;(@IY?D zqZC8412?d~<({&s6AM^5R(jF9rm?9+rWV9Z1g)=jttE_u3S1SPX!8p8OQ(EIo2F1P zFWs(mTP>SoEObiVi=Nn zn%KI_8z5((hF#+u?%M<%icpYaq2#&PBgn7CZd23?nWKhh;D@U~BK0^MeUxrxvB-Vu zmbDs9Q;yN=fudA5ZQhA$IfpN1N-<#7wsS9$mR^BlZW3jWW#>>Ty@OqV@Vzi$Qoyji zy_)%*gZ7@?IlmR!Dz{K>Z5S&Ww)xtMC|Ds~U!BzU5s~?j$ljeo4fwdQ%`iu#$vCvn(1@gKW2QAEr)*lsz-vr3vu5(h!9Y>oZ9-FK$`MwAN8~OmVCh7py1m*kc5x6--srB7=W_qG7Wd zh>57gO;fc>iw~}(-$Cfce47(C+s5}m3X+L&P(x%L=v+tFd%e|&Fs&!_8lfn?7N>=% z_(dI8xCMp7F5BrDl32L`4g9zS!HVVP-BA7jEp>$$TNp*9y(Y7-Qni> z>EWME)3-ldGF)_zToOpRDmXwoF4AQG#3Srvt$bI1?cJ!5a_D>Xvlat75tFr2IWg4V zaEAsk^&dQir3J@-+4t^KknP%lRLymoskG5Hw0CD5Sl(pIFt3eyEMyCB!XsSs;0{m6eW@J4ruz5Rgp(mc+o;VtAEi8QbWZ{5Nzf&Hc)&%TNc4H zF^0>MyB8^2eX(|0B14$1k)R=3Y$khu(TIoOiiR}1P=dbb2@m3e`p$*Ltq>_r>m8+z zF2!Fpk?W<9QR=8XxutcQ8{2875_|VYwCOW-=1hu~Su`x2JVPyv)~LM7sYm%@$6!bn z!{`n&N45%DJ(vShv3xo*Psr{8?a-xELU@GUh&unPTP^QEGkZ6Ye|$lsegBT6k?t@; z!u+z6Bdia2^oF1@^k*|fuF_0vRl~ygSFyV0K-7*OZHA18>R-VLb51?xI*-~-+hlax zlCP`FHwbIb#iYb{v)bfM;R%Py;_p`oSa%iUEjJfWdaIOxG+oPB0fT40;EIGHape4( z=pV9Y9EpcgV130Hv4Hs+Fg=S{er+6^> zD!AdOoS4*>Aakn;7_A<;^=8CxHN(9O~)&UG>U=diSmsx2=sAXi8 zoP^jHJG^RPJd#P`RcN$E$Ix2T6 zB_7$k=6dH9TrmN8-%Zi?BuQlG4l`&)@v;c=nX@o8u^2w2^Ly2xzBUGL8sD*A6K(Hk z(^8mEfx!F83rj4dFLd|&o~Pi8nZh_5Ez_m9tUN|KmrLiAOqc@VYNP7lwOFt@Vv9SG zLzH0u=(f#P92AQhx2^!l5Yk=i83QZ*IEtv>fyKW!K43p#;_d$OdQ0hGi&dj+G)tyk zYJd6{m)yh7oj+r~lYGj7hAQAppfzwV6UByS`xE03?IuRi>N9p+=vt-g4js|jSw<62 zrJh$I;-Eks9KG5(LQ&~6T49Pr;h!tAJgL8$6IJlya}vSTg_pZfY6&-Up7b`A(1);& zrfZ*AxdfA9<{E|@^DfP0e~;Lk3q9(P+NgQZA_PBl)()M~n-F4;O!{q)&l%Pq3Y{7| zC7`EBmX0tHlG!slS+~RB8e%6%RQ)o9JxhO{FZosG=VnfnQ=d33CI`T4F41EH63Iwb zsM=z?#LAsbqro`G#-t@GUKSs}Qa7hX z)4pg~WYLTGEj;XbAy%5>rub?t>TRKJl?H>w-tKMIPB`ud2)nf3L*U+BSIH20I2=n> zZyKC#$|{JJZYsuH;aY@TW#7Kg?n205=DhK%`zn$P5~6bBt%D-B@M6rML{YtkRh8X! z)NgQcg_^6!xXhNLlh^?{>?zvF3`dD|x{Z7FQdL(|7&~XfJSJY++DNb1URILAZCu}O zbGoWW=D+l0CiU~z2h%TL6p6^>JK4!853i3F z>vMH=^Z3B11Ky0f3Yo2hNuj486Ly)lBo3Ays{VateOoLlK~unO=8F|Ag7%N$^gjuv zOR}T+MXpTfp&5lHFv!kOB})&_IDP1iOj+MjtG@aIQK7b@inP#6Zpyv$W4n&wgh~@f zcImCMy$uOah>iu}Q;e&*QYtarXeB|TO1ziicpW*vbmU}Lq&ZB5_oN^xRJm-n0X2Zw zQ97n3S=CrnEv;EEpkX=!9iDlH4xQq3vxSVE4A#f^R)^C~&Db)RjZVhu$+mcK ze7AwrU{Zj5U9^P7%Mm@=8TB`$#e2`wUSn%fi4sGhZ}F@ijb}rRYC+a40^2RPXO2Ogx>CjbBd literal 13460 zcmb8VV{m3c*Df5}=ESyb+qN??C$?=nnb@{%+fF7yc=7wa>-X7rH~!k&zV9d0ch{TuXn#HV z#@8?nOC;>*!PW8W1AJF-SOfa(_4Vb=8@$i3pXllRJ7`WZ_O`D^zj7i9jc$P*3mi&N zW6yx&12dYY^h<4=zC>FoKR)KEN!Hx~E?QSh^xUz}Mt^ z%JlW8c@}Uh^@!$@?U4%-&!N?$JE9YzF$L|v#`K)S1!`)8(Wb{S>h(Z|)wSQDD!LccGD~8DM5&Wic!qmkJ&pCWb?2AZ@6$ z-ofaxV{lUInJ8D&423A@Y99ee=gP656`Tm)Qn01dLQLiC@hF z5^BK*u9yQgSb1paq>Bntj@PMa#0jRjn=|ve_xh4*Qfv z3XK_=i=0Ss$+){6xK1N zYSkorB_np^mG2!Xn#&EBi(jVF#VEw=y0$YSjYem>L6*iE{BWl}5WNF@m@wWu*v&Uk z0H`zl)pI!0-YH#$n#rhKTl==-GR!BMPEC`=vYTlI&u-hIvu>LF=g-i2SE?7(WiE;a z`$kolc1uIo9RE|{3TSSyas#7U6=uwic79o*e3(sPpfu=%Qb43tIyG*QIr%k9bgsE) zn6|co;}i@$w2P$mTESpUIZ7+dcXn&C1@VpAw6x8;i?72(;;+-|Ch4Nw3YRBYmq{2Z z?PSu^v*HQ0L~?G|;@InQPZo#b40S;)NG2(*Z0n%uvw`Dc3jK5--grIB5%h@65lSNx z4k}8WxEFO~KXw!t{D3z3d@StFO z3XO(g6w>6ZPv(xQXh$N+E~QE3{{QiE^HzON(SK= z<8@D7wU@y&KlB<$3ikOKUUE7|VNs7a!;B+=6D!=$998032rQT(XzRZ|OAdg5_yn8$ zH{us`LIgVa;uK;=#@%ypA0oErO#>6J=+qKqCcvFu@kVR)u56QLnnQAP$q&scU219wdviM?$%AMdr3;w#nX-)?-)ss&u&0y zL|!mWM=nCYQ?F{2dAMls zIh5Viw9#2pNx!jZ-$X$+Yu>?r9PvuXn`UD>ro%9W7) zGk;aCv0Xn2IjxAm7k|HdTJ*X4L@MbIwyt-LdcWm4)M`Cm${a3r_uU{OR(txcvGcUW zwlPb#u|!hkb=eSQ%%?|pg(jSkjprFDYNn{?vNk|RE$OWlx^p!tO&T&bO?h#2`hw6Z?i3rR(y`(4uHvCaLb-HtsP;0Ncm&P8cp?7mn?UGg>Rh$3ebstiBZ$j@FqXkr>*xZkqkcZMY>RC-sk!UZ&@WqthVua(Pr zZTK6k+G?pu=MmhvW^ zTaw_E^n$=QxmX+iEL3|yDsze97m9fqI}k;RT|&m@R^9Zc+#TKCN#E+@Yd^urb0+A| zzb)b;S_iy3c;*rYMC}K`|A)j7sSd|r-%qGpfI(kmoBsj?igl>iuJ?`PBouN6{7F z{SR+KF8yca=v!Yz>i>QDpHu%u3eo?Nfmf_&s&Qlhm2ME~KZ_%jR`4N8Wt-gNW8@~1 z-=-Sm3|{a?oUdSDJx_`-Z1UPCp$K%*9KDJ*Xc727b>4#@M%N4N2-cS5>}RUDZ)mYM#kE&W&Vw>-IG?*B~&(U1cL4!{2Lgn&CRVk_LU)E}HuEsJjDpmVuH z@R6dDWGscq{zt6r`M>)^WFLMI`BH?}yVbbtK&^w+j)pTa80EchNG99ZhmFzM8Au2o zkWJSCCj;toe(UPqciM-KFgu+lF7B8m(T9+Z6~B0a zMW0QC<3XVW_tY$2(u%NVaSUWM-hzIO4f}-{H}zhWBJe-d103YWx|g?){>p9oRE$h& zstuiNYWH5Jd(NSmlPTg1k2`9-cjscN(FYiza`BedXiXE`2)+{hDH~n3B4sCru|WWi zZysnfx}mvgk8nZ)L`1}CEZY*PGK|9D^con~Xbe9!2z>AtOB>oJnKR?4w6ChpPxGu~ z>Io9!Vm2%j%Oz4KT$x>fB62SoI}QS}(n4)*nG>rhUiz!=PBw%HkY6tk`UV*EF9`|( z{|EN_ls|xuCU1K2MyNdZ)9TNvAb(B`NP2s|jOll3dPY&ddQ6-2Xv?81sopH$a8`6IN@;LNDCg?aQ z$BkKG))mHTaW4~)u6knvl=bH8@s_`D=}Fsw+%M}!5%Dn5U+ZZYuBhZk(@TWHT@Uo6 zpY}t^8EbfdGL%<})>eDLZNypc<}boB*W2|gL(4*^ zP3W+I5M-F(tFb1z2tKoA@JF^l%N^0-Q4iIhp#bUL(t~l}NsS@*lA&dS)h&eTS|z6R zEcE*(mr!}DVP*agpM+AC)0`Kk<(hS}dKKGIyJntG^*NKFN+Ok-r(&v}6g`mrB+(S7 zIpQvF;BN@|ot=HZ=?9sPE9lN`z|6p8@ncI%7L-l~>>umU#)67bv1po3^Re_Pxk%Kb zaiFKE$42OJkT@Pvsm_~;0HxzpC-Q%@+PMYO^LuH z<+gp9Z{v%xzn&kQ%790-(WW~BI|@#a=FJA z*`)vQ2g5Ke(TvO{a)p;|C2T?Ht7x={9g%+=86t1}A>IlhsnKGazD0@Cl$?Kr*d|Qi zjb3b?Qaj?0A_t#u1Z76E)Q7W5%vW1XyS8?r@{_iNv~|{?JxQG}?A-ZyTO`17_;VD~ z5GIVP=Y=%!3S|~;svo_6@E-5Wzc_~bz5Io&&RV)X1k51eJZ-Jfq=egxsk?aFpiH(Z zQoG1qRX1whort({wfX_wFYGLlAkx8_)*ebDDl{kkq>C3c+8`A|kmKj#yRk%!e>@MN zg=m4tXX(Og)tWD@&82p0c8z++*hg2UyV|KeIl!xrr z-)aY`f()6r&y?I$vn73$*v;MNgP{Z7Ku>7S@CKuhH1L6 z)fdIuguv0plr4f-k27pAi{=|k+f+!Xf#DIdj1J=7xn7 zN&GL(cmoVG#xDHsmwg3_{zp;zk<0@{p8g}pp`mXFQlA5sGeg{brmv+fl z7lYT&I4xL{FhdNml`8H!Z=nUwn}?;2IGe6yq}1nZu3P_%h3#F$x;%hz`@GX?(EE3<)53Ph`gXw>UPe0t7tmJ-FpKH__VVV6FYutcAF(eC z^!RHHUhhO6EfVl#acFd>;`MFp4Mv^uV%uHx`E$JSmQig^wQa3(Yq-DUUoCBiTBQ!Q zU_@rRkizpG+bDEc<*N?OZHL`WgY%BeeORspGi%|woR({Pc18He&964^$5{0}74$vX zh|vsFJJeIRnedk@cRe>o6()VaEU^h^I9t-npjEF#?{f`u=NNgid&?N)ry!Y}8?Rf% z9;cn&Rz9JJ6r*y|o7M?u80}-$!zU{S8k=ISI-PDz1V>{E3DXSP@TVoVj)-&d<1H)O zI*$CzZ#G(;4;~~@oY?bZgU$?odG+MSVCU9zXWOsjCIIet&MnTl!ghUSk03}glA7pr zLasEVnyWe1efqpDORlpW;JZ^vcI?zTG-J9y1KysY8+BK;HOw2Z5K#hyvfoNvVD4^6uSzeZPzK1b1;G9>CuF z=PiHB?Won>=X|~2>q&x;w)5<-bGhNhaNG7BZ{SO_kAK_E+Pr|b^LCxu{@WOOIrNL6 zo4`qIUfyZm&Vj&d-cSQRkpoEi+vGrX0`mOFYfX*;yS~rMvx_f)6f;(3C2{@1RD(f1Gc z?mreDj7dw_Y;V0b;rD?=i1mvvYClUykQo zOl>zl+1uZL+X0x<=Pm;1z|`;0lfUov|C)hI-1^(E3oftIM&DOV%E0>w;I^-^##z*B z$GZRHe)Y?pFVDl@?(=|CQ>r;d;5N9@=vvPkf6ytBkZ|s|=u@~>X{&3$^8s0ZK=~*C z^H#I@vX-L%?Zg%+=_}h+*jxx3{-jAs?y~1hf6i+8W2MCg$$CYt8IT*1eFe^Ofh~x0Z2texk zxV&c4kQ1wJxq(WbZ8EC$y1S*QyA+??INQ{1wZxg{O}Q?MUX;MFGaKP6*L#qRfM;hN zPv$h(Da8c3f>?$zNI#hh3X;Cf-n>i|(EnHlESne;=2 z%MSbFXm!J;VeU1YP}fsb?wy#FSk>Lzk<@N$OCYbB_t)%p z-$c#Sfc2Kii-(y-*dNQPQl(7a_9rcBMQ$oZaAw4Y@z+ zzN&z&5AQTqk@>lOQ}rrwa7QY8vf5{)Q1yAPW+%F5v-)RFK$EE2!p>aZA{2x`H5x3O zaj!!FKQEC#eR?my-FSKW*XBcgwNd()b(7tO#>xqb3s>7T|mmQzy zuhxY6hbCGKX*RbyFa&Kc+l0b(tyv@}k(E%+!hSWKGI3MYy#K9t^b?YMT+QX+Mb7Ed zGuax$UuS26;j;FK>n`p6K0ifT&CS8`IG0+DfO<)LV(3xtY8)~`49IT>R19BuG)_Ub zCJPQLp zX^5FVyBuYBdX05A2fk>Co#+uQ?v+v`aegCtQRK|W+zSoC2TMAzfh7c`hFPw+diUvf zE`$;>{OG_Tv}RhMK*|V{{Db{%>SU#?o@#gCpSOXs|K4i*tpxbnysikoqr-WQ_Qvaq zD5mIEyUPjKIsLRB%X61H+x;AMC!EdvlQ}N;CgG{#`m?QUa8T!tTq%C38oSVXbBZwN zDSsJBo5+6Og(mZxub&TU91QDEV;aotXPQ-StsTy%MOt+MqoKLnV}M=adf&0)ueCp- zr-o7+S)2f1lFnT%^5ii!BSxhRT`hbZCO!|WPOj~VYmMQK zv(j;R7MlHaJe{0w_HG9U8*J-L&G~2HKMmBQ z$fM9pwS6nHF+u_06U+K4-kGmyWc%*#WF{#IJsPg%r_Z$YW8OsIo#hAI=p%t1cs>C~<$4xw#W! zOn26dv{J2VQ%-&8$V5Aj%d{fnSU*2_W_f{eWj~>K49;0|+BUD{UfgYV9K&BWr8YC1 zedc?fwjv8{cZ`$}k?l1*=8F1tylrI8FhevFu5qv*j~qDMT8IQ3dkxs`wg4kj=*f|f z<1LJ0m2Is%o7T7nB#ggL%IUoZySygG=a0_2E6fya)#P5sirwSdH5<+vm^6EvJJOvK zT?9$}#%oq76ye!nv1V>0p?_HO{-)Z8?Tqa3q<7pVUDVUerG4aHW@N07GR|-X>b?^G zfv;uXIy*fEBo<{M-kNMe?zFkftjo|&A{gZ7w9xaa5SdQl- zuLOUtkJAxFQ#)qlv9QxI2=-pRXR?VK=im3@=<4BXGO*W8+gQQ>b5gQ&_Svv!kFy;U z`jLrog=r~_#beVOovX{2#n6RRa(J(8VESd@q_#9h>Fp%HuPvW#LMhU{t3c=B`-SfE zj1G1XcXE|RbDt^Rk4K*f-@>9rj9JKPx0K8+g_11$y)MW35^8Y!C32qQ{QO^Q=2(e| zq53_3_n`AkZU!jz8a-9{O8LP;-my)yxUGGB|4z>ct9HZdeUE zSLT<_c{O*^>po%Mu}Nh31T!~ia|kkTj08P3W6%*RtnA387(sY_2TIWaq6uD(SlSf? zwATePEt^C5=l-#x5uB!CcA^~>idqgv+b+yn0WhT&QZf+GBD zgn(Z9sSjl>Z#I{8G>5TVJD`DqGMIZ;bDJGA7FmDuVK*Qq&rB^Pwb?CVI7}CB>NQ9GQ3R^*i1K&f(b#YbmW#P z=*Am@Sghz@VO>}v9804X&~DV^o zNG&f`u8!k+Djx>%r-u_xAKD$z6Wrt+u2C}I(sdai<$kKEqQRBK}8 zptT-nANGGJXv~CVdG(=uB!5CjZkGyrDN{pm3#^#`K&})^0K+?os01ynybF{JvJ~?1 zeo>cmI#(Bwu{!f>EF=R!PiS9Tu_B1!{-_Yn<{Dlw|5!PmvHdAu7@+R}vdx@|WsU<= z>46gBaN)eL5PEhfW96A;$NqI~DS#5QjCca!sk94y`%%s7ZmsLxP(Bx;W4wpA+qev9Ou?bE?J6S2M*e%5*Lc>@|ng1CY~L_5tLMj=MSEM4&#K)O|gzXM-dg2!>z>qe-4L|Xqa$qE*PduE=^Dj_Fk&XtNl)tz& zMlQ>vK_``r0^2Qz*p25|1+^(tQG-^dmlYJ_y0VO*lT2$KSYg{7EG4nk>M9 zicAT*6i8ECwk$4CBK(1&vYa4}t0tZWEw~kobru15*R@pp?kxngQ56CP2Y+x2Sf3`U zJC(~Py;}JO^}U2*g#(&(9PbK=rW+b~(R)R^|5bn;g&_wxh)7Rau`n>sxU z3&C9=SE>k8`$T`+?OZQ_AjRAcgMvG`_JVg5?Aakfv3kxCPyCGs<5~C?@mC887U7;n z$tIc@^=r6Z#ymREW`^MU9PWf?#(>W1&&sycrcY4-MLu#Te$P$fWX>HM8Dj-^f)~sW z_{P~2uHsRkC&dzi?)qNt|7&L*xp;yv!Pi1xt!E?95${oIJdC4YRot3+*`8k%_xtH8x4O}E)+9r z9~@iTing=||1o^oAy0${>P0jez`WMDaY=ISOIs2$4r~qe4~A{LIAO;Zytt;}zjsz) zi-vxDfG>0`OQ6PAoL@xwxJDiu4Hi0bvs9c9e0ZyD0Kv1*rdUou;h#kSe&$=54npMK zmXBw>H99gY`dE%Mcax~-eIjfA>2Frfo;jXDPt=~dKiN5-`aIvn>%~p)ln;PP0TUL% zpk}vFl$lVi;tKiHX@(E!PEmXgA;9q5OTp*mz#|l}-`YfA#7ici&fE}u&KNFK>SPhP z58gy&=7H;hk-3iz0S_pK@M0Fahe3mojw`E~`Rhm*NYm}S7QiL(*UI&++kMV-jv1+3 zKmnnLeYAz=hyQ}>ONav`liC2?KRlgt2-KQ&0>LJ|{vwH#jRULyF0BFf#|5YXfhT_f zA@?qLz!sp@d&XNpURazr@Gbztmdmc?&1m-{Rqg}2zHcj`n|aNRutvEtt*Ezv3`A8WAWo2>`B5 z!{IS3&~wCCZwB_$3(C|r&eh$?Iu;fTp2+DV6P6>+P@3AI{U=k zgtP$6fD8-#2zaBG(^Wk#0lPZIuAQ^G4!meZSu>;u|3oQjrQVf6OC!M+ON7ynyq5Ga z5iX)c?5w-RTlb7-w5wRkf^o5wQ`cK@JU<+M=`YG;dSU=Sn!Q3?NRf0X|e&`6p$30`& z8fp9zgwEgZg0VI@!!q1NS#?>!l)LUrQv8Gq_alZj{n)WWOu>OQ^vo2Tml+##bUYXc zGzr%J?ZHov-C31)y|pwV3g>OZ1EH>q@2>~n9`1w)qS!ilwL}*9m$*2XD;7x7L&uad zT3Py@qWTvXHQxsUk zTdAdEK6=-{;;d{LGvyZ(^F6D&RwYf*qcF!+WkEu8%BznJYeN@1lLS}& z`+ga*6FxthO!rd;>_@cD@5Ftk0+>W^4HY(Q zD&cQzjt9d>zy0W%k7e+%6wyzpf{Y7YX>(nuUD!-i(X!j$k`FkKx%}Q@lIV`PjMLV< zDY|kc@8qwiM)>6ITWhIXawpmey7MQx*^{-ipJ;Rewc453e|L z1mv(7{g~T|HF&Ofy<(B{XUXVH5>ir>x_FB)vwed0~X(q8-hi|RH4F6HGyNZnF8TwUh-TJExx zTg;JplbsqgjLw$h0TbJM|M70)h{qYabnF`n*=zwlTmT)MVQ@5ZstcMzO^>2PGLfnF!HL8T5) zB%@0OuNhX%D4e|+I=zM^$8_GC&VTJ}dSjV;sDP5J$(U6b25x@yAwXJ8Cy!sOyD zQytXA6gFUL{#_}ig4AI7UB^pEBqM_2Z?03?exOsniP>)ieH$KD66%BxX*g{q!zS`) zTia1%eKYKL<-G0KrYJX^iIpMxaiU|C-9w0Y#j^_A@C&Mp0O;zeGcZBYG7|PD+KerV zbJs5TT{u~vf`g__@zWZ`q+e0$1H95tRK^%}HK`!{Xv>tC;BDU!IEDAgU9Xg!gf_6V zsT6DBg|h-%6kfOuQ`lKeZbTp|sJpxJXB3=cbZFf$I*}*d7|7gIG=f?+p~DgRQM5w~ z^wnogJFqNrb2AZ>s?br)eJTayNIdb4=ZLD>r{y<%a8S2Be|tvuB}=?cE!YvyEbfQ- zqV;{!B!CRBrZB|XKrpA z1m4yfryRCL$$Ee!vQ;jWJYP_Q=926R!?obUD{m~J0a7eznW@<-2NQ<8C66A^x=S(g zGf#~Rw&*2_wSSqBizsUC)&7doKTT4Rs(Ge>djm1vClPzZG{Ffu0p*7pw51aV?Gm7< z>%HBA4VuvS=EjxMKYzgMYczz~-=G=Ya7RtyzTMa&PS$%X?B;Q*B8n2HYZAc_YBZ!g z&^o=O;*xz#v>tF^S4(;`on-@)H$u&&cqKEpC%~BcGMV`Wt8%e8LODFfxIiHk9XWN> zWtTYArFW8@P8cXonzn0yv3GLIFW9J~8W>$^N7u1Bkea$t?h1>uKQ+}S+TqHOW2$w? zM1$g19jnCguGTOjf0H9Lw3Izr*Xx23#(*$zEK_%X0#(l+D=0lMV{-sEhQYqTUB;$_ zkTrtJ|L^vU%bvdQkH90E7`C;XxtUP`!C77inUBKlU1oFVv0np-pq!b&cN|l(4W+_9r1ZeUs@~uA5G$y8IBH@8TjMucwnrF#> zXo=%@z39;Hkx|lx=8M|YG1oU&e=I2`sG=HmZ}tY^=V_doVcBb<`oQSecbaeb#cN{# zVlk3e^Tc}pP`F>*pp2^bC5%oDu^g6I;fp4Hm4W+)a<_1h1nyi}3K7JHSW+z^Pi;sE zKPWAA2>YpshS9}#s?7C!qg?T)l6)nFAl1Q$K5Rv!pX_Ax329R;#{4^Ar$&VQ;kC=k zLbr{%VbD6RsaqUc+S=V5YuI7C?j33#T8PA3+Gib!4{jGmSJKW;1xF{WYHRY_R?bCF z*TgB^--)*obFtu$1VwzIlZnGzmJD0~TJDB$hUi|NV(3E?PgX~?G0RC0NUydIcGzeo zK?Nf$D)SPSffBdt$3YRA(|Vz)#D~*ghuKKYG;7d&>xK3R2wa?+GNgvL&iQ z|Frwl;Nuv9vG0jDP6irjr8;4=K^qKu4*EK>y5UD~jOLYRK!!1Mp{a>PXY#_2EwE+N zYuX{ebcWo-5ch4ZokgXeOA04l3n}9c3XLq=oO|Wa^62BprzH@J@Bcyfu9vov@JCP0 zjT`}%2S~y{)=mZ;R!iG$t74O=J-s(}<#aAUHcZF|uk5>RRr>Shu|?&#Q&E)tBl$`F zo@kbMopXnF=@u^X$_HwW_X+8}@*g@8If+0YrOmRe(9+=y;K$OJS_aGZXnwFtAyr|(c>{j@^ChE{Iq{=MS6eIoc6%H6t| zcPoKKZ+;eYg(8;9>@oVgus!--dLbF>O!Kr(T3scOlO~yodoxK)b0|=I;+*qN`7D79 zU6n-VI#ASSe^AjY&t56Y&w*xak6=+B0{>vU4AP1p1$- zW8N80QO9MM%I*mkc`}7fU1KKk0osJtt);qV=x3f)_#87l%A_B&=4h;=T z49cZoqFj0_CZ5v*L(?jPbO-Z7Mb!zTsZD{t#i~?bYCP5*vU1M_G;a5CcUM(py|rB$!=5ftSA107~qdH?_b From 8f173b778f7653b434ecc7d4e168d92a02b1b07d Mon Sep 17 00:00:00 2001 From: "Dustin J. Mitchell" Date: Mon, 28 May 2012 14:41:24 -0500 Subject: [PATCH 114/136] fix win32 test error in test_scripts_runner.TestCreateMasterOptions.test_db_basedir --- master/buildbot/test/unit/test_scripts_runner.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/master/buildbot/test/unit/test_scripts_runner.py b/master/buildbot/test/unit/test_scripts_runner.py index 24ab9f32f5f..1f5ffb86942 100644 --- a/master/buildbot/test/unit/test_scripts_runner.py +++ b/master/buildbot/test/unit/test_scripts_runner.py @@ -21,7 +21,7 @@ import mock import cStringIO from twisted.trial import unittest -from twisted.python import usage +from twisted.python import usage, runtime from buildbot.scripts import base, runner from buildbot.test.util import misc @@ -187,8 +187,9 @@ def test_db_long(self): self.assertOptions(opts, exp) def test_db_basedir(self): - opts = self.parse('-f', '/foo/bar') - exp = self.defaults_and(force=True, basedir='/foo/bar') + path = r'c:\foo\bar' if runtime.platformType == "win32" else '/foo/bar' + opts = self.parse('-f', path) + exp = self.defaults_and(force=True, basedir=path) self.assertOptions(opts, exp) From e5916463847efb6b0e5584ee27bdc8698ae0513c Mon Sep 17 00:00:00 2001 From: "Dustin J. Mitchell" Date: Mon, 28 May 2012 16:25:13 -0500 Subject: [PATCH 115/136] fix double-slashifying in fake expanduser --- master/buildbot/test/unit/test_scripts_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/master/buildbot/test/unit/test_scripts_base.py b/master/buildbot/test/unit/test_scripts_base.py index 2d88c8fea85..c9dd61db29a 100644 --- a/master/buildbot/test/unit/test_scripts_base.py +++ b/master/buildbot/test/unit/test_scripts_base.py @@ -116,7 +116,7 @@ def do_loadOptionsFile(self, _here, exp): patches = [] def expanduser(p): - return p.replace('~', self.home + '/') + return p.replace('~/', self.home + '/') patches.append(self.patch(os.path, 'expanduser', expanduser)) old_dirname = os.path.dirname From 08060ca875d5990067ac353817adb626a003afbd Mon Sep 17 00:00:00 2001 From: "Dustin J. Mitchell" Date: Mon, 28 May 2012 16:46:36 -0500 Subject: [PATCH 116/136] fix tests for homedir searching on win32 On windows, buildbot looks for a config in %APPDATA%/buildbot, rather than $HOME/.buildbot. --- .../buildbot/test/unit/test_scripts_base.py | 24 ++++++++++++------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/master/buildbot/test/unit/test_scripts_base.py b/master/buildbot/test/unit/test_scripts_base.py index c9dd61db29a..9ec5b3d3762 100644 --- a/master/buildbot/test/unit/test_scripts_base.py +++ b/master/buildbot/test/unit/test_scripts_base.py @@ -21,7 +21,7 @@ from twisted.trial import unittest from buildbot.scripts import base from buildbot.test.util import dirs, misc -from twisted.python import usage +from twisted.python import usage, runtime class TestIBD(dirs.DirsMixin, misc.StdoutAssertionsMixin, unittest.TestCase): @@ -115,9 +115,14 @@ def do_loadOptionsFile(self, _here, exp): # avoid breaking other parts of the test system patches = [] - def expanduser(p): - return p.replace('~/', self.home + '/') - patches.append(self.patch(os.path, 'expanduser', expanduser)) + if runtime.platformType == 'win32': + from win32com.shell import shell + patches.append(self.patch(shell, 'SHGetFolderPath', + lambda *args : self.home)) + else: + def expanduser(p): + return p.replace('~/', self.home + '/') + patches.append(self.patch(os.path, 'expanduser', expanduser)) old_dirname = os.path.dirname def dirname(p): @@ -133,9 +138,9 @@ def dirname(p): for p in patches: p.restore() - def writeOptionsFile(self, dir, content): - os.makedirs(os.path.join(dir, '.buildbot')) - with open(os.path.join(dir, '.buildbot', 'options'), 'w') as f: + def writeOptionsFile(self, dir, content, bbdir='.buildbot'): + os.makedirs(os.path.join(dir, bbdir)) + with open(os.path.join(dir, bbdir, 'options'), 'w') as f: f.write(content) def test_loadOptionsFile_subdirs_not_found(self): @@ -160,7 +165,10 @@ def test_loadOptionsFile_subdirs_at_tip(self): def test_loadOptionsFile_subdirs_at_homedir(self): subdir = os.path.join(self.dir, 'a', 'b') os.makedirs(subdir) - self.writeOptionsFile(self.home, 'abc=123') + # on windows, the subdir of the home (well, appdata) dir + # is 'buildbot', not '.buildbot' + self.writeOptionsFile(self.home, 'abc=123', + 'buildbot' if runtime.platformType == 'win32' else '.buildbot') self.do_loadOptionsFile(_here=subdir, exp={'abc':123}) def test_loadOptionsFile_syntax_error(self): From 0de2aee19e55bb9ccb792042342bbab74f40ca0c Mon Sep 17 00:00:00 2001 From: "Dustin J. Mitchell" Date: Mon, 28 May 2012 17:05:24 -0500 Subject: [PATCH 117/136] don't expect interrupting a command to return a signal on windows --- master/buildbot/test/unit/test_steps_master.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/master/buildbot/test/unit/test_steps_master.py b/master/buildbot/test/unit/test_steps_master.py index a6b60d31d51..20903ca278c 100644 --- a/master/buildbot/test/unit/test_steps_master.py +++ b/master/buildbot/test/unit/test_steps_master.py @@ -73,7 +73,13 @@ def test_real_cmd_interrupted(self): self.setupStep( master.MasterShellCommand(command=cmd)) self.expectLogfile('stdio', "") - self.expectOutcome(result=EXCEPTION, status_text=["killed (9)", "interrupted"]) + if runtime.platformType == 'win32': + # windows doesn't have signals, so we don't get 'killed' + self.expectOutcome(result=EXCEPTION, + status_text=["failed (1)", "interrupted"]) + else: + self.expectOutcome(result=EXCEPTION, + status_text=["killed (9)", "interrupted"]) d = self.runStep() self.step.interrupt("KILL") return d From ab47f56fe1e441a761098234adfac9a27077a1f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Allard?= Date: Wed, 30 May 2012 12:30:24 +0300 Subject: [PATCH 118/136] Correct typo in parameter to RemoteCommand --- master/buildbot/process/buildstep.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/master/buildbot/process/buildstep.py b/master/buildbot/process/buildstep.py index 38cbcc98e74..6512bf43de3 100644 --- a/master/buildbot/process/buildstep.py +++ b/master/buildbot/process/buildstep.py @@ -372,7 +372,7 @@ def __init__(self, workdir, command, env=None, if interruptSignal is not None: args['interruptSignal'] = interruptSignal RemoteCommand.__init__(self, "shell", args, collectStdout=collectStdout, - sucessfulRC=successfulRC) + successfulRC=successfulRC) def _start(self): self.args['command'] = self.command From c8ed2a3eb4d15efd42cb2ddb6144098e850ae598 Mon Sep 17 00:00:00 2001 From: Tom Prince Date: Wed, 30 May 2012 21:11:29 -0600 Subject: [PATCH 119/136] Tests for master-side source steps timeout. Also includes missing timeout arguments discovered during testing. --- master/buildbot/steps/source/cvs.py | 3 +- master/buildbot/steps/source/svn.py | 6 +- .../test/unit/test_steps_source_bzr.py | 30 +++++++ .../test/unit/test_steps_source_cvs.py | 33 ++++++++ .../test/unit/test_steps_source_git.py | 70 ++++++++++++++++ .../test/unit/test_steps_source_mercurial.py | 46 +++++++++++ .../test/unit/test_steps_source_svn.py | 80 +++++++++++++++++++ 7 files changed, 265 insertions(+), 3 deletions(-) diff --git a/master/buildbot/steps/source/cvs.py b/master/buildbot/steps/source/cvs.py index feef851e608..968e8b0af20 100644 --- a/master/buildbot/steps/source/cvs.py +++ b/master/buildbot/steps/source/cvs.py @@ -152,7 +152,8 @@ def purge(self, ignore_ignores): command += ['--ignore'] cmd = buildstep.RemoteShellCommand(self.workdir, command, env=self.env, - logEnviron=self.logEnviron) + logEnviron=self.logEnviron, + timeout=self.timeout) cmd.useLog(self.stdio_log, False) d = self.runCommand(cmd) def evaluate(cmd): diff --git a/master/buildbot/steps/source/svn.py b/master/buildbot/steps/source/svn.py index a99624cb315..54a63e2873b 100644 --- a/master/buildbot/steps/source/svn.py +++ b/master/buildbot/steps/source/svn.py @@ -187,7 +187,7 @@ def copy(self): export_cmd.extend(['source', self.workdir]) cmd = buildstep.RemoteShellCommand('', export_cmd, - env=self.env, logEnviron=self.logEnviron) + env=self.env, logEnviron=self.logEnviron, timeout=self.timeout) cmd.useLog(self.stdio_log, False) yield self.runCommand(cmd) @@ -283,6 +283,7 @@ def parseGotRevision(self, _): cmd = buildstep.RemoteShellCommand(svnversion_dir, ['svnversion'], env=self.env, logEnviron=self.logEnviron, + timeout=self.timeout, collectStdout=True) cmd.useLog(self.stdio_log, False) d = self.runCommand(cmd) @@ -371,7 +372,8 @@ def removeFiles(self, files): def checkSvn(self): cmd = buildstep.RemoteShellCommand(self.workdir, ['svn', '--version'], env=self.env, - logEnviron=self.logEnviron) + logEnviron=self.logEnviron, + timeout=self.timeout) cmd.useLog(self.stdio_log, False) d = self.runCommand(cmd) def evaluate(cmd): diff --git a/master/buildbot/test/unit/test_steps_source_bzr.py b/master/buildbot/test/unit/test_steps_source_bzr.py index 3d1a8c217b2..6356a2a651a 100644 --- a/master/buildbot/test/unit/test_steps_source_bzr.py +++ b/master/buildbot/test/unit/test_steps_source_bzr.py @@ -53,6 +53,36 @@ def test_mode_full(self): self.expectOutcome(result=SUCCESS, status_text=["update"]) return self.runStep() + def test_mode_full_timeout(self): + self.setupStep( + bzr.Bzr(repourl='http://bzr.squid-cache.org/bzr/squid3/trunk', + mode='full', method='fresh', timeout=1)) + self.expectCommands( + ExpectShell(workdir='wkdir', + timeout=1, + command=['bzr', '--version']) + + 0, + Expect('stat', dict(file='wkdir/.bzr', + logEnviron=True)) + + 0, + ExpectShell(workdir='wkdir', + timeout=1, + command=['bzr', 'clean-tree', '--force']) + + 0, + ExpectShell(workdir='wkdir', + timeout=1, + command=['bzr', 'update']) + + 0, + ExpectShell(workdir='wkdir', + timeout=1, + command=['bzr', 'version-info', '--custom', "--template='{revno}"]) + + ExpectShell.log('stdio', + stdout='100') + + 0, + ) + self.expectOutcome(result=SUCCESS, status_text=["update"]) + return self.runStep() + def test_mode_full_revision(self): self.setupStep( bzr.Bzr(repourl='http://bzr.squid-cache.org/bzr/squid3/trunk', diff --git a/master/buildbot/test/unit/test_steps_source_cvs.py b/master/buildbot/test/unit/test_steps_source_cvs.py index 29f3ae3c536..42f0aa0008f 100644 --- a/master/buildbot/test/unit/test_steps_source_cvs.py +++ b/master/buildbot/test/unit/test_steps_source_cvs.py @@ -65,6 +65,39 @@ def test_mode_full_clean(self): self.expectOutcome(result=SUCCESS, status_text=["update"]) return self.runStep() + def test_mode_full_clean_timeout(self): + self.setupStep( + cvs.CVS(cvsroot=":pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot", + cvsmodule="mozilla/browser/", mode='full', method='clean', + login=True, timeout=1)) + self.expectCommands( + ExpectShell(workdir='wkdir', + timeout=1, + command=['cvs', '--version']) + + 0, + Expect('uploadFile', dict(blocksize=32768, maxsize=None, + slavesrc='Root', workdir='wkdir/CVS', + writer=ExpectRemoteRef(shell.StringFileWriter))) + + Expect.behavior(uploadString(':pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot')) + + 0, + Expect('uploadFile', dict(blocksize=32768, maxsize=None, + slavesrc='Repository', workdir='wkdir/CVS', + writer=ExpectRemoteRef(shell.StringFileWriter))) + + Expect.behavior(uploadString('mozilla/browser/')) + + 0, + ExpectShell(workdir='wkdir', + timeout=1, + command=['cvsdiscard']) + + 0, + ExpectShell(workdir='wkdir', + timeout=1, + command=['cvs', '-z3', 'update', '-dP']) + + 0, + ) + + self.expectOutcome(result=SUCCESS, status_text=["update"]) + return self.runStep() + def test_mode_full_clean_branch(self): self.setupStep( cvs.CVS(cvsroot=":pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot", diff --git a/master/buildbot/test/unit/test_steps_source_git.py b/master/buildbot/test/unit/test_steps_source_git.py index 0d7f73e2a5e..2b870eea16b 100644 --- a/master/buildbot/test/unit/test_steps_source_git.py +++ b/master/buildbot/test/unit/test_steps_source_git.py @@ -59,6 +59,76 @@ def test_mode_full_clean(self): self.expectProperty('got_revision', 'f6ad368298bd941e934a41f3babc827b2aa95a1d', 'Source') return self.runStep() + def test_mode_full_clean_timeout(self): + self.setupStep( + git.Git(repourl='http://github.com/buildbot/buildbot.git', + timeout=1, + mode='full', method='clean')) + self.expectCommands( + ExpectShell(workdir='wkdir', + timeout=1, + command=['git', '--version']) + + 0, + Expect('stat', dict(file='wkdir/.git', + logEnviron=True)) + + 0, + ExpectShell(workdir='wkdir', + timeout=1, + command=['git', 'clean', '-f', '-d']) + + 0, + ExpectShell(workdir='wkdir', + timeout=1, + command=['git', 'fetch', '-t', + 'http://github.com/buildbot/buildbot.git', + 'HEAD']) + + 0, + ExpectShell(workdir='wkdir', + timeout=1, + command=['git', 'reset', '--hard', 'FETCH_HEAD']) + + 0, + ExpectShell(workdir='wkdir', + timeout=1, + command=['git', 'rev-parse', 'HEAD']) + + ExpectShell.log('stdio', + stdout='f6ad368298bd941e934a41f3babc827b2aa95a1d') + + 0, + ) + self.expectOutcome(result=SUCCESS, status_text=["update"]) + self.expectProperty('got_revision', 'f6ad368298bd941e934a41f3babc827b2aa95a1d', 'Source') + return self.runStep() + + def test_mode_full_clean(self): + self.setupStep( + git.Git(repourl='http://github.com/buildbot/buildbot.git', + mode='full', method='clean')) + self.expectCommands( + ExpectShell(workdir='wkdir', + command=['git', '--version']) + + 0, + Expect('stat', dict(file='wkdir/.git', + logEnviron=True)) + + 0, + ExpectShell(workdir='wkdir', + command=['git', 'clean', '-f', '-d']) + + 0, + ExpectShell(workdir='wkdir', + command=['git', 'fetch', '-t', + 'http://github.com/buildbot/buildbot.git', + 'HEAD']) + + 0, + ExpectShell(workdir='wkdir', + command=['git', 'reset', '--hard', 'FETCH_HEAD']) + + 0, + ExpectShell(workdir='wkdir', + command=['git', 'rev-parse', 'HEAD']) + + ExpectShell.log('stdio', + stdout='f6ad368298bd941e934a41f3babc827b2aa95a1d') + + 0, + ) + self.expectOutcome(result=SUCCESS, status_text=["update"]) + self.expectProperty('got_revision', 'f6ad368298bd941e934a41f3babc827b2aa95a1d', 'Source') + return self.runStep() + def test_mode_full_clean_patch(self): self.setupStep( git.Git(repourl='http://github.com/buildbot/buildbot.git', diff --git a/master/buildbot/test/unit/test_steps_source_mercurial.py b/master/buildbot/test/unit/test_steps_source_mercurial.py index 104bce514a2..db2e03928c8 100644 --- a/master/buildbot/test/unit/test_steps_source_mercurial.py +++ b/master/buildbot/test/unit/test_steps_source_mercurial.py @@ -86,6 +86,52 @@ def test_mode_full_clean(self): self.expectOutcome(result=SUCCESS, status_text=["update"]) return self.runStep() + def test_mode_full_clean_timeout(self): + self.setupStep( + mercurial.Mercurial(repourl='http://hg.mozilla.org', + timeout=1, + mode='full', method='clean', branchType='inrepo')) + self.expectCommands( + ExpectShell(workdir='wkdir', + timeout=1, + command=['hg', '--verbose', '--version']) + + 0, + Expect('stat', dict(file='wkdir/.hg', + logEnviron=True)) + + 0, + ExpectShell(workdir='wkdir', + timeout=1, + command=['hg', '--verbose', '--config', + 'extensions.purge=', 'purge']) + + 0, + ExpectShell(workdir='wkdir', + timeout=1, + command=['hg', '--verbose', 'pull', + 'http://hg.mozilla.org']) + + 0, + ExpectShell(workdir='wkdir', + timeout=1, + command=['hg', '--verbose', 'identify', '--branch']) + + ExpectShell.log('stdio', + stdout='default') + + 0, + ExpectShell(workdir='wkdir', + timeout=1, + command=['hg', '--verbose', 'update', + '--clean']) + + 0, + ExpectShell(workdir='wkdir', + timeout=1, + command=['hg', '--verbose', 'identify', + '--id', '--debug']) + + ExpectShell.log('stdio', stdout='\n') + + ExpectShell.log('stdio', + stdout='f6ad368298bd941e934a41f3babc827b2aa95a1d') + + 0, + ) + self.expectOutcome(result=SUCCESS, status_text=["update"]) + return self.runStep() + def test_mode_full_clean_no_existing_repo(self): self.setupStep( mercurial.Mercurial(repourl='http://hg.mozilla.org', diff --git a/master/buildbot/test/unit/test_steps_source_svn.py b/master/buildbot/test/unit/test_steps_source_svn.py index 22eb9ddb6fe..d79709b0529 100644 --- a/master/buildbot/test/unit/test_steps_source_svn.py +++ b/master/buildbot/test/unit/test_steps_source_svn.py @@ -104,6 +104,44 @@ def test_mode_incremental(self): self.expectOutcome(result=SUCCESS, status_text=["update"]) return self.runStep() + def test_mode_incremental_timeout(self): + self.setupStep( + svn.SVN(repourl='http://svn.local/app/trunk', + mode='incremental',username='user', + timeout=1, + password='pass', extra_args=['--random'])) + self.expectCommands( + ExpectShell(workdir='wkdir', + timeout=1, + command=['svn', '--version']) + + 0, + Expect('stat', dict(file='wkdir/.svn', + logEnviron=True)) + + 0, + ExpectShell(workdir='wkdir', + timeout=1, + command=['svn', 'info', '--non-interactive', + '--no-auth-cache', '--username', 'user', + '--password', 'pass', '--random']) + + ExpectShell.log('stdio', + stdout="URL: http://svn.local/app/trunk") + + 0, + ExpectShell(workdir='wkdir', + timeout=1, + command=['svn', 'update', '--non-interactive', + '--no-auth-cache', '--username', 'user', + '--password', 'pass', '--random']) + + 0, + ExpectShell(workdir='wkdir', + timeout=1, + command=['svnversion']) + + ExpectShell.log('stdio', + stdout='100') + + 0, + ) + self.expectOutcome(result=SUCCESS, status_text=["update"]) + return self.runStep() + def test_mode_incremental_repourl_renderable(self): self.setupStep( svn.SVN(repourl=FakeRenderable('http://svn.local/trunk'), @@ -724,6 +762,48 @@ def test_mode_full_export(self): self.expectOutcome(result=SUCCESS, status_text=["update"]) return self.runStep() + def test_mode_full_export_timeout(self): + self.setupStep( + svn.SVN(repourl='http://svn.local/app/trunk', + timeout=1, + mode='full', method='export')) + self.expectCommands( + ExpectShell(workdir='wkdir', + timeout=1, + command=['svn', '--version']) + + 0, + Expect('rmdir', dict(dir='wkdir', + logEnviron=True)) + + 0, + Expect('stat', dict(file='source/.svn', + logEnviron=True)) + + 0, + ExpectShell(workdir='source', + timeout=1, + command=['svn', 'info', '--non-interactive', + '--no-auth-cache' ]) + + ExpectShell.log('stdio', + stdout="URL: http://svn.local/app/trunk") + + 0, + ExpectShell(workdir='source', + timeout=1, + command=['svn', 'update', '--non-interactive', + '--no-auth-cache']) + + 0, + ExpectShell(workdir='', + timeout=1, + command=['svn', 'export', 'source', 'wkdir']) + + 0, + ExpectShell(workdir='source', + timeout=1, + command=['svnversion']) + + ExpectShell.log('stdio', + stdout='100') + + 0, + ) + self.expectOutcome(result=SUCCESS, status_text=["update"]) + return self.runStep() + def test_mode_full_export_given_revision(self): self.setupStep( svn.SVN(repourl='http://svn.local/app/trunk', From 8c79548aeb0fa4ddcc60e0efcf2d1dd590a8c2c2 Mon Sep 17 00:00:00 2001 From: Tom Prince Date: Wed, 30 May 2012 21:21:56 -0600 Subject: [PATCH 120/136] Test that RemoteShellCommand's constructor can run with trivial arguments. This catches the error fixed in ab47f56fe1e441a761098234adfac9a27077a1f9. --- master/buildbot/test/interfaces/test_remotecommand.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/master/buildbot/test/interfaces/test_remotecommand.py b/master/buildbot/test/interfaces/test_remotecommand.py index f7bda5640e7..7fa6513fcaf 100644 --- a/master/buildbot/test/interfaces/test_remotecommand.py +++ b/master/buildbot/test/interfaces/test_remotecommand.py @@ -83,6 +83,9 @@ def test_signature_active(self): cmd = self.makeRemoteCommand() self.assertIsInstance(cmd.active, bool) + def test_RemoteShellCommand_constructor(self): + self.remoteShellCommandClass('wkdir', 'some-command') + class TestRunCommand(unittest.TestCase, Tests): From bee6af86dca8a17af667a7453e3c2e49ed3bee3f Mon Sep 17 00:00:00 2001 From: Edmund Wong Date: Wed, 30 May 2012 12:30:23 +0800 Subject: [PATCH 121/136] - Added check and tests for int values for caches --- master/buildbot/config.py | 4 ++++ master/buildbot/test/unit/test_config.py | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/master/buildbot/config.py b/master/buildbot/config.py index b73d331028c..c688f7c76bf 100644 --- a/master/buildbot/config.py +++ b/master/buildbot/config.py @@ -361,6 +361,10 @@ def load_caches(self, filename, config_dict, errors): if not isinstance(caches, dict): errors.addError("c['caches'] must be a dictionary") else: + vals = caches.values() + for x in vals: + if (not isinstance(x, int)): + errors.addError("value must be an int") self.caches.update(caches) if 'buildCacheSize' in config_dict: diff --git a/master/buildbot/test/unit/test_config.py b/master/buildbot/test/unit/test_config.py index cfbc1f90882..740fbdeac1a 100644 --- a/master/buildbot/test/unit/test_config.py +++ b/master/buildbot/test/unit/test_config.py @@ -546,6 +546,11 @@ def test_load_caches(self): self.errors) self.assertResults(caches=dict(Changes=10, Builds=15, foo=1)) + def test_load_caches_entries_test(self): + self.cfg.load_caches(self.filename, + dict(caches=dict(foo="1")), + self.errors) + self.assertConfigError(self.errors, "must be an int") def test_load_schedulers_defaults(self): self.cfg.load_schedulers(self.filename, {}, self.errors) From e80d3a0d3c1e8f3d51c074b201b945aa3b174f08 Mon Sep 17 00:00:00 2001 From: "Dustin J. Mitchell" Date: Wed, 30 May 2012 22:57:37 -0500 Subject: [PATCH 122/136] remove accidentally duplicated test --- .../test/unit/test_steps_source_git.py | 32 ------------------- 1 file changed, 32 deletions(-) diff --git a/master/buildbot/test/unit/test_steps_source_git.py b/master/buildbot/test/unit/test_steps_source_git.py index 2b870eea16b..d70545ef850 100644 --- a/master/buildbot/test/unit/test_steps_source_git.py +++ b/master/buildbot/test/unit/test_steps_source_git.py @@ -97,38 +97,6 @@ def test_mode_full_clean_timeout(self): self.expectProperty('got_revision', 'f6ad368298bd941e934a41f3babc827b2aa95a1d', 'Source') return self.runStep() - def test_mode_full_clean(self): - self.setupStep( - git.Git(repourl='http://github.com/buildbot/buildbot.git', - mode='full', method='clean')) - self.expectCommands( - ExpectShell(workdir='wkdir', - command=['git', '--version']) - + 0, - Expect('stat', dict(file='wkdir/.git', - logEnviron=True)) - + 0, - ExpectShell(workdir='wkdir', - command=['git', 'clean', '-f', '-d']) - + 0, - ExpectShell(workdir='wkdir', - command=['git', 'fetch', '-t', - 'http://github.com/buildbot/buildbot.git', - 'HEAD']) - + 0, - ExpectShell(workdir='wkdir', - command=['git', 'reset', '--hard', 'FETCH_HEAD']) - + 0, - ExpectShell(workdir='wkdir', - command=['git', 'rev-parse', 'HEAD']) - + ExpectShell.log('stdio', - stdout='f6ad368298bd941e934a41f3babc827b2aa95a1d') - + 0, - ) - self.expectOutcome(result=SUCCESS, status_text=["update"]) - self.expectProperty('got_revision', 'f6ad368298bd941e934a41f3babc827b2aa95a1d', 'Source') - return self.runStep() - def test_mode_full_clean_patch(self): self.setupStep( git.Git(repourl='http://github.com/buildbot/buildbot.git', From 5b22df96df9056ba01a47b27f2c889af48d9a623 Mon Sep 17 00:00:00 2001 From: Edmund Wong Date: Thu, 31 May 2012 12:33:34 +0800 Subject: [PATCH 123/136] Added check for ints for caches and added a test. --- master/buildbot/config.py | 9 +++++---- master/buildbot/test/unit/test_config.py | 3 ++- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/master/buildbot/config.py b/master/buildbot/config.py index c688f7c76bf..c95ff405327 100644 --- a/master/buildbot/config.py +++ b/master/buildbot/config.py @@ -361,10 +361,11 @@ def load_caches(self, filename, config_dict, errors): if not isinstance(caches, dict): errors.addError("c['caches'] must be a dictionary") else: - vals = caches.values() - for x in vals: - if (not isinstance(x, int)): - errors.addError("value must be an int") + valPairs = caches.items() + for (x, y) in valPairs: + if (not isinstance(y, int)): + errors.addError( + "value for cache size '%s' must be an integer" % x) self.caches.update(caches) if 'buildCacheSize' in config_dict: diff --git a/master/buildbot/test/unit/test_config.py b/master/buildbot/test/unit/test_config.py index 740fbdeac1a..30aceac7284 100644 --- a/master/buildbot/test/unit/test_config.py +++ b/master/buildbot/test/unit/test_config.py @@ -550,7 +550,8 @@ def test_load_caches_entries_test(self): self.cfg.load_caches(self.filename, dict(caches=dict(foo="1")), self.errors) - self.assertConfigError(self.errors, "must be an int") + self.assertConfigError(self.errors, + "value for cache size 'foo' must be an integer") def test_load_schedulers_defaults(self): self.cfg.load_schedulers(self.filename, {}, self.errors) From 5166fd6a031665e401f757001d238e49b032cd5e Mon Sep 17 00:00:00 2001 From: Edmund Wong Date: Thu, 31 May 2012 12:36:39 +0800 Subject: [PATCH 124/136] Added check for ints for caches and added a test. --- master/buildbot/config.py | 2 +- master/buildbot/test/unit/test_config.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/master/buildbot/config.py b/master/buildbot/config.py index c95ff405327..c33db5da10a 100644 --- a/master/buildbot/config.py +++ b/master/buildbot/config.py @@ -365,7 +365,7 @@ def load_caches(self, filename, config_dict, errors): for (x, y) in valPairs: if (not isinstance(y, int)): errors.addError( - "value for cache size '%s' must be an integer" % x) + "value for cache size '%s' must be an integer" % x) self.caches.update(caches) if 'buildCacheSize' in config_dict: diff --git a/master/buildbot/test/unit/test_config.py b/master/buildbot/test/unit/test_config.py index 30aceac7284..45da51bffff 100644 --- a/master/buildbot/test/unit/test_config.py +++ b/master/buildbot/test/unit/test_config.py @@ -551,7 +551,7 @@ def test_load_caches_entries_test(self): dict(caches=dict(foo="1")), self.errors) self.assertConfigError(self.errors, - "value for cache size 'foo' must be an integer") + "value for cache size 'foo' must be an integer") def test_load_schedulers_defaults(self): self.cfg.load_schedulers(self.filename, {}, self.errors) From 3231bb99250855203d28797f6829407386b3a496 Mon Sep 17 00:00:00 2001 From: Benoit Allard Date: Thu, 31 May 2012 10:34:13 +0200 Subject: [PATCH 125/136] hook a single master or a list of masters to mercurial --- master/buildbot/changes/hgbuildbot.py | 62 ++++++++++++++------------- 1 file changed, 32 insertions(+), 30 deletions(-) diff --git a/master/buildbot/changes/hgbuildbot.py b/master/buildbot/changes/hgbuildbot.py index 909c427ba24..b01a13ca805 100644 --- a/master/buildbot/changes/hgbuildbot.py +++ b/master/buildbot/changes/hgbuildbot.py @@ -47,8 +47,8 @@ def hook(ui, repo, hooktype, node=None, source=None, **kwargs): # read config parameters baseurl = ui.config('hgbuildbot', 'baseurl', ui.config('web', 'baseurl', '')) - master = ui.config('hgbuildbot', 'master') - if master: + masters = ui.configlist('hgbuildbot', 'master') + if masters: branchtype = ui.config('hgbuildbot', 'branchtype', 'inrepo') branch = ui.config('hgbuildbot', 'branch') fork = ui.configbool('hgbuildbot', 'fork', False) @@ -88,11 +88,8 @@ def hook(ui, repo, hooktype, node=None, source=None, **kwargs): auth = 'change:changepw' auth = auth.split(':', 1) - s = sendchange.Sender(master, auth=auth) - d = defer.Deferred() - reactor.callLater(0, d.callback, None) # process changesets - def _send(res, c): + def _send(res, s, c): if not fork: ui.status("rev %s sent\n" % c['revision']) return s.send(c['branch'], c['revision'], c['comments'], @@ -110,30 +107,35 @@ def _send(res, c): repository = strip(repo.root, stripcount) repository = baseurl + repository - for rev in xrange(start, end): - # send changeset - node = repo.changelog.node(rev) - manifest, user, (time, timezone), files, desc, extra = repo.changelog.read(node) - parents = filter(lambda p: not p == nullid, repo.changelog.parents(node)) - if branchtype == 'inrepo': - branch = extra['branch'] - is_merge = len(parents) > 1 - # merges don't always contain files, but at least one file is required by buildbot - if is_merge and not files: - files = ["merge"] - properties = {'is_merge': is_merge} - if branch: - branch = fromlocal(branch) - change = { - 'master': master, - 'username': fromlocal(user), - 'revision': hex(node), - 'comments': fromlocal(desc), - 'files': files, - 'branch': branch, - 'properties':properties - } - d.addCallback(_send, change) + for master in masters: + s = sendchange.Sender(master, auth=auth) + d = defer.Deferred() + reactor.callLater(0, d.callback, None) + + for rev in xrange(start, end): + # send changeset + node = repo.changelog.node(rev) + manifest, user, (time, timezone), files, desc, extra = repo.changelog.read(node) + parents = filter(lambda p: not p == nullid, repo.changelog.parents(node)) + if branchtype == 'inrepo': + branch = extra['branch'] + is_merge = len(parents) > 1 + # merges don't always contain files, but at least one file is required by buildbot + if is_merge and not files: + files = ["merge"] + properties = {'is_merge': is_merge} + if branch: + branch = fromlocal(branch) + change = { + 'master': master, + 'username': fromlocal(user), + 'revision': hex(node), + 'comments': fromlocal(desc), + 'files': files, + 'branch': branch, + 'properties':properties + } + d.addCallback(_send, s, change) def _printSuccess(res): ui.status(s.getSuccessString(res) + '\n') From 82e3bbbcc301fae672c8e37d721679e3e54da834 Mon Sep 17 00:00:00 2001 From: Harry Borkhuis Date: Thu, 31 May 2012 10:36:57 +0200 Subject: [PATCH 126/136] master list feature added to hgbuildbot documentation --- master/docs/manual/cfg-changesources.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/master/docs/manual/cfg-changesources.rst b/master/docs/manual/cfg-changesources.rst index ec0b134a355..d137dc4334c 100644 --- a/master/docs/manual/cfg-changesources.rst +++ b/master/docs/manual/cfg-changesources.rst @@ -511,6 +511,16 @@ hostname/portnumber as appropriate for your buildbot: master = buildmaster.example.org:9987 # .. other hgbuildbot parameters .. +The ``master`` configuration key allows to have more than one buildmaster +specification. The buildmasters have to be separated by a whitspace +or comma (see also 'hg help config'): + +.. code-block:: ini + + master = + buildmaster.example.org:9987 + buildmaster2.example.org:9989 + .. note:: Mercurial lets you define multiple ``changegroup`` hooks by giving them distinct names, like ``changegroup.foo`` and ``changegroup.bar``, which is why we use ``changegroup.buildbot`` From 4df4838968507f026493790b08507a46b3ed6609 Mon Sep 17 00:00:00 2001 From: Harry Borkhuis Date: Thu, 31 May 2012 11:38:26 +0200 Subject: [PATCH 127/136] indentation typo fixed --- master/docs/manual/cfg-changesources.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/master/docs/manual/cfg-changesources.rst b/master/docs/manual/cfg-changesources.rst index d137dc4334c..836f80dad43 100644 --- a/master/docs/manual/cfg-changesources.rst +++ b/master/docs/manual/cfg-changesources.rst @@ -519,7 +519,7 @@ or comma (see also 'hg help config'): master = buildmaster.example.org:9987 - buildmaster2.example.org:9989 + buildmaster2.example.org:9989 .. note:: Mercurial lets you define multiple ``changegroup`` hooks by giving them distinct names, like ``changegroup.foo`` and From e77702b83e5f63bced6e0891e721b7fef969f582 Mon Sep 17 00:00:00 2001 From: Edmund Wong Date: Thu, 31 May 2012 17:40:45 +0800 Subject: [PATCH 128/136] Fix ticket #268. --- master/buildbot/status/web/templates/build.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/master/buildbot/status/web/templates/build.html b/master/buildbot/status/web/templates/build.html index 2148486d66a..4c7dcd424d2 100644 --- a/master/buildbot/status/web/templates/build.html +++ b/master/buildbot/status/web/templates/build.html @@ -191,7 +191,7 @@

Forced Build Properties:

{% endfor %} -

Blamelist:

+

Annotate List:

{% if responsible_users %}
    From ccf072c16f53832cdb533908779be58418cf0fdb Mon Sep 17 00:00:00 2001 From: Edmund Wong Date: Thu, 31 May 2012 23:01:14 +0800 Subject: [PATCH 129/136] Changed "Annotate" to "Responsible Users" --- master/buildbot/status/web/templates/build.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/master/buildbot/status/web/templates/build.html b/master/buildbot/status/web/templates/build.html index 4c7dcd424d2..8b60b90b53a 100644 --- a/master/buildbot/status/web/templates/build.html +++ b/master/buildbot/status/web/templates/build.html @@ -191,7 +191,7 @@

    Forced Build Properties:

    {% endfor %} -

    Annotate List:

    +

    Responsible Users:

    {% if responsible_users %}
      From 3d968a1f193cdc692ac278d5b7cf368918eb986b Mon Sep 17 00:00:00 2001 From: tycho garen Date: Sat, 2 Jun 2012 08:16:20 -0400 Subject: [PATCH 130/136] changes to the style guide (language clean up and rst tweaks.) --- master/docs/developer/style.rst | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/master/docs/developer/style.rst b/master/docs/developer/style.rst index 7850833fc3c..bccd450b17d 100644 --- a/master/docs/developer/style.rst +++ b/master/docs/developer/style.rst @@ -9,7 +9,7 @@ the formatting of symbol names. The single exception in naming of functions and methods. Because Buildbot uses Twisted so heavily, and Twisted uses interCaps, Buildbot methods should do the -same. That is, methods and functions should be spelled with the first character +same. That is, you should spell methods and functions with the first character in lower-case, and the first letter of subsequent words capitalized, e.g., ``compareToOther`` or ``getChangesGreaterThan``. This point is not applied very consistently in Buildbot, but let's try to be consistent in new code. @@ -30,8 +30,8 @@ Just about anything might block - even getters and setters! Helpful Twisted Classes ~~~~~~~~~~~~~~~~~~~~~~~ -Twisted has some useful, but little-known classes. They are listed here with -brief descriptions, but you should consult the API documentation or source code +Twisted has some useful, but little-known classes. +Brief descriptions follow, but you should consult the API documentation or source code for the full details. :class:`twisted.internet.task.LoopingCall` @@ -39,8 +39,8 @@ for the full details. :class:`twisted.application.internet.TimerService` Similar to ``t.i.t.LoopingCall``, but implemented as a service that will - automatically start and stop the function calls when the service is started and - stopped. + automatically start and stop the function calls when the service starts and + stops. Sequences of Operations ~~~~~~~~~~~~~~~~~~~~~~~ @@ -134,11 +134,11 @@ The key points to notice here: * Use the decorator form of ``inlineCallbacks`` * In most cases, the result of a ``yield`` expression should be assigned to a variable. It can be used in a larger expression, but remember that Python - requires that it be enclosed in its own set of parentheses. + requires that you enclose the expression in its own set of parentheses. * Python does not permit returning a value from a generator, so statements like ``return xval + y`` are invalid. Instead, yield the result of ``defer.returnValue``. Although this function does cause an immediate - function exit, for clarity it should be followed by a bare ``return``, as in + function exit, for clarity follow it with a bare ``return``, as in the example, unless it is the last statement in a function. The great advantage of ``inlineCallbacks`` is that it allows you to use all @@ -157,7 +157,7 @@ operations, each time you wait for a Deferred, arbitrary other actions can take place. In general, you should try to perform actions atomically, but for the rare -times synchronization is required, the following might be useful: +situations that require synchronization, the following might be useful: * :py:class:`twisted.internet.defer.DeferredLock` * :py:func:`buildbot.util.misc.deferredLocked` @@ -168,8 +168,8 @@ Joining Sequences It's often the case that you'll want to perform multiple operations in parallel, and re-join the results at the end. For this purpose, you'll want to -use a `DeferredList -`_:: +use a `DeferredList `_ +:: def getRevInfo(revname): results = {} @@ -192,5 +192,5 @@ use a `DeferredList return d Here the deferred list will wait for both ``rev_parse_d`` and ``log_d`` to -fire, or for one of them to fail. Callbacks and errbacks can be attached to a +fire, or for one of them to fail. You may attach Callbacks and errbacks to a ``DeferredList`` just as for a deferred. From 88398012fd9ac672628a259f24d2a213a1a23b51 Mon Sep 17 00:00:00 2001 From: "Dustin J. Mitchell" Date: Sat, 2 Jun 2012 17:39:41 -0500 Subject: [PATCH 131/136] relnotes for pull 436 --- master/docs/release-notes.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/master/docs/release-notes.rst b/master/docs/release-notes.rst index de451fb1be4..01cd4d4db36 100644 --- a/master/docs/release-notes.rst +++ b/master/docs/release-notes.rst @@ -95,6 +95,8 @@ Features * A new ternary substitution operator ``:?:`` and ``:#?:`` to use with the ``Interpolate`` and ``WithProperties`` classes. +* The mercurial hook now supports multple masters. See :bb:pull:`436`. + Slave ----- From a94047bc8d69cd192d376ed653c05422c90cddbd Mon Sep 17 00:00:00 2001 From: John Carr Date: Sun, 3 Jun 2012 00:09:48 +0100 Subject: [PATCH 132/136] Add a web hook that can trigger named pollers --- master/buildbot/changes/base.py | 53 +++++++++---- master/buildbot/changes/bonsaipoller.py | 6 +- master/buildbot/changes/gitpoller.py | 7 +- master/buildbot/changes/p4poller.py | 6 +- master/buildbot/changes/svnpoller.py | 6 +- master/buildbot/status/web/hooks/poller.py | 46 +++++++++++ .../test_status_web_change_hooks_poller.py | 76 +++++++++++++++++++ master/docs/manual/cfg-statustargets.rst | 28 +++++++ 8 files changed, 206 insertions(+), 22 deletions(-) create mode 100644 master/buildbot/status/web/hooks/poller.py create mode 100644 master/buildbot/test/unit/test_status_web_change_hooks_poller.py diff --git a/master/buildbot/changes/base.py b/master/buildbot/changes/base.py index 07eefeedbcc..3af353e9b1e 100644 --- a/master/buildbot/changes/base.py +++ b/master/buildbot/changes/base.py @@ -35,6 +35,8 @@ class PollingChangeSource(ChangeSource): Utility subclass for ChangeSources that use some kind of periodic polling operation. Subclasses should define C{poll} and set C{self.pollInterval}. The rest is taken care of. + + Any subclass will be available via the "poller" webhook. """ pollInterval = 60 @@ -42,6 +44,24 @@ class PollingChangeSource(ChangeSource): _loop = None + def __init__(self, name=None, pollInterval=60*10): + if name: + self.setName(name) + self.pollInterval = pollInterval + + self.doPoll = util.misc.SerializedInvocation(self.doPoll) + + def doPoll(self): + """ + This is the method that is called by LoopingCall to actually poll. + It may also be called by change hooks to request a poll. + It is serialiazed - if you call it while a poll is in progress + then the 2nd invocation won't start until the 1st has finished. + """ + d = defer.maybeDeferred(self.poll) + d.addErrback(log.err, 'while polling for changes') + return d + def poll(self): """ Perform the polling operation, and return a deferred that will fire @@ -49,22 +69,27 @@ def poll(self): method will be called again after C{pollInterval} seconds. """ + def startLoop(self): + self._loop = task.LoopingCall(self.doPoll) + self._loop.start(self.pollInterval, now=False) + + def stopLoop(self): + if self._loop and self._loop.running: + self._loop.stop() + self._loop = None + def startService(self): ChangeSource.startService(self) - def do_poll(): - d = defer.maybeDeferred(self.poll) - d.addErrback(log.err, 'while polling for changes') - return d - - # delay starting the loop until the reactor is running, and do not - # run it immediately - if services are still starting up, they may - # miss an initial flood of changes - def start_loop(): - self._loop = task.LoopingCall(do_poll) - self._loop.start(self.pollInterval, now=False) - reactor.callWhenRunning(start_loop) + + # delay starting doing anything until the reactor is running - if + # services are still starting up, they may miss an initial flood of + # changes + if self.pollInterval: + reactor.callWhenRunning(self.startLoop) + else: + reactor.callWhenRunning(self.doPoll) def stopService(self): - if self._loop and self._loop.running: - self._loop.stop() + self.stopLoop() return ChangeSource.stopService(self) + diff --git a/master/buildbot/changes/bonsaipoller.py b/master/buildbot/changes/bonsaipoller.py index 56f5dab174a..9f252ee5c6a 100644 --- a/master/buildbot/changes/bonsaipoller.py +++ b/master/buildbot/changes/bonsaipoller.py @@ -207,14 +207,16 @@ class BonsaiPoller(base.PollingChangeSource): "module", "branch", "cvsroot"] def __init__(self, bonsaiURL, module, branch, tree="default", - cvsroot="/cvsroot", pollInterval=30, project=''): + cvsroot="/cvsroot", pollInterval=30, project='', name=None): + + base.PollingChangeSource.__init__(self, name=name, pollInterval=pollInterval) + self.bonsaiURL = bonsaiURL self.module = module self.branch = branch self.tree = tree self.cvsroot = cvsroot self.repository = module != 'all' and module or '' - self.pollInterval = pollInterval self.lastChange = time.time() self.lastPoll = time.time() diff --git a/master/buildbot/changes/gitpoller.py b/master/buildbot/changes/gitpoller.py index c9e2e79e7c1..ee394b48f21 100644 --- a/master/buildbot/changes/gitpoller.py +++ b/master/buildbot/changes/gitpoller.py @@ -36,15 +36,18 @@ def __init__(self, repourl, branch='master', gitbin='git', usetimestamps=True, category=None, project=None, pollinterval=-2, fetch_refspec=None, - encoding='utf-8'): + encoding='utf-8', name=None): + # for backward compatibility; the parameter used to be spelled with 'i' if pollinterval != -2: pollInterval = pollinterval + + base.PollingChangeSource.__init__(self, name=name, pollInterval=pollInterval) + if project is None: project = '' self.repourl = repourl self.branch = branch - self.pollInterval = pollInterval self.fetch_refspec = fetch_refspec self.encoding = encoding self.lastChange = time.time() diff --git a/master/buildbot/changes/p4poller.py b/master/buildbot/changes/p4poller.py index ffae1ce52b2..255570f0e9f 100644 --- a/master/buildbot/changes/p4poller.py +++ b/master/buildbot/changes/p4poller.py @@ -66,11 +66,14 @@ def __init__(self, p4port=None, p4user=None, p4passwd=None, p4base='//', p4bin='p4', split_file=lambda branchfile: (None, branchfile), pollInterval=60 * 10, histmax=None, pollinterval=-2, - encoding='utf8', project=None): + encoding='utf8', project=None, name=None): + # for backward compatibility; the parameter used to be spelled with 'i' if pollinterval != -2: pollInterval = pollinterval + base.PollingChangeSource.__init__(self, name=name, pollInterval=pollInterval) + if project is None: project = '' @@ -80,7 +83,6 @@ def __init__(self, p4port=None, p4user=None, p4passwd=None, self.p4base = p4base self.p4bin = p4bin self.split_file = split_file - self.pollInterval = pollInterval self.encoding = encoding self.project = project diff --git a/master/buildbot/changes/svnpoller.py b/master/buildbot/changes/svnpoller.py index 12bfe053727..fde04097856 100644 --- a/master/buildbot/changes/svnpoller.py +++ b/master/buildbot/changes/svnpoller.py @@ -66,11 +66,14 @@ def __init__(self, svnurl, split_file=None, pollInterval=10*60, histmax=100, svnbin='svn', revlinktmpl='', category=None, project='', cachepath=None, pollinterval=-2, - extra_args=None): + extra_args=None, name=None): + # for backward compatibility; the parameter used to be spelled with 'i' if pollinterval != -2: pollInterval = pollinterval + base.PollingChangeSource.__init__(self, name=name, pollInterval=pollInterval) + if svnurl.endswith("/"): svnurl = svnurl[:-1] # strip the trailing slash self.svnurl = svnurl @@ -85,7 +88,6 @@ def __init__(self, svnurl, split_file=None, # required for ssh-agent auth self.svnbin = svnbin - self.pollInterval = pollInterval self.histmax = histmax self._prefix = None self.category = category diff --git a/master/buildbot/status/web/hooks/poller.py b/master/buildbot/status/web/hooks/poller.py new file mode 100644 index 00000000000..01fd43af49b --- /dev/null +++ b/master/buildbot/status/web/hooks/poller.py @@ -0,0 +1,46 @@ +# This file is part of Buildbot. Buildbot is free software: you can +# redistribute it and/or modify it under the terms of the GNU General Public +# License as published by the Free Software Foundation, version 2. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +# details. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, write to the Free Software Foundation, Inc., 51 +# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Copyright Buildbot Team Members + +# This change hook allows GitHub or a hand crafted curl inovcation to "knock on +# the door" and trigger a change source to poll. + +from buildbot.changes.base import PollingChangeSource + + +def getChanges(req, options=None): + change_svc = req.site.buildbot_service.master.change_svc + + if not "poller" in req.args: + raise ValueError("Request missing parameter 'poller'") + + pollers = [] + + for pollername in req.args['poller']: + try: + source = change_svc.getServiceNamed(pollername) + except KeyError: + raise ValueError("No such change source '%s'" % pollername) + + if not isinstance(source, PollingChangeSource): + raise ValueError("No such polling change source '%s'" % pollername) + + pollers.append(source) + + for p in pollers: + source.doPoll() + + return [], None + + diff --git a/master/buildbot/test/unit/test_status_web_change_hooks_poller.py b/master/buildbot/test/unit/test_status_web_change_hooks_poller.py new file mode 100644 index 00000000000..f02ea29b472 --- /dev/null +++ b/master/buildbot/test/unit/test_status_web_change_hooks_poller.py @@ -0,0 +1,76 @@ +# This file is part of Buildbot. Buildbot is free software: you can +# redistribute it and/or modify it under the terms of the GNU General Public +# License as published by the Free Software Foundation, version 2. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +# details. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, write to the Free Software Foundation, Inc., 51 +# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Copyright Buildbot Team Members + +from twisted.trial import unittest +from twisted.internet import defer +from buildbot.changes import base +import buildbot.status.web.change_hook as change_hook +from buildbot.test.fake.web import FakeRequest + +class TestPollingChangeHook(unittest.TestCase): + class Subclass(base.PollingChangeSource): + pollInterval = None + called = False + + def poll(self): + self.called = True + + def setUp(self): + self.changeHook = change_hook.ChangeHookResource(dialects={'poller' : True}) + self.changesrc= self.Subclass() + + @defer.inlineCallbacks + def test_no_args(self): + self.request = FakeRequest(args={}) + self.request.uri = "/change_hook/poller" + self.request.method = "GET" + yield self.request.test_render(self.changeHook) + + expected = "Request missing parameter 'poller'" + self.assertEqual(self.request.written, expected) + self.request.setResponseCode.assert_called_with(400, expected) + + @defer.inlineCallbacks + def test_no_poller(self): + self.request = FakeRequest(args={"poller":["example"]}) + self.request.uri = "/change_hook/poller" + self.request.method = "GET" + self.request.master.change_svc.getServiceNamed.side_effect = KeyError + yield self.request.test_render(self.changeHook) + + expected = "No such change source 'example'" + self.assertEqual(self.request.written, expected) + self.request.setResponseCode.assert_called_with(400, expected) + + @defer.inlineCallbacks + def test_invalid_poller(self): + self.request = FakeRequest(args={"poller":["example"]}) + self.request.uri = "/change_hook/poller" + self.request.method = "GET" + yield self.request.test_render(self.changeHook) + + expected = "No such polling change source 'example'" + self.assertEqual(self.request.written, expected) + self.request.setResponseCode.assert_called_with(400, expected) + + @defer.inlineCallbacks + def test_trigger_poll(self): + self.request = FakeRequest(args={"poller":["example"]}) + self.request.uri = "/change_hook/poller" + self.request.method = "GET" + self.request.master.change_svc.getServiceNamed.return_value = self.changesrc + yield self.request.test_render(self.changeHook) + self.assertEqual(self.changesrc.called, True) + diff --git a/master/docs/manual/cfg-statustargets.rst b/master/docs/manual/cfg-statustargets.rst index b13ade92d83..e7dabf52fb5 100644 --- a/master/docs/manual/cfg-statustargets.rst +++ b/master/docs/manual/cfg-statustargets.rst @@ -709,6 +709,34 @@ that periodically poll the Google Code commit feed for changes. change_hook_dialects={'googlecode': {'secret_key': 'FSP3p-Ghdn4T0oqX', 'branch': 'master'}} +Poller hook +########### + +The poller hook allows you to use GET requests to trigger polling. One +advantage of this is your buildbot instance can (at start up) poll to get +changes that happened while it was down, but then you can still use a commit +hook to get fast notification of new changes. + +Suppose you have a poller configured like this:: + + c['change_source'] = SVNPoller( + name="amanda", + svnurl="https://amanda.svn.sourceforge.net/svnroot/amanda/amanda", + split_file=split_file_branches) + +And you configure your WebStatus to enable this hook:: + + c['status'].append(html.WebStatus( + …, + change_hook_dialects={'poller': True} + )) + +Then you will be able to trigger a poll of the SVN repository by poking the +``/change_hook/poller`` URL from a commit hook like this:: + + curl http://yourbuildbot/change_hook/poller?poller=amanda + + .. bb:status:: MailNotifier .. index:: single: email; MailNotifier From e1f505f1e540c9c783a25dc5ae39c66bf1f5cd50 Mon Sep 17 00:00:00 2001 From: John Carr Date: Sun, 3 Jun 2012 15:59:27 +0100 Subject: [PATCH 133/136] If 'poller' argument isn't provided, trigger all of them --- master/buildbot/status/web/hooks/poller.py | 26 ++++----- .../test_status_web_change_hooks_poller.py | 56 ++++++++++--------- master/docs/manual/cfg-statustargets.rst | 3 + 3 files changed, 45 insertions(+), 40 deletions(-) diff --git a/master/buildbot/status/web/hooks/poller.py b/master/buildbot/status/web/hooks/poller.py index 01fd43af49b..4ba20ff2bf4 100644 --- a/master/buildbot/status/web/hooks/poller.py +++ b/master/buildbot/status/web/hooks/poller.py @@ -21,26 +21,26 @@ def getChanges(req, options=None): change_svc = req.site.buildbot_service.master.change_svc - - if not "poller" in req.args: - raise ValueError("Request missing parameter 'poller'") + poll_all = not "poller" in req.args pollers = [] - for pollername in req.args['poller']: - try: - source = change_svc.getServiceNamed(pollername) - except KeyError: - raise ValueError("No such change source '%s'" % pollername) - + for source in change_svc: if not isinstance(source, PollingChangeSource): - raise ValueError("No such polling change source '%s'" % pollername) - + continue + if not hasattr(source, "name"): + continue + if not poll_all and not source.name in req.args['poller']: + continue pollers.append(source) + if not poll_all: + missing = set(req.args['poller']) - set(s.name for s in pollers) + if missing: + raise ValueError("Could not find pollers: %s" % ",".join(missing)) + for p in pollers: - source.doPoll() + p.doPoll() return [], None - diff --git a/master/buildbot/test/unit/test_status_web_change_hooks_poller.py b/master/buildbot/test/unit/test_status_web_change_hooks_poller.py index f02ea29b472..87573920f33 100644 --- a/master/buildbot/test/unit/test_status_web_change_hooks_poller.py +++ b/master/buildbot/test/unit/test_status_web_change_hooks_poller.py @@ -18,6 +18,8 @@ from buildbot.changes import base import buildbot.status.web.change_hook as change_hook from buildbot.test.fake.web import FakeRequest +from buildbot.changes.manager import ChangeManager + class TestPollingChangeHook(unittest.TestCase): class Subclass(base.PollingChangeSource): @@ -27,50 +29,50 @@ class Subclass(base.PollingChangeSource): def poll(self): self.called = True - def setUp(self): + def setUpRequest(self, args): self.changeHook = change_hook.ChangeHookResource(dialects={'poller' : True}) - self.changesrc= self.Subclass() - @defer.inlineCallbacks - def test_no_args(self): - self.request = FakeRequest(args={}) + self.request = FakeRequest(args=args) self.request.uri = "/change_hook/poller" self.request.method = "GET" - yield self.request.test_render(self.changeHook) - expected = "Request missing parameter 'poller'" - self.assertEqual(self.request.written, expected) - self.request.setResponseCode.assert_called_with(400, expected) + master = self.request.site.buildbot_service.master + master.change_svc = ChangeManager(master) + + self.changesrc = self.Subclass("example", None) + self.changesrc.setServiceParent(master.change_svc) + + anotherchangesrc = base.ChangeSource() + anotherchangesrc.setName("notapoller") + anotherchangesrc.setServiceParent(master.change_svc) + + return self.request.test_render(self.changeHook) @defer.inlineCallbacks - def test_no_poller(self): - self.request = FakeRequest(args={"poller":["example"]}) - self.request.uri = "/change_hook/poller" - self.request.method = "GET" - self.request.master.change_svc.getServiceNamed.side_effect = KeyError - yield self.request.test_render(self.changeHook) + def test_no_args(self): + yield self.setUpRequest({}) + self.assertEqual(self.request.written, "no changes found") + self.assertEqual(self.changesrc.called, True) - expected = "No such change source 'example'" + @defer.inlineCallbacks + def test_no_poller(self): + yield self.setUpRequest({"poller": ["nosuchpoller"]}) + expected = "Could not find pollers: nosuchpoller" self.assertEqual(self.request.written, expected) self.request.setResponseCode.assert_called_with(400, expected) + self.assertEqual(self.changesrc.called, False) @defer.inlineCallbacks def test_invalid_poller(self): - self.request = FakeRequest(args={"poller":["example"]}) - self.request.uri = "/change_hook/poller" - self.request.method = "GET" - yield self.request.test_render(self.changeHook) - - expected = "No such polling change source 'example'" + yield self.setUpRequest({"poller": ["notapoller"]}) + expected = "Could not find pollers: notapoller" self.assertEqual(self.request.written, expected) self.request.setResponseCode.assert_called_with(400, expected) + self.assertEqual(self.changesrc.called, False) @defer.inlineCallbacks def test_trigger_poll(self): - self.request = FakeRequest(args={"poller":["example"]}) - self.request.uri = "/change_hook/poller" - self.request.method = "GET" - self.request.master.change_svc.getServiceNamed.return_value = self.changesrc - yield self.request.test_render(self.changeHook) + yield self.setUpRequest({"poller": ["example"]}) + self.assertEqual(self.request.written, "no changes found") self.assertEqual(self.changesrc.called, True) diff --git a/master/docs/manual/cfg-statustargets.rst b/master/docs/manual/cfg-statustargets.rst index e7dabf52fb5..bdd37e1206e 100644 --- a/master/docs/manual/cfg-statustargets.rst +++ b/master/docs/manual/cfg-statustargets.rst @@ -736,6 +736,9 @@ Then you will be able to trigger a poll of the SVN repository by poking the curl http://yourbuildbot/change_hook/poller?poller=amanda +If no ``poller`` argument is provided then the hook will trigger polling of all +polling change sources. + .. bb:status:: MailNotifier From aa177f38517c47857b77d1262a16811d66f34e72 Mon Sep 17 00:00:00 2001 From: John Carr Date: Sun, 3 Jun 2012 16:09:38 +0100 Subject: [PATCH 134/136] Implement an allowed list --- master/buildbot/status/web/hooks/poller.py | 8 ++++++++ .../test_status_web_change_hooks_poller.py | 18 ++++++++++++++++-- master/docs/manual/cfg-statustargets.rst | 8 ++++++++ 3 files changed, 32 insertions(+), 2 deletions(-) diff --git a/master/buildbot/status/web/hooks/poller.py b/master/buildbot/status/web/hooks/poller.py index 4ba20ff2bf4..1ac29a4f599 100644 --- a/master/buildbot/status/web/hooks/poller.py +++ b/master/buildbot/status/web/hooks/poller.py @@ -23,6 +23,12 @@ def getChanges(req, options=None): change_svc = req.site.buildbot_service.master.change_svc poll_all = not "poller" in req.args + allow_all = True + allowed = [] + if isinstance(options, dict) and "allowed" in options: + allow_all = False + allowed = options["allowed"] + pollers = [] for source in change_svc: @@ -32,6 +38,8 @@ def getChanges(req, options=None): continue if not poll_all and not source.name in req.args['poller']: continue + if not allow_all and not source.name in allowed: + continue pollers.append(source) if not poll_all: diff --git a/master/buildbot/test/unit/test_status_web_change_hooks_poller.py b/master/buildbot/test/unit/test_status_web_change_hooks_poller.py index 87573920f33..e92689a6ca2 100644 --- a/master/buildbot/test/unit/test_status_web_change_hooks_poller.py +++ b/master/buildbot/test/unit/test_status_web_change_hooks_poller.py @@ -29,8 +29,8 @@ class Subclass(base.PollingChangeSource): def poll(self): self.called = True - def setUpRequest(self, args): - self.changeHook = change_hook.ChangeHookResource(dialects={'poller' : True}) + def setUpRequest(self, args, options=True): + self.changeHook = change_hook.ChangeHookResource(dialects={'poller' : options}) self.request = FakeRequest(args=args) self.request.uri = "/change_hook/poller" @@ -76,3 +76,17 @@ def test_trigger_poll(self): self.assertEqual(self.request.written, "no changes found") self.assertEqual(self.changesrc.called, True) + @defer.inlineCallbacks + def test_allowlist_deny(self): + yield self.setUpRequest({"poller": ["example"]}, options={"allowed": []}) + expected = "Could not find pollers: example" + self.assertEqual(self.request.written, expected) + self.request.setResponseCode.assert_called_with(400, expected) + self.assertEqual(self.changesrc.called, False) + + @defer.inlineCallbacks + def test_allowlist_allow(self): + yield self.setUpRequest({"poller": ["example"]}, options={"allowed": ["example"]}) + self.assertEqual(self.request.written, "no changes found") + self.assertEqual(self.changesrc.called, True) + diff --git a/master/docs/manual/cfg-statustargets.rst b/master/docs/manual/cfg-statustargets.rst index bdd37e1206e..21d69e25f46 100644 --- a/master/docs/manual/cfg-statustargets.rst +++ b/master/docs/manual/cfg-statustargets.rst @@ -739,6 +739,14 @@ Then you will be able to trigger a poll of the SVN repository by poking the If no ``poller`` argument is provided then the hook will trigger polling of all polling change sources. +You can restrict which pollers the webhook has access to using the ``allowed`` +option:: + + c['status'].append(html.WebStatus( + …, + change_hook_dialects={'poller': {'allowed': ['amanda']}} + )) + .. bb:status:: MailNotifier From 4c12f65d50595be12c24e393c72cac5c9ecb20f5 Mon Sep 17 00:00:00 2001 From: Tom Prince Date: Tue, 5 Jun 2012 15:33:49 -0600 Subject: [PATCH 135/136] poller-web-hook: Add a test of all allowed pollers. --- .../test_status_web_change_hooks_poller.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/master/buildbot/test/unit/test_status_web_change_hooks_poller.py b/master/buildbot/test/unit/test_status_web_change_hooks_poller.py index e92689a6ca2..fca1b66e0b3 100644 --- a/master/buildbot/test/unit/test_status_web_change_hooks_poller.py +++ b/master/buildbot/test/unit/test_status_web_change_hooks_poller.py @@ -42,6 +42,9 @@ def setUpRequest(self, args, options=True): self.changesrc = self.Subclass("example", None) self.changesrc.setServiceParent(master.change_svc) + self.disabledChangesrc = self.Subclass("disabled", None) + self.disabledChangesrc.setServiceParent(master.change_svc) + anotherchangesrc = base.ChangeSource() anotherchangesrc.setName("notapoller") anotherchangesrc.setServiceParent(master.change_svc) @@ -53,6 +56,7 @@ def test_no_args(self): yield self.setUpRequest({}) self.assertEqual(self.request.written, "no changes found") self.assertEqual(self.changesrc.called, True) + self.assertEqual(self.disabledChangesrc.called, True) @defer.inlineCallbacks def test_no_poller(self): @@ -61,6 +65,7 @@ def test_no_poller(self): self.assertEqual(self.request.written, expected) self.request.setResponseCode.assert_called_with(400, expected) self.assertEqual(self.changesrc.called, False) + self.assertEqual(self.disabledChangesrc.called, False) @defer.inlineCallbacks def test_invalid_poller(self): @@ -69,24 +74,34 @@ def test_invalid_poller(self): self.assertEqual(self.request.written, expected) self.request.setResponseCode.assert_called_with(400, expected) self.assertEqual(self.changesrc.called, False) + self.assertEqual(self.disabledChangesrc.called, False) @defer.inlineCallbacks def test_trigger_poll(self): yield self.setUpRequest({"poller": ["example"]}) self.assertEqual(self.request.written, "no changes found") self.assertEqual(self.changesrc.called, True) + self.assertEqual(self.disabledChangesrc.called, False) @defer.inlineCallbacks def test_allowlist_deny(self): - yield self.setUpRequest({"poller": ["example"]}, options={"allowed": []}) - expected = "Could not find pollers: example" + yield self.setUpRequest({"poller": ["disabled"]}, options={"allowed": ["example"]}) + expected = "Could not find pollers: disabled" self.assertEqual(self.request.written, expected) self.request.setResponseCode.assert_called_with(400, expected) self.assertEqual(self.changesrc.called, False) + self.assertEqual(self.disabledChangesrc.called, False) @defer.inlineCallbacks def test_allowlist_allow(self): yield self.setUpRequest({"poller": ["example"]}, options={"allowed": ["example"]}) self.assertEqual(self.request.written, "no changes found") self.assertEqual(self.changesrc.called, True) + self.assertEqual(self.disabledChangesrc.called, False) + @defer.inlineCallbacks + def test_allowlist_all(self): + yield self.setUpRequest({}, options={"allowed": ["example"]}) + self.assertEqual(self.request.written, "no changes found") + self.assertEqual(self.changesrc.called, True) + self.assertEqual(self.disabledChangesrc.called, False) From c1d4b2af6d25219cc119e1f296b37c85cad0e23f Mon Sep 17 00:00:00 2001 From: John Carr Date: Sun, 3 Jun 2012 01:04:22 +0100 Subject: [PATCH 136/136] Add a 'jinja_loaders' parameter to WebStatus for customizers who want to use additional Jina2 loaders --- master/buildbot/status/web/base.py | 12 ++++---- master/buildbot/status/web/baseweb.py | 10 +++++-- master/docs/manual/cfg-statustargets.rst | 36 ++++++++++++++++-------- 3 files changed, 39 insertions(+), 19 deletions(-) diff --git a/master/buildbot/status/web/base.py b/master/buildbot/status/web/base.py index 522cf0a671d..c4447ef32bd 100644 --- a/master/buildbot/status/web/base.py +++ b/master/buildbot/status/web/base.py @@ -481,7 +481,7 @@ def map_branches(branches): # jinja utilities def createJinjaEnv(revlink=None, changecommentlink=None, - repositories=None, projects=None): + repositories=None, projects=None, jinja_loaders=None): ''' Create a jinja environment changecommentlink is used to render HTML in the WebStatus and for mail changes @@ -503,10 +503,12 @@ def createJinjaEnv(revlink=None, changecommentlink=None, # See http://buildbot.net/trac/ticket/658 assert not hasattr(sys, "frozen"), 'Frozen config not supported with jinja (yet)' - default_loader = jinja2.PackageLoader('buildbot.status.web', 'templates') - root = os.path.join(os.getcwd(), 'templates') - loader = jinja2.ChoiceLoader([jinja2.FileSystemLoader(root), - default_loader]) + all_loaders = [jinja2.FileSystemLoader(os.path.join(os.getcwd(), 'templates'))] + if jinja_loaders: + all_loaders.extend(jinja_loaders) + all_loaders.append(jinja2.PackageLoader('buildbot.status.web', 'templates')) + loader = jinja2.ChoiceLoader(all_loaders) + env = jinja2.Environment(loader=loader, extensions=['jinja2.ext.i18n'], trim_blocks=True, diff --git a/master/buildbot/status/web/baseweb.py b/master/buildbot/status/web/baseweb.py index 6f1989a1436..ec5eaad77c4 100644 --- a/master/buildbot/status/web/baseweb.py +++ b/master/buildbot/status/web/baseweb.py @@ -148,7 +148,7 @@ def __init__(self, http_port=None, distrib_port=None, allowForce=None, order_console_by_time=False, changecommentlink=None, revlink=None, projects=None, repositories=None, authz=None, logRotateLength=None, maxRotatedFiles=None, - change_hook_dialects = {}, provide_feeds=None): + change_hook_dialects = {}, provide_feeds=None, jinja_loaders=None): """Run a web server that provides Buildbot status. @type http_port: int or L{twisted.application.strports} string @@ -266,6 +266,10 @@ def __init__(self, http_port=None, distrib_port=None, allowForce=None, Otherwise, a dictionary of strings of the type of feeds provided. Current possibilities are "atom", "json", and "rss" + + @type jinja_loaders: None or list + @param jinja_loaders: If not empty, a list of additional Jinja2 loader + objects to search for templates. """ service.MultiService.__init__(self) @@ -344,6 +348,8 @@ def __init__(self, http_port=None, distrib_port=None, allowForce=None, else: self.provide_feeds = provide_feeds + self.jinja_loaders = jinja_loaders + def setupUsualPages(self, numbuilds, num_events, num_events_max): #self.putChild("", IndexOrWaterfallRedirection()) self.putChild("waterfall", WaterfallStatusResource(num_events=num_events, @@ -402,7 +408,7 @@ def either(a,b): # a if a else b for py2.4 else: revlink = self.master.config.revlink self.templates = createJinjaEnv(revlink, self.changecommentlink, - self.repositories, self.projects) + self.repositories, self.projects, self.jinja_loaders) if not self.site: diff --git a/master/docs/manual/cfg-statustargets.rst b/master/docs/manual/cfg-statustargets.rst index b13ade92d83..00146a70f09 100644 --- a/master/docs/manual/cfg-statustargets.rst +++ b/master/docs/manual/cfg-statustargets.rst @@ -57,11 +57,8 @@ server and retrieve information about every build the buildbot knows about, as well as find out what the buildbot is currently working on. The first page you will see is the *Welcome Page*, which contains -links to all the other useful pages. By default, this page is served from -the :file:`status/web/templates/root.html` file in buildbot's library area. -If you'd like to override this page or the other templates found there, -copy the files you're interested in into a :file:`templates/` directory in -the buildmaster's base directory. +links to all the other useful pages. By default, this page is served from the +:file:`status/web/templates/root.html` file in buildbot's library area. One of the most complex resource provided by :class:`WebStatus` is the *Waterfall Display*, which shows a time-based chart of events. This @@ -86,25 +83,39 @@ in great detail below. Configuration +++++++++++++ -Buildbot now uses a templating system for the web interface. The source +The simplest possible configuration for WebStatus is:: + + from buildbot.status.html import WebStatus + c['status'].append(WebStatus(8080)) + +Buildbot uses a templating system for the web interface. The source of these templates can be found in the :file:`status/web/templates/` directory in buildbot's library area. You can override these templates by creating alternate versions in a :file:`templates/` directory within the buildmaster's base directory. +If that isn't enough you can also provide additional Jinja2 template loaders:: + + import jinja2 + myloaders = [ + jinja2.FileSystemLoader("/tmp/mypath"), + ] + + c['status'].append(html.WebStatus( + …, + jinja_loaders = myloaders, + )) + The first time a buildmaster is created, the :file:`public_html/` directory is populated with some sample files, which you will probably want to customize for your own project. These files are all static: the buildbot does not modify them in any way as it serves them to HTTP clients. -Note that templates in :file:`templates/` take precedence over static files in -:file:`public_html/`. :: +Templates in :file:`templates/` take precedence over static files in +:file:`public_html/`. - from buildbot.status.html import WebStatus - c['status'].append(WebStatus(8080)) - -Note that the initial :file:`robots.txt` file has Disallow lines for all of +The initial :file:`robots.txt` file has Disallow lines for all of the dynamically-generated buildbot pages, to discourage web spiders and search engines from consuming a lot of CPU time as they crawl through the entire history of your buildbot. If you are running the @@ -709,6 +720,7 @@ that periodically poll the Google Code commit feed for changes. change_hook_dialects={'googlecode': {'secret_key': 'FSP3p-Ghdn4T0oqX', 'branch': 'master'}} + .. bb:status:: MailNotifier .. index:: single: email; MailNotifier