From c203010282e3651673f98540b88c7302c7b71082 Mon Sep 17 00:00:00 2001 From: Mikhail Sobolev Date: Thu, 6 Nov 2014 19:24:52 +0200 Subject: [PATCH] Revert "Merge pull request #1323 from delanne/IRCContact" This reverts commit 918e6c0a62ab524f5177f0cb59c127698fc2c04c, reversing changes made to 38d595befd159d3655bd89c77c9947dc64f12dff. --- master/buildbot/config.py | 6 +- master/buildbot/process/builder.py | 10 + master/buildbot/schedulers/base.py | 8 +- master/buildbot/schedulers/basic.py | 105 ++++-- master/buildbot/schedulers/timed.py | 300 ++++++++++-------- master/buildbot/test/fake/fakemaster.py | 6 - master/buildbot/test/integration/test_www.py | 3 +- master/buildbot/test/unit/test_config.py | 28 +- .../test/unit/test_schedulers_base.py | 7 +- .../test/unit/test_schedulers_basic.py | 94 +++++- .../test/unit/test_schedulers_dependent.py | 2 +- .../unit/test_schedulers_timed_Nightly.py | 155 ++++----- ...est_schedulers_timed_NightlyTriggerable.py | 28 +- .../unit/test_schedulers_timed_Periodic.py | 8 +- .../test/unit/test_schedulers_trysched.py | 10 +- .../buildbot/test/unit/test_util_codebase.py | 102 ------ master/buildbot/test/unit/test_www_service.py | 2 +- master/buildbot/test/util/scheduler.py | 4 +- master/buildbot/util/codebase.py | 46 --- master/buildbot/www/service.py | 36 +-- master/contrib/Dockerfile | 4 +- master/docs/bbdocs/ext.py | 3 +- master/docs/developer/mq.rst | 2 +- master/docs/manual/cfg-schedulers.rst | 9 +- master/docs/manual/cfg-www.rst | 13 - master/docs/manual/cmdline.rst | 40 +-- master/docs/relnotes/index.rst | 8 - master/docs/tutorial/docker.rst | 17 +- www/base/guanlecoja/config.coffee | 4 +- www/base/src/app/layout.jade | 1 - www/codeparameter/guanlecoja/config.coffee | 2 +- www/console_view/guanlecoja/config.coffee | 2 +- www/waterfall_view/guanlecoja/config.coffee | 2 +- 33 files changed, 480 insertions(+), 587 deletions(-) delete mode 100644 master/buildbot/test/unit/test_util_codebase.py delete mode 100644 master/buildbot/util/codebase.py diff --git a/master/buildbot/config.py b/master/buildbot/config.py index 6735c8305e1..1791def67db 100644 --- a/master/buildbot/config.py +++ b/master/buildbot/config.py @@ -120,8 +120,7 @@ def __init__(self): url='http://localhost:8080/', plugins=dict(), auth=auth.NoAuth(), - avatar_methods=avatar.AvatarGravatar(), - logfileName='http.log', + avatar_methods=avatar.AvatarGravatar() ) _known_config_keys = set([ @@ -570,8 +569,7 @@ def load_www(self, filename, config_dict): www_cfg = config_dict['www'] allowed = set(['port', 'url', 'debug', 'json_cache_seconds', 'rest_minimum_version', 'allowed_origins', 'jsonp', - 'plugins', 'auth', 'avatar_methods', 'logfileName', - 'logRotateLength', 'maxRotatedFiles']) + 'plugins', 'auth', 'avatar_methods']) unknown = set(www_cfg.iterkeys()) - allowed if unknown: error("unknown www configuration parameter(s) %s" % diff --git a/master/buildbot/process/builder.py b/master/buildbot/process/builder.py index 9626cc4d0d8..a74948ad2ae 100644 --- a/master/buildbot/process/builder.py +++ b/master/buildbot/process/builder.py @@ -464,6 +464,9 @@ def buildFinished(self, build, sb): brids = [br.id for br in build.requests] d = self.master.data.updates.completeBuildRequests(brids, results, complete_at=complete_at) + d.addCallback(lambda _: + self._notify_completions(build.requests, results, + complete_at_epoch)) # nothing in particular to do with this deferred, so just log it if # it fails.. d.addErrback(log.err, 'while marking build requests as completed') @@ -473,6 +476,13 @@ def buildFinished(self, build, sb): self.updateBigStatus() + @defer.inlineCallbacks + def _notify_completions(self, requests, results, complete_at_epoch): + # send a message for each request + for br in requests: + yield self.master.data.updates.completeBuildRequests([br.id], results, + epoch2datetime(complete_at_epoch)) + def _resubmit_buildreqs(self, build): brids = [br.id for br in build.requests] d = self.master.data.updates.unclaimBuildRequests(brids) diff --git a/master/buildbot/schedulers/base.py b/master/buildbot/schedulers/base.py index 31202da4b58..9985bd4ead5 100644 --- a/master/buildbot/schedulers/base.py +++ b/master/buildbot/schedulers/base.py @@ -213,10 +213,7 @@ def addBuildsetForSourceStampsWithDefaults(self, reason, sourcestamps, def getCodebaseDict(self, codebase): # Hook for subclasses to change codebase parameters when a codebase does # not have a change associated with it. - try: - return defer.succeed(self.codebases[codebase]) - except KeyError: - return defer.fail() + return self.codebases[codebase] @defer.inlineCallbacks def addBuildsetForChanges(self, waited_for=False, reason='', @@ -238,8 +235,7 @@ def get_last_change_for_codebase(codebase): if codebase not in changesByCodebase: # codebase has no changes # create a sourcestamp that has no changes - cb = yield self.getCodebaseDict(codebase) - + cb = self.getCodebaseDict(codebase) ss = { 'codebase': codebase, 'repository': cb['repository'], diff --git a/master/buildbot/schedulers/basic.py b/master/buildbot/schedulers/basic.py index a9957ed8c12..1243ceadd62 100644 --- a/master/buildbot/schedulers/basic.py +++ b/master/buildbot/schedulers/basic.py @@ -16,11 +16,10 @@ from buildbot import config from buildbot import util from buildbot.changes import changes -from buildbot.changes.filter import ChangeFilter +from buildbot.changes import filter from buildbot.schedulers import base from buildbot.schedulers import dependent from buildbot.util import NotABranch -from buildbot.util.codebase import AbsoluteSourceStampsMixin from collections import defaultdict from twisted.internet import defer from twisted.internet import reactor @@ -82,24 +81,34 @@ def __init__(self, name, shouldntBeSet=NotSet, treeStableTimer=None, def getChangeFilter(self, branch, branches, change_filter, categories): raise NotImplementedError - @defer.inlineCallbacks - def activate(self): - yield base.BaseScheduler.activate(self) - yield self.startConsumingChanges(fileIsImportant=self.fileIsImportant, - change_filter=self.change_filter, - onlyImportant=self.onlyImportant) - - # if we have a treeStableTimer, if there are classified changes - # out there, start their timers again - if self.treeStableTimer: - yield self.scanExistingClassifiedChanges() + def preStartConsumingChanges(self): + # Hook for subclasses to setup before startConsumingChanges(). + return defer.succeed(None) - # otherwise, we don't care about classified + def activate(self): + d = base.BaseScheduler.activate(self) + d.addCallback(lambda _: + self.preStartConsumingChanges()) + d.addCallback(lambda _: + self.startConsumingChanges(fileIsImportant=self.fileIsImportant, + change_filter=self.change_filter, + onlyImportant=self.onlyImportant)) + + # if treeStableTimer is False, then we don't care about classified # changes, so get rid of any hanging around from previous # configurations + if not self.treeStableTimer: + d.addCallback(lambda _: + self.master.db.schedulers.flushChangeClassifications( + self.objectid)) + + # otherwise, if there are classified changes out there, start their + # treeStableTimers again else: - yield self.master.db.schedulers.flushChangeClassifications( - self.objectid) + d.addCallback(lambda _: + self.scanExistingClassifiedChanges()) + + return d @defer.inlineCallbacks def deactivate(self): @@ -127,11 +136,16 @@ def gotChange(self, change, important): timer_name = self.getTimerNameForChange(change) - # if we have a treeStableTimer + # if we have a treeStableTimer, then record the change's importance + # and: # - for an important change, start the timer # - for an unimportant change, reset the timer if it is running + d = self.master.db.schedulers.classifyChanges( + self.objectid, {change.number: important}) - if important or self._stable_timers[timer_name]: + def fix_timer(_): + if not important and not self._stable_timers[timer_name]: + return if self._stable_timers[timer_name]: self._stable_timers[timer_name].cancel() @@ -140,10 +154,8 @@ def fire_timer(): d.addErrback(log.err, "while firing stable timer") self._stable_timers[timer_name] = self._reactor.callLater( self.treeStableTimer, fire_timer) - - # record the change's importance - return self.master.db.schedulers.classifyChanges( - self.objectid, {change.number: important}) + d.addCallback(fix_timer) + return d @defer.inlineCallbacks def scanExistingClassifiedChanges(self): @@ -178,11 +190,13 @@ def getChangeClassificationsForTimer(self, objectid, timer_name): @util.deferredLocked('_stable_timers_lock') @defer.inlineCallbacks def stableTimerFired(self, timer_name): - # delete this now-fired timer, if the service has already been stopped - # then just bail out - if not self._stable_timers.pop(timer_name, None): + # if the service has already been stopped then just bail out + if not self._stable_timers[timer_name]: return + # delete this now-fired timer + del self._stable_timers[timer_name] + classifications = \ yield self.getChangeClassificationsForTimer(self.objectid, timer_name) @@ -205,22 +219,49 @@ def getPendingBuildTimes(self): return [timer.getTime() for timer in self._stable_timers.values() if timer and timer.active()] -class SingleBranchScheduler(BaseBasicScheduler, AbsoluteSourceStampsMixin): +class SingleBranchScheduler(BaseBasicScheduler): def __init__(self, name, createAbsoluteSourceStamps=False, **kwargs): + self._lastCodebases = {} self.createAbsoluteSourceStamps = createAbsoluteSourceStamps BaseBasicScheduler.__init__(self, name, **kwargs) - @defer.inlineCallbacks + def preStartConsumingChanges(self): + if self.createAbsoluteSourceStamps: + # load saved codebases + d = self.getState("lastCodebases", {}) + + def setLast(lastCodebases): + self._lastCodebases = lastCodebases + d.addCallback(setLast) + return d + else: + return defer.succeed(None) + def gotChange(self, change, important): + d = defer.succeed(None) + if self.createAbsoluteSourceStamps: - yield self.recordChange(change) + self._lastCodebases.setdefault(change.codebase, {}) + lastChange = self._lastCodebases[change.codebase].get('lastChange', -1) + + codebaseDict = dict(repository=change.repository, + branch=change.branch, + revision=change.revision, + lastChange=change.number) + + if change.number > lastChange: + self._lastCodebases[change.codebase] = codebaseDict + d.addCallback(lambda _: + self.setState('lastCodebases', self._lastCodebases)) - yield BaseBasicScheduler.gotChange(self, change, important) + d.addCallback(lambda _: + BaseBasicScheduler.gotChange(self, change, important)) + return d def getCodebaseDict(self, codebase): if self.createAbsoluteSourceStamps: - return AbsoluteSourceStampsMixin.getCodebaseDict(self, codebase) + return self._lastCodebases.get(codebase, self.codebases[codebase]) else: return self.codebases[codebase] @@ -234,7 +275,7 @@ def getChangeFilter(self, branch, branches, change_filter, categories): "the 'branches' argument is not allowed for " + "SingleBranchScheduler") - return ChangeFilter.fromSchedulerConstructorArgs( + return filter.ChangeFilter.fromSchedulerConstructorArgs( change_filter=change_filter, branch=branch, categories=categories) @@ -262,7 +303,7 @@ class AnyBranchScheduler(BaseBasicScheduler): def getChangeFilter(self, branch, branches, change_filter, categories): assert branch is NotABranch - return ChangeFilter.fromSchedulerConstructorArgs( + return filter.ChangeFilter.fromSchedulerConstructorArgs( change_filter=change_filter, branch=branches, categories=categories) diff --git a/master/buildbot/schedulers/timed.py b/master/buildbot/schedulers/timed.py index 33654e71f94..f263067c432 100644 --- a/master/buildbot/schedulers/timed.py +++ b/master/buildbot/schedulers/timed.py @@ -17,19 +17,18 @@ from buildbot import config from buildbot import util -from buildbot.changes.filter import ChangeFilter +from buildbot.changes import filter from buildbot.interfaces import ITriggerableScheduler from buildbot.process import buildstep from buildbot.process import properties from buildbot.schedulers import base from buildbot.util import croniter -from buildbot.util.codebase import AbsoluteSourceStampsMixin from twisted.internet import defer from twisted.internet import reactor from twisted.python import log -class Timed(base.BaseScheduler, AbsoluteSourceStampsMixin): +class Timed(base.BaseScheduler): """ Parent class for timed schedulers. This takes care of the (surprisingly @@ -37,17 +36,10 @@ class Timed(base.BaseScheduler, AbsoluteSourceStampsMixin): before the service stops. """ - compare_attrs = ('reason', 'createAbsoluteSourceStamps', 'onlyIfChanged', - 'branch', 'fileIsImportant', 'change_filter', 'onlyImportant') + compare_attrs = ('reason',) reason = '' - class NoBranch: - pass - - def __init__(self, name, builderNames, properties={}, reason='', - createAbsoluteSourceStamps=False, onlyIfChanged=False, - branch=NoBranch, change_filter=None, fileIsImportant=None, - onlyImportant=False, **kwargs): + def __init__(self, name, builderNames, properties={}, reason='', **kwargs): base.BaseScheduler.__init__(self, name, builderNames, properties, **kwargs) @@ -62,38 +54,33 @@ def __init__(self, name, builderNames, properties={}, reason='', self.actuateAtTimer = None self.reason = util.ascii2unicode(reason % {'name': name}) - self.branch = branch - self.change_filter = ChangeFilter.fromSchedulerConstructorArgs(change_filter=change_filter) - self.createAbsoluteSourceStamps = createAbsoluteSourceStamps - self.onlyIfChanged = onlyIfChanged - if fileIsImportant and not callable(fileIsImportant): - config.error( - "fileIsImportant must be a callable") - self.fileIsImportant = fileIsImportant - # If True, only important changes will be added to the buildset. - self.onlyImportant = onlyImportant + self._reactor = reactor # patched by tests - @defer.inlineCallbacks def activate(self): - yield base.BaseScheduler.activate(self) + d = base.BaseScheduler.activate(self) - # no need to lock this - # nothing else can run before the service is started + # no need to lock this; nothing else can run before the service is started self.actuateOk = True # get the scheduler's last_build time (note: only done at startup) - self.lastActuated = yield self.getState('last_build', None) + d.addCallback(lambda _: + self.getState('last_build', None)) + + def set_last(lastActuated): + self.lastActuated = lastActuated + d.addCallback(set_last) # schedule the next build - yield self.scheduleNextBuild() + d.addCallback(lambda _: self.scheduleNextBuild()) - if self.onlyIfChanged or self.createAbsoluteSourceStamps: - yield self.startConsumingChanges(fileIsImportant=self.fileIsImportant, - change_filter=self.change_filter, - onlyImportant=self.onlyImportant) - else: - yield self.master.db.schedulers.flushChangeClassifications(self.objectid) + # give subclasses a chance to start up + d.addCallback(lambda _: self.startTimedSchedulerService()) + return d + + def startTimedSchedulerService(self): + """Hook for subclasses to participate in the L{activate} process; + can return a Deferred""" @defer.inlineCallbacks def deactivate(self): @@ -117,59 +104,12 @@ def getPendingBuildTimes(self): # estimate return [self.actuateAt] - def gotChange(self, change, important): - # both important and unimportant changes on our branch are recorded, as - # we will include all such changes in any buildsets we start. Note - # that we must check the branch here because it is not included in the - # change filter. - if self.branch is not Timed.NoBranch and change.branch != self.branch: - return defer.succeed(None) # don't care about this change - - d = self.master.db.schedulers.classifyChanges( - self.objectid, {change.number: important}) - - if self.createAbsoluteSourceStamps: - d.addCallback(lambda _: self.recordChange(change)) - - return d + # Timed methods - @defer.inlineCallbacks def startBuild(self): - # use the collected changes to start a build - scheds = self.master.db.schedulers - classifications = yield scheds.getChangeClassifications(self.objectid) - - # if onlyIfChanged is True, then we will skip this build if no - # important changes have occurred since the last invocation - if self.onlyIfChanged and not any(classifications.itervalues()): - log.msg(("%s scheduler <%s>: skipping build " + - "- No important changes on configured branch") % - (self.__class__.__name__, self.name)) - return - - changeids = sorted(classifications.keys()) - - if changeids: - max_changeid = changeids[-1] # (changeids are sorted) - yield self.addBuildsetForChanges(reason=self.reason, - changeids=changeids) - yield scheds.flushChangeClassifications(self.objectid, - less_than=max_changeid + 1) - else: - # There are no changes, but onlyIfChanged is False, so start - # a build of the latest revision, whatever that is - sourcestamps = [dict(codebase=cb) for cb in self.codebases.keys()] - yield self.addBuildsetForSourceStampsWithDefaults( - reason=self.reason, - sourcestamps=sourcestamps) - - def getCodebaseDict(self, codebase): - if self.createAbsoluteSourceStamps: - return AbsoluteSourceStampsMixin.getCodebaseDict(self, codebase) - else: - return self.codebases[codebase] - - # Timed methods + """The time has come to start a new build. Returns a Deferred. + Override in subclasses.""" + raise NotImplementedError def getNextBuildTime(self, lastActuation): """ @@ -216,9 +156,8 @@ def set_timer(actuateAt): if actuateAt is not None: untilNext = self.actuateAt - now if untilNext == 0: - log.msg(("%s scheduler <%s>: missed scheduled build time" - " - building immediately") % - (self.__class__.__name__, self.name)) + log.msg(("%s: missed scheduled build time, so building " + "immediately") % self.name) self.actuateAtTimer = self._reactor.callLater(untilNext, self._actuate) d.addCallback(set_timer) @@ -253,17 +192,19 @@ def set_state_and_start(): class Periodic(Timed): - compare_attrs = ('periodicBuildTimer',) + compare_attrs = ('periodicBuildTimer', 'branch',) def __init__(self, name, builderNames, periodicBuildTimer, reason="The Periodic scheduler named '%(name)s' triggered this build", - **kwargs): + branch=None, properties={}, onlyImportant=False, + codebases=base.BaseScheduler.DEFAULT_CODEBASES): Timed.__init__(self, name=name, builderNames=builderNames, - reason=reason, **kwargs) + properties=properties, reason=reason, codebases=codebases) if periodicBuildTimer <= 0: config.error( "periodicBuildTimer must be positive") self.periodicBuildTimer = periodicBuildTimer + self.branch = branch def getNextBuildTime(self, lastActuated): if lastActuated is None: @@ -271,15 +212,19 @@ def getNextBuildTime(self, lastActuated): else: return defer.succeed(lastActuated + self.periodicBuildTimer) + def startBuild(self): + return self.addBuildsetForLatest(reason=self.reason, branch=self.branch) + class NightlyBase(Timed): compare_attrs = ('minute', 'hour', 'dayOfMonth', 'month', 'dayOfWeek') def __init__(self, name, builderNames, minute=0, hour='*', dayOfMonth='*', month='*', dayOfWeek='*', - **kwargs): + reason='NightlyBase(%(name)s)', + properties={}, codebases=base.BaseScheduler.DEFAULT_CODEBASES): Timed.__init__(self, name=name, builderNames=builderNames, - **kwargs) + reason=reason, properties=properties, codebases=codebases) self.minute = minute self.hour = hour @@ -314,14 +259,132 @@ def getNextBuildTime(self, lastActuated): class Nightly(NightlyBase): + compare_attrs = ('branch', 'onlyIfChanged', 'fileIsImportant', + 'change_filter', 'onlyImportant', 'createAbsoluteSourceStamps',) + + class NoBranch: + pass def __init__(self, name, builderNames, minute=0, hour='*', dayOfMonth='*', month='*', dayOfWeek='*', + branch=NoBranch, fileIsImportant=None, onlyIfChanged=False, + createAbsoluteSourceStamps=False, + properties={}, change_filter=None, onlyImportant=False, reason="The Nightly scheduler named '%(name)s' triggered this build", - **kwargs): + codebases=base.BaseScheduler.DEFAULT_CODEBASES): NightlyBase.__init__(self, name=name, builderNames=builderNames, minute=minute, hour=hour, dayOfWeek=dayOfWeek, dayOfMonth=dayOfMonth, - reason=reason, **kwargs) + properties=properties, codebases=codebases, reason=reason) + + # If True, only important changes will be added to the buildset. + self.onlyImportant = onlyImportant + + if fileIsImportant and not callable(fileIsImportant): + config.error( + "fileIsImportant must be a callable") + + if branch is Nightly.NoBranch: + config.error( + "Nightly parameter 'branch' is required") + + if createAbsoluteSourceStamps and not onlyIfChanged: + config.error( + "createAbsoluteSourceStamps can only be used with onlyIfChanged") + + self._lastCodebases = {} + self.branch = branch + self.onlyIfChanged = onlyIfChanged + self.createAbsoluteSourceStamps = createAbsoluteSourceStamps + self.fileIsImportant = fileIsImportant + self.change_filter = filter.ChangeFilter.fromSchedulerConstructorArgs( + change_filter=change_filter) + + def startTimedSchedulerService(self): + if self.onlyIfChanged: + d = self.preStartConsumingChanges() + + d.addCallback(lambda _: + self.startConsumingChanges(fileIsImportant=self.fileIsImportant, + change_filter=self.change_filter, + onlyImportant=self.onlyImportant)) + return d + else: + return self.master.db.schedulers.flushChangeClassifications(self.objectid) + + def preStartConsumingChanges(self): + if self.createAbsoluteSourceStamps: + # load saved codebases + d = self.getState("lastCodebases", {}) + + def setLast(lastCodebases): + self._lastCodebases = lastCodebases + d.addCallback(setLast) + return d + else: + return defer.succeed(None) + + def gotChange(self, change, important): + # both important and unimportant changes on our branch are recorded, as + # we will include all such changes in any buildsets we start. Note + # that we must check the branch here because it is not included in the + # change filter. + if change.branch != self.branch: + return defer.succeed(None) # don't care about this change + + d = self.master.db.schedulers.classifyChanges( + self.objectid, {change.number: important}) + + if self.createAbsoluteSourceStamps: + self._lastCodebases.setdefault(change.codebase, {}) + lastChange = self._lastCodebases[change.codebase].get('lastChange', -1) + + codebaseDict = dict(repository=change.repository, + branch=change.branch, + revision=change.revision, + lastChange=change.number) + + if change.number > lastChange: + self._lastCodebases[change.codebase] = codebaseDict + d.addCallback(lambda _: + self.setState('lastCodebases', self._lastCodebases)) + + return d + + def getCodebaseDict(self, codebase): + if self.createAbsoluteSourceStamps: + return self._lastCodebases.get(codebase, self.codebases[codebase]) + else: + return self.codebases[codebase] + + @defer.inlineCallbacks + def startBuild(self): + scheds = self.master.db.schedulers + # if onlyIfChanged is True, then we will skip this build if no + # important changes have occurred since the last invocation + if self.onlyIfChanged: + classifications = \ + yield scheds.getChangeClassifications(self.objectid) + + # see if we have any important changes + for imp in classifications.itervalues(): + if imp: + break + else: + log.msg(("Nightly Scheduler <%s>: skipping build " + + "- No important changes on configured branch") % self.name) + return + + changeids = sorted(classifications.keys()) + yield self.addBuildsetForChanges(reason=self.reason, + changeids=changeids) + + max_changeid = changeids[-1] # (changeids are sorted) + yield scheds.flushChangeClassifications(self.objectid, + less_than=max_changeid + 1) + else: + # start a build of the latest revision, whatever that is + yield self.addBuildsetForLatest(reason=self.reason, + branch=self.branch) class NightlyTriggerable(NightlyBase): @@ -330,10 +393,10 @@ class NightlyTriggerable(NightlyBase): def __init__(self, name, builderNames, minute=0, hour='*', dayOfMonth='*', month='*', dayOfWeek='*', reason="The NightlyTriggerable scheduler named '%(name)s' triggered this build", - **kwargs): + properties={}, codebases=base.BaseScheduler.DEFAULT_CODEBASES): NightlyBase.__init__(self, name=name, builderNames=builderNames, minute=minute, hour=hour, - dayOfWeek=dayOfWeek, dayOfMonth=dayOfMonth, reason=reason, - **kwargs) + dayOfWeek=dayOfWeek, dayOfMonth=dayOfMonth, properties=properties, reason=reason, + codebases=codebases) self._lastTrigger = None @@ -346,15 +409,11 @@ def activate(self): try: if isinstance(lastTrigger[0], list): self._lastTrigger = (lastTrigger[0], - properties.Properties.fromDict(lastTrigger[1]), - lastTrigger[2], - lastTrigger[3]) + properties.Properties.fromDict(lastTrigger[1])) # handle state from before Buildbot-0.9.0 elif isinstance(lastTrigger[0], dict): self._lastTrigger = (lastTrigger[0].values(), - properties.Properties.fromDict(lastTrigger[1]), - None, - None) + properties.Properties.fromDict(lastTrigger[1])) except Exception: pass # If the lastTrigger isn't of the right format, ignore it @@ -364,41 +423,34 @@ def activate(self): "could not load previous state; starting fresh", scheduler=self.name) - def trigger(self, waited_for, sourcestamps=None, set_props=None, - parent_buildid=None, parent_relationship=None): + def trigger(self, sourcestamps, set_props=None): """Trigger this scheduler with the given sourcestamp ID. Returns a deferred that will fire when the buildset is finished.""" assert isinstance(sourcestamps, list), \ "trigger requires a list of sourcestamps" - self._lastTrigger = (sourcestamps, - set_props, - parent_buildid, - parent_relationship) + self._lastTrigger = (sourcestamps, set_props) + # record the trigger in the db if set_props: propsDict = set_props.asDict() else: propsDict = {} - - # record the trigger in the db - d = self.setState('lastTrigger', (sourcestamps, - propsDict, - parent_buildid, - parent_relationship)) + d = self.setState('lastTrigger', + (sourcestamps, propsDict)) # Trigger expects a callback with the success of the triggered # build, if waitForFinish is True. # Just return SUCCESS, to indicate that the trigger was succesful, - # don't wait for the nightly to run. - return (defer.succeed((None, {})), d.addCallback(lambda _: buildstep.SUCCESS)) + # don't want for the nightly to run. + return d.addCallback(lambda _: buildstep.SUCCESS) @defer.inlineCallbacks def startBuild(self): if self._lastTrigger is None: - return + defer.returnValue(None) - (sourcestamps, set_props, parent_buildid, parent_relationship) = self._lastTrigger + (sourcestamps, set_props) = self._lastTrigger self._lastTrigger = None yield self.setState('lastTrigger', None) @@ -409,9 +461,5 @@ def startBuild(self): if set_props: props.updateFromProperties(set_props) - yield self.addBuildsetForSourceStampsWithDefaults( - reason=self.reason, - sourcestamps=sourcestamps, - properties=props, - parent_buildid=parent_buildid, - parent_relationship=parent_relationship) + yield self.addBuildsetForSourceStampsWithDefaults(reason=self.reason, + sourcestamps=sourcestamps, properties=props) diff --git a/master/buildbot/test/fake/fakemaster.py b/master/buildbot/test/fake/fakemaster.py index 78c3cde481b..35e220fb7bd 100644 --- a/master/buildbot/test/fake/fakemaster.py +++ b/master/buildbot/test/fake/fakemaster.py @@ -137,11 +137,6 @@ def addPointEvent(self, text): pass -class FakeLogRotation(object): - rotateLength = 42 - maxRotatedFiles = 42 - - class FakeMaster(object): """ @@ -164,7 +159,6 @@ def __init__(self, master_id=fakedb.FakeBuildRequestsComponent.MASTER_ID): self.name = 'fake:/master' self.masterid = master_id self.buildslaves = bslavemanager.FakeBuildslaveManager(self) - self.log_rotation = FakeLogRotation() def getObjectId(self): return defer.succeed(self._master_id) diff --git a/master/buildbot/test/integration/test_www.py b/master/buildbot/test/integration/test_www.py index 0d89aecb21c..19bdbf6e397 100644 --- a/master/buildbot/test/integration/test_www.py +++ b/master/buildbot/test/integration/test_www.py @@ -77,8 +77,7 @@ def setUp(self): debug=True, auth=auth.NoAuth(), url="not yet known", - avatar_methods=[], - logfileName='http.log') + avatar_methods=[]) master.www = wwwservice.WWWService(master) yield master.www.startService() yield master.www.reconfigService(master.config) diff --git a/master/buildbot/test/unit/test_config.py b/master/buildbot/test/unit/test_config.py index 7f20e0d058d..2f38492cf2f 100644 --- a/master/buildbot/test/unit/test_config.py +++ b/master/buildbot/test/unit/test_config.py @@ -60,8 +60,7 @@ manhole=None, www=dict(port=None, url='http://localhost:8080/', plugins={}, auth={'name': 'NoAuth'}, - avatar_methods={'name': 'gravatar'}, - logfileName='http.log'), + avatar_methods={'name': 'gravatar'}), ) @@ -802,16 +801,14 @@ def test_load_www_default(self): self.cfg.load_www(self.filename, {}) self.assertResults(www=dict(port=None, url='http://localhost:8080/', plugins={}, auth={'name': 'NoAuth'}, - avatar_methods={'name': 'gravatar'}, - logfileName='http.log')) + avatar_methods={'name': 'gravatar'})) def test_load_www_port(self): self.cfg.load_www(self.filename, dict(www=dict(port=9888))) self.assertResults(www=dict(port=9888, url='http://localhost:9888/', plugins={}, auth={'name': 'NoAuth'}, - avatar_methods={'name': 'gravatar'}, - logfileName='http.log')) + avatar_methods={'name': 'gravatar'})) def test_load_www_plugin(self): self.cfg.load_www(self.filename, @@ -819,16 +816,14 @@ def test_load_www_plugin(self): self.assertResults(www=dict(port=None, url='http://localhost:8080/', plugins={'waterfall': {'foo': 'bar'}}, auth={'name': 'NoAuth'}, - avatar_methods={'name': 'gravatar'}, - logfileName='http.log')) + avatar_methods={'name': 'gravatar'})) def test_load_www_url_no_slash(self): self.cfg.load_www(self.filename, dict(www=dict(url='http://foo', port=20))) self.assertResults(www=dict(port=20, url='http://foo/', plugins={}, auth={'name': 'NoAuth'}, - avatar_methods={'name': 'gravatar'}, - logfileName='http.log')) + avatar_methods={'name': 'gravatar'})) def test_load_www_allowed_origins(self): self.cfg.load_www(self.filename, @@ -837,17 +832,8 @@ def test_load_www_allowed_origins(self): self.assertResults(www=dict(port=None, url='http://foo/', allowed_origins=['a', 'b'], plugins={}, auth={'name': 'NoAuth'}, - avatar_methods={'name': 'gravatar'}, - logfileName='http.log')) - - def test_load_www_logfileName(self): - self.cfg.load_www(self.filename, - dict(www=dict(url='http://foo', - logfileName='http-access.log'))) - self.assertResults(www=dict(port=None, url='http://foo/', - plugins={}, auth={'name': 'NoAuth'}, - avatar_methods={'name': 'gravatar'}, - logfileName='http-access.log')) + avatar_methods={'name': 'gravatar'} + )) def test_load_www_unknown(self): self.cfg.load_www(self.filename, diff --git a/master/buildbot/test/unit/test_schedulers_base.py b/master/buildbot/test/unit/test_schedulers_base.py index feb2dc1c1ab..01467624d32 100644 --- a/master/buildbot/test/unit/test_schedulers_base.py +++ b/master/buildbot/test/unit/test_schedulers_base.py @@ -79,15 +79,14 @@ def test_constructor_codebases_invalid(self): self.assertRaises(config.ConfigErrors, lambda: self.makeScheduler(codebases=codebases)) - @defer.inlineCallbacks def test_getCodebaseDict(self): sched = self.makeScheduler(codebases={'lib': {'repository': 'librepo'}}) - cbd = yield sched.getCodebaseDict('lib') - self.assertEqual(cbd, {'repository': 'librepo'}) + self.assertEqual(sched.getCodebaseDict('lib'), + {'repository': 'librepo'}) def test_getCodebaseDict_not_found(self): sched = self.makeScheduler(codebases={'lib': {'repository': 'librepo'}}) - return self.assertFailure(sched.getCodebaseDict('app'), KeyError) + self.assertRaises(KeyError, lambda: sched.getCodebaseDict('app')) def test_listBuilderNames(self): sched = self.makeScheduler(builderNames=['x', 'y']) diff --git a/master/buildbot/test/unit/test_schedulers_basic.py b/master/buildbot/test/unit/test_schedulers_basic.py index 849247695c8..35e2328d727 100644 --- a/master/buildbot/test/unit/test_schedulers_basic.py +++ b/master/buildbot/test/unit/test_schedulers_basic.py @@ -159,6 +159,17 @@ def check(_): d.addCallback(lambda _: sched.deactivate()) return d + def test_activate_calls_preStartConsumingChanges(self): + sched = self.makeScheduler(self.Subclass) + sched.preStartConsumingChanges = mock.Mock( + return_value=defer.succeed(None)) + d = sched.activate() + + @d.addCallback + def check(_): + sched.preStartConsumingChanges.assert_called_with() + return d + def test_gotChange_no_treeStableTimer_unimportant(self): sched = self.makeScheduler( self.Subclass, treeStableTimer=None, branch='master') @@ -381,6 +392,49 @@ def check(_): d.addCallback(lambda _: sched.deactivate()) @defer.inlineCallbacks + def test_preStartConsumingChanges_empty(self): + sched = self.makeFullScheduler(name='test', builderNames=['test'], + treeStableTimer=None, branch='master', + codebases=self.codebases, + createAbsoluteSourceStamps=True) + + yield sched.preStartConsumingChanges() + self.assertEqual(sched._lastCodebases, {}) + + @defer.inlineCallbacks + def test_preStartConsumingChanges_no_createAbsoluteSourceStamps(self): + sched = self.makeFullScheduler(name='test', builderNames=['test'], + treeStableTimer=None, branch='master', + codebases=self.codebases) + + self.db.insertTestData([ + fakedb.Object(id=self.OBJECTID, name='test', + class_name='SingleBranchScheduler'), + fakedb.ObjectState(objectid=self.OBJECTID, name='lastCodebases', + value_json='{"a": {"branch": "master"}}')]) + + yield sched.preStartConsumingChanges() + self.assertEqual(sched._lastCodebases, {}) + + @defer.inlineCallbacks + def test_preStartConsumingChanges_existing(self): + sched = self.makeFullScheduler(name='test', builderNames=['test'], + treeStableTimer=None, branch='master', + codebases=self.codebases, + createAbsoluteSourceStamps=True) + + self.db.insertTestData([ + fakedb.Object(id=self.OBJECTID, name='test', + class_name='SingleBranchScheduler'), + fakedb.ObjectState(objectid=self.OBJECTID, name='lastCodebases', + value_json='{"a": {"branch": "master", "repository": "A", ' + '"revision": "1234:abc", "lastChange": 13}}')]) + + yield sched.preStartConsumingChanges() + self.assertEqual(sched._lastCodebases, { + 'a': {'branch': 'master', 'lastChange': 13, + 'repository': 'A', 'revision': '1234:abc'}}) + def test_gotChange_createAbsoluteSourceStamps_saveCodebase(self): # check codebase is stored after receiving change. sched = self.makeFullScheduler(name='test', builderNames=['test'], @@ -390,18 +444,22 @@ def test_gotChange_createAbsoluteSourceStamps_saveCodebase(self): self.db.insertTestData([ fakedb.Object(id=self.OBJECTID, name='test', class_name='SingleBranchScheduler')]) - yield sched.activate() + d = sched.activate() - yield sched.gotChange(self.mkch(codebase='a', revision='1234:abc', repository='A', number=0), True) - yield sched.gotChange(self.mkch(codebase='b', revision='2345:bcd', repository='B', number=1), True) + d.addCallback(lambda _: + sched.gotChange(self.mkch(codebase='a', revision='1234:abc', repository='A', number=0), True)) + d.addCallback(lambda _: + sched.gotChange(self.mkch(codebase='b', revision='2345:bcd', repository='B', number=1), True)) - self.db.state.assertState(self.OBJECTID, lastCodebases={ - 'a': dict(branch='master', repository='A', revision=u'1234:abc', lastChange=0), - 'b': dict(branch='master', repository='B', revision=u'2345:bcd', lastChange=1)}) + def check(_): + self.db.state.assertState(self.OBJECTID, lastCodebases={ + 'a': dict(branch='master', repository='A', revision=u'1234:abc', lastChange=0), + 'b': dict(branch='master', repository='B', revision=u'2345:bcd', lastChange=1)}) + d.addCallback(check) - yield sched.deactivate() + d.addCallback(lambda _: sched.deactivate()) + return d - @defer.inlineCallbacks def test_gotChange_createAbsoluteSourceStamps_older_change(self): # check codebase is not stored if it's older than the most recent sched = self.makeFullScheduler(name='test', builderNames=['test'], @@ -415,16 +473,22 @@ def test_gotChange_createAbsoluteSourceStamps_older_change(self): value_json='{"a": {"branch": "master", "repository": "A", ' '"revision": "5555:def", "lastChange": 20}}')]) - yield sched.activate() + d = sched.activate() - # this change is not recorded, since it's older than - # change 20 - yield sched.gotChange(self.mkch(codebase='a', revision='1234:abc', repository='A', number=10), True) + d.addCallback(lambda _: + # this change is not recorded, since it's older than + # change 20 + sched.gotChange(self.mkch(codebase='a', revision='1234:abc', + repository='A', number=10), True)) - self.db.state.assertState(self.OBJECTID, lastCodebases={ - 'a': dict(branch='master', repository='A', revision=u'5555:def', lastChange=20)}) + def check(_): + self.db.state.assertState(self.OBJECTID, + lastCodebases={'a': dict(branch='master', repository='A', + revision=u'5555:def', lastChange=20)}) + d.addCallback(check) - yield sched.deactivate() + d.addCallback(lambda _: sched.deactivate()) + return d @defer.inlineCallbacks def test_getCodebaseDict(self): diff --git a/master/buildbot/test/unit/test_schedulers_dependent.py b/master/buildbot/test/unit/test_schedulers_dependent.py index cccdd4bbba8..736592ec1b8 100644 --- a/master/buildbot/test/unit/test_schedulers_dependent.py +++ b/master/buildbot/test/unit/test_schedulers_dependent.py @@ -154,7 +154,7 @@ def do_test(self, scheduler_name, expect_subscription, # and check whether a buildset was added in response if expect_buildset: self.assertEqual(self.addBuildsetCalls, [ - ('addBuildsetForSourceStamps', dict( + ('addBuildsetForSourceStamp', dict( builderNames=None, # defaults external_idstring=None, properties=None, diff --git a/master/buildbot/test/unit/test_schedulers_timed_Nightly.py b/master/buildbot/test/unit/test_schedulers_timed_Nightly.py index e5a06cfde23..eaf972c5425 100644 --- a/master/buildbot/test/unit/test_schedulers_timed_Nightly.py +++ b/master/buildbot/test/unit/test_schedulers_timed_Nightly.py @@ -16,6 +16,7 @@ import mock import time +from buildbot import config from buildbot.changes import filter from buildbot.schedulers import timed from buildbot.test.fake import fakedb @@ -36,7 +37,8 @@ class Nightly(scheduler.SchedulerMixin, unittest.TestCase): # minutes past the hour) and subtracted before the time offset is reported. localtime_offset = time.timezone % 3600 - def makeScheduler(self, **kwargs): + def makeScheduler(self, firstBuildDuration=0, **kwargs): + sched = self.attachScheduler(timed.Nightly(**kwargs), self.OBJECTID, overrideBuildsetMethods=True) @@ -46,20 +48,32 @@ def makeScheduler(self, **kwargs): self.clock = sched._reactor = task.Clock() self.clock.advance(self.localtime_offset) # get to 0 min past the hour - self.addBuildsetCallTimes = [] - - def recordTimes(timeList, method): - def timedMethod(**kw): - timeList.append(self.clock.seconds() - self.localtime_offset) - return method(**kw) - return timedMethod - - sched.addBuildsetForSourceStampsWithDefaults = recordTimes( - self.addBuildsetCallTimes, - sched.addBuildsetForSourceStampsWithDefaults) - sched.addBuildsetForChanges = recordTimes( - self.addBuildsetCallTimes, - sched.addBuildsetForChanges) + # keep track of builds in self.events + self.events = [] + + def addBuildsetForLatest(reason='', external_idstring='', + branch=None, repository='', project=''): + self.assertIn('scheduler named', reason) + isFirst = (self.events == []) + self.events.append('B(%s)@%d' % (branch, + # show the offset as seconds past the GMT hour + self.clock.seconds() - self.localtime_offset)) + if isFirst and firstBuildDuration: + d = defer.Deferred() + self.clock.callLater(firstBuildDuration, d.callback, None) + return d + else: + return defer.succeed(None) + sched.addBuildsetForLatest = addBuildsetForLatest + + origAddBuildsetForChanges = sched.addBuildsetForChanges + + def addBuildsetForChanges(reason='', external_idstring=None, changeids=[]): + self.events.append('B%s@%d' % (repr(changeids).replace(' ', ''), + # show the offset as seconds past the GMT hour + self.clock.seconds() - self.localtime_offset)) + return origAddBuildsetForChanges(reason=reason, external_idstring=external_idstring, changeids=changeids) + sched.addBuildsetForChanges = addBuildsetForChanges # see self.assertConsumingChanges self.consumingChanges = None @@ -119,9 +133,13 @@ def test_constructor_change_filter(self): branch=None, change_filter=filter.ChangeFilter(category_re="fo+o")) assert sched.change_filter + def test_constructor_no_branch(self): + self.assertRaises(config.ConfigErrors, + lambda: self.makeScheduler(name='test', builderNames=['test'], + change_filter=filter.ChangeFilter(category_re="fo+o"))) + # end-to-end tests: let's see the scheduler in action - @defer.inlineCallbacks def test_iterations_simple(self): # note that Nightly works in local time, but the task.Clock() always # starts at midnight UTC, so be careful not to use times that are @@ -133,7 +151,7 @@ def test_iterations_simple(self): # add a change classification self.db.schedulers.fakeClassifications(self.OBJECTID, {19: True}) - yield sched.activate() + sched.activate() # check that the classification has been flushed, since this # invocation has not requested onlyIfChanged @@ -142,30 +160,12 @@ def test_iterations_simple(self): self.clock.advance(0) # let it get set up while self.clock.seconds() < self.localtime_offset + 30 * 60: self.clock.advance(60) - self.assertEqual(self.addBuildsetCallTimes, [600, 1200, 1260]) - self.assertEqual(self.addBuildsetCalls, [ - ('addBuildsetForSourceStampsWithDefaults', { - 'builderNames': None, - 'sourcestamps': [{'codebase': ''}], - 'properties': None, - 'reason': u"The Nightly scheduler named 'test' triggered this build", - 'waited_for': False}), - ('addBuildsetForSourceStampsWithDefaults', { - 'builderNames': None, - 'sourcestamps': [{'codebase': ''}], - 'properties': None, - 'reason': u"The Nightly scheduler named 'test' triggered this build", - 'waited_for': False}), - ('addBuildsetForSourceStampsWithDefaults', { - 'builderNames': None, - 'sourcestamps': [{'codebase': ''}], - 'properties': None, - 'reason': u"The Nightly scheduler named 'test' triggered this build", - 'waited_for': False})]) + self.assertEqual(self.events, ['B(None)@600', 'B(None)@1200', 'B(None)@1260']) self.db.state.assertStateByClass('test', 'Nightly', last_build=1260 + self.localtime_offset) - yield sched.deactivate() + d = sched.deactivate() + return d def test_iterations_simple_with_branch(self): # see timezone warning above @@ -177,14 +177,7 @@ def test_iterations_simple_with_branch(self): self.clock.advance(0) while self.clock.seconds() < self.localtime_offset + 10 * 60: self.clock.advance(60) - self.assertEqual(self.addBuildsetCallTimes, [300]) - self.assertEqual(self.addBuildsetCalls, [ - ('addBuildsetForSourceStampsWithDefaults', { - 'builderNames': None, - 'sourcestamps': [{'codebase': ''}], - 'properties': None, - 'reason': u"The Nightly scheduler named 'test' triggered this build", - 'waited_for': False})]) + self.assertEqual(self.events, ['B(master)@300']) self.db.state.assertStateByClass('test', 'Nightly', last_build=300 + self.localtime_offset) @@ -197,11 +190,10 @@ def do_test_iterations_onlyIfChanged(self, *changes_at, **kwargs): minute=[5, 25, 45], onlyIfChanged=True, fileIsImportant=fII, **kwargs) - return self.do_test_iterations_onlyIfChanged_test(fII, *changes_at) + self.do_test_iterations_onlyIfChanged_test(fII, *changes_at) - @defer.inlineCallbacks def do_test_iterations_onlyIfChanged_test(self, fII, *changes_at): - yield self.sched.activate() + self.sched.activate() # check that the scheduler has started to consume changes self.assertConsumingChanges(fileIsImportant=fII, change_filter=None, @@ -218,41 +210,37 @@ def do_test_iterations_onlyIfChanged_test(self, fII, *changes_at): self.localtime_offset + changes_at[0][0]): when, newchange, important = changes_at.pop(0) self.db.changes.fakeAddChangeInstance(newchange) - yield self.sched.gotChange(newchange, important).addErrback(log.err) + self.sched.gotChange(newchange, important).addErrback(log.err) # and advance the clock by a minute self.clock.advance(60) - @defer.inlineCallbacks def test_iterations_onlyIfChanged_no_changes(self): - yield self.do_test_iterations_onlyIfChanged() - self.assertEqual(self.addBuildsetCalls, []) + self.do_test_iterations_onlyIfChanged() + self.assertEqual(self.events, []) self.db.state.assertStateByClass('test', 'Nightly', last_build=1500 + self.localtime_offset) - yield self.sched.deactivate() + return self.sched.deactivate() - @defer.inlineCallbacks def test_iterations_onlyIfChanged_unimp_changes(self): - yield self.do_test_iterations_onlyIfChanged( + self.do_test_iterations_onlyIfChanged( (60, mock.Mock(), False), (600, mock.Mock(), False)) - self.assertEqual(self.addBuildsetCalls, []) + self.assertEqual(self.events, []) self.db.state.assertStateByClass('test', 'Nightly', last_build=1500 + self.localtime_offset) - yield self.sched.deactivate() + return self.sched.deactivate() - @defer.inlineCallbacks def test_iterations_onlyIfChanged_off_branch_changes(self): - yield self.do_test_iterations_onlyIfChanged( - (60, self.makeFakeChange(number=1, branch='testing'), True), - (1700, self.makeFakeChange(number=2, branch='staging'), True)) - self.assertEqual(self.addBuildsetCalls, []) + self.do_test_iterations_onlyIfChanged( + (60, self.makeFakeChange(branch='testing'), True), + (1700, self.makeFakeChange(branch='staging'), True)) + self.assertEqual(self.events, []) self.db.state.assertStateByClass('test', 'Nightly', last_build=1500 + self.localtime_offset) - yield self.sched.deactivate() + return self.sched.deactivate() - @defer.inlineCallbacks def test_iterations_onlyIfChanged_mixed_changes(self): - yield self.do_test_iterations_onlyIfChanged( + self.do_test_iterations_onlyIfChanged( (120, self.makeFakeChange(number=3, branch=None), False), (130, self.makeFakeChange(number=4, branch='offbranch'), True), (1200, self.makeFakeChange(number=5, branch=None), True), @@ -261,31 +249,22 @@ def test_iterations_onlyIfChanged_mixed_changes(self): # note that the changeid list includes the unimportant changes, but not the # off-branch changes, and note that no build took place at 300s, as no important # changes had yet arrived - self.assertEqual(self.addBuildsetCallTimes, [1500]) - self.assertEqual(self.addBuildsetCalls, [ - ('addBuildsetForChanges', { - 'builderNames': None, - 'changeids': [3, 5, 6], - 'external_idstring': None, - 'properties': None, - 'reason': u"The Nightly scheduler named 'test' triggered this build", - 'waited_for': False})]) + self.assertEqual(self.events, ['B[3,5,6]@1500']) self.db.state.assertStateByClass('test', 'Nightly', last_build=1500 + self.localtime_offset) - yield self.sched.deactivate() + return self.sched.deactivate() - @defer.inlineCallbacks def test_iterations_onlyIfChanged_createAbsoluteSourceStamps_oneChanged(self): # Test createAbsoluteSourceStamps=True when only one codebase has changed - yield self.do_test_iterations_onlyIfChanged( + self.do_test_iterations_onlyIfChanged( (120, self.makeFakeChange(number=3, codebase='a', revision='2345:bcd'), True), codebases={'a': {'repository': "", 'branch': 'master'}, 'b': {'repository': "", 'branch': 'master'}}, createAbsoluteSourceStamps=True) + self.assertEqual(self.events, ['B[3]@300']) self.db.state.assertStateByClass('test', 'Nightly', last_build=1500 + self.localtime_offset) # addBuildsetForChanges calls getCodebase, so this isn't too interesting - self.assertEqual(self.addBuildsetCallTimes, [300]) self.assertEqual(self.addBuildsetCalls, [ ('addBuildsetForChanges', { 'builderNames': None, @@ -296,9 +275,8 @@ def test_iterations_onlyIfChanged_createAbsoluteSourceStamps_oneChanged(self): 'waited_for': False})]) self.db.state.assertStateByClass('test', 'Nightly', lastCodebases={ 'a': dict(revision='2345:bcd', branch=None, repository='', lastChange=3)}) - yield self.sched.deactivate() + return self.sched.deactivate() - @defer.inlineCallbacks def test_iterations_onlyIfChanged_createAbsoluteSourceStamps_oneChanged_loadOther(self): # Test createAbsoluteSourceStamps=True when only one codebase has changed, # but the other was previously changed @@ -315,13 +293,13 @@ def test_iterations_onlyIfChanged_createAbsoluteSourceStamps_oneChanged_loadOthe fakedb.ObjectState(objectid=self.OBJECTID, name='lastCodebases', value_json='{"b": {"branch": "master", "repository": "B", "revision": "1234:abc", "lastChange": 2}}')]) - yield self.do_test_iterations_onlyIfChanged_test(fII, - (120, self.makeFakeChange(number=3, codebase='a', revision='2345:bcd'), True)) + self.do_test_iterations_onlyIfChanged_test(fII, + (120, self.makeFakeChange(number=3, codebase='a', revision='2345:bcd'), True)) + self.assertEqual(self.events, ['B[3]@300']) self.db.state.assertStateByClass('test', 'Nightly', last_build=1500 + self.localtime_offset) # addBuildsetForChanges calls getCodebase, so this isn't too interesting - self.assertEqual(self.addBuildsetCallTimes, [300]) self.assertEqual(self.addBuildsetCalls, [ ('addBuildsetForChanges', { 'builderNames': None, @@ -333,21 +311,20 @@ def test_iterations_onlyIfChanged_createAbsoluteSourceStamps_oneChanged_loadOthe self.db.state.assertStateByClass('test', 'Nightly', lastCodebases={ 'a': dict(revision='2345:bcd', branch=None, repository='', lastChange=3), 'b': dict(revision='1234:abc', branch="master", repository='B', lastChange=2)}) - yield self.sched.deactivate() + return self.sched.deactivate() - @defer.inlineCallbacks def test_iterations_onlyIfChanged_createAbsoluteSourceStamps_bothChanged(self): # Test createAbsoluteSourceStamps=True when both codebases have changed - yield self.do_test_iterations_onlyIfChanged( + self.do_test_iterations_onlyIfChanged( (120, self.makeFakeChange(number=3, codebase='a', revision='2345:bcd'), True), (122, self.makeFakeChange(number=4, codebase='b', revision='1234:abc'), True), codebases={'a': {'repository': "", 'branch': 'master'}, 'b': {'repository': "", 'branch': 'master'}}, createAbsoluteSourceStamps=True) + self.assertEqual(self.events, ['B[3,4]@300']) self.db.state.assertStateByClass('test', 'Nightly', last_build=1500 + self.localtime_offset) # addBuildsetForChanges calls getCodebase, so this isn't too interesting - self.assertEqual(self.addBuildsetCallTimes, [300]) self.assertEqual(self.addBuildsetCalls, [ ('addBuildsetForChanges', { 'builderNames': None, @@ -359,4 +336,4 @@ def test_iterations_onlyIfChanged_createAbsoluteSourceStamps_bothChanged(self): self.db.state.assertStateByClass('test', 'Nightly', lastCodebases={ 'a': dict(revision='2345:bcd', branch=None, repository='', lastChange=3), 'b': dict(revision='1234:abc', branch=None, repository='', lastChange=4)}) - yield self.sched.deactivate() + return self.sched.deactivate() diff --git a/master/buildbot/test/unit/test_schedulers_timed_NightlyTriggerable.py b/master/buildbot/test/unit/test_schedulers_timed_NightlyTriggerable.py index cc5d4c3e81b..763fcc33a9c 100644 --- a/master/buildbot/test/unit/test_schedulers_timed_NightlyTriggerable.py +++ b/master/buildbot/test/unit/test_schedulers_timed_NightlyTriggerable.py @@ -85,7 +85,7 @@ def test_timer_oneTrigger(self): sched.activate() - sched.trigger(False, [ + sched.trigger([ dict(revision='myrev', branch='br', project='p', repository='r', codebase='cb'), ], set_props=None) @@ -103,11 +103,11 @@ def test_timer_twoTriggers(self): sched.activate() - sched.trigger(False, [ + sched.trigger([ dict(codebase='cb', revision='myrev1', branch='br', project='p', repository='r') ], set_props=None) - sched.trigger(False, [ + sched.trigger([ dict(codebase='cb', revision='myrev2', branch='br', project='p', repository='r') ], set_props=None) @@ -126,7 +126,7 @@ def test_timer_oneTrigger_then_noBuild(self): sched.activate() - sched.trigger(False, [ + sched.trigger([ dict(codebase='cb', revision='myrev', branch='br', project='p', repository='r') ], set_props=None) @@ -149,7 +149,7 @@ def test_timer_oneTriggers_then_oneTrigger(self): sched.activate() - sched.trigger(False, [ + sched.trigger([ dict(codebase='cb', revision='myrev1', branch='br', project='p', repository='r') ], set_props=None) @@ -161,7 +161,7 @@ def test_timer_oneTriggers_then_oneTrigger(self): revision='myrev1'), ]) - sched.trigger(False, [ + sched.trigger([ dict(codebase='cb', revision='myrev2', branch='br', project='p', repository='r') ], set_props=None) @@ -179,7 +179,7 @@ def test_savedTrigger(self): self.db.insertTestData([ fakedb.Object(id=self.SCHEDULERID, name='test', class_name='NightlyTriggerable'), fakedb.ObjectState(objectid=self.SCHEDULERID, name='lastTrigger', - value_json='[ [ {"codebase": "cb", "project": "p", "repository": "r", "branch": "br", "revision": "myrev"} ], {}, null, null ]'), + value_json='[ [ {"codebase": "cb", "project": "p", "repository": "r", "branch": "br", "revision": "myrev"} ], {} ]'), ]) sched.activate() @@ -197,7 +197,7 @@ def test_savedTrigger_dict(self): self.db.insertTestData([ fakedb.Object(id=self.SCHEDULERID, name='test', class_name='NightlyTriggerable'), fakedb.ObjectState(objectid=self.SCHEDULERID, name='lastTrigger', - value_json='[ { "cb": {"codebase": "cb", "project": "p", "repository": "r", "branch": "br", "revision": "myrev"} }, {}, null, null ]'), + value_json='[ { "cb": {"codebase": "cb", "project": "p", "repository": "r", "branch": "br", "revision": "myrev"} }, {} ]'), ]) sched.activate() @@ -218,7 +218,7 @@ def test_saveTrigger(self): sched.activate() - (idsDeferred, d) = sched.trigger(False, [ + d = sched.trigger([ dict(codebase='cb', revision='myrev', branch='br', project='p', repository='r'), ], set_props=None) @@ -228,7 +228,7 @@ def cb(_): self.db.state.assertState(self.SCHEDULERID, lastTrigger=[[ dict(codebase='cb', revision='myrev', branch='br', project='p', repository='r'), - ], {}, None, None]) + ], {}]) return d @@ -241,7 +241,7 @@ def test_saveTrigger_noTrigger(self): sched.activate() - (idsDeferre, d) = sched.trigger(False, [ + d = sched.trigger([ dict(codebase='cb', revision='myrev', branch='br', project='p', repository='r'), ], set_props=None) @@ -263,7 +263,7 @@ def test_triggerProperties(self): sched.activate() - sched.trigger(False, [ + sched.trigger([ dict(codebase='cb', revision='myrev', branch='br', project='p', repository='r'), ], properties.Properties(testprop='test')) @@ -271,7 +271,7 @@ def test_triggerProperties(self): self.db.state.assertState(self.SCHEDULERID, lastTrigger=[[ dict(codebase='cb', revision='myrev', branch='br', project='p', repository='r'), - ], {'testprop': ['test', 'TEST']}, None, None]) + ], {'testprop': ['test', 'TEST']}]) self.clock.advance(60 * 60) # Run for 1h @@ -288,7 +288,7 @@ def test_savedProperties(self): self.db.insertTestData([ fakedb.Object(id=self.SCHEDULERID, name='test', class_name='NightlyTriggerable'), fakedb.ObjectState(objectid=self.SCHEDULERID, name='lastTrigger', - value_json='[ [ {"codebase": "cb", "project": "p", "repository": "r", "branch": "br", "revision": "myrev"} ], {"testprop": ["test", "TEST"]}, null, null ]'), + value_json='[ [ {"codebase": "cb", "project": "p", "repository": "r", "branch": "br", "revision": "myrev"} ], {"testprop": ["test", "TEST"]} ]'), ]) sched.activate() diff --git a/master/buildbot/test/unit/test_schedulers_timed_Periodic.py b/master/buildbot/test/unit/test_schedulers_timed_Periodic.py index 87c2b615b03..30a922b8922 100644 --- a/master/buildbot/test/unit/test_schedulers_timed_Periodic.py +++ b/master/buildbot/test/unit/test_schedulers_timed_Periodic.py @@ -38,11 +38,9 @@ def makeScheduler(self, firstBuildDuration=0, exp_branch=None, **kwargs): # keep track of builds in self.events self.events = [] - def addBuildsetForSourceStampsWithDefaults(reason, sourcestamps, - waited_for=False, properties=None, builderNames=None, - **kw): + def addBuildsetForLatest(reason=None, branch=None): self.assertIn('Periodic scheduler named', reason) - # TODO: check branch + self.assertEqual(branch, exp_branch) isFirst = (self.events == []) self.events.append('B@%d' % self.clock.seconds()) if isFirst and firstBuildDuration: @@ -51,7 +49,7 @@ def addBuildsetForSourceStampsWithDefaults(reason, sourcestamps, return d else: return defer.succeed(None) - sched.addBuildsetForSourceStampsWithDefaults = addBuildsetForSourceStampsWithDefaults + sched.addBuildsetForLatest = addBuildsetForLatest # handle state locally self.state = {} diff --git a/master/buildbot/test/unit/test_schedulers_trysched.py b/master/buildbot/test/unit/test_schedulers_trysched.py index 339c26a29c0..4cd14ffbc25 100644 --- a/master/buildbot/test/unit/test_schedulers_trysched.py +++ b/master/buildbot/test/unit/test_schedulers_trysched.py @@ -533,7 +533,7 @@ def test_handleJobFile(self): def check(_): self.assertEqual(self.addBuildsetCalls, [ - ('addBuildsetForSourceStamps', dict( + ('addBuildsetForSourceStamp', dict( builderNames=['buildera', 'builderb'], external_idstring=u'extid', properties={}, @@ -586,7 +586,7 @@ def test_handleJobFile_subset_builders(self): def check(_): self.assertEqual(self.addBuildsetCalls, [ - ('addBuildsetForSourceStamps', dict( + ('addBuildsetForSourceStamp', dict( builderNames=['buildera'], external_idstring=u'extid', properties={}, @@ -614,7 +614,7 @@ def test_handleJobFile_with_try_properties(self): def check(_): self.assertEqual(self.addBuildsetCalls, [ - ('addBuildsetForSourceStamps', dict( + ('addBuildsetForSourceStamp', dict( builderNames=['buildera', 'builderb'], external_idstring=u'extid', properties={'foo': ('bar', u'try build')}, @@ -688,7 +688,7 @@ def test_perspective_try(self): def check(_): self.assertEqual(self.addBuildsetCalls, [ - ('addBuildsetForSourceStamps', dict( + ('addBuildsetForSourceStamp', dict( builderNames=['a'], external_idstring=None, properties={'pr': ('op', u'try build')}, @@ -717,7 +717,7 @@ def test_perspective_try_who(self): def check(_): self.assertEqual(self.addBuildsetCalls, [ - ('addBuildsetForSourceStamps', dict( + ('addBuildsetForSourceStamp', dict( builderNames=['a'], external_idstring=None, properties={'pr': ('op', u'try build')}, diff --git a/master/buildbot/test/unit/test_util_codebase.py b/master/buildbot/test/unit/test_util_codebase.py deleted file mode 100644 index b36c16d85b9..00000000000 --- a/master/buildbot/test/unit/test_util_codebase.py +++ /dev/null @@ -1,102 +0,0 @@ -# This file is part of Buildbot. Buildbot is free software: you can -# redistribute it and/or modify it under the terms of the GNU General Public -# License as published by the Free Software Foundation, version 2. -# -# This program is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more -# details. -# -# You should have received a copy of the GNU General Public License along with -# this program; if not, write to the Free Software Foundation, Inc., 51 -# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# -# Copyright Buildbot Team Members - -from buildbot.test.fake import fakemaster -from buildbot.test.util import scheduler -from buildbot.util import codebase -from buildbot.util import state -from twisted.internet import defer -from twisted.trial import unittest - - -class FakeObject(codebase.AbsoluteSourceStampsMixin, state.StateMixin): - - name = 'fake-name' - - def __init__(self, master, codebases): - self.master = master - self.codebases = codebases - - -class TestAbsoluteSourceStampsMixin(unittest.TestCase, scheduler.SchedulerMixin): - - codebases = {'a': {'repository': '', 'branch': 'master'}, - 'b': {'repository': '', 'branch': 'master'}} - - def setUp(self): - self.master = fakemaster.make_master(wantDb=True, wantData=True, testcase=self) - self.db = self.master.db - self.object = FakeObject(self.master, self.codebases) - - def mkch(self, **kwargs): - ch = self.makeFakeChange(**kwargs) - self.master.db.changes.fakeAddChangeInstance(ch) - return ch - - @defer.inlineCallbacks - def test_getCodebaseDict(self): - cbd = yield self.object.getCodebaseDict('a') - self.assertEqual(cbd, {'repository': '', 'branch': 'master'}) - - def test_getCodebaseDict_not_found(self): - d = self.object.getCodebaseDict('c') - self.assertFailure(d, KeyError) - return d - - @defer.inlineCallbacks - def test_getCodebaseDict_existing(self): - self.db.state.fakeState('fake-name', 'FakeObject', - lastCodebases={'a': { - 'repository': 'A', - 'revision': '1234:abc', - 'branch': 'master', - 'lastChange': 10}}) - cbd = yield self.object.getCodebaseDict('a') - self.assertEqual(cbd, {'repository': 'A', 'revision': '1234:abc', - 'branch': 'master', 'lastChange': 10}) - cbd = yield self.object.getCodebaseDict('b') - self.assertEqual(cbd, {'repository': '', 'branch': 'master'}) - - @defer.inlineCallbacks - def test_recordChange(self): - yield self.object.recordChange(self.mkch(codebase='a', repository='A', revision='1234:abc', branch='master', number=10)) - self.db.state.assertStateByClass('fake-name', 'FakeObject', lastCodebases={ - 'a': {'repository': 'A', 'revision': '1234:abc', 'branch': 'master', 'lastChange': 10}}) - - @defer.inlineCallbacks - def test_recordChange_older(self): - self.db.state.fakeState('fake-name', 'FakeObject', - lastCodebases={'a': { - 'repository': 'A', - 'revision': '2345:bcd', - 'branch': 'master', - 'lastChange': 20}}) - yield self.object.getCodebaseDict('a') - yield self.object.recordChange(self.mkch(codebase='a', repository='A', revision='1234:abc', branch='master', number=10)) - self.db.state.assertStateByClass('fake-name', 'FakeObject', lastCodebases={ - 'a': {'repository': 'A', 'revision': '2345:bcd', 'branch': 'master', 'lastChange': 20}}) - - @defer.inlineCallbacks - def test_recordChange_newer(self): - self.db.state.fakeState('fake-name', 'FakeObject', - lastCodebases={'a': { - 'repository': 'A', - 'revision': '1234:abc', - 'branch': 'master', - 'lastChange': 10}}) - yield self.object.getCodebaseDict('a') - yield self.object.recordChange(self.mkch(codebase='a', repository='A', revision='2345:bcd', branch='master', number=20)) - self.db.state.assertStateByClass('fake-name', 'FakeObject', lastCodebases={ - 'a': {'repository': 'A', 'revision': '2345:bcd', 'branch': 'master', 'lastChange': 20}}) diff --git a/master/buildbot/test/unit/test_www_service.py b/master/buildbot/test/unit/test_www_service.py index 94193b985cf..4273aa1c551 100644 --- a/master/buildbot/test/unit/test_www_service.py +++ b/master/buildbot/test/unit/test_www_service.py @@ -43,7 +43,7 @@ def setUp(self): def makeConfig(self, **kwargs): pwd = os.getcwd() - w = dict(url='h:/', port=None, public_html=pwd, auth=auth.NoAuth(), logfileName='l') + w = dict(url='h:/', port=None, public_html=pwd, auth=auth.NoAuth()) w.update(kwargs) new_config = mock.Mock() new_config.www = w diff --git a/master/buildbot/test/util/scheduler.py b/master/buildbot/test/util/scheduler.py index 7182b8cc365..5b8ab3e8aa8 100644 --- a/master/buildbot/test/util/scheduler.py +++ b/master/buildbot/test/util/scheduler.py @@ -191,7 +191,7 @@ def _addBuildsetReturnValue(self, builderNames): def fake_addBuildsetForSourceStampsWithDefaults(self, reason, sourcestamps, waited_for=False, properties=None, builderNames=None, **kw): - properties = properties.asDict() if properties is not None else None + properties = properties.asDict() self.assertIsInstance(sourcestamps, list) sourcestamps.sort() self.addBuildsetCalls.append(('addBuildsetForSourceStampsWithDefaults', @@ -217,7 +217,7 @@ def fake_addBuildsetForSourceStamps(self, waited_for=False, sourcestamps=[], properties = properties.asDict() if properties is not None else None self.assertIsInstance(sourcestamps, list) sourcestamps.sort() - self.addBuildsetCalls.append(('addBuildsetForSourceStamps', + self.addBuildsetCalls.append(('addBuildsetForSourceStamp', dict(reason=reason, external_idstring=external_idstring, properties=properties, builderNames=builderNames, sourcestamps=sourcestamps))) diff --git a/master/buildbot/util/codebase.py b/master/buildbot/util/codebase.py deleted file mode 100644 index 3a1699efd3c..00000000000 --- a/master/buildbot/util/codebase.py +++ /dev/null @@ -1,46 +0,0 @@ -# This file is part of Buildbot. Buildbot is free software: you can -# redistribute it and/or modify it under the terms of the GNU General Public -# License as published by the Free Software Foundation, version 2. -# -# This program is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more -# details. -# -# You should have received a copy of the GNU General Public License along with -# this program; if not, write to the Free Software Foundation, Inc., 51 -# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# -# Copyright Buildbot Team Members - -from twisted.internet import defer - - -class AbsoluteSourceStampsMixin(object): - # record changes and revisions per codebase - - _lastCodebases = None - - @defer.inlineCallbacks - def getCodebaseDict(self, codebase): - assert self.codebases - - if self._lastCodebases is None: - self._lastCodebases = yield self.getState('lastCodebases', {}) - - # may fail with KeyError - defer.returnValue(self._lastCodebases.get(codebase, self.codebases[codebase])) - - @defer.inlineCallbacks - def recordChange(self, change): - codebase = yield self.getCodebaseDict(change.codebase) - lastChange = codebase.get('lastChange', -1) - - if change.number > lastChange: - self._lastCodebases[change.codebase] = { - 'repository': change.repository, - 'branch': change.branch, - 'revision': change.revision, - 'lastChange': change.number - } - yield self.setState('lastCodebases', self._lastCodebases) diff --git a/master/buildbot/www/service.py b/master/buildbot/www/service.py index 5f314e42359..ffba4851928 100644 --- a/master/buildbot/www/service.py +++ b/master/buildbot/www/service.py @@ -13,8 +13,6 @@ # # Copyright Buildbot Team Members -import os - from buildbot import config from buildbot.plugins.db import get_plugins from buildbot.util import service @@ -26,7 +24,6 @@ from buildbot.www import ws from twisted.application import strports from twisted.internet import defer -from twisted.python import log from twisted.web import server @@ -142,38 +139,7 @@ def setupSite(self, new_config): root.putChild('sse', sse.EventResource(self.master)) self.root = root - - def either(a, b): # a if a else b for py2.4 - if a: - return a - else: - return b - - rotateLength = either(new_config.www.get('logRotateLength'), self.master.log_rotation.rotateLength) - maxRotatedFiles = either(new_config.www.get('maxRotatedFiles'), self.master.log_rotation.maxRotatedFiles) - - class RotateLogSite(server.Site): - """ A Site that logs to a separate file: http.log, and rotate its logs """ - - def _openLogFile(self, path): - try: - from twisted.python.logfile import LogFile - log.msg("Setting up http.log rotating %s files of %s bytes each" % - (maxRotatedFiles, rotateLength)) - if hasattr(LogFile, "fromFullPath"): # not present in Twisted-2.5.0 - return LogFile.fromFullPath(path, rotateLength=rotateLength, maxRotatedFiles=maxRotatedFiles) - else: - log.msg("WebStatus: rotated http logs are not supported on this version of Twisted") - except ImportError, e: - log.msg("WebStatus: Unable to set up rotating http.log: %s" % e) - - # if all else fails, just call the parent method - return server.Site._openLogFile(self, path) - - httplog = None - if new_config.www['logfileName']: - httplog = os.path.abspath(os.path.join(self.master.basedir, new_config.www['logfileName'])) - self.site = RotateLogSite(root, logPath=httplog) + self.site = server.Site(root) # todo: need to store session infos in the db for multimaster # rough examination, it looks complicated, as all the session APIs are sync diff --git a/master/contrib/Dockerfile b/master/contrib/Dockerfile index afc8a31cfaf..0f728aee92e 100644 --- a/master/contrib/Dockerfile +++ b/master/contrib/Dockerfile @@ -31,7 +31,7 @@ # docker build -t buildbot - < Dockerfile # # # Run buildbot -# CONTAINER_ID=$(docker run -d -p 8010:8010 -p 22 buildbot) +# CONTAINER_ID=$(docker run -d buildbot) # # # Test buildbot master is listening # wget -qO- localhost:8010 @@ -51,7 +51,7 @@ # # make your buildbot playground useful for your own projects. # # # Log into container (username: admin password: admin) -# ssh -p $(docker port $CONTAINER_ID 22 | cut -d: -f 2) admin@localhost +# ssh -p $(docker port $CONTAINER_ID 22) admin@localhost # # Base docker image from ubuntu:12.04 diff --git a/master/docs/bbdocs/ext.py b/master/docs/bbdocs/ext.py index f96d58f6d9f..d6b16c911b8 100644 --- a/master/docs/bbdocs/ext.py +++ b/master/docs/bbdocs/ext.py @@ -110,8 +110,7 @@ def resolve_ref(cls, domain, env, fromdocname, builder, typ, target, node, try: todocname, targetname = targets[target] except KeyError: - env.warn(fromdocname, "Missing BB reference: bb:%s:%s" % (cls.ref_type, target), - node.line) + print "MISSING BB REFERENCE: bb:%s:%s" % (cls.ref_type, target) return None return make_refnode(builder, fromdocname, diff --git a/master/docs/developer/mq.rst b/master/docs/developer/mq.rst index 0d753e6f150..4f9cf60c4e3 100644 --- a/master/docs/developer/mq.rst +++ b/master/docs/developer/mq.rst @@ -156,7 +156,7 @@ Unit tests should be used to ensure this resiliency. Some related messages are sent at approximately the same time. Due to the non-blocking nature of message delivery, consumers should *not* assume that subsequent messages in a sequence remain queued. -For example, upon receipt of a :bb:event:`buildset.$bsid.new` message, it is already too late to try to subscribe to the associated build requests messages, as they may already have been consumed. +For example, upon receipt of a :bb:msg:`buildset.$bsid.new` message, it is already too late to try to subscribe to the associated build requests messages, as they may already have been consumed. Schema Changes ~~~~~~~~~~~~~~ diff --git a/master/docs/manual/cfg-schedulers.rst b/master/docs/manual/cfg-schedulers.rst index 63950b3675e..7fdd9c4eb35 100644 --- a/master/docs/manual/cfg-schedulers.rst +++ b/master/docs/manual/cfg-schedulers.rst @@ -395,14 +395,6 @@ The arguments to this scheduler are: ``onlyImportant`` -``createAbsoluteSourceStamps`` - See :ref:`Configuring-Schedulers`. - -``onlyIfChanged`` - If this is true, then builds will not be scheduled at the designated time - *unless* the specified branch has seen an important change since - the previous build. - ``reason`` See :ref:`Configuring-Schedulers`. @@ -455,6 +447,7 @@ The full list of parameters is: ``createAbsoluteSourceStamps`` See :ref:`Configuring-Schedulers`. + Note that ``fileIsImportant``, ``change_filter`` and ``createAbsoluteSourceStamps`` are only relevant if ``onlyIfChanged`` is ``True``. ``onlyIfChanged`` If this is true, then builds will not be scheduled at the designated time *unless* the specified branch has seen an important change since the previous build. diff --git a/master/docs/manual/cfg-www.rst b/master/docs/manual/cfg-www.rst index c4cf943c733..737448d1ddb 100644 --- a/master/docs/manual/cfg-www.rst +++ b/master/docs/manual/cfg-www.rst @@ -72,19 +72,6 @@ This server is configured with the :bb:cfg:`www` configuration key, which specif For use of corporate pictures, you can use LdapUserInfo, which can also acts as an avatar provider. See :ref:`Web-Authentication`. -``logfileName`` - Filename used for http access logs, relative to the master directory. - If set to ``None`` or the empty string, the content of the logs will land in the main :file:`twisted.log` log file. - (Default to ``http.log``) - -``logRotateLength`` - The amount of bytes after which the :file:`http.log` file will be rotated. - (Default to the same value as for the :file:`twisted.log` file, set in :file:`buildbot.tac`) - -``maxRotatedFiles`` - The amount of log files that will be kept when rotating - (Default to the same value as for the :file:`twisted.log` file, set in :file:`buildbot.tac`) - .. _Web-Authentication: Authentication plugins diff --git a/master/docs/manual/cmdline.rst b/master/docs/manual/cmdline.rst index 7e27cd2ffc6..1944cca3c70 100644 --- a/master/docs/manual/cmdline.rst +++ b/master/docs/manual/cmdline.rst @@ -374,12 +374,12 @@ It requires that you have a :class:`PBChangeSource` (:bb:chsrc:`PBChangeSource`) buildbot sendchange --master {MASTERHOST}:{PORT} --auth {USER}:{PASS} --who {USER} {FILENAMES..} -The :option:`--auth` option specifies the credentials to use to connect to the master, in the form ``user:pass``. +The :option:`auth` option specifies the credentials to use to connect to the master, in the form ``user:pass``. If the password is omitted, then sendchange will prompt for it. If both are omitted, the old default (username "change" and password "changepw") will be used. Note that this password is well-known, and should not be used on an internet-accessible port. -The :option:`--master` and :option:`--username` arguments can also be given in the options file (see :ref:`buildbot-config-directory`). +The :option:`master` and :option:`username` arguments can also be given in the options file (see :ref:`buildbot-config-directory`). There are other (optional) arguments which can influence the ``Change`` that gets submitted: --branch @@ -440,7 +440,7 @@ For details on how Buildbot manages users, see :ref:`Concepts-Users`. --master The :command:`user` command can be run virtually anywhere provided a location of the running buildmaster. - The :option:`--master` argument is of the form :samp:`{MASTERHOST}:{PORT}`. + The :option:`master` argument is of the form :samp:`{MASTERHOST}:{PORT}`. --username PB connection authentication that should match the arguments to `CommandlineUserManager`. @@ -449,35 +449,35 @@ For details on how Buildbot manages users, see :ref:`Concepts-Users`. PB connection authentication that should match the arguments to `CommandlineUserManager`. --op - There are four supported values for the :option:`--op` argument: ``add``, ``update``, ``remove``, and ``get``. + There are four supported values for the :option:`op` argument: :option:`add`, :option:`update`, :option:`remove`, and :option:`get`. Each are described in full in the following sections. --bb_username - Used with the :option:`--op=update` option, this sets the user's username for web authentication in the database. - It requires :option:`--bb_password` to be set along with it. + Used with the :option:`update` option, this sets the user's username for web authentication in the database. + It requires :option:`bb_password` to be set along with it. --bb_password - Also used with the :option:`--op=update` option, this sets the password portion of a user's web authentication credentials into the database. + Also used with the :option:`update` option, this sets the password portion of a user's web authentication credentials into the database. The password is first encrypted prior to storage for security reasons. --ids When working with users, you need to be able to refer to them by unique identifiers to find particular users in the database. - The :option:`--ids` option lets you specify a comma separated list of these identifiers for use with the :command:`user` command. + The :option:`ids` option lets you specify a comma separated list of these identifiers for use with the :command:`user` command. - The :option:`--ids` option is used only when using :option:`--op=remove` or :option:`--op=get`. + The :option:`ids` option is used only when using :option:`remove` or :option:`show`. --info Users are known in buildbot as a collection of attributes tied together by some unique identifier (see :ref:`Concepts-Users`). - These attributes are specified in the form ``{TYPE}={VALUE}`` when using the :option:`--info` option. + These attributes are specified in the form ``{TYPE}={VALUE}`` when using the :option:`info` option. These ``{TYPE}={VALUE}`` pairs are specified in a comma separated list, so for example: .. code-block:: none --info=svn=jdoe,git='John Doe ' - The :option:`--info` option can be specified multiple times in the :command:`user` command, as each specified option will be interpreted as a new user. - Note that :option:`--info` is only used with :option:`--op=add` or with :option:`--op=update`, and whenever you use :option:`--op=update` you need to specify the identifier of the user you want to update. - This is done by prepending the :option:`--info` arguments with ``{ID:}``. + The :option:`info` option can be specified multiple times in the :command:`user` command, as each specified option will be interpreted as a new user. + Note that :option:`info` is only used with :option:`add` or with :option:`update`, and whenever you use :option:`update` you need to specify the identifier of the user you want to update. + This is done by prepending the :option:`info` arguments with ``{ID:}``. If we were to update ``'jschmo'`` from the previous example, it would look like this: .. code-block:: none @@ -486,13 +486,13 @@ For details on how Buildbot manages users, see :ref:`Concepts-Users`. Note that :option:`--master`, :option:`--username`, :option:`--passwd`, and :option:`--op` are always required to issue the :command:`user` command. -The :option:`--master`, :option:`--username`, and :option:`--passwd` options can be specified in the option file with keywords ``user_master``, ``user_username``, and ``user_passwd``, respectively. -If ``user_master`` is not specified, then :option:`--master` from the options file will be used instead. +The :option:`--master`, :option:`--username`, and :option:`--passwd` options can be specified in the option file with keywords :option:`user_master`, :option:`user_username`, and :option:`user_passwd`, respectively. +If :option:`user_master` is not specified, then :option:`master` from the options file will be used instead. Below are examples of how each command should look. Whenever a :command:`user` command is successful, results will be shown to whoever issued the command. -For :option:`--op=add`: +For :option:`add`: .. code-block:: none @@ -500,7 +500,7 @@ For :option:`--op=add`: --username={USER} --passwd={USERPW} \ --info={TYPE}={VALUE},... -For :option:`--op=update`: +For :option:`update`: .. code-block:: none @@ -508,7 +508,7 @@ For :option:`--op=update`: --username={USER} --passwd={USERPW} \ --info={ID}:{TYPE}={VALUE},... -For :option:`--op=remove`: +For :option:`remove`: .. code-block:: none @@ -516,7 +516,7 @@ For :option:`--op=remove`: --username={USER} --passwd={USERPW} \ --ids={ID1},{ID2},... -For :option:`--op=get`: +For :option:`get`: .. code-block:: none @@ -524,7 +524,7 @@ For :option:`--op=get`: --username={USER} --passwd={USERPW} \ --ids={ID1},{ID2},... -A note on :option:`--op=update`: when updating the :option:`--bb_username` and :option:`--bb_password`, the :option:`--info` doesn't need to have additional ``{TYPE}={VALUE}`` pairs to update and can just take the ``{ID}`` portion. +A note on :option:`update`: when updating the :option:`bb_username` and :option:`bb_password`, the :option:`info` doesn't need to have additional ``{TYPE}={VALUE}`` pairs to update and can just take the ``{ID}`` portion. .. _buildbot-config-directory: diff --git a/master/docs/relnotes/index.rst b/master/docs/relnotes/index.rst index 0345013af98..0f9ca458bd3 100644 --- a/master/docs/relnotes/index.rst +++ b/master/docs/relnotes/index.rst @@ -222,12 +222,6 @@ Changes and Removals - Configuring ``codebases`` is now mandatory, and the deprecated ``branch``, ``repository``, ``project``, ``revision`` are not supported anymore in ForceScheduler - :py:meth:`buildbot.schedulers.forcesched.BaseParameter.updateFromKwargs` now takes a ``collector`` parameter used to collect all validation errors -* :bb:sched:`Periodic`, :bb:sched:`Nightly` and :bb:sched:`NightlyTriggerable` have the following changes: - - - The ``Periodic`` and ``Nightly`` schedulers can now consume changes and use ``onlyIfChanged`` and ``createAbsoluteTimestamps``. - - All "timed" schedulers now handle ``codebases`` the same way. Configuring ``codebases`` is strongly recommended. - Using the ``branch`` parameter is discouraged. - * Logs are now stored as Unicode strings, and thus must be decoded properly from the bytestrings provided by shell commands. By default this encoding is assumed to be UTF-8, but the :bb:cfg:`logEncoding` parameter can be used to select an alternative. Steps and individual logfiles can also override the global default. @@ -290,8 +284,6 @@ Changes for Developers These methods are not yet documented, and their interface is not stable. Consult the source code for details on the changes. -* The ``preStartConsumingChanges`` and ``startTimedSchedulerService`` hooks have been removed. - * The triggerable schedulers` ``trigger`` method now requires a list of sourcestamps, rather than a dictionary. * The :py:class:`~buildbot.sourcestamp.SourceStamp` class is no longer used. diff --git a/master/docs/tutorial/docker.rst b/master/docs/tutorial/docker.rst index 84ce9ca35a6..269d165ec8a 100644 --- a/master/docs/tutorial/docker.rst +++ b/master/docs/tutorial/docker.rst @@ -4,17 +4,12 @@ First Buildbot run with Docker ============================== -.. warning:: - The instruction in this document are based on an *old* Dockerfile, not complying with the state-of-the-art best practices (all components in one container, access via ssh, ...). - This is not the recommended way to deploy a Buildbot farm. - However, it offers an easy way to get a first-hand experience. - Docker_ is a tool that makes building and deploying custom environments a breeze. It uses lightweight linux containers (LXC) and performs quickly, making it a great instrument for the testing community. The next section includes a Docker pre-flight check. If it takes more that 3 minutes to get the 'Success' message for you, try the Buildbot pip-based :ref:`first run ` instead. -.. _Docker: https://www.docker.com +.. _Docker: http://www.docker.io Current Docker dependencies --------------------------- @@ -32,7 +27,7 @@ Installation .. code-block:: bash - sudo docker run -i busybox /bin/echo Success + sudo docker run -i busybox /bin/echo Success Building and running Buildbot ----------------------------- @@ -43,10 +38,10 @@ Building and running Buildbot wget https://raw.github.com/buildbot/buildbot/master/master/contrib/Dockerfile # Build the Buildbot container (it will take a few minutes to download packages) - docker build -t buildbot - < Dockerfile + sudo docker build -t buildbot - < Dockerfile # Run buildbot - CONTAINER_ID=$(docker run -d -p 8010:8010 -p 22 buildbot) + CONTAINER_ID=$(sudo docker run -d -p 127.0.0.1:8010:8010 buildbot) You should now be able to go to http://localhost:8010 and see a web page similar to: @@ -64,11 +59,11 @@ Playing with your Buildbot container ------------------------------------ If you've come this far, you have a Buildbot environment that you can freely experiment with. -You can access your container using ssh, the password is ``admin``: +You can access your container using ssh (username: admin, password: admin): .. code-block:: bash - ssh -p $(docker port $CONTAINER_ID 22 | cut -d: -f 2) admin@localhost + ssh -p $(sudo docker port $CONTAINER_ID 22) admin@localhost You've got a taste now, but you're probably curious for more. diff --git a/www/base/guanlecoja/config.coffee b/www/base/guanlecoja/config.coffee index 541e1eb6458..af62fe5a285 100644 --- a/www/base/guanlecoja/config.coffee +++ b/www/base/guanlecoja/config.coffee @@ -3,7 +3,7 @@ # This module contains all configuration for the build process # ### ############################################################################################### -ANGULAR_TAG = "~1.3.1" +ANGULAR_TAG = "~1.2.17" gulp = require("gulp") shell = require("gulp-shell") path = require("path") @@ -30,7 +30,7 @@ config = # JavaScript libraries (order matters) deps: "guanlecoja-ui": - version: '~1.3.0' + version: '~1.2.0' files: ['vendors.js', 'scripts.js'] moment: version: "~2.6.0" diff --git a/www/base/src/app/layout.jade b/www/base/src/app/layout.jade index 1202291065f..9b714f3c6a8 100644 --- a/www/base/src/app/layout.jade +++ b/www/base/src/app/layout.jade @@ -7,7 +7,6 @@ html.no-js(xmlns:ng='http://angularjs.org', xmlns:app='ignored') meta(name='description', content='Buildbot web UI') meta(name='viewport', content='initial-scale=1, minimum-scale=1, user-scalable=no, maximum-scale=1, width=device-width') link(rel='stylesheet', href='styles.css') - link(rel='icon', href='img/favicon.ico') body(ng-cloak, ng-app="app") block content diff --git a/www/codeparameter/guanlecoja/config.coffee b/www/codeparameter/guanlecoja/config.coffee index 7790a51fe3f..553911b3917 100644 --- a/www/codeparameter/guanlecoja/config.coffee +++ b/www/codeparameter/guanlecoja/config.coffee @@ -3,7 +3,7 @@ # This module contains all configuration for the build process # ### ############################################################################################### -ANGULAR_TAG = "~1.3.0" +ANGULAR_TAG = "~1.2.0" gulp = require("gulp") gulp.task "copyace", -> diff --git a/www/console_view/guanlecoja/config.coffee b/www/console_view/guanlecoja/config.coffee index 4fc4ca6810c..ddd4e9f783d 100644 --- a/www/console_view/guanlecoja/config.coffee +++ b/www/console_view/guanlecoja/config.coffee @@ -3,7 +3,7 @@ # This module contains all configuration for the build process # ### ############################################################################################### -ANGULAR_TAG = "~1.3.0" +ANGULAR_TAG = "~1.2.0" module.exports = ### ########################################################################################### diff --git a/www/waterfall_view/guanlecoja/config.coffee b/www/waterfall_view/guanlecoja/config.coffee index bbeebe98752..c016b5b0b2e 100644 --- a/www/waterfall_view/guanlecoja/config.coffee +++ b/www/waterfall_view/guanlecoja/config.coffee @@ -3,7 +3,7 @@ # This module contains all configuration for the build process # ### ############################################################################################### -ANGULAR_TAG = "~1.3.0" +ANGULAR_TAG = "~1.2.0" module.exports = ### ###########################################################################################