diff --git a/buildbot/db.py b/buildbot/db.py index 31ef38538d4..cbf29c1bd1c 100644 --- a/buildbot/db.py +++ b/buildbot/db.py @@ -347,7 +347,7 @@ def get_sqlite_dbapi_name(): # don't use built-in sqlite3 on 2.5 -- it has *bad* bugs if sys.version_info >= (2,6): import sqlite3 - sqlite_dbapi_name = "pysqlite2.dbapi2" + sqlite_dbapi_name = "sqlite3" else: raise return sqlite_dbapi_name diff --git a/buildbot/master.py b/buildbot/master.py index 800ff890880..33d768fe4e5 100644 --- a/buildbot/master.py +++ b/buildbot/master.py @@ -11,18 +11,17 @@ from twisted.internet import defer, reactor from twisted.spread import pb from twisted.cred import portal, checkers -from twisted.application import service, strports, internet +from twisted.application import service, strports +from twisted.application.internet import TimerService import buildbot # sibling imports from buildbot.util import now, safeTranslate from buildbot.pbutil import NewCredPerspective from buildbot.process.builder import Builder, IDLE -from buildbot.buildrequest import BuildRequest from buildbot.status.builder import Status, BuildSetStatus from buildbot.changes.changes import Change from buildbot.changes.manager import ChangeManager -from buildbot.sourcestamp import SourceStamp from buildbot.buildslave import BuildSlave from buildbot import interfaces, locks from buildbot.process.properties import Properties @@ -383,6 +382,8 @@ def _avatarAttached(self, p, mind): # TCPServer(self.site) # UNIXServer(ResourcePublisher(self.site)) +class _Unset: pass # marker + class BuildMaster(service.MultiService): debug = 0 manhole = None @@ -437,6 +438,7 @@ def __init__(self, basedir, configFileName="master.cfg", db=None): self.db = None self.db_url = None + self.db_poll_interval = _Unset if db: self.loadDatabase(db) @@ -542,6 +544,7 @@ def loadConfig(self, f, check_synchronously_only=False): # optional db_url = config.get("db_url", "sqlite:///state.sqlite") + db_poll_interval = config.get("db_poll_interval", None) debugPassword = config.get('debugPassword') manhole = config.get('manhole') status = config.get('status', []) @@ -629,6 +632,10 @@ def loadConfig(self, f, check_synchronously_only=False): raise KeyError("c['interlocks'] is no longer accepted") assert self.db_url is None or db_url == self.db_url, \ "Cannot change db_url after master has started" + assert db_poll_interval is None or isinstance(db_poll_interval, int), \ + "db_poll_interval must be an integer: seconds between polls" + assert self.db_poll_interval is _Unset or db_poll_interval == self.db_poll_interval, \ + "Cannot change db_poll_interval after master has started" assert isinstance(change_sources, (list, tuple)) for s in change_sources: @@ -785,7 +792,8 @@ def loadConfig(self, f, check_synchronously_only=False): self.buildHorizon = buildHorizon # Set up the database - d.addCallback(lambda res: self.loadConfig_Database(db_url)) + d.addCallback(lambda res: + self.loadConfig_Database(db_url, db_poll_interval)) # self.slaves: Disconnect any that were attached and removed from the # list. Update self.checker with the new list of passwords, including @@ -851,7 +859,7 @@ def _done(res): d.addErrback(log.err) return d - def loadDatabase(self, db_spec): + def loadDatabase(self, db_spec, db_poll_interval=None): if self.db: return self.db = open_db(db_spec) @@ -868,15 +876,26 @@ def loadDatabase(self, db_spec): self.scheduler_manager = sm sm.setServiceParent(self) - # it'd be nice if TimerService let us set now=False - t = internet.TimerService(30, sm.trigger) - t.setServiceParent(self) - # should we try to remove this? Periodic() requires at least one kick - def loadConfig_Database(self, db_url): + # Set db_poll_interval (perhaps to 30 seconds) if you are using + # multiple buildmasters that share a common database, such that the + # masters need to discover what each other is doing by polling the + # database. TODO: this will be replaced by the DBNotificationServer. + if db_poll_interval: + # it'd be nice if TimerService let us set now=False + t1 = TimerService(db_poll_interval, sm.trigger) + t1.setServiceParent(self) + t2 = TimerService(db_poll_interval, self.botmaster.loop.trigger) + t2.setServiceParent(self) + # adding schedulers (like when loadConfig happens) will trigger the + # scheduler loop at least once, which we need to jump-start things + # like Periodic. + + def loadConfig_Database(self, db_url, db_poll_interval): self.db_url = db_url + self.db_poll_interval = db_poll_interval db_spec = DB.from_url(db_url, self.basedir) - self.loadDatabase(db_spec) + self.loadDatabase(db_spec, db_poll_interval) def loadConfig_Slaves(self, new_slaves): # set up the Checker with the names and passwords of all valid bots diff --git a/buildbot/process/builder.py b/buildbot/process/builder.py index a070992b11c..a55b64fcee1 100644 --- a/buildbot/process/builder.py +++ b/buildbot/process/builder.py @@ -7,10 +7,9 @@ from twisted.application import service, internet from twisted.internet import defer -from buildbot import interfaces, util, buildrequest +from buildbot import interfaces, util from buildbot.status.progress import Expectations from buildbot.status.builder import RETRY -from buildbot.process import base from buildbot.process.properties import Properties from buildbot.eventual import eventually diff --git a/buildbot/process/subunitlogobserver.py b/buildbot/process/subunitlogobserver.py index e188e568db2..50ab07c6d51 100644 --- a/buildbot/process/subunitlogobserver.py +++ b/buildbot/process/subunitlogobserver.py @@ -1,11 +1,8 @@ # -*- test-case-name: buildbot.test.test_buildstep -*- from unittest import TestResult - -from buildbot.steps.shell import ShellCommand from buildbot.process import buildstep - class DiscardStream: """A trivial thunk used to discard passthrough content.""" diff --git a/buildbot/schedulers/manager.py b/buildbot/schedulers/manager.py index af308191acf..2a1e8bb700b 100644 --- a/buildbot/schedulers/manager.py +++ b/buildbot/schedulers/manager.py @@ -39,6 +39,7 @@ from twisted.python import log from buildbot import loop from buildbot.util import defaultdict +from buildbot.eventual import eventually class SchedulerManager(loop.MultiServiceLoop): def __init__(self, master, db, change_svc): @@ -94,6 +95,7 @@ def _attach(ign): for s in list(self): if s.upstream_name: self.upstream_subscribers[s.upstream_name].append(s) + eventually(self.trigger) d.addCallback(_attach) d.addErrback(log.err) return d diff --git a/buildbot/schedulers/timed.py b/buildbot/schedulers/timed.py index ba7ef3bf442..b08a54540f6 100644 --- a/buildbot/schedulers/timed.py +++ b/buildbot/schedulers/timed.py @@ -99,13 +99,14 @@ def _run(self, t): if last_build is None: self.start_HEAD_build(t) self.update_last_build(t, now) - return None + last_build = now when = last_build + self.periodicBuildTimer if when < now: self.start_HEAD_build(t) self.update_last_build(t, now) - return None - return when - now + 1.0 + last_build = now + when = now + self.periodicBuildTimer + return when + 1.0 class Nightly(_Base, ClassifierMixin, TimedBuildMixin): diff --git a/buildbot/status/web/buildstatus.py b/buildbot/status/web/buildstatus.py index 94010fc94e8..528c6556ba4 100644 --- a/buildbot/status/web/buildstatus.py +++ b/buildbot/status/web/buildstatus.py @@ -33,8 +33,9 @@ def content(self, request, ctx): # Display each step, starting by the last one. for i in range(len(build.getSteps()) - 1, -1, -1): - if build.getSteps()[i].getText(): - rows.append(IBox(build.getSteps()[i]).getBox(request).td(align="center")) + step = build.getSteps()[i] + if step.isStarted() and step.getText(): + rows.append(IBox(step).getBox(request).td(align="center")) # Display the bottom box with the build number in it. ctx['build'] = IBox(build).getBox(request).td(align="center") diff --git a/buildbot/test/runs/test_db.py b/buildbot/test/runs/test_db.py index dba9a5761d9..48eb6936720 100644 --- a/buildbot/test/runs/test_db.py +++ b/buildbot/test/runs/test_db.py @@ -48,6 +48,7 @@ from buildbot.changes.manager import ChangeManager from buildbot.test.pollmixin import PollMixin from buildbot.test.runutils import RunMixin +from buildbot.eventual import flushEventualQueue class ShouldFailMixin: @@ -285,6 +286,7 @@ def _check4(ign): self.failUnlessEqual(all[0].name, "one") self.failUnlessEqual(all[1].builderNames, ["builder-two-other"]) d.addCallback(_check4) + d.addCallback(flushEventualQueue) return d def stall(self, res, timeout): diff --git a/buildbot/test/runutils.py b/buildbot/test/runutils.py index 2d6b8f0e234..5128839563f 100644 --- a/buildbot/test/runutils.py +++ b/buildbot/test/runutils.py @@ -73,7 +73,7 @@ class MasterMixin: master = None basedir = None slave_basedir = None - def create_master(self): + def create_master(self, **kwargs): assert not self.master, "you called create_master twice" # probably because you subclassed RunMixin instead of MasterMixin self.slaves = {} @@ -81,7 +81,7 @@ def create_master(self): self.basedir = self.mktemp() basedir = self.basedir os.makedirs(basedir) - self.master = master.BuildMaster(basedir) + self.master = master.BuildMaster(basedir, **kwargs) spec = db.DB.from_url("sqlite:///state.sqlite", basedir=basedir) db.create_db(spec) self.master.loadDatabase(spec) diff --git a/buildbot/test/unit/test_config.py b/buildbot/test/unit/test_config.py index d7e033e33e3..bdfa73d1a36 100644 --- a/buildbot/test/unit/test_config.py +++ b/buildbot/test/unit/test_config.py @@ -6,13 +6,13 @@ from twisted.python import failure from twisted.internet import defer -from buildbot.master import BuildMaster -from buildbot import scheduler, db -from buildbot.eventual import flushEventualQueue +from buildbot import scheduler from twisted.application import service, internet from twisted.spread import pb from twisted.web.server import Site from twisted.web.distrib import ResourcePublisher +from buildbot import db +from buildbot.master import BuildMaster from buildbot.process.builder import Builder from buildbot.process.factory import BasicBuildFactory, ArgumentsInTheWrongPlace from buildbot.changes.pb import PBChangeSource @@ -32,7 +32,7 @@ def startService(self): return except ImportError: pass -from buildbot.test.runutils import ShouldFailMixin, StallMixin +from buildbot.test.runutils import ShouldFailMixin, StallMixin, MasterMixin emptyCfg = \ """ @@ -410,35 +410,50 @@ def startService(self): c['db_url'] = "sqlite:///new.sqlite" """ +dbPollIntervalCfg = emptyCfg + \ +""" +c['db_poll_interval'] = 2*60 # two minutes +""" + +dbPollIntervalCfg2 = emptyCfg + \ +""" +c['db_poll_interval'] = 3*60 # three minutes +""" + class TestDBUrl(unittest.TestCase): + # a dburl of "sqlite:///.." can use either the third-party sqlite3 + # module, or the stdlib pysqlite2.dbapi2 module, depending upon the + # version of python in use + SQLITE_NAMES = ["sqlite3", "pysqlite2.dbapi2"] + def testSQLiteRelative(self): basedir = "/foo/bar" d = db.DB.from_url("sqlite:///state.sqlite", basedir=basedir) - self.failUnlessEqual(d.dbapiName, "pysqlite2.dbapi2") + self.failUnlessIn(d.dbapiName, self.SQLITE_NAMES) self.failUnlessEqual(d.connargs, (os.path.join(basedir, "state.sqlite"),)) def testSQLiteBasedir(self): basedir = "/foo/bar" d = db.DB.from_url("sqlite:///%(basedir)s/baz/state.sqlite", basedir=basedir) - self.failUnlessEqual(d.dbapiName, "pysqlite2.dbapi2") + self.failUnlessIn(d.dbapiName, self.SQLITE_NAMES) self.failUnlessEqual(d.connargs, (os.path.join(basedir, "baz/state.sqlite"),)) def testSQLiteAbsolute(self): basedir = "/foo/bar" d = db.DB.from_url("sqlite:////tmp/state.sqlite", basedir=basedir) - self.failUnlessEqual(d.dbapiName, "pysqlite2.dbapi2") + self.failUnlessIn(d.dbapiName, self.SQLITE_NAMES) self.failUnlessEqual(d.connargs, ("/tmp/state.sqlite",)) def testSQLiteMemory(self): basedir = "/foo/bar" d = db.DB.from_url("sqlite://", basedir=basedir) - self.failUnlessEqual(d.dbapiName, "pysqlite2.dbapi2") + self.failUnlessIn(d.dbapiName, self.SQLITE_NAMES) self.failUnlessEqual(d.connargs, (":memory:",)) def testSQLiteArgs(self): basedir = "/foo/bar" d = db.DB.from_url("sqlite:///state.sqlite?foo=bar", basedir=basedir) - self.failUnlessEqual(d.dbapiName, "pysqlite2.dbapi2") + self.failUnlessIn(d.dbapiName, self.SQLITE_NAMES) self.failUnlessEqual(d.connargs, (os.path.join(basedir, "state.sqlite"),)) self.failUnlessEqual(d.connkw, dict(foo="bar")) @@ -492,34 +507,11 @@ def testMysqlArgs(self): self.failUnlessEqual(d.connkw, dict(host="somehost.com", db="database_name", foo="bar")) -class SetupBase: +class ConfigTest(MasterMixin, unittest.TestCase, ShouldFailMixin, StallMixin): def setUp(self): # this class generates several deprecation warnings, which the user # doesn't need to see. warnings.simplefilter('ignore', exceptions.DeprecationWarning) - self.serviceparent = service.MultiService() - self.serviceparent.startService() - - def tearDown(self): - d = self.serviceparent.stopService() - d.addCallback(flushEventualQueue) - return d - - def setup_master(self): - if not os.path.exists(self.basedir): - os.makedirs(self.basedir) - spec = db.DB("sqlite3", os.path.join(self.basedir, "state.sqlite")) - db.create_db(spec) - self.buildmaster = BuildMaster(self.basedir, db=spec) - self.buildmaster.readConfig = True - self.buildmaster.setServiceParent(self.serviceparent) - -class ConfigTest(SetupBase, unittest.TestCase, ShouldFailMixin, StallMixin): - def setUp(self): - # this class generates several deprecation warnings, which the user - # doesn't need to see. - warnings.simplefilter('ignore', exceptions.DeprecationWarning) - SetupBase.setUp(self) def failUnlessListsEquivalent(self, list1, list2): l1 = list1[:] @@ -572,14 +564,14 @@ def checkPorts(self, svc, expected): def testEmpty(self): self.basedir = "config/configtest/empty" - self.setup_master() - self.failUnlessRaises(KeyError, self.buildmaster.loadConfig, "") + self.create_master() + self.failUnlessRaises(KeyError, self.master.loadConfig, "") def testSimple(self): # covers slavePortnum, base checker passwords self.basedir = "config/configtest/simple" - self.setup_master() - master = self.buildmaster + self.create_master() + master = self.master d = master.loadConfig(emptyCfg) def _check(ign): @@ -600,8 +592,8 @@ def _check(ign): def testSlavePortnum(self): self.basedir = "config/configtest/slave_portnum" - self.setup_master() - master = self.buildmaster + self.create_master() + master = self.master d = master.loadConfig(emptyCfg) def _check1(ign): @@ -632,8 +624,8 @@ def _check3(ign): def testSlaves(self): self.basedir = "config/configtest/slaves" - self.setup_master() - master = self.buildmaster + self.create_master() + master = self.master d = master.loadConfig(emptyCfg) def _check1(ign): self.failUnlessEqual(master.botmaster.builders, {}) @@ -679,8 +671,8 @@ def _check5(ign): def testChangeSource(self): self.basedir = "config/configtest/changesource" - self.setup_master() - master = self.buildmaster + self.create_master() + master = self.master d = master.loadConfig(emptyCfg) def _check0(ign): self.failUnlessEqual(list(master.change_svc), []) @@ -694,18 +686,18 @@ def _check0(ign): d.addCallback(lambda ign: master.loadConfig(sourcesCfg)) def _check1(res): - self.failUnlessEqual(len(list(self.buildmaster.change_svc)), 1) - s1 = list(self.buildmaster.change_svc)[0] + self.failUnlessEqual(len(list(self.master.change_svc)), 1) + s1 = list(self.master.change_svc)[0] self.failUnless(isinstance(s1, PBChangeSource)) - self.failUnlessEqual(s1, list(self.buildmaster.change_svc)[0]) + self.failUnlessEqual(s1, list(self.master.change_svc)[0]) self.failUnless(s1.parent) # verify that unchanged sources are not interrupted - d1 = self.buildmaster.loadConfig(sourcesCfg) + d1 = self.master.loadConfig(sourcesCfg) def _check2(res): - self.failUnlessEqual(len(list(self.buildmaster.change_svc)), 1) - s2 = list(self.buildmaster.change_svc)[0] + self.failUnlessEqual(len(list(self.master.change_svc)), 1) + s2 = list(self.master.change_svc)[0] self.failUnlessIdentical(s1, s2) self.failUnless(s1.parent) d1.addCallback(_check2) @@ -713,10 +705,10 @@ def _check2(res): d.addCallback(_check1) # make sure we can get rid of the sources too - d.addCallback(lambda res: self.buildmaster.loadConfig(emptyCfg)) + d.addCallback(lambda res: self.master.loadConfig(emptyCfg)) def _check3(res): - self.failUnlessEqual(list(self.buildmaster.change_svc), []) + self.failUnlessEqual(list(self.master.change_svc), []) d.addCallback(_check3) return d @@ -724,11 +716,11 @@ def _check3(res): def testChangeSources(self): # make sure we can accept a list self.basedir = "config/configtest/changesources" - self.setup_master() + self.create_master() maildir = os.path.join(self.basedir, "maildir") maildir_new = os.path.join(self.basedir, "maildir", "new") os.makedirs(maildir_new) - master = self.buildmaster + master = self.master d = master.loadConfig(emptyCfg) def _check0(ign): self.failUnlessEqual(list(master.change_svc), []) @@ -745,8 +737,8 @@ def _check0(ign): d.addCallback(lambda ign: master.loadConfig(sourcesCfg)) def _check1(res): - self.failUnlessEqual(len(list(self.buildmaster.change_svc)), 2) - s1,s2 = list(self.buildmaster.change_svc) + self.failUnlessEqual(len(list(self.master.change_svc)), 2) + s1,s2 = list(self.master.change_svc) if isinstance(s2, PBChangeSource): s1,s2 = s2,s1 self.failUnless(isinstance(s1, PBChangeSource)) @@ -759,8 +751,8 @@ def _check1(res): def testSources(self): # test backwards compatibility. c['sources'] is deprecated. self.basedir = "config/configtest/sources" - self.setup_master() - master = self.buildmaster + self.create_master() + master = self.master d = master.loadConfig(emptyCfg) def _check0(ign): self.failUnlessEqual(list(master.change_svc), []) @@ -774,8 +766,8 @@ def _check0(ign): d.addCallback(lambda ign: master.loadConfig(sourcesCfg)) def _check1(res): - self.failUnlessEqual(len(list(self.buildmaster.change_svc)), 1) - s1 = list(self.buildmaster.change_svc)[0] + self.failUnlessEqual(len(list(self.master.change_svc)), 1) + s1 = list(self.master.change_svc)[0] self.failUnless(isinstance(s1, PBChangeSource)) self.failUnless(s1.parent) d.addCallback(_check1) @@ -789,8 +781,8 @@ def shouldBeFailure(self, res, *expected): def testSchedulerErrors(self): self.basedir = "config/configtest/schedulererrors" - self.setup_master() - master = self.buildmaster + self.create_master() + master = self.master d = master.loadConfig(emptyCfg) def _check1(ign): self.failUnlessEqual(master.allSchedulers(), []) @@ -798,7 +790,7 @@ def _check1(ign): def _test(ign, cfg, which, err, substr): self.shouldFail(err, which, substr, - self.buildmaster.loadConfig, cfg) + self.master.loadConfig, cfg) # c['schedulers'] must be a list badcfg = schedulersCfg + \ @@ -864,17 +856,17 @@ def _test(ign, cfg, which, err, substr): def testSchedulers(self): self.basedir = "config/configtest/schedulers" - self.setup_master() - master = self.buildmaster + self.create_master() + master = self.master d = master.loadConfig(emptyCfg) d.addCallback(lambda ign: self.failUnlessEqual(master.allSchedulers(), [])) - d.addCallback(lambda ign: self.buildmaster.loadConfig(schedulersCfg)) + d.addCallback(lambda ign: self.master.loadConfig(schedulersCfg)) d.addCallback(self._testSchedulers_1) return d def _testSchedulers_1(self, res): - sch = self.buildmaster.allSchedulers() + sch = self.master.allSchedulers() self.failUnlessEqual(len(sch), 1) s = sch[0] self.failUnless(isinstance(s, Scheduler)) @@ -888,11 +880,11 @@ def _testSchedulers_1(self, res): s1 = Scheduler('full', None, 60, ['builder1']) c['schedulers'] = [s1, Dependent('downstream', s1, ['builder1'])] """ - d = self.buildmaster.loadConfig(newcfg) + d = self.master.loadConfig(newcfg) d.addCallback(self._testSchedulers_2, newcfg) return d def _testSchedulers_2(self, res, newcfg): - sch = self.buildmaster.allSchedulers() + sch = self.master.allSchedulers() self.failUnlessEqual(len(sch), 2) s = sch[0] self.failUnless(isinstance(s, scheduler.Scheduler)) @@ -902,18 +894,18 @@ def _testSchedulers_2(self, res, newcfg): self.failUnlessEqual(s.builderNames, ['builder1']) # reloading the same config file should leave the schedulers in place - d = self.buildmaster.loadConfig(newcfg) + d = self.master.loadConfig(newcfg) d.addCallback(self._testSchedulers_3, sch) return d def _testSchedulers_3(self, res, sch1): - sch2 = self.buildmaster.allSchedulers() + sch2 = self.master.allSchedulers() self.failUnlessEqual(len(sch2), 2) sch1.sort() sch2.sort() self.failUnlessEqual(sch1, sch2) self.failUnlessIdentical(sch1[0], sch2[0]) self.failUnlessIdentical(sch1[1], sch2[1]) - sm = self.buildmaster.scheduler_manager + sm = self.master.scheduler_manager self.failUnlessIdentical(sch1[0].parent, sm) self.failUnlessIdentical(sch1[1].parent, sm) @@ -921,8 +913,8 @@ def _testSchedulers_3(self, res, sch1): def testBuilders(self): self.basedir = "config/configtest/builders" - self.setup_master() - master = self.buildmaster + self.create_master() + master = self.master bm = master.botmaster d = master.loadConfig(emptyCfg) def _check1(ign): @@ -1013,8 +1005,8 @@ def _check7(ign): def testWithProperties(self): self.basedir = "config/configtest/withproperties" - self.setup_master() - master = self.buildmaster + self.create_master() + master = self.master d = master.loadConfig(wpCfg1) def _check1(ign): self.failUnlessEqual(master.botmaster.builderNames, ["builder1"]) @@ -1054,8 +1046,8 @@ def testIRC(self): if not words: raise unittest.SkipTest("Twisted Words package is not installed") self.basedir = "config/configtest/IRC" - self.setup_master() - master = self.buildmaster + self.create_master() + master = self.master d = master.loadConfig(emptyCfg) e1 = {} d.addCallback(lambda res: self.checkIRC(master, e1)) @@ -1078,21 +1070,21 @@ def testIRC(self): def testWebPortnum(self): self.basedir = "config/configtest/webportnum" - self.setup_master() - master = self.buildmaster + self.create_master() + master = self.master d = master.loadConfig(webCfg1) def _check1(res): - ports = self.checkPorts(self.buildmaster, + ports = self.checkPorts(self.master, [(9999, pb.PBServerFactory), (9980, Site)]) p = ports[1] self.p = p # nothing should be changed d.addCallback(_check1) - d.addCallback(lambda res: self.buildmaster.loadConfig(webCfg1)) + d.addCallback(lambda res: self.master.loadConfig(webCfg1)) def _check2(res): - ports = self.checkPorts(self.buildmaster, + ports = self.checkPorts(self.master, [(9999, pb.PBServerFactory), (9980, Site)]) self.failUnlessIdentical(self.p, ports[1], "web port was changed even though " @@ -1101,65 +1093,65 @@ def _check2(res): # rebuilt on each reconfig #d.addCallback(_check2) - d.addCallback(lambda res: self.buildmaster.loadConfig(webCfg2)) + d.addCallback(lambda res: self.master.loadConfig(webCfg2)) # changes port to 9981 def _check3(p): - ports = self.checkPorts(self.buildmaster, + ports = self.checkPorts(self.master, [(9999, pb.PBServerFactory), (9981, Site)]) self.failIf(self.p is ports[1], "configuration was changed but web port was unchanged") d.addCallback(_check3) - d.addCallback(lambda res: self.buildmaster.loadConfig(webCfg3)) + d.addCallback(lambda res: self.master.loadConfig(webCfg3)) # make 9981 on only localhost def _check4(p): - ports = self.checkPorts(self.buildmaster, + ports = self.checkPorts(self.master, [(9999, pb.PBServerFactory), (9981, Site)]) self.failUnlessEqual(ports[1].kwargs['interface'], "127.0.0.1") d.addCallback(_check4) - d.addCallback(lambda res: self.buildmaster.loadConfig(emptyCfg)) + d.addCallback(lambda res: self.master.loadConfig(emptyCfg)) d.addCallback(lambda res: - self.checkPorts(self.buildmaster, + self.checkPorts(self.master, [(9999, pb.PBServerFactory)])) return d def testWebPathname(self): self.basedir = "config/configtest/webpathname" - self.setup_master() - master = self.buildmaster + self.create_master() + master = self.master d = master.loadConfig(webNameCfg1) def _check1(res): - self.checkPorts(self.buildmaster, + self.checkPorts(self.master, [(9999, pb.PBServerFactory), ('./foo.socket', pb.PBServerFactory)]) - unixports = self.UNIXports(self.buildmaster) + unixports = self.UNIXports(self.master) self.f = f = unixports[0].args[1] self.failUnless(isinstance(f.root, ResourcePublisher)) d.addCallback(_check1) - d.addCallback(lambda res: self.buildmaster.loadConfig(webNameCfg2)) + d.addCallback(lambda res: self.master.loadConfig(webNameCfg2)) def _check2(res): - self.checkPorts(self.buildmaster, + self.checkPorts(self.master, [(9999, pb.PBServerFactory), ('./bar.socket', pb.PBServerFactory)]) - newf = self.UNIXports(self.buildmaster)[0].args[1], + newf = self.UNIXports(self.master)[0].args[1], self.failIf(self.f is newf, "web factory was unchanged but " "configuration was changed") d.addCallback(_check2) - d.addCallback(lambda res: self.buildmaster.loadConfig(emptyCfg)) + d.addCallback(lambda res: self.master.loadConfig(emptyCfg)) d.addCallback(lambda res: - self.checkPorts(self.buildmaster, + self.checkPorts(self.master, [(9999, pb.PBServerFactory)])) return d def testDebugPassword(self): self.basedir = "config/configtest/debugpassword" - self.setup_master() - master = self.buildmaster + self.create_master() + master = self.master d = master.loadConfig(debugPasswordCfg) d.addCallback(lambda ign: @@ -1181,8 +1173,8 @@ def testDebugPassword(self): def testLocks(self): self.basedir = "config/configtest/locks" - self.setup_master() - master = self.buildmaster + self.create_master() + master = self.master botmaster = master.botmaster d = defer.succeed(None) sf = self.shouldFail @@ -1251,8 +1243,8 @@ def _check7(ign): def testNoChangeHorizon(self): self.basedir = "config/configtest/NoChangeHorizon" - self.setup_master() - master = self.buildmaster + self.create_master() + master = self.master sourcesCfg = emptyCfg + \ """ from buildbot.changes.pb import PBChangeSource @@ -1260,15 +1252,15 @@ def testNoChangeHorizon(self): """ d = master.loadConfig(sourcesCfg) def _check1(res): - self.failUnlessEqual(len(list(self.buildmaster.change_svc)), 1) - self.failUnlessEqual(self.buildmaster.change_svc.changeHorizon, 0) + self.failUnlessEqual(len(list(self.master.change_svc)), 1) + self.failUnlessEqual(self.master.change_svc.changeHorizon, 0) d.addCallback(_check1) return d def testChangeHorizon(self): self.basedir = "config/configtest/ChangeHorizon" - self.setup_master() - master = self.buildmaster + self.create_master() + master = self.master sourcesCfg = emptyCfg + \ """ from buildbot.changes.pb import PBChangeSource @@ -1277,43 +1269,83 @@ def testChangeHorizon(self): """ d = master.loadConfig(sourcesCfg) def _check1(res): - self.failUnlessEqual(len(list(self.buildmaster.change_svc)), 1) - self.failUnlessEqual(self.buildmaster.change_svc.changeHorizon, 5) + self.failUnlessEqual(len(list(self.master.change_svc)), 1) + self.failUnlessEqual(self.master.change_svc.changeHorizon, 5) d.addCallback(_check1) return d def testDBUrl(self): self.basedir = "config/configtest/DBUrl" - if not os.path.exists(self.basedir): - os.makedirs(self.basedir) - self.buildmaster = BuildMaster(self.basedir) - self.buildmaster.readConfig = True - self.buildmaster.setServiceParent(self.serviceparent) + self.slaves = {} + os.makedirs(self.basedir) + self.master = BuildMaster(self.basedir) + self.master.readConfig = True + self.master.startService() spec = db.DB.from_url("sqlite:///orig.sqlite", basedir=self.basedir) db.create_db(spec) - d = self.buildmaster.loadConfig(dburlCfg) + d = self.master.loadConfig(dburlCfg) def _check(ign): - self.failUnlessEqual(self.buildmaster.db_url, "sqlite:///orig.sqlite") + self.failUnlessEqual(self.master.db_url, "sqlite:///orig.sqlite") + timers = self.get_timers() + self.failIf(self.get_timers(), + "default should have no timers, but: %s" % timers) d.addCallback(_check) return d def testDBUrlChange(self): self.basedir = "config/configtest/DBUrlChange" - if not os.path.exists(self.basedir): - os.makedirs(self.basedir) - self.buildmaster = BuildMaster(self.basedir) - self.buildmaster.readConfig = True - self.buildmaster.setServiceParent(self.serviceparent) + self.slaves = {} + os.makedirs(self.basedir) + self.master = BuildMaster(self.basedir) + self.master.readConfig = True + self.master.startService() spec = db.DB.from_url("sqlite:///orig.sqlite", basedir=self.basedir) db.create_db(spec) - d = self.buildmaster.loadConfig(dburlCfg) + d = self.master.loadConfig(dburlCfg) def _check(ign): - self.failUnlessEqual(self.buildmaster.db_url, "sqlite:///orig.sqlite") + self.failUnlessEqual(self.master.db_url, "sqlite:///orig.sqlite") d.addCallback(_check) d.addCallback(lambda ign: self.shouldFail(AssertionError, "loadConfig", "Cannot change db_url after master has started", - self.buildmaster.loadConfig, dburlCfg1)) + self.master.loadConfig, dburlCfg1)) + return d + + def get_timers(self): + return [s for s in list(self.master) if isinstance(s, internet.TimerService)] + + def testDBPollInterval(self): + self.basedir = "config/configtest/DBPollInterval" + self.slaves = {} + os.makedirs(self.basedir) + spec = db.DB.from_url("sqlite:///state.sqlite", basedir=self.basedir) + db.create_db(spec) + self.master = BuildMaster(self.basedir) + self.master.readConfig = True + self.master.startService() + d = self.master.loadConfig(dbPollIntervalCfg) + def _check(ign): + self.failUnless(len(self.get_timers()) >= 2, list(self.master)) + d.addCallback(_check) + return d + + def testDBPollIntervalChange(self): + self.basedir = "config/configtest/DBPollIntervalChange" + self.slaves = {} + os.makedirs(self.basedir) + spec = db.DB.from_url("sqlite:///state.sqlite", basedir=self.basedir) + db.create_db(spec) + self.master = BuildMaster(self.basedir) + self.master.readConfig = True + self.master.startService() + d = self.master.loadConfig(dbPollIntervalCfg) + def _check(ign): + self.failUnless(len(self.get_timers()) >= 2, list(self.master)) + d.addCallback(_check) + + d.addCallback(lambda ign: self.shouldFail(AssertionError, "loadConfig", + "Cannot change db_poll_interval after master has started", + self.master.loadConfig, dbPollIntervalCfg2)) return d class ConfigElements(unittest.TestCase): @@ -1342,37 +1374,26 @@ def testSchedulers(self): self.failUnless(s3a in [s1, s2, s3]) - -class ConfigFileTest(SetupBase, unittest.TestCase): - - def testFindConfigFile(self): - self.basedir = "config/configfiletest/FindConfigFile" - self.setup_master() - +class ConfigFileTest(MasterMixin, unittest.TestCase): + def testFindConfigFile1(self): + self.basedir = "config/configfiletest/FindConfigFile1" + self.create_master() open(os.path.join(self.basedir, "master.cfg"), "w").write(emptyCfg) + d = self.master.loadTheConfigFile() + def _check(ign): + self.failUnlessEqual(self.master.slavePortnum, "tcp:9999") + d.addCallback(_check) + return d - d = self.buildmaster.loadTheConfigFile() + def testFindConfigFile2(self): + self.basedir = "config/configfiletest/FindConfigFile2" + self.create_master(configFileName="alternate.cfg") + cfg2 = emptyCfg + "c['slavePortnum'] = 9000\n" + open(os.path.join(self.basedir, "alternate.cfg"), "w").write(cfg2) + d = self.master.loadTheConfigFile() def _check(ign): - self.failUnlessEqual(self.buildmaster.slavePortnum, "tcp:9999") + self.failUnlessEqual(self.master.slavePortnum, "tcp:9000") d.addCallback(_check) - d.addCallback(lambda ign: self.buildmaster.disownServiceParent()) - def _setup2(ign): - basedir2 = "config/configfiletest/FindConfigFile_2" - if not os.path.exists(basedir2): - os.makedirs(basedir2) - spec = db.DB("sqlite3", os.path.join(basedir2, "state.sqlite")) - db.create_db(spec) - slaveportCfg = emptyCfg + "c['slavePortnum'] = 9000\n" - open(os.path.join(basedir2, "alternate.cfg"), "w").write(slaveportCfg) - m = BuildMaster(basedir2, "alternate.cfg", db=spec) - self.buildmaster = m - m.readConfig = True - m.setServiceParent(self.serviceparent) - return m.loadTheConfigFile() - d.addCallback(_setup2) - def _check2(ign): - self.failUnlessEqual(self.buildmaster.slavePortnum, "tcp:9000") - d.addCallback(_check2) return d @@ -1425,21 +1446,12 @@ def finishedStalling(res): c['status'] = [MySlowTarget('b')] """ -class StartService(unittest.TestCase): - def tearDown(self): - return self.master.stopService() +class StartService(MasterMixin, unittest.TestCase): def testStartService(self): self.basedir = "config/startservice/startservice" - if not os.path.exists(self.basedir): - os.makedirs(self.basedir) - spec = db.DB("sqlite3", os.path.join(self.basedir, "state.sqlite")) - db.create_db(spec) - self.master = m = BuildMaster(self.basedir, db=spec) - # inhibit the usual read-config-on-startup behavior - m.readConfig = True - m.startService() - d = m.loadConfig(startableEmptyCfg % 0) + self.create_master() + d = self.master.loadConfig(startableEmptyCfg % 0) d.addCallback(self._testStartService_0) return d @@ -1536,7 +1548,7 @@ def _testStartService_4(self, res): f1.addStep(BuildFactory()) # pass addStep something that's not a step or step class """ -class Factories(SetupBase, unittest.TestCase, ShouldFailMixin): +class Factories(MasterMixin, unittest.TestCase, ShouldFailMixin): def printExpecting(self, factory, args): factory_keys = factory[1].keys() factory_keys.sort() @@ -1586,8 +1598,8 @@ def failUnlessExpectedDarcs(self, factory, **kwargs): def testSteps(self): self.basedir = "config/factories/steps" - self.setup_master() - m = self.buildmaster + self.create_master() + m = self.master d = m.loadConfig(cfg1) def _check1(ign): b = m.botmaster.builders["builder1"] @@ -1606,8 +1618,8 @@ def _check1(ign): def testBadAddStepArguments(self): self.basedir = "config/factories/BadAddStepArguments" - self.setup_master() - m = self.buildmaster + self.create_master() + m = self.master d = self.shouldFail(ArgumentsInTheWrongPlace, "here", None, m.loadConfig, cfg1_bad) return d diff --git a/buildbot/test/unit/test_scheduler.py b/buildbot/test/unit/test_scheduler.py index 311cebedf04..7f75c50af98 100644 --- a/buildbot/test/unit/test_scheduler.py +++ b/buildbot/test/unit/test_scheduler.py @@ -18,20 +18,26 @@ def setSchedulers(self, *schedulers): def testPeriodic1(self): self.basedir = 'scheduler/Scheduling/testPeriodic1' self.create_master() + # updateSchedulers itself fires the first trigger d = self.setSchedulers(scheduler.Periodic("quickly", ["a","b"], 2)) - d.addCallback(lambda ign: self.master.scheduler_manager.trigger()) - d.addCallback(self.stall, 2) - d.addCallback(lambda ign: self.master.scheduler_manager.when_quiet()) + # there should be one buildset submitted at startup, and another a + # few seconds later. We wait until we've seen at least two. + def _pollf(): + bsids = self.master.db.get_active_buildset_ids() + return bool(len(bsids) > 1) + d.addCallback(lambda ign: self.poll(_pollf, 0.1)) d.addCallback(self._testPeriodic1_1) return d def _testPeriodic1_1(self, res): bsids = self.master.db.get_active_buildset_ids() self.failUnless(len(bsids) > 1) - (external_idstring, reason, ssid, complete, results) = self.master.db.get_buildset_info(bsids[0]) + (external_idstring, reason, ssid, + complete, results) = self.master.db.get_buildset_info(bsids[0]) reqs = self.master.db.get_buildrequestids_for_buildset(bsids[0]) self.failUnlessEqual(sorted(reqs.keys()), ["a","b"]) self.failUnlessEqual(reason, "The Periodic scheduler named 'quickly' triggered this build") + testPeriodic1.timeout = 20 def testNightly(self): # now == 15-Nov-2005, 00:05:36 AM . By using mktime, this is diff --git a/docs/cfg-buildsteps.texinfo b/docs/cfg-buildsteps.texinfo index efd6d0b6b96..8657d91fe39 100644 --- a/docs/cfg-buildsteps.texinfo +++ b/docs/cfg-buildsteps.texinfo @@ -501,8 +501,7 @@ like some build outputs that can be reused between builds. svn:ignore properties and global-ignores in subversion/config. @item always_purge -(optional): if set to True, always purge local changes after each -build. This is everything that would appear in a @code{svn status}. +(optional): if set to True, always purge local changes before updating. This deletes unversioned files and reverts everything that would appear in a @code{svn status}. @item depth (optional): Specify depth argument to achieve sparse checkout. Only available if slave has Subversion 1.5 or higher.