Refactor database loading code so that it can be specified via

master.cfg

buildbot/master.py

@@ -28,7 +28,7 @@
 from buildbot.process.properties import Properties
 from buildbot.config import BuilderConfig
 from buildbot.process.builder import BuilderControl
-from buildbot.db import open_db, DB # DB is used by buildbot.tac
+from buildbot.db import open_db, DB
 from buildbot.schedulers.manager import SchedulerManager
 from buildbot.loop import DelegateLoop
 
@@ -43,9 +43,8 @@ class BotMaster(service.MultiService):
 
     debug = 0
 
-    def __init__(self, db):
+    def __init__(self):
         service.MultiService.__init__(self)
-        self.db = db
         self.builders = {}
         self.builderNames = []
         # builders maps Builder names to instances of bb.p.builder.Builder,
@@ -399,9 +398,6 @@ def __init__(self, basedir, configFileName="master.cfg", db=None):
         self.setName("buildmaster")
         self.basedir = basedir
         self.configFileName = configFileName
-        if db is None:
-            db = DB("sqlite3", os.path.join(basedir, "state.sqlite"))
-        self.db = open_db(db)
 
         # the dispatcher is the realm in which all inbound connections are
         # looked up: slave builders, change notifications, status clients, and
@@ -429,27 +425,21 @@ def __init__(self, basedir, configFileName="master.cfg", db=None):
             hostname = "?"
         self.master_name = "%s:%s" % (hostname, os.path.abspath(self.basedir))
         self.master_incarnation = "pid%d-boot%d" % (os.getpid(), time.time())
-        self.botmaster = BotMaster(self.db)
-        self.db.subscribe_to("add-buildrequest",
-                             self.botmaster.trigger_add_buildrequest)
+
+        self.botmaster = BotMaster()
         self.botmaster.setName("botmaster")
         self.botmaster.setMasterName(self.master_name, self.master_incarnation)
         self.botmaster.setServiceParent(self)
-        dispatcher.botmaster = self.botmaster
+        self.dispatcher.botmaster = self.botmaster
 
-        sm = SchedulerManager(self, self.db, self.change_svc)
-        self.db.subscribe_to("add-change", sm.trigger_add_change)
-        self.db.subscribe_to("modify-buildset", sm.trigger_modify_buildset)
-        self.scheduler_manager = sm
-        sm.setServiceParent(self)
-        # it'd be nice if TimerService let us set now=False
-        t = internet.TimerService(30, sm.trigger)
-        t.setServiceParent(self)
-        # should we try to remove this? Periodic() requires at least one kick
-
-        self.status = Status(self.botmaster, self.db, self.basedir)
+        self.status = Status(self.botmaster, self.basedir)
         self.statusTargets = []
 
+        self.db = None
+        self.db_url = None
+        if db:
+            self.loadDatabase(db)
+
         self.readConfig = False
 
     def startService(self):
@@ -536,7 +526,7 @@ def loadConfig(self, f, check_synchronously_only=False):
                       "buildbotURL", "properties", "prioritizeBuilders",
                       "eventHorizon", "buildCacheSize", "logHorizon", "buildHorizon",
                       "changeHorizon", "logMaxSize", "logMaxTailSize",
-                      "logCompressionMethod",
+                      "logCompressionMethod", "db_url",
                       )
         for k in config.keys():
             if k not in known_keys:
@@ -551,6 +541,7 @@ def loadConfig(self, f, check_synchronously_only=False):
             #change_source = config['change_source']
 
             # optional
+            db_url = config.get("db_url", "sqlite:///state.sqlite")
             debugPassword = config.get('debugPassword')
             manhole = config.get('manhole')
             status = config.get('status', [])
@@ -791,6 +782,10 @@ def loadConfig(self, f, check_synchronously_only=False):
         self.logHorizon = logHorizon
         self.buildHorizon = buildHorizon
 
+        # Set up the database
+        # TODO: Warn if this changes
+        d.addCallback(lambda res: self.loadConfig_Database(db_url))
+
         # self.slaves: Disconnect any that were attached and removed from the
         # list. Update self.checker with the new list of passwords, including
         # debug/change/status.
@@ -855,6 +850,34 @@ def _done(res):
         d.addErrback(log.err)
         return d
 
+    def loadDatabase(self, db_spec):
+        if self.db:
+            return
+        self.db = open_db(db_spec)
+
+        self.botmaster.db = self.db
+        self.status.setDB(self.db)
+
+        self.db.subscribe_to("add-buildrequest",
+                             self.botmaster.trigger_add_buildrequest)
+
+        sm = SchedulerManager(self, self.db, self.change_svc)
+        self.db.subscribe_to("add-change", sm.trigger_add_change)
+        self.db.subscribe_to("modify-buildset", sm.trigger_modify_buildset)
+
+        self.scheduler_manager = sm
+        sm.setServiceParent(self)
+        # it'd be nice if TimerService let us set now=False
+        t = internet.TimerService(30, sm.trigger)
+        t.setServiceParent(self)
+        # should we try to remove this? Periodic() requires at least one kick
+
+    def loadConfig_Database(self, db_url):
+        assert self.db_url is None or db_url == self.db_url, "Cannot change db_url after master has started"
+        self.db_url = db_url
+        db_spec = DB.from_url(db_url, self.basedir)
+        self.loadDatabase(db_spec)
+
     def loadConfig_Slaves(self, new_slaves):
         # set up the Checker with the names and passwords of all valid bots
         self.checker.users = {} # violates abstraction, oh well

buildbot/status/builder.py

@@ -2200,7 +2200,7 @@ class Status:
     """
     implements(interfaces.IStatus)
 
-    def __init__(self, botmaster, db, basedir):
+    def __init__(self, botmaster, basedir):
         """
         @type  botmaster: L{buildbot.master.BotMaster}
         @param botmaster: the Status object uses C{.botmaster} to get at
@@ -2215,7 +2215,7 @@ def __init__(self, botmaster, db, basedir):
                         pickles) can be stored
         """
         self.botmaster = botmaster
-        self.db = db
+        self.db = None
         self.basedir = basedir
         self.watchers = []
         assert os.path.isdir(basedir)
@@ -2227,11 +2227,13 @@ def __init__(self, botmaster, db, basedir):
         self.logMaxTailSize = None
 
         self._buildreq_observers = util.DictOfSets()
-        self.db.subscribe_to("add-build", self._db_builds_changed)
         self._buildset_success_waiters = util.DictOfSets()
         self._buildset_finished_waiters = util.DictOfSets()
-        self.db.subscribe_to("modify-buildset", self._db_buildsets_changed)
 
+    def setDB(self, db):
+        self.db = db
+        self.db.subscribe_to("add-build", self._db_builds_changed)
+        self.db.subscribe_to("modify-buildset", self._db_buildsets_changed)
 
     # methods called by our clients
 

buildbot/test/runs/test_locks.py

@@ -304,7 +304,7 @@ def testGet(self):
         # BotMaster.locks, but they're small and shouldn't really cause a
         # problem.
 
-        b = master.BotMaster(None)
+        b = master.BotMaster()
         l1 = locks.MasterLock("one")
         l1a = locks.MasterLock("one")
         l2 = locks.MasterLock("one", maxCount=4)

buildbot/test/runutils.py

@@ -81,9 +81,10 @@ def create_master(self):
             self.basedir = self.mktemp()
         basedir = self.basedir
         os.makedirs(basedir)
-        spec = db.DB("sqlite3", os.path.join(basedir, "state.sqlite"))
+        self.master = master.BuildMaster(basedir)
+        spec = db.DB.from_url("sqlite:///state.sqlite", basedir=basedir)
         db.create_db(spec)
-        self.master = master.BuildMaster(basedir, db=spec)
+        self.master.loadDatabase(spec)
         self.master.readConfig = True
         self.master.startService()
         self.status = self.master.getStatus()
@@ -293,7 +294,7 @@ def setupBuildStepStatus(basedir):
     use."""
     os.mkdir(basedir)
     botmaster = None
-    s0 = builder.Status(botmaster, FakeDB(), basedir)
+    s0 = builder.Status(botmaster, basedir)
     s1 = s0.builderAdded("buildername", "buildername")
     s2 = builder.BuildStatus(s1, 1)
     s3 = builder.BuildStepStatus(s2)

docs/cfg-global.texinfo

@@ -1,6 +1,7 @@
 The keys in this section affect the operations of the buildmaster globally.
 
 @menu
+* Database Specification::
 * Project Definitions::
 * Log Handling::
 * Data Lifetime::
@@ -11,6 +12,26 @@ The keys in this section affect the operations of the buildmaster globally.
 * Debug Options::
 @end menu
 
+@node Database Specification
+@subsection Database Specification
+
+Buildbot requires a connection to a database to maintain certain state
+information, such as tracking pending build requests.  By default this is
+stored in a sqlite file called 'state.sqlite' in the base directory of your
+master.  This can be overridden with the @code{db_url} parameter.
+
+This parameter is of the form:
+
+driver://[username:password@@]host:port/database[?args]
+
+For sqlite databases, since there is no host and port, relative paths are
+specified with @code{sqlite:///} and absolute paths with @code{sqlite:////}
+
+@example
+c['db_url'] = "sqlite:///state.sqlite"
+c['db_url'] = "mysql://user:pass@@somehost.com/database_name"
+@end example
+
 @node Project Definitions
 @subsection Project Definitions