Merge remote branch 'dustin/master'

Merged catlee's c['db_url'] work with my test fixes and loop-trigger changes.

buildbot/changes/svnpoller.py

@@ -20,10 +20,6 @@ def _assert(condition, msg):
         return True
     raise AssertionError(msg)
 
-def dbgMsg(myString):
-    log.msg(myString)
-    return 1
-
 # these split_file_* functions are available for use as values to the
 # split_file= argument.
 def split_file_alwaystrunk(path):
@@ -292,7 +288,6 @@ def determine_prefix(self, output):
         try:
             doc = xml.dom.minidom.parseString(output)
         except xml.parsers.expat.ExpatError:
-            dbgMsg("_process_changes: ExpatError in %s" % output)
             log.msg("SVNPoller._determine_prefix_2: ExpatError in '%s'"
                     % output)
             raise
@@ -331,8 +326,7 @@ def parse_logs(self, output):
         try:
             doc = xml.dom.minidom.parseString(output)
         except xml.parsers.expat.ExpatError:
-            dbgMsg("_process_changes: ExpatError in %s" % output)
-            log.msg("SVNPoller._parse_changes: ExpatError in '%s'" % output)
+            log.msg("SVNPoller.parse_logs: ExpatError in '%s'" % output)
             raise
         logentries = doc.getElementsByTagName("logentry")
         return logentries
@@ -411,7 +405,7 @@ def create_changes(self, new_logentries):
                 if revision:
                     revlink = self.revlinktmpl % urllib.quote_plus(revision)
 
-            dbgMsg("Adding change revision %s" % (revision,))
+            log.msg("Adding change revision %s" % (revision,))
             # TODO: the rest of buildbot may not be ready for unicode 'who'
             # values
             author   = self._get_text(el, "author")
@@ -473,15 +467,13 @@ def submit_changes(self, changes):
             self.parent.addChange(c)
 
     def finished_ok(self, res):
-        log.msg("SVNPoller finished polling")
-        dbgMsg('_finished : %s' % res)
+        log.msg("SVNPoller finished polling %s" % res)
         assert self.working
         self.working = False
         return res
 
     def finished_failure(self, f):
-        log.msg("SVNPoller failed")
-        dbgMsg('_finished : %s' % f)
+        log.msg("SVNPoller failed %s" % f)
         assert self.working
         self.working = False
         return None # eat the failure

buildbot/db.py

@@ -20,6 +20,7 @@
 #
 # Contributor(s):
 #   Brian Warner <warner@lothar.com>
+#   Chris AtLee <catlee@mozilla.com>
 #
 # Alternatively, the contents of this file may be used under the terms of
 # either the GNU General Public License Version 2 or later (the "GPL"), or
@@ -35,7 +36,7 @@
 #
 # ***** END LICENSE BLOCK *****
 
-import sys, time, collections, base64, textwrap
+import sys, time, collections, base64, textwrap, os, cgi, re
 
 try:
     import simplejson
@@ -281,6 +282,61 @@ def __init__(self, dbapiName, *connargs, **connkw):
         self.connargs = connargs
         self.connkw = connkw
 
+    @classmethod
+    def from_url(cls, url, basedir=None):
+        """Parses a URL of the format
+        driver://[username:password@]host:port/database[?args]
+
+        and returns a DB object representing this URL
+        """
+        match = re.match(r"""
+        ^(?P<driver>\w+)://
+        (
+            ((?P<user>\w+)(:(?P<passwd>\S+))?@)?
+            ((?P<host>[-A-Za-z0-9.]+)(:(?P<port>\d+))?)?/
+            (?P<database>\S+?)(\?(?P<args>.*))?
+        )?$""", url, re.X)
+        if not match:
+            raise ValueError("Malformed url")
+
+        d = match.groupdict()
+        driver = d['driver']
+        user = d['user']
+        passwd = d['passwd']
+        host = d['host']
+        port = d['port']
+        if port is not None:
+            port = int(port)
+        database = d['database']
+        args = {}
+        if d['args']:
+            for key, value in cgi.parse_qsl(d['args']):
+                args[key] = value
+
+        if driver == "sqlite":
+            # user, passwd, host, and port must all be None
+            if not user == passwd == host == port == None:
+                raise ValueError("user, passwd, host, port must all be None")
+            if not database:
+                database = ":memory:"
+            else:
+                database = database % dict(basedir=basedir)
+                database = os.path.join(basedir, database)
+            return cls("sqlite3", database, **args)
+        elif driver == "mysql":
+            args['host'] = host
+            args['db'] = database
+            if user:
+                args['user'] = user
+            if passwd:
+                args['passwd'] = passwd
+            if port:
+                args['port'] = port
+
+            return cls("MySQLdb", **args)
+        else:
+            raise ValueError("Unsupported dbapi %s" % driver)
+
 def get_sqlite_dbapi_name():
     # see which dbapi we can use, and import it as 'buildbot.db.sqlite3'
     sqlite_dbapi_name = None

buildbot/master.py

@@ -26,7 +26,7 @@
 from buildbot.process.properties import Properties
 from buildbot.config import BuilderConfig
 from buildbot.process.builder import BuilderControl
-from buildbot.db import open_db, DB # DB is used by buildbot.tac
+from buildbot.db import open_db, DB
 from buildbot.schedulers.manager import SchedulerManager
 from buildbot.loop import DelegateLoop
 
@@ -41,9 +41,8 @@ class BotMaster(service.MultiService):
 
     debug = 0
 
-    def __init__(self, db):
+    def __init__(self):
         service.MultiService.__init__(self)
-        self.db = db
         self.builders = {}
         self.builderNames = []
         # builders maps Builder names to instances of bb.p.builder.Builder,
@@ -397,9 +396,6 @@ def __init__(self, basedir, configFileName="master.cfg", db=None):
         self.setName("buildmaster")
         self.basedir = basedir
         self.configFileName = configFileName
-        if db is None:
-            db = DB("sqlite3", os.path.join(basedir, "state.sqlite"))
-        self.db = open_db(db)
 
         # the dispatcher is the realm in which all inbound connections are
         # looked up: slave builders, change notifications, status clients, and
@@ -427,37 +423,21 @@ def __init__(self, basedir, configFileName="master.cfg", db=None):
             hostname = "?"
         self.master_name = "%s:%s" % (hostname, os.path.abspath(self.basedir))
         self.master_incarnation = "pid%d-boot%d" % (os.getpid(), time.time())
-        self.botmaster = BotMaster(self.db)
-        self.db.subscribe_to("add-buildrequest",
-                             self.botmaster.trigger_add_buildrequest)
+
+        self.botmaster = BotMaster()
         self.botmaster.setName("botmaster")
         self.botmaster.setMasterName(self.master_name, self.master_incarnation)
         self.botmaster.setServiceParent(self)
-        dispatcher.botmaster = self.botmaster
-
-        sm = SchedulerManager(self, self.db, self.change_svc)
-        self.db.subscribe_to("add-change", sm.trigger_add_change)
-        self.db.subscribe_to("modify-buildset", sm.trigger_modify_buildset)
-        self.scheduler_manager = sm
-        sm.setServiceParent(self)
-        # TODO: this will turn into a config knob somehow. Set it if you are
-        # using multiple buildmasters that share a common database, such that
-        # the masters need to discover what each other is doing by polling
-        # the database. TODO: this will turn into the DBNotificationServer.
-        just_poll = False
-        if just_poll:
-            # it'd be nice if TimerService let us set now=False
-            t1 = internet.TimerService(30, sm.trigger)
-            t1.setServiceParent(self)
-            t2 = internet.TimerService(30, self.botmaster.loop.trigger)
-            t2.setServiceParent(self)
-        # adding schedulers (like when loadConfig happens) will trigger the
-        # scheduler loop at least once, which we need to jump-start things
-        # like Periodic.
+        self.dispatcher.botmaster = self.botmaster
 
-        self.status = Status(self.botmaster, self.db, self.basedir)
+        self.status = Status(self.botmaster, self.basedir)
         self.statusTargets = []
 
+        self.db = None
+        self.db_url = None
+        if db:
+            self.loadDatabase(db)
+
         self.readConfig = False
 
     def startService(self):
@@ -544,7 +524,7 @@ def loadConfig(self, f, check_synchronously_only=False):
                       "buildbotURL", "properties", "prioritizeBuilders",
                       "eventHorizon", "buildCacheSize", "logHorizon", "buildHorizon",
                       "changeHorizon", "logMaxSize", "logMaxTailSize",
-                      "logCompressionMethod",
+                      "logCompressionMethod", "db_url",
                       )
         for k in config.keys():
             if k not in known_keys:
@@ -559,6 +539,7 @@ def loadConfig(self, f, check_synchronously_only=False):
             #change_source = config['change_source']
 
             # optional
+            db_url = config.get("db_url", "sqlite:///state.sqlite")
             debugPassword = config.get('debugPassword')
             manhole = config.get('manhole')
             status = config.get('status', [])
@@ -644,6 +625,8 @@ def loadConfig(self, f, check_synchronously_only=False):
                     "reserved name '%s' used for a bot" % s.slavename)
         if config.has_key('interlocks'):
             raise KeyError("c['interlocks'] is no longer accepted")
+        assert self.db_url is None or db_url == self.db_url, \
+                "Cannot change db_url after master has started"
 
         assert isinstance(change_sources, (list, tuple))
         for s in change_sources:
@@ -799,6 +782,9 @@ def loadConfig(self, f, check_synchronously_only=False):
         self.logHorizon = logHorizon
         self.buildHorizon = buildHorizon
 
+        # Set up the database
+        d.addCallback(lambda res: self.loadConfig_Database(db_url))
+
         # self.slaves: Disconnect any that were attached and removed from the
         # list. Update self.checker with the new list of passwords, including
         # debug/change/status.
@@ -863,6 +849,44 @@ def _done(res):
         d.addErrback(log.err)
         return d
 
+    def loadDatabase(self, db_spec):
+        if self.db:
+            return
+        self.db = open_db(db_spec)
+
+        self.botmaster.db = self.db
+        self.status.setDB(self.db)
+
+        self.db.subscribe_to("add-buildrequest",
+                             self.botmaster.trigger_add_buildrequest)
+
+        sm = SchedulerManager(self, self.db, self.change_svc)
+        self.db.subscribe_to("add-change", sm.trigger_add_change)
+        self.db.subscribe_to("modify-buildset", sm.trigger_modify_buildset)
+
+        self.scheduler_manager = sm
+        sm.setServiceParent(self)
+
+        # TODO: this will turn into a config knob somehow. Set it if you are
+        # using multiple buildmasters that share a common database, such that
+        # the masters need to discover what each other is doing by polling
+        # the database. TODO: this will turn into the DBNotificationServer.
+        just_poll = False
+        if just_poll:
+            # it'd be nice if TimerService let us set now=False
+            t1 = internet.TimerService(30, sm.trigger)
+            t1.setServiceParent(self)
+            t2 = internet.TimerService(30, self.botmaster.loop.trigger)
+            t2.setServiceParent(self)
+        # adding schedulers (like when loadConfig happens) will trigger the
+        # scheduler loop at least once, which we need to jump-start things
+        # like Periodic.
+
+    def loadConfig_Database(self, db_url):
+        self.db_url = db_url
+        db_spec = DB.from_url(db_url, self.basedir)
+        self.loadDatabase(db_spec)
+
     def loadConfig_Slaves(self, new_slaves):
         # set up the Checker with the names and passwords of all valid bots
         self.checker.users = {} # violates abstraction, oh well

buildbot/schedulers/basic.py

@@ -90,12 +90,16 @@ def get_initial_state(self, max_changeid):
 
     def get_state(self, t):
         return self.parent.db.scheduler_get_state(self.schedulerid, t)
+
     def set_state(self, t, state):
         self.parent.db.scheduler_set_state(self.schedulerid, t, state)
 
     def listBuilderNames(self):
         return self.builderNames
 
+    def getPendingBuildTimes(self):
+        return []
+
     def create_buildset(self, ssid, reason, t, props=_None, builderNames=None):
         db = self.parent.db
         if props is _None:
@@ -176,9 +180,6 @@ def __init__(self, name, branch, treeStableTimer, builderNames,
     def get_initial_state(self, max_changeid):
         return {"last_processed": max_changeid}
 
-    def getPendingBuildTimes(self):
-        return []
-
     def changeIsRelevant(self, change):
         if change.branch != self.branch:
             return False
@@ -326,9 +327,6 @@ def __init__(self, name, upstream, builderNames,
         # DB.
         self.upstream_name = upstream.name
 
-    def getPendingBuildTimes(self):
-        return []
-
     def buildSetSubmitted(self, bsid, t):
         db = self.parent.db
         db.scheduler_subscribe_to_buildset(self.schedulerid, bsid, t)

buildbot/schedulers/triggerable.py

@@ -54,9 +54,6 @@ def __init__(self, name, builderNames, properties={}, categories=None):
     def get_initial_state(self, max_changeid):
         return {}
 
-    def getPendingBuildTimes(self):
-        return []
-
     def trigger(self, ss, set_props=None):
         """Trigger this scheduler. Returns a deferred that will fire when the
         buildset is finished.

buildbot/schedulers/trysched.py

@@ -58,10 +58,6 @@ def run(self):
         # triggered by external events, not DB changes or timers
         return None
 
-    def getPendingBuildTimes(self):
-        # we can't predict what the developers are going to do in the future
-        return []
-
     def filterBuilderList(self, builderNames):
         # self.builderNames is the configured list of builders
         # available for try.  If the user supplies a list of builders,