Add schedulers to the data API

Schedulers are only active on one master at a time.  This commit adds
support for schedulers to be configured and started (as Twisted
services), but not active.  The scheduler objects poll regularly and
become active if the scheduler is not active on any other master.

When a master shuts down - either normally or due to a heartbeat timeout
- it drops ownership of all schedulers for which it was active.

Other than that, this adds some basic endpoints and reflects the
findSchedulerById and setSchedulerMaster db methods as update methods.

README.md

@@ -96,6 +96,7 @@ For each resource type, we'll need the following (based on "Adding Resource Type
 
 * A resource-type module and class, with unit tests
 * Docs for that resource type
+* Fake versions of all update methods
 * Type validators for the resource type
 * New endpoints, with unit tests
 * Docs for the endpoints
@@ -107,25 +108,14 @@ It's safe to leave tasks that have significant prerequisites - particularly the
 
 The outstanding resource types are:
 
-* scheduler (underlying DB API complete)
 * buildrequest
 * build
 * step
 * logfile
 * buildslave
 * changesource
 
-#### Schedulers ####
-
-There is no scheduler resource type yet, although schedulers are in place in the database API so that other tables can refer to a scheduler ID.
-Support needs to be added for schedulers on a particular master to "claim" the name for themselves, deactivate themselves if already running on another master, and periodically poll for the opportunity to pick up the role.
-This gives us automatic failover for schedulers in a multi-master configuration, and also means that all masters can run with identical configurations, which may make configuration management easier.
-
-When a master becomes inactive (either on its own, or having failed its heartbeat check), its schedulers should be marked inactive by the data API and suitable messages sent.
-
-It should be possible to list the master on which a scheduler is running at e.g., `/scheduler/:schedulerid/master`
-
-#### Buildrequests ####
+### Notes on Build Requests ###
 
 Buildrequests include a `builderid` field which will need to be determined from the builders table.
 It should probably be an error to schedule a build on a builder that does not exist, as that build will not be executed.
@@ -164,6 +154,7 @@ Tasks:
 * Assign `urn`s to objects, and use those to correlate messages with objects.
 * Define the proper means of synchronizing messages and resources for each resource type.
   This information should be sufficient to reliably predict resource contents based on only on messages.
+* Remove `listBuilderNames` and `getPendingBuildTimes` methods from BaseScheduler
 
 ## Status Rewrites ##
 

master/buildbot/data/connector.py

@@ -44,6 +44,7 @@ class DataConnector(service.Service):
         'buildbot.data.changes',
         'buildbot.data.masters',
         'buildbot.data.sourcestamps',
+        'buildbot.data.schedulers',
     ]
 
     def __init__(self, master):

master/buildbot/data/exceptions.py

@@ -13,6 +13,13 @@
 #
 # Copyright Buildbot Team Members
 
+# copy some exceptions from the DB layer
+from buildbot.db.schedulers import SchedulerAlreadyClaimedError
+
+_hush_pyflakes = [
+    SchedulerAlreadyClaimedError
+]
+
 class DataException(Exception):
     pass
 
@@ -23,6 +30,7 @@ class InvalidPathError(DataException):
 class InvalidOptionException(DataException):
     "An option was invalid"
     pass
+
 class InvalidActionException(DataException):
     "Action is not supported"
     pass

master/buildbot/data/masters.py

@@ -116,6 +116,8 @@ def _masterDeactivated(self, masterid, name):
         # common code for deactivating a master
         yield self.master.data.rtypes.builder._masterDeactivated(
                                                     masterid=masterid)
+        yield self.master.data.rtypes.scheduler._masterDeactivated(
+                                                    masterid=masterid)
         self.produceEvent(
             dict(masterid=masterid, name=name, active=False),
             'stopped')

master/buildbot/data/schedulers.py

@@ -0,0 +1,95 @@
+# This file is part of Buildbot.  Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
+from twisted.internet import defer
+from buildbot.data import base
+
+class Db2DataMixin(object):
+
+    @defer.inlineCallbacks
+    def db2data(self, dbdict):
+        master = None
+        if dbdict['masterid'] is not None:
+            master = yield self.master.data.get({},
+                                    ('master', dbdict['masterid']))
+        data = {
+            'schedulerid': dbdict['id'],
+            'name': dbdict['name'],
+            'master': master,
+            'link': base.Link(('scheduler', str(dbdict['id']))),
+        }
+        defer.returnValue(data)
+
+
+class SchedulerEndpoint(Db2DataMixin, base.Endpoint):
+
+    pathPatterns = [
+        ( 'scheduler', 'i:schedulerid' ),
+        ( 'master', 'i:masterid', 'scheduler', 'i:schedulerid' ),
+    ]
+
+    @defer.inlineCallbacks
+    def get(self, options, kwargs):
+        dbdict = yield self.master.db.schedulers.getScheduler(
+                                                        kwargs['schedulerid'])
+        if 'masterid' in kwargs:
+            if dbdict['masterid'] != kwargs['masterid']:
+                return
+        yield defer.returnValue((yield self.db2data(dbdict))
+                                if dbdict else None)
+
+
+class SchedulersEndpoint(Db2DataMixin, base.Endpoint):
+
+    pathPatterns = [
+        ( 'scheduler', ),
+        ( 'master', 'i:masterid', 'scheduler', ),
+    ]
+    rootLinkName = 'schedulers'
+
+    @defer.inlineCallbacks
+    def get(self, options, kwargs):
+        schedulers = yield self.master.db.schedulers.getSchedulers(
+                                masterid=kwargs.get('masterid'))
+        yield defer.returnValue(
+                [ (yield self.db2data(schdict)) for schdict in schedulers ])
+
+    def startConsuming(self, callback, options, kwargs):
+        return self.master.mq.startConsuming(callback,
+                ('scheduler', None, None))
+
+
+class SchedulerResourceType(base.ResourceType):
+
+    type = "scheduler"
+    endpoints = [ SchedulerEndpoint, SchedulersEndpoint ]
+    keyFields = [ 'schedulerid' ]
+
+    @base.updateMethod
+    def findSchedulerId(self, name):
+        return self.master.db.schedulers.findSchedulerId(name)
+
+    @base.updateMethod
+    def setSchedulerMaster(self, schedulerid, masterid):
+        # the db method raises the same exception as documented
+        return self.master.db.schedulers.setSchedulerMaster(
+                                            schedulerid, masterid)
+
+    @defer.inlineCallbacks
+    def _masterDeactivated(self, masterid):
+        schedulers = yield self.master.db.schedulers.getSchedulers(
+                                masterid=masterid)
+        for sch in schedulers:
+            yield self.master.db.schedulers.setSchedulerMaster(sch['id'], None)