add DB API for builders

README.md

@@ -105,7 +105,6 @@ So, they're listed here.
 Don't include the schema changes needed to implement the status stuff here; those will come when we implement the status stuff.
 
 * Remove ``is_dir`` from the changes table (and ignore/remove it everywhere else)
-* Add a ``builders`` table with provisions to indicate which masters are running which builders
 * Add a ``changesources`` table, similar to schedulers
 
 For each of the config-objects tables (masters, builders, schedulesr, changesources):

master/buildbot/db/builders.py

@@ -0,0 +1,89 @@
+# This file is part of Buildbot.  Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
+import hashlib
+import sqlalchemy as sa
+from buildbot.db import base
+
+class BuildersConnectorComponent(base.DBConnectorComponent):
+
+    def findBuilderId(self, name):
+        tbl = self.db.model.builders
+        return self.findSomethingId(
+                tbl=tbl,
+                whereclause=(tbl.c.name == name),
+                insert_values=dict(
+                    name=name,
+                    name_hash=hashlib.sha1(name).hexdigest(),
+                    ))
+
+    def getBuilder(self, builderid):
+        d = self.getBuilders(_builderid=builderid)
+        @d.addCallback
+        def first(bldrs):
+            if bldrs:
+                return bldrs[0]
+            else:
+                return None
+        return d
+
+    def addBuilderMaster(self, builderid=None, masterid=None):
+        def thd(conn, no_recurse=False):
+            tbl = self.db.model.builder_masters
+            q = tbl.insert()
+            conn.execute(q, builderid=builderid, masterid=masterid)
+        return self.db.pool.do(thd)
+
+    def removeBuilderMaster(self, builderid=None, masterid=None):
+        def thd(conn, no_recurse=False):
+            tbl = self.db.model.builder_masters
+            conn.execute(tbl.delete(
+                whereclause=((tbl.c.builderid == builderid)
+                           & (tbl.c.masterid == masterid))))
+        return self.db.pool.do(thd)
+
+    def getBuilders(self, masterid=None, _builderid=None):
+        def thd(conn):
+            bldr_tbl = self.db.model.builders
+            bm_tbl = self.db.model.builder_masters
+            j = bldr_tbl.outerjoin(bm_tbl)
+            # if we want to filter by masterid, we must join to builder_masters
+            # again, so we can still get the full set of masters for each
+            # builder
+            if masterid is not None:
+                limiting_bm_tbl = bm_tbl.alias('limiting_bm')
+                j = j.join(limiting_bm_tbl,
+                    onclause=(bldr_tbl.c.id == limiting_bm_tbl.c.builderid))
+            q = sa.select(
+                [bldr_tbl.c.id, bldr_tbl.c.name, bm_tbl.c.masterid],
+                from_obj=[j],
+                order_by=[bldr_tbl.c.id, bm_tbl.c.masterid])
+            if masterid is not None:
+                # filter the masterid from the limiting table
+                q = q.where(limiting_bm_tbl.c.masterid == masterid)
+            if _builderid is not None:
+                q = q.where(bldr_tbl.c.id == _builderid)
+
+            # now group those by builderid, aggregating by masterid
+            rv = []
+            last = None
+            for row in conn.execute(q).fetchall():
+                if not last or row['id'] != last['id']:
+                    last = dict(id=row.id, name=row.name, masterids=[])
+                    rv.append(last)
+                if row['masterid']:
+                    last['masterids'].append(row['masterid'])
+            return rv
+        return self.db.pool.do(thd)

master/buildbot/db/connector.py

@@ -21,7 +21,7 @@
 from buildbot.db import enginestrategy, exceptions
 from buildbot.db import pool, model, changes, schedulers, sourcestamps
 from buildbot.db import sourcestampsets, state, buildsets, buildrequests
-from buildbot.db import builds, users, masters
+from buildbot.db import builds, users, masters, builders
 
 upgrade_message = textwrap.dedent("""\
 
@@ -70,6 +70,7 @@ def __init__(self, master, basedir):
         self.builds = builds.BuildsConnectorComponent(self)
         self.users = users.UsersConnectorComponent(self)
         self.masters = masters.MastersConnectorComponent(self)
+        self.builders = builders.BuildersConnectorComponent(self)
 
         self.cleanup_timer = internet.TimerService(self.CLEANUP_PERIOD,
                 self._doCleanup)

master/buildbot/test/fake/fakedb.py

@@ -367,6 +367,30 @@ class Master(Row):
     id_column = 'id'
     hash_pairs = [ ( 'name', 'name_hash' ) ]
 
+class Builder(Row):
+    table = "builders"
+
+    defaults = dict(
+        id = None,
+        name = 'some:builder',
+        name_hash = None,
+    )
+
+    id_column = 'id'
+    hash_pairs = [ ( 'name', 'name_hash' ) ]
+
+class BuilderMaster(Row):
+    table = "builder_masters"
+
+    defaults = dict(
+        id = None,
+        builderid = None,
+        masterid = None
+    )
+
+    id_column = 'id'
+    required_columns = ( 'builderid', 'masterid' )
+
 # Fake DB Components
 
 class FakeDBComponent(object):
@@ -1292,11 +1316,6 @@ def setMasterState(self, masterid, active, _reactor=reactor):
         else:
             return defer.succeed(False)
 
-    def markMasterInactive(self, masterid):
-        if masterid in self.masters:
-            self.masters[masterid]['active'] = False
-        return defer.succeed(None)
-
     def getMaster(self, masterid):
         if masterid in self.masters:
             return defer.succeed(self.masters[masterid])
@@ -1305,6 +1324,75 @@ def getMaster(self, masterid):
     def getMasters(self):
         return defer.succeed(self.masters.values())
 
+    # test helpers
+
+    def markMasterInactive(self, masterid):
+        if masterid in self.masters:
+            self.masters[masterid]['active'] = False
+        return defer.succeed(None)
+
+
+class FakeBuildersComponent(FakeDBComponent):
+
+    def setUp(self):
+        self.builders = {}
+        self.builder_masters = {}
+
+    def insertTestData(self, rows):
+        for row in rows:
+            if isinstance(row, Builder):
+                self.builders[row.id] = dict(
+                        id=row.id,
+                        name=row.name)
+            if isinstance(row, BuilderMaster):
+                self.builder_masters[row.id] = \
+                        (row.builderid, row.masterid)
+
+    def findBuilderId(self, name, _reactor=reactor):
+        for m in self.builders.itervalues():
+            if m['name'] == name:
+                return defer.succeed(m['id'])
+        id = len(self.builders) + 1
+        self.builders[id] = dict(
+            id=id,
+            name=name)
+        return defer.succeed(id)
+
+    def addBuilderMaster(self, builderid=None, masterid=None):
+        self.insertTestData([
+            BuilderMaster(builderid=builderid, masterid=masterid),
+        ])
+        return defer.succeed(None)
+
+    def removeBuilderMaster(self, builderid=None, masterid=None):
+        for id, tup in self.builder_masters.iteritems():
+            if tup == (builderid, masterid):
+                del self.builder_masters[id]
+                break
+        return defer.succeed(None)
+
+    def getBuilder(self, builderid):
+        if builderid in self.builders:
+            masterids = [ bm[1] for bm in self.builder_masters.itervalues()
+                        if bm[0] == builderid ]
+            bldr = self.builders[builderid].copy()
+            bldr['masterids'] = sorted(masterids)
+            return defer.succeed(bldr)
+        return defer.succeed(None)
+
+    def getBuilders(self, masterid=None):
+        rv = []
+        for builderid, bldr in self.builders.iteritems():
+            masterids = [ bm[1] for bm in self.builder_masters.itervalues()
+                        if bm[0] == builderid ]
+            bldr = bldr.copy()
+            bldr['masterids'] = sorted(masterids)
+            rv.append(bldr)
+        if masterid is not None:
+            rv = [ bd for bd in rv
+                   if masterid in bd['masterids'] ]
+        return defer.succeed(rv)
+
 
 class FakeDBConnector(object):
     """
@@ -1341,6 +1429,8 @@ def __init__(self, testcase):
         self._components.append(comp)
         self.masters = comp = FakeMastersComponent(self, testcase)
         self._components.append(comp)
+        self.builders = comp = FakeBuildersComponent(self, testcase)
+        self._components.append(comp)
 
     def setup(self):
         self.is_setup = True