Add schedulers and scheduler_masters tables

README.md

@@ -87,6 +87,13 @@ Don't include the schema changes needed to implement the status stuff here; thos
 * Remove ``is_dir`` from the changes table (and ignore/remove it everywhere else)
 * Add a ``builders`` table with provisions to indicate which masters are running which builders
 * Add a ``schedulers`` table with provisions for masters to lock a particular scheduler name
+
+ * New table (DONE)
+ * Migration script (DONE) + tests (DONE)
+ * DB API module + docs, type verifier, and interface tests
+ * Fake implementation that passes interface tests
+ * Add TODO for data API implementation
+
 * Add a ``changesources`` table, similar to schedulers
 
 For each of the config-objects tables (masters, builders, schedulesr, changesources):

master/buildbot/db/migrate/versions/024_add_schedulers_table.py

@@ -0,0 +1,101 @@
+# This file is part of Buildbot.  Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
+import sqlalchemy as sa
+
+def upgrade(migrate_engine):
+
+    # drop the old scheduler_changes table (that references objects).  This
+    # loses a small, transient bit of information (processed but un-built
+    # changes), during a cluster-wide downtime.
+
+    metadata = sa.MetaData()
+    metadata.bind = migrate_engine
+    scheduler_changes = sa.Table('scheduler_changes', metadata,
+        sa.Column('objectid', sa.Integer),
+        sa.Column('changeid', sa.Integer),
+        # ..
+    )
+
+    idx = sa.Index('scheduler_changes_objectid', scheduler_changes.c.objectid)
+    idx.drop()
+    idx = sa.Index('scheduler_changes_changeid', scheduler_changes.c.changeid)
+    idx.drop()
+    idx = sa.Index('scheduler_changes_unique', scheduler_changes.c.objectid,
+                    scheduler_changes.c.changeid, unique=True)
+    idx.drop()
+    scheduler_changes.drop()
+
+    # now create the new tables (with new metadata since we're using the same
+    # table name)
+
+    metadata = sa.MetaData()
+    metadata.bind = migrate_engine
+
+    sa.Table('masters', metadata,
+        sa.Column('id', sa.Integer,  primary_key=True),
+        # ..
+    )
+
+    sa.Table('changes', metadata,
+        sa.Column('changeid', sa.Integer,  primary_key=True),
+        # ..
+    )
+
+    schedulers = sa.Table('schedulers', metadata,
+        sa.Column("id", sa.Integer, primary_key=True),
+        sa.Column('name', sa.Text, nullable=False),
+        sa.Column('name_hash', sa.String(40), nullable=False),
+    )
+
+    scheduler_masters = sa.Table('scheduler_masters', metadata,
+        sa.Column('schedulerid', sa.Integer, sa.ForeignKey('schedulers.id'),
+            nullable=False),
+        sa.Column('masterid', sa.Integer, sa.ForeignKey('masters.id'),
+            nullable=False),
+    )
+
+    scheduler_changes = sa.Table('scheduler_changes', metadata,
+        sa.Column('schedulerid', sa.Integer, sa.ForeignKey('schedulers.id')),
+        sa.Column('changeid', sa.Integer, sa.ForeignKey('changes.changeid')),
+        # true (nonzero) if this change is important to this scheduler
+        sa.Column('important', sa.Integer),
+    )
+
+    # create the new tables
+    schedulers.create()
+    scheduler_masters.create()
+    scheduler_changes.create()
+
+    # and the indices
+    idx = sa.Index('scheduler_name_hash', schedulers.c.name_hash, unique=True)
+    idx.create()
+    idx = sa.Index('scheduler_masters_schedulerid',
+            scheduler_masters.c.schedulerid, unique=True)
+    idx.create()
+    idx = sa.Index('scheduler_masters_identity',
+            scheduler_masters.c.schedulerid, scheduler_masters.c.masterid,
+            unique=True)
+    idx.create()
+    idx = sa.Index('scheduler_changes_schedulerid',
+            scheduler_changes.c.schedulerid)
+    idx.create()
+    idx = sa.Index('scheduler_changes_changeid',
+            scheduler_changes.c.changeid)
+    idx.create()
+    idx = sa.Index('scheduler_changes_unique',
+            scheduler_changes.c.schedulerid, scheduler_changes.c.changeid,
+            unique=True)
+    idx.create()

master/buildbot/db/model.py

@@ -282,13 +282,35 @@ class Model(base.DBConnectorComponent):
 
     # schedulers
 
+    # The schedulers table gives a unique identifier to each scheduler.  It
+    # also links to other tables used to ensure only one master runs each
+    # scheduler, and to track changes that a scheduler may trigger a build for
+    # later.
+    schedulers = sa.Table('schedulers', metadata,
+        sa.Column("id", sa.Integer, primary_key=True),
+
+        # name for this scheduler, as given in the configuration, plus a hash
+        # of that name used for a unique index
+        sa.Column('name', sa.Text, nullable=False),
+        sa.Column('name_hash', sa.String(40), nullable=False),
+    )
+
+    # This links schedulers to the master where they are running.  A scheduler
+    # linked to a master that is inactive can be unlinked by any master.
+    scheduler_masters = sa.Table('scheduler_masters', metadata,
+        sa.Column('schedulerid', sa.Integer, sa.ForeignKey('schedulers.id'),
+            nullable=False),
+        sa.Column('masterid', sa.Integer, sa.ForeignKey('masters.id'),
+            nullable=False),
+    )
+
     # This table references "classified" changes that have not yet been
     # "processed".  That is, the scheduler has looked at these changes and
     # determined that something should be done, but that hasn't happened yet.
     # Rows are deleted from this table as soon as the scheduler is done with
     # the change.
     scheduler_changes = sa.Table('scheduler_changes', metadata,
-        sa.Column('objectid', sa.Integer, sa.ForeignKey('objects.id')),
+        sa.Column('schedulerid', sa.Integer, sa.ForeignKey('schedulers.id')),
         sa.Column('changeid', sa.Integer, sa.ForeignKey('changes.changeid')),
         # true (nonzero) if this change is important to this scheduler
         sa.Column('important', sa.Integer),
@@ -388,9 +410,15 @@ class Model(base.DBConnectorComponent):
     sa.Index('changes_when_timestamp', changes.c.when_timestamp)
     sa.Index('change_files_changeid', change_files.c.changeid)
     sa.Index('change_properties_changeid', change_properties.c.changeid)
-    sa.Index('scheduler_changes_objectid', scheduler_changes.c.objectid)
+    sa.Index('scheduler_name_hash', schedulers.c.name_hash, unique=True)
+    sa.Index('scheduler_masters_schedulerid', scheduler_masters.c.schedulerid,
+            unique=True)
+    sa.Index('scheduler_masters_identity',
+            scheduler_masters.c.schedulerid, scheduler_masters.c.masterid,
+            unique=True)
+    sa.Index('scheduler_changes_schedulerid', scheduler_changes.c.schedulerid)
     sa.Index('scheduler_changes_changeid', scheduler_changes.c.changeid)
-    sa.Index('scheduler_changes_unique', scheduler_changes.c.objectid,
+    sa.Index('scheduler_changes_unique', scheduler_changes.c.schedulerid,
             scheduler_changes.c.changeid, unique=True)
     sa.Index('sourcestamp_changes_sourcestampid',
             sourcestamp_changes.c.sourcestampid)

master/buildbot/test/fake/fakedb.py

@@ -230,16 +230,39 @@ class SourceStamp(Row):
     id_column = 'id'
 
 
+class Scheduler(Row):
+    table = "schedulers"
+
+    defaults = dict(
+        id = None,
+        name = 'schname',
+        name_hash = None,
+    )
+
+    id_column = 'id'
+    hash_pairs = [ ( 'name', 'name_hash' ) ]
+
+
+class SchedulerMaster(Row):
+    table = "scheduler_masters"
+
+    defaults = dict(
+        schedulerid = None,
+        masterid = None,
+    )
+
+    required_columns = ( 'schedulerid', 'masterid' )
+
 class SchedulerChange(Row):
     table = "scheduler_changes"
 
     defaults = dict(
-        objectid = None,
+        schedulerid = None,
         changeid = None,
         important = 1,
     )
 
-    required_columns = ( 'objectid', 'changeid' )
+    required_columns = ( 'schedulerid', 'changeid' )
 
 
 class Buildset(Row):

master/buildbot/test/unit/test_db_migrate_versions_024_add_schedulers_table.py

@@ -0,0 +1,83 @@
+# This file is part of Buildbot.  Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
+import sqlalchemy as sa
+from twisted.trial import unittest
+from buildbot.test.util import migration
+
+class Migration(migration.MigrateTestMixin, unittest.TestCase):
+
+    def setUp(self):
+        return self.setUpMigrateTest()
+
+    def tearDown(self):
+        return self.tearDownMigrateTest()
+
+    def test_migration(self):
+        def setup_thd(conn):
+            metadata = sa.MetaData()
+            metadata.bind = conn
+            scheduler_changes = sa.Table('scheduler_changes', metadata,
+                sa.Column('objectid', sa.Integer),
+                sa.Column('changeid', sa.Integer),
+                # ..
+            )
+            scheduler_changes.create()
+
+            sa.Table('masters', metadata,
+                sa.Column('id', sa.Integer,  primary_key=True),
+                # ..
+            ).create()
+
+            sa.Table('changes', metadata,
+                sa.Column('changeid', sa.Integer,  primary_key=True),
+                # ..
+            ).create()
+
+            idx = sa.Index('scheduler_changes_objectid',
+                    scheduler_changes.c.objectid)
+            idx.create()
+            idx = sa.Index('scheduler_changes_changeid',
+                    scheduler_changes.c.changeid)
+            idx.create()
+            idx = sa.Index('scheduler_changes_unique',
+                    scheduler_changes.c.objectid, scheduler_changes.c.changeid,
+                    unique=True)
+            idx.create()
+
+        def verify_thd(conn):
+            metadata = sa.MetaData()
+            metadata.bind = conn
+
+            schedulers = sa.Table('schedulers', metadata, autoload=True)
+            scheduler_masters = sa.Table('scheduler_masters', metadata,
+                    autoload=True)
+            scheduler_changes = sa.Table('scheduler_changes', metadata,
+                    autoload=True)
+
+            q = sa.select([ schedulers.c.id, schedulers.c.name,
+                            schedulers.c.name_hash ])
+            self.assertEqual(conn.execute(q).fetchall(), [])
+
+            q = sa.select([ scheduler_masters.c.schedulerid,
+                            scheduler_masters.c.masterid ])
+            self.assertEqual(conn.execute(q).fetchall(), [])
+
+            q = sa.select([ scheduler_changes.c.schedulerid,
+                            scheduler_changes.c.changeid,
+                            scheduler_changes.c.important ])
+            self.assertEqual(conn.execute(q).fetchall(), [])
+
+        return self.do_test_migration(23, 24, setup_thd, verify_thd)