Merge branch 'v5' of git://github.com/catlee/buildbot

* 'v5' of git://github.com/catlee/buildbot:
  Create v5 schema that adds a bunch of indexes

Fixes buildbot#829 (hopefully)

buildbot/db/schema/manager.py

@@ -2,7 +2,7 @@
 
 # note that schema modules are not loaded unless an upgrade is taking place
 
-CURRENT_VERSION = 4
+CURRENT_VERSION = 5
 
 class DBSchemaManager(object):
     """

buildbot/db/schema/v5.py

@@ -0,0 +1,55 @@
+from buildbot.db.schema import base
+from buildbot.db.exceptions import DatabaseNotReadyError
+
+class Upgrader(base.Upgrader):
+    def upgrade(self):
+        self.add_index("buildrequests", "buildsetid")
+        self.add_index("buildrequests", "buildername", 255)
+        self.add_index("buildrequests", "complete")
+        self.add_index("buildrequests", "claimed_at")
+        self.add_index("buildrequests", "claimed_by_name", 255)
+
+        self.add_index("builds", "number")
+        self.add_index("builds", "brid")
+
+        self.add_index("buildsets", "complete")
+        self.add_index("buildsets", "submitted_at")
+
+        self.add_index("buildset_properties", "buildsetid")
+
+        self.add_index("changes", "branch", 255)
+        self.add_index("changes", "revision", 255)
+        self.add_index("changes", "author", 255)
+        self.add_index("changes", "category", 255)
+        self.add_index("changes", "when_timestamp")
+
+        self.add_index("change_files", "changeid")
+        self.add_index("change_links", "changeid")
+        self.add_index("change_properties", "changeid")
+
+        # schedulers already has an index
+
+        self.add_index("scheduler_changes", "schedulerid")
+        self.add_index("scheduler_changes", "changeid")
+
+        self.add_index("scheduler_upstream_buildsets", "buildsetid")
+        self.add_index("scheduler_upstream_buildsets", "schedulerid")
+        self.add_index("scheduler_upstream_buildsets", "active")
+
+        # sourcestamps are only queried by id, no need for additional indexes
+
+        self.add_index("sourcestamp_changes", "sourcestampid")
+
+        self.set_version()
+
+    def add_index(self, table, column, length=None):
+        lengthstr=""
+        if length is not None and self.dbapiName == 'MySQLdb':
+            lengthstr = " (%i)" % length
+        q = "CREATE INDEX `%(table)s_%(column)s` ON `%(table)s` (`%(column)s`%(lengthstr)s)"
+        cursor = self.conn.cursor()
+        cursor.execute(q % locals())
+
+    def set_version(self):
+        c = self.conn.cursor()
+        c.execute("""UPDATE version set version = 5 where version = 4""")

buildbot/test/unit/test_db_schema_master.py

@@ -8,7 +8,7 @@
 from twisted.trial import unittest
 
 from buildbot.db.schema import manager
-from buildbot.test.fake import fakedb
+from buildbot.db import dbspec
 
 class Thing(object):
     # simple object-with-attributes for use in faking pickled objects
@@ -23,9 +23,7 @@ def setUp(self):
             shutil.rmtree(self.basedir)
         os.makedirs(self.basedir)
 
-        # use an in-memory DB for speed
-        self.conn = fakedb.get_sqlite_memory_connection()
-        self.spec = fakedb.FakeDBSpec(conn=self.conn)
+        self.spec = dbspec.DBSpec.from_url("sqlite:///state.sqlite", self.basedir)
 
         self.sm = manager.DBSchemaManager(self.spec, self.basedir)
 
@@ -36,7 +34,7 @@ def assertDatabaseOKEmpty(self):
         assert that the database is an upgrade of an empty db
         """
         errs = []
-        c = self.conn.cursor()
+        c = self.spec.get_sync_connection().cursor()
 
         # check the version
         c.execute("SELECT * FROM version")
@@ -100,7 +98,7 @@ def assertDatabaseOKFull(self):
         assert that the database is an upgrade of the db created by fill_basedir
         """
         errs = []
-        c = self.conn.cursor()
+        c = self.spec.get_sync_connection().cursor()
 
         # check the version
         c.execute("SELECT * FROM version")
@@ -173,16 +171,17 @@ def assertDatabaseOKFull(self):
     def test_get_current_version(self):
         # this is as much a reminder to write tests for the new version
         # as a test of the (very trivial) method
-        self.assertEqual(self.sm.get_current_version(), 4)
+        self.assertEqual(self.sm.get_current_version(), 5)
 
     def test_get_db_version_empty(self):
         self.assertEqual(self.sm.get_db_version(), 0)
 
     def test_get_db_version_int(self):
-        c = self.conn.cursor()
+        conn = self.spec.get_sync_connection()
+        c = conn.cursor()
         c.execute("CREATE TABLE version (`version` integer)")
         c.execute("INSERT INTO version values (17)")
-        self.assertEqual(self.sm.get_db_version(), 17)
+        self.assertEqual(self.sm.get_db_version(conn), 17)
 
     def test_is_current_empty(self):
         self.assertFalse(self.sm.is_current())
@@ -202,7 +201,7 @@ def test_upgrade_full(self):
 
     def test_scheduler_name_uniqueness(self):
         self.sm.upgrade(quiet=True)
-        c = self.conn.cursor()
+        c = self.spec.get_sync_connection().cursor()
         c.execute("""INSERT INTO schedulers (`name`, `class_name`, `state`)
                                              VALUES ('s1', 'Nightly', '')""")
         c.execute("""INSERT INTO schedulers (`name`, `class_name`, `state`)
@@ -218,17 +217,15 @@ def setUp(self):
             shutil.rmtree(self.basedir)
         os.makedirs(self.basedir)
 
-        self.conn = MySQLdb.connect(user="buildbot_test", db="buildbot_test", passwd="buildbot_test", use_unicode=True, charset='utf8')
+        self.spec = dbspec.DBSpec.from_url("mysql://buildbot_test:buildbot_test@localhost/buildbot_test")
+
         # Drop all previous tables
-        cur = self.conn.cursor()
+        cur = self.spec.get_sync_connection().cursor()
         cur.execute("SHOW TABLES")
         for row in cur.fetchall():
             cur.execute("DROP TABLE %s" % row[0])
         cur.execute("COMMIT")
 
-        self.spec = fakedb.FakeDBSpec(conn=self.conn)
-        self.spec.get_dbapi = lambda: MySQLdb
-
         self.sm = manager.DBSchemaManager(self.spec, self.basedir)
 
 try: