Merge pull request #1316 from djmitche/bug2986

Update migration 037 and tests to work on Postgres

Fixes ticket:2986

master/buildbot/db/migrate/versions/037_buildrequests_builderid.py

@@ -86,33 +86,14 @@ def remove_buildername(migrate_engine):
 
     sa.Table('builders', metadata, autoload=True)
     sa.Table('buildsets', metadata, autoload=True)
+    buildrequests = sa.Table('buildrequests', metadata, autoload=True)
 
     # Specify what the new table should look like
-    buildrequests = sa.Table('buildrequests', metadata,
-                             sa.Column('id', sa.Integer, primary_key=True),
-                             sa.Column('buildsetid', sa.Integer, sa.ForeignKey("buildsets.id"),
-                                       nullable=False),
-                             sa.Column('builderid', sa.Integer, sa.ForeignKey('builders.id'),
-                                       nullable=False),
-                             sa.Column('priority', sa.Integer, nullable=False,
-                                       server_default=sa.DefaultClause("0")),
-                             sa.Column('complete', sa.Integer,
-                                       server_default=sa.DefaultClause("0")),
-                             sa.Column('results', sa.SmallInteger),
-                             sa.Column('submitted_at', sa.Integer, nullable=False),
-                             sa.Column('complete_at', sa.Integer),
-                             sa.Column('waited_for', sa.SmallInteger,
-                                       server_default=sa.DefaultClause("0")),
-                             )
-    changeset.drop_column(
-        sa.Column('buildername', sa.String(length=256), nullable=False),
-        table=buildrequests,
-        metadata=metadata,
-        engine=migrate_engine)
+    buildrequests.c.buildername.drop()
 
     changeset.alter_column(
         sa.Column('builderid', sa.Integer, sa.ForeignKey("builders.id"), nullable=False),
-        table='buildrequests',
+        table=buildrequests,
         metadata=metadata,
         engine=migrate_engine)
 
@@ -129,6 +110,5 @@ def upgrade(migrate_engine):
     add_new_schema_parts(migrate_engine)
     # migrate the data to new tables
     migrate_data(migrate_engine)
-
     # Finally remove the buildername column
     remove_buildername(migrate_engine)

master/buildbot/db/schedulers.py

@@ -47,6 +47,8 @@ def thd(conn):
                                  important=imp_int)
                 except (sqlalchemy.exc.ProgrammingError,
                         sqlalchemy.exc.IntegrityError):
+                    transaction.rollback()
+                    transaction = conn.begin()
                     # insert failed, so try an update
                     conn.execute(upd_q,
                                  wc_changeid=changeid,

master/buildbot/test/unit/test_db_buildsets.py

@@ -41,6 +41,8 @@ def setUpTests(self):
         # set up a sourcestamp with id 234 for use below
         return self.insertTestData([
             fakedb.SourceStamp(id=234),
+            fakedb.Builder(id=1, name='bldr1'),
+            fakedb.Builder(id=2, name='bldr2'),
         ])
 
     def test_signature_addBuildset(self):
@@ -461,7 +463,7 @@ def test_addBuildset_simple(self):
         d = defer.succeed(None)
         d.addCallback(lambda _:
                       self.db.buildsets.addBuildset(sourcestamps=[234], reason='because',
-                                                    properties={}, builderids=[42], external_idstring='extid',
+                                                    properties={}, builderids=[2], external_idstring='extid',
                                                     waited_for=True, _reactor=self.clock))
 
         def check(xxx_todo_changeme):
@@ -486,7 +488,7 @@ def thd(conn):
                                   u'complete', u'results', u'submitted_at',
                                   u'complete_at', u'waited_for'])
                 self.assertEqual(r.fetchall(),
-                                 [(bsid, brids[42], 42, 0, 0,
+                                 [(bsid, brids[2], 2, 0, 0,
                                    -1, self.now, None, 1)])
 
                 # one buildset_sourcestamps row
@@ -502,7 +504,7 @@ def test_addBuildset_bigger(self):
         d = defer.succeed(None)
         d.addCallback(lambda _:
                       self.db.buildsets.addBuildset(sourcestamps=[234], reason='because',
-                                                    waited_for=False, properties=props, builderids=[11, 12]))
+                                                    waited_for=False, properties=props, builderids=[1, 2]))
 
         def check(xxx_todo_changeme1):
             (bsid, brids) = xxx_todo_changeme1
@@ -539,7 +541,7 @@ def thd(conn):
                 # we don't know which of the brids is assigned to which
                 # buildername, but either one will do
                 self.assertEqual(sorted(rows),
-                                 [(bsid, brids[11], 11), (bsid, brids[12], 12)])
+                                 [(bsid, brids[1], 1), (bsid, brids[2], 2)])
             return self.db.pool.do(thd)
         d.addCallback(check)
         return d
@@ -570,7 +572,7 @@ def setUp(self):
         d = self.setUpConnectorComponent(
             table_names=['patches', 'buildsets', 'buildset_properties',
                          'objects', 'buildrequests', 'sourcestamps',
-                         'buildset_sourcestamps'])
+                         'buildset_sourcestamps', 'builders'])
 
         @d.addCallback
         def finish_setup(_):

master/buildbot/test/unit/test_db_migrate_versions_037_buildrequests_builderid.py

@@ -0,0 +1,130 @@
+# This file is part of Buildbot.  Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
+import sqlalchemy as sa
+
+from buildbot.test.util import migration
+from twisted.trial import unittest
+
+
+class Migration(migration.MigrateTestMixin, unittest.TestCase):
+
+    def setUp(self):
+        return self.setUpMigrateTest()
+
+    def tearDown(self):
+        return self.tearDownMigrateTest()
+
+    def test_migration(self):
+        def setup_thd(conn):
+            metadata = sa.MetaData()
+            metadata.bind = conn
+
+            builders = sa.Table(
+                'builders', metadata,
+                sa.Column('id', sa.Integer, primary_key=True),
+                sa.Column('name', sa.Text, nullable=False),
+                sa.Column('name_hash', sa.String(40), nullable=False),
+                )
+            builders.create()
+
+            buildsets = sa.Table(
+                'buildsets', metadata,
+                sa.Column('id', sa.Integer, primary_key=True),
+                sa.Column('external_idstring', sa.String(256)),
+                sa.Column('reason', sa.String(256)),
+                sa.Column('submitted_at', sa.Integer, nullable=False),
+                sa.Column('complete', sa.SmallInteger, nullable=False,
+                          server_default=sa.DefaultClause("0")),
+                sa.Column('complete_at', sa.Integer),
+                sa.Column('results', sa.SmallInteger),
+                sa.Column('parent_buildid', sa.Integer),
+                sa.Column('parent_relationship', sa.Text),
+                )
+            buildsets.create()
+
+            buildrequests = sa.Table(
+                'buildrequests', metadata,
+                sa.Column('id', sa.Integer, primary_key=True),
+                sa.Column('buildsetid', sa.Integer,
+                          sa.ForeignKey("buildsets.id"), nullable=False),
+                sa.Column('buildername', sa.String(length=256),
+                          nullable=False),
+                sa.Column('priority', sa.Integer, nullable=False,
+                          server_default=sa.DefaultClause("0")),
+                sa.Column('complete', sa.Integer,
+                          server_default=sa.DefaultClause("0")),
+                sa.Column('results', sa.SmallInteger),
+                sa.Column('submitted_at', sa.Integer, nullable=False),
+                sa.Column('complete_at', sa.Integer),
+                sa.Column('waited_for', sa.SmallInteger,
+                          server_default=sa.DefaultClause("0")),
+                )
+            buildrequests.create()
+
+            idx = sa.Index('buildrequests_buildsetid',
+                           buildrequests.c.buildsetid)
+            idx.create()
+            idx = sa.Index('buildrequests_buildername',
+                           buildrequests.c.buildername)
+            idx.create()
+            idx = sa.Index('buildrequests_complete', buildrequests.c.complete)
+            idx.create()
+            idx = sa.Index('buildsets_complete', buildsets.c.complete)
+            idx.create()
+            idx = sa.Index('buildsets_submitted_at', buildsets.c.submitted_at)
+            idx.create()
+
+            brargs = dict(buildsetid=10, priority=1, submitted_at=1234)
+            conn.execute(buildsets.insert(), id=10, submitted_at=1233)
+            conn.execute(builders.insert(), id=20, name='bldr1',
+                         name_hash='88103b2fbeb05bdd81c066b58a11bcf9b0d29300')
+            conn.execute(buildrequests.insert(),
+                         id=30, buildername='bldr1', **brargs)
+            conn.execute(buildrequests.insert(),
+                         id=31, buildername='bldr1', **brargs)
+            conn.execute(buildrequests.insert(),
+                         id=32, buildername='bldr2', **brargs)
+            self.assertTrue(hasattr(buildrequests.c, 'buildername'))
+            self.assertFalse(hasattr(buildrequests.c, 'builderid'))
+
+        def verify_thd(conn):
+            metadata = sa.MetaData()
+            metadata.bind = conn
+
+            buildrequests = sa.Table('buildrequests', metadata, autoload=True)
+            builders = sa.Table('builders', metadata, autoload=True)
+
+            self.assertFalse(hasattr(buildrequests.c, 'buildername'))
+            self.assertTrue(hasattr(buildrequests.c, 'builderid'))
+
+            self.assertEqual(
+                sorted([i.name for i in buildrequests.indexes]), [
+                    'buildrequests_builderid',
+                    'buildrequests_buildsetid',
+                    'buildrequests_complete',
+                    ])
+
+            # get the new builderid
+            bldr2_id = conn.execute(
+                sa.select(
+                    [builders.c.id],
+                    whereclause=(builders.c.name == 'bldr2'))).first()[0]
+            res = conn.execute(
+                sa.select([buildrequests.c.id, buildrequests.c.builderid]))
+            self.assertEqual(sorted(map(tuple, res)),
+                             [(30, 20), (31, 20), (32, bldr2_id)])
+
+        return self.do_test_migration(36, 37, setup_thd, verify_thd)