Set postgres sequence values to allow inserts

This fixes #2119, where the explicit inserts done as Buildbot imports
changes from pickles skip the call to nextval('changes_changeid_seq'),
and thus leave that sequence value out of sync with the table contents.

master/buildbot/db/migrate/versions/021_fix_postgres_sequences.py

@@ -0,0 +1,52 @@
+# This file is part of Buildbot.  Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
+import sqlalchemy as sa
+
+def upgrade(migrate_engine):
+    # see bug #2119
+
+    # this only applies to postgres
+    if migrate_engine.dialect.name != 'postgresql':
+        return
+
+    metadata = sa.MetaData()
+    metadata.bind = migrate_engine
+
+    to_fix = [
+        'buildrequests.id',
+        'builds.id',
+        'buildsets.id',
+        'changes.changeid',
+        'patches.id',
+        'sourcestampsets.id',
+        'sourcestamps.id',
+        'objects.id',
+        'users.uid',
+    ]
+
+    for col in to_fix:
+        tbl_name, col_name = col.split('.')
+        tbl = sa.Table(tbl_name, metadata, autoload=True)
+        col = tbl.c[col_name]
+
+        res = migrate_engine.execute(sa.select([ sa.func.max(col) ]))
+        max = res.fetchall()[0][0]
+
+        if max:
+            seq_name = "%s_%s_seq" % (tbl_name, col_name)
+            r = migrate_engine.execute("SELECT setval('%s', %d)"
+                            % (seq_name, max))
+            r.close()

master/buildbot/test/integration/test_upgrade.py

@@ -112,7 +112,7 @@ def setUpUpgradeTest(self):
 
         querylog.log_from_engine(self.db.pool.engine)
 
-    @defer.deferredGenerator # note tearDown doesn't handle Deferreds anyway
+    @defer.deferredGenerator
     def tearDownUpgradeTest(self):
         if self.use_real_db:
             wfd = defer.waitForDeferred(
@@ -129,7 +129,7 @@ def setUp(self):
         return self.setUpUpgradeTest()
 
     def tearDown(self):
-        self.tearDownUpgradeTest()
+        return self.tearDownUpgradeTest()
 
     def assertModelMatches(self):
         # this patch only applies to sqlalchemy-migrate-0.7.x.  We prefer to
@@ -236,6 +236,10 @@ class UpgradeTestV075(UpgradeTestMixin,
 
     source_tarball = "master-0-7-5.tgz"
 
+    # this test can use a real DB because 0.7.5 was pre-DB, so there's no
+    # expectation that the MySQL or Postgres DB will have anything in it.
+    use_real_db = True
+
     def verify_thd(self, conn):
         "verify the contents of the db - run in a thread"
         # note that this will all change if we re-create the tarball!
@@ -267,6 +271,14 @@ def verify_thd(self, conn):
         ])
         self.failUnlessEqual(filenames, expected)
 
+        # check that the change table's primary-key sequence is correct by
+        # trying to insert a new row.  This assumes that other sequences are
+        # handled correctly, if this one is.
+        r = conn.execute(model.changes.insert(),
+                dict(author='foo', comments='foo', is_dir=0,
+                    when_timestamp=123, repository='', project=''))
+        self.assertEqual(r.inserted_primary_key[0], 3)
+
     def fix_pickle_encoding(self, old_encoding):
         """Do the equivalent of master/contrib/fix_pickle_encoding.py"""
         changes_file = os.path.join(self.basedir, "changes.pck")

master/buildbot/test/unit/test_db_migrate_versions_011_add_buildrequest_claims.py

@@ -85,7 +85,7 @@ def create_tables_thd(self, conn):
 
     # tests
 
-    def test_foo(self):
+    def test_migrate(self):
         def setup_thd(conn):
             self.create_tables_thd(conn)
 

master/buildbot/test/unit/test_db_migrate_versions_021_fix_postgres_sequences.py

@@ -0,0 +1,76 @@
+# This file is part of Buildbot.  Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
+import sqlalchemy as sa
+from twisted.trial import unittest
+from buildbot.test.util import migration
+
+class Migration(migration.MigrateTestMixin, unittest.TestCase):
+
+    def setUp(self):
+        return self.setUpMigrateTest()
+
+    def tearDown(self):
+        return self.tearDownMigrateTest()
+
+    cols = [
+        'buildrequests.id',
+        'builds.id',
+        'buildsets.id',
+        'changes.changeid',
+        'patches.id',
+        'sourcestampsets.id',
+        'sourcestamps.id',
+        'objects.id',
+        'users.uid',
+    ]
+
+    # tests
+
+    def test_update(self):
+        def setup_thd(conn):
+            metadata = sa.MetaData()
+            metadata.bind = conn
+
+            # insert a row into each table, giving an explicit id column so
+            # that the sequence is not advanced correctly, but leave no rows in
+            # one table to test that corner case
+            for i, col in enumerate(self.cols):
+                tbl_name, col_name = col.split('.')
+                tbl = sa.Table(tbl_name, metadata,
+                        sa.Column(col_name, sa.Integer, primary_key=True))
+                tbl.create()
+                if i > 1:
+                    conn.execute(tbl.insert(), { col_name : i })
+
+        def verify_thd(conn):
+            metadata = sa.MetaData()
+            metadata.bind = conn
+
+            # try inserting *without* an ID, and verify that the resulting ID
+            # is as expected
+            for i, col in enumerate(self.cols):
+                print i, col
+                tbl_name, col_name = col.split('.')
+                tbl = sa.Table(tbl_name, metadata,
+                        sa.Column(col_name, sa.Integer, primary_key=True))
+                r = conn.execute(tbl.insert(), {})
+                if i > 1:
+                    exp = i+1
+                else:
+                    exp = 1
+                self.assertEqual(r.inserted_primary_key[0], exp)
+
+        return self.do_test_migration(20, 21, setup_thd, verify_thd)