DB2 migration support

DB2 will not allow you to rename a table if it's got a
constraint on it (a unique or foreign key constraint).

This fix changes the migrations so that the unique or FK
constraints are dropped from tables before renaming and then
restoring the unique FK constraints. This works for DB2 and
other DBMSs that support FK constraints such as MySQL with
InnoDB and PostgreSQL.

Also, for DB2, give a name to the unique constraints so that
they can be manipulated.

Fixes bug 1188785

Change-Id: I7cf6ab42084e43d827ed827c64025e61e72a4672

keystone/common/sql/migrate_repo/versions/001_add_initial_tables.py

@@ -54,12 +54,28 @@ def upgrade(migrate_engine):
         sql.Column('name', sql.String(255), unique=True, nullable=False))
     role_table.create(migrate_engine, checkfirst=True)
 
-    tenant_table = sql.Table(
-        'tenant',
-        meta,
-        sql.Column('id', sql.String(64), primary_key=True),
-        sql.Column('name', sql.String(64), unique=True, nullable=False),
-        sql.Column('extra', sql.Text()))
+    if migrate_engine.name == 'ibm_db_sa':
+        # NOTE(blk-u): SQLAlchemy for PostgreSQL picks the name tenant_name_key
+        # for the unique constraint, but for DB2 doesn't give the UC a name
+        # unless we tell it to and there is no DDL to alter a column to drop
+        # an unnamed unique constraint, so this code creates a named unique
+        # constraint on the name column rather than an unnamed one.
+        # (This is used in migration 16.)
+        tenant_table = sql.Table(
+            'tenant',
+            meta,
+            sql.Column('id', sql.String(64), primary_key=True),
+            sql.Column('name', sql.String(64), nullable=False),
+            sql.Column('extra', sql.Text()),
+            sql.UniqueConstraint('name', name='tenant_name_key'))
+    else:
+        tenant_table = sql.Table(
+            'tenant',
+            meta,
+            sql.Column('id', sql.String(64), primary_key=True),
+            sql.Column('name', sql.String(64), unique=True, nullable=False),
+            sql.Column('extra', sql.Text()))
+
     tenant_table.create(migrate_engine, checkfirst=True)
 
     metadata_table = sql.Table(
@@ -79,12 +95,28 @@ def upgrade(migrate_engine):
         sql.Column('tenant_id', sql.String(64)))
     ec2_credential_table.create(migrate_engine, checkfirst=True)
 
-    user_table = sql.Table(
-        'user',
-        meta,
-        sql.Column('id', sql.String(64), primary_key=True),
-        sql.Column('name', sql.String(64), unique=True, nullable=False),
-        sql.Column('extra', sql.Text()))
+    if migrate_engine.name == 'ibm_db_sa':
+        # NOTE(blk-u): SQLAlchemy for PostgreSQL picks the name user_name_key
+        # for the unique constraint, but for DB2 doesn't give the UC a name
+        # unless we tell it to and there is no DDL to alter a column to drop
+        # an unnamed unique constraint, so this code creates a named unique
+        # constraint on the name column rather than an unnamed one.
+        # (This is used in migration 16.)
+        user_table = sql.Table(
+            'user',
+            meta,
+            sql.Column('id', sql.String(64), primary_key=True),
+            sql.Column('name', sql.String(64), nullable=False),
+            sql.Column('extra', sql.Text()),
+            sql.UniqueConstraint('name', name='user_name_key'))
+    else:
+        user_table = sql.Table(
+            'user',
+            meta,
+            sql.Column('id', sql.String(64), primary_key=True),
+            sql.Column('name', sql.String(64), unique=True, nullable=False),
+            sql.Column('extra', sql.Text()))
+
     user_table.create(migrate_engine, checkfirst=True)
 
     user_tenant_membership_table = sql.Table(

keystone/common/sql/migrate_repo/versions/011_endpoints_v3.py

@@ -16,14 +16,23 @@
 
 import sqlalchemy as sql
 
+from keystone.common.sql import migration_helpers
+
 
 def upgrade(migrate_engine):
     """Create API-version specific endpoint tables."""
     meta = sql.MetaData()
     meta.bind = migrate_engine
 
     legacy_table = sql.Table('endpoint', meta, autoload=True)
-    legacy_table.rename('endpoint_v2')
+
+    renames = {'endpoint_v2': legacy_table}
+    service_table = sql.Table('service', meta, autoload=True)
+    constraints = [{'table': legacy_table,
+                    'fk_column': 'service_id',
+                    'ref_column': service_table.c.id}]
+    migration_helpers.rename_tables_with_constraints(renames, constraints,
+                                                     migrate_engine)
 
     sql.Table('service', meta, autoload=True)
     new_table = sql.Table(
@@ -51,4 +60,11 @@ def downgrade(migrate_engine):
     new_table.drop()
 
     legacy_table = sql.Table('endpoint_v2', meta, autoload=True)
-    legacy_table.rename('endpoint')
+
+    renames = {'endpoint': legacy_table}
+    service_table = sql.Table('service', meta, autoload=True)
+    constraints = [{'table': legacy_table,
+                    'fk_column': 'service_id',
+                    'ref_column': service_table.c.id}]
+    migration_helpers.rename_tables_with_constraints(renames, constraints,
+                                                     migrate_engine)

keystone/common/sql/migrate_repo/versions/013_drop_legacy_endpoints.py

@@ -16,6 +16,8 @@
 
 import sqlalchemy as sql
 
+from keystone.common.sql import migration_helpers
+
 
 def upgrade(migrate_engine):
     """Replace API-version specific endpoint tables with one based on v3."""
@@ -26,7 +28,14 @@ def upgrade(migrate_engine):
     legacy_table.drop()
 
     new_table = sql.Table('endpoint_v3', meta, autoload=True)
-    new_table.rename('endpoint')
+
+    renames = {'endpoint': new_table}
+    service_table = sql.Table('service', meta, autoload=True)
+    constraints = [{'table': new_table,
+                    'fk_column': 'service_id',
+                    'ref_column': service_table.c.id}]
+    migration_helpers.rename_tables_with_constraints(renames, constraints,
+                                                     migrate_engine)
 
 
 def downgrade(migrate_engine):
@@ -35,7 +44,14 @@ def downgrade(migrate_engine):
     meta.bind = migrate_engine
 
     new_table = sql.Table('endpoint', meta, autoload=True)
-    new_table.rename('endpoint_v3')
+
+    renames = {'endpoint_v3': new_table}
+    service_table = sql.Table('service', meta, autoload=True)
+    constraints = [{'table': new_table,
+                    'fk_column': 'service_id',
+                    'ref_column': service_table.c.id}]
+    migration_helpers.rename_tables_with_constraints(renames, constraints,
+                                                     migrate_engine)
 
     sql.Table('service', meta, autoload=True)
     legacy_table = sql.Table(

keystone/common/sql/migrate_repo/versions/015_tenant_to_project.py

@@ -1,19 +1,66 @@
 import sqlalchemy as sql
 from sqlalchemy.orm import sessionmaker
 
+from keystone.common.sql import migration_helpers
+
+
+def rename_with_constraints(meta, legacy_project_table_name,
+                            new_project_table_name,
+                            legacy_user_project_membership_table_name,
+                            new_user_project_membership_table_name):
+    # Not all RDBMSs support renaming a table that has foreign key constraints
+    # on it, so drop FK constraints before renaming and then replace FKs
+    # afterwards.
+
+    credential_table = sql.Table('credential', meta, autoload=True)
+    group_project_meta_table = sql.Table('group_project_metadata', meta,
+                                         autoload=True)
+    project_table = sql.Table(legacy_project_table_name, meta, autoload=True)
+    user_project_membership_table = sql.Table(
+        legacy_user_project_membership_table_name, meta, autoload=True)
+    user_table = sql.Table('user', meta, autoload=True)
+
+    constraints = [{'table': credential_table,
+                    'fk_column': 'project_id',
+                    'ref_column': project_table.c.id},
+                   {'table': group_project_meta_table,
+                    'fk_column': 'project_id',
+                    'ref_column': project_table.c.id},
+                   {'table': user_project_membership_table,
+                    'fk_column': 'tenant_id',
+                    'ref_column': project_table.c.id},
+                   {'table': user_project_membership_table,
+                    'fk_column': 'user_id',
+                    'ref_column': user_table.c.id}]
+
+    renames = {
+        new_project_table_name: project_table,
+        new_user_project_membership_table_name: user_project_membership_table}
+
+    migration_helpers.rename_tables_with_constraints(renames, constraints,
+                                                     meta.bind)
+
 
 def upgrade_with_rename(meta, migrate_engine):
-    legacy_table = sql.Table('tenant', meta, autoload=True)
-    legacy_table.rename('project')
-    legacy_table = sql.Table('user_tenant_membership', meta, autoload=True)
-    legacy_table.rename('user_project_membership')
+    legacy_project_table_name = 'tenant'
+    new_project_table_name = 'project'
+    legacy_user_project_membership_table_name = 'user_tenant_membership'
+    new_user_project_membership_table_name = 'user_project_membership'
+    rename_with_constraints(meta, legacy_project_table_name,
+                            new_project_table_name,
+                            legacy_user_project_membership_table_name,
+                            new_user_project_membership_table_name)
 
 
 def downgrade_with_rename(meta, migrate_engine):
-    upgrade_table = sql.Table('project', meta, autoload=True)
-    upgrade_table.rename('tenant')
-    upgrade_table = sql.Table('user_project_membership', meta, autoload=True)
-    upgrade_table.rename('user_tenant_membership')
+    legacy_project_table_name = 'project'
+    new_project_table_name = 'tenant'
+    legacy_user_project_membership_table_name = 'user_project_membership'
+    new_user_project_membership_table_name = 'user_tenant_membership'
+    rename_with_constraints(meta, legacy_project_table_name,
+                            new_project_table_name,
+                            legacy_user_project_membership_table_name,
+                            new_user_project_membership_table_name)
 
 
 def upgrade_with_copy(meta, migrate_engine):

keystone/common/sql/migrate_repo/versions/016_normalize_domain_ids.py

@@ -381,7 +381,9 @@ def downgrade_user_table_with_col_drop(meta, migrate_engine, session):
     # Revert uniqueness settings for the name attribute
     session.execute('ALTER TABLE "user" DROP CONSTRAINT '
                     'user_dom_name_unique;')
-    session.execute('ALTER TABLE "user" ADD UNIQUE (name);')
+    # specify the constraint name so it can be referenced later
+    session.execute('ALTER TABLE "user" ADD CONSTRAINT user_name_key '
+                    'UNIQUE (name);')
     session.commit()
     # And now go ahead an drop the domain_id column
     sql.Table('domain', meta, autoload=True)

keystone/common/sql/migration_helpers.py

@@ -56,3 +56,27 @@ def add_constraints(constraints):
             columns=[getattr(constraint_def['table'].c,
                              constraint_def['fk_column'])],
             refcolumns=[constraint_def['ref_column']]).create()
+
+
+def rename_tables_with_constraints(renames, constraints, engine):
+    """Renames tables with foreign key constraints.
+
+    Tables are renamed after first removing constraints. The constraints are
+    replaced after the rename is complete.
+
+    This works on databases that don't support renaming tables that have
+    constraints on them (DB2).
+
+    `renames` is a dict, mapping {'to_table_name': from_table, ...}
+    """
+
+    if engine.name != 'sqlite':
+        # Sqlite doesn't support constraints, so nothing to remove.
+        remove_constraints(constraints)
+
+    for to_table_name in renames:
+        from_table = renames[to_table_name]
+        from_table.rename(to_table_name)
+
+    if engine != 'sqlite':
+        add_constraints(constraints)

tests/_sql_livetest.py

@@ -35,3 +35,11 @@ def config_files(self):
                  _config_file_list[:])
         files.append("backend_mysql.conf")
         return files
+
+
+class Db2MigrateTests(test_sql_upgrade.SqlUpgradeTests):
+    def config_files(self):
+        files = (test_sql_upgrade.SqlUpgradeTests.
+                 _config_file_list[:])
+        files.append("backend_db2.conf")
+        return files

tests/backend_db2.conf

@@ -0,0 +1,4 @@
+#Used for running the Migrate tests against a live DB2 Server
+#See _sql_livetest.py
+[sql]
+connection = ibm_db_sa://keystone:keystone@/staktest?charset=utf8

tests/test_sql_upgrade.py

@@ -597,27 +597,30 @@ def test_metadata_table_migration(self):
         user_project_metadata_table = sqlalchemy.Table(
             'user_project_metadata', self.metadata, autoload=True)
 
-        r = session.execute('select data from metadata where '
-                            'user_id=:user and tenant_id=:tenant',
-                            {'user': user['id'], 'tenant': project['id']})
+        s = sqlalchemy.select([metadata_table.c.data]).where(
+            (metadata_table.c.user_id == user['id']) &
+            (metadata_table.c.tenant_id == project['id']))
+        r = session.execute(s)
         test_project1 = json.loads(r.fetchone()['data'])
         self.assertEqual(len(test_project1['roles']), 1)
         self.assertIn(role['id'], test_project1['roles'])
 
         # Test user in project2 has role2
-        r = session.execute('select data from metadata where '
-                            'user_id=:user and tenant_id=:tenant',
-                            {'user': user['id'], 'tenant': project2['id']})
+        s = sqlalchemy.select([metadata_table.c.data]).where(
+            (metadata_table.c.user_id == user['id']) &
+            (metadata_table.c.tenant_id == project2['id']))
+        r = session.execute(s)
         test_project2 = json.loads(r.fetchone()['data'])
         self.assertEqual(len(test_project2['roles']), 1)
         self.assertIn(role2['id'], test_project2['roles'])
 
         # Test for user in project has role in user_project_metadata
         # Migration 17 does not properly migrate this data, so this should
         # be None.
-        r = session.execute('select data from user_project_metadata where '
-                            'user_id=:user and project_id=:project',
-                            {'user': user['id'], 'project': project['id']})
+        s = sqlalchemy.select([user_project_metadata_table.c.data]).where(
+            (user_project_metadata_table.c.user_id == user['id']) &
+            (user_project_metadata_table.c.project_id == project['id']))
+        r = session.execute(s)
         self.assertIsNone(r.fetchone())
 
         # Create a conflicting user-project in user_project_metadata with
@@ -638,9 +641,10 @@ def test_metadata_table_migration(self):
         # The user-project pairs should have all roles from the previous
         # metadata table in addition to any roles currently in
         # user_project_metadata
-        r = session.execute('select data from user_project_metadata where '
-                            'user_id=:user and project_id=:project',
-                            {'user': user['id'], 'project': project['id']})
+        s = sqlalchemy.select([user_project_metadata_table.c.data]).where(
+            (user_project_metadata_table.c.user_id == user['id']) &
+            (user_project_metadata_table.c.project_id == project['id']))
+        r = session.execute(s)
         role_ids = json.loads(r.fetchone()['data'])['roles']
         self.assertEqual(len(role_ids), 3)
         self.assertIn(CONF.member_role_id, role_ids)
@@ -649,9 +653,10 @@ def test_metadata_table_migration(self):
 
         # pairs that only existed in old metadata table should be in
         # user_project_metadata
-        r = session.execute('select data from user_project_metadata where '
-                            'user_id=:user and project_id=:project',
-                            {'user': user['id'], 'project': project2['id']})
+        s = sqlalchemy.select([user_project_metadata_table.c.data]).where(
+            (user_project_metadata_table.c.user_id == user['id']) &
+            (user_project_metadata_table.c.project_id == project2['id']))
+        r = session.execute(s)
         role_ids = json.loads(r.fetchone()['data'])['roles']
         self.assertEqual(len(role_ids), 2)
         self.assertIn(CONF.member_role_id, role_ids)