Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
Use InnoDB for MySQL
This change adds a migration to convert any non-InnoDB tables to
InnoDB.

On some systems, the default engine is MyISAM, which doesn't
support features used by Keystone (foreign keys).

The approach is the same as what's used in Nova. A test is added
to ensure that all tables use InnoDB after migration. The test
passes when all the tables are mysql_engine='InnoDB'. This is
accomplished by adding a new migration that migrates all the
tables that aren't InnoDB to InnoDB.

Fixes bug 1191110.

Change-Id: I220f7642f5468c5cf4194f248210f90ff983b6e5
  • Loading branch information
Brant Knudson committed Jul 10, 2013
1 parent 25277d0 commit cd8fa2b
Show file tree
Hide file tree
Showing 3 changed files with 183 additions and 0 deletions.
@@ -0,0 +1,143 @@

import sqlalchemy as sql
from sqlalchemy import MetaData

from keystone.common.sql import migration_helpers


def upgrade(migrate_engine):
# Upgrade operations go here. Don't create your own engine;
# bind migrate_engine to your metadata

if migrate_engine.name != 'mysql':
# InnoDB / MyISAM only applies to MySQL.
return

# This is a list of all the tables that might have been created with MyISAM
# rather than InnoDB.
tables = [
'credential',
'domain',
'ec2_credential',
'endpoint',
'group',
'group_domain_metadata',
'group_project_metadata',
'policy',
'project',
'role',
'service',
'token',
'trust',
'trust_role',
'user',
'user_domain_metadata',
'user_group_membership',
'user_project_metadata',
]

meta = MetaData()
meta.bind = migrate_engine

domain_table = sql.Table('domain', meta, autoload=True)
endpoint_table = sql.Table('endpoint', meta, autoload=True)
group_table = sql.Table('group', meta, autoload=True)
group_domain_metadata_table = sql.Table('group_domain_metadata', meta,
autoload=True)
group_project_metadata_table = sql.Table('group_project_metadata', meta,
autoload=True)
project_table = sql.Table('project', meta, autoload=True)
service_table = sql.Table('service', meta, autoload=True)
user_table = sql.Table('user', meta, autoload=True)
user_domain_metadata_table = sql.Table('user_domain_metadata', meta,
autoload=True)
user_group_membership_table = sql.Table('user_group_membership', meta,
autoload=True)

# Mapping of table name to the constraints on that table,
# so we can create them.
table_constraints = {
'endpoint': [{'table': endpoint_table,
'fk_column': 'service_id',
'ref_column': service_table.c.id},
],
'group': [{'table': group_table,
'fk_column': 'domain_id',
'ref_column': domain_table.c.id},
],
'group_domain_metadata': [{'table': group_domain_metadata_table,
'fk_column': 'domain_id',
'ref_column': domain_table.c.id},
],
'group_project_metadata': [{'table': group_project_metadata_table,
'fk_column': 'project_id',
'ref_column': project_table.c.id},
],
'project': [{'table': project_table,
'fk_column': 'domain_id',
'ref_column': domain_table.c.id},
],
'user': [{'table': user_table,
'fk_column': 'domain_id',
'ref_column': domain_table.c.id},
],
'user_domain_metadata': [{'table': user_domain_metadata_table,
'fk_column': 'domain_id',
'ref_column': domain_table.c.id},
],
'user_group_membership': [{'table': user_group_membership_table,
'fk_column': 'user_id',
'ref_column': user_table.c.id},
{'table': user_group_membership_table,
'fk_column': 'group_id',
'ref_column': group_table.c.id},
],
'user_project_metadata': [{'table': group_project_metadata_table,
'fk_column': 'project_id',
'ref_column': project_table.c.id},
],
}

# Maps a table name to the tables that reference it as a FK constraint
# (See the map above).
ref_tables_map = {
'service': ['endpoint', ],
'domain': ['group', 'group_domain_metadata', 'project', 'user',
'user_domain_metadata', ],
'project': ['group_project_metadata', 'user_project_metadata', ],
'user': ['user_group_membership', ],
'group': ['user_group_membership', ],
}

# The names of tables that need to have their FKs added.
fk_table_names = set()

d = migrate_engine.execute("SHOW TABLE STATUS WHERE Engine!='InnoDB';")
for row in d.fetchall():
table_name = row[0]

if table_name not in tables:
# Skip this table since it's not a Keystone table.
continue

migrate_engine.execute("ALTER TABLE `%s` Engine=InnoDB" % table_name)

# Will add the FKs to the table if any of
# a) the table itself was converted
# b) the tables that the table referenced were converted

if table_name in table_constraints:
fk_table_names.add(table_name)

ref_tables = ref_tables_map.get(table_name, [])
for other_table_name in ref_tables:
fk_table_names.add(other_table_name)

# Now add all the FK constraints to those tables
for table_name in fk_table_names:
constraints = table_constraints.get(table_name)
migration_helpers.add_constraints(constraints)


def downgrade(migrate_engine):
pass
13 changes: 13 additions & 0 deletions keystone/common/sql/migration_helpers.py
Expand Up @@ -52,6 +52,19 @@ def remove_constraints(constraints):

def add_constraints(constraints):
for constraint_def in constraints:

if constraint_def['table'].kwargs.get('mysql_engine') == 'MyISAM':
# Don't try to create constraint when using MyISAM because it's
# not supported.
continue

ref_col = constraint_def['ref_column']
ref_engine = ref_col.table.kwargs.get('mysql_engine')
if ref_engine == 'MyISAM':
# Don't try to create constraint when using MyISAM because it's
# not supported.
continue

migrate.ForeignKeyConstraint(
columns=[getattr(constraint_def['table'].c,
constraint_def['fk_column'])],
Expand Down
27 changes: 27 additions & 0 deletions tests/test_sql_upgrade.py
Expand Up @@ -506,6 +506,10 @@ def insert_dict(self, session, table_name, d):

def test_downgrade_to_0(self):
self.upgrade(self.max_version)

if self.engine.name == 'mysql':
self._mysql_check_all_tables_innodb()

self.downgrade(0)
for table_name in ["user", "token", "role", "user_tenant_membership",
"metadata"]:
Expand Down Expand Up @@ -961,3 +965,26 @@ def _migrate(self, version, repository=None, downgrade=False):
for ver, change in changeset:
self.schema.runchange(ver, change, changeset.step)
self.assertEqual(self.schema.version, version)

def _mysql_check_all_tables_innodb(self):
database = self.engine.url.database

connection = self.engine.connect()
# sanity check
total = connection.execute("SELECT count(*) "
"from information_schema.TABLES "
"where TABLE_SCHEMA='%(database)s'" %
locals())
self.assertTrue(total.scalar() > 0, "No tables found. Wrong schema?")

noninnodb = connection.execute("SELECT table_name "
"from information_schema.TABLES "
"where TABLE_SCHEMA='%(database)s' "
"and ENGINE!='InnoDB' "
"and TABLE_NAME!='migrate_version'" %
locals())
names = [x[0] for x in noninnodb]
self.assertEqual(names, [],
"Non-InnoDB tables exist")

connection.close()

0 comments on commit cd8fa2b

Please sign in to comment.