Migrate roles from metadata to user_project_metadata

migration 017 did not migrate existing roles from the metadata table
that was created in 001. Adding a migration (20) that compares the roles
in the metadata table (if any) and joins them to the new
user_project_metadata role that matches the user and tenant from the old
table. Also adding subsequent tests to check both of the issues above.
bug 1131087

Change-Id: I00ea6043d949c9c358827e25f05c63515fe5dea8

keystone/common/sql/migrate_repo/versions/020_migrate_metadata_table_roles.py

@@ -0,0 +1,104 @@
+import json
+import uuid
+
+import sqlalchemy as sql
+from sqlalchemy import orm
+
+from keystone import config
+from keystone import exception
+
+
+CONF = config.CONF
+
+
+def upgrade(migrate_engine):
+    meta = sql.MetaData()
+    meta.bind = migrate_engine
+
+    user_table = sql.Table('user', meta, autoload=True)
+    role_table = sql.Table('role', meta, autoload=True)
+    project_table = sql.Table('project', meta, autoload=True)
+    new_metadata_table = sql.Table('user_project_metadata',
+                                   meta,
+                                   autoload=True)
+
+    conn = migrate_engine.connect()
+
+    old_metadata_table = sql.Table('metadata', meta, autoload=True)
+    session = sql.orm.sessionmaker(bind=migrate_engine)()
+
+    for metadata in session.query(old_metadata_table):
+        if not config.CONF.member_role_id in metadata.data:
+            data = json.loads(metadata.data)
+            data['roles'].append(config.CONF.member_role_id)
+        else:
+            data = metadata.data
+
+        r = session.query(new_metadata_table).filter_by(
+            user_id=metadata.user_id,
+            project_id=metadata.tenant_id).first()
+
+        if r is not None:
+            # roles should be the union of the two role lists
+            old_roles = data['roles']
+            new_roles = json.loads(r.data)['roles']
+            data['roles'] = list(set(old_roles) | set(new_roles))
+            q = new_metadata_table.update().where(
+                new_metadata_table.c.user_id == metadata.user_id and
+                new_metadata_table.c.project_id == metadata.tenant_id).values(
+                    data=json.dumps(data))
+        else:
+            q = new_metadata_table.insert().values(
+                user_id=metadata.user_id,
+                project_id=metadata.tenant_id,
+                data=json.dumps(data))
+
+        conn.execute(q)
+
+    session.close()
+    old_metadata_table.drop()
+
+
+def downgrade(migrate_engine):
+    meta = sql.MetaData()
+    meta.bind = migrate_engine
+
+    user_table = sql.Table('user', meta, autoload=True)
+    project_table = sql.Table('project', meta, autoload=True)
+
+    metadata_table = sql.Table(
+        'metadata',
+        meta,
+        sql.Column(
+            'user_id',
+            sql.String(64),
+            sql.ForeignKey('user.id'),
+            primary_key=True),
+        sql.Column(
+            'tenant_id',
+            sql.String(64),
+            primary_key=True),
+        sql.Column('data',
+                   sql.Text()))
+    metadata_table.create(migrate_engine, checkfirst=True)
+
+    user_project_metadata_table = sql.Table(
+        'user_project_metadata',
+        meta,
+        autoload=True)
+
+    metadata_table = sql.Table(
+        'metadata',
+        meta,
+        autoload=True)
+
+    session = sql.orm.sessionmaker(bind=migrate_engine)()
+
+    for metadata in session.query(user_project_metadata_table):
+        if 'roles' in metadata:
+            roles = json.loads(metadata.data)
+            ins = (metadata_table.insert()
+                   .values(user_id=metadata.user_id,
+                           tenant_id=metadata.project_id))
+
+    session.close()

tests/test_sql_upgrade.py

@@ -35,7 +35,6 @@
 from keystone.common import sql
 from keystone.common.sql import migration
 from keystone import config
-from keystone import exception
 from keystone import test
 
 import default_fixtures
@@ -440,6 +439,137 @@ def test_upgrade_add_domain_tables(self):
         self.assertTableColumns('user_domain_metadata',
                                 ['user_id', 'domain_id', 'data'])
 
+    def test_metadata_table_migration(self):
+        # Scaffolding
+        session = self.Session()
+
+        self.upgrade(16)
+        domain_table = sqlalchemy.Table('domain', self.metadata, autoload=True)
+        user_table = sqlalchemy.Table('user', self.metadata, autoload=True)
+        role_table = sqlalchemy.Table('role', self.metadata, autoload=True)
+        project_table = sqlalchemy.Table(
+            'project', self.metadata, autoload=True)
+        metadata_table = sqlalchemy.Table(
+            'metadata', self.metadata, autoload=True)
+
+        # Create a Domain
+        domain = {'id': uuid.uuid4().hex,
+                  'name': uuid.uuid4().hex,
+                  'enabled': True}
+        self.engine.execute(domain_table.insert().values(domain))
+
+        # Create a Project
+        project = {'id': uuid.uuid4().hex,
+                   'name': uuid.uuid4().hex,
+                   'domain_id': domain['id'],
+                   'extra': "{}"}
+        self.engine.execute(project_table.insert().values(project))
+
+        # Create another Project
+        project2 = {'id': uuid.uuid4().hex,
+                    'name': uuid.uuid4().hex,
+                    'domain_id': domain['id'],
+                    'extra': "{}"}
+        self.engine.execute(project_table.insert().values(project2))
+
+        # Create a User
+        user = {'id': uuid.uuid4().hex,
+                'name': uuid.uuid4().hex,
+                'domain_id': domain['id'],
+                'password': uuid.uuid4().hex,
+                'enabled': True,
+                'extra': json.dumps({})}
+        self.engine.execute(user_table.insert().values(user))
+
+        # Create a Role
+        role = {'id': uuid.uuid4().hex,
+                'name': uuid.uuid4().hex}
+        self.engine.execute(role_table.insert().values(role))
+
+        # And another role
+        role2 = {'id': uuid.uuid4().hex,
+                 'name': uuid.uuid4().hex}
+        self.engine.execute(role_table.insert().values(role2))
+
+        # Grant Role to User
+        role_grant = {'user_id': user['id'],
+                      'tenant_id': project['id'],
+                      'data': json.dumps({"roles": [role['id']]})}
+        self.engine.execute(metadata_table.insert().values(role_grant))
+
+        role_grant = {'user_id': user['id'],
+                      'tenant_id': project2['id'],
+                      'data': json.dumps({"roles": [role2['id']]})}
+        self.engine.execute(metadata_table.insert().values(role_grant))
+
+        session.commit()
+
+        self.upgrade(17)
+
+        user_project_metadata_table = sqlalchemy.Table(
+            'user_project_metadata', self.metadata, autoload=True)
+
+        # Test user in project has role
+        r = session.execute('select data from metadata where user_id="%s"'
+                            'and tenant_id="%s"' %
+                            (user['id'], project['id']))
+        test_project1 = json.loads(r.fetchone()['data'])
+        self.assertEqual(len(test_project1['roles']), 1)
+        self.assertIn(role['id'], test_project1['roles'])
+
+        # Test user in project2 has role2
+        r = session.execute('select data from metadata where user_id="%s"'
+                            ' and tenant_id="%s"' %
+                            (user['id'], project2['id']))
+        test_project2 = json.loads(r.fetchone()['data'])
+        self.assertEqual(len(test_project2['roles']), 1)
+        self.assertIn(role2['id'], test_project2['roles'])
+
+        # Test for user in project has role in user_project_metadata
+        # Migration 17 does not properly migrate this data, so this should
+        # be None.
+        r = session.execute('select data from user_project_metadata where '
+                            'user_id="%s" and project_id="%s"' %
+                            (user['id'], project['id']))
+        self.assertIsNone(r.fetchone())
+
+        # Create a conflicting user-project in user_project_metadata with
+        # a different role
+        data = json.dumps({"roles": [role2['id']]})
+        role_grant = {'user_id': user['id'],
+                      'project_id': project['id'],
+                      'data': data}
+        cmd = user_project_metadata_table.insert().values(role_grant)
+        self.engine.execute(cmd)
+        # End Scaffolding
+
+        # Migrate to 20
+        self.upgrade(20)
+
+        # The user-project pairs should have all roles from the previous
+        # metadata table in addition to any roles currently in
+        # user_project_metadata
+        r = session.execute('select data from user_project_metadata '
+                            'where user_id="%s" and project_id="%s"' %
+                            (user['id'], project['id']))
+        role_ids = json.loads(r.fetchone()['data'])['roles']
+        self.assertEqual(len(role_ids), 3)
+        self.assertIn(CONF.member_role_id, role_ids)
+        self.assertIn(role['id'], role_ids)
+        self.assertIn(role2['id'], role_ids)
+
+        # pairs that only existed in old metadata table should be in
+        # user_project_metadata
+        r = session.execute('select data from user_project_metadata where '
+                            'user_id="%s" and project_id="%s"' %
+                            (user['id'], project2['id']))
+        role_ids = json.loads(r.fetchone()['data'])['roles']
+        self.assertEqual(len(role_ids), 2)
+        self.assertIn(CONF.member_role_id, role_ids)
+        self.assertIn(role2['id'], role_ids)
+
+        self.assertTableDoesNotExist('metadata')
+
     def test_upgrade_default_roles(self):
         def count_member_roles():
             session = self.Session()
@@ -673,9 +803,7 @@ def assertTableDoesNotExist(self, table_name):
         try:
             temp_metadata = sqlalchemy.MetaData()
             temp_metadata.bind = self.engine
-            table = sqlalchemy.Table(table_name,
-                                     temp_metadata,
-                                     autoload=True)
+            sqlalchemy.Table(table_name, temp_metadata, autoload=True)
         except sqlalchemy.exc.NoSuchTableError:
             pass
         else: