Key manager drivers support schema migration.

* Script to run migration swift-key-manager-sync.
* KeyDriver interface extended by sync method.
* Intergration with SQLAlchemy Migration for SQLDriver.

Change-Id: I2c482e994668fcb55c6a3b11158ae3c65d43d33e

MANIFEST.in

@@ -3,6 +3,8 @@ include CHANGELOG README.md
 include babel.cfg
 include test/sample.conf
 include tox.ini
+include swift/common/key_manager/drivers/sql/migrate_repo/README
+include swift/common/key_manager/drivers/sql/migrate_repo/migrate.cfg
 graft doc
 graft etc
 graft locale

bin/swift-key-manager-sync

@@ -0,0 +1,29 @@
+#!/usr/bin/env python
+# Copyright (c) 2010-2012 OpenStack, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from optparse import OptionParser
+from paste.deploy import loadwsgi
+
+from swift.common.utils import parse_options
+from swift.common.key_manager import migration
+
+
+if __name__ == '__main__':
+    parser = OptionParser("%prog PROXY_SERVER_CONFIG [options]")
+    parser.add_option('-f', '--filter-section', default='key-manager',
+                      help='Key manager filter section.')
+    conf_file, options = parse_options(parser=parser)
+    migration.synchronize(conf_file, options['filter_section'])

setup.py

@@ -35,6 +35,7 @@
     author_email='openstack-admins@lists.launchpad.net',
     url='https://launchpad.net/swift',
     packages=find_packages(exclude=['test', 'bin']),
+    include_package_data=True,
     test_suite='nose.collector',
     classifiers=[
         'Development Status :: 5 - Production/Stable',
@@ -66,6 +67,7 @@
         'bin/swift-form-signature',
         'bin/swift-get-nodes',
         'bin/swift-init',
+        'bin/swift-key-manager-sync',
         'bin/swift-object-auditor',
         'bin/swift-object-expirer',
         'bin/swift-object-info',

swift/common/key_manager/drivers/base.py

@@ -47,3 +47,10 @@ def get_key_id(self, account):
         """
         raise NotImplementedError("Not implemented get_key_id function. "
                                   "Maybe incorrect driver")
+
+    def sync(self):
+        """
+        Some synchronization operations can be achieved in cases of changing
+        a structure of data storage.
+        """
+        raise NotImplementedError()

swift/common/key_manager/drivers/dummy.py

@@ -16,10 +16,10 @@
 
 import hashlib
 
-from .base import KeyDriver
+from swift.common.key_manager.drivers import base
 
 
-class DummyDriver(KeyDriver):
+class DummyDriver(base.KeyDriver):
     """
     Dummy key manager dirver which just return md5sum of account name
     and key_id equal key.
@@ -42,3 +42,10 @@ def get_key_id(self, account):
         :returns: key which actually equal to md5sum of account
         """
         return hashlib.md5(account).hexdigest()
+
+    def sync(self):
+        """
+        The dummy driver doesn't need to synchronize data storage
+        schemas.
+        """
+        pass

swift/common/key_manager/drivers/fake.py

@@ -18,10 +18,10 @@
 Fake methods for testing other implemented components.
 """
 
-from .base import KeyDriver
+from swift.common.key_manager.drivers import base
 
 
-class FakeDriver(KeyDriver):
+class FakeDriver(base.KeyDriver):
     """ Fake driver for testing key store services """
 
     def get_key(self, key_id):
@@ -43,3 +43,10 @@ def get_key_id(self, account):
         """
         key_id = 12345
         return key_id
+
+    def sync(self):
+        """
+        The fake driver doesn't need to synchronize data storage
+        schemas.
+        """
+        pass

swift/common/key_manager/drivers/sql/__init__.py

@@ -0,0 +1,18 @@
+# Copyright (c) 2010-2012 OpenStack, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Implementation of a key management driver to store into SQL.
+"""
+from swift.common.key_manager.drivers.sql.driver import SQLDriver

swift/common/key_manager/drivers/sql.py → swift/common/key_manager/drivers/sql/driver.py

@@ -27,8 +27,11 @@
 from sqlalchemy import create_engine, exc
 from sqlalchemy.schema import MetaData, Table, Column
 from sqlalchemy.types import String, Integer
+from migrate.versioning import api as versioning_api
+from migrate import exceptions as versioning_exceptions
 
-from .base import KeyDriver
+from swift.common.key_manager.drivers import base
+from swift.common.key_manager.drivers.sql import migrate_repo
 
 
 meta = MetaData()
@@ -39,14 +42,14 @@
 )
 
 
-class SQLDriver(KeyDriver):
+class SQLDriver(base.KeyDriver):
     """
     Driver for cooperation proxy and object servers with keys storage.
     """
     default_connection_attempts = 5
     default_connection_url = 'sqlite:///keystore.sqlite'
 
-    def __init__(self, conf, initialize_table=True):
+    def __init__(self, conf, initialize_table=False):
         """
         Initialization function.
 
@@ -161,3 +164,14 @@ def find_value(self, col_name, val):
         column = key_info_table.c[col_name]
         # use select method
         return key_info_table.select(column == val).execute().fetchall()
+
+    def sync(self):
+        """
+        Migrate database schemas.
+        """
+        repo_path = os.path.abspath(os.path.dirname(migrate_repo.__file__))
+        try:
+            versioning_api.upgrade(self.connection_url, repo_path)
+        except versioning_exceptions.DatabaseNotControlledError:
+            versioning_api.version_control(self.connection_url, repo_path)
+            versioning_api.upgrade(self.connection_url, repo_path)

swift/common/key_manager/drivers/sql/migrate_repo/README

@@ -0,0 +1,4 @@
+This is a database migration repository.
+
+More information at
+http://code.google.com/p/sqlalchemy-migrate/

swift/common/key_manager/drivers/sql/migrate_repo/manage.py

@@ -0,0 +1,5 @@
+#!/usr/bin/env python
+from migrate.versioning.shell import main
+
+if __name__ == '__main__':
+    main(debug='False')

swift/common/key_manager/drivers/sql/migrate_repo/migrate.cfg

@@ -0,0 +1,25 @@
+[db_settings]
+# Used to identify which repository this database is versioned under.
+# You can use the name of your project.
+repository_id=swift-key-manager
+
+# The name of the database table used to track the schema version.
+# This name shouldn't already be used by your project.
+# If this is changed once a database is under version control, you'll need to
+# change the table name in each database too.
+version_table=migrate_version
+
+# When committing a change script, Migrate will attempt to generate the
+# sql for all supported databases; normally, if one of them fails - probably
+# because you don't have that database installed - it is ignored and the
+# commit continues, perhaps ending successfully.
+# Databases in this list MUST compile successfully during a commit, or the
+# entire commit will fail. List the databases your application will actually
+# be using to ensure your updates to that database work properly.
+# This must be a list; example: ['postgres','sqlite']
+required_dbs=[]
+
+# When creating new change scripts, Migrate will stamp the new script with
+# a version number. By default this is latest_version + 1. You can set this
+# to 'true' to tell Migrate to use the UTC timestamp instead.
+use_timestamp_numbering=False

swift/common/key_manager/drivers/sql/migrate_repo/versions/001_Initialize_tables.py

@@ -0,0 +1,31 @@
+# Copyright (c) 2010-2012 OpenStack, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from sqlalchemy import MetaData, Table, Column, String, Integer
+
+
+def upgrade(migrate_engine):
+    meta = MetaData(bind=migrate_engine)
+    table = Table('key_info', meta,
+                  Column('account', String(30)),
+                  Column('key_id', Integer, primary_key=True,
+                         autoincrement=True),
+                  Column('encryption_key', String(30)))
+    table.create()
+
+
+def downgrade(migrate_engine):
+    meta = MetaData(bind=migrate_engine)
+    table = Table('key_info', meta, autoload=True)
+    table.remove()

swift/common/key_manager/migration.py

@@ -0,0 +1,50 @@
+# Copyright (c) 2010-2012 OpenStack, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Management the schema of key store.
+"""
+from paste.deploy import loadwsgi
+
+from swift.common.utils import create_instance
+from swift.common.key_manager.drivers.base import KeyDriver
+
+
+def migrate(conf, driver):
+    """
+    Upgrading the schemes of the key store.
+
+    :param conf: Application configuration.
+    :param driver: Import path of a driver.
+    """
+    key_manager = create_instance(driver, KeyDriver, conf)
+    key_manager.sync()
+
+
+def synchronize(conf_file, filter_section):
+    """
+    Process schema synchronization.
+
+    :param conf_file: Filename of configuration path.
+    :param filter_section: Name of key_management filter section.
+    """
+    context = loadwsgi.loadcontext(loadwsgi.FILTER, "config:%s" % (conf_file,),
+                                   name=filter_section)
+    conf = context.config()
+    driver = conf.get('crypto_keystore_driver')
+    if driver:
+        #NOTE(ikharin): The operation of data schema synchronization is
+        #               not required for the default driver if it's not
+        #               specified into configuration.
+        migrate(conf, driver)

test/unit/common/test_key_manager.py

@@ -19,8 +19,10 @@
 import mock
 
 from sqlalchemy import exc
+from migrate.exceptions import DatabaseNotControlledError
 
-from swift.common.key_manager.drivers.sql import SQLDriver, key_info_table
+from swift.common.key_manager.drivers.sql import SQLDriver
+from swift.common.key_manager.drivers.sql.driver import key_info_table
 
 
 class TestSQLDriver(unittest.TestCase):
@@ -128,6 +130,30 @@ def test_get_key(self):
         for val in test:
             self.assertRaises(TypeError, self.key_driver.get_key, val)
 
+    @mock.patch('migrate.versioning.api.upgrade')
+    def test_sync_success(self, mock_upgrade):
+        """
+        Successful migration.
+        """
+        self.key_driver.create_table()
+
+        self.key_driver.sync()
+        mock_upgrade.assert_called_once_with(self.url, mock.ANY)
+
+    @mock.patch('migrate.versioning.api.version_control')
+    @mock.patch('migrate.versioning.api.upgrade')
+    def test_sync_failed(self, mock_upgrade, mock_version_control):
+        """
+        Version control table doesn't exist.
+        """
+        self.key_driver.create_table()
+
+        mock_upgrade.side_effect = [DatabaseNotControlledError, None]
+        self.key_driver.sync()
+        mock_upgrade.assert_has_calls(2 * [mock.call(self.url, mock.ANY)])
+        mock_version_control.assert_called_once_with(self.url, mock.ANY)
+
+
 class TestSQLDriverReconnection(unittest.TestCase):
     def setUp(self):
         """
@@ -136,9 +162,9 @@ def setUp(self):
         """
         self.conf = {'crypto_keystore_sql_url': 'fake'}
         self.patcher = mock.patch(
-            'swift.common.key_manager.drivers.sql.create_engine')
+            'swift.common.key_manager.drivers.sql.driver.create_engine')
         self.mock_create_engine = self.patcher.start()
-        self.key_driver = SQLDriver(self.conf)#, initialize_table=False)
+        self.key_driver = SQLDriver(self.conf, initialize_table=True)
         self.mock_connect = self.mock_create_engine.return_value.connect
 
     def tearDown(self):

tools/pip-requires

@@ -5,5 +5,6 @@ pastedeploy>=1.3.3
 simplejson>=2.0.9
 xattr>=0.4
 python-swiftclient
-sqlalchemy
+sqlalchemy==0.7.2
+sqlalchemy-migrate==0.7.2
 M2Crypto