Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Key manager drivers support schema migration.

* Script to run migration swift-key-manager-sync.
* KeyDriver interface extended by sync method.
* Intergration with SQLAlchemy Migration for SQLDriver.

Change-Id: I2c482e994668fcb55c6a3b11158ae3c65d43d33e
  • Loading branch information...
commit 6b2a1759679d944c4835530056ccec911e4f6b7a 1 parent 350bfd2
@akscram akscram authored skraynev committed
View
2  MANIFEST.in
@@ -3,6 +3,8 @@ include CHANGELOG README.md
include babel.cfg
include test/sample.conf
include tox.ini
+include swift/common/key_manager/drivers/sql/migrate_repo/README
+include swift/common/key_manager/drivers/sql/migrate_repo/migrate.cfg
graft doc
graft etc
graft locale
View
29 bin/swift-key-manager-sync
@@ -0,0 +1,29 @@
+#!/usr/bin/env python
+# Copyright (c) 2010-2012 OpenStack, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from optparse import OptionParser
+from paste.deploy import loadwsgi
+
+from swift.common.utils import parse_options
+from swift.common.key_manager import migration
+
+
+if __name__ == '__main__':
+ parser = OptionParser("%prog PROXY_SERVER_CONFIG [options]")
+ parser.add_option('-f', '--filter-section', default='key-manager',
+ help='Key manager filter section.')
+ conf_file, options = parse_options(parser=parser)
+ migration.synchronize(conf_file, options['filter_section'])
View
2  setup.py
@@ -35,6 +35,7 @@
author_email='openstack-admins@lists.launchpad.net',
url='https://launchpad.net/swift',
packages=find_packages(exclude=['test', 'bin']),
+ include_package_data=True,
test_suite='nose.collector',
classifiers=[
'Development Status :: 5 - Production/Stable',
@@ -66,6 +67,7 @@
'bin/swift-form-signature',
'bin/swift-get-nodes',
'bin/swift-init',
+ 'bin/swift-key-manager-sync',
'bin/swift-object-auditor',
'bin/swift-object-expirer',
'bin/swift-object-info',
View
7 swift/common/key_manager/drivers/base.py
@@ -47,3 +47,10 @@ def get_key_id(self, account):
"""
raise NotImplementedError("Not implemented get_key_id function. "
"Maybe incorrect driver")
+
+ def sync(self):
+ """
+ Some synchronization operations can be achieved in cases of changing
+ a structure of data storage.
+ """
+ raise NotImplementedError()
View
11 swift/common/key_manager/drivers/dummy.py
@@ -16,10 +16,10 @@
import hashlib
-from .base import KeyDriver
+from swift.common.key_manager.drivers import base
-class DummyDriver(KeyDriver):
+class DummyDriver(base.KeyDriver):
"""
Dummy key manager dirver which just return md5sum of account name
and key_id equal key.
@@ -42,3 +42,10 @@ def get_key_id(self, account):
:returns: key which actually equal to md5sum of account
"""
return hashlib.md5(account).hexdigest()
+
+ def sync(self):
+ """
+ The dummy driver doesn't need to synchronize data storage
+ schemas.
+ """
+ pass
View
11 swift/common/key_manager/drivers/fake.py
@@ -18,10 +18,10 @@
Fake methods for testing other implemented components.
"""
-from .base import KeyDriver
+from swift.common.key_manager.drivers import base
-class FakeDriver(KeyDriver):
+class FakeDriver(base.KeyDriver):
""" Fake driver for testing key store services """
def get_key(self, key_id):
@@ -43,3 +43,10 @@ def get_key_id(self, account):
"""
key_id = 12345
return key_id
+
+ def sync(self):
+ """
+ The fake driver doesn't need to synchronize data storage
+ schemas.
+ """
+ pass
View
18 swift/common/key_manager/drivers/sql/__init__.py
@@ -0,0 +1,18 @@
+# Copyright (c) 2010-2012 OpenStack, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Implementation of a key management driver to store into SQL.
+"""
+from swift.common.key_manager.drivers.sql.driver import SQLDriver
View
20 swift/common/key_manager/drivers/sql.py → swift/common/key_manager/drivers/sql/driver.py
@@ -27,8 +27,11 @@
from sqlalchemy import create_engine, exc
from sqlalchemy.schema import MetaData, Table, Column
from sqlalchemy.types import String, Integer
+from migrate.versioning import api as versioning_api
+from migrate import exceptions as versioning_exceptions
-from .base import KeyDriver
+from swift.common.key_manager.drivers import base
+from swift.common.key_manager.drivers.sql import migrate_repo
meta = MetaData()
@@ -39,14 +42,14 @@
)
-class SQLDriver(KeyDriver):
+class SQLDriver(base.KeyDriver):
"""
Driver for cooperation proxy and object servers with keys storage.
"""
default_connection_attempts = 5
default_connection_url = 'sqlite:///keystore.sqlite'
- def __init__(self, conf, initialize_table=True):
+ def __init__(self, conf, initialize_table=False):
"""
Initialization function.
@@ -161,3 +164,14 @@ def find_value(self, col_name, val):
column = key_info_table.c[col_name]
# use select method
return key_info_table.select(column == val).execute().fetchall()
+
+ def sync(self):
+ """
+ Migrate database schemas.
+ """
+ repo_path = os.path.abspath(os.path.dirname(migrate_repo.__file__))
+ try:
+ versioning_api.upgrade(self.connection_url, repo_path)
+ except versioning_exceptions.DatabaseNotControlledError:
+ versioning_api.version_control(self.connection_url, repo_path)
+ versioning_api.upgrade(self.connection_url, repo_path)
View
4 swift/common/key_manager/drivers/sql/migrate_repo/README
@@ -0,0 +1,4 @@
+This is a database migration repository.
+
+More information at
+http://code.google.com/p/sqlalchemy-migrate/
View
0  swift/common/key_manager/drivers/sql/migrate_repo/__init__.py
No changes.
View
5 swift/common/key_manager/drivers/sql/migrate_repo/manage.py
@@ -0,0 +1,5 @@
+#!/usr/bin/env python
+from migrate.versioning.shell import main
+
+if __name__ == '__main__':
+ main(debug='False')
View
25 swift/common/key_manager/drivers/sql/migrate_repo/migrate.cfg
@@ -0,0 +1,25 @@
+[db_settings]
+# Used to identify which repository this database is versioned under.
+# You can use the name of your project.
+repository_id=swift-key-manager
+
+# The name of the database table used to track the schema version.
+# This name shouldn't already be used by your project.
+# If this is changed once a database is under version control, you'll need to
+# change the table name in each database too.
+version_table=migrate_version
+
+# When committing a change script, Migrate will attempt to generate the
+# sql for all supported databases; normally, if one of them fails - probably
+# because you don't have that database installed - it is ignored and the
+# commit continues, perhaps ending successfully.
+# Databases in this list MUST compile successfully during a commit, or the
+# entire commit will fail. List the databases your application will actually
+# be using to ensure your updates to that database work properly.
+# This must be a list; example: ['postgres','sqlite']
+required_dbs=[]
+
+# When creating new change scripts, Migrate will stamp the new script with
+# a version number. By default this is latest_version + 1. You can set this
+# to 'true' to tell Migrate to use the UTC timestamp instead.
+use_timestamp_numbering=False
View
31 swift/common/key_manager/drivers/sql/migrate_repo/versions/001_Initialize_tables.py
@@ -0,0 +1,31 @@
+# Copyright (c) 2010-2012 OpenStack, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from sqlalchemy import MetaData, Table, Column, String, Integer
+
+
+def upgrade(migrate_engine):
+ meta = MetaData(bind=migrate_engine)
+ table = Table('key_info', meta,
+ Column('account', String(30)),
+ Column('key_id', Integer, primary_key=True,
+ autoincrement=True),
+ Column('encryption_key', String(30)))
+ table.create()
+
+
+def downgrade(migrate_engine):
+ meta = MetaData(bind=migrate_engine)
+ table = Table('key_info', meta, autoload=True)
+ table.remove()
View
0  swift/common/key_manager/drivers/sql/migrate_repo/versions/__init__.py
No changes.
View
50 swift/common/key_manager/migration.py
@@ -0,0 +1,50 @@
+# Copyright (c) 2010-2012 OpenStack, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Management the schema of key store.
+"""
+from paste.deploy import loadwsgi
+
+from swift.common.utils import create_instance
+from swift.common.key_manager.drivers.base import KeyDriver
+
+
+def migrate(conf, driver):
+ """
+ Upgrading the schemes of the key store.
+
+ :param conf: Application configuration.
+ :param driver: Import path of a driver.
+ """
+ key_manager = create_instance(driver, KeyDriver, conf)
+ key_manager.sync()
+
+
+def synchronize(conf_file, filter_section):
+ """
+ Process schema synchronization.
+
+ :param conf_file: Filename of configuration path.
+ :param filter_section: Name of key_management filter section.
+ """
+ context = loadwsgi.loadcontext(loadwsgi.FILTER, "config:%s" % (conf_file,),
+ name=filter_section)
+ conf = context.config()
+ driver = conf.get('crypto_keystore_driver')
+ if driver:
+ #NOTE(ikharin): The operation of data schema synchronization is
+ # not required for the default driver if it's not
+ # specified into configuration.
+ migrate(conf, driver)
View
32 test/unit/common/test_key_manager.py
@@ -19,8 +19,10 @@
import mock
from sqlalchemy import exc
+from migrate.exceptions import DatabaseNotControlledError
-from swift.common.key_manager.drivers.sql import SQLDriver, key_info_table
+from swift.common.key_manager.drivers.sql import SQLDriver
+from swift.common.key_manager.drivers.sql.driver import key_info_table
class TestSQLDriver(unittest.TestCase):
@@ -128,6 +130,30 @@ def test_get_key(self):
for val in test:
self.assertRaises(TypeError, self.key_driver.get_key, val)
+ @mock.patch('migrate.versioning.api.upgrade')
+ def test_sync_success(self, mock_upgrade):
+ """
+ Successful migration.
+ """
+ self.key_driver.create_table()
+
+ self.key_driver.sync()
+ mock_upgrade.assert_called_once_with(self.url, mock.ANY)
+
+ @mock.patch('migrate.versioning.api.version_control')
+ @mock.patch('migrate.versioning.api.upgrade')
+ def test_sync_failed(self, mock_upgrade, mock_version_control):
+ """
+ Version control table doesn't exist.
+ """
+ self.key_driver.create_table()
+
+ mock_upgrade.side_effect = [DatabaseNotControlledError, None]
+ self.key_driver.sync()
+ mock_upgrade.assert_has_calls(2 * [mock.call(self.url, mock.ANY)])
+ mock_version_control.assert_called_once_with(self.url, mock.ANY)
+
+
class TestSQLDriverReconnection(unittest.TestCase):
def setUp(self):
"""
@@ -136,9 +162,9 @@ def setUp(self):
"""
self.conf = {'crypto_keystore_sql_url': 'fake'}
self.patcher = mock.patch(
- 'swift.common.key_manager.drivers.sql.create_engine')
+ 'swift.common.key_manager.drivers.sql.driver.create_engine')
self.mock_create_engine = self.patcher.start()
- self.key_driver = SQLDriver(self.conf)#, initialize_table=False)
+ self.key_driver = SQLDriver(self.conf, initialize_table=True)
self.mock_connect = self.mock_create_engine.return_value.connect
def tearDown(self):
View
3  tools/pip-requires
@@ -5,5 +5,6 @@ pastedeploy>=1.3.3
simplejson>=2.0.9
xattr>=0.4
python-swiftclient
-sqlalchemy
+sqlalchemy==0.7.2
+sqlalchemy-migrate==0.7.2
M2Crypto
Please sign in to comment.
Something went wrong with that request. Please try again.