diff --git a/MANIFEST.in b/MANIFEST.in index 73ea9e2e..8ef7cd23 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,6 +3,8 @@ include CHANGELOG README.md include babel.cfg include test/sample.conf include tox.ini +include swift/common/key_manager/drivers/sql/migrate_repo/README +include swift/common/key_manager/drivers/sql/migrate_repo/migrate.cfg graft doc graft etc graft locale diff --git a/bin/swift-key-manager-sync b/bin/swift-key-manager-sync new file mode 100755 index 00000000..7391c5fd --- /dev/null +++ b/bin/swift-key-manager-sync @@ -0,0 +1,29 @@ +#!/usr/bin/env python +# Copyright (c) 2010-2012 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from optparse import OptionParser +from paste.deploy import loadwsgi + +from swift.common.utils import parse_options +from swift.common.key_manager import migration + + +if __name__ == '__main__': + parser = OptionParser("%prog PROXY_SERVER_CONFIG [options]") + parser.add_option('-f', '--filter-section', default='key-manager', + help='Key manager filter section.') + conf_file, options = parse_options(parser=parser) + migration.synchronize(conf_file, options['filter_section']) diff --git a/setup.py b/setup.py index cf92c2de..9099aa75 100644 --- a/setup.py +++ b/setup.py @@ -35,6 +35,7 @@ author_email='openstack-admins@lists.launchpad.net', url='https://launchpad.net/swift', packages=find_packages(exclude=['test', 'bin']), + include_package_data=True, test_suite='nose.collector', classifiers=[ 'Development Status :: 5 - Production/Stable', @@ -66,6 +67,7 @@ 'bin/swift-form-signature', 'bin/swift-get-nodes', 'bin/swift-init', + 'bin/swift-key-manager-sync', 'bin/swift-object-auditor', 'bin/swift-object-expirer', 'bin/swift-object-info', diff --git a/swift/common/key_manager/drivers/base.py b/swift/common/key_manager/drivers/base.py index 2b364799..9481927f 100644 --- a/swift/common/key_manager/drivers/base.py +++ b/swift/common/key_manager/drivers/base.py @@ -47,3 +47,10 @@ def get_key_id(self, account): """ raise NotImplementedError("Not implemented get_key_id function. " "Maybe incorrect driver") + + def sync(self): + """ + Some synchronization operations can be achieved in cases of changing + a structure of data storage. + """ + raise NotImplementedError() diff --git a/swift/common/key_manager/drivers/dummy.py b/swift/common/key_manager/drivers/dummy.py index 6ab13184..1fe6c0ff 100644 --- a/swift/common/key_manager/drivers/dummy.py +++ b/swift/common/key_manager/drivers/dummy.py @@ -16,10 +16,10 @@ import hashlib -from .base import KeyDriver +from swift.common.key_manager.drivers import base -class DummyDriver(KeyDriver): +class DummyDriver(base.KeyDriver): """ Dummy key manager dirver which just return md5sum of account name and key_id equal key. @@ -42,3 +42,10 @@ def get_key_id(self, account): :returns: key which actually equal to md5sum of account """ return hashlib.md5(account).hexdigest() + + def sync(self): + """ + The dummy driver doesn't need to synchronize data storage + schemas. + """ + pass diff --git a/swift/common/key_manager/drivers/fake.py b/swift/common/key_manager/drivers/fake.py index fe6fd985..d5326f29 100644 --- a/swift/common/key_manager/drivers/fake.py +++ b/swift/common/key_manager/drivers/fake.py @@ -18,10 +18,10 @@ Fake methods for testing other implemented components. """ -from .base import KeyDriver +from swift.common.key_manager.drivers import base -class FakeDriver(KeyDriver): +class FakeDriver(base.KeyDriver): """ Fake driver for testing key store services """ def get_key(self, key_id): @@ -43,3 +43,10 @@ def get_key_id(self, account): """ key_id = 12345 return key_id + + def sync(self): + """ + The fake driver doesn't need to synchronize data storage + schemas. + """ + pass diff --git a/swift/common/key_manager/drivers/sql/__init__.py b/swift/common/key_manager/drivers/sql/__init__.py new file mode 100644 index 00000000..034057f8 --- /dev/null +++ b/swift/common/key_manager/drivers/sql/__init__.py @@ -0,0 +1,18 @@ +# Copyright (c) 2010-2012 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Implementation of a key management driver to store into SQL. +""" +from swift.common.key_manager.drivers.sql.driver import SQLDriver diff --git a/swift/common/key_manager/drivers/sql.py b/swift/common/key_manager/drivers/sql/driver.py similarity index 87% rename from swift/common/key_manager/drivers/sql.py rename to swift/common/key_manager/drivers/sql/driver.py index 7e0b8ea8..2e3c70f4 100644 --- a/swift/common/key_manager/drivers/sql.py +++ b/swift/common/key_manager/drivers/sql/driver.py @@ -27,8 +27,11 @@ from sqlalchemy import create_engine, exc from sqlalchemy.schema import MetaData, Table, Column from sqlalchemy.types import String, Integer +from migrate.versioning import api as versioning_api +from migrate import exceptions as versioning_exceptions -from .base import KeyDriver +from swift.common.key_manager.drivers import base +from swift.common.key_manager.drivers.sql import migrate_repo meta = MetaData() @@ -39,14 +42,14 @@ ) -class SQLDriver(KeyDriver): +class SQLDriver(base.KeyDriver): """ Driver for cooperation proxy and object servers with keys storage. """ default_connection_attempts = 5 default_connection_url = 'sqlite:///keystore.sqlite' - def __init__(self, conf, initialize_table=True): + def __init__(self, conf, initialize_table=False): """ Initialization function. @@ -161,3 +164,14 @@ def find_value(self, col_name, val): column = key_info_table.c[col_name] # use select method return key_info_table.select(column == val).execute().fetchall() + + def sync(self): + """ + Migrate database schemas. + """ + repo_path = os.path.abspath(os.path.dirname(migrate_repo.__file__)) + try: + versioning_api.upgrade(self.connection_url, repo_path) + except versioning_exceptions.DatabaseNotControlledError: + versioning_api.version_control(self.connection_url, repo_path) + versioning_api.upgrade(self.connection_url, repo_path) diff --git a/swift/common/key_manager/drivers/sql/migrate_repo/README b/swift/common/key_manager/drivers/sql/migrate_repo/README new file mode 100644 index 00000000..6218f8ca --- /dev/null +++ b/swift/common/key_manager/drivers/sql/migrate_repo/README @@ -0,0 +1,4 @@ +This is a database migration repository. + +More information at +http://code.google.com/p/sqlalchemy-migrate/ diff --git a/swift/common/key_manager/drivers/sql/migrate_repo/__init__.py b/swift/common/key_manager/drivers/sql/migrate_repo/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/swift/common/key_manager/drivers/sql/migrate_repo/manage.py b/swift/common/key_manager/drivers/sql/migrate_repo/manage.py new file mode 100644 index 00000000..39fa3892 --- /dev/null +++ b/swift/common/key_manager/drivers/sql/migrate_repo/manage.py @@ -0,0 +1,5 @@ +#!/usr/bin/env python +from migrate.versioning.shell import main + +if __name__ == '__main__': + main(debug='False') diff --git a/swift/common/key_manager/drivers/sql/migrate_repo/migrate.cfg b/swift/common/key_manager/drivers/sql/migrate_repo/migrate.cfg new file mode 100644 index 00000000..719406cf --- /dev/null +++ b/swift/common/key_manager/drivers/sql/migrate_repo/migrate.cfg @@ -0,0 +1,25 @@ +[db_settings] +# Used to identify which repository this database is versioned under. +# You can use the name of your project. +repository_id=swift-key-manager + +# The name of the database table used to track the schema version. +# This name shouldn't already be used by your project. +# If this is changed once a database is under version control, you'll need to +# change the table name in each database too. +version_table=migrate_version + +# When committing a change script, Migrate will attempt to generate the +# sql for all supported databases; normally, if one of them fails - probably +# because you don't have that database installed - it is ignored and the +# commit continues, perhaps ending successfully. +# Databases in this list MUST compile successfully during a commit, or the +# entire commit will fail. List the databases your application will actually +# be using to ensure your updates to that database work properly. +# This must be a list; example: ['postgres','sqlite'] +required_dbs=[] + +# When creating new change scripts, Migrate will stamp the new script with +# a version number. By default this is latest_version + 1. You can set this +# to 'true' to tell Migrate to use the UTC timestamp instead. +use_timestamp_numbering=False diff --git a/swift/common/key_manager/drivers/sql/migrate_repo/versions/001_Initialize_tables.py b/swift/common/key_manager/drivers/sql/migrate_repo/versions/001_Initialize_tables.py new file mode 100644 index 00000000..25776ade --- /dev/null +++ b/swift/common/key_manager/drivers/sql/migrate_repo/versions/001_Initialize_tables.py @@ -0,0 +1,31 @@ +# Copyright (c) 2010-2012 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from sqlalchemy import MetaData, Table, Column, String, Integer + + +def upgrade(migrate_engine): + meta = MetaData(bind=migrate_engine) + table = Table('key_info', meta, + Column('account', String(30)), + Column('key_id', Integer, primary_key=True, + autoincrement=True), + Column('encryption_key', String(30))) + table.create() + + +def downgrade(migrate_engine): + meta = MetaData(bind=migrate_engine) + table = Table('key_info', meta, autoload=True) + table.remove() diff --git a/swift/common/key_manager/drivers/sql/migrate_repo/versions/__init__.py b/swift/common/key_manager/drivers/sql/migrate_repo/versions/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/swift/common/key_manager/migration.py b/swift/common/key_manager/migration.py new file mode 100644 index 00000000..354b0151 --- /dev/null +++ b/swift/common/key_manager/migration.py @@ -0,0 +1,50 @@ +# Copyright (c) 2010-2012 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Management the schema of key store. +""" +from paste.deploy import loadwsgi + +from swift.common.utils import create_instance +from swift.common.key_manager.drivers.base import KeyDriver + + +def migrate(conf, driver): + """ + Upgrading the schemes of the key store. + + :param conf: Application configuration. + :param driver: Import path of a driver. + """ + key_manager = create_instance(driver, KeyDriver, conf) + key_manager.sync() + + +def synchronize(conf_file, filter_section): + """ + Process schema synchronization. + + :param conf_file: Filename of configuration path. + :param filter_section: Name of key_management filter section. + """ + context = loadwsgi.loadcontext(loadwsgi.FILTER, "config:%s" % (conf_file,), + name=filter_section) + conf = context.config() + driver = conf.get('crypto_keystore_driver') + if driver: + #NOTE(ikharin): The operation of data schema synchronization is + # not required for the default driver if it's not + # specified into configuration. + migrate(conf, driver) diff --git a/test/unit/common/test_key_manager.py b/test/unit/common/test_key_manager.py index 480181c5..54c8578d 100644 --- a/test/unit/common/test_key_manager.py +++ b/test/unit/common/test_key_manager.py @@ -19,8 +19,10 @@ import mock from sqlalchemy import exc +from migrate.exceptions import DatabaseNotControlledError -from swift.common.key_manager.drivers.sql import SQLDriver, key_info_table +from swift.common.key_manager.drivers.sql import SQLDriver +from swift.common.key_manager.drivers.sql.driver import key_info_table class TestSQLDriver(unittest.TestCase): @@ -128,6 +130,30 @@ def test_get_key(self): for val in test: self.assertRaises(TypeError, self.key_driver.get_key, val) + @mock.patch('migrate.versioning.api.upgrade') + def test_sync_success(self, mock_upgrade): + """ + Successful migration. + """ + self.key_driver.create_table() + + self.key_driver.sync() + mock_upgrade.assert_called_once_with(self.url, mock.ANY) + + @mock.patch('migrate.versioning.api.version_control') + @mock.patch('migrate.versioning.api.upgrade') + def test_sync_failed(self, mock_upgrade, mock_version_control): + """ + Version control table doesn't exist. + """ + self.key_driver.create_table() + + mock_upgrade.side_effect = [DatabaseNotControlledError, None] + self.key_driver.sync() + mock_upgrade.assert_has_calls(2 * [mock.call(self.url, mock.ANY)]) + mock_version_control.assert_called_once_with(self.url, mock.ANY) + + class TestSQLDriverReconnection(unittest.TestCase): def setUp(self): """ @@ -136,9 +162,9 @@ def setUp(self): """ self.conf = {'crypto_keystore_sql_url': 'fake'} self.patcher = mock.patch( - 'swift.common.key_manager.drivers.sql.create_engine') + 'swift.common.key_manager.drivers.sql.driver.create_engine') self.mock_create_engine = self.patcher.start() - self.key_driver = SQLDriver(self.conf)#, initialize_table=False) + self.key_driver = SQLDriver(self.conf, initialize_table=True) self.mock_connect = self.mock_create_engine.return_value.connect def tearDown(self): diff --git a/tools/pip-requires b/tools/pip-requires index 75874310..c45b8a32 100644 --- a/tools/pip-requires +++ b/tools/pip-requires @@ -5,5 +5,6 @@ pastedeploy>=1.3.3 simplejson>=2.0.9 xattr>=0.4 python-swiftclient -sqlalchemy +sqlalchemy==0.7.2 +sqlalchemy-migrate==0.7.2 M2Crypto