From e10ab58c6d7861d6ad1afbadcdf1e0f094c3f272 Mon Sep 17 00:00:00 2001 From: Alexander Bashmakov Date: Fri, 2 Sep 2016 20:57:21 +0000 Subject: [PATCH] WIP: Initial alembic migrations implementation. Includes Hemanth's alembic migrations defintions, updates to glance-manage utility to use alembic instead of sqlalchemy, and updates to unit tests. NOTE: technical debt exists for this patch: 1. It was discovered that no functional tests exist for the glance-manage utility, some should probably be added. 2. unit/test_mirations.py module should be updated to reflect the move to alembic. --- doc/source/db.rst | 4 +- doc/source/man/glancemanage.rst | 9 +- glance/cmd/manage.py | 133 ++++++++--- glance/db/migration.py | 10 + glance/db/sqlalchemy/alembic.ini | 68 ++++++ .../db/sqlalchemy/alembic_migrations/README | 1 + .../sqlalchemy/alembic_migrations/__init__.py | 0 .../add_artifacts_tables.py | 224 ++++++++++++++++++ .../alembic_migrations/add_images_tables.py | 201 ++++++++++++++++ .../alembic_migrations/add_metadefs_tables.py | 171 +++++++++++++ .../alembic_migrations/add_tasks_tables.py | 66 ++++++ .../sqlalchemy/alembic_migrations/alembic.ini | 68 ++++++ .../db/sqlalchemy/alembic_migrations/env.py | 88 +++++++ .../sqlalchemy/alembic_migrations/migrate.cfg | 20 ++ .../alembic_migrations/script.py.mako | 24 ++ .../versions/liberty_initial.py | 44 ++++ .../mitaka01_add_image_created_updated_idx.py | 51 ++++ .../mitaka02_update_metadef_os_nova_server.py | 46 ++++ .../newton_contract01_drop_is_public.py | 52 ++++ .../newton_expand01_add_visibility.py | 55 +++++ glance/tests/unit/test_manage.py | 141 +++++------ 21 files changed, 1369 insertions(+), 107 deletions(-) create mode 100644 glance/db/sqlalchemy/alembic.ini create mode 100644 glance/db/sqlalchemy/alembic_migrations/README create mode 100644 glance/db/sqlalchemy/alembic_migrations/__init__.py create mode 100644 glance/db/sqlalchemy/alembic_migrations/add_artifacts_tables.py create mode 100644 glance/db/sqlalchemy/alembic_migrations/add_images_tables.py create mode 100644 glance/db/sqlalchemy/alembic_migrations/add_metadefs_tables.py create mode 100644 glance/db/sqlalchemy/alembic_migrations/add_tasks_tables.py create mode 100644 glance/db/sqlalchemy/alembic_migrations/alembic.ini create mode 100644 glance/db/sqlalchemy/alembic_migrations/env.py create mode 100644 glance/db/sqlalchemy/alembic_migrations/migrate.cfg create mode 100644 glance/db/sqlalchemy/alembic_migrations/script.py.mako create mode 100644 glance/db/sqlalchemy/alembic_migrations/versions/liberty_initial.py create mode 100644 glance/db/sqlalchemy/alembic_migrations/versions/mitaka01_add_image_created_updated_idx.py create mode 100644 glance/db/sqlalchemy/alembic_migrations/versions/mitaka02_update_metadef_os_nova_server.py create mode 100644 glance/db/sqlalchemy/alembic_migrations/versions/newton_contract01_drop_is_public.py create mode 100644 glance/db/sqlalchemy/alembic_migrations/versions/newton_expand01_add_visibility.py diff --git a/doc/source/db.rst b/doc/source/db.rst index a57052a005..fc0a1501d2 100644 --- a/doc/source/db.rst +++ b/doc/source/db.rst @@ -29,9 +29,9 @@ The commands should be executed as a subcommand of 'db': Sync the Database ----------------- - glance-manage db sync + glance-manage db sync -Place a database under migration control and upgrade, creating it first if necessary. +Place an existing database under migration control and upgrade it. Determining the Database Version diff --git a/doc/source/man/glancemanage.rst b/doc/source/man/glancemanage.rst index 921c645ea4..b8addf3cba 100644 --- a/doc/source/man/glancemanage.rst +++ b/doc/source/man/glancemanage.rst @@ -53,9 +53,8 @@ COMMANDS **db_version_control** Place the database under migration control. - **db_sync ** - Place a database under migration control and upgrade, creating - it first if necessary. + **db_sync ** + Place an existing database under migration control and upgrade it. **db_export_metadefs** Export the metadata definitions into json format. By default the @@ -78,10 +77,6 @@ OPTIONS .. include:: general_options.rst - **--sql_connection=CONN_STRING** - A proper SQLAlchemy connection string as described - `here `_ - .. include:: footer.rst CONFIGURATION diff --git a/glance/cmd/manage.py b/glance/cmd/manage.py index 390729fb99..fbf77f9bb0 100644 --- a/glance/cmd/manage.py +++ b/glance/cmd/manage.py @@ -39,6 +39,10 @@ if os.path.exists(os.path.join(possible_topdir, 'glance', '__init__.py')): sys.path.insert(0, possible_topdir) +from alembic import command as alembic_command +from alembic import config as alembic_config +from alembic import migration as alembic_migration + from oslo_config import cfg from oslo_db.sqlalchemy import migration from oslo_log import log as logging @@ -73,39 +77,66 @@ def __init__(self): def version(self): """Print database's current migration level""" - print(migration.db_version(db_api.get_engine(), - db_migration.MIGRATE_REPO_PATH, - db_migration.INIT_VERSION)) + current_heads = _get_current_alembic_heads() + if current_heads: + # Migrations are managed by alembic + for head in current_heads: + print(head) + else: + # Migrations are managed by legacy versioning scheme + print(_get_current_legacy_head()) @args('--version', metavar='', help='Database version') - def upgrade(self, version=None): + def upgrade(self, version='heads'): """Upgrade the database's migration level""" - migration.db_sync(db_api.get_engine(), - db_migration.MIGRATE_REPO_PATH, - version) + a_config = _get_alembic_config() + + if not _get_current_alembic_heads(): + head = _get_current_legacy_head() + if head == 42: + alembic_command.stamp(a_config, 'liberty') + elif head == 43: + alembic_command.stamp(a_config, 'mitaka01') + elif head == 44: + alembic_command.stamp(a_config, 'mitaka02') + elif head != 0: + sys.exit(("Unknown database state, " + "please delete and run db_sync")) + + alembic_command.upgrade(a_config, version) + print("Upgraded to revision:", version) @args('--version', metavar='', help='Database version') - def version_control(self, version=None): + def version_control(self, version=db_migration.ALEMBIC_INIT_VERSION): """Place a database under migration control""" - migration.db_version_control(db_api.get_engine(), - db_migration.MIGRATE_REPO_PATH, - version) + a_config = _get_alembic_config() + alembic_command.stamp(a_config, version) + print("Stamped revision:", version) @args('--version', metavar='', help='Database version') - @args('--current_version', metavar='', - help='Current Database version') - def sync(self, version=None, current_version=None): + def sync(self, version='heads'): """ - Place a database under migration control and upgrade it, - creating first if necessary. + Place an existing database under migration control and upgrade it. """ - if current_version not in (None, 'None'): - migration.db_version_control(db_api.get_engine(), - db_migration.MIGRATE_REPO_PATH, - version=current_version) - migration.db_sync(db_api.get_engine(), - db_migration.MIGRATE_REPO_PATH, - version) + # TODO(abashmak): check if glance database exists and create if not. + # Note: this functionality never existed in previous versions, so + # this would be a robustness enhancements + self.upgrade(version) + + def expand(self): + """Run the expansion phase of a rolling upgrade procedure.""" + self.upgrade(db_migration.EXPAND_BRANCH) + + def data_migrate(self): + """Run the data migration phase of a rolling upgrade procedure.""" + raise NotImplementedError(("Data migration is currently done as" + " part of the contract phase")) + + def contract(self): + """Run the contraction phase of a rolling upgrade procedure.""" + # TODO(abashmak): determine if need to check here for expansion + # (and later data migration) being completed, to prevent operator error + self.upgrade(db_migration.CONTRACT_BRANCH) @args('--path', metavar='', help='Path to the directory or file ' 'where json metadata is stored') @@ -185,9 +216,17 @@ def upgrade(self, version=None): def version_control(self, version=None): self.command_object.version_control(CONF.command.version) - def sync(self, version=None, current_version=None): - self.command_object.sync(CONF.command.version, - CONF.command.current_version) + def sync(self, version=None): + self.command_object.sync(CONF.command.version) + + def expand(self): + self.command_object.expand() + + def data_migrate(self): + self.command_object.data_migrate() + + def contract(self): + self.command_object.contract() def load_metadefs(self, path=None, merge=False, prefer_new=False, overwrite=False): @@ -224,9 +263,20 @@ def add_legacy_command_parsers(command_object, subparsers): parser = subparsers.add_parser('db_sync') parser.set_defaults(action_fn=legacy_command_object.sync) parser.add_argument('version', nargs='?') - parser.add_argument('current_version', nargs='?') parser.set_defaults(action='db_sync') + parser = subparsers.add_parser('db_expand') + parser.set_defaults(action_fn=legacy_command_object.expand) + parser.set_defaults(action='db_expand') + + parser = subparsers.add_parser('db_data_migrate') + parser.set_defaults(action_fn=legacy_command_object.data_migrate) + parser.set_defaults(action='db_data_migrate') + + parser = subparsers.add_parser('db_contract') + parser.set_defaults(action_fn=legacy_command_object.contract) + parser.set_defaults(action='db_contract') + parser = subparsers.add_parser('db_load_metadefs') parser.set_defaults(action_fn=legacy_command_object.load_metadefs) parser.add_argument('path', nargs='?') @@ -253,7 +303,7 @@ def add_command_parsers(subparsers): category_subparsers = parser.add_subparsers(dest='action') - for (action, action_fn) in methods_of(command_object): + for (action, action_fn) in _methods_of(command_object): parser = category_subparsers.add_parser(action) action_kwargs = [] @@ -289,7 +339,32 @@ def add_command_parsers(subparsers): } -def methods_of(obj): +def _get_alembic_config(): + """Return a valid alembic config object""" + # TODO(abashmak) there has to be a better way to do this + ini_path = os.path.join(os.path.dirname(__file__), + '../db/sqlalchemy/alembic.ini') + config = alembic_config.Config(os.path.abspath(ini_path)) + dbconn = str(db_api.get_engine().url) + config.set_main_option('sqlalchemy.url', dbconn) + return config + + +def _get_current_alembic_heads(): + """Return current heads (if any) from the alembic migration table""" + engine = db_api.get_engine() + conn = engine.connect() + context = alembic_migration.MigrationContext.configure(conn) + return context.get_current_heads() + + +def _get_current_legacy_head(): + return migration.db_version(db_api.get_engine(), + db_migration.MIGRATE_REPO_PATH, + db_migration.INIT_VERSION) + + +def _methods_of(obj): """Get all callable methods of an object that don't start with underscore returns a list of tuples of the form (method_name, method) diff --git a/glance/db/migration.py b/glance/db/migration.py index 3709f0686d..ab4ad8791f 100644 --- a/glance/db/migration.py +++ b/glance/db/migration.py @@ -22,6 +22,7 @@ import os import threading + from oslo_config import cfg from oslo_db import options as db_options from stevedore import driver @@ -31,6 +32,14 @@ _IMPL = None _LOCK = threading.Lock() +EXPAND_BRANCH = 'expand' +CONTRACT_BRANCH = 'contract' +MIGRATION_BRANCHES = (EXPAND_BRANCH, CONTRACT_BRANCH) +MITAKA = 'mitaka' +NEWTON = 'newton' +OCATA = 'ocata' +RELEASES = (MITAKA, NEWTON, OCATA) + db_options.set_defaults(cfg.CONF) @@ -46,6 +55,7 @@ def get_backend(): return _IMPL INIT_VERSION = 0 +ALEMBIC_INIT_VERSION = 'liberty' MIGRATE_REPO_PATH = os.path.join( os.path.abspath(os.path.dirname(__file__)), diff --git a/glance/db/sqlalchemy/alembic.ini b/glance/db/sqlalchemy/alembic.ini new file mode 100644 index 0000000000..ade1a92b07 --- /dev/null +++ b/glance/db/sqlalchemy/alembic.ini @@ -0,0 +1,68 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts +script_location = %(here)s/alembic_migrations + +# template used to generate migration files +# file_template = %%(rev)s_%%(slug)s + +# max length of characters to apply to the +# "slug" field +#truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +# version location specification; this defaults +# to alembic_migrations/versions. When using multiple version +# directories, initial revisions must be specified with --version-path +# version_locations = %(here)s/bar %(here)s/bat alembic_migrations/versions + +# the output encoding used when revision files +# are written from script.py.mako +# output_encoding = utf-8 + +sqlalchemy.url = mysql://root:alexstack@localhost/glance + + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/glance/db/sqlalchemy/alembic_migrations/README b/glance/db/sqlalchemy/alembic_migrations/README new file mode 100644 index 0000000000..98e4f9c44e --- /dev/null +++ b/glance/db/sqlalchemy/alembic_migrations/README @@ -0,0 +1 @@ +Generic single-database configuration. \ No newline at end of file diff --git a/glance/db/sqlalchemy/alembic_migrations/__init__.py b/glance/db/sqlalchemy/alembic_migrations/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/glance/db/sqlalchemy/alembic_migrations/add_artifacts_tables.py b/glance/db/sqlalchemy/alembic_migrations/add_artifacts_tables.py new file mode 100644 index 0000000000..31c008d17a --- /dev/null +++ b/glance/db/sqlalchemy/alembic_migrations/add_artifacts_tables.py @@ -0,0 +1,224 @@ +# Copyright 2013 OpenStack Foundation +# Copyright 2013 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from alembic import op +from sqlalchemy.schema import ( + Column, PrimaryKeyConstraint, ForeignKeyConstraint) + +from glance.db.sqlalchemy.migrate_repo.schema import ( + Boolean, DateTime, Integer, BigInteger, String, Text, Numeric) # noqa + + +def _add_artifacts_table(): + op.create_table('artifacts', + Column('id', String(length=36), nullable=False), + Column('name', String(length=255), nullable=False), + Column('type_name', String(length=255), nullable=False), + Column('type_version_prefix', + BigInteger(), + nullable=False), + Column('type_version_suffix', + String(length=255), + nullable=True), + Column('type_version_meta', + String(length=255), + nullable=True), + Column('version_prefix', BigInteger(), nullable=False), + Column('version_suffix', + String(length=255), + nullable=True), + Column('version_meta', String(length=255), nullable=True), + Column('description', Text(), nullable=True), + Column('visibility', String(length=32), nullable=False), + Column('state', String(length=32), nullable=False), + Column('owner', String(length=255), nullable=False), + Column('created_at', DateTime(), nullable=False), + Column('updated_at', DateTime(), nullable=False), + Column('deleted_at', DateTime(), nullable=True), + Column('published_at', DateTime(), nullable=True), + PrimaryKeyConstraint('id'), + mysql_engine='InnoDB', + mysql_charset='utf8', + extend_existing=True) + + op.create_index('ix_artifact_name_and_version', + 'artifacts', + ['name', 'version_prefix', 'version_suffix'], + unique=False) + op.create_index('ix_artifact_owner', 'artifacts', ['owner'], unique=False) + op.create_index('ix_artifact_state', 'artifacts', ['state'], unique=False) + op.create_index('ix_artifact_type', + 'artifacts', + ['type_name', + 'type_version_prefix', + 'type_version_suffix'], + unique=False) + op.create_index('ix_artifact_visibility', + 'artifacts', + ['visibility'], + unique=False) + + +def _add_artifact_blobs_table(): + op.create_table('artifact_blobs', + Column('id', String(length=36), nullable=False), + Column('artifact_id', String(length=36), nullable=False), + Column('size', BigInteger(), nullable=False), + Column('checksum', String(length=32), nullable=True), + Column('name', String(length=255), nullable=False), + Column('item_key', String(length=329), nullable=True), + Column('position', Integer(), nullable=True), + Column('created_at', DateTime(), nullable=False), + Column('updated_at', DateTime(), nullable=False), + ForeignKeyConstraint(['artifact_id'], ['artifacts.id'], ), + PrimaryKeyConstraint('id'), + mysql_engine='InnoDB', + mysql_charset='utf8', + extend_existing=True) + + op.create_index('ix_artifact_blobs_artifact_id', + 'artifact_blobs', + ['artifact_id'], + unique=False) + op.create_index('ix_artifact_blobs_name', + 'artifact_blobs', + ['name'], + unique=False) + + +def _add_artifact_dependencies_table(): + op.create_table('artifact_dependencies', + Column('id', String(length=36), nullable=False), + Column('artifact_source', + String(length=36), + nullable=False), + Column('artifact_dest', String(length=36), nullable=False), + Column('artifact_origin', + String(length=36), + nullable=False), + Column('is_direct', Boolean(), nullable=False), + Column('position', Integer(), nullable=True), + Column('name', String(length=36), nullable=True), + Column('created_at', DateTime(), nullable=False), + Column('updated_at', DateTime(), nullable=False), + ForeignKeyConstraint(['artifact_dest'], + ['artifacts.id'], ), + ForeignKeyConstraint(['artifact_origin'], + ['artifacts.id'], ), + ForeignKeyConstraint(['artifact_source'], + ['artifacts.id'], ), + PrimaryKeyConstraint('id'), + mysql_engine='InnoDB', + mysql_charset='utf8', + extend_existing=True) + + op.create_index('ix_artifact_dependencies_dest_id', + 'artifact_dependencies', + ['artifact_dest'], + unique=False) + op.create_index('ix_artifact_dependencies_direct_dependencies', + 'artifact_dependencies', + ['artifact_source', 'is_direct'], + unique=False) + op.create_index('ix_artifact_dependencies_origin_id', + 'artifact_dependencies', + ['artifact_origin'], + unique=False) + op.create_index('ix_artifact_dependencies_source_id', + 'artifact_dependencies', + ['artifact_source'], + unique=False) + + +def _add_artifact_properties_table(): + op.create_table('artifact_properties', + Column('id', String(length=36), nullable=False), + Column('artifact_id', String(length=36), nullable=False), + Column('name', String(length=255), nullable=False), + Column('string_value', String(length=255), nullable=True), + Column('int_value', Integer(), nullable=True), + Column('numeric_value', Numeric(), nullable=True), + Column('bool_value', Boolean(), nullable=True), + Column('text_value', Text(), nullable=True), + Column('created_at', DateTime(), nullable=False), + Column('updated_at', DateTime(), nullable=False), + Column('position', Integer(), nullable=True), + ForeignKeyConstraint(['artifact_id'], ['artifacts.id'], ), + PrimaryKeyConstraint('id'), + mysql_engine='InnoDB', + mysql_charset='utf8', + extend_existing=True) + + op.create_index('ix_artifact_properties_artifact_id', + 'artifact_properties', + ['artifact_id'], + unique=False) + op.create_index('ix_artifact_properties_name', + 'artifact_properties', + ['name'], + unique=False) + + +def _add_artifact_tags_table(): + op.create_table('artifact_tags', + Column('id', String(length=36), nullable=False), + Column('artifact_id', String(length=36), nullable=False), + Column('value', String(length=255), nullable=False), + Column('created_at', DateTime(), nullable=False), + Column('updated_at', DateTime(), nullable=False), + ForeignKeyConstraint(['artifact_id'], ['artifacts.id'], ), + PrimaryKeyConstraint('id'), + mysql_engine='InnoDB', + mysql_charset='utf8', + extend_existing=True) + + op.create_index('ix_artifact_tags_artifact_id', + 'artifact_tags', + ['artifact_id'], + unique=False) + op.create_index('ix_artifact_tags_artifact_id_tag_value', + 'artifact_tags', + ['artifact_id', 'value'], + unique=False) + + +def _add_artifact_blob_locations_table(): + op.create_table('artifact_blob_locations', + Column('id', String(length=36), nullable=False), + Column('blob_id', String(length=36), nullable=False), + Column('value', Text(), nullable=False), + Column('created_at', DateTime(), nullable=False), + Column('updated_at', DateTime(), nullable=False), + Column('position', Integer(), nullable=True), + Column('status', String(length=36), nullable=True), + ForeignKeyConstraint(['blob_id'], ['artifact_blobs.id'], ), + PrimaryKeyConstraint('id'), + mysql_engine='InnoDB', + mysql_charset='utf8', + extend_existing=True) + + op.create_index('ix_artifact_blob_locations_blob_id', + 'artifact_blob_locations', + ['blob_id'], + unique=False) + + +def upgrade(): + _add_artifacts_table() + _add_artifact_blobs_table() + _add_artifact_dependencies_table() + _add_artifact_properties_table() + _add_artifact_tags_table() + _add_artifact_blob_locations_table() diff --git a/glance/db/sqlalchemy/alembic_migrations/add_images_tables.py b/glance/db/sqlalchemy/alembic_migrations/add_images_tables.py new file mode 100644 index 0000000000..a5b0cbf7cc --- /dev/null +++ b/glance/db/sqlalchemy/alembic_migrations/add_images_tables.py @@ -0,0 +1,201 @@ +# Copyright 2013 OpenStack Foundation +# Copyright 2013 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from alembic import op +from sqlalchemy import sql +from sqlalchemy.schema import ( + Column, PrimaryKeyConstraint, ForeignKeyConstraint, UniqueConstraint) + +from glance.db.sqlalchemy.migrate_repo.schema import ( + Boolean, DateTime, Integer, BigInteger, String, Text) # noqa +from glance.db.sqlalchemy.models import JSONEncodedDict + + +def _add_images_table(): + op.create_table('images', + Column('id', String(length=36), nullable=False), + Column('name', String(length=255), nullable=True), + Column('size', BigInteger(), nullable=True), + Column('status', String(length=30), nullable=False), + Column('is_public', Boolean(), nullable=False), + Column('created_at', DateTime(), nullable=False), + Column('updated_at', DateTime(), nullable=True), + Column('deleted_at', DateTime(), nullable=True), + Column('deleted', Boolean(), nullable=False), + Column('disk_format', String(length=20), nullable=True), + Column('container_format', + String(length=20), + nullable=True), + Column('checksum', String(length=32), nullable=True), + Column('owner', String(length=255), nullable=True), + Column('min_disk', Integer(), nullable=False), + Column('min_ram', Integer(), nullable=False), + Column('protected', + Boolean(), + server_default=sql.false(), + nullable=False), + Column('virtual_size', BigInteger(), nullable=True), + PrimaryKeyConstraint('id'), + mysql_engine='InnoDB', + mysql_charset='utf8', + extend_existing=True) + + op.create_index('checksum_image_idx', + 'images', + ['checksum'], + unique=False) + op.create_index('ix_images_deleted', + 'images', + ['deleted'], + unique=False) + op.create_index('ix_images_is_public', + 'images', + ['is_public'], + unique=False) + op.create_index('owner_image_idx', + 'images', + ['owner'], + unique=False) + + +def _add_image_properties_table(): + op.create_table('image_properties', + Column('id', Integer(), nullable=False), + Column('image_id', String(length=36), nullable=False), + Column('name', String(length=255), nullable=False), + Column('value', Text(), nullable=True), + Column('created_at', DateTime(), nullable=False), + Column('updated_at', DateTime(), nullable=True), + Column('deleted_at', DateTime(), nullable=True), + Column('deleted', Boolean(), nullable=False), + PrimaryKeyConstraint('id'), + ForeignKeyConstraint(['image_id'], ['images.id'], ), + UniqueConstraint('image_id', + 'name', + name='ix_image_properties_image_id_name'), + mysql_engine='InnoDB', + mysql_charset='utf8', + extend_existing=True) + + op.create_index('ix_image_properties_deleted', + 'image_properties', + ['deleted'], + unique=False) + op.create_index('ix_image_properties_image_id', + 'image_properties', + ['image_id'], + unique=False) + + +def _add_image_locations_table(): + op.create_table('image_locations', + Column('id', Integer(), nullable=False), + Column('image_id', String(length=36), nullable=False), + Column('value', Text(), nullable=False), + Column('created_at', DateTime(), nullable=False), + Column('updated_at', DateTime(), nullable=True), + Column('deleted_at', DateTime(), nullable=True), + Column('deleted', Boolean(), nullable=False), + Column('meta_data', JSONEncodedDict(), nullable=True), + Column('status', + String(length=30), + server_default='active', + nullable=False), + PrimaryKeyConstraint('id'), + ForeignKeyConstraint(['image_id'], ['images.id'], ), + mysql_engine='InnoDB', + mysql_charset='utf8', + extend_existing=True) + + op.create_index('ix_image_locations_deleted', + 'image_locations', + ['deleted'], + unique=False) + op.create_index('ix_image_locations_image_id', + 'image_locations', + ['image_id'], + unique=False) + + +def _add_image_members_table(): + deleted_member_constraint = 'image_members_image_id_member_deleted_at_key' + op.create_table('image_members', + Column('id', Integer(), nullable=False), + Column('image_id', String(length=36), nullable=False), + Column('member', String(length=255), nullable=False), + Column('can_share', Boolean(), nullable=False), + Column('created_at', DateTime(), nullable=False), + Column('updated_at', DateTime(), nullable=True), + Column('deleted_at', DateTime(), nullable=True), + Column('deleted', Boolean(), nullable=False), + Column('status', + String(length=20), + server_default='pending', + nullable=False), + ForeignKeyConstraint(['image_id'], ['images.id'], ), + PrimaryKeyConstraint('id'), + UniqueConstraint('image_id', + 'member', + 'deleted_at', + name=deleted_member_constraint), + mysql_engine='InnoDB', + mysql_charset='utf8', + extend_existing=True) + + op.create_index('ix_image_members_deleted', + 'image_members', + ['deleted'], + unique=False) + op.create_index('ix_image_members_image_id', + 'image_members', + ['image_id'], + unique=False) + op.create_index('ix_image_members_image_id_member', + 'image_members', + ['image_id', 'member'], + unique=False) + + +def _add_images_tags_table(): + op.create_table('image_tags', + Column('id', Integer(), nullable=False), + Column('image_id', String(length=36), nullable=False), + Column('value', String(length=255), nullable=False), + Column('created_at', DateTime(), nullable=False), + Column('updated_at', DateTime(), nullable=True), + Column('deleted_at', DateTime(), nullable=True), + Column('deleted', Boolean(), nullable=False), + ForeignKeyConstraint(['image_id'], ['images.id'], ), + PrimaryKeyConstraint('id'), + mysql_engine='InnoDB', + mysql_charset='utf8', + extend_existing=True) + + op.create_index('ix_image_tags_image_id', + 'image_tags', + ['image_id'], + unique=False) + op.create_index('ix_image_tags_image_id_tag_value', + 'image_tags', + ['image_id', 'value'], + unique=False) + + +def upgrade(): + _add_images_table() + _add_image_properties_table() + _add_image_locations_table() + _add_image_members_table() + _add_images_tags_table() diff --git a/glance/db/sqlalchemy/alembic_migrations/add_metadefs_tables.py b/glance/db/sqlalchemy/alembic_migrations/add_metadefs_tables.py new file mode 100644 index 0000000000..ee5aea763d --- /dev/null +++ b/glance/db/sqlalchemy/alembic_migrations/add_metadefs_tables.py @@ -0,0 +1,171 @@ +# Copyright 2013 OpenStack Foundation +# Copyright 2013 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from alembic import op +from sqlalchemy.schema import ( + Column, PrimaryKeyConstraint, ForeignKeyConstraint, UniqueConstraint) + +from glance.db.sqlalchemy.migrate_repo.schema import ( + Boolean, DateTime, Integer, String, Text) # noqa +from glance.db.sqlalchemy.models import JSONEncodedDict + + +def _add_metadef_namespaces_table(): + op.create_table('metadef_namespaces', + Column('id', Integer(), nullable=False), + Column('namespace', String(length=80), nullable=False), + Column('display_name', String(length=80), nullable=True), + Column('description', Text(), nullable=True), + Column('visibility', String(length=32), nullable=True), + Column('protected', Boolean(), nullable=True), + Column('owner', String(length=255), nullable=False), + Column('created_at', DateTime(), nullable=False), + Column('updated_at', DateTime(), nullable=True), + PrimaryKeyConstraint('id'), + UniqueConstraint('namespace', + name='uq_metadef_namespaces_namespace'), + mysql_engine='InnoDB', + mysql_charset='utf8', + extend_existing=True) + + op.create_index('ix_metadef_namespaces_owner', + 'metadef_namespaces', + ['owner'], + unique=False) + + +def _add_metadef_resource_types_table(): + op.create_table('metadef_resource_types', + Column('id', Integer(), nullable=False), + Column('name', String(length=80), nullable=False), + Column('protected', Boolean(), nullable=False), + Column('created_at', DateTime(), nullable=False), + Column('updated_at', DateTime(), nullable=True), + PrimaryKeyConstraint('id'), + UniqueConstraint('name', + name='uq_metadef_resource_types_name'), + mysql_engine='InnoDB', + mysql_charset='utf8', + extend_existing=True) + + +def _add_metadef_namespace_resource_types_table(): + op.create_table('metadef_namespace_resource_types', + Column('resource_type_id', Integer(), nullable=False), + Column('namespace_id', Integer(), nullable=False), + Column('properties_target', + String(length=80), + nullable=True), + Column('prefix', String(length=80), nullable=True), + Column('created_at', DateTime(), nullable=False), + Column('updated_at', DateTime(), nullable=True), + ForeignKeyConstraint(['namespace_id'], + ['metadef_namespaces.id'], ), + ForeignKeyConstraint(['resource_type_id'], + ['metadef_resource_types.id'], ), + PrimaryKeyConstraint('resource_type_id', 'namespace_id'), + mysql_engine='InnoDB', + mysql_charset='utf8', + extend_existing=True) + + op.create_index('ix_metadef_ns_res_types_namespace_id', + 'metadef_namespace_resource_types', + ['namespace_id'], + unique=False) + + +def _add_metadef_objects_table(): + ns_id_name_constraint = 'uq_metadef_objects_namespace_id_name' + + op.create_table('metadef_objects', + Column('id', Integer(), nullable=False), + Column('namespace_id', Integer(), nullable=False), + Column('name', String(length=80), nullable=False), + Column('description', Text(), nullable=True), + Column('required', Text(), nullable=True), + Column('json_schema', JSONEncodedDict(), nullable=False), + Column('created_at', DateTime(), nullable=False), + Column('updated_at', DateTime(), nullable=True), + ForeignKeyConstraint(['namespace_id'], + ['metadef_namespaces.id'], ), + PrimaryKeyConstraint('id'), + UniqueConstraint('namespace_id', + 'name', + name=ns_id_name_constraint), + mysql_engine='InnoDB', + mysql_charset='utf8', + extend_existing=True) + + op.create_index('ix_metadef_objects_name', + 'metadef_objects', + ['name'], + unique=False) + + +def _add_metadef_properties_table(): + ns_id_name_constraint = 'uq_metadef_properties_namespace_id_name' + op.create_table('metadef_properties', + Column('id', Integer(), nullable=False), + Column('namespace_id', Integer(), nullable=False), + Column('name', String(length=80), nullable=False), + Column('json_schema', JSONEncodedDict(), nullable=False), + Column('created_at', DateTime(), nullable=False), + Column('updated_at', DateTime(), nullable=True), + ForeignKeyConstraint(['namespace_id'], + ['metadef_namespaces.id'], ), + PrimaryKeyConstraint('id'), + UniqueConstraint('namespace_id', + 'name', + name=ns_id_name_constraint), + mysql_engine='InnoDB', + mysql_charset='utf8', + extend_existing=True) + + op.create_index('ix_metadef_properties_name', + 'metadef_properties', + ['name'], + unique=False) + + +def _add_metadef_tags_table(): + op.create_table('metadef_tags', + Column('id', Integer(), nullable=False), + Column('namespace_id', Integer(), nullable=False), + Column('name', String(length=80), nullable=False), + Column('created_at', DateTime(), nullable=False), + Column('updated_at', DateTime(), nullable=True), + ForeignKeyConstraint(['namespace_id'], + ['metadef_namespaces.id'], ), + PrimaryKeyConstraint('id'), + UniqueConstraint('namespace_id', + 'name', + name='uq_metadef_tags_namespace_id_name'), + mysql_engine='InnoDB', + mysql_charset='utf8', + extend_existing=True) + + op.create_index('ix_metadef_tags_name', + 'metadef_tags', + ['name'], + unique=False) + + +def upgrade(): + _add_metadef_namespaces_table() + _add_metadef_resource_types_table() + _add_metadef_namespace_resource_types_table() + _add_metadef_objects_table() + _add_metadef_properties_table() + _add_metadef_tags_table() diff --git a/glance/db/sqlalchemy/alembic_migrations/add_tasks_tables.py b/glance/db/sqlalchemy/alembic_migrations/add_tasks_tables.py new file mode 100644 index 0000000000..fe2f29d753 --- /dev/null +++ b/glance/db/sqlalchemy/alembic_migrations/add_tasks_tables.py @@ -0,0 +1,66 @@ +# Copyright 2013 OpenStack Foundation +# Copyright 2013 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from alembic import op +from sqlalchemy.schema import ( + Column, PrimaryKeyConstraint, ForeignKeyConstraint) + +from glance.db.sqlalchemy.migrate_repo.schema import ( + Boolean, DateTime, String, Text) # noqa +from glance.db.sqlalchemy.models import JSONEncodedDict + + +def _add_tasks_table(): + op.create_table('tasks', + Column('id', String(length=36), nullable=False), + Column('type', String(length=30), nullable=False), + Column('status', String(length=30), nullable=False), + Column('owner', String(length=255), nullable=False), + Column('expires_at', DateTime(), nullable=True), + Column('created_at', DateTime(), nullable=False), + Column('updated_at', DateTime(), nullable=True), + Column('deleted_at', DateTime(), nullable=True), + Column('deleted', Boolean(), nullable=False), + PrimaryKeyConstraint('id'), + mysql_engine='InnoDB', + mysql_charset='utf8', + extend_existing=True) + + op.create_index('ix_tasks_deleted', 'tasks', ['deleted'], unique=False) + op.create_index('ix_tasks_owner', 'tasks', ['owner'], unique=False) + op.create_index('ix_tasks_status', 'tasks', ['status'], unique=False) + op.create_index('ix_tasks_type', 'tasks', ['type'], unique=False) + op.create_index('ix_tasks_updated_at', + 'tasks', + ['updated_at'], + unique=False) + + +def _add_task_info_table(): + op.create_table('task_info', + Column('task_id', String(length=36), nullable=False), + Column('input', JSONEncodedDict(), nullable=True), + Column('result', JSONEncodedDict(), nullable=True), + Column('message', Text(), nullable=True), + ForeignKeyConstraint(['task_id'], ['tasks.id'], ), + PrimaryKeyConstraint('task_id'), + mysql_engine='InnoDB', + mysql_charset='utf8', + extend_existing=True) + + +def upgrade(): + _add_tasks_table() + _add_task_info_table() diff --git a/glance/db/sqlalchemy/alembic_migrations/alembic.ini b/glance/db/sqlalchemy/alembic_migrations/alembic.ini new file mode 100644 index 0000000000..ade1a92b07 --- /dev/null +++ b/glance/db/sqlalchemy/alembic_migrations/alembic.ini @@ -0,0 +1,68 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts +script_location = %(here)s/alembic_migrations + +# template used to generate migration files +# file_template = %%(rev)s_%%(slug)s + +# max length of characters to apply to the +# "slug" field +#truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +# version location specification; this defaults +# to alembic_migrations/versions. When using multiple version +# directories, initial revisions must be specified with --version-path +# version_locations = %(here)s/bar %(here)s/bat alembic_migrations/versions + +# the output encoding used when revision files +# are written from script.py.mako +# output_encoding = utf-8 + +sqlalchemy.url = mysql://root:alexstack@localhost/glance + + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/glance/db/sqlalchemy/alembic_migrations/env.py b/glance/db/sqlalchemy/alembic_migrations/env.py new file mode 100644 index 0000000000..c0087facde --- /dev/null +++ b/glance/db/sqlalchemy/alembic_migrations/env.py @@ -0,0 +1,88 @@ +# Copyright 2013 OpenStack Foundation +# Copyright 2013 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from __future__ import with_statement +from alembic import context +from sqlalchemy import engine_from_config, pool +from logging.config import fileConfig + +from glance.db.sqlalchemy import models_metadef + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +fileConfig(config.config_file_name) + +# add your model's MetaData object here +# for 'autogenerate' support +# from myapp import mymodel +# target_metadata = mymodel.Base.metadata + +target_metadata = models_metadef.BASE_DICT.metadata + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + + +def run_migrations_offline(): + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, target_metadata=target_metadata, literal_binds=True) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online(): + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + connectable = engine_from_config( + config.get_section(config.config_ini_section), + prefix='sqlalchemy.', + poolclass=pool.NullPool) + + with connectable.connect() as connection: + context.configure( + connection=connection, + target_metadata=target_metadata + ) + + with context.begin_transaction(): + context.run_migrations() + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/glance/db/sqlalchemy/alembic_migrations/migrate.cfg b/glance/db/sqlalchemy/alembic_migrations/migrate.cfg new file mode 100644 index 0000000000..8ddf0500ab --- /dev/null +++ b/glance/db/sqlalchemy/alembic_migrations/migrate.cfg @@ -0,0 +1,20 @@ +[db_settings] +# Used to identify which repository this database is versioned under. +# You can use the name of your project. +repository_id=Glance Migrations + +# The name of the database table used to track the schema version. +# This name shouldn't already be used by your project. +# If this is changed once a database is under version control, you'll need to +# change the table name in each database too. +version_table=alembic_version + +# When committing a change script, Migrate will attempt to generate the +# sql for all supported databases; normally, if one of them fails - probably +# because you don't have that database installed - it is ignored and the +# commit continues, perhaps ending successfully. +# Databases in this list MUST compile successfully during a commit, or the +# entire commit will fail. List the databases your application will actually +# be using to ensure your updates to that database work properly. +# This must be a list; example: ['postgres','sqlite'] +required_dbs=[] diff --git a/glance/db/sqlalchemy/alembic_migrations/script.py.mako b/glance/db/sqlalchemy/alembic_migrations/script.py.mako new file mode 100644 index 0000000000..43c09401bc --- /dev/null +++ b/glance/db/sqlalchemy/alembic_migrations/script.py.mako @@ -0,0 +1,24 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +def upgrade(): + ${upgrades if upgrades else "pass"} + + +def downgrade(): + ${downgrades if downgrades else "pass"} diff --git a/glance/db/sqlalchemy/alembic_migrations/versions/liberty_initial.py b/glance/db/sqlalchemy/alembic_migrations/versions/liberty_initial.py new file mode 100644 index 0000000000..9dbf4d8fed --- /dev/null +++ b/glance/db/sqlalchemy/alembic_migrations/versions/liberty_initial.py @@ -0,0 +1,44 @@ +# Copyright 2013 OpenStack Foundation +# Copyright 2013 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""liberty initial + +Revision ID: liberty +Revises: +Create Date: 2016-08-03 16:06:59.657433 + +""" + +from glance.db.sqlalchemy.alembic_migrations import add_artifacts_tables +from glance.db.sqlalchemy.alembic_migrations import add_images_tables +from glance.db.sqlalchemy.alembic_migrations import add_metadefs_tables +from glance.db.sqlalchemy.alembic_migrations import add_tasks_tables + +# revision identifiers, used by Alembic. +revision = 'liberty' +down_revision = None +branch_labels = None +depends_on = None + + +def upgrade(): + add_images_tables.upgrade() + add_tasks_tables.upgrade() + add_metadefs_tables.upgrade() + add_artifacts_tables.upgrade() + + +def downgrade(): + raise NotImplementedError diff --git a/glance/db/sqlalchemy/alembic_migrations/versions/mitaka01_add_image_created_updated_idx.py b/glance/db/sqlalchemy/alembic_migrations/versions/mitaka01_add_image_created_updated_idx.py new file mode 100644 index 0000000000..1dea1bc06b --- /dev/null +++ b/glance/db/sqlalchemy/alembic_migrations/versions/mitaka01_add_image_created_updated_idx.py @@ -0,0 +1,51 @@ +# Copyright 2013 OpenStack Foundation +# Copyright 2013 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""add index on created_at and updated_at columns of 'images' table + +Revision ID: mitaka01 +Revises: liberty +Create Date: 2016-08-03 17:19:35.306161 + +""" + +from alembic import op +from sqlalchemy import MetaData, Table, Index + + +# revision identifiers, used by Alembic. +revision = 'mitaka01' +down_revision = 'liberty' +branch_labels = None +depends_on = None + +CREATED_AT_INDEX = 'created_at_image_idx' +UPDATED_AT_INDEX = 'updated_at_image_idx' + + +def upgrade(): + migrate_engine = op.get_bind() + meta = MetaData(bind=migrate_engine) + + images = Table('images', meta, autoload=True) + + created_index = Index(CREATED_AT_INDEX, images.c.created_at) + created_index.create(migrate_engine) + updated_index = Index(UPDATED_AT_INDEX, images.c.updated_at) + updated_index.create(migrate_engine) + + +def downgrade(): + raise NotImplementedError diff --git a/glance/db/sqlalchemy/alembic_migrations/versions/mitaka02_update_metadef_os_nova_server.py b/glance/db/sqlalchemy/alembic_migrations/versions/mitaka02_update_metadef_os_nova_server.py new file mode 100644 index 0000000000..7af53a805d --- /dev/null +++ b/glance/db/sqlalchemy/alembic_migrations/versions/mitaka02_update_metadef_os_nova_server.py @@ -0,0 +1,46 @@ +# Copyright 2013 OpenStack Foundation +# Copyright 2013 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""update metadef os_nova_server + +Revision ID: mitaka02 +Revises: mitaka01 +Create Date: 2016-08-03 17:23:23.041663 + +""" + +from alembic import op +from sqlalchemy import MetaData, Table + + +# revision identifiers, used by Alembic. +revision = 'mitaka02' +down_revision = 'mitaka01' +branch_labels = None +depends_on = None + + +def upgrade(): + migrate_engine = op.get_bind() + meta = MetaData(bind=migrate_engine) + + resource_types_table = Table('metadef_resource_types', meta, autoload=True) + + resource_types_table.update(values={'name': 'OS::Nova::Server'}).where( + resource_types_table.c.name == 'OS::Nova::Instance').execute() + + +def downgrade(): + raise NotImplementedError diff --git a/glance/db/sqlalchemy/alembic_migrations/versions/newton_contract01_drop_is_public.py b/glance/db/sqlalchemy/alembic_migrations/versions/newton_contract01_drop_is_public.py new file mode 100644 index 0000000000..7a1ed061fc --- /dev/null +++ b/glance/db/sqlalchemy/alembic_migrations/versions/newton_contract01_drop_is_public.py @@ -0,0 +1,52 @@ +# Copyright 2013 OpenStack Foundation +# Copyright 2013 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""drop is_public + +Revision ID: newton_contract01 +Revises: mitaka02 +Create Date: 2016-08-04 12:22:01.294934 + +""" + +from alembic import op +from sqlalchemy import MetaData, Table + +from glance.db import migration + +# revision identifiers, used by Alembic. +revision = 'newton_contract01' +down_revision = 'mitaka02' +branch_labels = (migration.CONTRACT_BRANCH,) +depends_on = None + + +def upgrade(): + meta = MetaData(bind=op.get_bind()) + images = Table('images', meta, autoload=True) + + op.execute(images.update(values={'visibility': 'public'}).where( + images.c.is_public)) + + # TODO(abashmak) uncomment below when community images codebase + # including updates to tests gets merged, either upstream or in + # this 'feature' breanch + # with op.batch_alter_table("images") as batch_op: + # batch_op.drop_index('ix_images_is_public') + # batch_op.drop_column('is_public') + + +def downgrade(): + raise NotImplementedError diff --git a/glance/db/sqlalchemy/alembic_migrations/versions/newton_expand01_add_visibility.py b/glance/db/sqlalchemy/alembic_migrations/versions/newton_expand01_add_visibility.py new file mode 100644 index 0000000000..f66a5a184a --- /dev/null +++ b/glance/db/sqlalchemy/alembic_migrations/versions/newton_expand01_add_visibility.py @@ -0,0 +1,55 @@ +# Copyright 2013 OpenStack Foundation +# Copyright 2013 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""add visibility + +Revision ID: newton_expand01 +Revises: mitaka02 +Create Date: 2016-08-04 12:08:08.258679 + +""" +from alembic import op +from sqlalchemy import MetaData, Column, Enum + +from glance.db import migration + +# revision identifiers, used by Alembic. +revision = 'newton_expand01' +down_revision = 'mitaka02' +branch_labels = (migration.EXPAND_BRANCH,) +depends_on = None + + +def upgrade(): + meta = MetaData(bind=op.get_bind()) + + # TODO(tsymanczyk): would be nice if this enum weren't copypaste + enum = Enum('private', 'public', 'shared', 'community', + metadata=meta, + name='image_visibility') + enum.create() + + op.add_column('images', Column('visibility', + enum, + nullable=False, + server_default='private')) + op.create_index('visibility_image_idx', + 'images', + ['visibility'], + unique=False) + + +def downgrade(): + raise NotImplementedError diff --git a/glance/tests/unit/test_manage.py b/glance/tests/unit/test_manage.py index 16a76aba70..0d127fc316 100644 --- a/glance/tests/unit/test_manage.py +++ b/glance/tests/unit/test_manage.py @@ -15,11 +15,9 @@ import fixtures import mock -from oslo_db.sqlalchemy import migration from six.moves import StringIO from glance.cmd import manage -from glance.db import migration as db_migration from glance.db.sqlalchemy import api as db_api from glance.db.sqlalchemy import metadata as db_metadata from glance.tests import utils as test_utils @@ -51,48 +49,50 @@ def _main_test_helper(self, argv, func_name=None, *exp_args, **exp_kwargs): class TestLegacyManage(TestManageBase): - @mock.patch.object(migration, 'db_version') - def test_legacy_db_version(self, db_version): - with mock.patch('sys.stdout', new_callable=StringIO): - self._main_test_helper(['glance.cmd.manage', 'db_version'], - migration.db_version, - db_api.get_engine(), - db_migration.MIGRATE_REPO_PATH, 0) + @mock.patch.object(manage.DbCommands, 'version') + def test_legacy_db_version(self, db_upgrade): + self._main_test_helper(['glance.cmd.manage', 'db_version'], + manage.DbCommands.version) - @mock.patch.object(migration, 'db_sync') + @mock.patch.object(manage.DbCommands, 'sync') def test_legacy_db_sync(self, db_sync): self._main_test_helper(['glance.cmd.manage', 'db_sync'], - migration.db_sync, - db_api.get_engine(), - db_migration.MIGRATE_REPO_PATH, None) + manage.DbCommands.sync, None) - @mock.patch.object(migration, 'db_sync') - def test_legacy_db_upgrade(self, db_sync): + @mock.patch.object(manage.DbCommands, 'upgrade') + def test_legacy_db_upgrade(self, db_upgrade): self._main_test_helper(['glance.cmd.manage', 'db_upgrade'], - migration.db_sync, - db_api.get_engine(), - db_migration.MIGRATE_REPO_PATH, None) + manage.DbCommands.upgrade, None) - @mock.patch.object(migration, 'db_version_control') + @mock.patch.object(manage.DbCommands, 'version_control') def test_legacy_db_version_control(self, db_version_control): self._main_test_helper(['glance.cmd.manage', 'db_version_control'], - migration.db_version_control, - db_api.get_engine(), - db_migration.MIGRATE_REPO_PATH, None) + manage.DbCommands.version_control, None) - @mock.patch.object(migration, 'db_sync') + @mock.patch.object(manage.DbCommands, 'sync') def test_legacy_db_sync_version(self, db_sync): - self._main_test_helper(['glance.cmd.manage', 'db_sync', '20'], - migration.db_sync, - db_api.get_engine(), - db_migration.MIGRATE_REPO_PATH, '20') + self._main_test_helper(['glance.cmd.manage', 'db_sync', 'liberty'], + manage.DbCommands.sync, 'liberty') - @mock.patch.object(migration, 'db_sync') - def test_legacy_db_upgrade_version(self, db_sync): - self._main_test_helper(['glance.cmd.manage', 'db_upgrade', '20'], - migration.db_sync, - db_api.get_engine(), - db_migration.MIGRATE_REPO_PATH, '20') + @mock.patch.object(manage.DbCommands, 'upgrade') + def test_legacy_db_upgrade_version(self, db_upgrade): + self._main_test_helper(['glance.cmd.manage', 'db_upgrade', 'liberty'], + manage.DbCommands.upgrade, 'liberty') + + @mock.patch.object(manage.DbCommands, 'expand') + def test_legacy_db_expand(self, db_expand): + self._main_test_helper(['glance.cmd.manage', 'db_expand'], + manage.DbCommands.expand) + + @mock.patch.object(manage.DbCommands, 'data_migrate') + def test_legacy_db_data_migrate(self, db_data_migrate): + self._main_test_helper(['glance.cmd.manage', 'db_data_migrate'], + manage.DbCommands.data_migrate) + + @mock.patch.object(manage.DbCommands, 'contract') + def test_legacy_db_contract(self, db_contract): + self._main_test_helper(['glance.cmd.manage', 'db_contract'], + manage.DbCommands.contract) def test_db_metadefs_unload(self): db_metadata.db_unload_metadefs = mock.Mock() @@ -157,48 +157,51 @@ def test_db_metadefs_export_with_specified_path(self): class TestManage(TestManageBase): - @mock.patch.object(migration, 'db_version') - def test_db_version(self, db_version): - with mock.patch('sys.stdout', new_callable=StringIO): - self._main_test_helper(['glance.cmd.manage', 'db', 'version'], - migration.db_version, - db_api.get_engine(), - db_migration.MIGRATE_REPO_PATH, 0) + @mock.patch.object(manage.DbCommands, 'version') + def test_db_version(self, version): + self._main_test_helper(['glance.cmd.manage', 'db', 'version'], + manage.DbCommands.version) - @mock.patch.object(migration, 'db_sync') - def test_db_sync(self, db_sync): + @mock.patch.object(manage.DbCommands, 'sync') + def test_db_sync(self, sync): self._main_test_helper(['glance.cmd.manage', 'db', 'sync'], - migration.db_sync, - db_api.get_engine(), - db_migration.MIGRATE_REPO_PATH, None) + manage.DbCommands.sync) - @mock.patch.object(migration, 'db_sync') - def test_db_upgrade(self, db_sync): + @mock.patch.object(manage.DbCommands, 'upgrade') + def test_db_upgrade(self, upgrade): self._main_test_helper(['glance.cmd.manage', 'db', 'upgrade'], - migration.db_sync, - db_api.get_engine(), - db_migration.MIGRATE_REPO_PATH, None) + manage.DbCommands.upgrade) - @mock.patch.object(migration, 'db_version_control') - def test_db_version_control(self, db_version_control): + @mock.patch.object(manage.DbCommands, 'version_control') + def test_db_version_control(self, version_control): self._main_test_helper(['glance.cmd.manage', 'db', 'version_control'], - migration.db_version_control, - db_api.get_engine(), - db_migration.MIGRATE_REPO_PATH, None) - - @mock.patch.object(migration, 'db_sync') - def test_db_sync_version(self, db_sync): - self._main_test_helper(['glance.cmd.manage', 'db', 'sync', '20'], - migration.db_sync, - db_api.get_engine(), - db_migration.MIGRATE_REPO_PATH, '20') - - @mock.patch.object(migration, 'db_sync') - def test_db_upgrade_version(self, db_sync): - self._main_test_helper(['glance.cmd.manage', 'db', 'upgrade', '20'], - migration.db_sync, - db_api.get_engine(), - db_migration.MIGRATE_REPO_PATH, '20') + manage.DbCommands.version_control) + + @mock.patch.object(manage.DbCommands, 'sync') + def test_db_sync_version(self, sync): + self._main_test_helper(['glance.cmd.manage', 'db', 'sync', 'liberty'], + manage.DbCommands.sync, 'liberty') + + @mock.patch.object(manage.DbCommands, 'upgrade') + def test_db_upgrade_version(self, upgrade): + self._main_test_helper(['glance.cmd.manage', 'db', + 'upgrade', 'liberty'], + manage.DbCommands.upgrade, 'liberty') + + @mock.patch.object(manage.DbCommands, 'expand') + def test_db_expand(self, expand): + self._main_test_helper(['glance.cmd.manage', 'db', 'expand'], + manage.DbCommands.expand) + + @mock.patch.object(manage.DbCommands, 'data_migrate') + def test_db_data_migrate(self, data_migrate): + self._main_test_helper(['glance.cmd.manage', 'db', 'data_migrate'], + manage.DbCommands.data_migrate) + + @mock.patch.object(manage.DbCommands, 'contract') + def test_db_contract(self, contract): + self._main_test_helper(['glance.cmd.manage', 'db', 'contract'], + manage.DbCommands.contract) def test_db_metadefs_unload(self): db_metadata.db_unload_metadefs = mock.Mock()