From 32f5085bea29515cd5b2267ad176ae2120badcbe Mon Sep 17 00:00:00 2001 From: Manuel Marin Date: Tue, 19 Feb 2019 15:06:35 -0400 Subject: [PATCH] Issue #21 updated, migrate only when explicitly asked for --- spinedatabase_api/__init__.py | 4 +- spinedatabase_api/database_mapping.py | 36 ++++++++++--- spinedatabase_api/diff_database_mapping.py | 61 +++++++++++----------- spinedatabase_api/exception.py | 9 ++++ 4 files changed, 72 insertions(+), 38 deletions(-) diff --git a/spinedatabase_api/__init__.py b/spinedatabase_api/__init__.py index 7c508884..252bae7b 100644 --- a/spinedatabase_api/__init__.py +++ b/spinedatabase_api/__init__.py @@ -1,7 +1,7 @@ from .database_mapping import DatabaseMapping from .diff_database_mapping import DiffDatabaseMapping -from .exception import SpineDBAPIError, SpineIntegrityError, SpineTableNotFoundError, \ - RecordNotFoundError, ParameterValueError +from .exception import SpineDBAPIError, SpineIntegrityError, SpineDBVersionError, \ + SpineTableNotFoundError, RecordNotFoundError, ParameterValueError from .helpers import create_new_spine_database, copy_database, merge_database, is_unlocked from .import_functions import import_data, import_object_classes, import_objects, \ import_object_parameters, import_object_parameter_values, import_relationship_classes, \ diff --git a/spinedatabase_api/database_mapping.py b/spinedatabase_api/database_mapping.py index 631962e9..0b835a0d 100644 --- a/spinedatabase_api/database_mapping.py +++ b/spinedatabase_api/database_mapping.py @@ -24,6 +24,7 @@ :date: 11.8.2018 """ +import os import time import logging from sqlalchemy import create_engine, false, distinct, func, MetaData, event @@ -31,8 +32,13 @@ from sqlalchemy.orm import Session, aliased from sqlalchemy.pool import StaticPool from sqlalchemy.exc import NoSuchTableError, DBAPIError, DatabaseError -from .exception import SpineDBAPIError, SpineTableNotFoundError, RecordNotFoundError, ParameterValueError -from .helpers import custom_generate_relationship, attr_dict, upgrade_to_head +from alembic.migration import MigrationContext +from alembic.script import ScriptDirectory +from alembic.config import Config +from alembic import command +from .exception import SpineDBAPIError, SpineDBVersionError, SpineTableNotFoundError, \ + RecordNotFoundError, ParameterValueError +from .helpers import custom_generate_relationship, attr_dict from datetime import datetime, timezone # TODO: Consider returning lists of dict (with _asdict()) rather than queries, @@ -49,7 +55,7 @@ class DatabaseMapping(object): db_url (str): The database url formatted according to sqlalchemy rules username (str): The user name """ - def __init__(self, db_url, username=None, create_all=True): + def __init__(self, db_url, username=None, create_all=True, migrate=False): """Initialize class.""" self.db_url = db_url self.username = username @@ -68,14 +74,14 @@ def __init__(self, db_url, username=None, create_all=True): self.ParameterDefinitionTag = None self.ParameterEnum = None self.Commit = None - upgrade_to_head(db_url) if create_all: self.create_engine_and_session() + self.check_db_version(migrate=migrate) self.create_mapping() # self.create_triggers() def create_engine_and_session(self): - """Create engine connected to self.db_url and session.""" + """Create engine connected to self.db_url and corresponding session.""" try: self.engine = create_engine(self.db_url) self.engine.connect() @@ -94,6 +100,24 @@ def create_engine_and_session(self): # raise SpineDBAPIError(msg) self.session = Session(self.engine, autoflush=False) + def check_db_version(self, migrate=False): + """Check if database is the latest version and raise a SpineDBVersionError if not. + If migrate is True, then don't raise the error and upgrade the database instead. + """ + path = os.path.dirname(__file__) # NOTE: this assumes this file and alembic.ini are on the same path + config = Config(os.path.join(path, "alembic.ini")) + config.set_main_option("script_location", "spinedatabase_api:alembic") + config.set_main_option("sqlalchemy.url", self.db_url) + script = ScriptDirectory.from_config(config) + head = script.get_current_head() + context = MigrationContext.configure(self.engine.connect()) + current = context.get_current_revision() + if current == head: + return + if not migrate: + raise SpineDBVersionError(url=self.db_url, current=current, head=head) + command.upgrade(config, "head") + def create_mapping(self): """Create ORM.""" try: @@ -666,7 +690,7 @@ def parameter_enum_list(self, id_list=None): qry = self.session.query( self.ParameterEnum.id.label("id"), self.ParameterEnum.name.label("name"), - self.ParameterEnum.value.label("value_index"), + self.ParameterEnum.value_index.label("value_index"), self.ParameterEnum.value.label("value")) if id_list is not None: qry = qry.filter(self.ParameterEnum.id.in_(id_list)) diff --git a/spinedatabase_api/diff_database_mapping.py b/spinedatabase_api/diff_database_mapping.py index 827dc2be..1820ebed 100644 --- a/spinedatabase_api/diff_database_mapping.py +++ b/spinedatabase_api/diff_database_mapping.py @@ -43,9 +43,9 @@ class DiffDatabaseMapping(DatabaseMapping): In a nutshell, it works by creating a new bunch of tables to hold differences with respect to original tables. """ - def __init__(self, db_url, username=None, create_all=True): + def __init__(self, db_url, username=None, create_all=True, migrate=False): """Initialize class.""" - super().__init__(db_url, username=username, create_all=False) + super().__init__(db_url, username=username, create_all=create_all, migrate=migrate) # Diff meta, Base and tables self.diff_prefix = None self.diff_metadata = None @@ -69,11 +69,9 @@ def __init__(self, db_url, username=None, create_all=True): # Initialize stuff self.init_diff_dicts() if create_all: - self.create_engine_and_session() - self.create_mapping() self.create_diff_tables_and_mapping() self.init_next_id() - # self.create_triggers() + # self.create_diff_triggers() def has_pending_changes(self): """Return True if there are uncommitted changes. Otherwise return False.""" @@ -222,13 +220,12 @@ def init_next_id(self): self.close() raise SpineTableNotFoundError(table) - def create_triggers(self): + def create_diff_triggers(self): """Create ad-hoc triggers. NOTE: Not in use at the moment. Cascade delete is implemented in the `remove_items` method. TODO: is there a way to synch this with our CREATE TRIGGER statements from `helpers.create_new_spine_database`? """ - super().create_triggers() @event.listens_for(self.DiffObjectClass, 'after_delete') def receive_after_object_class_delete(mapper, connection, object_class): @event.listens_for(self.session, "after_flush", once=True) @@ -1750,7 +1747,7 @@ def check_wide_parameter_enums_for_update(self, *wide_kwargs_list, raise_intgr_e parameter_enum_dict = { x.id: { "name": x.name, - "element_list": x.element_list.split(",") + "value_list": x.value_list.split(",") } for x in self.wide_parameter_enum_list() } parameter_enum_names = {x.name for x in self.wide_parameter_enum_list()} @@ -2725,30 +2722,35 @@ def update_wide_parameter_enums(self, *wide_kwargs_list, raise_intgr_error=True) """Update parameter enums. NOTE: It's too difficult to do it cleanly, so we just remove and then add. """ + checked_wide_kwargs_list, intgr_error_log = self.check_wide_parameter_enums_for_update( + *wide_kwargs_list, raise_intgr_error=raise_intgr_error) + wide_parameter_enum_dict = {x.id: x._asdict() for x in self.wide_parameter_enum_list()} + updated_ids = set() + item_list = list() + for wide_kwargs in checked_wide_kwargs_list: + try: + id = wide_kwargs['id'] + updated_wide_kwargs = wide_parameter_enum_dict[id] + except KeyError: + continue + updated_ids.add(id) + updated_wide_kwargs.update(wide_kwargs) + for k, value in enumerate(updated_wide_kwargs['value_list']): + updated_narrow_kwargs = { + 'id': id, + 'name': updated_wide_kwargs['name'], + 'value_index': k, + 'value': value + } + item_list.append(updated_narrow_kwargs) try: - wide_parameter_enum_dict = {x.id: x._asdict() for x in self.wide_parameter_enum_list()} - updated_ids = set() - item_list = list() - for wide_kwargs in wide_kwargs_list: - try: - id = wide_kwargs['id'] - updated_wide_kwargs = wide_parameter_enum_dict[id] - except KeyError: - continue - updated_ids.add(id) - updated_wide_kwargs.update(wide_kwargs) - for k, value in enumerate(updated_wide_kwargs['value_list']): - updated_narrow_kwargs = { - 'id': id, - 'name': updated_wide_kwargs['name'], - 'value_index': k, - 'value': value - } - item_list.append(updated_narrow_kwargs) - self.remove_items(parameter_enum_ids=updated_ids) + self.session.query(self.DiffParameterEnum).filter(self.DiffParameterEnum.id.in_(updated_ids)).\ + delete(synchronize_session=False) self.session.bulk_insert_mappings(self.DiffParameterEnum, item_list) self.session.commit() self.new_item_id["parameter_enum"].update(updated_ids) + self.removed_item_id["parameter_enum"].update(updated_ids) + self.touched_item_id["parameter_enum"].update(updated_ids) updated_item_list = self.wide_parameter_enum_list(id_list=updated_ids) if not raise_intgr_error: return updated_item_list, intgr_error_log @@ -3076,8 +3078,7 @@ def _remove_cascade_parameter_definition_tags(self, ids, diff_ids, removed_item_ def _remove_cascade_parameter_enums(self, ids, diff_ids, removed_item_id, removed_diff_item_id): """Remove parameter enums and all related items. - TODO: Should we remove parameter definitions here? Do we care if they have invalid enum? - If we *do* remove parameter definitions then we need to fix `update_wide_parameter_enum` + TODO: Should we remove parameter definitions here? Set their enum_id to NULL? """ removed_item_id.setdefault("parameter_enum", set()).update(ids) removed_diff_item_id.setdefault("parameter_enum", set()).update(diff_ids) diff --git a/spinedatabase_api/exception.py b/spinedatabase_api/exception.py index e3effe79..cac4a4b4 100644 --- a/spinedatabase_api/exception.py +++ b/spinedatabase_api/exception.py @@ -38,6 +38,15 @@ def __init__(self, msg=None): super().__init__(msg) +class SpineDBVersionError(SpineDBAPIError): + """Database version error.""" + def __init__(self, url=None, current=None, head=None): + super().__init__(msg="The database at '{}' is not the latest version.".format(url)) + self.url = url + self.current = current + self.head = head + + class SpineTableNotFoundError(SpineDBAPIError): """Can't find one of the tables.""" def __init__(self, table):