Issue #21 updated, migrate only when explicitly asked for

spinedatabase_api/__init__.py

@@ -1,7 +1,7 @@
 from .database_mapping import DatabaseMapping
 from .diff_database_mapping import DiffDatabaseMapping
-from .exception import SpineDBAPIError, SpineIntegrityError, SpineTableNotFoundError, \
-    RecordNotFoundError, ParameterValueError
+from .exception import SpineDBAPIError, SpineIntegrityError, SpineDBVersionError, \
+    SpineTableNotFoundError, RecordNotFoundError, ParameterValueError
 from .helpers import create_new_spine_database, copy_database, merge_database, is_unlocked
 from .import_functions import import_data, import_object_classes, import_objects, \
     import_object_parameters, import_object_parameter_values, import_relationship_classes, \

spinedatabase_api/database_mapping.py

@@ -24,15 +24,21 @@
 :date:   11.8.2018
 """
 
+import os
 import time
 import logging
 from sqlalchemy import create_engine, false, distinct, func, MetaData, event
 from sqlalchemy.ext.automap import automap_base
 from sqlalchemy.orm import Session, aliased
 from sqlalchemy.pool import StaticPool
 from sqlalchemy.exc import NoSuchTableError, DBAPIError, DatabaseError
-from .exception import SpineDBAPIError, SpineTableNotFoundError, RecordNotFoundError, ParameterValueError
-from .helpers import custom_generate_relationship, attr_dict, upgrade_to_head
+from alembic.migration import MigrationContext
+from alembic.script import ScriptDirectory
+from alembic.config import Config
+from alembic import command
+from .exception import SpineDBAPIError, SpineDBVersionError, SpineTableNotFoundError, \
+    RecordNotFoundError, ParameterValueError
+from .helpers import custom_generate_relationship, attr_dict
 from datetime import datetime, timezone
 
 # TODO: Consider returning lists of dict (with _asdict()) rather than queries,
@@ -49,7 +55,7 @@ class DatabaseMapping(object):
         db_url (str): The database url formatted according to sqlalchemy rules
         username (str): The user name
     """
-    def __init__(self, db_url, username=None, create_all=True):
+    def __init__(self, db_url, username=None, create_all=True, migrate=False):
         """Initialize class."""
         self.db_url = db_url
         self.username = username
@@ -68,14 +74,14 @@ def __init__(self, db_url, username=None, create_all=True):
         self.ParameterDefinitionTag = None
         self.ParameterEnum = None
         self.Commit = None
-        upgrade_to_head(db_url)
         if create_all:
             self.create_engine_and_session()
+            self.check_db_version(migrate=migrate)
             self.create_mapping()
             # self.create_triggers()
 
     def create_engine_and_session(self):
-        """Create engine connected to self.db_url and session."""
+        """Create engine connected to self.db_url and corresponding session."""
         try:
             self.engine = create_engine(self.db_url)
             self.engine.connect()
@@ -94,6 +100,24 @@ def create_engine_and_session(self):
             #     raise SpineDBAPIError(msg)
         self.session = Session(self.engine, autoflush=False)
 
+    def check_db_version(self, migrate=False):
+        """Check if database is the latest version and raise a SpineDBVersionError if not.
+        If migrate is True, then don't raise the error and upgrade the database instead.
+        """
+        path = os.path.dirname(__file__)  # NOTE: this assumes this file and alembic.ini are on the same path
+        config = Config(os.path.join(path, "alembic.ini"))
+        config.set_main_option("script_location", "spinedatabase_api:alembic")
+        config.set_main_option("sqlalchemy.url", self.db_url)
+        script = ScriptDirectory.from_config(config)
+        head = script.get_current_head()
+        context = MigrationContext.configure(self.engine.connect())
+        current = context.get_current_revision()
+        if current == head:
+            return
+        if not migrate:
+            raise SpineDBVersionError(url=self.db_url, current=current, head=head)
+        command.upgrade(config, "head")
+
     def create_mapping(self):
         """Create ORM."""
         try:
@@ -666,7 +690,7 @@ def parameter_enum_list(self, id_list=None):
         qry = self.session.query(
             self.ParameterEnum.id.label("id"),
             self.ParameterEnum.name.label("name"),
-            self.ParameterEnum.value.label("value_index"),
+            self.ParameterEnum.value_index.label("value_index"),
             self.ParameterEnum.value.label("value"))
         if id_list is not None:
             qry = qry.filter(self.ParameterEnum.id.in_(id_list))

spinedatabase_api/diff_database_mapping.py

@@ -43,9 +43,9 @@ class DiffDatabaseMapping(DatabaseMapping):
     In a nutshell, it works by creating a new bunch of tables to hold differences
     with respect to original tables.
     """
-    def __init__(self, db_url, username=None, create_all=True):
+    def __init__(self, db_url, username=None, create_all=True, migrate=False):
         """Initialize class."""
-        super().__init__(db_url, username=username, create_all=False)
+        super().__init__(db_url, username=username, create_all=create_all, migrate=migrate)
         # Diff meta, Base and tables
         self.diff_prefix = None
         self.diff_metadata = None
@@ -69,11 +69,9 @@ def __init__(self, db_url, username=None, create_all=True):
         # Initialize stuff
         self.init_diff_dicts()
         if create_all:
-            self.create_engine_and_session()
-            self.create_mapping()
             self.create_diff_tables_and_mapping()
             self.init_next_id()
-            # self.create_triggers()
+            # self.create_diff_triggers()
 
     def has_pending_changes(self):
         """Return True if there are uncommitted changes. Otherwise return False."""
@@ -222,13 +220,12 @@ def init_next_id(self):
             self.close()
             raise SpineTableNotFoundError(table)
 
-    def create_triggers(self):
+    def create_diff_triggers(self):
         """Create ad-hoc triggers.
         NOTE: Not in use at the moment. Cascade delete is implemented in the `remove_items` method.
         TODO: is there a way to synch this with our CREATE TRIGGER statements
         from `helpers.create_new_spine_database`?
         """
-        super().create_triggers()
         @event.listens_for(self.DiffObjectClass, 'after_delete')
         def receive_after_object_class_delete(mapper, connection, object_class):
             @event.listens_for(self.session, "after_flush", once=True)
@@ -1750,7 +1747,7 @@ def check_wide_parameter_enums_for_update(self, *wide_kwargs_list, raise_intgr_e
         parameter_enum_dict = {
             x.id: {
                 "name": x.name,
-                "element_list": x.element_list.split(",")
+                "value_list": x.value_list.split(",")
             } for x in self.wide_parameter_enum_list()
         }
         parameter_enum_names = {x.name for x in self.wide_parameter_enum_list()}
@@ -2725,30 +2722,35 @@ def update_wide_parameter_enums(self, *wide_kwargs_list, raise_intgr_error=True)
         """Update parameter enums.
         NOTE: It's too difficult to do it cleanly, so we just remove and then add.
         """
+        checked_wide_kwargs_list, intgr_error_log = self.check_wide_parameter_enums_for_update(
+            *wide_kwargs_list, raise_intgr_error=raise_intgr_error)
+        wide_parameter_enum_dict = {x.id: x._asdict() for x in self.wide_parameter_enum_list()}
+        updated_ids = set()
+        item_list = list()
+        for wide_kwargs in checked_wide_kwargs_list:
+            try:
+                id = wide_kwargs['id']
+                updated_wide_kwargs = wide_parameter_enum_dict[id]
+            except KeyError:
+                continue
+            updated_ids.add(id)
+            updated_wide_kwargs.update(wide_kwargs)
+            for k, value in enumerate(updated_wide_kwargs['value_list']):
+                updated_narrow_kwargs = {
+                    'id': id,
+                    'name': updated_wide_kwargs['name'],
+                    'value_index': k,
+                    'value': value
+                }
+                item_list.append(updated_narrow_kwargs)
         try:
-            wide_parameter_enum_dict = {x.id: x._asdict() for x in self.wide_parameter_enum_list()}
-            updated_ids = set()
-            item_list = list()
-            for wide_kwargs in wide_kwargs_list:
-                try:
-                    id = wide_kwargs['id']
-                    updated_wide_kwargs = wide_parameter_enum_dict[id]
-                except KeyError:
-                    continue
-                updated_ids.add(id)
-                updated_wide_kwargs.update(wide_kwargs)
-                for k, value in enumerate(updated_wide_kwargs['value_list']):
-                    updated_narrow_kwargs = {
-                        'id': id,
-                        'name': updated_wide_kwargs['name'],
-                        'value_index': k,
-                        'value': value
-                    }
-                    item_list.append(updated_narrow_kwargs)
-            self.remove_items(parameter_enum_ids=updated_ids)
+            self.session.query(self.DiffParameterEnum).filter(self.DiffParameterEnum.id.in_(updated_ids)).\
+                delete(synchronize_session=False)
             self.session.bulk_insert_mappings(self.DiffParameterEnum, item_list)
             self.session.commit()
             self.new_item_id["parameter_enum"].update(updated_ids)
+            self.removed_item_id["parameter_enum"].update(updated_ids)
+            self.touched_item_id["parameter_enum"].update(updated_ids)
             updated_item_list = self.wide_parameter_enum_list(id_list=updated_ids)
             if not raise_intgr_error:
                 return updated_item_list, intgr_error_log
@@ -3076,8 +3078,7 @@ def _remove_cascade_parameter_definition_tags(self, ids, diff_ids, removed_item_
 
     def _remove_cascade_parameter_enums(self, ids, diff_ids, removed_item_id, removed_diff_item_id):
         """Remove parameter enums and all related items.
-        TODO: Should we remove parameter definitions here? Do we care if they have invalid enum?
-        If we *do* remove parameter definitions then we need to fix `update_wide_parameter_enum`
+        TODO: Should we remove parameter definitions here? Set their enum_id to NULL?
         """
         removed_item_id.setdefault("parameter_enum", set()).update(ids)
         removed_diff_item_id.setdefault("parameter_enum", set()).update(diff_ids)

spinedatabase_api/exception.py

@@ -38,6 +38,15 @@ def __init__(self, msg=None):
         super().__init__(msg)
 
 
+class SpineDBVersionError(SpineDBAPIError):
+    """Database version error."""
+    def __init__(self, url=None, current=None, head=None):
+        super().__init__(msg="The database at '{}' is not the latest version.".format(url))
+        self.url = url
+        self.current = current
+        self.head = head
+
+
 class SpineTableNotFoundError(SpineDBAPIError):
     """Can't find one of the tables."""
     def __init__(self, table):