Add migrations

aiida/backends/djsite/db/__init__.py

@@ -7,5 +7,3 @@
 # For further information on the license, see the LICENSE.txt file        #
 # For further information please visit http://www.aiida.net               #
 ###########################################################################
-
-

aiida/backends/djsite/db/migrations/0015_invalidating_node_hash.py

@@ -22,6 +22,7 @@
 # Currently valid hash key
 _HASH_EXTRA_KEY = '_aiida_hash'
 
+
 def notify_user(apps, schema_editor):
     echo_warning("Invalidating all the hashes of all the nodes. Please run verdi rehash", bold=True)
 
@@ -37,7 +38,6 @@ class Migration(migrations.Migration):
         migrations.RunPython(notify_user, reverse_code=notify_user),
         migrations.RunSQL(
             """ DELETE FROM db_dbextra WHERE key='""" + _HASH_EXTRA_KEY + """';""",
-            reverse_sql=""" DELETE FROM db_dbextra 
-                          WHERE key='""" + _HASH_EXTRA_KEY + """';"""),
+            reverse_sql=""" DELETE FROM db_dbextra WHERE key='""" + _HASH_EXTRA_KEY + """';"""),
         upgrade_schema_version(REVISION, DOWN_REVISION)
     ]

aiida/backends/djsite/db/migrations/0023_calc_job_option_attribute_keys.py

@@ -0,0 +1,115 @@
+# -*- coding: utf-8 -*-
+###########################################################################
+# Copyright (c), The AiiDA team. All rights reserved.                     #
+# This file is part of the AiiDA code.                                    #
+#                                                                         #
+# The code is hosted on GitHub at https://github.com/aiidateam/aiida_core #
+# For further information on the license, see the LICENSE.txt file        #
+# For further information please visit http://www.aiida.net               #
+###########################################################################
+# pylint: disable=invalid-name,too-few-public-methods
+"""Migration of CalcJobNode attributes for metadata options whose key changed."""
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+from __future__ import absolute_import
+
+# Remove when https://github.com/PyCQA/pylint/issues/1931 is fixed
+# pylint: disable=no-name-in-module,import-error
+from django.db import migrations
+
+from aiida.backends.djsite.db.migrations import upgrade_schema_version
+
+REVISION = '1.0.23'
+DOWN_REVISION = '1.0.22'
+
+
+class Migration(migrations.Migration):
+    """Migration of CalcJobNode attributes for metadata options whose key changed.
+
+    Renamed attribute keys:
+
+      * `custom_environment_variables` -> `environment_variables`
+      * `jobresource_params` -> `resources`
+      * `_process_label` -> `process_label`
+      * `parser` -> `parser_name`
+
+    Deleted attributes:
+      * `linkname_retrieved` (We do not actually delete it just in case some relies on it)
+
+    """
+
+    dependencies = [
+        ('db', '0022_dbgroup_type_string_change_content'),
+    ]
+
+    operations = [
+        migrations.RunSQL(
+            sql="""
+            UPDATE db_dbattribute AS attribute
+            SET key =  regexp_replace(attribute.key, '^custom_environment_variables', 'environment_variables')
+            FROM db_dbnode AS node
+            WHERE
+                attribute.key like 'custom_environment_variables%' AND
+                node.type = 'node.process.calculation.calcjob.CalcJobNode.';
+            -- custom_environment_variables -> environment_variables
+
+            UPDATE db_dbattribute AS attribute
+            SET key =  regexp_replace(attribute.key, '^jobresource_params', 'resources')
+            FROM db_dbnode AS node
+            WHERE
+                attribute.key like 'jobresource_params%' AND
+                node.type = 'node.process.calculation.calcjob.CalcJobNode.';
+            -- jobresource_params -> resources
+
+            UPDATE db_dbattribute AS attribute
+            SET key =  regexp_replace(attribute.key, '^_process_label', 'process_label')
+            FROM db_dbnode AS node
+            WHERE
+                attribute.key like '_process_label' AND
+                node.type like 'node.process.%';
+            -- _process_label -> process_label
+
+            UPDATE db_dbattribute AS attribute
+            SET key =  regexp_replace(attribute.key, '^parser', 'parser_name')
+            FROM db_dbnode AS node
+            WHERE
+                attribute.key like 'parser%' AND
+                node.type = 'node.process.calculation.calcjob.CalcJobNode.';
+            -- parser -> parser_name
+            """,
+            reverse_sql="""
+            UPDATE db_dbattribute AS attribute
+            SET key =  regexp_replace(attribute.key, '^environment_variables', 'custom_environment_variables')
+            FROM db_dbnode AS node
+            WHERE
+                attribute.key like 'environment_variables%' AND
+                node.type = 'node.process.calculation.calcjob.CalcJobNode.';
+            -- environment_variables -> custom_environment_variables
+
+            UPDATE db_dbattribute AS attribute
+            SET key =  regexp_replace(attribute.key, '^resources', 'jobresource_params')
+            FROM db_dbnode AS node
+            WHERE
+                attribute.key like 'resources%' AND
+                node.type = 'node.process.calculation.calcjob.CalcJobNode.';
+            -- resources -> jobresource_params
+
+            UPDATE db_dbattribute AS attribute
+            SET key =  regexp_replace(attribute.key, '^process_label', '_process_label')
+            FROM db_dbnode AS node
+            WHERE
+                attribute.key like 'process_label%' AND
+                node.type like 'node.process.%';
+            -- process_label -> _process_label
+
+            UPDATE db_dbattribute AS attribute
+            SET key =  regexp_replace(attribute.key, '^parser_name', 'parser')
+            FROM db_dbnode AS node
+            WHERE
+                attribute.key like 'parser_name%' AND
+                node.type = 'node.process.calculation.calcjob.CalcJobNode.';
+            -- parser_name -> parser
+            """),
+        upgrade_schema_version(REVISION, DOWN_REVISION)
+    ]

aiida/backends/djsite/db/migrations/__init__.py

@@ -11,7 +11,8 @@
 from __future__ import print_function
 from __future__ import absolute_import
 
-LATEST_MIGRATION = '0022_dbgroup_type_string_change_content'
+LATEST_MIGRATION = '0023_calc_job_option_attribute_keys'
+
 
 def _update_schema_version(version, apps, schema_editor):
     from aiida.backends.djsite.utils import set_db_schema_version
@@ -31,7 +32,7 @@ def current_schema_version():
     # Have to use this ugly way of importing because the django migration
     # files start with numbers which are not a valid package name
     latest_migration = __import__(
-        "aiida.backends.djsite.db.migrations.{}".format(LATEST_MIGRATION),
+        'aiida.backends.djsite.db.migrations.{}'.format(LATEST_MIGRATION),
         fromlist=['REVISION']
     )
     return latest_migration.REVISION

aiida/backends/djsite/db/subtests/migrations.py

@@ -179,6 +179,62 @@ def test_uuid_untouched(self):
         n = load_node(self.node_id)
         self.assertEqual(self.node_uuid, n.uuid)
 
+
+class TestCalcAttributeKeysMigration(TestMigrations):
+
+    migrate_from = '0022_dbgroup_type_string_change_content'
+    migrate_to = '0023_calc_job_option_attribute_keys'
+
+    KEY_RESOURCES_OLD = 'jobresource_params'
+    KEY_RESOURCES_NEW = 'resources'
+    KEY_PARSER_NAME_OLD = 'parser'
+    KEY_PARSER_NAME_NEW = 'parser_name'
+    KEY_PROCESS_LABEL_OLD = '_process_label'
+    KEY_PROCESS_LABEL_NEW = 'process_label'
+    KEY_ENVIRONMENT_VARIABLES_OLD = 'custom_environment_variables'
+    KEY_ENVIRONMENT_VARIABLES_NEW = 'environment_variables'
+
+    def setUpBeforeMigration(self, apps):
+        from aiida.orm.node.process import WorkflowNode, CalcJobNode
+
+        self.process_label = 'TestLabel'
+        self.resources = {'number_machines': 1}
+        self.environment_variables = {}
+        self.parser_name = 'aiida.parsers:parser'
+
+        self.node_work = WorkflowNode()
+        self.node_work._set_attr(self.KEY_PROCESS_LABEL_OLD, self.process_label)
+        self.node_work.store()
+
+        self.node_calc = CalcJobNode(computer=self.computer)
+        self.node_calc._validate = lambda: True  # Need to disable the validation because we cannot set `resources`
+        self.node_calc._set_attr(self.KEY_PROCESS_LABEL_OLD, self.process_label)
+        self.node_calc._set_attr(self.KEY_RESOURCES_OLD, self.resources)
+        self.node_calc._set_attr(self.KEY_ENVIRONMENT_VARIABLES_OLD, self.environment_variables)
+        self.node_calc._set_attr(self.KEY_PARSER_NAME_OLD, self.parser_name)
+        self.node_calc.store()
+
+    def test_attribute_key_changes(self):
+        """Verify that the keys are successfully changed of the affected attributes."""
+        from aiida.orm import load_node
+
+        NOT_FOUND = tuple([0])
+
+        node_work = load_node(self.node_work.pk)
+        self.assertEqual(node_work.get_attr(self.KEY_PROCESS_LABEL_NEW), self.process_label)
+        self.assertEqual(node_work.get_attr(self.KEY_PROCESS_LABEL_OLD, default=NOT_FOUND), NOT_FOUND)
+
+        node_calc = load_node(self.node_calc.pk)
+        self.assertEqual(node_calc.get_attr(self.KEY_PROCESS_LABEL_NEW), self.process_label)
+        self.assertEqual(node_calc.get_attr(self.KEY_RESOURCES_NEW), self.resources)
+        self.assertEqual(node_calc.get_attr(self.KEY_ENVIRONMENT_VARIABLES_NEW), self.environment_variables)
+        self.assertEqual(node_calc.get_attr(self.KEY_PARSER_NAME_NEW), self.parser_name)
+        self.assertEqual(node_calc.get_attr(self.KEY_PROCESS_LABEL_OLD, default=NOT_FOUND), NOT_FOUND)
+        self.assertEqual(node_calc.get_attr(self.KEY_RESOURCES_OLD, default=NOT_FOUND), NOT_FOUND)
+        self.assertEqual(node_calc.get_attr(self.KEY_ENVIRONMENT_VARIABLES_OLD, default=NOT_FOUND), NOT_FOUND)
+        self.assertEqual(node_calc.get_attr(self.KEY_PARSER_NAME_OLD, default=NOT_FOUND), NOT_FOUND)
+
+
 class TestGroupRenamingMigration(TestMigrations):
 
     migrate_from = '0021_dbgroup_name_to_label_type_to_type_string'

aiida/backends/sqlalchemy/migrations/versions/7ca08c391c49_calc_job_option_attribute_keys.py

@@ -0,0 +1,98 @@
+# -*- coding: utf-8 -*-
+###########################################################################
+# Copyright (c), The AiiDA team. All rights reserved.                     #
+# This file is part of the AiiDA code.                                    #
+#                                                                         #
+# The code is hosted on GitHub at https://github.com/aiidateam/aiida_core #
+# For further information on the license, see the LICENSE.txt file        #
+# For further information please visit http://www.aiida.net               #
+###########################################################################
+# pylint: disable=invalid-name,no-member
+"""Migration of CalcJobNode attributes for metadata options whose key changed.
+
+Revision ID: 7ca08c391c49
+Revises: e72ad251bcdb
+Create Date: 2019-01-15 15:03:43.876133
+
+"""
+from __future__ import absolute_import
+from __future__ import print_function
+
+# Remove when https://github.com/PyCQA/pylint/issues/1931 is fixed
+# pylint: disable=no-name-in-module,import-error
+from alembic import op
+from sqlalchemy.sql import text
+
+# revision identifiers, used by Alembic.
+revision = '7ca08c391c49'
+down_revision = 'e72ad251bcdb'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    """Migration of CalcJobNode attributes for metadata options whose key changed.
+
+    Renamed attribute keys:
+
+      * `custom_environment_variables` -> `environment_variables`
+      * `jobresource_params` -> `resources`
+      * `_process_label` -> `process_label`
+      * `parser` -> `parser_name`
+
+    Deleted attributes:
+      * `linkname_retrieved` (We do not actually delete it just in case some relies on it)
+
+    """
+    conn = op.get_bind()
+
+    statement = text("""
+        UPDATE db_dbnode
+        SET attributes = jsonb_set(attributes, '{environment_variables}', to_jsonb(attributes->>'custom_environment_variables'))
+        WHERE
+            attributes ? 'custom_environment_variables' AND
+            type = 'node.process.calculation.calcjob.CalcJobNode.';
+        UPDATE db_dbnode SET attributes = attributes - 'custom_environment_variables'
+        WHERE
+            attributes ? 'custom_environment_variables' AND
+            type = 'node.process.calculation.calcjob.CalcJobNode.';
+        -- custom_environment_variables -> environment_variables
+
+        UPDATE db_dbnode
+        SET attributes = jsonb_set(attributes, '{resources}', to_jsonb(attributes->>'jobresource_params'))
+        WHERE
+            attributes ? 'jobresource_params' AND
+            type = 'node.process.calculation.calcjob.CalcJobNode.';
+        UPDATE db_dbnode SET attributes = attributes - 'jobresource_params'
+        WHERE
+            attributes ? 'jobresource_params' AND
+            type = 'node.process.calculation.calcjob.CalcJobNode.';
+        -- jobresource_params -> resources
+
+        UPDATE db_dbnode
+        SET attributes = jsonb_set(attributes, '{process_label}', to_jsonb(attributes->>'_process_label'))
+        WHERE
+            attributes ? '_process_label' AND
+            type = 'node.process.calculation.calcjob.CalcJobNode.';
+        UPDATE db_dbnode SET attributes = attributes - '_process_label'
+        WHERE
+            attributes ? '_process_label' AND
+            type like 'node.process.%';
+        -- _process_label -> process_label
+
+        UPDATE db_dbnode
+        SET attributes = jsonb_set(attributes, '{parser_name}', to_jsonb(attributes->>'parser'))
+        WHERE
+            attributes ? 'parser' AND
+            type = 'node.process.calculation.calcjob.CalcJobNode.';
+        UPDATE db_dbnode SET attributes = attributes - 'parser'
+        WHERE
+            attributes ? 'parser' AND
+            type = 'node.process.calculation.calcjob.CalcJobNode.';
+        -- parser -> parser_name
+        """)
+    conn.execute(statement)
+
+
+def downgrade():
+    pass

aiida/orm/node/process/process.py

@@ -33,8 +33,8 @@ class ProcessNode(Sealable, Node):
     EXCEPTION_KEY = 'exception'
     EXIT_MESSAGE_KEY = 'exit_message'
     EXIT_STATUS_KEY = 'exit_status'
-    PROCESS_LABEL_KEY = '_process_label'
     PROCESS_PAUSED_KEY = 'paused'
+    PROCESS_LABEL_KEY = 'process_label'
     PROCESS_STATE_KEY = 'process_state'
     PROCESS_STATUS_KEY = 'process_status'