Implement newly formalized graph traversal rules for export (#3262)

Thew newly formalized graph traversal rules are implemented in the
export interface. All twelve rules are used in traversing the graph
based on an initial set of nodes, but only a few can have the default
value overridden. Certain rules were decided not to be changeable and so
passing them will result in an exception. Also the command line interface
only exposed the toggleable flags.

The graph traversal logic has been moved from the export function itself
to a utility called `retrieve_linked_nodes`. Some duplication of queries
has been removed to improve the efficiency.

Archives now must be self-consistent in terms of the provenance graph
it represents, i.e., all nodes mentioned in the list of links have to
also be present in the archive. The danglink link checks can therefore
be removed from the export and import logic.

aiida/backends/tests/tools/importexport/orm/test_calculations.py

@@ -58,7 +58,7 @@ def max_(**kwargs):
         not_wanted_uuids = [v.uuid for v in (b, c, d)]
         # At this point we export the generated data
         filename1 = os.path.join(temp_dir, 'export1.tar.gz')
-        export([res], outfile=filename1, silent=True, return_reversed=True)
+        export([res], outfile=filename1, silent=True, return_backward=True)
         self.clean_db()
         self.insert_data()
         import_data(filename1, silent=True)

aiida/cmdline/commands/cmd_export.py

@@ -7,7 +7,7 @@
 # For further information on the license, see the LICENSE.txt file        #
 # For further information please visit http://www.aiida.net               #
 ###########################################################################
-# pylint: disable=too-many-arguments,import-error
+# pylint: disable=too-many-arguments,import-error,too-many-locals
 """`verdi export` command."""
 from __future__ import division
 from __future__ import print_function
@@ -24,6 +24,7 @@
 from aiida.cmdline.params import options
 from aiida.cmdline.utils import decorators
 from aiida.cmdline.utils import echo
+from aiida.tools.importexport import LINK_FLAGS
 
 
 @verdi.group('export')
@@ -70,28 +71,40 @@ def inspect(archive, version, data, meta_data):
 @options.ARCHIVE_FORMAT()
 @options.FORCE(help='overwrite output file if it already exists')
 @click.option(
-    '--input-forward/--no-input-forward',
-    default=False,
+    '--input-calc-forward/--no-input-calc-forward',
+    default=LINK_FLAGS['input_calc_forward'],
     show_default=True,
-    help='Follow forward INPUT links (recursively) when calculating the node set to export.'
+    help='Follow forward INPUT_CALC links (recursively) when calculating the node set to export.'
 )
 @click.option(
-    '--create-reversed/--no-create-reversed',
-    default=True,
+    '--input-work-forward/--no-input-work-forward',
+    default=LINK_FLAGS['input_work_forward'],
+    show_default=True,
+    help='Follow forward INPUT_WORK links (recursively) when calculating the node set to export.'
+)
+@click.option(
+    '--create-backward/--no-create-backward',
+    default=LINK_FLAGS['create_backward'],
     show_default=True,
     help='Follow reverse CREATE links (recursively) when calculating the node set to export.'
 )
 @click.option(
-    '--return-reversed/--no-return-reversed',
-    default=False,
+    '--return-backward/--no-return-backward',
+    default=LINK_FLAGS['return_backward'],
     show_default=True,
     help='Follow reverse RETURN links (recursively) when calculating the node set to export.'
 )
 @click.option(
-    '--call-reversed/--no-call-reversed',
-    default=False,
+    '--call-calc-backward/--no-call-calc-backward',
+    default=LINK_FLAGS['call_calc_backward'],
+    show_default=True,
+    help='Follow reverse CALL_CALC links (recursively) when calculating the node set to export.'
+)
+@click.option(
+    '--call-work-backward/--no-call-work-backward',
+    default=LINK_FLAGS['call_work_backward'],
     show_default=True,
-    help='Follow reverse CALL links (recursively) when calculating the node set to export.'
+    help='Follow reverse CALL_WORK links (recursively) when calculating the node set to export.'
 )
 @click.option(
     '--include-logs/--exclude-logs',
@@ -107,8 +120,8 @@ def inspect(archive, version, data, meta_data):
 )
 @decorators.with_dbenv()
 def create(
-    output_file, codes, computers, groups, nodes, archive_format, force, input_forward, create_reversed,
-    return_reversed, call_reversed, include_comments, include_logs
+    output_file, codes, computers, groups, nodes, archive_format, force, input_calc_forward, input_work_forward,
+    create_backward, return_backward, call_calc_backward, call_work_backward, include_comments, include_logs
 ):
     """
     Export parts of the AiiDA database to file for sharing.
@@ -133,10 +146,12 @@ def create(
         entities.extend(nodes)
 
     kwargs = {
-        'input_forward': input_forward,
-        'create_reversed': create_reversed,
-        'return_reversed': return_reversed,
-        'call_reversed': call_reversed,
+        'input_calc_forward': input_calc_forward,
+        'input_work_forward': input_work_forward,
+        'create_backward': create_backward,
+        'return_backward': return_backward,
+        'call_calc_backward': call_calc_backward,
+        'call_work_backward': call_work_backward,
         'include_comments': include_comments,
         'include_logs': include_logs,
         'overwrite': force

aiida/tools/importexport/__init__.py

@@ -8,7 +8,13 @@
 # For further information please visit http://www.aiida.net               #
 ###########################################################################
 # pylint: disable=wildcard-import,undefined-variable
-"""Provides import/export functionalities."""
+"""Provides import/export functionalities.
+
+To see history/git blame prior to the move to aiida.tools.importexport,
+explore tree: https://github.com/aiidateam/aiida-core/tree/eebef392c81e8b130834a92e1d7abf5e2e30b3ce
+Functionality: <tree>/aiida/orm/importexport.py
+Tests: <tree>/aiida/backends/tests/test_export_and_import.py
+"""
 from __future__ import division
 from __future__ import print_function
 from __future__ import absolute_import

aiida/tools/importexport/common/config.py

@@ -15,7 +15,7 @@
 
 from aiida.orm import Computer, Group, GroupTypeString, Node, User, Log, Comment
 
-__all__ = ('EXPORT_VERSION',)
+__all__ = ('EXPORT_VERSION', 'LINK_FLAGS')
 
 # Current export version
 EXPORT_VERSION = '0.7'
@@ -26,6 +26,23 @@
 # The name of the subfolder in which the node files are stored
 NODES_EXPORT_SUBFOLDER = 'nodes'
 
+# Default rules for following Links, when creating the graph of Nodes to export
+# Structure: key: <LinkType>_<direction-to-follow>, value: <Follow?>
+LINK_FLAGS = {
+    'input_calc_forward': False,  # Togglable
+    'input_calc_backward': True,
+    'create_forward': True,
+    'create_backward': True,  # Togglable
+    'return_forward': True,
+    'return_backward': False,  # Togglable
+    'input_work_forward': False,  # Togglable
+    'input_work_backward': True,
+    'call_calc_forward': True,
+    'call_calc_backward': False,  # Togglable
+    'call_work_forward': True,
+    'call_work_backward': False  # Togglable
+}
+
 # Giving names to the various entities. Attributes and links are not AiiDA
 # entities but we will refer to them as entities in the file (to simplify
 # references to them).