a49 - Morphology pipelines (#686)

* wip, recommit later

* Morphology pipeline can store end result

* give underlying cast error

* Fixed error with `has_own_init` checking incorrect class

* Requirement handlers can check if the node data is written in shortform

* Preferably parse swc with inhouse swc parser

* Added NameSelector shorthand

* Fixed unserializable `FileDependency` attributes

* Fixed pipeline calls

* Added morphology creation from swc data

* Added stable morpho comparison and allowed indicesless MorphologySet

* Removed useless print commands

* removed debug traceback print

* Fixed missing exception imports

* Added `pipeline_method` type handler

* Allow methods of Morphology to be used in morpho pipeline

* updated func to new method shortcut syntax

* Added `detach` method to branch

* Try to sanitize `rotation` input as euler degrees Rotation

* docstring

* Added `assertNotClose`

* test morphology pipelines

* fix parametrized pipeline call

* Fixed method shortcuts with fallback to regular func import

* return morpho from stale load_object code path

* 3.8 backwards compat

---------

Co-authored-by: Francesco Sheiban <frances.j.shei@gmail.com>

bsb/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "4.0.0a48"
+__version__ = "4.0.0a49"
 
 import functools
 

bsb/config/_attrs.py

@@ -1,6 +1,8 @@
 """
     An attrs-inspired class annotation system, but my A stands for amateuristic.
 """
+import traceback
+
 import errr
 
 from ._hooks import run_hook
@@ -24,6 +26,7 @@
     CfgReferenceError,
     BootError,
 )
+from ..services import MPI
 import builtins
 
 
@@ -386,6 +389,7 @@ def _boot_nodes(top_node, scaffold):
             run_hook(node, "boot")
         except Exception as e:
             errr.wrap(BootError, e, prepend=f"Failed to boot {node}:")
+    MPI.barrier()
 
 
 def _unset_nodes(top_node):
@@ -586,7 +590,7 @@ def _preset(self, index, item):
             item = self._config_type(item, _parent=self, _key=index)
             try:
                 item._config_index = index
-            except Exception:
+            except Exception as e:
                 pass
             return item
         except (RequirementError, CastError) as e:
@@ -599,10 +603,10 @@ def _preset(self, index, item):
             if not e.node:
                 e.node, e.attr = self, index
             raise
-        except Exception:
+        except Exception as e:
             raise CastError(
                 f"Couldn't cast element {index} from '{item}'"
-                + f" into a {self._config_type.__name__}"
+                + f" into a {self._config_type.__name__}: {e}"
             )
 
     def _postset(self, items):

bsb/config/_config.py

@@ -2,7 +2,7 @@
 from ..cell_types import CellType
 from ._attrs import _boot_nodes
 from ..placement import PlacementStrategy
-from ..storage._files import CodeDependencyNode
+from ..storage._files import CodeDependencyNode, MorphologyDependencyNode
 from ..storage.interfaces import StorageNode
 from ..connectivity import ConnectionStrategy
 from ..simulation.simulation import Simulation
@@ -48,6 +48,7 @@ class Configuration:
 
     name = attr()
     components = list(type=CodeDependencyNode)
+    morphologies = list(type=MorphologyDependencyNode)
     storage = attr(type=StorageNode, required=True)
     network = attr(type=NetworkNode, required=True)
     regions = dict(type=Region)

bsb/config/_make.py

@@ -21,6 +21,14 @@
 import types
 
 
+def _has_own_init(meta_subject, kwargs):
+    try:
+        determined_class = meta_subject.__new__.class_determinant(meta_subject, kwargs)
+        return overrides(determined_class, "__init__", mro=True)
+    except Exception:
+        return overrides(meta_subject, "__init__", mro=True)
+
+
 def make_metaclass(cls):
     # We make a `NodeMeta` class for each decorated node class, in compliance with any
     # metaclasses they might already have (to prevent metaclass confusion).
@@ -36,7 +44,7 @@ def make_metaclass(cls):
     # and keep the object reference that the user gives them
     class ConfigArgRewrite:
         def __call__(meta_subject, *args, _parent=None, _key=None, **kwargs):
-            has_own_init = overrides(meta_subject, "__init__", mro=True)
+            has_own_init = _has_own_init(meta_subject, kwargs)
             # Rewrite the arguments
             primer = args[0] if args else None
             if isinstance(primer, meta_subject):
@@ -55,7 +63,7 @@ def __call__(meta_subject, *args, _parent=None, _key=None, **kwargs):
                 raise ValueError(f"Unexpected positional argument '{primer}'")
             # Call the base class's new with internal arguments
             instance = meta_subject.__new__(
-                meta_subject, _parent=_parent, _key=_key, **kwargs
+                meta_subject, *args, _parent=_parent, _key=_key, **kwargs
             )
             instance._config_pos_init = getattr(instance, "_config_pos_init", False)
             # Call the end user's __init__ with the rewritten arguments, if one is defined
@@ -100,6 +108,12 @@ def __new__(cls, *args, **kwargs):
     return NodeMeta
 
 
+class NodeKwargs(dict):
+    def __init__(self, instance, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.is_shortform = getattr(instance, "_config_pos_init", False)
+
+
 def compose_nodes(*node_classes):
     """
     Create a composite mixin class of the given classes. Inherit from the returned class
@@ -187,17 +201,20 @@ def compile_new(node_cls, dynamic=False, pluggable=False, root=False):
     else:
         class_determinant = _node_determinant
 
-    def __new__(_cls, _parent=None, _key=None, **kwargs):
+    def __new__(_cls, *args, _parent=None, _key=None, **kwargs):
         ncls = class_determinant(_cls, kwargs)
         instance = object.__new__(ncls)
+        instance._config_pos_init = bool(len(args))
         _set_pk(instance, _parent, _key)
         if root:
             instance._config_isfinished = False
         instance.__post_new__(**kwargs)
         if _cls is not ncls:
-            instance.__init__(**kwargs)
+            instance.__init__(*args, **kwargs)
         return instance
 
+    __new__.class_determinant = class_determinant
+
     return __new__
 
 
@@ -215,8 +232,11 @@ def _set_pk(obj, parent, key):
             _setattr(obj, a.attr_name, key)
 
 
-def _check_required(cls, attr, kwargs):
+def _check_required(instance, attr, kwargs):
+    # We use `self.__class__`, not `cls`, to get the proper child class.
+    cls = instance.__class__
     dynamic_root = getattr(cls, "_config_dynamic_root", None)
+    kwargs = NodeKwargs(instance, kwargs)
     if dynamic_root is not None:
         dynamic_attr = dynamic_root._config_dynamic_attr
         # If we are checking the dynamic attribute, but we're already a dynamic subclass,
@@ -241,8 +261,7 @@ def __post_new__(self, _parent=None, _key=None, **kwargs):
             name = attr.attr_name
             value = values[name] = leftovers.pop(name, None)
             try:
-                # We use `self.__class__`, not `cls`, to get the proper child class.
-                if value is None and _check_required(self.__class__, attr, kwargs):
+                if value is None and _check_required(self, attr, kwargs):
                     raise RequirementError(f"Missing required attribute '{name}'")
             except RequirementError as e:
                 # Catch both our own and possible `attr.required` RequirementErrors

bsb/config/types.py

@@ -3,9 +3,9 @@
 import inspect
 import math
 import numpy as np
-
+from weakref import WeakKeyDictionary
 from ._compile import _reserved_kw_passes, _wrap_reserved
-from ._make import _load_class, _load_object
+from ._make import _load_object
 from ..exceptions import (
     ClassMapMissingError,
     CastError,
@@ -91,7 +91,7 @@ def or_(*type_args):
     :raises: TypeError if none of the given type validators can cast the value.
     :rtype: Callable
     """
-    handler_name = "any of: " + ", ".join(map(lambda x: x.__name__, type_args))
+    handler_name = "any of: " + ", ".join(x.__name__ for x in type_args)
     # Make sure to wrap all type handlers so that they accept the parent and key args.
     type_args = [_wrap_reserved(t) for t in type_args]
 
@@ -141,22 +141,23 @@ def __init__(self, module_path=None):
         self._module_path = module_path
 
     def __call__(self, value):
-        msg = f"Could not import object {value}."
+        msg = f"Could not import '{value}': "
         try:
             obj = _load_object(value, self._module_path)
             obj._cfg_inv = value
-        except Exception:
-            raise TypeError(msg)
+        except Exception as e:
+            raise TypeError(msg + builtins.str(e))
         return obj
 
     def __inv__(self, value):
         return getattr(value, "_cfg_inv", value)
 
+    @property
     def __name__(self):
         return "object"
 
 
-class class_(TypeHandler):
+class class_(object_):
     """
     Type validator. Attempts to import the value as the name of a class, relative to
     the `module_path` entries, absolute or just returning it if it is already a class.
@@ -169,20 +170,20 @@ class class_(TypeHandler):
     :rtype: Callable
     """
 
-    def __init__(self, module_path=None):
-        self._module_path = module_path
-
     def __call__(self, value):
-        try:
-            return _load_class(value, self._module_path)
-        except Exception:
-            raise TypeError("Could not import {} as a class".format(value))
+        if inspect.isclass(value):
+            return value
+        obj = super().__call__(value)
+        if not inspect.isclass(obj):
+            raise TypeError(f"'{value}' is not a class, got {builtins.type(obj)} instead")
+        return obj
 
     def __inv__(self, value):
         if not inspect.isclass(value):
             value = type(value)
         return f"{value.__module__}.{value.__name__}"
 
+    @property
     def __name__(self):
         return "class"
 
@@ -209,10 +210,57 @@ def __call__(self, value):
     def __inv__(self, value):
         return f"{value.__module__}.{value.__name__}"
 
+    @property
     def __name__(self):
         return "function"
 
 
+class method(function_):
+    def __init__(self, class_name):
+        self._class = class_name
+
+    def __call__(self, value):
+        parent = class_()(self._class)
+        obj = getattr(parent, value)
+        if not callable(obj):
+            raise TypeError(f"Could not import '{value}' as a method of `{self._class}`.")
+        return obj
+
+    def __inv__(self, value):
+        return value.__name__
+
+    @property
+    def __name__(self):
+        return f"method of '{self._class}'"
+
+
+class WeakInverter:
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._map = WeakKeyDictionary()
+
+    def store_value(self, value, result):
+        self._map[result] = value
+
+    def __inv__(self, value):
+        return self._map.get(value, value)
+
+
+class method_shortcut(WeakInverter, method, function_):
+    def __call__(self, value):
+        try:
+            obj = method.__call__(self, value)
+        except TypeError:
+            try:
+                obj = function_.__call__(self, value)
+            except TypeError:
+                raise TypeError(
+                    f"Could not import '{value}' as a function or a method of `{self._class}`."
+                ) from None
+        self.store_value(value, obj)
+        return obj
+
+
 def str(strip=False, lower=False, upper=False):
     """
     Type validator. Attempts to cast the value to an str, optionally with some sanitation.
@@ -650,6 +698,13 @@ def requirement(section):
     return requirement
 
 
+def shortform():
+    def requirement(section):
+        return not section.is_shortform
+
+    return requirement
+
+
 class ndarray(TypeHandler):
     """
     Type validator numpy arrays.