Automatic documentation generation for configuration nodes (#709)

* wip, recommit later

* Morphology pipeline can store end result

* give underlying cast error

* Fixed error with `has_own_init` checking incorrect class

* Requirement handlers can check if the node data is written in shortform

* Preferably parse swc with inhouse swc parser

* Added NameSelector shorthand

* Fixed unserializable `FileDependency` attributes

* Fixed pipeline calls

* Added morphology creation from swc data

* Added stable morpho comparison and allowed indicesless MorphologySet

* Removed useless print commands

* added `hint` kwarg to pass hint example values to autoconfig

* expose more type information on attr

* don't attempt to invert `None`

* renamed and exposed `get_config_attributes`

* use more robust `getattr` instead of dict lookup in module refs

* added parser lookup methods

* added syntax specification and `generate` method to parser iface

* return already callable objects directly from `function_` type handler

* pass (unenforced) type information to the property arg

* autoconfig all the nodes in the reference

* bumped checkout action

* Fixed inversion of method shortcut

* catch method's attribute error and reraise as typeerror

---------

Co-authored-by: Francesco Sheiban <frances.j.shei@gmail.com>

.github/workflows/build.yml

@@ -10,7 +10,7 @@ jobs:
       matrix:
         python-version: ["3.8", "3.9", "3.10"]
     steps:
-    - uses: actions/checkout@v2.5.0
+    - uses: actions/checkout@v3.5.0
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v4.3.0
       with:

bsb/config/__init__.py

@@ -33,7 +33,7 @@
     ConfigurationAttribute,
 )
 from .._util import ichain
-from ._make import walk_node_attributes, walk_nodes, compose_nodes
+from ._make import walk_node_attributes, walk_nodes, compose_nodes, get_config_attributes
 from ._hooks import on, before, after, run_hook, has_hook
 from .. import plugins
 from ..exceptions import ConfigTemplateNotFoundError, ParserError, PluginError
@@ -67,6 +67,7 @@ def __init__(self, name):
     file = staticmethod(file)
 
     walk_node_attributes = staticmethod(walk_node_attributes)
+    get_config_attributes = staticmethod(get_config_attributes)
     walk_nodes = staticmethod(walk_nodes)
     compose_nodes = staticmethod(compose_nodes)
     on = staticmethod(on)

bsb/config/_attrs.py

@@ -7,6 +7,7 @@
 
 from ._hooks import run_hook
 from ._make import (
+    MISSING,
     compile_class,
     compile_postnew,
     compile_new,
@@ -419,6 +420,7 @@ def __init__(
         required=False,
         key=False,
         unset=False,
+        hint=MISSING,
     ):
         if not callable(required):
             self.required = lambda s: required
@@ -427,8 +429,9 @@ def __init__(
         self.key = key
         self.default = default
         self.call_default = call_default
-        self.type = self._get_type(type)
+        self.type = self._set_type(type)
         self.unset = unset
+        self.hint = hint
 
     def __set_name__(self, owner, name):
         self.attr_name = name
@@ -460,6 +463,7 @@ def __set__(self, instance, value):
                 e.node, e.attr = instance, self.attr_name
             raise
         except Exception as e:
+            traceback.print_exc()
             raise CastError(
                 f"Couldn't cast '{value}' into {self.type.__name__}: {e}",
                 instance,
@@ -471,7 +475,8 @@ def __set__(self, instance, value):
         if _is_booted(root):
             _boot_nodes(value, root.scaffold)
 
-    def _get_type(self, type):
+    def _set_type(self, type):
+        self._config_type = type
         # Determine type of the attribute
         if not type and self.default is not None:
             if self.should_call_default():
@@ -485,9 +490,20 @@ def _get_type(self, type):
         t = _wrap_reserved(t)
         return t
 
+    def get_type(self):
+        return self._config_type
+
+    def get_hint(self):
+        if hasattr(self.type, "__hint__"):
+            return self.type.__hint__()
+        return self.hint
+
     def get_node_name(self, instance):
         return instance.get_node_name() + "." + self.attr_name
 
+    def is_node_type(self):
+        return hasattr(self._config_type, "_config_attrs")
+
     def tree(self, instance):
         val = _getattr(instance, self.attr_name)
         # Allow subnodes and other class values to convert themselves to their tree
@@ -496,7 +512,7 @@ def tree(self, instance):
             val = val.__tree__()
         # Check if the type handler specifies any inversion function to convert tree
         # values back to how they were found in the document.
-        if hasattr(self.type, "__inv__"):
+        if hasattr(self.type, "__inv__") and val is not None:
             val = self.type.__inv__(val)
         return val
 
@@ -587,7 +603,7 @@ def _fromiter(self, start, items):
 
     def _preset(self, index, item):
         try:
-            item = self._config_type(item, _parent=self, _key=index)
+            item = self._elem_type(item, _parent=self, _key=index)
             try:
                 item._config_index = index
             except Exception as e:
@@ -596,7 +612,7 @@ def _preset(self, index, item):
         except (RequirementError, CastError) as e:
             e.args = (
                 f"Couldn't cast element {index} from '{item}'"
-                + f" into a {self._config_type.__name__}. "
+                + f" into a {self._elem_type.__name__}. "
                 + e.msg,
                 *e.args,
             )
@@ -606,7 +622,7 @@ def _preset(self, index, item):
         except Exception as e:
             raise CastError(
                 f"Couldn't cast element {index} from '{item}'"
-                + f" into a {self._config_type.__name__}: {e}"
+                + f" into a {self._elem_type.__name__}: {e}"
             )
 
     def _postset(self, items):
@@ -640,7 +656,7 @@ def fill(self, value, _parent, _key=None):
         _cfglist = cfglist()
         _cfglist._config_parent = _parent
         _cfglist._config_attr = self
-        _cfglist._config_type = self.child_type
+        _cfglist._elem_type = self.child_type
         if isinstance(value, builtins.dict):
             raise CastError(f"Dictionary `{value}` given where list is expected.")
         _cfglist.extend(value or builtins.list())
@@ -651,8 +667,8 @@ def fill(self, value, _parent, _key=None):
             )
         return _cfglist
 
-    def _get_type(self, type):
-        self.child_type = super()._get_type(type)
+    def _set_type(self, type):
+        self.child_type = super()._set_type(type)
         return self.fill
 
     def tree(self, instance):
@@ -673,7 +689,7 @@ def __setitem__(self, key, value):
         if key in self:
             _unset_nodes(self[key])
         try:
-            value = self._config_type(value, _parent=self, _key=key)
+            value = self._elem_type(value, _parent=self, _key=key)
         except (RequirementError, CastError) as e:
             if not (hasattr(e, "node") and e.node):
                 e.node, e.attr = self, key
@@ -683,7 +699,7 @@ def __setitem__(self, key, value):
 
             raise CastError(
                 "Couldn't cast {}.{} from '{}' into a {}".format(
-                    self.get_node_name(), key, value, self._config_type.__name__
+                    self.get_node_name(), key, value, self._elem_type.__name__
                 )
                 + "\n"
                 + traceback.format_exc()
@@ -702,7 +718,7 @@ def add(self, key, *args, **kwargs):
                 f"{self.get_node_name()} already contains '{key}'."
                 + " Use `node[key] = value` if you want to overwrite it."
             )
-        self[key] = value = self._config_type(*args, _parent=self, _key=key, **kwargs)
+        self[key] = value = self._elem_type(*args, _parent=self, _key=key, **kwargs)
         return value
 
     def clear(self):
@@ -760,7 +776,7 @@ def get_node_name(self):
 class cfgdictcopy(builtins.dict):
     def __init__(self, other):
         super().__init__(other)
-        self._config_type = other._config_type
+        self._elem_type = other._elem_type
         self._copied_from = other
 
     @builtins.property
@@ -787,12 +803,12 @@ def fill(self, value, _parent, _key=None):
         _cfgdict._config_parent = _parent
         _cfgdict._config_key = _key
         _cfgdict._config_attr = self
-        _cfgdict._config_type = self.child_type
+        _cfgdict._elem_type = self.child_type
         _cfgdict.update(value or builtins.dict())
         return _cfgdict
 
-    def _get_type(self, type):
-        self.child_type = super()._get_type(type)
+    def _set_type(self, type):
+        self.child_type = super()._set_type(type)
         return self.fill
 
     def tree(self, instance):

bsb/config/_make.py

@@ -225,7 +225,7 @@ def _set_pk(obj, parent, key):
         obj._config_attr_order = []
     if not hasattr(obj, "_config_state"):
         obj._config_state = {}
-    for a in _get_class_config_attrs(obj.__class__).values():
+    for a in get_config_attributes(obj.__class__).values():
         if a.key:
             from ._attrs import _setattr
 
@@ -252,7 +252,7 @@ def _check_required(instance, attr, kwargs):
 
 def compile_postnew(cls):
     def __post_new__(self, _parent=None, _key=None, **kwargs):
-        attrs = _get_class_config_attrs(self.__class__)
+        attrs = get_config_attributes(self.__class__)
         self._config_attr_order = list(kwargs.keys())
         catch_attrs = [a for a in attrs.values() if hasattr(a, "__catch__")]
         leftovers = kwargs.copy()
@@ -341,7 +341,7 @@ def _bubble_up_warnings(log):
             warn(str(m), w.category, stacklevel=4)
 
 
-def _get_class_config_attrs(cls):
+def get_config_attributes(cls):
     attrs = {}
     for p_cls in reversed(cls.__mro__):
         if hasattr(p_cls, "_config_attrs"):
@@ -522,24 +522,24 @@ def _get_module_object(object_name, module_name, object_path):
         tmp = list(reversed(sys.path))
         tmp.remove(os.getcwd())
         sys.path = list(reversed(tmp))
-    module_dict = module_ref.__dict__
-    if object_name not in module_dict:
+    try:
+        return getattr(module_ref, object_name)
+    except Exception:
         raise DynamicObjectNotFoundError(f"'{object_path}' not found.")
-    return module_dict[object_name]
 
 
 def make_dictable(node_cls):
     def __contains__(self, attr):
-        return attr in _get_class_config_attrs(self.__class__)
+        return attr in get_config_attributes(self.__class__)
 
     def __getitem__(self, attr):
-        if attr in _get_class_config_attrs(self.__class__):
+        if attr in get_config_attributes(self.__class__):
             return getattr(self, attr)
         else:
             raise KeyError(attr)
 
     def __iter__(self):
-        return (attr for attr in _get_class_config_attrs(self.__class__))
+        return (attr for attr in get_config_attributes(self.__class__))
 
     node_cls.__contains__ = __contains__
     node_cls.__getitem__ = __getitem__
@@ -553,7 +553,7 @@ def get_tree(instance):
             inv = instance.__inv__()
             instance._config_inv = False
             return inv
-        attrs = _get_class_config_attrs(instance.__class__)
+        attrs = get_config_attributes(instance.__class__)
         catch_attrs = [a for a in attrs.values() if hasattr(a, "__catch__")]
         tree = {}
         for name in instance._config_attr_order:

bsb/config/parsers/__init__.py

@@ -1,2 +1,12 @@
 from ._parser import Parser
 from .json import JsonParser
+
+
+def get_parser_classes():
+    from ...plugins import discover
+
+    return discover("config.parsers")
+
+
+def get_parser(parser):
+    return get_parser_classes()[parser]()

bsb/config/parsers/_parser.py

@@ -5,3 +5,7 @@ class Parser(abc.ABC):
     @abc.abstractmethod
     def parse(self, content, path=None):
         pass
+
+    @abc.abstractmethod
+    def generate(self, tree, pretty=False):
+        pass

bsb/config/parsers/json.py

@@ -149,6 +149,7 @@ class JsonParser(Parser):
 
     data_description = "JSON"
     data_extensions = ("json",)
+    data_syntax = "json"
 
     def parse(self, content, path=None):
         # Parses the content. If path is set it's used as the root for the multi-document
@@ -170,6 +171,12 @@ def parse(self, content, path=None):
         meta = {"path": path}
         return content, meta
 
+    def generate(self, tree, pretty=False):
+        if pretty:
+            return json.dumps(tree, indent=4)
+        else:
+            return json.dumps(tree)
+
     def _traverse(self, node, iter):
         # Iterates over all values in `iter` and checks for import keys, recursion or refs
         # Also wraps all nodes in their `parsed_*` counterparts.

bsb/config/types.py

@@ -202,6 +202,8 @@ class function_(object_):
     """
 
     def __call__(self, value):
+        if callable(value):
+            return value
         obj = super().__call__(value)
         if not callable(obj):
             raise TypeError(f"Could not import {value} as a callable function.")
@@ -217,11 +219,15 @@ def __name__(self):
 
 class method(function_):
     def __init__(self, class_name):
+        super().__init__()
         self._class = class_name
 
     def __call__(self, value):
         parent = class_()(self._class)
-        obj = getattr(parent, value)
+        try:
+            obj = getattr(parent, value)
+        except AttributeError as e:
+            raise TypeError(builtins.str(e)) from None
         if not callable(obj):
             raise TypeError(f"Could not import '{value}' as a method of `{self._class}`.")
         return obj
@@ -246,7 +252,7 @@ def __inv__(self, value):
         return self._map.get(value, value)
 
 
-class method_shortcut(WeakInverter, method, function_):
+class method_shortcut(method, function_):
     def __call__(self, value):
         try:
             obj = method.__call__(self, value)
@@ -257,9 +263,18 @@ def __call__(self, value):
                 raise TypeError(
                     f"Could not import '{value}' as a function or a method of `{self._class}`."
                 ) from None
-        self.store_value(value, obj)
         return obj
 
+    def __inv__(self, value):
+        if inspect.isfunction(value):
+            try:
+                method.__call__(self, value.__name__)
+                return method.__inv__(self, value)
+            except TypeError:
+                return function_.__inv__(self, value)
+        else:
+            return value
+
 
 def str(strip=False, lower=False, upper=False):
     """

bsb/topology/partition.py

@@ -495,7 +495,7 @@ class AllenStructure(NrrdVoxels, classmap_entry="allen"):
     """Name or acronym of the region to filter within the annotation volume according to the AMBRH.
     If struct_name is set, then struct_id should not be set."""
 
-    @config.property
+    @config.property(type=int)
     def voxel_size(self):
         """Size of each voxel."""
         return self._voxel_size if self._voxel_size is not None else 25
@@ -504,11 +504,11 @@ def voxel_size(self):
     def voxel_size(self, value):
         self._voxel_size = value
 
-    @config.property
+    @config.property(type=bool)
     def mask_only(self):
         return self.source is None and len(self.sources) == 0
 
-    @config.property
+    @config.property(type=str)
     @functools.cache
     def mask_source(self):
         if hasattr(self, "_annotations_file"):

docs/conf.py

@@ -64,6 +64,7 @@
     "zwembad",
     "arbor",
     "morphio",
+    "nrrd",
 ]
 
 intersphinx_mapping = {