File dependencies & csv import (#675)

* `config` accesible as script option

* Fixed cell type `str` erroring

* bootable & file attributes added

* Added FileDependency and FileScheme classes

* removed varargs from `discover`

* Deprecated `in_classmap`

* removed `stream` from interface, added file finding

* removed spatial dimension error

* Link with default config

* track whether a node was instantiated with pos args

* let `__tree__` pass through `__inv__` if provided

* added component bootstrapping

* Added filesystem engine with FileStore support

* removed old linking util

* removed links from core

* get config from the path where it used to be, from options. or basta

* removed links from project options

* moved file code to `_files.py`

* refactored code loading

* added `function_` type handler

* renamed exception

* added `file` to config module

* added fs plugin to setup

* removed doc note on links

* fixed ckdtree import

* changed name of engines

* switched to FileNodes

* updated tests

* added shortcut imports

* removed unused root arg and fixed dynamic attr check. closes #663

* removed ConfigFileAttribute in favor of FileDependencyNode

* Fixed config properties not returning the descriptor in class access

* added tests for #663

* improved example

* removed abstract Voxels from classmap

* updated to private imports

* added `provide_stream/locally` up to node in class hierarchy

* moved file attr boot to file node boot

* wip import placement

* Add function to return voxel id from position.
Rename variable in function snap_to_grid.

* added tqdm progress bars to importer

* Rename function index_of to coordinate_of and create index_of for VoxelSet.
coordinate_of provides 3d ids while index_of provides index in position list.

* Add all unmanaged partitions to a group. closes #493

* added import connect stub

* set chunk size on Chunk if it is known and missing

* moved imports to prevent circular import

* made progress bar optional, removed dead code

* removed dead imports

* fixed chunk id instability

* Fixed a bug where runtime modification and ref resolving would dupe refs

* shortened

* fixed setting dimensions on np array

* fixed tqdm semver

* fixed infinite loop

* fixed queue imposing single chunk pre roi

* check flush every 10k iterations

* fixed chunk comparators

* added missing ps interface methods

* overwrite content on update

* Added CsvImportConnectivity

* bumped bsb-hdf5 dependency

* added `overwrite` to fs engine

* fixed docs

---------

Co-authored-by: drodarie <d.rodarie@gmail.com>

bsb/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "4.0.0a46"
+__version__ = "4.0.0a47"
 
 import functools
 

bsb/_options.py

@@ -75,6 +75,7 @@ class ConfigOption(
     BsbOption,
     name="config",
     cli=("c", "config"),
+    script=("config",),
     project=("config",),
     env=("BSB_CONFIG_FILE",),
 ):

bsb/cell_types.py

@@ -67,12 +67,14 @@ def __lt__(self, other):
 
     @obj_str_insert
     def __repr__(self):
-        if hasattr(self, "scaffold"):
-            cells_placed = len(self.get_placement_set())
+        try:
             placements = len(self.get_placement())
-        else:
-            cells_placed = 0
+        except Exception:
             placements = "?"
+        try:
+            cells_placed = len(self.get_placement_set())
+        except Exception:
+            cells_placed = 0
         return f"'{self.name}', {cells_placed} cells, {placements} placement strategies"
 
     def get_placement(self):

bsb/cli/commands/_projects.py

@@ -55,21 +55,13 @@ def handler(self, context):
                     "tools": {
                         "bsb": {
                             "config": output,
-                            "links": {
-                                "config": "auto",
-                                "morpho": ["sys", "morphologies.hdf5", "newer"],
-                            },
                         }
                     }
                 },
                 f,
             )
-        init_path = root / name / "__init__.py"
-        place_path = root / name / "placement.py"
-        conn_path = root / name / "connectome.py"
-        if not init_path.exists():
-            with open(init_path, "w") as f:
-                f.write("\n")
+        place_path = root / "placement.py"
+        conn_path = root / "connectome.py"
         if not place_path.exists():
             with open(place_path, "w") as f:
                 f.write("from bsb.placement import PlacementStrategy\n")

bsb/config/__init__.py

@@ -18,6 +18,7 @@
     attr,
     list,
     dict,
+    file,
     node,
     root,
     dynamic,
@@ -63,6 +64,7 @@ def __init__(self, name):
     root = staticmethod(root)
     dynamic = staticmethod(dynamic)
     pluggable = staticmethod(pluggable)
+    file = staticmethod(file)
 
     walk_node_attributes = staticmethod(walk_node_attributes)
     walk_nodes = staticmethod(walk_nodes)
@@ -199,6 +201,10 @@ def file_has_parser_ext(kv):
 
 
 def _from_parsed(self, parser_name, tree, meta, file=None):
+    if "components" in tree:
+        from ._config import _bootstrap_components
+
+        _bootstrap_components(tree["components"])
     conf = self.Configuration(tree)
     conf._parser = parser_name
     conf._meta = meta

bsb/config/_attrs.py

@@ -56,7 +56,7 @@ def node(node_cls, root=False, dynamic=False, pluggable=False):
             else:
                 attrs[k] = v
     node_cls._config_attrs = attrs
-    node_cls.__post_new__ = compile_postnew(node_cls, root=root)
+    node_cls.__post_new__ = compile_postnew(node_cls)
     node_cls._config_isroot = root
     if root:
         node_cls.__post_new__ = wrap_root_postnew(node_cls.__post_new__)
@@ -138,6 +138,7 @@ def _dynamic(node_cls, class_attr, attr_name, config):
     node_cls._config_dynamic_attr = attr_name
     # Other than that compile the dynamic class like a regular node class
     node_cls = node(node_cls, dynamic=config)
+
     if config.auto_classmap or config.classmap:
         node_cls._config_dynamic_classmap = config.classmap or {}
     # This adds the parent class to its own classmap, which for subclasses happens in init
@@ -318,6 +319,16 @@ def unset():
     return ConfigurationAttribute(unset=True)
 
 
+def file(**kwargs):
+    """
+    Create a file dependency attribute.
+    """
+    from ..storage import FileDependencyNode
+
+    kwargs.setdefault("type", FileDependencyNode)
+    return attr(**kwargs)
+
+
 def _setattr(instance, name, value):
     instance.__dict__["_" + name] = value
 
@@ -362,6 +373,15 @@ def _root_is_booted(obj):
 def _boot_nodes(top_node, scaffold):
     for node in walk_nodes(top_node):
         node.scaffold = scaffold
+        # Boot attributes
+        for attr in getattr(node, "_config_attrs", {}).values():
+            booted = {None}
+            for cls in type(node).__mro__:
+                cls_attr = getattr(cls, attr.attr_name, None)
+                if (boot := getattr(cls_attr, "__boot__", None)) and boot not in booted:
+                    boot(node, scaffold)
+                    booted.add(boot)
+        # Boot node hook
         try:
             run_hook(node, "boot")
         except Exception as e:
@@ -931,9 +951,6 @@ def __ref__(self, instance, root):
             remote, remote_keys = self._prepare_self(instance, root)
         except NoReferenceAttributeSignal:  # pragma: nocover
             return None
-        if _hasattr(instance, self.attr_name):
-            remote_keys.extend(_getattr(instance, self.attr_name))
-            remote_keys = builtins.list(set(remote_keys))
         return self.resolve_reference_list(instance, remote, remote_keys)
 
     def resolve_reference_list(self, instance, remote, remote_keys):
@@ -1001,7 +1018,7 @@ def setter(self, f):
 
     def __get__(self, instance, owner):
         if instance is None:
-            return owner
+            return self
         return self.fget(instance)
 
     def __set__(self, instance, value):

bsb/config/_config.py

@@ -1,16 +1,17 @@
-from . import attr, dict, root, node, types
+from . import attr, list, dict, root, node, types
 from ..cell_types import CellType
 from ._attrs import _boot_nodes
 from ..placement import PlacementStrategy
+from ..storage._files import CodeDependencyNode
 from ..storage.interfaces import StorageNode
 from ..connectivity import ConnectionStrategy
 from ..simulation.simulation import Simulation
 from ..postprocessing import PostProcessingHook
-from ..exceptions import UnmanagedPartitionError
 from .._util import merge_dicts
 from ..topology import (
     get_partitions,
     create_topology,
+    RegionGroup,
     Region,
     Partition,
 )
@@ -46,6 +47,7 @@ class Configuration:
     """
 
     name = attr()
+    components = list(type=CodeDependencyNode)
     storage = attr(type=StorageNode, required=True)
     network = attr(type=NetworkNode, required=True)
     regions = dict(type=Region)
@@ -85,7 +87,10 @@ def _bootstrap(self, scaffold):
         # If there are any partitions not part of the topology, raise an error
         if unmanaged := set(self.partitions.values()) - get_partitions([topology]):
             p = "', '".join(p.name for p in unmanaged)
-            raise UnmanagedPartitionError(f"Please make '{p}' part of a Region.")
+            r = scaffold.regions.add(
+                "__unmanaged__", RegionGroup(children=builtins.list(unmanaged))
+            )
+            topology.children.append(r)
         # Activate the scaffold property of each config node
         _boot_nodes(self, scaffold)
         self._config_isbooted = True
@@ -101,3 +106,10 @@ def __str__(self):
 
     def __repr__(self):
         return f"{type(self).__qualname__}({self})"
+
+
+def _bootstrap_components(components, file_store=None):
+    for component in components:
+        component_node = CodeDependencyNode(component)
+        component_node.file_store = file_store
+        component_node.load_object()

bsb/config/_make.py

@@ -7,7 +7,7 @@
     DynamicClassError,
     UnresolvedClassCastError,
     PluginError,
-    DynamicClassNotFoundError,
+    DynamicObjectNotFoundError,
 )
 from ..reporting import warn
 from ._hooks import overrides
@@ -82,6 +82,7 @@ def __call__(meta_subject, *args, _parent=None, _key=None, **kwargs):
                         + f": e.g. 'def __init__{sig}'"
                     ) from None
                 else:
+                    instance._config_pos_init = bool(len(args))
                     instance.__init__(*args, **kwargs)
             return instance
 
@@ -203,7 +204,22 @@ def _set_pk(obj, parent, key):
             _setattr(obj, a.attr_name, key)
 
 
-def compile_postnew(cls, root=False):
+def _check_required(cls, attr, kwargs):
+    dynamic_root = getattr(cls, "_config_dynamic_root", None)
+    if dynamic_root is not None:
+        dynamic_attr = dynamic_root._config_dynamic_attr
+        # If we are checking the dynamic attribute, but we're already a dynamic subclass,
+        # we skip the required check.
+        return (
+            attr.attr_name == dynamic_attr
+            and cls is dynamic_root
+            and attr.required(kwargs)
+        ) or (attr.attr_name != dynamic_attr and attr.required(kwargs))
+    else:
+        return attr.required(kwargs)
+
+
+def compile_postnew(cls):
     def __post_new__(self, _parent=None, _key=None, **kwargs):
         attrs = _get_class_config_attrs(self.__class__)
         self._config_attr_order = list(kwargs.keys())
@@ -214,7 +230,8 @@ def __post_new__(self, _parent=None, _key=None, **kwargs):
             name = attr.attr_name
             value = values[name] = leftovers.pop(name, None)
             try:
-                if value is None and attr.required(kwargs):
+                # We use `self.__class__`, not `cls`, to get the proper child class.
+                if value is None and _check_required(self.__class__, attr, kwargs):
                     raise RequirementError(f"Missing required attribute '{name}'")
             except RequirementError as e:
                 # Catch both our own and possible `attr.required` RequirementErrors
@@ -233,7 +250,6 @@ def __post_new__(self, _parent=None, _key=None, **kwargs):
             else:
                 setattr(self, name, value)
                 attr.flag_dirty(self)
-        # # TODO: catch attrs
         for key, value in leftovers.items():
             try:
                 _try_catch_attrs(self, catch_attrs, key, value)
@@ -433,13 +449,8 @@ def _load_class(cfg_classname, module_path, interface=None, classmap=None):
         class_ref = cfg_classname
         class_name = cfg_classname.__name__
     else:
-        class_parts = cfg_classname.split(".")
-        class_name = class_parts[-1]
-        module_name = ".".join(class_parts[:-1])
-        if module_name == "":
-            class_ref = _search_module_path(class_name, module_path, cfg_classname)
-        else:
-            class_ref = _get_module_class(class_name, module_name, cfg_classname)
+        class_ref = _load_object(cfg_classname, module_path)
+        class_name = class_ref.__name__
 
     def qualname(cls):
         return cls.__module__ + "." + cls.__name__
@@ -453,15 +464,27 @@ def qualname(cls):
     return class_ref
 
 
+def _load_object(object_path, module_path):
+    class_parts = object_path.split(".")
+    object_name = class_parts[-1]
+    module_name = ".".join(class_parts[:-1])
+    if not module_name:
+        object_ref = _search_module_path(object_name, module_path, object_path)
+    else:
+        object_ref = _get_module_object(object_name, module_name, object_path)
+
+    return object_ref
+
+
 def _search_module_path(class_name, module_path, cfg_classname):
     for module_name in module_path:
         module_dict = sys.modules[module_name].__dict__
         if class_name in module_dict:
             return module_dict[class_name]
-    raise DynamicClassNotFoundError("Class not found: " + cfg_classname)
+    raise DynamicObjectNotFoundError("Class not found: " + cfg_classname)
 
 
-def _get_module_class(class_name, module_name, cfg_classname):
+def _get_module_object(object_name, module_name, object_path):
     sys.path.append(os.getcwd())
     try:
         module_ref = importlib.import_module(module_name)
@@ -470,9 +493,9 @@ def _get_module_class(class_name, module_name, cfg_classname):
         tmp.remove(os.getcwd())
         sys.path = list(reversed(tmp))
     module_dict = module_ref.__dict__
-    if class_name not in module_dict:
-        raise DynamicClassNotFoundError("Class not found: " + cfg_classname)
-    return module_dict[class_name]
+    if object_name not in module_dict:
+        raise DynamicObjectNotFoundError(f"'{object_path}' not found.")
+    return module_dict[object_name]
 
 
 def make_dictable(node_cls):
@@ -495,6 +518,11 @@ def __iter__(self):
 
 def make_tree(node_cls):
     def get_tree(instance):
+        if hasattr(instance, "__inv__") and not getattr(instance, "_config_inv", None):
+            instance._config_inv = True
+            inv = instance.__inv__()
+            instance._config_inv = False
+            return inv
         attrs = _get_class_config_attrs(instance.__class__)
         catch_attrs = [a for a in attrs.values() if hasattr(a, "__catch__")]
         tree = {}

bsb/config/types.py

@@ -5,7 +5,7 @@
 import numpy as np
 
 from ._compile import _reserved_kw_passes, _wrap_reserved
-from ._make import _load_class
+from ._make import _load_class, _load_object
 from ..exceptions import (
     ClassMapMissingError,
     CastError,
@@ -156,6 +156,42 @@ def __name__(self):
         return "class"
 
 
+class function_(TypeHandler):
+    """
+    Type validator. Attempts to import the value, absolute, or relative to the
+    `module_path` entries.
+
+    :param module_path: List of the modules that should be searched when doing a
+      relative import.
+    :type module_path: list[str]
+    :raises: TypeError when value can't be cast.
+    :returns: Type validator function
+    :rtype: Callable
+    """
+
+    def __init__(self, module_path=None):
+        self._module_path = module_path
+
+    def __call__(self, value):
+        msg = f"Could not import {value} as a callable function."
+        try:
+            obj = _load_object(value, self._module_path)
+        except Exception:
+            raise TypeError(msg)
+        else:
+            if not callable(obj):
+                raise TypeError(msg)
+            return obj
+
+    def __inv__(self, value):
+        if not inspect.isclass(value):
+            value = type(value)
+        return f"{value.__module__}.{value.__name__}"
+
+    def __name__(self):
+        return "class"
+
+
 def str(strip=False, lower=False, upper=False):
     """
     Type validator. Attempts to cast the value to an str, optionally with some sanitation.

bsb/connectivity/__init__.py

@@ -1,4 +1,5 @@
 from .strategy import ConnectionStrategy
 from .detailed import *
 from .general import *
+from .import_ import CsvImportConnectivity
 from .detailed.fiber_intersection import FiberTransform, QuiverTransform