Fix/requirements second version (#844)

* Fix README.md links

* Fix PackageRequirement, ConfigurationListAttribute and ConfigurationDictAttribute.

* Fix CodeDependencyNode. Inv function was providing the resolved path instead of the original str.

* 2nd fix for CodeDependencyNode, tree_of should either be based on __tree__ or __inv__.

* Revert changes on ConfigurationListAttribute and ConfigurationDictAttribute and PackageRequirement.
Now expose __inv__ of fill function for ConfigurationListAttribute and ConfigurationDictAttribute

* Add attr to the CodeDependencyNode __inv__

* Fix test_particle_vd (forgot to retrieve 1)

* Fix test_particle_vd (forgot to retrieve 1) and PackageRequirement __inv__

* Fix test_code_dependency_node for folders with a `.` inside.

* Change CodeDependencyNode. Now allow both loading of python file or module.

* Resolve relative file paths for CodeDependencyNode for files that are outside the root folder.

* Solution 3 for ConfigurationListAttribute and ConfigurationDictAttribute: use a wrapper to expose __inv__ instead of setting it as a static attribute.

* Function tree_of -> elif back to if.
Add test in CodeDependencyNode __inv__ function (similar to FileDependencyNode)

* Take review feedback into account.

* Add forgotten test_file

* fix duplicate occurrence of BoxTree in public API

* Add test for duplicate api key

* Update tests/data/code_dependency.py

* Fix morphologies __init__.py

---------

Co-authored-by: Robin De Schepper <robin.deschepper93@gmail.com>

.github/devops/generate_public_api.py

@@ -36,6 +36,10 @@ def get_public_api_map():
                         if isinstance(el, ast.Constant)
                     ]
         for api in module_api:
+            if api in public_api_map:
+                raise RuntimeError(
+                    f"Duplicate api key: bsb.{module}.{api} and bsb.{public_api_map[api]}.{api}"
+                )
             public_api_map[api] = module
 
     return public_api_map

README.md

@@ -1,7 +1,7 @@
 [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
 [![Documentation Status](https://readthedocs.org/projects/bsb/badge/?version=latest)](https://bsb.readthedocs.io/en/latest/?badge=latest)
-[![Build Status](https://travis-ci.com/dbbs-lab/bsb.svg?branch=master)](https://travis-ci.com/dbbs-lab/bsb)
-[![codecov](https://codecov.io/gh/dbbs-lab/bsb/branch/master/graph/badge.svg)](https://codecov.io/gh/dbbs-lab/bsb)
+[![Build Status](https://travis-ci.com/dbbs-lab/bsb-core.svg?branch=main)](https://travis-ci.com/dbbs-lab/bsb-core)
+[![codecov](https://codecov.io/gh/dbbs-lab/bsb-core/branch/main/graph/badge.svg)](https://codecov.io/gh/dbbs-lab/bsb-core)
 
 <h3>:closed_book: Read the documentation on https://bsb.readthedocs.io/en/latest</h3>
 

bsb/__init__.py

@@ -142,7 +142,7 @@ def __dir__():
 BaseCommand: typing.Type["bsb.cli.commands.BaseCommand"]
 BidirectionalContact: typing.Type["bsb.postprocessing.BidirectionalContact"]
 BootError: typing.Type["bsb.exceptions.BootError"]
-BoxTree: typing.Type["bsb.voxels.BoxTree"]
+BoxTree: typing.Type["bsb.trees.BoxTree"]
 BoxTreeInterface: typing.Type["bsb.trees.BoxTreeInterface"]
 Branch: typing.Type["bsb.morphologies.Branch"]
 BranchLocTargetting: typing.Type["bsb.simulation.targetting.BranchLocTargetting"]

bsb/config/_attrs.py

@@ -3,6 +3,7 @@
 """
 
 import builtins
+from functools import wraps
 
 import errr
 
@@ -476,7 +477,7 @@ def __set__(self, instance, value):
                 self.attr_name,
             ) from e
         self.flag_dirty(instance)
-        # The value was cast to its intented type and the new value can be set.
+        # The value was cast to its intended type and the new value can be set.
         self.fset(instance, value)
         root = _strict_root(instance)
         if _is_booted(root):
@@ -687,7 +688,15 @@ def fill(self, value, _parent, _key=None):
 
     def _set_type(self, type, key=None):
         self.child_type = super()._set_type(type, key=False)
-        return self.fill
+
+        @wraps(self.fill)
+        def wrapper(*args, **kwargs):
+            return self.fill(*args, **kwargs)
+
+        # Expose children __inv__ function if it exists
+        if hasattr(self.child_type, "__inv__"):
+            setattr(wrapper, "__inv__", self.child_type.__inv__)
+        return wrapper
 
     def tree(self, instance):
         val = _getattr(instance, self.attr_name)
@@ -838,7 +847,15 @@ def fill(self, value, _parent, _key=None):
 
     def _set_type(self, type, key=None):
         self.child_type = super()._set_type(type, key=False)
-        return self.fill
+
+        @wraps(self.fill)
+        def wrapper(*args, **kwargs):
+            return self.fill(*args, **kwargs)
+
+        # Expose children __inv__ function if it exists
+        if hasattr(self.child_type, "__inv__"):
+            setattr(wrapper, "__inv__", self.child_type.__inv__)
+        return wrapper
 
     def tree(self, instance):
         val = _getattr(instance, self.attr_name).items()

bsb/config/types.py

@@ -780,14 +780,16 @@ class PackageRequirement(TypeHandler):
     def __call__(self, value):
         from packaging.requirements import Requirement
 
-        return Requirement(value)
+        requirement = Requirement(value)
+        requirement._cfg_inv = value
+        return requirement
 
     @property
     def __name__(self):
         return "package requirement"
 
     def __inv__(self, value):
-        return str(value)
+        return getattr(value, "_cfg_inv", builtins.str(value))
 
     def __hint__(self):
         return "numpy==1.24.0"

bsb/morphologies/__init__.py

@@ -31,12 +31,6 @@
 from ..voxels import VoxelSet
 
 
-def parse_morphology_file(file, **kwargs):
-    from .parsers import parse_morphology_file
-
-    return parse_morphology_file(file, **kwargs)
-
-
 class MorphologySet:
     """
     Associates a set of :class:`StoredMorphologies
@@ -1730,5 +1724,4 @@ def _morpho_to_swc(morpho):
     "RotationSet",
     "SubTree",
     "branch_iter",
-    "parse_morphology_file",
 ]

bsb/storage/_files.py

@@ -400,24 +400,44 @@ def get_stored_file(self):
 
 @config.node
 class CodeDependencyNode(FileDependencyNode):
+    """
+    Allow the loading of external code during network loading.
+    """
+
     module: str = config.attr(type=str, required=types.shortform())
+    """Should be either the path to a python file or a import like string"""
     attr: str = config.attr(type=str)
+    """Attribute to extract from the loaded script"""
 
     @config.property
     def file(self):
+        import os
+
         if getattr(self, "scaffold", None) is not None:
             file_store = self.scaffold.files
         else:
             file_store = None
-        return FileDependency(
-            self.module.replace(".", _os.sep) + ".py", file_store=file_store
-        )
+        if os.path.isfile(self.module):
+            # Convert potential relative path to absolute path
+            module_file = os.path.abspath(os.path.join(os.getcwd(), self.module))
+        else:
+            # Module like string converted to a path string relative to current folder
+            module_file = "./" + self.module.replace(".", _os.sep) + ".py"
+        return FileDependency(module_file, file_store=file_store)
 
     def __init__(self, module=None, **kwargs):
         super().__init__(**kwargs)
         if module is not None:
             self.module = module
 
+    def __inv__(self):
+        if not isinstance(self, CodeDependencyNode):
+            return self
+        res = {"module": getattr(self, "module")}
+        if self.attr is not None:
+            res["attr"] = self.attr
+        return res
+
     def load_object(self):
         import importlib.util
         import sys
@@ -429,7 +449,7 @@ def load_object(self):
                 module = importlib.util.module_from_spec(spec)
                 sys.modules[self.module] = module
                 spec.loader.exec_module(module)
-                return module if self.attr is None else module[self.attr]
+                return module if self.attr is None else getattr(module, self.attr)
         finally:
             tmp = list(reversed(sys.path))
             tmp.remove(_os.getcwd())

bsb/voxels.py

@@ -637,4 +637,4 @@ def _squash_zero(arr):
     return np.where(np.isclose(arr, 0), np.finfo(float).max, arr)
 
 
-__all__ = ["BoxTree", "VoxelData", "VoxelSet"]
+__all__ = ["VoxelData", "VoxelSet"]

tests/data/code_dependency.py

@@ -0,0 +1 @@
+from bsb_test.configs.double_neuron import tree

tests/test_configuration.py

@@ -1,5 +1,6 @@
 import inspect
 import json
+import os.path
 import sys
 import unittest
 
@@ -10,12 +11,14 @@
     get_test_config,
     list_test_configs,
 )
+from bsb_test.configs import get_test_config_module
 
 import bsb
 from bsb import (
     CastError,
     CfgReferenceError,
     ClassMapMissingError,
+    CodeDependencyNode,
     ConfigurationError,
     ConfigurationWarning,
     DynamicClassInheritanceError,
@@ -28,6 +31,7 @@
     UnfitClassCastError,
     UnresolvedClassCastError,
     config,
+    from_storage,
 )
 from bsb._package_spec import get_missing_requirement_reason
 from bsb.config import Configuration, _attrs, compose_nodes, types
@@ -1253,6 +1257,37 @@ class TestClass:
         with self.assertRaises(RequirementError):
             TestClass(a="1", b="6", c="3")
 
+    def test_code_dependency_node(self):
+        @config.node
+        class Test:
+            c = config.attr(type=CodeDependencyNode)
+
+        module = get_test_config_module("double_neuron")
+        script = str(module.__file__)
+        # Test with a module like string
+        import_str = os.path.relpath(
+            os.path.join(os.path.dirname(__file__), "data/code_dependency")
+        ).replace(os.sep, ".")
+        b = Test(
+            c=import_str,
+            _parent=TestRoot(),
+        )
+        self.assertEqual(b.c.load_object().tree, module.tree)
+        # test with a file
+        b = Test(
+            c={"module": script},
+            _parent=TestRoot(),
+        )
+        # Test variable tree inside the file.
+        self.assertEqual(b.c.load_object().tree, module.tree)
+        self.assertEqual(b.__tree__(), {"c": {"module": script}})
+        # Test with relative path
+        b = Test(
+            c={"module": os.path.relpath(script), "attr": "tree"},
+            _parent=TestRoot(),
+        )
+        self.assertEqual(b.c.load_object(), module.tree)
+
 
 @config.dynamic(
     type=types.in_classmap(),
@@ -1649,7 +1684,7 @@ def test_composite_node(self):
         assert type(self.tested.attrC == config.ConfigurationAttribute)
 
 
-class TestPackageRequirements(unittest.TestCase):
+class TestPackageRequirements(RandomStorageFixture, unittest.TestCase, engine_name="fs"):
     def test_basic_version(self):
         self.assertIsNone(get_missing_requirement_reason("bsb-core==" + bsb.__version__))
 
@@ -1669,3 +1704,15 @@ def test_uninstalled_package(self):
         self.assertIsNotNone(get_missing_requirement_reason("bsb-core-soup==4.0"))
         with self.assertWarns(PackageRequirementWarning):
             Configuration.default(packages=["bsb-core-soup==4.0"])
+
+    def test_installed_package(self):
+        self.assertIsNone(get_missing_requirement_reason(f"bsb-core~={bsb.__version__}"))
+        # Should produce no warnings
+        cfg = Configuration.default(packages=[f"bsb-core~={bsb.__version__}"])
+        # Checking that the config with package requirements can be saved in storage
+        self.network = Scaffold(cfg, self.storage)
+        # Checking if the config
+        network2 = from_storage(self.storage.root)
+        self.assertEqual(
+            self.network.configuration.packages, network2.configuration.packages
+        )

tests/test_placement.py

@@ -335,8 +335,8 @@ def test_particle_vd(self):
         self.assertTrue(np.allclose([78, 15, 7, 26], counts, atol=1), "densities incorr")
         network.compile(clear=True)
         ps = network.get_placement_set("test_cell")
-        self.assertGreater(len(ps), 126)  # rounded down values -1
-        self.assertLess(len(ps), 132)  # rouded up values + 1
+        self.assertGreater(len(ps), 125)  # rounded down values -1
+        self.assertLess(len(ps), 132)  # rounded up values + 1
 
     def _config_packing_fact(self):
         return Configuration.default(