Feature/recursive parsing (#847)

* Add recursion for document resolution in ReferenceParser

* Merge ReferenceConfigParser and ConfigurationParser.
There is no benefit to have them both.
Forward path to parse config in get_linked_config

* Fix tests adding mock get_configuration_parser_classes function

* Add mock parser to test recursive parsing

* Separate reference and import features from default ConfigurationParser class.
Now implemented as the mixin class ParsesReferences that decorates the parse function.

* Add API test pre-commit

* Implement review feedback

* Use unittest mock patch instead of function overriding.

.pre-commit-config.yaml

@@ -7,4 +7,10 @@ repos:
     rev: 5.12.0
     hooks:
       - id: isort
-        name: isort (python)
+        name: isort (python)
+  - repo: local
+    hooks:
+      - id: api-test
+        name: api-test
+        entry: python3 .github/devops/generate_public_api.py
+        language: system

bsb/__init__.py

@@ -294,6 +294,7 @@ def __dir__():
 ParameterError: typing.Type["bsb.exceptions.ParameterError"]
 ParameterValue: typing.Type["bsb.simulation.parameter.ParameterValue"]
 ParserError: typing.Type["bsb.exceptions.ParserError"]
+ParsesReferences: typing.Type["bsb.config.parsers.ParsesReferences"]
 Partition: typing.Type["bsb.topology.partition.Partition"]
 PlacementError: typing.Type["bsb.exceptions.PlacementError"]
 PlacementIndications: typing.Type["bsb.cell_types.PlacementIndications"]
@@ -314,7 +315,6 @@ def __dir__():
 ReadOnlyOptionError: typing.Type["bsb.exceptions.ReadOnlyOptionError"]
 RedoError: typing.Type["bsb.exceptions.RedoError"]
 Reference: typing.Type["bsb.config.refs.Reference"]
-ReferenceParser: typing.Type["bsb.config.parsers.ReferenceParser"]
 Region: typing.Type["bsb.topology.region.Region"]
 RegionGroup: typing.Type["bsb.topology.region.RegionGroup"]
 ReificationError: typing.Type["bsb.exceptions.ReificationError"]

bsb/config/parsers.py

@@ -31,58 +31,32 @@ def generate(self, tree, pretty=False):  # pragma: nocover
         pass
 
 
-class ReferenceParser(ConfigurationParser):
+class ParsesReferences:
     """
-    Parser plugin class to parse configuration files with references and imports.
+    Mixin to decorate parse function of ConfigurationParser.
+    Allows for imports and references inside configuration files.
     """
 
-    def parse_content(self, content):
-        if isinstance(content, str):
-            content = parsed_dict(self.from_str(content))
-        elif isinstance(content, dict):
-            content = parsed_dict(content)
-        return content
-
-    @abc.abstractmethod
-    def from_str(self, content):  # pragma: nocover
-        """
-        Parse dictionary from string content.
-
-        :param str content: content to parse
-        :return: parsed dictionary
-        :rtype: dict
-        """
-        pass
-
-    @abc.abstractmethod
-    def load_content(self, stream):  # pragma: nocover
-        """
-        Read content from file object and return parsed dictionary.
-
-        :param stream: python file object
-        :return: parsed dictionary
-        :rtype: dict
-        """
-        pass
+    def __init_subclass__(cls, **kwargs):
+        super().__init_subclass__(**kwargs)
+        parse = cls.parse
 
-    def parse(self, content, path=None):
-        # Parses the content. If path is set it's used as the root for the multi-document
-        # features. During parsing the references (refs & imps) are stored. After parsing
-        # the other documents are parsed by the standard file loader (so no recursion yet)
-        # After loading all required documents the references are resolved and all values
-        # copied over to their final destination.
-        content = self.parse_content(content)
-        self.root = content
-        self.path = path or os.getcwd()
-        self.is_doc = path and not os.path.isdir(path)
-        self.references = []
-        self.documents = {}
-        self._traverse(content, content.items())
-        self.resolved_documents = {}
-        self._resolve_documents()
-        self._resolve_references()
-        meta = {"path": path}
-        return content, meta
+        def parse_with_references(self, content, path=None):
+            """Traverse the parsed tree and resolve any `$ref` and `$import`"""
+            content, meta = parse(self, content, path)
+            content = parsed_dict(content)
+            self.root = content
+            self.path = path or os.getcwd()
+            self.is_doc = path and not os.path.isdir(path)
+            self.references = []
+            self.documents = {}
+            self._traverse(content, content.items())
+            self.resolved_documents = {}
+            self._resolve_documents()
+            self._resolve_references()
+            return content, meta
+
+        cls.parse = parse_with_references
 
     def _traverse(self, node, iter):
         # Iterates over all values in `iter` and checks for import keys, recursion or refs
@@ -111,14 +85,15 @@ def _is_reference(self, key):
     def _is_import(self, key):
         return key == "$import"
 
-    @staticmethod
-    def _get_ref_document(ref, base=None):
+    def _get_ref_document(self, ref, base=None):
         if "#" not in ref or ref.split("#")[0] == "":
             return None
         doc = ref.split("#")[0]
         if not os.path.isabs(doc):
             if not base:
-                base = os.getcwd()
+                # reference should be relative to the current configuration file
+                # to avoid recurrence issues.
+                base = os.path.dirname(self.path)
             elif not os.path.isdir(base):
                 base = os.path.dirname(base)
                 if not os.path.exists(base):
@@ -166,9 +141,13 @@ def _resolve_documents(self):
             if file is None:
                 content = self.root
             else:
-                # We could open another ReferenceParser to easily recurse.
+                from . import _try_parsers
+
+                parser_classes = get_configuration_parser_classes()
+                ext = file.split(".")[-1]
                 with open(file, "r") as f:
-                    content = self.load_content(f)
+                    content = f.read()
+                    _, content, _ = _try_parsers(content, parser_classes, ext, path=file)
             try:
                 self.resolved_documents[file] = self._resolve_document(content, refs)
             except FileReferenceError as jre:
@@ -233,7 +212,7 @@ def get_configuration_parser(parser, **kwargs):
 
 __all__ = [
     "ConfigurationParser",
-    "ReferenceParser",
+    "ParsesReferences",
     "get_configuration_parser",
     "get_configuration_parser_classes",
 ]

bsb/core.py

@@ -87,7 +87,7 @@ def _get_linked_config(storage=None):
         path = cfg._meta.get("path", None)
     if path and os.path.exists(path):
         with open(path, "r") as f:
-            cfg = bsb.config.parse_configuration_file(f)
+            cfg = bsb.config.parse_configuration_file(f, path=path)
             return cfg
     else:
         return None

tests/data/configs/far/targetme.bla

@@ -0,0 +1,25 @@
+<
+  "this": <
+    "key": <
+      "was": "in another folder",
+      "can": <
+        "i": <
+          "be": "imported"
+        >
+      >,
+      "with": <
+        "this": "string"
+      >,
+      "in": <
+        "my": "dict",
+        "or": <
+          "will": "it",
+          "give": "an",
+          "error": <
+            "on": "import"
+          >
+        >
+      >
+    >
+  >
+>

tests/test_parsers.py

@@ -1,24 +1,44 @@
 import ast
 import pathlib
 import unittest
+from unittest.mock import patch
 
-from bsb.config.parsers import ReferenceParser, get_configuration_parser
+from bsb.config.parsers import (
+    ConfigurationParser,
+    ParsesReferences,
+    get_configuration_parser,
+)
 from bsb.exceptions import ConfigurationWarning, FileReferenceError, PluginError
 
 
 def get_content(file: str):
     return (pathlib.Path(__file__).parent / "data/configs" / file).read_text()
 
 
-class RefParserMock(ReferenceParser):
+class RefParserMock(ParsesReferences, ConfigurationParser):
     data_description = "txt"
     data_extensions = ("txt",)
 
-    def from_str(self, content):
-        return ast.literal_eval(content)
+    def parse(self, content, path=None):
+        if isinstance(content, str):
+            content = ast.literal_eval(content)
+        return content, {"meta": path}
+
+    def generate(self, tree, pretty=False):
+        # Should not be called.
+        pass
 
-    def load_content(self, stream):
-        return ast.literal_eval(stream.read())
+
+class RefParserMock2(ParsesReferences, ConfigurationParser):
+    data_description = "bla"
+    data_extensions = ("bla",)
+
+    def parse(self, content, path=None):
+        if isinstance(content, str):
+            content = content.replace("<", "{")
+            content = content.replace(">", "}")
+            content = ast.literal_eval(content)
+        return content, {"meta": path}
 
     def generate(self, tree, pretty=False):
         # Should not be called.
@@ -83,12 +103,15 @@ def test_indoc_reference(self):
         with self.assertRaises(FileReferenceError, msg="Should raise 'ref not a dict'"):
             tree, meta = self.parser.parse(content)
 
-    def test_far_references(self):
+    @patch("bsb.config.parsers.get_configuration_parser_classes")
+    def test_far_references(self, get_content_mock):
+        # Override get_configuration_parser to manually register RefParserMock
+        get_content_mock.return_value = {"txt": RefParserMock, "bla": RefParserMock2}
         content = {
             "refs": {
                 "whats the": {"$ref": "basics.txt#/nest me hard"},
                 "and": {"$ref": "indoc_reference.txt#/refs/whats the"},
-                "far": {"$ref": "far/targetme.txt#/this/key"},
+                "far": {"$ref": "far/targetme.bla#/this/key"},
             },
             "target": {"for": "another"},
         }
@@ -103,7 +126,10 @@ def test_far_references(self):
         self.assertIn("oh yea", tree["refs"]["whats the"])
         self.assertEqual("just like that", tree["refs"]["whats the"]["oh yea"])
 
-    def test_double_ref(self):
+    @patch("bsb.config.parsers.get_configuration_parser_classes")
+    def test_double_ref(self, get_content_mock):
+        # Override get_configuration_parser to manually register RefParserMock
+        get_content_mock.return_value = {"txt": RefParserMock, "bla": RefParserMock2}
         tree, meta = self.parser.parse(
             get_content("doubleref.txt"),
             path=str(
@@ -116,7 +142,10 @@ def test_double_ref(self):
         self.assertIn("for", tree["refs"]["whats the"])
         self.assertIn("another", tree["refs"]["whats the"]["for"])
 
-    def test_ref_str(self):
+    @patch("bsb.config.parsers.get_configuration_parser_classes")
+    def test_ref_str(self, get_content_mock):
+        # Override get_configuration_parser to manually register RefParserMock
+        get_content_mock.return_value = {"txt": RefParserMock, "bla": RefParserMock2}
         tree, meta = self.parser.parse(
             get_content("doubleref.txt"),
             path=str(
@@ -130,7 +159,11 @@ def test_ref_str(self):
             wstr.endswith("/bsb-core/tests/data/configs/indoc_reference.txt#/target'>")
         )
 
-    def test_wrong_ref(self):
+    @patch("bsb.config.parsers.get_configuration_parser_classes")
+    def test_wrong_ref(self, get_content_mock):
+        # Override get_configuration_parser to manually register RefParserMock
+        get_content_mock.return_value = {"txt": RefParserMock, "bla": RefParserMock2}
+
         content = {"refs": {"whats the": {"$ref": "basics.txt#/oooooooooooooo"}}}
         with self.assertRaises(FileReferenceError, msg="ref should not exist"):
             self.parser.parse(