NSETM-2114 partial circuit config (#194)

clarification for partial circuit config

CHANGELOG.rst

@@ -1,6 +1,10 @@
 Changelog
 =========
 
+Improvements
+~~~~~~~~~~~~
+- Clarification for partial circuit configs
+
 Version v1.0.5
 --------------
 

bluepysnap/circuit.py

@@ -16,15 +16,18 @@
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
 """Access to circuit data."""
+import logging
 
 from cached_property import cached_property
 
-from bluepysnap.config import CircuitConfig
+from bluepysnap.config import CircuitConfig, CircuitConfigStatus
 from bluepysnap.edges import Edges
 from bluepysnap.exceptions import BluepySnapError
 from bluepysnap.node_sets import NodeSets
 from bluepysnap.nodes import Nodes
 
+L = logging.getLogger(__name__)
+
 
 class Circuit:
     """Access to circuit data."""
@@ -41,6 +44,12 @@ def __init__(self, config):
         self._circuit_config_path = config
         self._config = CircuitConfig.from_config(config)
 
+        if self.partial_config:
+            L.info(
+                "Loaded PARTIAL circuit config. Functionality may be limited. "
+                "It is up to the user to be diligent when accessing properties."
+            )
+
     @property
     def to_libsonata(self):
         """Libsonata instance of the circuit."""
@@ -82,6 +91,11 @@ def edges(self):
         """Access to edge population(s). See :py:class:`~bluepysnap.edges.Edges`."""
         return Edges(self)
 
+    @cached_property
+    def partial_config(self):
+        """Check partiality of the config."""
+        return self._config.status == CircuitConfigStatus.partial
+
     def __getstate__(self):
         """Make Circuits pickle-able, without storing state of caches."""
         return self._circuit_config_path

bluepysnap/circuit_validation.py

@@ -66,6 +66,10 @@ def fatal(message):
     return Error(Error.FATAL, message)
 
 
+def _check_partial_circuit_config(config):
+    return config.get("metadata", {}).get("status") == "partial"
+
+
 def _check_components_dir(name, components):
     """Checks existence of directory within Sonata config 'components'.
 
@@ -619,6 +623,14 @@ def validate(config_file, skip_slow, only_errors=False, print_errors=True):
     if "networks" in config:
         errors += validate_networks(config, skip_slow)
 
+    if _check_partial_circuit_config(config):
+        message = (
+            "The Circuit config is partial. Validity cannot be established "
+            "for partial configs as it depends on the intended use. "
+        )
+        L.warning(message)
+        errors.append(Error(Error.WARNING, message))
+
     if only_errors:
         errors = [e for e in errors if e.level == Error.FATAL]
 

bluepysnap/config.py

@@ -23,6 +23,7 @@
 from pathlib import Path
 
 import libsonata
+from libsonata import CircuitConfigStatus
 
 from bluepysnap.exceptions import BluepySnapError
 
@@ -184,6 +185,11 @@ def edge_populations(self):
         """Access edge population configs."""
         return self._populations["edges"]
 
+    @property
+    def status(self) -> CircuitConfigStatus:
+        """Return status of the config."""
+        return self._libsonata.config_status
+
     @staticmethod
     def _resolve_population_configs(config):
         """Resolves population configs for the node and edge populations."""

bluepysnap/network.py

@@ -123,6 +123,9 @@ def _get_ids_from_pop(self, fun_to_apply, returned_ids_cls, sample=None, limit=N
         Returns:
             CircuitNodeIds/CircuitEdgeIds: containing the IDs and the populations.
         """
+        if not self.population_names:
+            raise BluepySnapError("Cannot create CircuitIds for empty population.")
+
         str_type = f"<U{max(len(pop) for pop in self.population_names)}"
         ids = []
         populations = []

setup.py

@@ -46,7 +46,7 @@ def __init__(self, *args, **kwargs):
         "cached_property>=1.0",
         "h5py>=3.0.1,<4.0.0",
         "jsonschema>=4.0.0,<5.0.0",
-        "libsonata>=0.1.17,<1.0.0",
+        "libsonata>=0.1.20,<1.0.0",
         "morphio>=3.0.0,<4.0.0",
         "morph-tool>=2.4.3,<3.0.0",
         "numpy>=1.8",

tests/data/circuit_config.json

@@ -14,27 +14,27 @@
       {
         "nodes_file": "$NETWORK_DIR/nodes.h5",
         "populations": {
-            "default": {
-                "type": "biophysical"
-              },
-            "default2": {
-                "type": "biophysical",
-                "spatial_segment_index_dir": "path/to/node/dir"
-            }
+          "default": {
+            "type": "biophysical"
+          },
+          "default2": {
+            "type": "biophysical",
+            "spatial_segment_index_dir": "path/to/node/dir"
+          }
         }
       }
     ],
     "edges": [
       {
         "edges_file": "$NETWORK_DIR/edges.h5",
         "populations": {
-            "default":{
-                "type": "chemical"
-              },
-            "default2":{
-                "type": "chemical",
-                "spatial_synapse_index_dir": "path/to/edge/dir"
-            }
+          "default": {
+            "type": "chemical"
+          },
+          "default2": {
+            "type": "chemical",
+            "spatial_synapse_index_dir": "path/to/edge/dir"
+          }
         }
       }
     ]

tests/test_circuit_validation.py

@@ -541,3 +541,19 @@ def test_no_duplicate_population_names():
             Error(Error.FATAL, 'Already have population "default" in config for type "nodes"'),
             Error(Error.FATAL, 'Already have population "default2" in config for type "nodes"'),
         }
+
+
+def test_partial_config_warning():
+    with copy_test_data() as (_, config_copy_path):
+        with edit_config(config_copy_path) as config:
+            config["metadata"] = {"status": "partial"}
+        errors = validate(str(config_copy_path))
+        assert errors == {
+            Error(
+                Error.WARNING,
+                (
+                    "The Circuit config is partial. Validity cannot be established "
+                    "for partial configs as it depends on the intended use. "
+                ),
+            ),
+        }

tests/test_config/test_partial_config.py

@@ -0,0 +1,126 @@
+import json
+import logging
+import tempfile
+
+import pytest
+
+import bluepysnap.circuit as test_module
+from bluepysnap.exceptions import BluepySnapError
+from bluepysnap.sonata_constants import Edge, Node
+
+from utils import TEST_DATA_DIR
+
+
+def test_partial_circuit_config_minimal():
+    config = {
+        "metadata": {"status": "partial"},
+        "networks": {
+            "nodes": [
+                {
+                    # missing `nodes_file` rises libsonata exception
+                    "nodes_file": str(TEST_DATA_DIR / "nodes.h5"),
+                    "populations": {
+                        "default": {},
+                    },
+                }
+            ],
+            "edges": [
+                {
+                    # missing `edges_file` rises libsonata exception
+                    "edges_file": str(TEST_DATA_DIR / "edges.h5"),
+                    "populations": {
+                        "default": {},
+                    },
+                }
+            ],
+        },
+    }
+    with tempfile.NamedTemporaryFile(mode="w+") as config_file:
+        config_file.write(json.dumps(config))
+        config_file.flush()
+        circuit = test_module.Circuit(config_file.name)
+
+    assert circuit.get_node_population_config("default")
+    assert circuit.nodes
+    assert circuit.nodes["default"].type == "biophysical"
+    assert circuit.nodes["default"].size == 3
+    assert circuit.nodes["default"].source_in_edges() == {"default"}
+    assert circuit.nodes["default"].target_in_edges() == {"default"}
+    assert circuit.nodes["default"].config is not None
+    assert circuit.nodes["default"].property_names is not None
+    assert circuit.nodes["default"].container_property_names(Node) is not None
+    assert circuit.nodes["default"].container_property_names(Edge) == []
+    assert circuit.nodes["default"].property_values("layer") == {2, 6}
+    assert circuit.nodes["default"].property_dtypes is not None
+    assert list(circuit.nodes["default"].ids()) == [0, 1, 2]
+    assert circuit.nodes["default"].get() is not None
+    assert circuit.nodes["default"].positions() is not None
+    assert circuit.nodes["default"].orientations() is not None
+    assert circuit.nodes["default"].count() == 3
+    assert circuit.nodes["default"].morph is not None
+    assert circuit.nodes["default"].models is not None
+    assert circuit.nodes["default"].h5_filepath.endswith("/snap/tests/data/nodes.h5")
+    assert circuit.nodes["default"]._properties.spatial_segment_index_dir == ""
+
+    assert circuit.nodes.population_names == ["default"]
+    assert list(circuit.nodes.values())
+
+    assert [item.id for item in circuit.nodes.ids()] == [0, 1, 2]
+
+    assert circuit.get_edge_population_config("default")
+    assert circuit.edges
+    assert circuit.edges["default"].type == "chemical"
+    assert circuit.edges["default"].size == 4
+    assert circuit.edges["default"].source is not None
+    assert circuit.edges["default"].target is not None
+    assert circuit.edges["default"].config is not None
+    assert circuit.edges["default"].property_names is not None
+    assert circuit.edges["default"].property_dtypes is not None
+    assert circuit.edges["default"].container_property_names(Node) == []
+    assert circuit.edges["default"].container_property_names(Edge) is not None
+    assert list(circuit.edges["default"].ids()) == [0, 1, 2, 3]
+    assert list(circuit.edges["default"].get([1, 2], None)) == [1, 2]
+    assert circuit.edges["default"].positions([1, 2], "afferent", "center") is not None
+    assert list(circuit.edges["default"].afferent_nodes(None)) == [0, 2]
+    assert list(circuit.edges["default"].efferent_nodes(None)) == [0, 1]
+    assert list(circuit.edges["default"].pathway_edges(0)) == [1, 2]
+    assert list(circuit.edges["default"].afferent_edges(0)) == [0]
+    assert list(circuit.edges["default"].efferent_edges(0)) == [1, 2]
+    assert circuit.edges["default"].iter_connections() is not None
+    assert circuit.edges["default"].h5_filepath.endswith("/snap/tests/data/edges.h5")
+    assert circuit.nodes["default"]._properties.spatial_segment_index_dir == ""
+
+    assert circuit.edges.population_names == ["default"]
+    assert list(circuit.edges.values())
+
+    assert [item.id for item in circuit.edges.ids()] == [0, 1, 2, 3]
+
+
+def test_partial_circuit_config_log(caplog):
+    caplog.set_level(logging.INFO)
+
+    config = {"metadata": {"status": "partial"}}
+
+    with tempfile.NamedTemporaryFile(mode="w+") as config_file:
+        config_file.write(json.dumps(config))
+        config_file.flush()
+        test_module.Circuit(config_file.name)
+
+    assert "Loaded PARTIAL circuit config" in caplog.text
+
+
+def test_partial_circuit_config_empty():
+    config = {"metadata": {"status": "partial"}}
+    with tempfile.NamedTemporaryFile(mode="w+") as config_file:
+        config_file.write(json.dumps(config))
+        config_file.flush()
+        circuit = test_module.Circuit(config_file.name)
+
+    with pytest.raises(BluepySnapError):
+        assert circuit.get_node_population_config("default")
+    with pytest.raises(BluepySnapError):
+        assert circuit.get_edge_population_config("default")
+    with pytest.raises(BluepySnapError):
+        circuit.nodes.ids()
+    with pytest.raises(BluepySnapError):
+        circuit.edges.ids()

tests/test_edges/test_edge_population.py

@@ -655,7 +655,7 @@ def test_pickle(self, tmp_path):
         with open(pickle_path, "rb") as fd:
             edge_population = pickle.load(fd)
 
-        assert pickle_path.stat().st_size <= 250
+        assert pickle_path.stat().st_size < 260
         assert edge_population.name == "default"
 
 

tests/test_edges/test_edges.py

@@ -784,5 +784,5 @@ def test_pickle(self, tmp_path):
         with open(pickle_path, "rb") as fd:
             test_obj = pickle.load(fd)
 
-        assert pickle_path.stat().st_size < 170
+        assert pickle_path.stat().st_size < 180
         assert test_obj.size == 8

tests/test_nodes/test_node_population.py

@@ -633,7 +633,7 @@ def test_pickle(self, tmp_path):
         with open(pickle_path, "rb") as fd:
             test_obj = pickle.load(fd)
 
-        assert pickle_path.stat().st_size <= 200
+        assert pickle_path.stat().st_size < 210
         assert test_obj.size == 3
 
 

tests/test_nodes/test_nodes.py

@@ -426,5 +426,5 @@ def test_pickle(self, tmp_path):
         with open(pickle_path, "rb") as fd:
             test_obj = pickle.load(fd)
 
-        assert pickle_path.stat().st_size < 170
+        assert pickle_path.stat().st_size < 180
         assert test_obj.size == 7

tests/test_simulation.py

@@ -113,5 +113,5 @@ def test_pickle(tmp_path):
     with open(pickle_path, "rb") as fd:
         simulation = pickle.load(fd)
 
-    assert pickle_path.stat().st_size < 190
+    assert pickle_path.stat().st_size < 200
     assert simulation.dt == 0.01