Added functionalities regarding the newest addition to BBP sonata spe…

…c. (#142)

* Added functionalities regarding the newest addition to BBP sonata spec.

* alternate morphology directories
* population config overwriting "components"
* population type available in NodePopulation/EdgePopulation
* updated circuit_validator to be closer to BBP spec
* Updated CHANGELOG

Co-authored-by: aleksei sanin <aleksei.sanin@epfl.ch>
Co-authored-by: Gianluca Ficarelli <26835404+GianlucaFicarelli@users.noreply.github.com>

.gitignore

@@ -1,5 +1,6 @@
 doc/build
 doc/source/submodules
+coverage.xml
 .coverage
 .eggs
 .tox

CHANGELOG.rst

@@ -1,6 +1,20 @@
 Changelog
 =========
 
+Version v0.13.0
+---------------
+
+New Features
+~~~~~~~~~~~~~
+- Node/edge populations are now supported in config
+- Population type available in NodePopulation/EdgePopulation
+- Population config (if given) overwrites the "components" config for that population
+- Alternate morphology directories (.h5, .asc) are now supported
+
+Improvements
+~~~~~~~~~~~~~~
+- Update circuit validation for the current BBP sonata spec
+
 Version v0.12.1
 ---------------
 

bluepysnap/bbp.py

@@ -19,6 +19,9 @@
 
 from bluepysnap.sonata_constants import DYNAMICS_PREFIX, Edge, Node
 
+NODE_TYPES = {'biophysical', 'virtual', 'astrocyte', 'single_compartment', 'point_neuron'}
+EDGE_TYPES = {'chemical', 'electrical', 'synapse_astrocyte', 'endfoot'}
+
 
 class Cell(Node):
     """Cell property names."""

bluepysnap/circuit_validation.py

@@ -9,6 +9,8 @@
 
 from bluepysnap import BluepySnapError
 from bluepysnap.config import Config
+from bluepysnap.morph import EXTENSIONS_MAPPING
+from bluepysnap.bbp import NODE_TYPES, EDGE_TYPES
 
 MAX_MISSING_FILES_DISPLAY = 10
 
@@ -143,6 +145,8 @@ def _check_required_datasets(config):
         types_file = nodes_dict.get('node_types_file')
         if types_file is not None and not Path(types_file).is_file():
             errors.append(fatal('Invalid "node_types_file": {}'.format(types_file)))
+        if not nodes_dict.get('populations'):
+            errors.append(BbpError(Error.FATAL, 'No "populations" defined in config "nodes"'))
 
     for edges_dict in edges:
         edges_file = edges_dict.get('edges_file')
@@ -151,6 +155,8 @@ def _check_required_datasets(config):
         types_file = edges_dict.get('edge_types_file')
         if types_file is not None and not Path(types_file).is_file():
             errors.append(fatal('Invalid "edge_types_file": {}'.format(types_file)))
+        if not edges_dict.get('populations'):
+            errors.append(BbpError(Error.FATAL, 'No "populations" defined in config "edges"'))
 
     return errors
 
@@ -275,18 +281,17 @@ def _check_multi_groups(group_id_h5, group_index_h5, population):
     return []
 
 
-def _check_bio_nodes_group(group_df, group, config):
+def _check_bio_nodes_group(group_df, group, population):
     """Checks biophysical nodes group for errors.
 
     Args:
         group_df (pd.DataFrame): nodes group as a dataframe
         group (h5py.Group): nodes group in nodes .h5 file
-        config (dict): resolved bluepysnap config
+        population (dict): a merged dictionary (current population and 'components' in config)
 
     Returns:
         list: List of errors, empty if no errors
     """
-
     def _check_rotations():
         """Checks for proper rotation fields."""
         angle_fields = {'rotation_angle_xaxis', 'rotation_angle_yaxis', 'rotation_angle_zaxis'}
@@ -308,48 +313,81 @@ def _check_rotations():
     if missing_fields:
         errors.append(fatal('Group {} of {} misses biophysical fields: {}'.
                             format(group_name, group.file.filename, missing_fields)))
-    if 'components' not in config:
-        errors.append(fatal('No "components" in config'))
-        return errors
-    components = config['components']
-    errors += _check_components_dir('morphologies_dir', components)
-    errors += _check_components_dir('biophysical_neuron_models_dir', components)
+
+    errors += _check_components_dir('morphologies_dir', population)
+    errors += _check_components_dir('biophysical_neuron_models_dir', population)
+
+    morph_dirs = set()
+    if 'morphologies_dir' in population:
+        morph_dirs = {(population['morphologies_dir'], 'swc')}
+    if 'alternate_morphologies' in population:
+        for morph_type, morph_path in population['alternate_morphologies'].items():
+            errors += _check_components_dir(morph_type, population['alternate_morphologies'])
+            for extension, _type in EXTENSIONS_MAPPING.items():
+                if _type == morph_type:
+                    morph_dirs |= {(morph_path, extension)}
     if errors:
         return errors
     _check_rotations()
-    errors += _check_files(
-        'morphology: {}[{}]'.format(group_name, group.file.filename),
-        (Path(components['morphologies_dir'], m + '.swc') for m in group_df['morphology']),
-        Error.WARNING)
-    bio_path = Path(components['biophysical_neuron_models_dir'])
+    for morph_path, extension in morph_dirs:
+        errors += _check_files(
+            'morphology: {}[{}]'.format(group_name, group.file.filename),
+            (Path(morph_path, m + '.' + extension) for m in group_df['morphology']),
+            Error.WARNING)
+    bio_path = Path(population['biophysical_neuron_models_dir'])
     errors += _check_files(
         'model_template: {}[{}]'.format(group_name, group.file.filename),
         (bio_path / _get_model_template_file(m) for m in group_df['model_template']),
         Error.WARNING)
     return errors
 
 
-def _check_nodes_group(group_df, group, config):
+def _check_nodes_group(group_df, group, config, population):
     """Validates nodes group in nodes population.
 
     Args:
         group_df (pd.DataFrame): nodes group in nodes .h5 file
         group (h5py.Group): nodes group in nodes .h5 file
         config (dict): resolved bluepysnap config
+        population (dict): definition of current population in config
 
     Returns:
         list: List of errors, empty if no errors
     """
+    errors = []
+    if 'type' in population and population['type'] not in NODE_TYPES:
+        errors.append(BbpError(Error.WARNING, 'Invalid node type: {}'.format(population['type'])))
     if 'model_type' not in group_df.columns:
-        return [fatal('Group {} of {} misses "model_type" field'
-                      .format(_get_group_name(group, parents=1), group.file.filename))]
+        return errors + [fatal('Group {} of {} misses "model_type" field'
+                               .format(_get_group_name(group, parents=1), group.file.filename))]
     if group_df['model_type'][0] == 'virtual':
-        return []
+        return errors
     if 'model_template' not in group_df.columns:
-        return [fatal('Group {} of {} misses "model_template" field'
-                      .format(_get_group_name(group, parents=1), group.file.filename))]
+        return errors + [fatal('Group {} of {} misses "model_template" field'
+                               .format(_get_group_name(group, parents=1), group.file.filename))]
     elif group_df['model_type'][0] == 'biophysical':
-        return _check_bio_nodes_group(group_df, group, config)
+        if 'components' not in config:
+            return errors + [fatal('No "components" in config')]
+        population = {**config['components'], **population}
+        return errors + _check_bio_nodes_group(group_df, group, population)
+    return errors
+
+
+def _check_populations_config(populations_config, populations_h5, file_name):
+    """Validates the keys (population names) in populations dictionaries.
+
+    Args:
+        populations_config (dict): edge/node populations in config nodes/edges
+        populations_h5 (h5py.Group): nodes/edges group in the h5 file
+        file_name (str): the name of the h5 file
+
+    Returns:
+        list: List of errors, empty if no errors
+    """
+    not_found = set(populations_config) - set(populations_h5)
+    if not_found:
+        return [fatal('populations not found in {}:\n{}'.format(
+            file_name, ''.join(f'\t{p}\n' for p in not_found)))]
     return []
 
 
@@ -371,6 +409,10 @@ def _check_nodes_population(nodes_dict, config):
         nodes = _get_h5_data(h5f, 'nodes')
         if not nodes or len(nodes) == 0:
             return [fatal('No "nodes" in {}.'.format(nodes_file))]
+        populations_config = nodes_dict.get('populations', dict())
+        errors += _check_populations_config(populations_config, nodes, nodes_file)
+        if len(errors) > 0:
+            return errors
         for population_name in nodes:
             population = nodes[population_name]
             groups = _get_population_groups(population)
@@ -381,13 +423,14 @@ def _check_nodes_population(nodes_dict, config):
                 return [fatal('Population {} of {} misses datasets {}'.
                               format(population_name, nodes_file, missing_datasets))]
             if len(groups) > 1:
-                m_errors = _check_multi_groups(population['node_group_id'],
-                                               population['node_group_index'], population)
-                if len(m_errors) > 0:
-                    return m_errors
+                errors += _check_multi_groups(population['node_group_id'],
+                                              population['node_group_index'], population)
+                if len(errors) > 0:
+                    return errors
             for group in groups:
                 group_df = _nodes_group_to_dataframe(group, node_types_file, population)
-                errors += _check_nodes_group(group_df, group, config)
+                errors += _check_nodes_group(group_df, group, config,
+                                             populations_config.get(population_name, dict()))
     return errors
 
 
@@ -572,6 +615,16 @@ def _check_edges_population(edges_dict, nodes):
             errors.append(fatal('No "edges" in {}.'.format(edges_file)))
             return errors
 
+        populations = edges_dict.get('populations', dict())
+        errors += _check_populations_config(populations, edges, edges_file)
+        if len(errors) > 0:
+            return errors
+
+        for populations_config in populations.values():
+            if 'type' in populations_config and populations_config['type'] not in EDGE_TYPES:
+                errors.append(BbpError(Error.WARNING,
+                                       'Invalid edge type: {}'.format(populations_config['type'])))
+
         for population_name in edges:
             population_path = '/edges/' + population_name
             population = h5f[population_path]

bluepysnap/config.py

@@ -23,6 +23,20 @@
 from bluepysnap import utils
 from bluepysnap.exceptions import BluepySnapError
 
+# List of keys which are expected to have paths
+EXPECTED_PATH_KEYS = {'morphologies_dir',
+                      'biophysical_neuron_models_dir',
+                      'vasculature_file',
+                      'vasculature_mesh',
+                      'end_feet_area',
+                      'neurolucida-asc',
+                      'h5v1',
+                      'edges_file',
+                      'nodes_file',
+                      'edges_type_file',
+                      'nodes_type_file',
+                      'node_sets_file'}
+
 
 class Config:
     """SONATA network config parser.
@@ -81,7 +95,7 @@ def _resolve_manifest(manifest, configdir):
 
         return result
 
-    def _resolve_string(self, value):
+    def _resolve_string(self, value, key):
         # not a startswith to detect the badly placed anchors
         if '$' in value:
             vs = [
@@ -94,21 +108,21 @@ def _resolve_string(self, value):
                                       "Please verify your '$' usage.".format(value))
             return str(Path(*vs))
         # only way to know if value is a relative path or a normal string
-        elif value.startswith('.'):
+        elif value.startswith('.') or key in EXPECTED_PATH_KEYS:
             if self.manifest['${configdir}'] is not None:
                 return str(Path(self.manifest['${configdir}'], value).resolve())
             raise BluepySnapError("Dictionary config with relative paths is not allowed.")
         else:
             # we cannot know if a string is a path or not if it does not contain anchor or .
             return value
 
-    def _resolve(self, value):
+    def _resolve(self, value, key=None):
         if isinstance(value, Mapping):
             return {
-                k: self._resolve(v) for k, v in value.items()
+                k: self._resolve(v, k) for k, v in value.items()
             }
         elif isinstance(value, str):
-            return self._resolve_string(value)
+            return self._resolve_string(value, key)
         elif isinstance(value, Iterable):
             return [self._resolve(v) for v in value]
         else:

bluepysnap/edges.py

@@ -18,6 +18,7 @@
 """Edge population access."""
 import inspect
 from collections.abc import Mapping
+from copy import deepcopy
 
 import libsonata
 import numpy as np
@@ -29,7 +30,7 @@
 from bluepysnap.network import NetworkObject
 from bluepysnap.exceptions import BluepySnapError
 from bluepysnap.circuit_ids import CircuitEdgeId, CircuitEdgeIds, CircuitNodeId, CircuitNodeIds
-from bluepysnap.sonata_constants import DYNAMICS_PREFIX, Edge, ConstContainer
+from bluepysnap.sonata_constants import DEFAULT_EDGE_TYPE, DYNAMICS_PREFIX, Edge, ConstContainer
 from bluepysnap import utils
 from bluepysnap.utils import Deprecate, IDS_DTYPE
 from bluepysnap._doctools import AbstractDocSubstitutionMeta
@@ -303,7 +304,8 @@ def __init__(self, config, circuit):
             EdgeStorage: A EdgeStorage object.
         """
         self._h5_filepath = config['edges_file']
-        self._csv_filepath = config['edge_types_file']
+        self._csv_filepath = config.get('edge_types_file')
+        self._populations_config = config.get('populations', {})
         self._circuit = circuit
         self._populations = {}
 
@@ -335,7 +337,11 @@ def circuit(self):
     def population(self, population_name):
         """Access the different populations from the storage."""
         if population_name not in self._populations:
-            self._populations[population_name] = EdgePopulation(self, population_name)
+            population_config = self._populations_config.get(population_name, {})
+
+            self._populations[population_name] = EdgePopulation(
+                self, population_name, population_config)
+
         return self._populations[population_name]
 
 
@@ -356,16 +362,18 @@ def _estimate_range_size(func, node_ids, n=3):
 class EdgePopulation:
     """Edge population access."""
 
-    def __init__(self, edge_storage, population_name):
+    def __init__(self, edge_storage, population_name, population_config=None):
         """Initializes a EdgePopulation object from a EdgeStorage and a population name.
 
         Args:
             edge_storage (EdgeStorage): the edge storage containing the edge population
             population_name (str): the name of the edge population
+            population_config (dict): the config for the population
 
         Returns:
             EdgePopulation: An EdgePopulation object.
         """
+        self._config = population_config or {}
         self._edge_storage = edge_storage
         self.name = population_name
 
@@ -390,6 +398,18 @@ def _nodes(self, population_name):
         result = self._edge_storage.circuit.nodes[population_name]
         return result
 
+    @cached_property
+    def config(self):
+        """Population config dictionary combined with the components dictionary."""
+        components = deepcopy(self._edge_storage.circuit.config.get('components', {}))
+        components.update(self._config)
+        return components
+
+    @property
+    def type(self):
+        """Population type."""
+        return self.config.get('type', DEFAULT_EDGE_TYPE)
+
     @cached_property
     def source(self):
         """Source NodePopulation."""