Simulation (#669)

* changed plugin names

* linting

* `simulation/__init__.py` is no longer a barrel file

* minimized adapter interface while we rework

* reworked simulation backends in prep of move at eof rework

* Pass simulation object around, time for adapters!

* removed nodes.py

* added compile submodule with some tricky circulars

* removed references to config.nodes module

* updated references to stuff that's now in _compile

* added content of `config.nodes` to other files

* moved distributions to new file

* Distributions can be more easily constructed from constant

* Fixed distribution type handler errors

* Removed `from_hdf5`. Added generic `from_storage`

* removed `missing_ok` test, as `missing_ok` doesn't appear anywhere.

* Added parameter class

* Allow `from_file` to open paths

* Fixed `__inv__` of distributions

* Add arguments to parser before the options

* Factored out `open_storage`

* linted, added `reconfigure` and `simulate` commands and use `from_file`

* simulation CLI command with configurable sim source

* Fixed `from_content` error message

* Import devices module to load classmap

* Moved `relay` to CellModel. closes #638

* linting

* Fixed extra sims

* Added `by_label` targetting stub

* removed device_protocol reference for now

* fixed up neuron config

* moved `validate_prepare` to connection model. depends on strategy now

* Fixed empty branch errors

* Single sim: composition through Cosimulation

* Started on rework of NEURON adapter

* store scaffold reference. transfer simulation vars.

* Added `get_chunk_stats` and `read_only` mode. closes bsb-hdf5#5

* Quick Neo-results stub rewrite

* Added `immutable` decorator

* Linting

* wip CSIterator

* updated requirements

* ConnectivityIterator can produce scoped and global ids

* added `chunklist` (sorted list of chunks)

* Fixed main

* ignore json

* If dynamic attr isn't required, fallback to class name after default

* lint

* Parameter spaceholders, models/strats should provide meaningful params

* tweaks

* made `class_` type handler invert back to class name

* fixed result init

* gave simulations a name key

* fixed scaffold access in load balance

* write sim results to uuid

* `class_` inversion looks up type if value isn't a class

* return result

* pass simulation data along where it's needed

* fixed recorders, results

* post prepare hooks

* messy adapter WIP: lets us run a sim with cells for now

* cell model instantiation through Arborize strategy

* fixed deps and imports

* Fixed ConnectivityIterator.__copy__() method (#651)

Co-authored-by: Robin De Schepper <robin.deschepper93@gmail.com>

* added conn_type_ref

* fixed conn type ref

* cleaned up imports

* bumped version

* wip connections

* wip connections

* cast `from_id` input to int

* sort cell types by name

* numpy compatible len check

* `pre` and `post` renamed to `pre_type` and `post_type`

* sortable cell models

* added sim util functions

* Public CSIterator now iterates `pre`, `post` blocks

* chunk context no longer vararg

* added connections

* f-string

* rewrote targetting

* removed dead adapter code

* adapted neuron devices to new dynamic class structure

* wip voltage clamp

* added `up` traversal for refs, fixed SimCellModel ref

* fixed sim result recording

* fixed targetting signatures

* added neuron specific result recording

* refactored signature

* removed relays from sim, replaced more `from_hdf5` refs

* Fixed docbuild

* typed chunklist

* removed occurence of `relay`

* fixed load all tests

* cleaned up imports

* bumped

* re-added exception imports

* cache transition hit

* If `boot` hook errors, show which node it is

* fixed distribution inv

* added a bit of type hints to the `Scaffold`class

* added BootError

* removed prints

* added comment

* removed old device interface

* removed old device mixins

* fixed `_map_transmitters` with 0 transmitters

* removed old sim config

* also try `1.1.1.1` IP for connection helper

* changed imports

* fixed connectivity tests

* fixed imports

* updated unittest tests

* bumped to a45

* bumped to a46

* bumped requirements

* removed deprecated test

* fixed deprecated overflow cast

* Refactored expectedFailures

* fixed doc build

Co-authored-by: alessiomarta <81438612+alessiomarta@users.noreply.github.com>

.github/workflows/docs.yml

@@ -21,7 +21,6 @@ jobs:
       run: |
         python -m pip install pip
         pip install -r docs/requirements.txt
-        pip install sphinx==4.4.0
     - name: Install self
       run: |
         pip install -e .

.gitignore

@@ -4,6 +4,7 @@ __pycache__/
 *.hdf5
 *.swc
 *.h5
+*.json
 build/
 dist/
 bsb-*/

.readthedocs.yml

@@ -16,4 +16,6 @@ formats: all
 python:
   version: 3.8
   install:
+    - method: pip
+      path: .
     - requirements: docs/requirements.txt

bsb/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "4.0.0a43"
+__version__ = "4.0.0a46"
 
 import functools
 

bsb/__main__.py

@@ -1,3 +1,3 @@
 from . import cli
 
-cli.scaffold_cli()
+cli.handle_cli()

bsb/_encoding.py

@@ -75,7 +75,8 @@ def transition(point):
                 # Check if this new combination of labels already is assigned an id.
                 for k, v in self.labels.items():
                     if trans_labels == v:
-                        # Transition labels already exist, return it
+                        # Transition labels already exist, store and return it
+                        _transitions[point] = k
                         return k
                 else:
                     # Transition labels are a new combination, store them under a new id.

bsb/_util.py

@@ -25,7 +25,7 @@ def obj_str_insert(__str__):
     @functools.wraps(__str__)
     def wrapper(self):
         obj_str = object.__repr__(self)
-        return obj_str.replace("at 0x", __str__(self) + " at 0x")
+        return obj_str.replace("at 0x", f"{__str__(self)} at 0x")
 
     return wrapper
 
@@ -45,7 +45,7 @@ def suppress_stdout():
 
 
 def get_qualified_class_name(x):
-    return x.__class__.__module__ + "." + str(x.__class__.__name__)
+    return f"{x.__class__.__module__}.{str(x.__class__.__name__)}"
 
 
 def listify_input(value):
@@ -59,15 +59,15 @@ def listify_input(value):
         return [str]
     try:
         return list(value)
-    except:
+    except Exception:
         return [value]
 
 
-def sanitize_ndarray(input, shape, dtype=None):
+def sanitize_ndarray(arr_input, shape, dtype=None):
     kwargs = {"copy": False}
     if dtype is not None:
         kwargs["dtype"] = dtype
-    arr = _np.array(input, **kwargs)
+    arr = _np.array(arr_input, **kwargs)
     arr.shape = shape
     return arr
 
@@ -188,11 +188,25 @@ def resolve_order(cls, objects):
                     if not c.is_after_satisfied(sorting_objects)
                 )
                 raise _OrderError(
-                    f"Couldn't resolve order, probably a circular dependency including: {circulars}"
+                    f"Couldn't resolve order, probably a circular dependency including: "
+                    f"{circulars}"
                 )
         # Return the sorted array.
         return sorting_objects
 
 
+def immutable():
+    def immutable_decorator(f):
+        @functools.wraps(f)
+        def immutable_action(self, *args, **kwargs):
+            new_instance = self.__copy__()
+            f(new_instance, *args, **kwargs)
+            return new_instance
+
+        return immutable_action
+
+    return immutable_decorator
+
+
 def unique(iter_: typing.Iterable[typing.Any]):
     return [*set(iter_)]

bsb/cell_types.py

@@ -4,12 +4,8 @@
 
 from . import config
 from .config import types
-from .placement import PlacementStrategy
 from .placement.indicator import PlacementIndications
-from ._util import SortableByAfter
-from .exceptions import *
 from ._util import obj_str_insert
-import abc
 
 
 @config.node
@@ -53,11 +49,6 @@ class CellType:
     """
     Plotting information about the cell type, such as color and labels.
     """
-    relay = config.attr(type=bool, default=False)
-    """
-    Whether this cell type is a relay type. Relay types, during simulation, instantly
-    transmit incoming spikes to their targets.
-    """
     entity = config.attr(type=bool, default=False)
     """
     Whether this cell type is an entity type. Entity types don't have representations in
@@ -68,6 +59,12 @@ def __boot__(self):
         storage = self.scaffold.storage
         storage._PlacementSet.require(storage._engine, self)
 
+    def __lt__(self, other):
+        try:
+            return self.name < other.name
+        except Exception:
+            return True
+
     @obj_str_insert
     def __repr__(self):
         if hasattr(self, "scaffold"):

bsb/cli/commands/__init__.py

@@ -6,9 +6,8 @@
 :class:`BsbCommand` if you want more freedom in what exactly constitutes a command to the
 BSB.
 """
-import abc
 import argparse
-from ...exceptions import *
+from ...exceptions import CommandError
 from ...reporting import report
 
 
@@ -65,8 +64,8 @@ def add_to_parser(self, parent, context, locals, level):
         locals = locals.copy()
         locals.update(self.get_options())
         parser = parent.add_parser(self.name)
-        self.add_parser_options(parser, context, locals, level)
         self.add_parser_arguments(parser)
+        self.add_parser_options(parser, context, locals, level)
         parser.set_defaults(handler=self.execute_handler)
         self.add_subparsers(parser, context, self._subcommands, locals, level)
         return parser

bsb/cli/commands/_commands.py

@@ -1,13 +1,17 @@
 """
 Contains builtin commands.
 """
+from uuid import uuid4
 
 from . import BaseCommand
 from ...option import BsbOption
-from ...exceptions import *
 from ..._options import ConfigOption
-from . import _projects
+from ...core import from_storage, Scaffold
+from ...storage import open_storage
+from ...config import from_file
+from ...exceptions import NodeNotFoundError
 import itertools
+import errr
 
 
 class XScale(BsbOption, name="x", cli=("x",), env=("BSB_CONFIG_NETWORK_X",)):
@@ -127,11 +131,7 @@ def add_parser_arguments(self, parser):
 
 class BsbCompile(BaseCommand, name="compile"):
     def handler(self, context):
-        from ...config import from_json
-        from ...core import Scaffold
-
-        cfg = from_json(context.config)
-        # Bootstrap the scaffold and clear the storage if not in append mode
+        cfg = from_file(context.config)
         network = Scaffold(cfg)
         network.resize(context.x, context.y, context.z)
         network.compile(
@@ -175,9 +175,39 @@ def add_parser_arguments(self, parser):
         pass
 
 
+class BsbReconfigure(BaseCommand, name="reconfigure"):
+    def handler(self, context):
+        cfg = from_file(context.config)
+        # Bootstrap the scaffold and clear the storage if not in append mode
+        storage = open_storage(context.arguments.network)
+        storage.store_active_config(cfg)
+
+    def get_options(self):
+        return {
+            "config": ConfigOption(positional=True),
+        }
+
+    def add_parser_arguments(self, parser):
+        parser.add_argument("network")
+
+
 class BsbSimulate(BaseCommand, name="simulate"):
     def handler(self, context):
-        pass
+        network = from_storage(context.arguments.network)
+        config_option = context.options["config"]
+        sim_name = context.arguments.simulation
+        if config_option.is_set("cli"):
+            extra_simulations = from_file(context.config).simulations
+            for name, sim in extra_simulations.items():
+                if name not in network.simulations and name == sim_name:
+                    network.simulations[sim_name] = sim
+        try:
+            result = network.run_simulation(sim_name)
+        except NodeNotFoundError as e:
+            append = ", " if len(network.simulations) else ""
+            append += ", ".join(f"'{name}'" for name in extra_simulations.keys())
+            errr.wrap(type(e), e, append=append)
+        result.write(f"{uuid4()}.nio", "ow")
 
     def get_options(self):
         return {
@@ -186,7 +216,8 @@ def get_options(self):
         }
 
     def add_parser_arguments(self, parser):
-        pass
+        parser.add_argument("network")
+        parser.add_argument("simulation")
 
 
 class CacheCommand(BaseCommand, name="cache"):  # pragma: nocover
@@ -204,7 +235,7 @@ def handler(self, context):
             files = [*_cache_path.iterdir()]
             maxlen = 5
             try:
-                maxlen = max(maxlen, max(len(l.name) for l in files))
+                maxlen = max(maxlen, max(len(file.name) for file in files))
             except ValueError:
                 print("Cache is empty")
             else:

bsb/config/__init__.py

@@ -8,7 +8,6 @@
 
 import os
 import sys
-import os
 import glob
 import itertools
 from shutil import copy2 as copy_file
@@ -32,18 +31,19 @@
     catch_all,
     ConfigurationAttribute,
 )
+from .._util import ichain
 from ._make import walk_node_attributes, walk_nodes
 from ._hooks import on, before, after, run_hook, has_hook
 from .. import plugins
-from ..exceptions import *
+from ..exceptions import ConfigTemplateNotFoundError, ParserError, PluginError
 
 
 _path = __path__
 ConfigurationAttribute.__module__ = __name__
 
 
 class ConfigurationModule:
-    from . import types, refs, nodes
+    from . import types, refs
 
     def __init__(self, name):
         self.__name__ = name
@@ -75,7 +75,7 @@ def __init__(self, name):
     _parser_classes = {}
 
     # The __path__ attribute needs to be retained to mark this module as a package with
-    # submodules (config.nodes, config.refs, config.parsers.json, ...)
+    # submodules (config.refs, config.parsers.json, ...)
     __path__ = _path
 
     # Load the Configuration class on demand, not on import, to avoid circular
@@ -86,7 +86,7 @@ def Configuration(self):
             from ._config import Configuration
 
             self._cfg_cls = Configuration
-            self._cfg_cls.__module__ = __name__
+            assert self._cfg_cls.__module__ == __name__
         return self._cfg_cls
 
     @builtins.property
@@ -100,7 +100,7 @@ def get_parser(self, parser_name):
 
         Configuration trees can be cast into Configuration objects.
         """
-        if not parser_name in self._parser_classes:
+        if parser_name not in self._parser_classes:
             raise PluginError("Configuration parser '{}' not found".format(parser_name))
         return self._parser_classes[parser_name]()
 
@@ -132,6 +132,9 @@ def copy_template(self, template, output="network_configuration.json", path=None
         copy_file(files[0], output)
 
     def from_file(self, file):
+        if not hasattr(file, "read"):
+            with open(file, "r") as f:
+                return self.from_file(f)
         path = getattr(file, "name", None)
         if path is not None:
             path = os.path.abspath(path)
@@ -172,7 +175,10 @@ def _parser_method_docs(parser):
 
 def _try_parsers(content, classes, ext=None, path=None):  # pragma: nocover
     if ext is not None:
-        file_has_parser_ext = lambda kv: ext in getattr(kv[1], "data_extensions", ())
+
+        def file_has_parser_ext(kv):
+            return ext in getattr(kv[1], "data_extensions", ())
+
         classes = builtins.dict(sorted(classes.items(), key=file_has_parser_ext))
     exc = {}
     for name, cls in classes.items():
@@ -183,9 +189,13 @@ def _try_parsers(content, classes, ext=None, path=None):  # pragma: nocover
         else:
             return (name, tree, meta)
     msges = [
-        (f"Can't parse with {n}:", traceback.format_exception(e)) for n, e in exc.items()
+        (
+            f"Can't parse contents with '{n}':\n",
+            "".join(traceback.format_exception(type(e), e, e.__traceback__)),
+        )
+        for n, e in exc.items()
     ]
-    raise ParserError("\n".join(msges))
+    raise ParserError("\n".join(ichain(msges)))
 
 
 def _from_parsed(self, parser_name, tree, meta, file=None):

bsb/config/_attrs.py

@@ -1,6 +1,7 @@
 """
     An attrs-inspired class annotation system, but my A stands for amateuristic.
 """
+import errr
 
 from ._hooks import run_hook
 from ._make import (
@@ -15,12 +16,13 @@
     walk_nodes,
     _resolve_references,
 )
-from .types import _wrap_reserved
+from ._compile import _wrap_reserved
 from ..exceptions import (
     RequirementError,
     NoReferenceAttributeSignal,
     CastError,
     CfgReferenceError,
+    BootError,
 )
 import builtins
 
@@ -360,7 +362,10 @@ def _root_is_booted(obj):
 def _boot_nodes(top_node, scaffold):
     for node in walk_nodes(top_node):
         node.scaffold = scaffold
-        run_hook(node, "boot")
+        try:
+            run_hook(node, "boot")
+        except Exception as e:
+            errr.wrap(BootError, e, prepend=f"Failed to boot {node}:")
 
 
 def _unset_nodes(top_node):