Switch to multi-simulation adapters by default (#804)

* allow multiple simulations to be ran at the same time

* b4 - multi sim

* updated naming and api of AdapterProgress

* cast any non `FileDependency` inputs

* Add `log_exc` to log exception to cache files. Log broken options

* simulator plugin typing

* switch to official black action

* fix black

* bump test deps

.github/workflows/black.yml

@@ -9,7 +9,7 @@ jobs:
 
     steps:
     - uses: actions/checkout@v4
-    - name: Black Check
-      uses: jpetrucciani/black-check@24.1.1
+    - uses: psf/black@stable
       with:
-        path: 'bsb'
+        options: "--check --verbose"
+        version: "24.1.1"

bsb/core.py

@@ -413,7 +413,7 @@ def run_pipelines(self, pipelines=None, DEBUG=True):
             pool.execute()
 
     @meter()
-    def run_simulation(self, simulation_name: str, quit=False):
+    def run_simulation(self, simulation_name: str):
         """
         Run a simulation starting from the default single-instance adapter.
 
@@ -422,7 +422,7 @@ def run_simulation(self, simulation_name: str, quit=False):
         """
         simulation = self.get_simulation(simulation_name)
         adapter = get_simulation_adapter(simulation.simulator)
-        return adapter.simulate(simulation)
+        return adapter.simulate(simulation)[0]
 
     def get_simulation(self, sim_name: str) -> "Simulation":
         """

bsb/morphologies/parsers/parser.py

@@ -45,7 +45,7 @@ class BsbParser(MorphologyParser, classmap_entry="bsb"):
     def parse(self, file: typing.Union["FileDependency", str]):
         from ...storage import FileDependency
 
-        if isinstance(file, str):
+        if not isinstance(file, FileDependency):
             file = FileDependency(file)
         content, encoding = file.get_content(check_store=False)
         return self.parse_content(content.decode(encoding or "utf8"))

bsb/option.py

@@ -10,6 +10,7 @@
 import toml
 
 from .exceptions import OptionError
+from .reporting import warn
 
 
 class OptionDescriptor:
@@ -281,7 +282,8 @@ def get(self, prio=None):
                 return self.env
             return self.get_default()
         except Exception as e:
-            print(e)
+            warn(f"Error retrieving option '{self.name}'.", log_exc=e)
+            return self.get_default()
 
     def is_set(self, slug):
         if descriptor := getattr(type(self), slug, None):

bsb/reporting.py

@@ -91,7 +91,7 @@ def report(*message, level=2, ongoing=False, token=None, nodes=None, all_nodes=F
             print(message, end="\n" if not ongoing else "\r", flush=True)
 
 
-def warn(message, category=None, stacklevel=2):
+def warn(message, category=None, stacklevel=2, log_exc=None):
     """
     Send a warning.
 
@@ -101,7 +101,18 @@ def warn(message, category=None, stacklevel=2):
     """
     from . import options
 
-    if options.verbosity > 0:
+    if log_exc:
+        import traceback
+
+        from .storage._util import cache
+
+        log = f"{message}\n\n{traceback.format_exception(type(log_exc), log_exc, log_exc.__traceback__)}"
+        id = cache.files.store(log)
+        path = cache.files.id_to_file_path(id)
+        message += f" See '{path}' for full error log."
+
+    # Avoid infinite loop looking up verbosity when verbosity option is broken.
+    if "Error retrieving option 'verbosity'" in message or options.verbosity > 0:
         if _report_file:
             with open(_report_file, "a") as f:
                 f.write(_encode(str(category or "warning"), message))

bsb/simulation/__init__.py

@@ -10,7 +10,7 @@ class SimulationBackendPlugin:
     Simulation: _Sim
 
 
-def get_simulation_adapter(name):
+def get_simulation_adapter(name: str):
     from ._backends import get_simulation_adapters
 
     return get_simulation_adapters()[name]

bsb/simulation/_backends.py

@@ -1,10 +1,16 @@
 import functools
+import typing
 
 from .. import plugins
 
+if typing.TYPE_CHECKING:
+    from . import SimulationBackendPlugin
+    from .adapter import SimulatorAdapter
+    from .simulation import Simulation
+
 
 @functools.cache
-def get_backends():
+def get_backends() -> dict[str, "SimulationBackendPlugin"]:
     backends = plugins.discover("simulation_backends")
     for backend in backends.values():
         plugins._decorate_advert(backend.Simulation, backend._bsb_entry_point)
@@ -13,10 +19,10 @@ def get_backends():
 
 
 @functools.cache
-def get_simulation_nodes():
+def get_simulation_nodes() -> dict[str, "Simulation"]:
     return {name: plugin.Simulation for name, plugin in get_backends().items()}
 
 
 @functools.cache
-def get_simulation_adapters():
+def get_simulation_adapters() -> dict[str, "SimulatorAdapter"]:
     return {name: plugin.Adapter() for name, plugin in get_backends().items()}

bsb/simulation/adapter.py

@@ -1,5 +1,11 @@
 import abc
+import itertools
+import types
 import typing
+from contextlib import ExitStack
+from time import time
+
+import numpy as np
 
 from .results import SimulationResult
 
@@ -9,6 +15,36 @@
     from .simulation import Simulation
 
 
+class AdapterProgress:
+    def __init__(self, duration):
+        self._duration = duration
+        self._start = self._last_tick = time()
+        self._ticks = 0
+
+    def tick(self, step):
+        """
+        Report simulation progress.
+        """
+        now = time()
+        tic = now - self._last_tick
+        self._ticks += 1
+        el = now - self._start
+        progress = types.SimpleNamespace(
+            progression=step, duration=self._duration, time=time(), tick=tic, elapsed=el
+        )
+        self._last_tick = now
+        return progress
+
+    def steps(self, step=1):
+        steps = itertools.chain(np.arange(0, self._duration, step), (self._duration,))
+        a, b = itertools.tee(steps)
+        next(b, None)
+        yield from zip(a, b)
+
+    def complete(self):
+        return
+
+
 class SimulationData:
     def __init__(self, simulation: "Simulation", result=None):
         self.chunks = None
@@ -25,18 +61,29 @@ def __init__(self, simulation: "Simulation", result=None):
 
 class SimulatorAdapter(abc.ABC):
     def __init__(self):
+        self._progress_listeners = []
         self.simdata: dict["Simulation", "SimulationData"] = dict()
 
-    def simulate(self, simulation):
+    def simulate(self, *simulations, post_prepare=None, comm=None):
         """
-        Simulate the given simulation.
+        Simulate the given simulations.
         """
-        with simulation.scaffold.storage.read_only():
-            data = self.prepare(simulation)
-            for hook in simulation.post_prepare:
-                hook(self, simulation, data)
-            result = self.run(simulation)
-            return self.collect(simulation, data, result)
+        with ExitStack() as context:
+            for simulation in simulations:
+                context.enter_context(simulation.scaffold.storage.read_only())
+            alldata = []
+            for simulation in simulations:
+                data = self.prepare(simulation)
+                alldata.append(data)
+                for hook in simulation.post_prepare:
+                    hook(self, simulation, data)
+            if post_prepare:
+                post_prepare(self, simulations, alldata)
+            results = self.run(*simulations)
+            return [
+                self.collect(simulation, data, result)
+                for simulation, result in zip(simulations, results)
+            ]
 
     @abc.abstractmethod
     def prepare(self, simulation, comm=None):
@@ -51,15 +98,18 @@ def prepare(self, simulation, comm=None):
         pass
 
     @abc.abstractmethod
-    def run(self, simulation):
+    def run(self, *simulations, comm=None):
         """
         Fire up the prepared adapter.
         """
         pass
 
-    def collect(self, simulation, simdata, simresult):
+    def collect(self, simulation, simdata, simresult, comm=None):
         """
         Collect the output of a simulation that completed
         """
         simresult.flush()
         return simresult
+
+    def add_progress_listener(self, listener):
+        self._progress_listeners.append(listener)

bsb/simulation/simulation.py

@@ -30,6 +30,7 @@ def __init__(self, progression, duration, time):
 @config.pluggable(key="simulator", plugin_name="simulation backend")
 class Simulation:
     scaffold: "Scaffold"
+    simulator: str
     name: str = config.attr(key=True)
     duration: float = config.attr(type=float, required=True)
     cell_models: cfgdict[CellModel] = config.slot(type=CellModel, required=True)
@@ -45,45 +46,6 @@ class Simulation:
     def __plugins__():
         return get_simulation_nodes()
 
-    def start_progress(self, duration):
-        """
-        Start a progress meter.
-        """
-        self._progdur = duration
-        self._progstart = self._last_progtic = time()
-        self._progtics = 0
-
-    def progress(self, step):
-        """
-        Report simulation progress.
-        """
-        now = time()
-        tic = now - self._last_progtic
-        self._progtics += 1
-        el = now - self._progstart
-        # report(
-        #     f"Simulated {step}/{self._progdur}ms.",
-        #     f"{el:.2f}s elapsed.",
-        #     f"Simulated tick in {tic:.2f}.",
-        #     f"Avg tick {el / self._progtics:.4f}s",
-        #     level=3,
-        #     ongoing=False,
-        # )
-        progress = types.SimpleNamespace(
-            progression=step, duration=self._progdur, time=time()
-        )
-        self._last_progtic = now
-        return progress
-
-    def step_progress(self, duration, step=1):
-        steps = itertools.chain(np.arange(0, duration), (duration,))
-        a, b = itertools.tee(steps)
-        next(b, None)
-        yield from zip(a, b)
-
-    def add_progress_listener(self, listener):
-        self._progress_listeners.append(listener)
-
     def get_model_of(
         self, type: typing.Union["CellType", "ConnectionStrategy"]
     ) -> typing.Optional[typing.Union["CellModel", "ConnectionModel"]]:

pyproject.toml

@@ -64,9 +64,9 @@ parallel = [
     "mpilock~=1.1"
 ]
 test = [
-    "bsb-arbor==0.0.0b0",
+    "bsb-arbor==0.0.0b1",
     "bsb-hdf5==1.0.0b1",
-    "bsb-test==0.0.0b7",
+    "bsb-test==0.0.0b8",
     "coverage~=7.3",
 ]
 docs = [