Added a TTY listener, workflow, and concurrent scheduling (#817)

* add dashing TTYListener, let listeners be context managers

* add context, description, and str properties

* add dashing dep

* improve context passing for FunctionJob

* pass morphology function job context

* remove placeholder text

* BSB --> yellow

* wip show summary of tasks per submitting component

* renamed PoolStatus members

* JobPool is a mandatory context manager

* use dashing fork

* tally ongoing jobs

* fix bug with redo + skip_placement/skip_connectivity

* hopefully improved error

* threaded and error safe scheduling (but not concurrent execution)

* fix level

* delete old progress loop

* fix redo+only wiping everything

* remove test tty setup

* tty! 🎉

* reenable postprocessing

* schedule on thread with future, wait for it (serial)

* display jobs while they are being scheduled

* added concurrent scheduling, workflows, and sanitized output on workers

* fix context manager duck typing

* fix tests

* fix test of fallback function submitter

* fix content

* don't use TTY when terminal is 0x0 (happens under `mpirun`)

* adapt tests to new pool timeline

* comment on the occurence of partially saved morphologies

* update bsb-hdf5 test dep to fix morphology KeyError

* fix error check

* speed up root import (no more runtime ast checking required)

* remove report file support (will reintroduce better logging and capture)

* move ast generation into the devops file itself

* fix root

* fix profiling import hook

* fix broken test?

* cba

* test schedule

.github/devops/generate_public_api.py

@@ -1,21 +1,57 @@
+import ast
 import difflib
+import functools
 import sys
 from pathlib import Path
 
-from bsb import _get_public_api_map
+
+def _assign_targets(assign: ast.Assign, id_: str):
+    return any(
+        target.id == id_ for target in assign.targets if isinstance(target, ast.Name)
+    )
+
+
+@functools.cache
+def get_public_api_map():
+    root = Path(__file__).parent.parent.parent / "bsb"
+
+    public_api_map = {}
+    for file in root.rglob("*.py"):
+        module_parts = file.relative_to(root).parts
+        module = ".".join(
+            module_parts[:-1]
+            + ((module_parts[-1][:-3],) if module_parts[-1] != "__init__.py" else tuple())
+        )
+        module_api = []
+        for assign in ast.parse(file.read_text()).body:
+            if isinstance(assign, ast.Assign):
+                is_api = _assign_targets(assign, "__api__")
+                is_either = is_api or _assign_targets(assign, "__all__")
+                if ((is_either and not module_api) or is_api) and isinstance(
+                    assign.value, ast.List
+                ):
+                    module_api = [
+                        el.value
+                        for el in assign.value.elts
+                        if isinstance(el, ast.Constant)
+                    ]
+        for api in module_api:
+            public_api_map[api] = module
+
+    return public_api_map
 
 
 def public_annotations():
     annotations = []
-    for api, module in _get_public_api_map().items():
+    for api, module in get_public_api_map().items():
         annotation = f'"bsb.{module}.{api}"'
         if api[0].isupper():
             annotation = f"typing.Type[{annotation}]"
         annotations.append(f"{api}: {annotation}")
 
     lines = [
         "if typing.TYPE_CHECKING:",
-        *sorted(f"  import bsb.{module}" for module in {*_get_public_api_map().values()}),
+        *sorted(f"  import bsb.{module}" for module in {*get_public_api_map().values()}),
         "",
         *sorted(annotations),
         "",

bsb/__init__.py

@@ -9,12 +9,13 @@
 
 __version__ = "4.0.0-b10"
 
-import ast
 import functools
 import importlib
 import sys
 import typing
-from pathlib import Path
+import warnings
+
+import bsb.exceptions as _exc
 
 # Patch functools on 3.8
 try:
@@ -30,76 +31,47 @@ def _register(self, cls, method=None):  # pragma: nocover
 
     functools.singledispatchmethod.register = _register
 
+
+# Always show all scaffold warnings
+for e in _exc.__dict__.values():
+    if isinstance(e, type) and issubclass(e, Warning):
+        warnings.simplefilter("always", e)
+
 try:
     from .options import profiling as _pr
 except Exception:
-    _pr = False
-
-if _pr:
-    from .profiling import activate_session
-
-    session = activate_session()
-    meter = session.meter("root_module")
-    meter.start()
-
-from . import reporting
-
-reporting.setup_reporting()
+    pass
+else:
+    if _pr:
+        from .profiling import activate_session
 
-if _pr:
-    meter.stop()
+        activate_session()
 
 
-def _assign_targets(assign: ast.Assign, id_: str):
-    return any(
-        target.id == id_ for target in assign.targets if isinstance(target, ast.Name)
-    )
-
-
-@functools.cache
-def _get_public_api_map():
-    root = Path(__file__).parent
-
-    public_api_map = {}
-    for file in root.rglob("*.py"):
-        module_parts = file.relative_to(root).parts
-        module = ".".join(
-            module_parts[:-1]
-            + ((module_parts[-1][:-3],) if module_parts[-1] != "__init__.py" else tuple())
-        )
-        module_api = []
-        for assign in ast.parse(file.read_text()).body:
-            if isinstance(assign, ast.Assign):
-                is_api = _assign_targets(assign, "__api__")
-                is_either = is_api or _assign_targets(assign, "__all__")
-                if ((is_either and not module_api) or is_api) and isinstance(
-                    assign.value, ast.List
-                ):
-                    module_api = [
-                        el.value
-                        for el in assign.value.elts
-                        if isinstance(el, ast.Constant)
-                    ]
-        for api in module_api:
-            public_api_map[api] = module
-
-    return public_api_map
+def _get_annotation_submodule(name: str):
+    annotation = __annotations__.get(name, None)
+    if annotation:
+        type_ = typing.get_args(annotation)
+        if type_:
+            # typing.Type["bsb.submodule.name"]
+            annotation = type_[0].__forward_arg__
+        return annotation[4 : -len(name) - 1]
 
 
 @functools.cache
 def __getattr__(name):
     if name == "config":
         return object.__getattribute__(sys.modules[__name__], name)
-    api = _get_public_api_map()
-    module = api.get(name, None)
+    module = _get_annotation_submodule(name)
     if module is None:
         return object.__getattribute__(sys.modules[__name__], name)
     else:
         return getattr(importlib.import_module("." + module, package="bsb"), name)
 
 
+@functools.cache
 def __dir__():
-    return [*_get_public_api_map().keys()]
+    return [*__annotations__.keys()]
 
 
 # Do not modify: autogenerated public API type annotations of the `bsb` module
@@ -164,6 +136,8 @@ def __dir__():
 AdapterError: typing.Type["bsb.exceptions.AdapterError"]
 AdapterProgress: typing.Type["bsb.simulation.adapter.AdapterProgress"]
 AdaptiveNeighbourhood: typing.Type["bsb.placement.particle.AdaptiveNeighbourhood"]
+AfterConnectivityHook: typing.Type["bsb.postprocessing.AfterConnectivityHook"]
+AfterPlacementHook: typing.Type["bsb.postprocessing.AfterPlacementHook"]
 AllToAll: typing.Type["bsb.connectivity.general.AllToAll"]
 AllenApiError: typing.Type["bsb.exceptions.AllenApiError"]
 AllenStructure: typing.Type["bsb.topology.partition.AllenStructure"]
@@ -274,7 +248,9 @@ def __dir__():
 InvertedRoI: typing.Type["bsb.mixins.InvertedRoI"]
 JobCancelledError: typing.Type["bsb.exceptions.JobCancelledError"]
 JobPool: typing.Type["bsb.services.JobPool"]
+JobPoolContextError: typing.Type["bsb.exceptions.JobPoolContextError"]
 JobPoolError: typing.Type["bsb.exceptions.JobPoolError"]
+JobSchedulingError: typing.Type["bsb.exceptions.JobSchedulingError"]
 JsonImportError: typing.Type["bsb.exceptions.JsonImportError"]
 JsonParseError: typing.Type["bsb.exceptions.JsonParseError"]
 JsonReferenceError: typing.Type["bsb.exceptions.JsonReferenceError"]
@@ -287,7 +263,6 @@ def __dir__():
 MPILock: typing.Type["bsb.services.MPILock"]
 Meter: typing.Type["bsb.profiling.Meter"]
 MissingActiveConfigError: typing.Type["bsb.exceptions.MissingActiveConfigError"]
-MissingAxon: typing.Type["bsb.postprocessing.MissingAxon"]
 MissingMorphologyError: typing.Type["bsb.exceptions.MissingMorphologyError"]
 MissingSourceError: typing.Type["bsb.exceptions.MissingSourceError"]
 MorphIOParser: typing.Type["bsb.morphologies.parsers.parser.MorphIOParser"]
@@ -351,7 +326,6 @@ def __dir__():
 PlacementWarning: typing.Type["bsb.exceptions.PlacementWarning"]
 Plotting: typing.Type["bsb.cell_types.Plotting"]
 PluginError: typing.Type["bsb.exceptions.PluginError"]
-PostProcessingHook: typing.Type["bsb.postprocessing.PostProcessingHook"]
 ProfilingSession: typing.Type["bsb.profiling.ProfilingSession"]
 ProgressEvent: typing.Type["bsb.simulation.simulation.ProgressEvent"]
 ProjectOptionDescriptor: typing.Type["bsb.option.ProjectOptionDescriptor"]
@@ -456,13 +430,10 @@ def __dir__():
 get_parser_classes: "bsb.config.parsers.get_parser_classes"
 get_partitions: "bsb.topology.get_partitions"
 get_project_option: "bsb.options.get_project_option"
-get_report_file: "bsb.reporting.get_report_file"
 get_root_regions: "bsb.topology.get_root_regions"
 get_simulation_adapter: "bsb.simulation.get_simulation_adapter"
 handle_cli: "bsb.cli.handle_cli"
 handle_command: "bsb.cli.handle_command"
-in_notebook: "bsb.reporting.in_notebook"
-in_pytest: "bsb.reporting.in_pytest"
 init_engines: "bsb.storage.init_engines"
 is_module_option_set: "bsb.options.is_module_option_set"
 is_partition: "bsb.topology.is_partition"
@@ -478,16 +449,13 @@ def __dir__():
 parse_morphology_file: "bsb.morphologies.parsers.parse_morphology_file"
 parsers: "bsb.config.parsers"
 read: "bsb.options.read"
-read_report_file: "bsb.reporting.read_report_file"
 refs: "bsb.config.refs"
 register_engine: "bsb.storage.register_engine"
 register_option: "bsb.options.register_option"
 register_service: "bsb.services.register_service"
 report: "bsb.reporting.report"
 reset_module_option: "bsb.options.reset_module_option"
 set_module_option: "bsb.options.set_module_option"
-set_report_file: "bsb.reporting.set_report_file"
-setup_reporting: "bsb.reporting.setup_reporting"
 store: "bsb.options.store"
 types: "bsb.config.types"
 unregister_option: "bsb.options.unregister_option"

bsb/config/_config.py

@@ -7,7 +7,7 @@
 from ..cell_types import CellType
 from ..connectivity import ConnectionStrategy
 from ..placement import PlacementStrategy
-from ..postprocessing import PostProcessingHook
+from ..postprocessing import AfterPlacementHook
 from ..simulation.simulation import Simulation
 from ..storage._files import (
     CodeDependencyNode,
@@ -122,8 +122,8 @@ class Configuration:
     """
     Network placement strategies
     """
-    after_placement: cfgdict[str, PostProcessingHook] = config.dict(
-        type=PostProcessingHook,
+    after_placement: cfgdict[str, AfterPlacementHook] = config.dict(
+        type=AfterPlacementHook,
     )
     connectivity: cfgdict[str, ConnectionStrategy] = config.dict(
         type=ConnectionStrategy,
@@ -132,8 +132,8 @@ class Configuration:
     """
     Network connectivity strategies
     """
-    after_connectivity: cfgdict[str, PostProcessingHook] = config.dict(
-        type=PostProcessingHook,
+    after_connectivity: cfgdict[str, AfterPlacementHook] = config.dict(
+        type=AfterPlacementHook,
     )
     simulations: cfgdict[str, Simulation] = config.dict(
         type=Simulation,