Address offline services (in this case NeuroMorpho) (#790)

* remove `language_version` spec

* use `skipIfOffline` for NM tests

* add a test that confirms warning when NM is down

* use test-b2 to fix nest simulator errors during test_compilation

.pre-commit-config.yaml

@@ -3,7 +3,6 @@ repos:
     rev: 23.11.0
     hooks:
       - id: black
-        language_version: python3.11
   - repo: https://github.com/pycqa/isort
     rev: 5.12.0
     hooks:

bsb/storage/_files.py

@@ -22,6 +22,7 @@
 from .._util import obj_str_insert
 from ..config import types
 from ..config._attrs import cfglist
+from ..reporting import warn
 
 if _tp.TYPE_CHECKING:
     from ..morphologies import Morphology
@@ -270,6 +271,9 @@ def provide_stream(self, file):
     def create_session(self):
         return _rq.Session()
 
+    def get_base_url(self):
+        raise NotImplementedError("Base UrlScheme has no fixed base URL.")
+
 
 class NeuroMorphoScheme(UrlScheme):
     _nm_url = "https://neuromorpho.org/"
@@ -286,17 +290,24 @@ def get_nm_meta(self, file: FileDependency):
         # urlparse gives lowercase, so slice out the original cased NM name
         idx = file.uri.lower().find(name)
         name = file.uri[idx : (idx + len(name))]
-        try:
-            with self.create_session() as session:
+        with self.create_session() as session:
+            try:
                 res = session.get(self._nm_url + self._meta + name)
-        except Exception as e:
-            return {"archive": "none", "neuron_name": "none"}
-        if res.status_code == 404:
-            raise IOError(f"'{name}' is not a valid NeuroMorpho name.")
-        elif res.status_code != 200:
-            raise IOError("NeuroMorpho API error: " + res.text)
+            except Exception as e:
+                return {"archive": "none", "neuron_name": "none"}
+            if res.status_code == 404:
+                res = session.get(self._nm_url)
+                if res.status_code != 200 or "Service Interruption Notice" in res.text:
+                    warn(f"NeuroMorpho.org is down, can't retrieve morphology '{name}'.")
+                    return {"archive": "none", "neuron_name": "none"}
+                raise IOError(f"'{name}' is not a valid NeuroMorpho name.")
+            elif res.status_code != 200:
+                raise IOError("NeuroMorpho API error: " + res.text)
         return res.json()
 
+    def get_base_url(self):
+        return self._nm_url
+
     def get_meta(self, file: FileDependency):
         meta = super().get_meta(file)
         meta["neuromorpho_data"] = self.get_nm_meta(file)

pyproject.toml

@@ -67,7 +67,7 @@ parallel = [
 test = [
     "coverage~=7.3",
     "bsb-hdf5==1.0.0b1",
-    "bsb-test==0.0.0b0",
+    "bsb-test==0.0.0b2",
     "bsb-json==0.0.0b0"
 ]
 docs = [

tests/test_morphologies.py

@@ -7,7 +7,12 @@
 
 import h5py
 import numpy as np
-from bsb_test import NumpyTestCase, RandomStorageFixture, get_morphology_path
+from bsb_test import (
+    NumpyTestCase,
+    RandomStorageFixture,
+    get_morphology_path,
+    skipIfOffline,
+)
 from scipy.spatial.transform import Rotation
 
 from bsb._encoding import EncodedLabels
@@ -17,7 +22,11 @@
 from bsb.morphologies import Branch, Morphology, MorphologySet, RotationSet
 from bsb.services import MPI
 from bsb.storage import Storage
-from bsb.storage._files import MorphologyDependencyNode, MorphologyOperation
+from bsb.storage._files import (
+    MorphologyDependencyNode,
+    MorphologyOperation,
+    NeuroMorphoScheme,
+)
 from bsb.storage.interfaces import StoredMorphology
 
 
@@ -1060,6 +1069,7 @@ def test_arrays(self):
 class TestPipelines(
     NumpyTestCase, RandomStorageFixture, unittest.TestCase, engine_name="hdf5"
 ):
+    @skipIfOffline(scheme=NeuroMorphoScheme())
     def test_pipeline_functions(self):
         m1 = MorphologyDependencyNode(file="nm://hippo-1264-9")
         m2 = MorphologyDependencyNode(

tests/test_util.py

@@ -1,7 +1,12 @@
 import unittest
 
 import numpy as np
-from bsb_test import FixedPosConfigFixture, NumpyTestCase, RandomStorageFixture
+from bsb_test import (
+    FixedPosConfigFixture,
+    NumpyTestCase,
+    RandomStorageFixture,
+    skipIfOffline,
+)
 from scipy.spatial.transform import Rotation
 
 from bsb._util import rotation_matrix_from_vectors
@@ -66,9 +71,21 @@ def test_rotation_matrix_from_vectors(self):
 
 
 class TestUriSchemes(unittest.TestCase):
+    @skipIfOffline(scheme=NeuroMorphoScheme())
     def test_nm_scheme(self):
         file = FileDependency("nm://AX2_scaled", Scaffold().files)
         self.assertIs(NeuroMorphoScheme, type(file._scheme), "Expected NM scheme")
         meta = file.get_meta()
         self.assertIn("neuromorpho_data", meta)
         self.assertEqual(130892, meta["neuromorpho_data"]["neuron_id"])
+
+    def test_nm_scheme_down(self):
+        url = NeuroMorphoScheme._nm_url
+        # Consistently trigger a 404 response in the NM scheme
+        NeuroMorphoScheme._nm_url = "https://google.com/404"
+        try:
+            file = FileDependency("nm://AX2_scaled", Scaffold().files)
+            with self.assertWarns(UserWarning) as w:
+                file.get_meta()
+        finally:
+            NeuroMorphoScheme._nm_url = url