Better handling of fatal errors (SystemExit) in the test_case decorator

optimade/validator/utils.py

@@ -120,8 +120,10 @@ def get(self, request: str):
             retries += 1
             try:
                 self.response = requests.get(self.last_request)
-            except requests.exceptions.ConnectionError:
-                sys.exit(f"No response from server at {self.last_request}")
+            except requests.exceptions.ConnectionError as exc:
+                sys.exit(
+                    f"{exc.__class__.__name__}: No response from server at {self.last_request}, please check the URL."
+                )
             except requests.exceptions.MissingSchema:
                 sys.exit(
                     f"Unable to make request on {self.last_request}, did you mean http://{self.last_request}?"
@@ -208,11 +210,19 @@ def wrapper(
                 msg = f"{exc.__class__.__name__}: {exc}"
                 raise InternalError(msg)
 
-        except Exception as exc:
+        # Catch SystemExit here so that we can pass it through to the finally block,
+        # but prepare to immediately throw it
+        except (Exception, SystemExit) as exc:
             result = exc
             traceback = tb.format_exc()
 
         finally:
+            # This catches the case of the Client throwing a SystemExit if the server
+            # did not respond, and the case of the validator "fail-fast"'ing and throwing
+            # a SystemExit below
+            if isinstance(result, SystemExit):
+                raise SystemExit(result)
+
             display_request = None
             try:
                 display_request = validator.client.last_request

tests/validator/test_utils.py

@@ -336,3 +336,25 @@ def test_fail_fast_test_case():
     assert validator.internal_failure_messages[-1][1] == [
         "FileNotFoundError: Internal error"
     ]
+
+
+def test_that_system_exit_is_fatal_in_test_case():
+    """Check that test_case handles treates `SystemExit` as fatal."""
+    validator = ImplementationValidator(
+        base_url="http://example.org", verbosity=0, fail_fast=False
+    )
+
+    with pytest.raises(SystemExit, match="Fatal error"):
+        dummy_test_case(
+            validator,
+            ({"test": "dict"}, "message"),
+            request="test_request",
+            raise_exception=SystemExit("Fatal error"),
+            optional=True,
+        )
+
+    assert validator.success_count == 0
+    assert validator.optional_success_count == 0
+    assert validator.failure_count == 0
+    assert validator.optional_failure_count == 0
+    assert validator.internal_failure_count == 0