analyze: load the bundled detector metrics file

docs/source/detector_metrics.rst

@@ -89,7 +89,7 @@ Accessing Detector Metrics
 
 Detector evaluation metrics are stored in a JSON file located at::
 
-    data/detectors-eval/detector_metrics_summary.json
+    data/detectors_eval/detector_metrics_summary.json
 
 This file is updated when detector evaluations are performed against labeled benchmark datasets.
 

garak/_plugins.py

@@ -46,7 +46,7 @@ class PluginCache:
     _detector_metrics_filename = (
         _config.transient.package_dir
         / "data"
-        / "detectors-eval"
+        / "detectors_eval"
         / "detector_metrics_summary.json"
     )
     _plugin_cache_dict = None

garak/analyze/detector_metrics.py

@@ -4,11 +4,11 @@
 import json
 from json import JSONDecodeError
 import logging
-from pathlib import Path
 from typing import Optional, Tuple
 
 from garak import _config
 from garak.data import path as data_path
+from garak.exception import GarakException
 
 
 # Module-level cache for singleton instance
@@ -19,9 +19,19 @@ class DetectorMetrics:
     """Helper for managing detector performance metrics (sensitivity/specificity)"""
 
     def _load_metrics(self) -> bool:
-        metrics_file = (
-            Path(data_path) / "detectors_eval" / "detector_metrics_summary.json"
-        )
+        # resolve through LocalDataPath so the bundled copy under the package is
+        # found, and a user copy under the data dir can take precedence; wrapping
+        # this in Path() would pin the lookup to the data dir alone
+        try:
+            metrics_file = (
+                data_path / "detectors_eval" / "detector_metrics_summary.json"
+            )
+        except GarakException as e:
+            logging.debug(
+                "Detector metrics file not available: %s. Using default metrics (Se=1.0, Sp=1.0).",
+                e,
+            )
+            return False
 
         try:
             with open(metrics_file, "r", encoding="utf-8") as f:

tests/analyze/test_detector_metrics.py

@@ -209,3 +209,53 @@ def test_get_detector_metrics_returns_singleton():
 
     # Clean up cache
     garak.analyze.detector_metrics._detector_metrics_cache = None
+
+
+def test_bundled_metrics_file_loads():
+    """The bundled metrics summary must load from the path it ships at.
+
+    Regression test for #2003: the loader wrapped the data path in `Path()`,
+    which drops `LocalDataPath` resolution and pins the lookup to the data dir
+    rather than falling back to the bundled copy, and the directory name did
+    not match. The failure surfaced only as a debug log, leaving every detector
+    on the (1.0, 1.0) fallback.
+    """
+    garak.analyze.detector_metrics._detector_metrics_cache = None
+    dm = garak.analyze.detector_metrics.DetectorMetrics()
+
+    assert dm.metrics_loaded, "bundled detector metrics summary should load"
+    assert dm._data.get("results"), "loaded metrics should carry a results block"
+
+
+def test_bundled_metrics_are_served_not_defaults():
+    """Detectors in the bundled file report their measured Se/Sp.
+
+    Guards the failure mode of #2003, where a silently absent file made every
+    lookup indistinguishable from a perfect detector.
+    """
+    garak.analyze.detector_metrics._detector_metrics_cache = None
+    dm = garak.analyze.detector_metrics.DetectorMetrics()
+    results = dm._data.get("results", {})
+    assert results, "bundled metrics should describe at least one detector"
+
+    off_default = 0
+    for detector_name, detector_data in results.items():
+        metrics = detector_data.get("metrics", {})
+        expected_se = metrics.get("hit_sensitivity")
+        expected_sp = metrics.get("hit_specificity")
+        if expected_se is None or expected_sp is None:
+            continue
+
+        se, sp = dm.get_detector_se_sp(detector_name)
+        assert (se, sp) == (
+            expected_se,
+            expected_sp,
+        ), f"{detector_name} should report its measured Se/Sp"
+
+        if (se, sp) != (1.0, 1.0):
+            off_default += 1
+
+    assert off_default, (
+        "at least one bundled detector should differ from the (1.0, 1.0) fallback, "
+        "otherwise this test cannot distinguish loaded metrics from defaults"
+    )