From 7d8bb684b9f6e82747201275e87c7948d9f2de91 Mon Sep 17 00:00:00 2001 From: Imran Ahamed Date: Fri, 31 Jul 2026 00:26:57 -0500 Subject: [PATCH 1/2] analyze: load the bundled detector metrics file DetectorMetrics._load_metrics never found the bundled metrics summary, so get_detector_se_sp returned the (1.0, 1.0) fallback for every detector and ci_calculator computed intervals as if each detector were perfect. The failure was only logged at debug level. garak.data.path is a LocalDataPath that searches the data dir then the package dir on each join. Wrapping it in Path() pinned the lookup to the data dir, where nothing ships, and the directory was spelled detectors_eval while the data ships as detectors-eval. Resolve through LocalDataPath and use the shipped directory name. A genuinely missing resource raises GarakException rather than surfacing as FileNotFoundError, so catch it to keep the graceful fallback. Closes #2003 Signed-off-by: Imran Ahamed --- garak/analyze/detector_metrics.py | 18 +++++++--- tests/analyze/test_detector_metrics.py | 49 ++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 4 deletions(-) diff --git a/garak/analyze/detector_metrics.py b/garak/analyze/detector_metrics.py index 5858ce15b..e6294ed88 100644 --- a/garak/analyze/detector_metrics.py +++ b/garak/analyze/detector_metrics.py @@ -4,11 +4,11 @@ import json from json import JSONDecodeError import logging -from pathlib import Path from typing import Optional, Tuple from garak import _config from garak.data import path as data_path +from garak.exception import GarakException # Module-level cache for singleton instance @@ -19,9 +19,19 @@ class DetectorMetrics: """Helper for managing detector performance metrics (sensitivity/specificity)""" def _load_metrics(self) -> bool: - metrics_file = ( - Path(data_path) / "detectors_eval" / "detector_metrics_summary.json" - ) + # resolve through LocalDataPath so the bundled copy under the package is + # found, and a user copy under the data dir can take precedence; wrapping + # this in Path() would pin the lookup to the data dir alone + try: + metrics_file = ( + data_path / "detectors-eval" / "detector_metrics_summary.json" + ) + except GarakException as e: + logging.debug( + "Detector metrics file not available: %s. Using default metrics (Se=1.0, Sp=1.0).", + e, + ) + return False try: with open(metrics_file, "r", encoding="utf-8") as f: diff --git a/tests/analyze/test_detector_metrics.py b/tests/analyze/test_detector_metrics.py index 019f04a9d..cc3560426 100644 --- a/tests/analyze/test_detector_metrics.py +++ b/tests/analyze/test_detector_metrics.py @@ -209,3 +209,52 @@ def test_get_detector_metrics_returns_singleton(): # Clean up cache garak.analyze.detector_metrics._detector_metrics_cache = None + + +def test_bundled_metrics_file_loads(): + """The bundled metrics summary must load from the path it ships at. + + Regression test for #2003: the loader looked for a `detectors_eval` + directory while the data ships as `detectors-eval`, so the file was never + found. The failure surfaced only as a debug log, leaving every detector on + the (1.0, 1.0) fallback. + """ + garak.analyze.detector_metrics._detector_metrics_cache = None + dm = garak.analyze.detector_metrics.DetectorMetrics() + + assert dm.metrics_loaded, "bundled detector metrics summary should load" + assert dm._data.get("results"), "loaded metrics should carry a results block" + + +def test_bundled_metrics_are_served_not_defaults(): + """Detectors in the bundled file report their measured Se/Sp. + + Guards the failure mode of #2003, where a silently absent file made every + lookup indistinguishable from a perfect detector. + """ + garak.analyze.detector_metrics._detector_metrics_cache = None + dm = garak.analyze.detector_metrics.DetectorMetrics() + results = dm._data.get("results", {}) + assert results, "bundled metrics should describe at least one detector" + + off_default = 0 + for detector_name, detector_data in results.items(): + metrics = detector_data.get("metrics", {}) + expected_se = metrics.get("hit_sensitivity") + expected_sp = metrics.get("hit_specificity") + if expected_se is None or expected_sp is None: + continue + + se, sp = dm.get_detector_se_sp(detector_name) + assert (se, sp) == ( + expected_se, + expected_sp, + ), f"{detector_name} should report its measured Se/Sp" + + if (se, sp) != (1.0, 1.0): + off_default += 1 + + assert off_default, ( + "at least one bundled detector should differ from the (1.0, 1.0) fallback, " + "otherwise this test cannot distinguish loaded metrics from defaults" + ) From 69a745531df726fa1cd3c206dc06ef8cdb485e46 Mon Sep 17 00:00:00 2001 From: Imran Ahamed Date: Fri, 31 Jul 2026 11:18:39 -0500 Subject: [PATCH 2/2] analyze: rename detectors-eval data dir to detectors_eval Per review, align the data directory with the project convention of underscores in directory names; detectors-eval was the only hyphenated directory under garak/data. The loader keeps the name it already used and _plugins.py and the detector metrics doc move to the new name. Refs #2003 Signed-off-by: Imran Ahamed --- docs/source/detector_metrics.rst | 2 +- garak/_plugins.py | 2 +- garak/analyze/detector_metrics.py | 2 +- .../detector_metrics_summary.json | 0 tests/analyze/test_detector_metrics.py | 9 +++++---- 5 files changed, 8 insertions(+), 7 deletions(-) rename garak/data/{detectors-eval => detectors_eval}/detector_metrics_summary.json (100%) diff --git a/docs/source/detector_metrics.rst b/docs/source/detector_metrics.rst index 8bae428f2..385026907 100644 --- a/docs/source/detector_metrics.rst +++ b/docs/source/detector_metrics.rst @@ -89,7 +89,7 @@ Accessing Detector Metrics Detector evaluation metrics are stored in a JSON file located at:: - data/detectors-eval/detector_metrics_summary.json + data/detectors_eval/detector_metrics_summary.json This file is updated when detector evaluations are performed against labeled benchmark datasets. diff --git a/garak/_plugins.py b/garak/_plugins.py index f2400ba1d..430aa54f4 100644 --- a/garak/_plugins.py +++ b/garak/_plugins.py @@ -46,7 +46,7 @@ class PluginCache: _detector_metrics_filename = ( _config.transient.package_dir / "data" - / "detectors-eval" + / "detectors_eval" / "detector_metrics_summary.json" ) _plugin_cache_dict = None diff --git a/garak/analyze/detector_metrics.py b/garak/analyze/detector_metrics.py index e6294ed88..2ccc95e3a 100644 --- a/garak/analyze/detector_metrics.py +++ b/garak/analyze/detector_metrics.py @@ -24,7 +24,7 @@ def _load_metrics(self) -> bool: # this in Path() would pin the lookup to the data dir alone try: metrics_file = ( - data_path / "detectors-eval" / "detector_metrics_summary.json" + data_path / "detectors_eval" / "detector_metrics_summary.json" ) except GarakException as e: logging.debug( diff --git a/garak/data/detectors-eval/detector_metrics_summary.json b/garak/data/detectors_eval/detector_metrics_summary.json similarity index 100% rename from garak/data/detectors-eval/detector_metrics_summary.json rename to garak/data/detectors_eval/detector_metrics_summary.json diff --git a/tests/analyze/test_detector_metrics.py b/tests/analyze/test_detector_metrics.py index cc3560426..1d683cf0e 100644 --- a/tests/analyze/test_detector_metrics.py +++ b/tests/analyze/test_detector_metrics.py @@ -214,10 +214,11 @@ def test_get_detector_metrics_returns_singleton(): def test_bundled_metrics_file_loads(): """The bundled metrics summary must load from the path it ships at. - Regression test for #2003: the loader looked for a `detectors_eval` - directory while the data ships as `detectors-eval`, so the file was never - found. The failure surfaced only as a debug log, leaving every detector on - the (1.0, 1.0) fallback. + Regression test for #2003: the loader wrapped the data path in `Path()`, + which drops `LocalDataPath` resolution and pins the lookup to the data dir + rather than falling back to the bundled copy, and the directory name did + not match. The failure surfaced only as a debug log, leaving every detector + on the (1.0, 1.0) fallback. """ garak.analyze.detector_metrics._detector_metrics_cache = None dm = garak.analyze.detector_metrics.DetectorMetrics()