[bench] pipeline: fix citation counting, oracle validation, and defensive improvements

.claude/settings.local.json

@@ -0,0 +1,28 @@
+{
+  "permissions": {
+    "allow": [
+      "Bash(python *)",
+      "Bash(where python3 *)",
+      "Bash(where py *)",
+      "Bash(where python *)",
+      "PowerShell(python3 -m unittest tests.test_diff -v)",
+      "Bash(C:\\\\Users\\\\mstar\\\\AppData\\\\Local\\\\Python\\\\bin\\\\python3.exe *)",
+      "Bash(echo \"exit: $?\")",
+      "PowerShell(New-Item *)",
+      "PowerShell(Get-Command *)",
+      "PowerShell(py *)",
+      "PowerShell($hasPkg = Test-Path \"package.json\"; $hasCargo = Test-Path \"Cargo.toml\"; $hasMake = Test-Path \"Makefile\"; $hasTsc = Test-Path \"tsconfig.json\"; \"package.json=$hasPkg Cargo.toml=$hasCargo Makefile=$hasMake tsconfig.json=$hasTsc\")",
+      "PowerShell(git reset *)",
+      "PowerShell(git fetch *)",
+      "PowerShell(git push *)",
+      "PowerShell(gh pr *)",
+      "PowerShell(gh run *)",
+      "Bash(Get-ChildItem -Path \"C:\\\\Users\\\\mstar\\\\Bench\" -Recurse -File)",
+      "Bash(Select-Object -ExpandProperty FullName)",
+      "Bash(Get-ChildItem -Path \"C:\\\\Users\\\\mstar\\\\Bench\" -Recurse -Force)",
+      "Bash(Select-Object FullName)",
+      "Bash(python3 *)",
+      "Bash([ -f \"$dir/__init__.py\" ])"
+    ]
+  }
+}

cli/commands.py

@@ -134,6 +134,15 @@ def cmd_stats() -> int:
                 for cid in citations:
                     if isinstance(cid, str):
                         citation_counts[cid] = citation_counts.get(cid, 0) + 1
+                    elif isinstance(cid, dict):
+                        raw = cid.get("constraint_id")
+                        if isinstance(raw, str):
+                            citation_counts[raw] = citation_counts.get(raw, 0) + 1
+                    else:
+                        print(
+                            f"[bench cli] unexpected citation type: {type(cid).__name__}",
+                            file=sys.stderr,
+                        )
 
     most_cited: tuple[str, int] | None = None
     if citation_counts:
@@ -272,7 +281,10 @@ def _print_entry_line(entry: dict) -> None:
     if verdict == "VETO":
         citations: Any = oracle_dict.get("constraint_citations")
         if isinstance(citations, list) and citations:
-            cite_str: str = ", ".join(str(c) for c in citations if c)
+            cite_str: str = ", ".join(
+                c.get("constraint_id", str(c)) if isinstance(c, dict) else str(c)
+                for c in citations if c
+            )
             if cite_str:
                 print(f"      citations: {cite_str}")
 

ledger/ledger-meta.json

@@ -1,6 +1,6 @@
 {
-  "entry_count": 30,
-  "latest_hash": "1a7290ffaa46157d61fff765c7dcb8811d044fb1609a99c25df096b027e0d66c",
+  "entry_count": 41,
+  "latest_hash": "1b08e964398d77dc86e73946bbe5eaef2beaf0f21c2c149b0d0dac20c1856066",
   "created": "2026-04-22T04:00:32.627154+00:00",
-  "last_updated": "2026-04-22T06:06:36.399036+00:00"
+  "last_updated": "2026-05-15T04:52:17.529652+00:00"
 }

pipeline/oracle.py

@@ -220,6 +220,8 @@ def _validate_oracle_response(response: dict[str, Any]) -> bool:
     citations: Any = response.get("constraint_citations")
     if not isinstance(citations, list):
         return False
+    if verdict == "VETO" and len(citations) == 0:
+        return False
     for citation in citations:
         if not isinstance(citation, dict):
             return False

pipeline/runner.py

@@ -105,7 +105,8 @@ def run_governance_pipeline(
         defender_result = run_defender(
             diff_info, constitution, constitution_hash, challenger_result
         )
-        _accumulate_tokens(accumulated, defender_result.get("_tokens"))
+    _accumulate_tokens(accumulated, defender_result.get("_tokens"))
+    if challenger_result.get("status") != "CLEAR":
         if defender_result.get("status") == "PIPELINE_ERROR":
             return _finalize(
                 {

utils/api.py

@@ -261,10 +261,16 @@ def _openrouter_call(
         *messages,
     ]
 
+    api_key = os.environ.get("OPENROUTER_API_KEY")
+    if not api_key:
+        raise _ProviderError(
+            "openrouter: OPENROUTER_API_KEY environment variable is not set"
+        )
+
     try:
         client = openai.OpenAI(
             base_url=_OPENROUTER_BASE_URL,
-            api_key=os.environ.get("OPENROUTER_API_KEY"),
+            api_key=api_key,
         )
         response = client.chat.completions.create(
             model=routed_model,

utils/viewer.py

@@ -110,6 +110,15 @@ def _compute_stats(entries: list[dict]) -> dict:
                 for cid in citations:
                     if isinstance(cid, str):
                         citation_counts[cid] = citation_counts.get(cid, 0) + 1
+                    elif isinstance(cid, dict):
+                        raw = cid.get("constraint_id")
+                        if isinstance(raw, str):
+                            citation_counts[raw] = citation_counts.get(raw, 0) + 1
+                    else:
+                        print(
+                            f"[bench viewer] unexpected citation type: {type(cid).__name__}",
+                            file=sys.stderr,
+                        )
 
     most_cited: tuple[str, int] | None = None
     if citation_counts: