fix(ci): harden hourly GitHub API gate

.github/workflows/hourly-product-development.yml

@@ -37,7 +37,7 @@ jobs:
   develop-product-gap:
     name: Discover and package one bounded product gap
     runs-on: ubuntu-24.04
-    timeout-minutes: 45
+    timeout-minutes: 180
     permissions:
       actions: read
       checks: read
@@ -53,8 +53,9 @@ jobs:
         with:
           egress-policy: block
           disable-telemetry: true
-          allowed-endpoints: |
+          allowed-endpoints: >-
             api.github.com:443
+            cafe.github.com:443
             codeload.github.com:443
             github.com:443
             integrate.api.nvidia.com:443
@@ -69,13 +70,11 @@ jobs:
             files.pythonhosted.org:443
             pypi.org:443
 
-
-      - name: Enforce the credential, queue, and exact-main gate
+      - name: Enforce the deterministic queue and exact-main gate
         id: gate
         shell: bash
         env:
           GH_TOKEN: ${{ github.token }}
-          NIM_UPSTREAM_API_KEY: ${{ secrets.NVIDIA_NIM_API_KEY }}
           CURRENT_RUN_ID: ${{ github.run_id }}
           DRY_RUN: ${{ inputs.dry_run || false }}
         run: |
@@ -85,19 +84,13 @@ jobs:
             echo "base_sha="
           } >>"$GITHUB_OUTPUT"
 
-          if [ -z "${NIM_UPSTREAM_API_KEY:-}" ]; then
-            echo "NVIDIA_NIM_API_KEY is not configured; autonomous development stopped safely." \
-              >>"$GITHUB_STEP_SUMMARY"
-            exit 0
-          fi
-
           open_pr_file="${RUNNER_TEMP}/keyverse-open-pulls.json"
           if ! gh api \
             -H "Accept: application/vnd.github+json" \
             "repos/${GITHUB_REPOSITORY}/pulls?state=open&per_page=1" \
             >"$open_pr_file"; then
-            echo "::warning::Unable to list open pull requests; refusing to create work."
-            exit 0
+            echo "::error::Unable to list open pull requests; GitHub inventory is unavailable."
+            exit 1
           fi
           if ! open_pr_count="$(python3 - "$open_pr_file" <<'PY'
           import json
@@ -110,8 +103,8 @@ jobs:
           print(len(payload))
           PY
           )"; then
-            echo "::warning::Unable to interpret the open pull-request response; refusing to create work."
-            exit 0
+            echo "::error::Unable to interpret the open pull-request response; GitHub inventory is malformed."
+            exit 1
           fi
           if [ "$open_pr_count" -ne 0 ]; then
             echo "An open pull request exists; the protected PR loop owns this hour." \
@@ -125,12 +118,12 @@ jobs:
               "repos/${GITHUB_REPOSITORY}/commits/${DEFAULT_BRANCH}" \
               --jq '.sha'
           )"; then
-            echo "::warning::Unable to resolve the default-branch head; refusing to create work."
-            exit 0
+            echo "::error::Unable to resolve the default-branch head; GitHub inventory is unavailable."
+            exit 1
           fi
           if ! [[ "$base_sha" =~ ^[0-9a-f]{40}$ ]]; then
-            echo "::warning::The default-branch head was malformed; refusing to create work."
-            exit 0
+            echo "::error::The default-branch head was malformed; GitHub inventory is invalid."
+            exit 1
           fi
 
           workflow_runs_file="${RUNNER_TEMP}/keyverse-main-workflow-runs.json"
@@ -140,25 +133,30 @@ jobs:
             --slurp \
             "repos/${GITHUB_REPOSITORY}/actions/runs?branch=${DEFAULT_BRANCH}&head_sha=${base_sha}&per_page=100" \
             >"$workflow_runs_file"; then
-            echo "::warning::Unable to read default-branch workflow evidence; refusing to create work."
-            exit 0
+            echo "::error::Unable to read default-branch workflow evidence; GitHub inventory is unavailable."
+            exit 1
           fi
-          if ! python3 - "$workflow_runs_file" "$CORE_WORKFLOWS" <<'PY'
+          workflow_evidence_status=0
+          python3 - "$workflow_runs_file" "$CORE_WORKFLOWS" <<'PY' || workflow_evidence_status=$?
           import json
           import sys
 
-          with open(sys.argv[1], encoding="utf-8") as stream:
-              pages = json.load(stream)
-          required = set(json.loads(sys.argv[2]))
+          try:
+              with open(sys.argv[1], encoding="utf-8") as stream:
+                  pages = json.load(stream)
+              required = set(json.loads(sys.argv[2]))
+          except (OSError, json.JSONDecodeError, TypeError, ValueError) as exc:
+              print(f"Malformed workflow-run evidence: {exc}", file=sys.stderr)
+              raise SystemExit(2) from exc
           if not isinstance(pages, list) or not required:
-              raise SystemExit("Unsupported workflow-run response shape")
+              raise SystemExit(2)
 
           runs = []
           for page in pages:
               if not isinstance(page, dict) or not isinstance(
                   page.get("workflow_runs"), list
               ):
-                  raise SystemExit("Unsupported workflow-run response shape")
+                  raise SystemExit(2)
               runs.extend(page["workflow_runs"])
 
           latest = {}
@@ -175,26 +173,41 @@ jobs:
 
           missing = sorted(required.difference(latest))
           if missing:
-              raise SystemExit(
+              print(
                   "Missing required default-branch workflow evidence: "
-                  + ", ".join(missing)
+                  + ", ".join(missing),
+                  file=sys.stderr,
               )
+              raise SystemExit(3)
           unhealthy = sorted(
               name
               for name, run in latest.items()
               if run.get("status") != "completed"
               or run.get("conclusion") != "success"
           )
           if unhealthy:
-              raise SystemExit(
+              print(
                   "Default branch has pending or unsuccessful required workflow evidence: "
-                  + ", ".join(unhealthy)
+                  + ", ".join(unhealthy),
+                  file=sys.stderr,
               )
+              raise SystemExit(3)
           PY
-          then
-            echo "::warning::Default-branch core workflow evidence is incomplete or unhealthy; refusing to create work."
-            exit 0
-          fi
+          case "$workflow_evidence_status" in
+            0) ;;
+            2)
+              echo "::error::Unable to interpret default-branch workflow evidence; GitHub inventory is malformed."
+              exit 1
+              ;;
+            3)
+              echo "::warning::Default-branch core workflow evidence is incomplete or unhealthy; refusing to create work."
+              exit 0
+              ;;
+            *)
+              echo "::error::Default-branch workflow evidence parser failed unexpectedly."
+              exit 1
+              ;;
+          esac
 
           check_runs_file="${RUNNER_TEMP}/keyverse-main-check-runs.json"
           if ! gh api \
@@ -203,25 +216,30 @@ jobs:
             --slurp \
             "repos/${GITHUB_REPOSITORY}/commits/${base_sha}/check-runs?per_page=100" \
             >"$check_runs_file"; then
-            echo "::warning::Unable to read default-branch check evidence; refusing to create work."
-            exit 0
+            echo "::error::Unable to read default-branch check evidence; GitHub inventory is unavailable."
+            exit 1
           fi
-          if ! python3 - "$check_runs_file" "$CURRENT_RUN_ID" <<'PY'
+          check_evidence_status=0
+          python3 - "$check_runs_file" "$CURRENT_RUN_ID" <<'PY' || check_evidence_status=$?
           import json
           import sys
 
-          with open(sys.argv[1], encoding="utf-8") as stream:
-              pages = json.load(stream)
+          try:
+              with open(sys.argv[1], encoding="utf-8") as stream:
+                  pages = json.load(stream)
+          except (OSError, json.JSONDecodeError, TypeError, ValueError) as exc:
+              print(f"Malformed check-run evidence: {exc}", file=sys.stderr)
+              raise SystemExit(2) from exc
           current_run_fragment = f"/actions/runs/{sys.argv[2]}"
           if not isinstance(pages, list):
-              raise SystemExit("Unsupported check-run response shape")
+              raise SystemExit(2)
 
           checks = []
           for page in pages:
               if not isinstance(page, dict) or not isinstance(
                   page.get("check_runs"), list
               ):
-                  raise SystemExit("Unsupported check-run response shape")
+                  raise SystemExit(2)
               checks.extend(page["check_runs"])
 
           latest = {}
@@ -241,27 +259,41 @@ jobs:
                   latest[key] = check
 
           if not latest:
-              raise SystemExit("Missing latest default-branch check evidence")
-          accepted = {"success", "neutral", "skipped"}
+              print("Missing latest default-branch check evidence", file=sys.stderr)
+              raise SystemExit(3)
+          accepted = {"success"}
           unhealthy = sorted(
               f"{key[0]}/{key[1]}"
               for key, check in latest.items()
               if check.get("status") != "completed"
               or check.get("conclusion") not in accepted
           )
           if unhealthy:
-              raise SystemExit(
+              print(
                   "Default branch has pending or unsuccessful latest check evidence: "
-                  + ", ".join(unhealthy)
+                  + ", ".join(unhealthy),
+                  file=sys.stderr,
               )
+              raise SystemExit(3)
           PY
-          then
-            echo "::warning::Default-branch check evidence is incomplete or unhealthy; refusing to create work."
-            exit 0
-          fi
+          case "$check_evidence_status" in
+            0) ;;
+            2)
+              echo "::error::Unable to interpret default-branch check evidence; GitHub inventory is malformed."
+              exit 1
+              ;;
+            3)
+              echo "::warning::Default-branch check evidence is incomplete or unhealthy; refusing to create work."
+              exit 0
+              ;;
+            *)
+              echo "::error::Default-branch check evidence parser failed unexpectedly."
+              exit 1
+              ;;
+          esac
 
           if [ "$DRY_RUN" = "true" ]; then
-            echo "Dry run: the NVIDIA NIM OpenCode development gate is ready." \
+            echo "Dry run: deterministic repository gates are healthy; model access was not requested." \
               >>"$GITHUB_STEP_SUMMARY"
             exit 0
           fi
@@ -396,6 +428,10 @@ jobs:
           NIM_UPSTREAM_API_KEY: ${{ secrets.NVIDIA_NIM_API_KEY }}
         run: |
           set -euo pipefail
+          if [ -z "${NIM_UPSTREAM_API_KEY:-}" ]; then
+            echo "::error::NVIDIA_NIM_API_KEY is required only for model-backed development."
+            exit 1
+          fi
           umask 077
           proxy_log="${RUNNER_TEMP}/keyverse-nim-proxy.log"
           proxy_pid="${RUNNER_TEMP}/keyverse-nim-proxy.pid"
@@ -542,8 +578,6 @@ jobs:
         if: steps.gate.outputs.develop == 'true'
         id: package
         shell: bash
-        env:
-          KEYVERSE_FORBIDDEN_SECRET: ${{ secrets.NVIDIA_NIM_API_KEY }}
         run: |
           set -euo pipefail
           artifact_dir="${RUNNER_TEMP}/hourly-product-change"
@@ -585,8 +619,9 @@ jobs:
         with:
           egress-policy: block
           disable-telemetry: true
-          allowed-endpoints: |
+          allowed-endpoints: >-
             api.github.com:443
+            cafe.github.com:443
             github.com:443
             objects.githubusercontent.com:443
             raw.githubusercontent.com:443
@@ -598,7 +633,6 @@ jobs:
             files.pythonhosted.org:443
             pypi.org:443
 
-
       - name: Check out a fresh protected branch
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
@@ -726,8 +760,9 @@ jobs:
         with:
           egress-policy: block
           disable-telemetry: true
-          allowed-endpoints: |
+          allowed-endpoints: >-
             api.github.com:443
+            cafe.github.com:443
             github.com:443
             objects.githubusercontent.com:443
             results-receiver.actions.githubusercontent.com:443

services/account_unification/tests/test_hourly_product_development.py

@@ -49,12 +49,25 @@ def _harden_runner_endpoints(job_name: str) -> tuple[str, ...]:
         assert isinstance(inputs, dict)
         endpoint_block = inputs.get("allowed-endpoints")
         assert isinstance(endpoint_block, str)
-        return tuple(
-            line.strip() for line in endpoint_block.splitlines() if line.strip()
-        )
+        return tuple(endpoint_block.split())
     raise AssertionError(f"{job_name} has no harden-runner endpoint policy")
 
 
+def _step_by_name(job_name: str, step_name: str) -> dict[str, object]:
+    """Return one exact named workflow step from ``job_name``."""
+    jobs = _workflow_document().get("jobs")
+    assert isinstance(jobs, dict)
+    job = jobs.get(job_name)
+    assert isinstance(job, dict)
+    steps = job.get("steps")
+    assert isinstance(steps, list)
+
+    for step in steps:
+        if isinstance(step, dict) and step.get("name") == step_name:
+            return step
+    raise AssertionError(f"{job_name} has no step named {step_name}")
+
+
 def _permissions_block(source: str, marker: str, terminator: str) -> str:
     """Return one indentation-sensitive workflow permissions block."""
     block_start = source.index(marker)
@@ -69,7 +82,7 @@ def test_product_development_runs_hourly_without_cancelling_a_decision() -> None
     assert 'cron: "41 * * * *"' in workflow
     assert "hourly-product-development-${{ github.repository }}" in workflow
     assert "cancel-in-progress: false" in workflow
-    assert "timeout-minutes: 45" in workflow
+    assert "timeout-minutes: 180" in workflow
     assert "timeout-minutes: 30" in workflow
     assert "timeout-minutes: 15" in workflow
 
@@ -128,9 +141,9 @@ def test_nim_credential_is_brokered_outside_the_agent_environment() -> None:
 
     assert "Start the loopback-only NIM credential broker" in workflow
     assert "NIM_UPSTREAM_API_KEY: ${{ secrets.NVIDIA_NIM_API_KEY }}" in workflow
+    assert workflow.count("${{ secrets.NVIDIA_NIM_API_KEY }}") == 1
     assert "NVIDIA_API_KEY=keyverse-local-broker" in workflow
     assert "env -i" in workflow
-    assert "KEYVERSE_FORBIDDEN_SECRET: ${{ secrets.NVIDIA_NIM_API_KEY }}" in workflow
     assert "NVIDIA_API_KEY=${{ secrets.NVIDIA_NIM_API_KEY }}" not in workflow
 
 
@@ -145,10 +158,22 @@ def test_product_development_does_not_reuse_review_agent_credentials() -> None:
 
 
 def test_product_development_fails_closed_without_queue_ownership() -> None:
-    """Missing NIM access, unhealthy main, or open work suppresses the agent."""
+    """Unhealthy main or open work stops before entering the model-backed path."""
     workflow = _workflow_source()
-
-    assert "NVIDIA_NIM_API_KEY is not configured" in workflow
+    broker = _step_by_name(
+        "develop-product-gap",
+        "Start the loopback-only NIM credential broker",
+    )
+    broker_env = broker.get("env")
+    broker_run = broker.get("run")
+
+    assert isinstance(broker_env, dict)
+    assert isinstance(broker_run, str)
+    assert broker.get("if") == "steps.gate.outputs.develop == 'true'"
+    assert broker_env.get("NIM_UPSTREAM_API_KEY") == (
+        "${{ secrets.NVIDIA_NIM_API_KEY }}"
+    )
+    assert "NVIDIA_NIM_API_KEY is required only for model-backed development" in broker_run
     assert "pulls?state=open&per_page=1" in workflow
     assert "An open pull request exists" in workflow
     assert "CORE_WORKFLOWS" in workflow