From 3a274ab95bc98690689466d31dbd54e2717bde9a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 8 Aug 2026 19:02:06 +0900 Subject: [PATCH 01/12] test(ci): reproduce hourly GitHub API gate incident --- .../test_hourly_product_incident_contract.py | 193 ++++++++++++++++++ 1 file changed, 193 insertions(+) create mode 100644 services/account_unification/tests/test_hourly_product_incident_contract.py diff --git a/services/account_unification/tests/test_hourly_product_incident_contract.py b/services/account_unification/tests/test_hourly_product_incident_contract.py new file mode 100644 index 0000000..0c944c9 --- /dev/null +++ b/services/account_unification/tests/test_hourly_product_incident_contract.py @@ -0,0 +1,193 @@ +"""Incident regressions for the fail-closed hourly product-development workflow.""" +from __future__ import annotations + +from pathlib import Path + +import yaml + + +EXPECTED_ENDPOINTS = { + "develop-product-gap": ( + "api.github.com:443", + "cafe.github.com:443", + "codeload.github.com:443", + "github.com:443", + "integrate.api.nvidia.com:443", + "objects.githubusercontent.com:443", + "raw.githubusercontent.com:443", + "registry.npmjs.org:443", + "release-assets.githubusercontent.com:443", + "releases.astral.sh:443", + "results-receiver.actions.githubusercontent.com:443", + "*.actions.githubusercontent.com:443", + "*.blob.core.windows.net:443", + "files.pythonhosted.org:443", + "pypi.org:443", + ), + "reverify-product-gap": ( + "api.github.com:443", + "cafe.github.com:443", + "github.com:443", + "objects.githubusercontent.com:443", + "raw.githubusercontent.com:443", + "release-assets.githubusercontent.com:443", + "releases.astral.sh:443", + "results-receiver.actions.githubusercontent.com:443", + "*.actions.githubusercontent.com:443", + "*.blob.core.windows.net:443", + "files.pythonhosted.org:443", + "pypi.org:443", + ), + "publish-product-gap": ( + "api.github.com:443", + "cafe.github.com:443", + "github.com:443", + "objects.githubusercontent.com:443", + "results-receiver.actions.githubusercontent.com:443", + "*.actions.githubusercontent.com:443", + "*.blob.core.windows.net:443", + ), +} + + +def _repository_root() -> Path: + """Return the Keyverse repository root from this test module.""" + return Path(__file__).resolve().parents[3] + + +def _workflow_source() -> str: + """Return the hourly product-development workflow as reviewed text.""" + return ( + _repository_root() + / ".github" + / "workflows" + / "hourly-product-development.yml" + ).read_text(encoding="utf-8") + + +def _workflow_document() -> dict[str, object]: + """Parse the hourly workflow into a mapping for structural assertions.""" + document = yaml.safe_load(_workflow_source()) + assert isinstance(document, dict) + return document + + +def _job(job_name: str) -> dict[str, object]: + """Return one named workflow job as a mapping.""" + jobs = _workflow_document().get("jobs") + assert isinstance(jobs, dict) + job = jobs.get(job_name) + assert isinstance(job, dict) + return job + + +def _steps(job_name: str) -> list[dict[str, object]]: + """Return mapping-valued steps for one named job.""" + steps = _job(job_name).get("steps") + assert isinstance(steps, list) + return [step for step in steps if isinstance(step, dict)] + + +def _step_by_id(job_name: str, step_id: str) -> dict[str, object]: + """Return the exact step carrying ``step_id`` in ``job_name``.""" + for step in _steps(job_name): + if step.get("id") == step_id: + return step + raise AssertionError(f"{job_name} has no step id {step_id}") + + +def _step_by_name(job_name: str, step_name: str) -> dict[str, object]: + """Return the exact step named ``step_name`` in ``job_name``.""" + for step in _steps(job_name): + if step.get("name") == step_name: + return step + raise AssertionError(f"{job_name} has no step named {step_name}") + + +def _harden_runner_endpoints(job_name: str) -> tuple[str, ...]: + """Return the exact ordered Harden Runner endpoint allowlist for a job.""" + for step in _steps(job_name): + action = step.get("uses") + if not isinstance(action, str) or not action.startswith( + "step-security/harden-runner@" + ): + continue + inputs = step.get("with") + assert isinstance(inputs, dict) + assert inputs.get("egress-policy") == "block" + endpoint_block = inputs.get("allowed-endpoints") + assert isinstance(endpoint_block, str) + return tuple( + line.strip() for line in endpoint_block.splitlines() if line.strip() + ) + raise AssertionError(f"{job_name} has no Harden Runner step") + + +def test_github_api_jobs_use_exact_fail_closed_endpoint_sets() -> None: + """Every GitHub-API phase permits only its reviewed exact endpoint set.""" + for job_name, expected in EXPECTED_ENDPOINTS.items(): + actual = _harden_runner_endpoints(job_name) + assert actual == expected + assert "api.github.com:443.evil" not in actual + assert "*.github.com:443" not in actual + + +def test_deterministic_repository_gates_precede_optional_model_credential() -> None: + """Queue, main, release evidence, and dry-run gates run before model access.""" + gate = _step_by_id("develop-product-gap", "gate") + gate_run = gate.get("run") + gate_env = gate.get("env") + assert isinstance(gate_run, str) + assert isinstance(gate_env, dict) + + ordered_markers = ( + "pulls?state=open&per_page=1", + "commits/${DEFAULT_BRANCH}", + "actions/runs?branch=${DEFAULT_BRANCH}", + "commits/${base_sha}/check-runs?per_page=100", + 'if [ "$DRY_RUN" = "true" ]; then', + ) + positions = tuple(gate_run.index(marker) for marker in ordered_markers) + assert positions == tuple(sorted(positions)) + assert "NIM_UPSTREAM_API_KEY" not in gate_env + assert "NIM_UPSTREAM_API_KEY" not in gate_run + assert "NVIDIA_NIM_API_KEY" not in gate_run + + +def test_github_inventory_transport_failures_are_not_false_green() -> None: + """GitHub inventory transport/shape failures terminate the gate unsuccessfully.""" + gate = _step_by_id("develop-product-gap", "gate") + gate_run = gate.get("run") + assert isinstance(gate_run, str) + + failure_messages = ( + "Unable to list open pull requests", + "Unable to interpret the open pull-request response", + "Unable to resolve the default-branch head", + "The default-branch head was malformed", + "Unable to read default-branch workflow evidence", + "Unable to read default-branch check evidence", + ) + for message in failure_messages: + marker = f'echo "::error::{message}' + start = gate_run.index(marker) + branch_tail = gate_run[start : start + 320] + assert "exit 1" in branch_tail + assert f"::warning::{message}" not in gate_run + + +def test_nvidia_secret_is_required_only_on_the_model_backed_path() -> None: + """The NVIDIA secret is checked only after deterministic gates select development.""" + broker = _step_by_name( + "develop-product-gap", + "Start the loopback-only NIM credential broker", + ) + broker_env = broker.get("env") + broker_run = broker.get("run") + assert isinstance(broker_env, dict) + assert isinstance(broker_run, str) + + assert broker_env.get("NIM_UPSTREAM_API_KEY") == "${{ secrets.NVIDIA_NIM_API_KEY }}" + assert 'if [ -z "${NIM_UPSTREAM_API_KEY:-}" ]; then' in broker_run + assert "NVIDIA_NIM_API_KEY is required only for model-backed development" in broker_run + assert "exit 1" in broker_run From a1481e116519f9857b4fc3595c49787348818314 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 8 Aug 2026 19:04:33 +0900 Subject: [PATCH 02/12] fix(ci): fail closed on GitHub inventory egress --- .../workflows/hourly-product-development.yml | 44 +++++++++---------- 1 file changed, 21 insertions(+), 23 deletions(-) diff --git a/.github/workflows/hourly-product-development.yml b/.github/workflows/hourly-product-development.yml index 0699637..2424656 100644 --- a/.github/workflows/hourly-product-development.yml +++ b/.github/workflows/hourly-product-development.yml @@ -55,6 +55,7 @@ jobs: disable-telemetry: true allowed-endpoints: | api.github.com:443 + cafe.github.com:443 codeload.github.com:443 github.com:443 integrate.api.nvidia.com:443 @@ -69,13 +70,11 @@ jobs: files.pythonhosted.org:443 pypi.org:443 - - - name: Enforce the credential, queue, and exact-main gate + - name: Enforce the deterministic queue and exact-main gate id: gate shell: bash env: GH_TOKEN: ${{ github.token }} - NIM_UPSTREAM_API_KEY: ${{ secrets.NVIDIA_NIM_API_KEY }} CURRENT_RUN_ID: ${{ github.run_id }} DRY_RUN: ${{ inputs.dry_run || false }} run: | @@ -85,19 +84,13 @@ jobs: echo "base_sha=" } >>"$GITHUB_OUTPUT" - if [ -z "${NIM_UPSTREAM_API_KEY:-}" ]; then - echo "NVIDIA_NIM_API_KEY is not configured; autonomous development stopped safely." \ - >>"$GITHUB_STEP_SUMMARY" - exit 0 - fi - open_pr_file="${RUNNER_TEMP}/keyverse-open-pulls.json" if ! gh api \ -H "Accept: application/vnd.github+json" \ "repos/${GITHUB_REPOSITORY}/pulls?state=open&per_page=1" \ >"$open_pr_file"; then - echo "::warning::Unable to list open pull requests; refusing to create work." - exit 0 + echo "::error::Unable to list open pull requests; GitHub inventory is unavailable." + exit 1 fi if ! open_pr_count="$(python3 - "$open_pr_file" <<'PY' import json @@ -110,8 +103,8 @@ jobs: print(len(payload)) PY )"; then - echo "::warning::Unable to interpret the open pull-request response; refusing to create work." - exit 0 + echo "::error::Unable to interpret the open pull-request response; GitHub inventory is malformed." + exit 1 fi if [ "$open_pr_count" -ne 0 ]; then echo "An open pull request exists; the protected PR loop owns this hour." \ @@ -125,12 +118,12 @@ jobs: "repos/${GITHUB_REPOSITORY}/commits/${DEFAULT_BRANCH}" \ --jq '.sha' )"; then - echo "::warning::Unable to resolve the default-branch head; refusing to create work." - exit 0 + echo "::error::Unable to resolve the default-branch head; GitHub inventory is unavailable." + exit 1 fi if ! [[ "$base_sha" =~ ^[0-9a-f]{40}$ ]]; then - echo "::warning::The default-branch head was malformed; refusing to create work." - exit 0 + echo "::error::The default-branch head was malformed; GitHub inventory is invalid." + exit 1 fi workflow_runs_file="${RUNNER_TEMP}/keyverse-main-workflow-runs.json" @@ -140,8 +133,8 @@ jobs: --slurp \ "repos/${GITHUB_REPOSITORY}/actions/runs?branch=${DEFAULT_BRANCH}&head_sha=${base_sha}&per_page=100" \ >"$workflow_runs_file"; then - echo "::warning::Unable to read default-branch workflow evidence; refusing to create work." - exit 0 + echo "::error::Unable to read default-branch workflow evidence; GitHub inventory is unavailable." + exit 1 fi if ! python3 - "$workflow_runs_file" "$CORE_WORKFLOWS" <<'PY' import json @@ -203,8 +196,8 @@ jobs: --slurp \ "repos/${GITHUB_REPOSITORY}/commits/${base_sha}/check-runs?per_page=100" \ >"$check_runs_file"; then - echo "::warning::Unable to read default-branch check evidence; refusing to create work." - exit 0 + echo "::error::Unable to read default-branch check evidence; GitHub inventory is unavailable." + exit 1 fi if ! python3 - "$check_runs_file" "$CURRENT_RUN_ID" <<'PY' import json @@ -261,7 +254,7 @@ jobs: fi if [ "$DRY_RUN" = "true" ]; then - echo "Dry run: the NVIDIA NIM OpenCode development gate is ready." \ + echo "Dry run: deterministic repository gates are healthy; model access was not requested." \ >>"$GITHUB_STEP_SUMMARY" exit 0 fi @@ -396,6 +389,10 @@ jobs: NIM_UPSTREAM_API_KEY: ${{ secrets.NVIDIA_NIM_API_KEY }} run: | set -euo pipefail + if [ -z "${NIM_UPSTREAM_API_KEY:-}" ]; then + echo "::error::NVIDIA_NIM_API_KEY is required only for model-backed development." + exit 1 + fi umask 077 proxy_log="${RUNNER_TEMP}/keyverse-nim-proxy.log" proxy_pid="${RUNNER_TEMP}/keyverse-nim-proxy.pid" @@ -587,6 +584,7 @@ jobs: disable-telemetry: true allowed-endpoints: | api.github.com:443 + cafe.github.com:443 github.com:443 objects.githubusercontent.com:443 raw.githubusercontent.com:443 @@ -598,7 +596,6 @@ jobs: files.pythonhosted.org:443 pypi.org:443 - - name: Check out a fresh protected branch uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -728,6 +725,7 @@ jobs: disable-telemetry: true allowed-endpoints: | api.github.com:443 + cafe.github.com:443 github.com:443 objects.githubusercontent.com:443 results-receiver.actions.githubusercontent.com:443 From d1802174f73b5b2a3b7f8511e7816c8763d01ec0 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 8 Aug 2026 19:06:08 +0900 Subject: [PATCH 03/12] test(ci): align hourly model-path credential contract --- .../tests/test_hourly_product_development.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/account_unification/tests/test_hourly_product_development.py b/services/account_unification/tests/test_hourly_product_development.py index 328ed50..8455311 100644 --- a/services/account_unification/tests/test_hourly_product_development.py +++ b/services/account_unification/tests/test_hourly_product_development.py @@ -145,10 +145,10 @@ def test_product_development_does_not_reuse_review_agent_credentials() -> None: def test_product_development_fails_closed_without_queue_ownership() -> None: - """Missing NIM access, unhealthy main, or open work suppresses the agent.""" + """Unhealthy main or open work stops before entering the model-backed path.""" workflow = _workflow_source() - assert "NVIDIA_NIM_API_KEY is not configured" in workflow + assert "NVIDIA_NIM_API_KEY is required only for model-backed development" in workflow assert "pulls?state=open&per_page=1" in workflow assert "An open pull request exists" in workflow assert "CORE_WORKFLOWS" in workflow From da5c8a5722e7e75de3fb82b486a0c1e087c2fb02 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 8 Aug 2026 20:05:53 +0900 Subject: [PATCH 04/12] test(ci): require runtime-safe Harden Runner endpoint scalar --- .../test_hourly_product_incident_contract.py | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/services/account_unification/tests/test_hourly_product_incident_contract.py b/services/account_unification/tests/test_hourly_product_incident_contract.py index 0c944c9..1ece947 100644 --- a/services/account_unification/tests/test_hourly_product_incident_contract.py +++ b/services/account_unification/tests/test_hourly_product_incident_contract.py @@ -104,8 +104,8 @@ def _step_by_name(job_name: str, step_name: str) -> dict[str, object]: raise AssertionError(f"{job_name} has no step named {step_name}") -def _harden_runner_endpoints(job_name: str) -> tuple[str, ...]: - """Return the exact ordered Harden Runner endpoint allowlist for a job.""" +def _harden_runner_endpoint_scalar(job_name: str) -> str: + """Return the serialized Harden Runner endpoint input for one workflow job.""" for step in _steps(job_name): action = step.get("uses") if not isinstance(action, str) or not action.startswith( @@ -117,12 +117,15 @@ def _harden_runner_endpoints(job_name: str) -> tuple[str, ...]: assert inputs.get("egress-policy") == "block" endpoint_block = inputs.get("allowed-endpoints") assert isinstance(endpoint_block, str) - return tuple( - line.strip() for line in endpoint_block.splitlines() if line.strip() - ) + return endpoint_block raise AssertionError(f"{job_name} has no Harden Runner step") +def _harden_runner_endpoints(job_name: str) -> tuple[str, ...]: + """Return the exact ordered Harden Runner endpoint allowlist for a job.""" + return tuple(_harden_runner_endpoint_scalar(job_name).split()) + + def test_github_api_jobs_use_exact_fail_closed_endpoint_sets() -> None: """Every GitHub-API phase permits only its reviewed exact endpoint set.""" for job_name, expected in EXPECTED_ENDPOINTS.items(): @@ -132,6 +135,14 @@ def test_github_api_jobs_use_exact_fail_closed_endpoint_sets() -> None: assert "*.github.com:443" not in actual +def test_harden_runner_endpoint_input_is_space_delimited_for_runtime() -> None: + """Harden Runner receives one folded, space-delimited endpoint scalar per job.""" + for job_name, expected in EXPECTED_ENDPOINTS.items(): + endpoint_scalar = _harden_runner_endpoint_scalar(job_name) + assert endpoint_scalar == " ".join(expected) + assert "\n" not in endpoint_scalar + + def test_deterministic_repository_gates_precede_optional_model_credential() -> None: """Queue, main, release evidence, and dry-run gates run before model access.""" gate = _step_by_id("develop-product-gap", "gate") From f0d3bfbb6580926dd81a9f16de48680041b8dce9 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 8 Aug 2026 21:10:50 +0900 Subject: [PATCH 05/12] test(ci): bind NVIDIA secret to model-backed step --- .../tests/test_hourly_product_development.py | 31 +++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/services/account_unification/tests/test_hourly_product_development.py b/services/account_unification/tests/test_hourly_product_development.py index 8455311..1d4b711 100644 --- a/services/account_unification/tests/test_hourly_product_development.py +++ b/services/account_unification/tests/test_hourly_product_development.py @@ -55,6 +55,21 @@ def _harden_runner_endpoints(job_name: str) -> tuple[str, ...]: raise AssertionError(f"{job_name} has no harden-runner endpoint policy") +def _step_by_name(job_name: str, step_name: str) -> dict[str, object]: + """Return one exact named workflow step from ``job_name``.""" + jobs = _workflow_document().get("jobs") + assert isinstance(jobs, dict) + job = jobs.get(job_name) + assert isinstance(job, dict) + steps = job.get("steps") + assert isinstance(steps, list) + + for step in steps: + if isinstance(step, dict) and step.get("name") == step_name: + return step + raise AssertionError(f"{job_name} has no step named {step_name}") + + def _permissions_block(source: str, marker: str, terminator: str) -> str: """Return one indentation-sensitive workflow permissions block.""" block_start = source.index(marker) @@ -147,8 +162,20 @@ def test_product_development_does_not_reuse_review_agent_credentials() -> None: def test_product_development_fails_closed_without_queue_ownership() -> None: """Unhealthy main or open work stops before entering the model-backed path.""" workflow = _workflow_source() - - assert "NVIDIA_NIM_API_KEY is required only for model-backed development" in workflow + broker = _step_by_name( + "develop-product-gap", + "Start the loopback-only NIM credential broker", + ) + broker_env = broker.get("env") + broker_run = broker.get("run") + + assert isinstance(broker_env, dict) + assert isinstance(broker_run, str) + assert broker.get("if") == "steps.gate.outputs.develop == 'true'" + assert broker_env.get("NIM_UPSTREAM_API_KEY") == ( + "${{ secrets.NVIDIA_NIM_API_KEY }}" + ) + assert "NVIDIA_NIM_API_KEY is required only for model-backed development" in broker_run assert "pulls?state=open&per_page=1" in workflow assert "An open pull request exists" in workflow assert "CORE_WORKFLOWS" in workflow From dee4419a4d0857de7f0507f69fa2747606f2ecde Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 8 Aug 2026 21:11:27 +0900 Subject: [PATCH 06/12] test(ci): fail closed on malformed GitHub evidence --- .../test_hourly_product_incident_contract.py | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/services/account_unification/tests/test_hourly_product_incident_contract.py b/services/account_unification/tests/test_hourly_product_incident_contract.py index 1ece947..e6c9187 100644 --- a/services/account_unification/tests/test_hourly_product_incident_contract.py +++ b/services/account_unification/tests/test_hourly_product_incident_contract.py @@ -186,6 +186,36 @@ def test_github_inventory_transport_failures_are_not_false_green() -> None: assert "exit 1" in branch_tail assert f"::warning::{message}" not in gate_run + malformed_evidence_contracts = ( + ( + "Unsupported workflow-run response shape", + "raise SystemExit(2)", + "workflow_evidence_status", + "Unable to interpret default-branch workflow evidence", + ), + ( + "Unsupported check-run response shape", + "raise SystemExit(2)", + "check_evidence_status", + "Unable to interpret default-branch check evidence", + ), + ) + for parser_marker, malformed_exit, status_name, error_message in ( + malformed_evidence_contracts + ): + parser_start = gate_run.index(parser_marker) + parser_tail = gate_run[parser_start : parser_start + 180] + assert malformed_exit in parser_tail + assert f"{status_name}=$?" in gate_run + assert f"::error::{error_message}" in gate_run + + assert 'workflow_evidence_status=3' not in gate_run + assert 'check_evidence_status=3' not in gate_run + assert "Missing required default-branch workflow evidence" in gate_run + assert "Default branch has pending or unsuccessful required workflow evidence" in gate_run + assert "Missing latest default-branch check evidence" in gate_run + assert "Default branch has pending or unsuccessful latest check evidence" in gate_run + def test_nvidia_secret_is_required_only_on_the_model_backed_path() -> None: """The NVIDIA secret is checked only after deterministic gates select development.""" From f9f69ddef213a4e4f0566cfb02576426c9f5f48e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 8 Aug 2026 21:13:19 +0900 Subject: [PATCH 07/12] fix(ci): preserve Harden Runner ports and malformed evidence failures --- .../workflows/hourly-product-development.yml | 97 +++++++++++++------ 1 file changed, 68 insertions(+), 29 deletions(-) diff --git a/.github/workflows/hourly-product-development.yml b/.github/workflows/hourly-product-development.yml index 2424656..0493a61 100644 --- a/.github/workflows/hourly-product-development.yml +++ b/.github/workflows/hourly-product-development.yml @@ -53,7 +53,7 @@ jobs: with: egress-policy: block disable-telemetry: true - allowed-endpoints: | + allowed-endpoints: >- api.github.com:443 cafe.github.com:443 codeload.github.com:443 @@ -136,22 +136,27 @@ jobs: echo "::error::Unable to read default-branch workflow evidence; GitHub inventory is unavailable." exit 1 fi - if ! python3 - "$workflow_runs_file" "$CORE_WORKFLOWS" <<'PY' + workflow_evidence_status=0 + python3 - "$workflow_runs_file" "$CORE_WORKFLOWS" <<'PY' || workflow_evidence_status=$? import json import sys - with open(sys.argv[1], encoding="utf-8") as stream: - pages = json.load(stream) - required = set(json.loads(sys.argv[2])) + try: + with open(sys.argv[1], encoding="utf-8") as stream: + pages = json.load(stream) + required = set(json.loads(sys.argv[2])) + except (OSError, json.JSONDecodeError, TypeError, ValueError) as exc: + print(f"Malformed workflow-run evidence: {exc}", file=sys.stderr) + raise SystemExit(2) from exc if not isinstance(pages, list) or not required: - raise SystemExit("Unsupported workflow-run response shape") + raise SystemExit(2) runs = [] for page in pages: if not isinstance(page, dict) or not isinstance( page.get("workflow_runs"), list ): - raise SystemExit("Unsupported workflow-run response shape") + raise SystemExit(2) runs.extend(page["workflow_runs"]) latest = {} @@ -168,10 +173,12 @@ jobs: missing = sorted(required.difference(latest)) if missing: - raise SystemExit( + print( "Missing required default-branch workflow evidence: " - + ", ".join(missing) + + ", ".join(missing), + file=sys.stderr, ) + raise SystemExit(3) unhealthy = sorted( name for name, run in latest.items() @@ -179,15 +186,28 @@ jobs: or run.get("conclusion") != "success" ) if unhealthy: - raise SystemExit( + print( "Default branch has pending or unsuccessful required workflow evidence: " - + ", ".join(unhealthy) + + ", ".join(unhealthy), + file=sys.stderr, ) + raise SystemExit(3) PY - then - echo "::warning::Default-branch core workflow evidence is incomplete or unhealthy; refusing to create work." - exit 0 - fi + case "$workflow_evidence_status" in + 0) ;; + 2) + echo "::error::Unable to interpret default-branch workflow evidence; GitHub inventory is malformed." + exit 1 + ;; + 3) + echo "::warning::Default-branch core workflow evidence is incomplete or unhealthy; refusing to create work." + exit 0 + ;; + *) + echo "::error::Default-branch workflow evidence parser failed unexpectedly." + exit 1 + ;; + esac check_runs_file="${RUNNER_TEMP}/keyverse-main-check-runs.json" if ! gh api \ @@ -199,22 +219,27 @@ jobs: echo "::error::Unable to read default-branch check evidence; GitHub inventory is unavailable." exit 1 fi - if ! python3 - "$check_runs_file" "$CURRENT_RUN_ID" <<'PY' + check_evidence_status=0 + python3 - "$check_runs_file" "$CURRENT_RUN_ID" <<'PY' || check_evidence_status=$? import json import sys - with open(sys.argv[1], encoding="utf-8") as stream: - pages = json.load(stream) + try: + with open(sys.argv[1], encoding="utf-8") as stream: + pages = json.load(stream) + except (OSError, json.JSONDecodeError, TypeError, ValueError) as exc: + print(f"Malformed check-run evidence: {exc}", file=sys.stderr) + raise SystemExit(2) from exc current_run_fragment = f"/actions/runs/{sys.argv[2]}" if not isinstance(pages, list): - raise SystemExit("Unsupported check-run response shape") + raise SystemExit(2) checks = [] for page in pages: if not isinstance(page, dict) or not isinstance( page.get("check_runs"), list ): - raise SystemExit("Unsupported check-run response shape") + raise SystemExit(2) checks.extend(page["check_runs"]) latest = {} @@ -234,7 +259,8 @@ jobs: latest[key] = check if not latest: - raise SystemExit("Missing latest default-branch check evidence") + print("Missing latest default-branch check evidence", file=sys.stderr) + raise SystemExit(3) accepted = {"success", "neutral", "skipped"} unhealthy = sorted( f"{key[0]}/{key[1]}" @@ -243,15 +269,28 @@ jobs: or check.get("conclusion") not in accepted ) if unhealthy: - raise SystemExit( + print( "Default branch has pending or unsuccessful latest check evidence: " - + ", ".join(unhealthy) + + ", ".join(unhealthy), + file=sys.stderr, ) + raise SystemExit(3) PY - then - echo "::warning::Default-branch check evidence is incomplete or unhealthy; refusing to create work." - exit 0 - fi + case "$check_evidence_status" in + 0) ;; + 2) + echo "::error::Unable to interpret default-branch check evidence; GitHub inventory is malformed." + exit 1 + ;; + 3) + echo "::warning::Default-branch check evidence is incomplete or unhealthy; refusing to create work." + exit 0 + ;; + *) + echo "::error::Default-branch check evidence parser failed unexpectedly." + exit 1 + ;; + esac if [ "$DRY_RUN" = "true" ]; then echo "Dry run: deterministic repository gates are healthy; model access was not requested." \ @@ -582,7 +621,7 @@ jobs: with: egress-policy: block disable-telemetry: true - allowed-endpoints: | + allowed-endpoints: >- api.github.com:443 cafe.github.com:443 github.com:443 @@ -723,7 +762,7 @@ jobs: with: egress-policy: block disable-telemetry: true - allowed-endpoints: | + allowed-endpoints: >- api.github.com:443 cafe.github.com:443 github.com:443 From da7e49ffd0d6ee5e3fb24931510018d69f2d568d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 8 Aug 2026 21:15:33 +0900 Subject: [PATCH 08/12] test(ci): parse folded Harden Runner endpoints --- .../tests/test_hourly_product_development.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/services/account_unification/tests/test_hourly_product_development.py b/services/account_unification/tests/test_hourly_product_development.py index 1d4b711..b0d6929 100644 --- a/services/account_unification/tests/test_hourly_product_development.py +++ b/services/account_unification/tests/test_hourly_product_development.py @@ -49,9 +49,7 @@ def _harden_runner_endpoints(job_name: str) -> tuple[str, ...]: assert isinstance(inputs, dict) endpoint_block = inputs.get("allowed-endpoints") assert isinstance(endpoint_block, str) - return tuple( - line.strip() for line in endpoint_block.splitlines() if line.strip() - ) + return tuple(endpoint_block.split()) raise AssertionError(f"{job_name} has no harden-runner endpoint policy") From f24d4b46e344f0f2c375c65fafcb3d9410284fa2 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 8 Aug 2026 21:16:04 +0900 Subject: [PATCH 09/12] test(ci): bind malformed evidence assertions to parser structure --- .../tests/test_hourly_product_incident_contract.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/account_unification/tests/test_hourly_product_incident_contract.py b/services/account_unification/tests/test_hourly_product_incident_contract.py index e6c9187..c4fdd1f 100644 --- a/services/account_unification/tests/test_hourly_product_incident_contract.py +++ b/services/account_unification/tests/test_hourly_product_incident_contract.py @@ -188,13 +188,13 @@ def test_github_inventory_transport_failures_are_not_false_green() -> None: malformed_evidence_contracts = ( ( - "Unsupported workflow-run response shape", + "if not isinstance(pages, list) or not required:", "raise SystemExit(2)", "workflow_evidence_status", "Unable to interpret default-branch workflow evidence", ), ( - "Unsupported check-run response shape", + "if not isinstance(pages, list):", "raise SystemExit(2)", "check_evidence_status", "Unable to interpret default-branch check evidence", From d6eea5e47a8d7ac0f6f0f3fce3991c6fd1ed096d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 9 Aug 2026 00:33:16 +0900 Subject: [PATCH 10/12] test(ci): expose hourly feasibility gaps --- .../test_hourly_product_incident_contract.py | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/services/account_unification/tests/test_hourly_product_incident_contract.py b/services/account_unification/tests/test_hourly_product_incident_contract.py index c4fdd1f..22f493c 100644 --- a/services/account_unification/tests/test_hourly_product_incident_contract.py +++ b/services/account_unification/tests/test_hourly_product_incident_contract.py @@ -217,6 +217,50 @@ def test_github_inventory_transport_failures_are_not_false_green() -> None: assert "Default branch has pending or unsuccessful latest check evidence" in gate_run +def test_default_branch_check_evidence_requires_success() -> None: + """Neutral or skipped default-main checks never qualify as healthy evidence.""" + gate = _step_by_id("develop-product-gap", "gate") + gate_run = gate.get("run") + assert isinstance(gate_run, str) + + accepted_start = gate_run.index("accepted =") + accepted_block = gate_run[accepted_start : accepted_start + 120] + assert 'accepted = {"success"}' in accepted_block + assert '"neutral"' not in accepted_block + assert '"skipped"' not in accepted_block + + +def test_model_fallback_budget_fits_outer_job_timeout() -> None: + """All sequential model candidates plus setup reserve fit the job deadline.""" + document = _workflow_document() + env = document.get("env") + assert isinstance(env, dict) + candidates = str(env.get("OPENCODE_MODEL_CANDIDATES", "")).split() + per_model_seconds = int(str(env.get("OPENCODE_RUN_TIMEOUT_SECONDS", "0"))) + timeout_minutes = int(str(_job("develop-product-gap").get("timeout-minutes", 0))) + + assert candidates + setup_and_packaging_reserve_seconds = 15 * 60 + assert timeout_minutes * 60 >= ( + len(candidates) * per_model_seconds + setup_and_packaging_reserve_seconds + ) + + +def test_nvidia_secret_is_materialized_only_by_broker() -> None: + """The raw NVIDIA secret exists only in the conditional loopback broker step.""" + secret_expression = "${{ secrets.NVIDIA_NIM_API_KEY }}" + materializing_steps: list[str] = [] + for step in _steps("develop-product-gap"): + env = step.get("env") + if not isinstance(env, dict) or secret_expression not in env.values(): + continue + name = step.get("name") + assert isinstance(name, str) + materializing_steps.append(name) + + assert materializing_steps == ["Start the loopback-only NIM credential broker"] + + def test_nvidia_secret_is_required_only_on_the_model_backed_path() -> None: """The NVIDIA secret is checked only after deterministic gates select development.""" broker = _step_by_name( From 35975b79feef5bd6714013ae38bd584b4d9e18c8 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 9 Aug 2026 00:39:10 +0900 Subject: [PATCH 11/12] fix(ci): close hourly feasibility gaps --- .github/workflows/hourly-product-development.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/hourly-product-development.yml b/.github/workflows/hourly-product-development.yml index 0493a61..da09279 100644 --- a/.github/workflows/hourly-product-development.yml +++ b/.github/workflows/hourly-product-development.yml @@ -37,7 +37,7 @@ jobs: develop-product-gap: name: Discover and package one bounded product gap runs-on: ubuntu-24.04 - timeout-minutes: 45 + timeout-minutes: 180 permissions: actions: read checks: read @@ -261,7 +261,7 @@ jobs: if not latest: print("Missing latest default-branch check evidence", file=sys.stderr) raise SystemExit(3) - accepted = {"success", "neutral", "skipped"} + accepted = {"success"} unhealthy = sorted( f"{key[0]}/{key[1]}" for key, check in latest.items() @@ -578,8 +578,6 @@ jobs: if: steps.gate.outputs.develop == 'true' id: package shell: bash - env: - KEYVERSE_FORBIDDEN_SECRET: ${{ secrets.NVIDIA_NIM_API_KEY }} run: | set -euo pipefail artifact_dir="${RUNNER_TEMP}/hourly-product-change" From 2b572dd20c45e91853bdb9c20ba4f7b2bc6c625c Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 9 Aug 2026 00:41:11 +0900 Subject: [PATCH 12/12] test(ci): align hourly boundary contracts --- .../tests/test_hourly_product_development.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/account_unification/tests/test_hourly_product_development.py b/services/account_unification/tests/test_hourly_product_development.py index b0d6929..eb50768 100644 --- a/services/account_unification/tests/test_hourly_product_development.py +++ b/services/account_unification/tests/test_hourly_product_development.py @@ -82,7 +82,7 @@ def test_product_development_runs_hourly_without_cancelling_a_decision() -> None assert 'cron: "41 * * * *"' in workflow assert "hourly-product-development-${{ github.repository }}" in workflow assert "cancel-in-progress: false" in workflow - assert "timeout-minutes: 45" in workflow + assert "timeout-minutes: 180" in workflow assert "timeout-minutes: 30" in workflow assert "timeout-minutes: 15" in workflow @@ -141,9 +141,9 @@ def test_nim_credential_is_brokered_outside_the_agent_environment() -> None: assert "Start the loopback-only NIM credential broker" in workflow assert "NIM_UPSTREAM_API_KEY: ${{ secrets.NVIDIA_NIM_API_KEY }}" in workflow + assert workflow.count("${{ secrets.NVIDIA_NIM_API_KEY }}") == 1 assert "NVIDIA_API_KEY=keyverse-local-broker" in workflow assert "env -i" in workflow - assert "KEYVERSE_FORBIDDEN_SECRET: ${{ secrets.NVIDIA_NIM_API_KEY }}" in workflow assert "NVIDIA_API_KEY=${{ secrets.NVIDIA_NIM_API_KEY }}" not in workflow