diff --git a/.github/workflows/hourly-product-development.yml b/.github/workflows/hourly-product-development.yml index 0699637..da09279 100644 --- a/.github/workflows/hourly-product-development.yml +++ b/.github/workflows/hourly-product-development.yml @@ -37,7 +37,7 @@ jobs: develop-product-gap: name: Discover and package one bounded product gap runs-on: ubuntu-24.04 - timeout-minutes: 45 + timeout-minutes: 180 permissions: actions: read checks: read @@ -53,8 +53,9 @@ jobs: with: egress-policy: block disable-telemetry: true - allowed-endpoints: | + allowed-endpoints: >- api.github.com:443 + cafe.github.com:443 codeload.github.com:443 github.com:443 integrate.api.nvidia.com:443 @@ -69,13 +70,11 @@ jobs: files.pythonhosted.org:443 pypi.org:443 - - - name: Enforce the credential, queue, and exact-main gate + - name: Enforce the deterministic queue and exact-main gate id: gate shell: bash env: GH_TOKEN: ${{ github.token }} - NIM_UPSTREAM_API_KEY: ${{ secrets.NVIDIA_NIM_API_KEY }} CURRENT_RUN_ID: ${{ github.run_id }} DRY_RUN: ${{ inputs.dry_run || false }} run: | @@ -85,19 +84,13 @@ jobs: echo "base_sha=" } >>"$GITHUB_OUTPUT" - if [ -z "${NIM_UPSTREAM_API_KEY:-}" ]; then - echo "NVIDIA_NIM_API_KEY is not configured; autonomous development stopped safely." \ - >>"$GITHUB_STEP_SUMMARY" - exit 0 - fi - open_pr_file="${RUNNER_TEMP}/keyverse-open-pulls.json" if ! gh api \ -H "Accept: application/vnd.github+json" \ "repos/${GITHUB_REPOSITORY}/pulls?state=open&per_page=1" \ >"$open_pr_file"; then - echo "::warning::Unable to list open pull requests; refusing to create work." - exit 0 + echo "::error::Unable to list open pull requests; GitHub inventory is unavailable." + exit 1 fi if ! open_pr_count="$(python3 - "$open_pr_file" <<'PY' import json @@ -110,8 +103,8 @@ jobs: print(len(payload)) PY )"; then - echo "::warning::Unable to interpret the open pull-request response; refusing to create work." - exit 0 + echo "::error::Unable to interpret the open pull-request response; GitHub inventory is malformed." + exit 1 fi if [ "$open_pr_count" -ne 0 ]; then echo "An open pull request exists; the protected PR loop owns this hour." \ @@ -125,12 +118,12 @@ jobs: "repos/${GITHUB_REPOSITORY}/commits/${DEFAULT_BRANCH}" \ --jq '.sha' )"; then - echo "::warning::Unable to resolve the default-branch head; refusing to create work." - exit 0 + echo "::error::Unable to resolve the default-branch head; GitHub inventory is unavailable." + exit 1 fi if ! [[ "$base_sha" =~ ^[0-9a-f]{40}$ ]]; then - echo "::warning::The default-branch head was malformed; refusing to create work." - exit 0 + echo "::error::The default-branch head was malformed; GitHub inventory is invalid." + exit 1 fi workflow_runs_file="${RUNNER_TEMP}/keyverse-main-workflow-runs.json" @@ -140,25 +133,30 @@ jobs: --slurp \ "repos/${GITHUB_REPOSITORY}/actions/runs?branch=${DEFAULT_BRANCH}&head_sha=${base_sha}&per_page=100" \ >"$workflow_runs_file"; then - echo "::warning::Unable to read default-branch workflow evidence; refusing to create work." - exit 0 + echo "::error::Unable to read default-branch workflow evidence; GitHub inventory is unavailable." + exit 1 fi - if ! python3 - "$workflow_runs_file" "$CORE_WORKFLOWS" <<'PY' + workflow_evidence_status=0 + python3 - "$workflow_runs_file" "$CORE_WORKFLOWS" <<'PY' || workflow_evidence_status=$? import json import sys - with open(sys.argv[1], encoding="utf-8") as stream: - pages = json.load(stream) - required = set(json.loads(sys.argv[2])) + try: + with open(sys.argv[1], encoding="utf-8") as stream: + pages = json.load(stream) + required = set(json.loads(sys.argv[2])) + except (OSError, json.JSONDecodeError, TypeError, ValueError) as exc: + print(f"Malformed workflow-run evidence: {exc}", file=sys.stderr) + raise SystemExit(2) from exc if not isinstance(pages, list) or not required: - raise SystemExit("Unsupported workflow-run response shape") + raise SystemExit(2) runs = [] for page in pages: if not isinstance(page, dict) or not isinstance( page.get("workflow_runs"), list ): - raise SystemExit("Unsupported workflow-run response shape") + raise SystemExit(2) runs.extend(page["workflow_runs"]) latest = {} @@ -175,10 +173,12 @@ jobs: missing = sorted(required.difference(latest)) if missing: - raise SystemExit( + print( "Missing required default-branch workflow evidence: " - + ", ".join(missing) + + ", ".join(missing), + file=sys.stderr, ) + raise SystemExit(3) unhealthy = sorted( name for name, run in latest.items() @@ -186,15 +186,28 @@ jobs: or run.get("conclusion") != "success" ) if unhealthy: - raise SystemExit( + print( "Default branch has pending or unsuccessful required workflow evidence: " - + ", ".join(unhealthy) + + ", ".join(unhealthy), + file=sys.stderr, ) + raise SystemExit(3) PY - then - echo "::warning::Default-branch core workflow evidence is incomplete or unhealthy; refusing to create work." - exit 0 - fi + case "$workflow_evidence_status" in + 0) ;; + 2) + echo "::error::Unable to interpret default-branch workflow evidence; GitHub inventory is malformed." + exit 1 + ;; + 3) + echo "::warning::Default-branch core workflow evidence is incomplete or unhealthy; refusing to create work." + exit 0 + ;; + *) + echo "::error::Default-branch workflow evidence parser failed unexpectedly." + exit 1 + ;; + esac check_runs_file="${RUNNER_TEMP}/keyverse-main-check-runs.json" if ! gh api \ @@ -203,25 +216,30 @@ jobs: --slurp \ "repos/${GITHUB_REPOSITORY}/commits/${base_sha}/check-runs?per_page=100" \ >"$check_runs_file"; then - echo "::warning::Unable to read default-branch check evidence; refusing to create work." - exit 0 + echo "::error::Unable to read default-branch check evidence; GitHub inventory is unavailable." + exit 1 fi - if ! python3 - "$check_runs_file" "$CURRENT_RUN_ID" <<'PY' + check_evidence_status=0 + python3 - "$check_runs_file" "$CURRENT_RUN_ID" <<'PY' || check_evidence_status=$? import json import sys - with open(sys.argv[1], encoding="utf-8") as stream: - pages = json.load(stream) + try: + with open(sys.argv[1], encoding="utf-8") as stream: + pages = json.load(stream) + except (OSError, json.JSONDecodeError, TypeError, ValueError) as exc: + print(f"Malformed check-run evidence: {exc}", file=sys.stderr) + raise SystemExit(2) from exc current_run_fragment = f"/actions/runs/{sys.argv[2]}" if not isinstance(pages, list): - raise SystemExit("Unsupported check-run response shape") + raise SystemExit(2) checks = [] for page in pages: if not isinstance(page, dict) or not isinstance( page.get("check_runs"), list ): - raise SystemExit("Unsupported check-run response shape") + raise SystemExit(2) checks.extend(page["check_runs"]) latest = {} @@ -241,8 +259,9 @@ jobs: latest[key] = check if not latest: - raise SystemExit("Missing latest default-branch check evidence") - accepted = {"success", "neutral", "skipped"} + print("Missing latest default-branch check evidence", file=sys.stderr) + raise SystemExit(3) + accepted = {"success"} unhealthy = sorted( f"{key[0]}/{key[1]}" for key, check in latest.items() @@ -250,18 +269,31 @@ jobs: or check.get("conclusion") not in accepted ) if unhealthy: - raise SystemExit( + print( "Default branch has pending or unsuccessful latest check evidence: " - + ", ".join(unhealthy) + + ", ".join(unhealthy), + file=sys.stderr, ) + raise SystemExit(3) PY - then - echo "::warning::Default-branch check evidence is incomplete or unhealthy; refusing to create work." - exit 0 - fi + case "$check_evidence_status" in + 0) ;; + 2) + echo "::error::Unable to interpret default-branch check evidence; GitHub inventory is malformed." + exit 1 + ;; + 3) + echo "::warning::Default-branch check evidence is incomplete or unhealthy; refusing to create work." + exit 0 + ;; + *) + echo "::error::Default-branch check evidence parser failed unexpectedly." + exit 1 + ;; + esac if [ "$DRY_RUN" = "true" ]; then - echo "Dry run: the NVIDIA NIM OpenCode development gate is ready." \ + echo "Dry run: deterministic repository gates are healthy; model access was not requested." \ >>"$GITHUB_STEP_SUMMARY" exit 0 fi @@ -396,6 +428,10 @@ jobs: NIM_UPSTREAM_API_KEY: ${{ secrets.NVIDIA_NIM_API_KEY }} run: | set -euo pipefail + if [ -z "${NIM_UPSTREAM_API_KEY:-}" ]; then + echo "::error::NVIDIA_NIM_API_KEY is required only for model-backed development." + exit 1 + fi umask 077 proxy_log="${RUNNER_TEMP}/keyverse-nim-proxy.log" proxy_pid="${RUNNER_TEMP}/keyverse-nim-proxy.pid" @@ -542,8 +578,6 @@ jobs: if: steps.gate.outputs.develop == 'true' id: package shell: bash - env: - KEYVERSE_FORBIDDEN_SECRET: ${{ secrets.NVIDIA_NIM_API_KEY }} run: | set -euo pipefail artifact_dir="${RUNNER_TEMP}/hourly-product-change" @@ -585,8 +619,9 @@ jobs: with: egress-policy: block disable-telemetry: true - allowed-endpoints: | + allowed-endpoints: >- api.github.com:443 + cafe.github.com:443 github.com:443 objects.githubusercontent.com:443 raw.githubusercontent.com:443 @@ -598,7 +633,6 @@ jobs: files.pythonhosted.org:443 pypi.org:443 - - name: Check out a fresh protected branch uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -726,8 +760,9 @@ jobs: with: egress-policy: block disable-telemetry: true - allowed-endpoints: | + allowed-endpoints: >- api.github.com:443 + cafe.github.com:443 github.com:443 objects.githubusercontent.com:443 results-receiver.actions.githubusercontent.com:443 diff --git a/services/account_unification/tests/test_hourly_product_development.py b/services/account_unification/tests/test_hourly_product_development.py index 328ed50..eb50768 100644 --- a/services/account_unification/tests/test_hourly_product_development.py +++ b/services/account_unification/tests/test_hourly_product_development.py @@ -49,12 +49,25 @@ def _harden_runner_endpoints(job_name: str) -> tuple[str, ...]: assert isinstance(inputs, dict) endpoint_block = inputs.get("allowed-endpoints") assert isinstance(endpoint_block, str) - return tuple( - line.strip() for line in endpoint_block.splitlines() if line.strip() - ) + return tuple(endpoint_block.split()) raise AssertionError(f"{job_name} has no harden-runner endpoint policy") +def _step_by_name(job_name: str, step_name: str) -> dict[str, object]: + """Return one exact named workflow step from ``job_name``.""" + jobs = _workflow_document().get("jobs") + assert isinstance(jobs, dict) + job = jobs.get(job_name) + assert isinstance(job, dict) + steps = job.get("steps") + assert isinstance(steps, list) + + for step in steps: + if isinstance(step, dict) and step.get("name") == step_name: + return step + raise AssertionError(f"{job_name} has no step named {step_name}") + + def _permissions_block(source: str, marker: str, terminator: str) -> str: """Return one indentation-sensitive workflow permissions block.""" block_start = source.index(marker) @@ -69,7 +82,7 @@ def test_product_development_runs_hourly_without_cancelling_a_decision() -> None assert 'cron: "41 * * * *"' in workflow assert "hourly-product-development-${{ github.repository }}" in workflow assert "cancel-in-progress: false" in workflow - assert "timeout-minutes: 45" in workflow + assert "timeout-minutes: 180" in workflow assert "timeout-minutes: 30" in workflow assert "timeout-minutes: 15" in workflow @@ -128,9 +141,9 @@ def test_nim_credential_is_brokered_outside_the_agent_environment() -> None: assert "Start the loopback-only NIM credential broker" in workflow assert "NIM_UPSTREAM_API_KEY: ${{ secrets.NVIDIA_NIM_API_KEY }}" in workflow + assert workflow.count("${{ secrets.NVIDIA_NIM_API_KEY }}") == 1 assert "NVIDIA_API_KEY=keyverse-local-broker" in workflow assert "env -i" in workflow - assert "KEYVERSE_FORBIDDEN_SECRET: ${{ secrets.NVIDIA_NIM_API_KEY }}" in workflow assert "NVIDIA_API_KEY=${{ secrets.NVIDIA_NIM_API_KEY }}" not in workflow @@ -145,10 +158,22 @@ def test_product_development_does_not_reuse_review_agent_credentials() -> None: def test_product_development_fails_closed_without_queue_ownership() -> None: - """Missing NIM access, unhealthy main, or open work suppresses the agent.""" + """Unhealthy main or open work stops before entering the model-backed path.""" workflow = _workflow_source() - - assert "NVIDIA_NIM_API_KEY is not configured" in workflow + broker = _step_by_name( + "develop-product-gap", + "Start the loopback-only NIM credential broker", + ) + broker_env = broker.get("env") + broker_run = broker.get("run") + + assert isinstance(broker_env, dict) + assert isinstance(broker_run, str) + assert broker.get("if") == "steps.gate.outputs.develop == 'true'" + assert broker_env.get("NIM_UPSTREAM_API_KEY") == ( + "${{ secrets.NVIDIA_NIM_API_KEY }}" + ) + assert "NVIDIA_NIM_API_KEY is required only for model-backed development" in broker_run assert "pulls?state=open&per_page=1" in workflow assert "An open pull request exists" in workflow assert "CORE_WORKFLOWS" in workflow diff --git a/services/account_unification/tests/test_hourly_product_incident_contract.py b/services/account_unification/tests/test_hourly_product_incident_contract.py new file mode 100644 index 0000000..22f493c --- /dev/null +++ b/services/account_unification/tests/test_hourly_product_incident_contract.py @@ -0,0 +1,278 @@ +"""Incident regressions for the fail-closed hourly product-development workflow.""" +from __future__ import annotations + +from pathlib import Path + +import yaml + + +EXPECTED_ENDPOINTS = { + "develop-product-gap": ( + "api.github.com:443", + "cafe.github.com:443", + "codeload.github.com:443", + "github.com:443", + "integrate.api.nvidia.com:443", + "objects.githubusercontent.com:443", + "raw.githubusercontent.com:443", + "registry.npmjs.org:443", + "release-assets.githubusercontent.com:443", + "releases.astral.sh:443", + "results-receiver.actions.githubusercontent.com:443", + "*.actions.githubusercontent.com:443", + "*.blob.core.windows.net:443", + "files.pythonhosted.org:443", + "pypi.org:443", + ), + "reverify-product-gap": ( + "api.github.com:443", + "cafe.github.com:443", + "github.com:443", + "objects.githubusercontent.com:443", + "raw.githubusercontent.com:443", + "release-assets.githubusercontent.com:443", + "releases.astral.sh:443", + "results-receiver.actions.githubusercontent.com:443", + "*.actions.githubusercontent.com:443", + "*.blob.core.windows.net:443", + "files.pythonhosted.org:443", + "pypi.org:443", + ), + "publish-product-gap": ( + "api.github.com:443", + "cafe.github.com:443", + "github.com:443", + "objects.githubusercontent.com:443", + "results-receiver.actions.githubusercontent.com:443", + "*.actions.githubusercontent.com:443", + "*.blob.core.windows.net:443", + ), +} + + +def _repository_root() -> Path: + """Return the Keyverse repository root from this test module.""" + return Path(__file__).resolve().parents[3] + + +def _workflow_source() -> str: + """Return the hourly product-development workflow as reviewed text.""" + return ( + _repository_root() + / ".github" + / "workflows" + / "hourly-product-development.yml" + ).read_text(encoding="utf-8") + + +def _workflow_document() -> dict[str, object]: + """Parse the hourly workflow into a mapping for structural assertions.""" + document = yaml.safe_load(_workflow_source()) + assert isinstance(document, dict) + return document + + +def _job(job_name: str) -> dict[str, object]: + """Return one named workflow job as a mapping.""" + jobs = _workflow_document().get("jobs") + assert isinstance(jobs, dict) + job = jobs.get(job_name) + assert isinstance(job, dict) + return job + + +def _steps(job_name: str) -> list[dict[str, object]]: + """Return mapping-valued steps for one named job.""" + steps = _job(job_name).get("steps") + assert isinstance(steps, list) + return [step for step in steps if isinstance(step, dict)] + + +def _step_by_id(job_name: str, step_id: str) -> dict[str, object]: + """Return the exact step carrying ``step_id`` in ``job_name``.""" + for step in _steps(job_name): + if step.get("id") == step_id: + return step + raise AssertionError(f"{job_name} has no step id {step_id}") + + +def _step_by_name(job_name: str, step_name: str) -> dict[str, object]: + """Return the exact step named ``step_name`` in ``job_name``.""" + for step in _steps(job_name): + if step.get("name") == step_name: + return step + raise AssertionError(f"{job_name} has no step named {step_name}") + + +def _harden_runner_endpoint_scalar(job_name: str) -> str: + """Return the serialized Harden Runner endpoint input for one workflow job.""" + for step in _steps(job_name): + action = step.get("uses") + if not isinstance(action, str) or not action.startswith( + "step-security/harden-runner@" + ): + continue + inputs = step.get("with") + assert isinstance(inputs, dict) + assert inputs.get("egress-policy") == "block" + endpoint_block = inputs.get("allowed-endpoints") + assert isinstance(endpoint_block, str) + return endpoint_block + raise AssertionError(f"{job_name} has no Harden Runner step") + + +def _harden_runner_endpoints(job_name: str) -> tuple[str, ...]: + """Return the exact ordered Harden Runner endpoint allowlist for a job.""" + return tuple(_harden_runner_endpoint_scalar(job_name).split()) + + +def test_github_api_jobs_use_exact_fail_closed_endpoint_sets() -> None: + """Every GitHub-API phase permits only its reviewed exact endpoint set.""" + for job_name, expected in EXPECTED_ENDPOINTS.items(): + actual = _harden_runner_endpoints(job_name) + assert actual == expected + assert "api.github.com:443.evil" not in actual + assert "*.github.com:443" not in actual + + +def test_harden_runner_endpoint_input_is_space_delimited_for_runtime() -> None: + """Harden Runner receives one folded, space-delimited endpoint scalar per job.""" + for job_name, expected in EXPECTED_ENDPOINTS.items(): + endpoint_scalar = _harden_runner_endpoint_scalar(job_name) + assert endpoint_scalar == " ".join(expected) + assert "\n" not in endpoint_scalar + + +def test_deterministic_repository_gates_precede_optional_model_credential() -> None: + """Queue, main, release evidence, and dry-run gates run before model access.""" + gate = _step_by_id("develop-product-gap", "gate") + gate_run = gate.get("run") + gate_env = gate.get("env") + assert isinstance(gate_run, str) + assert isinstance(gate_env, dict) + + ordered_markers = ( + "pulls?state=open&per_page=1", + "commits/${DEFAULT_BRANCH}", + "actions/runs?branch=${DEFAULT_BRANCH}", + "commits/${base_sha}/check-runs?per_page=100", + 'if [ "$DRY_RUN" = "true" ]; then', + ) + positions = tuple(gate_run.index(marker) for marker in ordered_markers) + assert positions == tuple(sorted(positions)) + assert "NIM_UPSTREAM_API_KEY" not in gate_env + assert "NIM_UPSTREAM_API_KEY" not in gate_run + assert "NVIDIA_NIM_API_KEY" not in gate_run + + +def test_github_inventory_transport_failures_are_not_false_green() -> None: + """GitHub inventory transport/shape failures terminate the gate unsuccessfully.""" + gate = _step_by_id("develop-product-gap", "gate") + gate_run = gate.get("run") + assert isinstance(gate_run, str) + + failure_messages = ( + "Unable to list open pull requests", + "Unable to interpret the open pull-request response", + "Unable to resolve the default-branch head", + "The default-branch head was malformed", + "Unable to read default-branch workflow evidence", + "Unable to read default-branch check evidence", + ) + for message in failure_messages: + marker = f'echo "::error::{message}' + start = gate_run.index(marker) + branch_tail = gate_run[start : start + 320] + assert "exit 1" in branch_tail + assert f"::warning::{message}" not in gate_run + + malformed_evidence_contracts = ( + ( + "if not isinstance(pages, list) or not required:", + "raise SystemExit(2)", + "workflow_evidence_status", + "Unable to interpret default-branch workflow evidence", + ), + ( + "if not isinstance(pages, list):", + "raise SystemExit(2)", + "check_evidence_status", + "Unable to interpret default-branch check evidence", + ), + ) + for parser_marker, malformed_exit, status_name, error_message in ( + malformed_evidence_contracts + ): + parser_start = gate_run.index(parser_marker) + parser_tail = gate_run[parser_start : parser_start + 180] + assert malformed_exit in parser_tail + assert f"{status_name}=$?" in gate_run + assert f"::error::{error_message}" in gate_run + + assert 'workflow_evidence_status=3' not in gate_run + assert 'check_evidence_status=3' not in gate_run + assert "Missing required default-branch workflow evidence" in gate_run + assert "Default branch has pending or unsuccessful required workflow evidence" in gate_run + assert "Missing latest default-branch check evidence" in gate_run + assert "Default branch has pending or unsuccessful latest check evidence" in gate_run + + +def test_default_branch_check_evidence_requires_success() -> None: + """Neutral or skipped default-main checks never qualify as healthy evidence.""" + gate = _step_by_id("develop-product-gap", "gate") + gate_run = gate.get("run") + assert isinstance(gate_run, str) + + accepted_start = gate_run.index("accepted =") + accepted_block = gate_run[accepted_start : accepted_start + 120] + assert 'accepted = {"success"}' in accepted_block + assert '"neutral"' not in accepted_block + assert '"skipped"' not in accepted_block + + +def test_model_fallback_budget_fits_outer_job_timeout() -> None: + """All sequential model candidates plus setup reserve fit the job deadline.""" + document = _workflow_document() + env = document.get("env") + assert isinstance(env, dict) + candidates = str(env.get("OPENCODE_MODEL_CANDIDATES", "")).split() + per_model_seconds = int(str(env.get("OPENCODE_RUN_TIMEOUT_SECONDS", "0"))) + timeout_minutes = int(str(_job("develop-product-gap").get("timeout-minutes", 0))) + + assert candidates + setup_and_packaging_reserve_seconds = 15 * 60 + assert timeout_minutes * 60 >= ( + len(candidates) * per_model_seconds + setup_and_packaging_reserve_seconds + ) + + +def test_nvidia_secret_is_materialized_only_by_broker() -> None: + """The raw NVIDIA secret exists only in the conditional loopback broker step.""" + secret_expression = "${{ secrets.NVIDIA_NIM_API_KEY }}" + materializing_steps: list[str] = [] + for step in _steps("develop-product-gap"): + env = step.get("env") + if not isinstance(env, dict) or secret_expression not in env.values(): + continue + name = step.get("name") + assert isinstance(name, str) + materializing_steps.append(name) + + assert materializing_steps == ["Start the loopback-only NIM credential broker"] + + +def test_nvidia_secret_is_required_only_on_the_model_backed_path() -> None: + """The NVIDIA secret is checked only after deterministic gates select development.""" + broker = _step_by_name( + "develop-product-gap", + "Start the loopback-only NIM credential broker", + ) + broker_env = broker.get("env") + broker_run = broker.get("run") + assert isinstance(broker_env, dict) + assert isinstance(broker_run, str) + + assert broker_env.get("NIM_UPSTREAM_API_KEY") == "${{ secrets.NVIDIA_NIM_API_KEY }}" + assert 'if [ -z "${NIM_UPSTREAM_API_KEY:-}" ]; then' in broker_run + assert "NVIDIA_NIM_API_KEY is required only for model-backed development" in broker_run + assert "exit 1" in broker_run