From 9a76665fa59afbbd0464259f91371d5704e44b1b Mon Sep 17 00:00:00 2001 From: Tim Stranske Date: Fri, 10 Jul 2026 10:49:02 -0500 Subject: [PATCH 1/9] Add immutable read-only model profile trial runner --- .../reusable-model-profile-trial.yml | 297 +++++++++++ scripts/model_profile_trial_contract.py | 475 ++++++++++++++++++ .../test_model_profile_trial_contract.py | 232 +++++++++ 3 files changed, 1004 insertions(+) create mode 100644 .github/workflows/reusable-model-profile-trial.yml create mode 100644 scripts/model_profile_trial_contract.py create mode 100644 tests/scripts/test_model_profile_trial_contract.py diff --git a/.github/workflows/reusable-model-profile-trial.yml b/.github/workflows/reusable-model-profile-trial.yml new file mode 100644 index 000000000..f211ee6e6 --- /dev/null +++ b/.github/workflows/reusable-model-profile-trial.yml @@ -0,0 +1,297 @@ +name: Reusable Model Profile Trial + +on: + workflow_call: + inputs: + trial_id: + description: Frozen Orchestrator trial identifier. + required: true + type: string + request_id: + description: Replay-stable request identifier from the trial bridge. + required: true + type: string + request_hash: + description: Replay hash of the exact bridge request. + required: true + type: string + trial_run_id: + description: Unique worker run identifier for this arm. + required: true + type: string + profile_id: + description: Exact Sol, Terra, or Luna execution profile ID. + required: true + type: string + packet_hash: + description: SHA-256 identity of the frozen common task packet. + required: true + type: string + launch_ordinal: + description: Randomized arm launch position, from 1 through 3. + required: true + type: number + expected_source_sha: + description: Full immutable Workflows source SHA for this arm. + required: true + type: string + secrets: + CODEX_AUTH_JSON: + description: JSON contents of Codex subscription auth. + required: true + +permissions: + contents: read + +concurrency: + group: model-profile-trial-${{ inputs.trial_id }}-${{ inputs.trial_run_id }} + cancel-in-progress: false + +jobs: + run-single-arm: + name: Read-only ${{ inputs.profile_id }} arm + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: read + steps: + - name: Checkout exact source + uses: actions/checkout@v7 + with: + ref: ${{ inputs.expected_source_sha }} + persist-credentials: false + + - name: Set up Node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6 + with: + node-version: "24" + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.14" + + - name: Validate frozen inputs and registry profile + id: profile + env: + PROFILE_ID: ${{ inputs.profile_id }} + EXPECTED_SOURCE_SHA: ${{ inputs.expected_source_sha }} + TRIAL_ID: ${{ inputs.trial_id }} + REQUEST_ID: ${{ inputs.request_id }} + REQUEST_HASH: ${{ inputs.request_hash }} + TRIAL_RUN_ID: ${{ inputs.trial_run_id }} + PACKET_HASH: ${{ inputs.packet_hash }} + LAUNCH_ORDINAL: ${{ inputs.launch_ordinal }} + run: | + set -euo pipefail + source_sha="$(git rev-parse HEAD)" + if [ "$source_sha" != "$EXPECTED_SOURCE_SHA" ]; then + echo "::error::Checkout SHA does not match expected_source_sha." + exit 1 + fi + if ! printf '%s' "$EXPECTED_SOURCE_SHA" | grep -Eq '^[0-9a-f]{40}$'; then + echo "::error::expected_source_sha must be a full lowercase Git SHA." + exit 1 + fi + if ! printf '%s' "$REQUEST_HASH" | grep -Eq '^sha256:[0-9a-f]{64}$'; then + echo "::error::request_hash is not a canonical SHA-256 identity." + exit 1 + fi + if ! printf '%s' "$PACKET_HASH" | grep -Eq '^sha256:[0-9a-f]{64}$'; then + echo "::error::packet_hash is not a canonical SHA-256 identity." + exit 1 + fi + case "$LAUNCH_ORDINAL" in + 1|2|3) ;; + *) echo "::error::launch_ordinal must be 1, 2, or 3."; exit 1 ;; + esac + for value in "$TRIAL_ID" "$REQUEST_ID" "$TRIAL_RUN_ID" "$PROFILE_ID"; do + if ! printf '%s' "$value" | grep -Eq '^[A-Za-z0-9._:/-]{1,200}$'; then + echo "::error::Trial identifiers must be bounded stable identifiers." + exit 1 + fi + done + + # Ruby's standard-library YAML parser avoids an unpinned Python + # dependency solely to read the authoritative agent registry. + ruby -ryaml -rjson -e \ + 'puts JSON.generate(YAML.safe_load(File.read(ARGV[0]), aliases: false))' \ + .github/agents/registry.yml > "$RUNNER_TEMP/agent-registry.json" + + python scripts/model_profile_trial_contract.py resolve \ + --registry-json "$RUNNER_TEMP/agent-registry.json" \ + --model-registry config/model_registry.json \ + --profile-id "$PROFILE_ID" \ + --output "$RUNNER_TEMP/resolved-profile.json" + echo "source-sha-before=$source_sha" >> "$GITHUB_OUTPUT" + + - name: Install exact Codex CLI + id: cli + run: | + set -euo pipefail + npm install -g "@openai/codex@0.144.1" + cli_version="$(codex --version | tr '\n' ' ' | sed -e 's/[[:space:]]*$//')" + if ! printf '%s' "$cli_version" | grep -Eq '(^|[[:space:]])0\.144\.1($|[[:space:]])'; then + echo "::error::Installed Codex CLI is not exactly 0.144.1." + exit 1 + fi + echo "cli-version=$cli_version" >> "$GITHUB_OUTPUT" + + - name: Configure isolated Codex subscription auth + env: + CODEX_AUTH_JSON: ${{ secrets.CODEX_AUTH_JSON }} + run: | + set -euo pipefail + if [ -z "$CODEX_AUTH_JSON" ]; then + echo "::error::CODEX_AUTH_JSON is required; this runner never refreshes auth." + exit 1 + fi + install -d -m 700 "$RUNNER_TEMP/codex-trial-home" + printf '%s' "$CODEX_AUTH_JSON" > "$RUNNER_TEMP/codex-trial-home/auth.json" + chmod 600 "$RUNNER_TEMP/codex-trial-home/auth.json" + + - name: Prepare fixed read-only canary packet + env: + PACKET_HASH: ${{ inputs.packet_hash }} + PROFILE_ID: ${{ inputs.profile_id }} + TRIAL_RUN_ID: ${{ inputs.trial_run_id }} + EXPECTED_SOURCE_SHA: ${{ inputs.expected_source_sha }} + run: | + set -euo pipefail + { + echo "You are one arm of a frozen read-only model-profile plumbing canary." + echo "Inspect this checkout only enough to confirm that it is readable." + echo "Do not edit, create, commit, push, comment, or refresh auth." + echo "Do not invoke another evaluator." + echo "The source checkout must remain byte-for-byte clean." + echo "Return one concise line containing all four exact values below:" + echo "packet_hash=$PACKET_HASH" + echo "source_sha=$EXPECTED_SOURCE_SHA" + echo "profile_id=$PROFILE_ID" + echo "trial_run_id=$TRIAL_RUN_ID" + } > "$RUNNER_TEMP/model-profile-trial-prompt.txt" + + - name: Run one Codex arm in read-only mode + id: codex + continue-on-error: true + env: + CODEX_HOME: ${{ runner.temp }}/codex-trial-home + REQUESTED_MODEL: ${{ steps.profile.outputs.model }} + run: | + set -uo pipefail + session_stream="$RUNNER_TEMP/codex-trial-session.jsonl" + stderr_log="$RUNNER_TEMP/codex-trial-stderr.log" + stderr_pipe="$RUNNER_TEMP/codex-trial-stderr.pipe" + final_message="$RUNNER_TEMP/codex-trial-final.txt" + : > "$session_stream" + : > "$stderr_log" + : > "$final_message" + rm -f "$stderr_pipe" + mkfifo "$stderr_pipe" + python scripts/model_profile_trial_contract.py bound-stream \ + --output "$stderr_log" --max-bytes 65536 \ + < "$stderr_pipe" & + stderr_capture_pid=$! + + codex exec \ + --json \ + --ignore-user-config \ + --strict-config \ + --skip-git-repo-check \ + --sandbox read-only \ + --model "$REQUESTED_MODEL" \ + -c 'model_reasoning_effort="high"' \ + --output-last-message "$final_message" \ + "$(cat "$RUNNER_TEMP/model-profile-trial-prompt.txt")" \ + > "$session_stream" \ + 2> "$stderr_pipe" + exit_code=$? + wait "$stderr_capture_pid" + rm -f "$stderr_pipe" + echo "exit-code=$exit_code" >> "$GITHUB_OUTPUT" + exit "$exit_code" + + - name: Verify source integrity + id: source_after + if: ${{ always() }} + env: + SOURCE_SHA_BEFORE: ${{ steps.profile.outputs.source-sha-before }} + EXPECTED_SOURCE_SHA: ${{ inputs.expected_source_sha }} + run: | + set -euo pipefail + source_sha_after="$(git rev-parse HEAD)" + source_clean=true + if [ "$SOURCE_SHA_BEFORE" != "$EXPECTED_SOURCE_SHA" ] || \ + [ "$source_sha_after" != "$EXPECTED_SOURCE_SHA" ] || \ + [ -n "$(git status --porcelain --untracked-files=all)" ]; then + source_clean=false + fi + echo "source-sha-after=$source_sha_after" >> "$GITHUB_OUTPUT" + echo "source-clean=$source_clean" >> "$GITHUB_OUTPUT" + + - name: Emit strict quarantine artifact + id: artifact + if: ${{ always() }} + env: + CODEX_HOME: ${{ runner.temp }}/codex-trial-home + CODEX_EXIT_CODE: ${{ steps.codex.outputs.exit-code || '1' }} + SOURCE_SHA_BEFORE: >- + ${{ steps.profile.outputs.source-sha-before || inputs.expected_source_sha }} + SOURCE_SHA_AFTER: >- + ${{ steps.source_after.outputs.source-sha-after || inputs.expected_source_sha }} + SOURCE_CLEAN: ${{ steps.source_after.outputs.source-clean || 'false' }} + REQUESTED_MODEL: ${{ steps.profile.outputs.model }} + RUNNER_VERSION: ${{ steps.profile.outputs.runner-ref }} + CLI_VERSION: ${{ steps.cli.outputs.cli-version || 'unknown' }} + TRIAL_ID: ${{ inputs.trial_id }} + REQUEST_ID: ${{ inputs.request_id }} + REQUEST_HASH: ${{ inputs.request_hash }} + TRIAL_RUN_ID: ${{ inputs.trial_run_id }} + PROFILE_ID: ${{ inputs.profile_id }} + LAUNCH_ORDINAL: ${{ inputs.launch_ordinal }} + PACKET_HASH: ${{ inputs.packet_hash }} + EXPECTED_SOURCE_SHA: ${{ inputs.expected_source_sha }} + run: | + set -euo pipefail + python scripts/model_profile_trial_contract.py artifact \ + --trial-id "$TRIAL_ID" \ + --request-id "$REQUEST_ID" \ + --request-hash "$REQUEST_HASH" \ + --trial-run-id "$TRIAL_RUN_ID" \ + --profile-id "$PROFILE_ID" \ + --launch-ordinal "$LAUNCH_ORDINAL" \ + --packet-hash "$PACKET_HASH" \ + --expected-source-sha "$EXPECTED_SOURCE_SHA" \ + --source-sha-before "$SOURCE_SHA_BEFORE" \ + --source-sha-after "$SOURCE_SHA_AFTER" \ + --requested-model "$REQUESTED_MODEL" \ + --runner-version "$RUNNER_VERSION" \ + --cli-version "$CLI_VERSION" \ + --session-stream "$RUNNER_TEMP/codex-trial-session.jsonl" \ + --codex-home "$CODEX_HOME" \ + --final-message "$RUNNER_TEMP/codex-trial-final.txt" \ + --exit-code "$CODEX_EXIT_CODE" \ + --source-clean "$SOURCE_CLEAN" \ + --output "$RUNNER_TEMP/model-profile-trial-attempt.json" + + - name: Upload unique trial attempt + if: ${{ always() && steps.artifact.outcome == 'success' }} + uses: actions/upload-artifact@v7 + with: + name: model-trial-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/model-profile-trial-attempt.json + if-no-files-found: error + retention-days: 30 + + - name: Enforce successful no-fallback identity canary + if: ${{ always() }} + env: + ARTIFACT_STATUS: ${{ steps.artifact.outputs.status }} + FALLBACK_REASON: ${{ steps.artifact.outputs.fallback-reason }} + CODEX_OUTCOME: ${{ steps.codex.outcome }} + run: | + set -euo pipefail + if [ "$CODEX_OUTCOME" != "success" ] || [ "$ARTIFACT_STATUS" != "success" ]; then + echo "::error::Trial arm failed closed: ${FALLBACK_REASON:-codex_cli_failed}" + exit 1 + fi diff --git a/scripts/model_profile_trial_contract.py b/scripts/model_profile_trial_contract.py new file mode 100644 index 000000000..e68d14871 --- /dev/null +++ b/scripts/model_profile_trial_contract.py @@ -0,0 +1,475 @@ +#!/usr/bin/env python3 +"""Validate and emit the read-only Codex model-profile trial contract. + +This helper is intentionally deterministic. It resolves a single registry +profile before provider execution and turns Codex's own persisted session +``turn_context`` into a strict, quarantine-only attempt artifact afterwards. +It does not call a provider, score an answer, write GitHub state, or infer +provider-resolved identity. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +from pathlib import Path +from typing import Any + + +ARTIFACT_SCHEMA = "workflows.model-profile-trial-result/v1" +IDENTITY_AUTHORITY = "workflows-read-only-trial-artifact/v1" +EXPECTED_CLI_VERSION = "0.144.1" +EXPECTED_PROFILES = { + "codex-5.6-sol-high": "gpt-5.6-sol", + "codex-5.6-terra-high": "gpt-5.6-terra", + "codex-5.6-luna-high": "gpt-5.6-luna", +} +SHA256_RE = re.compile(r"^sha256:[0-9a-f]{64}$") +SOURCE_SHA_RE = re.compile(r"^[0-9a-f]{40}$") +PINNED_RUNNER_RE = re.compile( + r"^stranske/Workflows/\.github/workflows/" + r"reusable-model-profile-trial\.yml@[0-9a-f]{40}$" +) +SAFE_ID_RE = re.compile(r"^[A-Za-z0-9._:/-]{1,200}$") + +ARTIFACT_FIELDS = { + "schema", + "version", + "trial_id", + "request_id", + "request_hash", + "run_id", + "profile_id", + "launch_ordinal", + "packet_hash", + "acknowledged", + "status", + "requested_model", + "selected_model", + "reported_model", + "provider_resolved_provider", + "provider_resolved_model", + "fallback_reason", + "runner_version", + "cli_version", + "thread_id", + "source_sha_before", + "source_sha_after", +} + + +class ContractError(ValueError): + """The frozen trial contract was missing, malformed, or inconsistent.""" + + +def _load_json(path: Path) -> dict[str, Any]: + try: + value = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: + raise ContractError(f"unable to read strict JSON: {path}") from exc + if not isinstance(value, dict): + raise ContractError(f"JSON root must be an object: {path}") + return value + + +def _require_safe_id(label: str, value: str) -> str: + text = str(value or "").strip() + if not SAFE_ID_RE.fullmatch(text): + raise ContractError(f"{label} must be a bounded stable identifier") + return text + + +def _require_hash(label: str, value: str) -> str: + text = str(value or "").strip() + if not SHA256_RE.fullmatch(text): + raise ContractError(f"{label} must be sha256:<64 lowercase hex>") + return text + + +def _require_source_sha(label: str, value: str) -> str: + text = str(value or "").strip() + if not SOURCE_SHA_RE.fullmatch(text): + raise ContractError(f"{label} must be a full lowercase Git SHA") + return text + + +def resolve_profile( + registry: dict[str, Any], model_registry: dict[str, Any], profile_id: str +) -> dict[str, Any]: + """Return the exact single-arm execution contract or fail closed.""" + profile_id = _require_safe_id("profile_id", profile_id) + expected_model = EXPECTED_PROFILES.get(profile_id) + if expected_model is None: + raise ContractError(f"profile_id is not a Sol/Terra/Luna trial profile: {profile_id}") + + profiles = registry.get("execution_profiles") + if not isinstance(profiles, dict) or not isinstance(profiles.get(profile_id), dict): + raise ContractError(f"execution profile missing from registry: {profile_id}") + profile = dict(profiles[profile_id]) + contract = registry.get("model_profile_trial_contract") + if not isinstance(contract, dict): + raise ContractError("model_profile_trial_contract missing from registry") + + expected_profile = { + "agent": "codex", + "model": expected_model, + "fallback_model": "gpt-5.5", + "runner": "reusable-model-profile-trial", + "capacity_pool": "codex-standard", + "safety": "read-only", + "lifecycle": "trial", + "reasoning_effort": "high", + "permission_mode": "read-only", + } + for field, expected in expected_profile.items(): + if profile.get(field) != expected: + raise ContractError( + f"execution profile {profile_id} {field} mismatch: " + f"expected {expected!r}" + ) + + expected_contract = { + "mode": "read-only", + "artifact_schema": ARTIFACT_SCHEMA, + "identity_authority": IDENTITY_AUTHORITY, + "cli_version": EXPECTED_CLI_VERSION, + "runtime_fallback_allowed": False, + "auxiliary_evaluator_allowed": False, + } + for field, expected in expected_contract.items(): + if contract.get(field) != expected: + raise ContractError(f"trial contract {field} mismatch: expected {expected!r}") + + runner_ref = str(contract.get("runner_ref") or "") + if not PINNED_RUNNER_RE.fullmatch(runner_ref): + raise ContractError("trial contract runner_ref is not an immutable reusable workflow ref") + if profile.get("runner_ref") != runner_ref: + raise ContractError(f"execution profile {profile_id} runner_ref drifted") + + models = model_registry.get("models") + if not isinstance(models, list): + raise ContractError("model registry missing models array") + matches = [row for row in models if isinstance(row, dict) and row.get("model_id") == expected_model] + if len(matches) != 1: + raise ContractError(f"model registry must contain one exact row for {expected_model}") + model = matches[0] + if model.get("provider") != "openai": + raise ContractError(f"model registry provider mismatch for {expected_model}") + if model.get("worker_profile") is not True or model.get("lifecycle") != "trial": + raise ContractError(f"model registry trial-worker metadata mismatch for {expected_model}") + + return { + "profile_id": profile_id, + "model": expected_model, + "reasoning_effort": "high", + "permission_mode": "read-only", + "capacity_pool": "codex-standard", + "runner_ref": runner_ref, + "identity_authority": IDENTITY_AUTHORITY, + } + + +def _iter_jsonl(path: Path): + if not path.is_file(): + return + for line_number, raw in enumerate(path.read_text(encoding="utf-8").splitlines(), 1): + if not raw.strip(): + continue + try: + value = json.loads(raw) + except json.JSONDecodeError as exc: + raise ContractError(f"invalid JSONL at {path}:{line_number}") from exc + if isinstance(value, dict): + yield value + + +def extract_thread_id(session_stream: Path) -> str | None: + values = { + str(event.get("thread_id")) + for event in _iter_jsonl(session_stream) or () + if event.get("type") == "thread.started" and event.get("thread_id") + } + if len(values) > 1: + raise ContractError("Codex stream contains multiple thread ids") + return next(iter(values), None) + + +def extract_reported_identity(codex_home: Path, thread_id: str | None) -> tuple[str | None, str | None]: + """Read model and effort only from the matching persisted turn_context.""" + if not thread_id: + return None, None + matching_files: list[Path] = [] + for path in sorted((codex_home / "sessions").glob("**/*.jsonl")): + for event in _iter_jsonl(path) or (): + payload = event.get("payload") if event.get("type") == "session_meta" else None + if isinstance(payload, dict) and payload.get("id") == thread_id: + matching_files.append(path) + break + if len(matching_files) != 1: + return None, None + + models: set[str] = set() + efforts: set[str] = set() + for event in _iter_jsonl(matching_files[0]) or (): + if event.get("type") != "turn_context" or not isinstance(event.get("payload"), dict): + continue + payload = event["payload"] + if payload.get("model"): + models.add(str(payload["model"])) + effort = payload.get("effort") + if not effort: + settings = ((payload.get("collaboration_mode") or {}).get("settings") or {}) + effort = settings.get("reasoning_effort") + if effort: + efforts.add(str(effort)) + if len(models) != 1 or len(efforts) != 1: + return None, None + return next(iter(models)), next(iter(efforts)) + + +def _cli_version_number(cli_version: str) -> str | None: + match = re.search(r"(?:^|\s)(\d+\.\d+\.\d+)(?:\s|$)", str(cli_version or "")) + return match.group(1) if match else None + + +def build_artifact( + *, + trial_id: str, + request_id: str, + request_hash: str, + trial_run_id: str, + profile_id: str, + launch_ordinal: int, + packet_hash: str, + expected_source_sha: str, + source_sha_before: str, + source_sha_after: str, + requested_model: str, + runner_version: str, + cli_version: str, + session_stream: Path, + codex_home: Path, + final_message: Path, + exit_code: int, + source_clean: bool, +) -> dict[str, Any]: + """Build one strict attempt artifact, including failed canaries.""" + trial_id = _require_safe_id("trial_id", trial_id) + request_id = _require_safe_id("request_id", request_id) + trial_run_id = _require_safe_id("trial_run_id", trial_run_id) + profile_id = _require_safe_id("profile_id", profile_id) + request_hash = _require_hash("request_hash", request_hash) + packet_hash = _require_hash("packet_hash", packet_hash) + expected_source_sha = _require_source_sha("expected_source_sha", expected_source_sha) + source_sha_before = _require_source_sha("source_sha_before", source_sha_before) + source_sha_after = _require_source_sha("source_sha_after", source_sha_after) + if profile_id not in EXPECTED_PROFILES or requested_model != EXPECTED_PROFILES[profile_id]: + raise ContractError("requested model does not match the exact profile") + if not 1 <= int(launch_ordinal) <= 3: + raise ContractError("launch_ordinal must be between 1 and 3") + if not PINNED_RUNNER_RE.fullmatch(str(runner_version or "")): + raise ContractError("runner_version is not the pinned reusable trial workflow") + + identity_parse_failed = False + try: + thread_id = extract_thread_id(session_stream) + reported_model, reported_effort = extract_reported_identity(codex_home, thread_id) + except (ContractError, OSError, UnicodeDecodeError): + # Provider/CLI failures may leave a partial stream. Preserve a strict + # failure artifact instead of losing the attempt to parser noise. + identity_parse_failed = True + thread_id = None + reported_model = None + reported_effort = None + final_text = final_message.read_text(encoding="utf-8") if final_message.is_file() else "" + acknowledgement_tokens = ( + f"packet_hash={packet_hash}", + f"source_sha={expected_source_sha}", + f"profile_id={profile_id}", + f"trial_run_id={trial_run_id}", + ) + acknowledged = all(token in final_text for token in acknowledgement_tokens) + + failures: list[str] = [] + if exit_code != 0: + failures.append("codex_cli_failed") + if identity_parse_failed: + failures.append("session_identity_invalid") + if _cli_version_number(cli_version) != EXPECTED_CLI_VERSION: + failures.append("cli_version_mismatch") + if source_sha_before != expected_source_sha: + failures.append("source_sha_before_mismatch") + if source_sha_after != expected_source_sha or source_sha_after != source_sha_before: + failures.append("source_sha_changed") + if not source_clean: + failures.append("source_tree_changed") + if not acknowledged: + failures.append("packet_not_acknowledged") + if not thread_id: + failures.append("thread_id_missing") + if not reported_model: + failures.append("reported_model_missing") + elif reported_model != requested_model: + failures.append("reported_model_mismatch") + if not reported_effort: + failures.append("reported_reasoning_effort_missing") + elif reported_effort != "high": + failures.append("reported_reasoning_effort_mismatch") + + fallback_reason = failures[0] if failures else None + artifact = { + "schema": ARTIFACT_SCHEMA, + "version": 1, + "trial_id": trial_id, + "request_id": request_id, + "request_hash": request_hash, + "run_id": trial_run_id, + "profile_id": profile_id, + "launch_ordinal": int(launch_ordinal), + "packet_hash": packet_hash, + "acknowledged": acknowledged, + "status": "failed" if failures else "success", + "requested_model": requested_model, + "selected_model": requested_model, + "reported_model": reported_model, + "provider_resolved_provider": None, + "provider_resolved_model": None, + "fallback_reason": fallback_reason, + "runner_version": runner_version, + "cli_version": cli_version, + "thread_id": thread_id, + "source_sha_before": source_sha_before, + "source_sha_after": source_sha_after, + } + if set(artifact) != ARTIFACT_FIELDS: + raise AssertionError("strict trial artifact schema drifted") + return artifact + + +def _write_github_output(values: dict[str, Any]) -> None: + path = os.environ.get("GITHUB_OUTPUT") + if not path: + return + with open(path, "a", encoding="utf-8") as handle: + for key, value in values.items(): + handle.write(f"{key}={value}\n") + + +def _resolve_command(args: argparse.Namespace) -> int: + registry = _load_json(Path(args.registry_json)) + models = _load_json(Path(args.model_registry)) + resolved = resolve_profile(registry, models, args.profile_id) + Path(args.output).write_text(json.dumps(resolved, indent=2, sort_keys=True) + "\n") + _write_github_output({key.replace("_", "-"): value for key, value in resolved.items()}) + return 0 + + +def _artifact_command(args: argparse.Namespace) -> int: + artifact = build_artifact( + trial_id=args.trial_id, + request_id=args.request_id, + request_hash=args.request_hash, + trial_run_id=args.trial_run_id, + profile_id=args.profile_id, + launch_ordinal=args.launch_ordinal, + packet_hash=args.packet_hash, + expected_source_sha=args.expected_source_sha, + source_sha_before=args.source_sha_before, + source_sha_after=args.source_sha_after, + requested_model=args.requested_model, + runner_version=args.runner_version, + cli_version=args.cli_version, + session_stream=Path(args.session_stream), + codex_home=Path(args.codex_home), + final_message=Path(args.final_message), + exit_code=args.exit_code, + source_clean=args.source_clean == "true", + ) + output = Path(args.output) + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(json.dumps(artifact, indent=2, sort_keys=True) + "\n") + _write_github_output( + { + "status": artifact["status"], + "fallback-reason": artifact["fallback_reason"] or "", + "thread-id": artifact["thread_id"] or "", + "reported-model": artifact["reported_model"] or "", + } + ) + return 0 + + +def _bound_stream_command(args: argparse.Namespace) -> int: + """Drain stdin while retaining only a bounded diagnostic prefix.""" + output = Path(args.output) + output.parent.mkdir(parents=True, exist_ok=True) + remaining = int(args.max_bytes) + with output.open("wb") as handle: + while True: + chunk = os.sys.stdin.buffer.read(8192) + if not chunk: + break + if remaining > 0: + retained = chunk[:remaining] + handle.write(retained) + remaining -= len(retained) + return 0 + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers(dest="command", required=True) + + resolve = subparsers.add_parser("resolve") + resolve.add_argument("--registry-json", required=True) + resolve.add_argument("--model-registry", required=True) + resolve.add_argument("--profile-id", required=True) + resolve.add_argument("--output", required=True) + resolve.set_defaults(func=_resolve_command) + + artifact = subparsers.add_parser("artifact") + for name in ( + "trial-id", + "request-id", + "request-hash", + "trial-run-id", + "profile-id", + "packet-hash", + "expected-source-sha", + "source-sha-before", + "source-sha-after", + "requested-model", + "runner-version", + "cli-version", + "session-stream", + "codex-home", + "final-message", + "source-clean", + "output", + ): + artifact.add_argument(f"--{name}", required=True) + artifact.add_argument("--launch-ordinal", required=True, type=int) + artifact.add_argument("--exit-code", required=True, type=int) + artifact.set_defaults(func=_artifact_command) + + bound_stream = subparsers.add_parser("bound-stream") + bound_stream.add_argument("--output", required=True) + bound_stream.add_argument("--max-bytes", type=int, default=65536) + bound_stream.set_defaults(func=_bound_stream_command) + return parser + + +def main() -> int: + args = _parser().parse_args() + try: + return int(args.func(args)) + except ContractError as exc: + print(f"model-profile-trial contract error: {exc}", file=os.sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/scripts/test_model_profile_trial_contract.py b/tests/scripts/test_model_profile_trial_contract.py new file mode 100644 index 000000000..216b9ee70 --- /dev/null +++ b/tests/scripts/test_model_profile_trial_contract.py @@ -0,0 +1,232 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from scripts import model_profile_trial_contract as contract + + +PINNED_REF = ( + "stranske/Workflows/.github/workflows/" + "reusable-model-profile-trial.yml@" + "1" * 40 +) + + +def _registries(): + registry = { + "model_profile_trial_contract": { + "mode": "read-only", + "artifact_schema": contract.ARTIFACT_SCHEMA, + "identity_authority": contract.IDENTITY_AUTHORITY, + "runner_ref": PINNED_REF, + "cli_version": contract.EXPECTED_CLI_VERSION, + "runtime_fallback_allowed": False, + "auxiliary_evaluator_allowed": False, + }, + "execution_profiles": {}, + } + models = {"models": []} + for profile_id, model in contract.EXPECTED_PROFILES.items(): + registry["execution_profiles"][profile_id] = { + "agent": "codex", + "model": model, + "fallback_model": "gpt-5.5", + "runner": "reusable-model-profile-trial", + "runner_ref": PINNED_REF, + "capacity_pool": "codex-standard", + "safety": "read-only", + "lifecycle": "trial", + "reasoning_effort": "high", + "permission_mode": "read-only", + } + models["models"].append( + { + "model_id": model, + "provider": "openai", + "worker_profile": True, + "lifecycle": "trial", + } + ) + return registry, models + + +def _session(tmp_path: Path, *, model: str = "gpt-5.6-sol", effort: str = "high"): + tmp_path.mkdir(parents=True, exist_ok=True) + thread_id = "019f-trial-thread" + stream = tmp_path / "stream.jsonl" + stream.write_text(json.dumps({"type": "thread.started", "thread_id": thread_id}) + "\n") + codex_home = tmp_path / "codex-home" + rollout = codex_home / "sessions" / "2026" / "07" / "10" / "rollout.jsonl" + rollout.parent.mkdir(parents=True) + rollout.write_text( + "\n".join( + [ + json.dumps({"type": "session_meta", "payload": {"id": thread_id}}), + json.dumps( + { + "type": "turn_context", + "payload": {"model": model, "effort": effort}, + } + ), + ] + ) + + "\n" + ) + return stream, codex_home, thread_id + + +def _artifact(tmp_path: Path, **overrides): + packet_hash = "sha256:" + "a" * 64 + source_sha = "b" * 40 + stream, codex_home, _thread_id = _session( + tmp_path, + model=overrides.pop("reported_model", "gpt-5.6-sol"), + effort=overrides.pop("reported_effort", "high"), + ) + final_message = tmp_path / "final.txt" + final_message.write_text( + f"packet_hash={packet_hash} source_sha={source_sha} " + "profile_id=codex-5.6-sol-high trial_run_id=trial-run:one\n" + ) + values = { + "trial_id": "trial:canary", + "request_id": "trial-request:one", + "request_hash": "sha256:" + "c" * 64, + "trial_run_id": "trial-run:one", + "profile_id": "codex-5.6-sol-high", + "launch_ordinal": 2, + "packet_hash": packet_hash, + "expected_source_sha": source_sha, + "source_sha_before": source_sha, + "source_sha_after": source_sha, + "requested_model": "gpt-5.6-sol", + "runner_version": PINNED_REF, + "cli_version": "codex-cli 0.144.1", + "session_stream": stream, + "codex_home": codex_home, + "final_message": final_message, + "exit_code": 0, + "source_clean": True, + } + values.update(overrides) + return contract.build_artifact(**values) + + +def test_resolve_profile_requires_exact_read_only_pinned_contract(): + registry, models = _registries() + resolved = contract.resolve_profile(registry, models, "codex-5.6-terra-high") + assert resolved == { + "profile_id": "codex-5.6-terra-high", + "model": "gpt-5.6-terra", + "reasoning_effort": "high", + "permission_mode": "read-only", + "capacity_pool": "codex-standard", + "runner_ref": PINNED_REF, + "identity_authority": contract.IDENTITY_AUTHORITY, + } + + registry["execution_profiles"]["codex-5.6-terra-high"]["permission_mode"] = ( + "workspace-write" + ) + with pytest.raises(contract.ContractError, match="permission_mode mismatch"): + contract.resolve_profile(registry, models, "codex-5.6-terra-high") + + +def test_success_artifact_uses_session_turn_context_and_keeps_provider_null(tmp_path): + artifact = _artifact(tmp_path) + assert set(artifact) == contract.ARTIFACT_FIELDS + assert artifact["status"] == "success" + assert artifact["reported_model"] == "gpt-5.6-sol" + assert artifact["provider_resolved_provider"] is None + assert artifact["provider_resolved_model"] is None + assert artifact["fallback_reason"] is None + assert artifact["thread_id"] == "019f-trial-thread" + assert artifact["source_sha_before"] == artifact["source_sha_after"] + assert artifact["launch_ordinal"] == 2 + + +def test_artifact_fails_closed_on_model_effort_or_source_drift(tmp_path): + mismatch = _artifact(tmp_path / "model", reported_model="gpt-5.5") + assert mismatch["status"] == "failed" + assert mismatch["fallback_reason"] == "reported_model_mismatch" + + effort = _artifact(tmp_path / "effort", reported_effort="medium") + assert effort["status"] == "failed" + assert effort["fallback_reason"] == "reported_reasoning_effort_mismatch" + + source = _artifact(tmp_path / "source", source_sha_after="d" * 40) + assert source["status"] == "failed" + assert source["fallback_reason"] == "source_sha_changed" + + +def test_thread_identity_must_come_from_exact_matching_rollout(tmp_path): + artifact = _artifact(tmp_path) + assert artifact["reported_model"] == "gpt-5.6-sol" + rollout = next((tmp_path / "codex-home" / "sessions").glob("**/*.jsonl")) + rollout.write_text( + json.dumps({"type": "turn_context", "payload": {"model": "gpt-5.6-sol", "effort": "high"}}) + + "\n" + ) + packet_hash = "sha256:" + "a" * 64 + source_sha = "b" * 40 + final_message = tmp_path / "final.txt" + final_message.write_text( + f"packet_hash={packet_hash} source_sha={source_sha} " + "profile_id=codex-5.6-sol-high trial_run_id=trial-run:one\n" + ) + broken = contract.build_artifact( + trial_id="trial:canary", + request_id="trial-request:one", + request_hash="sha256:" + "c" * 64, + trial_run_id="trial-run:one", + profile_id="codex-5.6-sol-high", + launch_ordinal=1, + packet_hash=packet_hash, + expected_source_sha=source_sha, + source_sha_before=source_sha, + source_sha_after=source_sha, + requested_model="gpt-5.6-sol", + runner_version=PINNED_REF, + cli_version="codex-cli 0.144.1", + session_stream=tmp_path / "stream.jsonl", + codex_home=tmp_path / "codex-home", + final_message=final_message, + exit_code=0, + source_clean=True, + ) + assert broken["reported_model"] is None + assert broken["fallback_reason"] == "reported_model_missing" + + +def test_failed_cli_with_malformed_stream_still_emits_failure_artifact(tmp_path): + packet_hash = "sha256:" + "a" * 64 + source_sha = "b" * 40 + stream = tmp_path / "broken-stream.jsonl" + stream.write_text("not-json\n") + final_message = tmp_path / "final.txt" + final_message.write_text("") + artifact = contract.build_artifact( + trial_id="trial:canary", + request_id="trial-request:one", + request_hash="sha256:" + "c" * 64, + trial_run_id="trial-run:one", + profile_id="codex-5.6-sol-high", + launch_ordinal=1, + packet_hash=packet_hash, + expected_source_sha=source_sha, + source_sha_before=source_sha, + source_sha_after=source_sha, + requested_model="gpt-5.6-sol", + runner_version=PINNED_REF, + cli_version="codex-cli 0.144.1", + session_stream=stream, + codex_home=tmp_path / "codex-home", + final_message=final_message, + exit_code=1, + source_clean=True, + ) + assert artifact["status"] == "failed" + assert artifact["fallback_reason"] == "codex_cli_failed" + assert set(artifact) == contract.ARTIFACT_FIELDS From 42f63468e18f470dd1c5516237b2c5b5def284e1 Mon Sep 17 00:00:00 2001 From: Tim Stranske Date: Fri, 10 Jul 2026 10:49:44 -0500 Subject: [PATCH 2/9] Wire pinned Sol Terra Luna trial dispatch --- .github/agents/registry.yml | 38 +++++- .../scripts/__tests__/agent-registry.test.js | 4 +- .github/workflows/README.md | 1 + .../workflows/agents-model-profile-trial.yml | 65 ++++++++++ README.md | 5 +- docs/WORKFLOW_GUIDE.md | 15 ++- docs/ci/WORKFLOWS.md | 2 + docs/ci/WORKFLOW_SYSTEM.md | 9 +- scripts/validate_template_completeness.py | 1 + scripts/validate_workflow_yaml.py | 5 +- .../consumer-repo/.github/agents/registry.yml | 38 +++++- .../test_model_profile_trial_workflows.py | 113 ++++++++++++++++++ tests/workflows/test_workflow_naming.py | 2 + 13 files changed, 279 insertions(+), 19 deletions(-) create mode 100644 .github/workflows/agents-model-profile-trial.yml create mode 100644 tests/workflows/test_model_profile_trial_workflows.py diff --git a/.github/agents/registry.yml b/.github/agents/registry.yml index 40c7972e7..a0e7a0f82 100644 --- a/.github/agents/registry.yml +++ b/.github/agents/registry.yml @@ -5,6 +5,23 @@ default_agent: codex # Shared keepalive marker prefix (agent-agnostic) keepalive_marker_prefix: agent-keepalive +# Dedicated instrumentation-only contract for the Sol/Terra/Luna plumbing +# canary. The reusable workflow ref is replaced with its exact first-commit +# SHA before the dispatch shim is merged. It intentionally cannot fall back, +# mutate source, invoke an evaluator, or claim provider-resolved identity. +model_profile_trial_contract: + mode: read-only + artifact_schema: workflows.model-profile-trial-result/v1 + identity_authority: workflows-read-only-trial-artifact/v1 + runner_ref: stranske/Workflows/.github/workflows/reusable-model-profile-trial.yml@9a76665fa59afbbd0464259f91371d5704e44b1b + cli_version: 0.144.1 + runtime_fallback_allowed: false + auxiliary_evaluator_allowed: false + provider_resolved_identity: unavailable + capacity_pool_mapping: + orchestrator: codex-subscription + workflows: codex-standard + execution_profiles: codex-default: agent: codex @@ -26,26 +43,35 @@ execution_profiles: agent: codex model: gpt-5.6-sol fallback_model: gpt-5.5 - runner: reusable-codex-run + runner: reusable-model-profile-trial + runner_ref: stranske/Workflows/.github/workflows/reusable-model-profile-trial.yml@9a76665fa59afbbd0464259f91371d5704e44b1b capacity_pool: codex-standard - safety: standard + safety: read-only lifecycle: trial + reasoning_effort: high + permission_mode: read-only codex-5.6-terra-high: agent: codex model: gpt-5.6-terra fallback_model: gpt-5.5 - runner: reusable-codex-run + runner: reusable-model-profile-trial + runner_ref: stranske/Workflows/.github/workflows/reusable-model-profile-trial.yml@9a76665fa59afbbd0464259f91371d5704e44b1b capacity_pool: codex-standard - safety: standard + safety: read-only lifecycle: trial + reasoning_effort: high + permission_mode: read-only codex-5.6-luna-high: agent: codex model: gpt-5.6-luna fallback_model: gpt-5.5 - runner: reusable-codex-run + runner: reusable-model-profile-trial + runner_ref: stranske/Workflows/.github/workflows/reusable-model-profile-trial.yml@9a76665fa59afbbd0464259f91371d5704e44b1b capacity_pool: codex-standard - safety: standard + safety: read-only lifecycle: trial + reasoning_effort: high + permission_mode: read-only agents: codex: diff --git a/.github/scripts/__tests__/agent-registry.test.js b/.github/scripts/__tests__/agent-registry.test.js index 1505bee32..a7fd30603 100644 --- a/.github/scripts/__tests__/agent-registry.test.js +++ b/.github/scripts/__tests__/agent-registry.test.js @@ -238,9 +238,11 @@ test('resolveExecutionProfile exposes the explicit Sol Terra Luna trial profiles assert.equal(profile.agent, 'codex'); assert.equal(profile.model, model); assert.equal(profile.fallback_model, fallback); - assert.equal(profile.runner, 'reusable-codex-run'); + assert.equal(profile.runner, 'reusable-model-profile-trial'); assert.equal(profile.capacity_pool, 'codex-standard'); assert.equal(profile.lifecycle, 'trial'); + assert.equal(profile.reasoning_effort, 'high'); + assert.equal(profile.permission_mode, 'read-only'); } }); diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 261dcbfd4..023ccad7f 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -12,6 +12,7 @@ Core layers: - Minimal invariant CI (`pr-11-ci-smoke.yml`): lean push/PR workflow that installs the project once on Python 3.12, sanity-checks imports, and executes the invariant tests from Issue #3651 so regressions surface quickly. - Gate summary (`pr-00-gate.yml` post-CI jobs): integrated post-CI reporting that batches small hygiene fixes, posts Gate summaries, and manages trivial failure remediation using the composite autofix action. - Agents orchestration (`agents-70-orchestrator.yml` + `reusable-16-agents.yml`): single entry point for Codex readiness, bootstrap, diagnostics, and watchdog sweeps. Use the [Agent task issue template][agent-task-template] (auto-labels `agents` + `agent:codex`) to raise work for Codex; the issue bridge listens for `agent:codex` and hands issues to the orchestrator. Legacy consumer shims remain removed following Issue #2650. +- Model-profile trial (`agents-model-profile-trial.yml` + `reusable-model-profile-trial.yml`): Workflows-only manual transport for one immutable, read-only Sol/Terra/Luna instrumentation arm. It uploads quarantine-only identity telemetry and has no commit, push, comment, auth-refresh, evaluator, or provider-resolution path. - PR metadata management (`agents-pr-meta.yml`): serializes Codex activation commands and PR body decoration through dedicated jobs that share a concurrency group keyed by PR number. This prevents marker thrash while keeping activation dispatch responsive. - Agents intake + orchestration (`agents-63-issue-intake.yml`, `agents-70-orchestrator.yml`, `reusable-16-agents.yml`): unified entry point for ChatGPT topic imports and Codex readiness/bridge sweeps. Use the [Agent task issue template][agent-task-template] (auto-labels `agents` + `agent:codex`) to raise work for Codex; the intake workflow handles both label-triggered bridges and manual dispatch while keeping parsing and bridge logic centralised. - Codex belt automation (`agents-71-codex-belt-dispatcher.yml`, `agents-72-codex-belt-worker.yml`, `agents-73-codex-belt-conveyor.yml`): hands-off conveyor for labelled issues—dispatcher selects `agent:codex` + `status:ready` issues and prepares a `codex/issue-*` branch, worker opens or refreshes the PR with labels/assignees, and conveyor merges after Gate success before re-queuing the dispatcher. diff --git a/.github/workflows/agents-model-profile-trial.yml b/.github/workflows/agents-model-profile-trial.yml new file mode 100644 index 000000000..7f6a2997c --- /dev/null +++ b/.github/workflows/agents-model-profile-trial.yml @@ -0,0 +1,65 @@ +name: Agents Model Profile Trial + +on: + workflow_dispatch: + inputs: + trial_id: + description: Frozen Orchestrator trial identifier. + required: true + type: string + request_id: + description: Replay-stable request identifier from the trial bridge. + required: true + type: string + request_hash: + description: Replay hash of the exact bridge request. + required: true + type: string + trial_run_id: + description: Unique worker run identifier for this arm. + required: true + type: string + profile_id: + description: Exact Sol, Terra, or Luna execution profile ID. + required: true + type: choice + options: + - codex-5.6-sol-high + - codex-5.6-terra-high + - codex-5.6-luna-high + packet_hash: + description: SHA-256 identity of the frozen common task packet. + required: true + type: string + launch_ordinal: + description: Randomized arm launch position, from 1 through 3. + required: true + type: choice + options: + - "1" + - "2" + - "3" + expected_source_sha: + description: Full immutable Workflows source SHA for this arm. + required: true + type: string + +permissions: + contents: read + +jobs: + trial: + uses: stranske/Workflows/.github/workflows/reusable-model-profile-trial.yml@9a76665fa59afbbd0464259f91371d5704e44b1b + permissions: + contents: read + with: + trial_id: ${{ inputs.trial_id }} + request_id: ${{ inputs.request_id }} + request_hash: ${{ inputs.request_hash }} + trial_run_id: ${{ inputs.trial_run_id }} + profile_id: ${{ inputs.profile_id }} + packet_hash: ${{ inputs.packet_hash }} + launch_ordinal: ${{ fromJSON(inputs.launch_ordinal) }} + expected_source_sha: ${{ inputs.expected_source_sha }} + secrets: + CODEX_AUTH_JSON: ${{ secrets.CODEX_AUTH_JSON }} diff --git a/README.md b/README.md index 622098adb..9b3e09e2d 100644 --- a/README.md +++ b/README.md @@ -87,7 +87,7 @@ After PR merge, applying a `verify:*` label (typically `verify:evaluate` via aut ### Reusable Workflows (.github/workflows) - CI: `reusable-10-ci-python.yml`, `reusable-11-ci-node.yml`, `reusable-12-ci-docker.yml`, `reusable-13-cross-repo-smoke.yml` -- Agent automation: `reusable-16-agents.yml`, `reusable-18-autofix.yml`, `reusable-20-pr-meta.yml`, `reusable-codex-run.yml`, `reusable-claude-run.yml`, `reusable-cursor-run.yml`, `reusable-gemini-run.yml` +- Agent automation: `reusable-16-agents.yml`, `reusable-18-autofix.yml`, `reusable-20-pr-meta.yml`, `reusable-codex-run.yml`, `reusable-model-profile-trial.yml`, `reusable-claude-run.yml`, `reusable-cursor-run.yml`, `reusable-gemini-run.yml` - Agent helpers: `reusable-agents-issue-bridge.yml`, `reusable-agents-pr-health.yml`, `reusable-agents-verifier.yml`, `reusable-bot-comment-handler.yml`, `reusable-pr-context.yml` - Conformance: `reusable-backplane-conformance.yml` - Orchestration: `reusable-70-orchestrator-init.yml`, `reusable-70-orchestrator-main.yml` @@ -97,6 +97,9 @@ After PR merge, applying a `verify:*` label (typically `verify:evaluate` via aut - Gate: `pr-00-gate.yml` (single PR-required check) - Maintenance & health: `maint-*`, `health-*` - Agents: `agents-*` (auto-pilot, verifier, keepalive, issue-intake, pr-meta) +- Model-profile canary: `agents-model-profile-trial.yml` dispatches exactly one + pinned, read-only Sol/Terra/Luna arm and uploads quarantine-only identity + telemetry. It cannot commit, push, comment, refresh auth, or score the arm. ### Composite Actions (.github/actions) diff --git a/docs/WORKFLOW_GUIDE.md b/docs/WORKFLOW_GUIDE.md index e2ae97334..6c4dcfed0 100644 --- a/docs/WORKFLOW_GUIDE.md +++ b/docs/WORKFLOW_GUIDE.md @@ -20,8 +20,8 @@ operational detail for the kept set. | `pr-` | Pull-request CI wrappers | `pr-00-gate.yml`, `pr-11-ci-smoke.yml` | | `maint-` | Post-CI maintenance and self-tests | `maint-45-cosmetic-repair.yml`, `maint-46-post-ci.yml`, `maint-47-disable-legacy-workflows.yml`, `maint-50-tool-version-check.yml`, `maint-52-validate-workflows.yml`, `maint-60-release.yml`, `maint-61-release-please.yml`, `maint-coverage-guard.yml` | | `health-` | Repository health & policy checks | `health-40-sweep.yml`, `health-40-repo-selfcheck.yml`, `health-41-repo-health.yml`, `health-42-actionlint.yml`, `health-43-ci-signature-guard.yml`, `health-44-gate-branch-protection.yml`, `health-50-security-scan.yml` | -| `agents-` | Agent orchestration entry points | `agents-63-issue-intake.yml`, `agents-64-verify-agent-assignment.yml`, `agents-70-orchestrator.yml`, `agents-71-codex-belt-dispatcher.yml`, `agents-72-codex-belt-worker.yml`, `agents-73-codex-belt-conveyor.yml`, `agents-80-pr-event-hub.yml`, `agents-81-gate-followups.yml`, `agents-guard.yml`, `agents-pr-meta.yml`, `agents-moderate-connector.yml`, `agents-keepalive-*.yml`, `agents-debug-issue-event.yml` | -| `reusable-` | Reusable composites invoked by other workflows | `reusable-10-ci-python.yml`, `reusable-12-ci-docker.yml`, `reusable-13-cross-repo-smoke.yml`, `reusable-16-agents.yml`, `reusable-18-autofix.yml`, `reusable-agents-issue-bridge.yml` | +| `agents-` | Agent orchestration entry points | `agents-63-issue-intake.yml`, `agents-64-verify-agent-assignment.yml`, `agents-70-orchestrator.yml`, `agents-71-codex-belt-dispatcher.yml`, `agents-72-codex-belt-worker.yml`, `agents-73-codex-belt-conveyor.yml`, `agents-model-profile-trial.yml`, `agents-80-pr-event-hub.yml`, `agents-81-gate-followups.yml`, `agents-guard.yml`, `agents-pr-meta.yml`, `agents-moderate-connector.yml`, `agents-keepalive-*.yml`, `agents-debug-issue-event.yml` | +| `reusable-` | Reusable composites invoked by other workflows | `reusable-10-ci-python.yml`, `reusable-12-ci-docker.yml`, `reusable-13-cross-repo-smoke.yml`, `reusable-16-agents.yml`, `reusable-18-autofix.yml`, `reusable-model-profile-trial.yml`, `reusable-agents-issue-bridge.yml` | | `selftest-` | Manual self-tests & experiments | `selftest-reusable-ci.yml` | | `autofix.yml` | CI autofix loop | `autofix.yml` | @@ -87,6 +87,11 @@ _Inline Gate helper_ - **`health-keepalive-e2e.yml`** — PR-only safeguard that runs through the keepalive orchestration helpers on every workflow change and, when labeled `e2e:codex-ping`, executes a minimal real Codex call to prove the reusable runner still works. The default orchestration leg now uses just the installation token (no extra GitHub App mints) to keep API load low. ### Agents & Issues +- **`agents-model-profile-trial.yml`** — Manual single-arm shim for the + instrumentation-only Sol/Terra/Luna plumbing trial. It forwards only frozen + trial/request/profile/hash/source inputs and subscription auth to the + immutable `reusable-model-profile-trial.yml` ref recorded in the registry. + It is not a keepalive, evaluator, merge, or learning lane. - **Consumer default entry points** — New consumer repos should install the template-managed pair `agents-80-pr-event-hub.yml` + `agents-81-gate-followups.yml` (along with `agents-verifier.yml`, `pr-00-gate.yml`, `AGENTS.md`, and `CLAUDE.md`). Treat `agents-pr-meta-v4.yml` as Workflows-local infrastructure, not a default consumer setup file. - **`agents-63-issue-intake.yml`** — Canonical front door for Codex issues. It normalizes ChatGPT export blobs / topic lists, dedupes and validates the resulting queue, optionally re-formats the new issues via LangChain, and drives the label-enforced issue bridge when `agent_bridge` mode is selected. The workflow now relies solely on the installation token + the shared API client (no ad-hoc GitHub App token mints), so manual reruns and workflow_call invocations stay lightweight while still enforcing the single-agent label contract before dispatching work to the belt. - **`agents-64-verify-agent-assignment.yml`** — Workflow-call validator that enforces the single `agent:*` label contract and confirms the assignee belongs to the approved automation roster. It now runs entirely on the default workflow token with the shared retry helper (no bespoke GitHub App mint), so verification hooks stay lightweight for orchestrator and issue-bridge callers. @@ -152,6 +157,12 @@ _Inline Gate helper_ - **`reusable-70-orchestrator-main.yml`** — Executes the orchestrator stages (keepalive gate, readiness, preflight, diagnostics, bootstrap, watchdog, keepalive sweep) using the outputs from the init workflow. Requires the GitHub App token when PATs aren’t available so keepalive writes still run under `agents-workflows-bot`. - **`reusable-bot-comment-handler.yml`** — Collects unresolved bot review comments, generates a per-agent prompt, and dispatches the appropriate runner. Prefers GitHub App client ID auth, records the selected App auth mode, keeps a warning-only legacy App ID fallback, and still falls back to `service_bot_pat` or `GITHUB_TOKEN` so consumer repos don’t have to configure extra secrets. - **`reusable-codex-run.yml`** — Codex execution wrapper that checks out the target PR branch, installs the pinned Codex CLI, runs the prompt, and pushes commits when the GitHub App token is available (otherwise drops to read-only mode using `GITHUB_TOKEN`). +- **`reusable-model-profile-trial.yml`** — Dedicated read-only Codex trial + worker. It validates the registry profile and immutable source SHA, installs + exactly Codex CLI 0.144.1, requests high reasoning once with no fallback, + reads the CLI-reported model from the matching persisted `turn_context`, and + uploads one strict `workflows.model-profile-trial-result/v1` artifact. + Provider-resolved identity remains null and the artifact is quarantine-only. - **`reusable-claude-run.yml`** — Claude CLI wrapper for keepalive/autofix scenarios. It mints the Workflows GitHub App token when available so branch pushes can mirror Codex parity, reuses the shared setup-api-client checkout, hardens the Workflows scripts checkout (detecting blobless-clone ghost dirs and reinstalling @octokit deps), and exposes inputs for prompt files, sandbox/safety flags, runtime caps, and appendices. Use it anywhere Claude needs to run the same branch-update loop as Codex. - **`reusable-agents-issue-bridge.yml`** — Shared issue→PR bridge used by `agents-63-issue-intake.yml`; reads `.github/agents/registry.yml` to honor each agent’s branch prefix + assignee list, applies invite/create modes, and now relies solely on the shared token chain (service bot / owner PAT / default token) without minting extra App tokens. - **`reusable-agents-verifier.yml`** — Post-merge verifier reusable that waits for CI (tracking every configured workflow until each has both started and completed), builds PR context, runs checkbox/evaluate/compare modes via the agent verifier stack, and opens follow-up issues when acceptance criteria fail. It mints the Workflows GitHub App token up front so both the caller repo and the Workflows scripts checkout succeed for private/same-repo callers, then falls back to `GITHUB_TOKEN` automatically when App secrets are absent. diff --git a/docs/ci/WORKFLOWS.md b/docs/ci/WORKFLOWS.md index 993fde0d7..5f3eeaf8c 100644 --- a/docs/ci/WORKFLOWS.md +++ b/docs/ci/WORKFLOWS.md @@ -100,6 +100,7 @@ The gate uses the shared `.github/scripts/detect-changes.js` helper to decide wh * [`reusable-pr-context.yml`](../../.github/workflows/reusable-pr-context.yml) fetches comprehensive PR context via a single GraphQL query (60-80% API reduction vs REST). Returns PR metadata, labels, files, reviews, comments, and CI status as job outputs for downstream consumption. * [`reusable-codex-run.yml`](../../.github/workflows/reusable-codex-run.yml) exposes a reusable Codex runner with prompt-file input, sandbox/safety defaults, artifact upload, and commit/push handling so keepalive, autofix, and verifier wrappers can share the same execution surface. +* [`reusable-model-profile-trial.yml`](../../.github/workflows/reusable-model-profile-trial.yml) is a separate immutable, read-only worker for one Sol/Terra/Luna instrumentation arm. It installs exactly Codex CLI 0.144.1, enforces high reasoning and the registry-backed `codex-standard` capacity mapping, and emits a strict quarantine artifact with requested/selected/CLI-reported identity plus source SHA before/after. It never commits, pushes, comments, refreshes auth, or invokes an evaluator; provider-resolved identity is explicitly null. * Optional exported Orchestrator skill context for opener/closer Codex lanes is documented in [`ORCHESTRATOR_SKILL_CONTEXT.md`](ORCHESTRATOR_SKILL_CONTEXT.md). Repos opt in via `.github/orchestrator_skill.json`; remote runs receive exported instructions only, not local Orchestrator runtime access. * [`reusable-claude-run.yml`](../../.github/workflows/reusable-claude-run.yml) exposes a reusable Claude runner that builds a prompt from a file + optional appendix, runs the configured Claude CLI, and publishes the output and summary as artifacts. * [`reusable-cursor-run.yml`](../../.github/workflows/reusable-cursor-run.yml) exposes a reusable Cursor runner that builds a prompt from a file + optional appendix, runs the `cursor-agent` CLI headlessly (`-p --force --output-format text`, authenticated via `CURSOR_API_KEY`), and publishes the output and summary as artifacts. Keepalive routes `agent:cursor` PRs here. @@ -135,6 +136,7 @@ The agent workflows coordinate Codex and chat orchestration across topics: Consumer default note: `agents-pr-meta-v4.yml` is a Workflows-repo service workflow. The default consumer installation uses the consumer-template `agents-80-pr-event-hub.yml` PR event hub and `agents-81-gate-followups.yml` workflows (plus `agents-verifier.yml`, `pr-00-gate.yml`, `AGENTS.md`, and `CLAUDE.md`). * [`agents-70-orchestrator.yml`](../../.github/workflows/agents-70-orchestrator.yml) is the thin dispatcher that triggers the orchestrator init and main phases. It calls [`reusable-70-orchestrator-init.yml`](../../.github/workflows/reusable-70-orchestrator-init.yml) for initialization (rate limit checks, token preflight, parameter resolution) and [`reusable-70-orchestrator-main.yml`](../../.github/workflows/reusable-70-orchestrator-main.yml) for the main keepalive and belt operations. +* [`agents-model-profile-trial.yml`](../../.github/workflows/agents-model-profile-trial.yml) is the manual single-arm dispatch shim for the frozen Sol/Terra/Luna plumbing canary. The shim calls only the exact reusable-workflow commit recorded in `.github/agents/registry.yml`; three-arm ordering and capacity reservation remain Orchestrator responsibilities. * Required permissions: `actions: write`, `contents: write`, and `pull-requests: write` at the workflow root so nested branch-sync and keepalive post-work steps can request their scopes without startup failure. * [`agents-keepalive-loop.yml`](../../.github/workflows/agents-keepalive-loop.yml) listens for Gate completion (and the optional `agent:codex` label event) to continue keepalive work in a GitHub-native loop: it inspects PR checklists/config, gates on Gate success, dispatches `reusable-codex-run` with the keepalive prompt, updates a single summary comment, and pauses with a `needs-human` label when tasks complete, limits are reached, or repeated failures occur. * [`agents-keepalive-sweep.yml`](../../.github/workflows/agents-keepalive-sweep.yml) is an hourly level-based resync (#2267): it re-dispatches `agents-keepalive-loop.yml` for every open `agent:*` PR so a silent zero-commit round (which emits no follow-up event) is re-evaluated instead of stalling. It makes no dispatch decision itself — the loop's fingerprint/debounce keeps unchanged PRs a no-op and the operator guardrails (`agents:paused` / `needs-human`) prevent re-dispatch of paused/blocked PRs. diff --git a/docs/ci/WORKFLOW_SYSTEM.md b/docs/ci/WORKFLOW_SYSTEM.md index 3e38da63c..0bf624e3c 100644 --- a/docs/ci/WORKFLOW_SYSTEM.md +++ b/docs/ci/WORKFLOW_SYSTEM.md @@ -385,7 +385,7 @@ fires where” without diving into the full tables: - **Health 45 Agents Guard.** [workflow history](https://github.com/stranske/Workflows/actions/workflows/agents-guard.yml). - **Error checking, linting, and testing topology** - **Primary workflows.** `reusable-10-ci-python.yml`, `reusable-12-ci-docker.yml`, - `reusable-13-cross-repo-smoke.yml`, `reusable-16-agents.yml`, `reusable-18-autofix.yml`, `reusable-20-pr-meta.yml`, `reusable-agents-issue-bridge.yml`, `reusable-agents-pr-health.yml`, `reusable-agents-verifier.yml`, `reusable-backplane-conformance.yml`, `reusable-bot-comment-handler.yml`, `reusable-claude-run.yml`, `reusable-codex-run.yml`, `reusable-cursor-run.yml`, `reusable-gemini-run.yml`, `reusable-pr-context.yml`, and `selftest-reusable-ci.yml`. + `reusable-13-cross-repo-smoke.yml`, `reusable-16-agents.yml`, `reusable-18-autofix.yml`, `reusable-20-pr-meta.yml`, `reusable-agents-issue-bridge.yml`, `reusable-agents-pr-health.yml`, `reusable-agents-verifier.yml`, `reusable-backplane-conformance.yml`, `reusable-bot-comment-handler.yml`, `reusable-claude-run.yml`, `reusable-codex-run.yml`, `reusable-model-profile-trial.yml`, `reusable-cursor-run.yml`, `reusable-gemini-run.yml`, `reusable-pr-context.yml`, and `selftest-reusable-ci.yml`. - **Triggers.** Invoked via `workflow_call` by Gate, Gate summary job, and manual reruns. `selftest-reusable-ci.yml` handles the nightly rehearsal (cron at 06:30 UTC) and manual publication modes via `workflow_dispatch`. @@ -470,7 +470,7 @@ status updates: | --- | --- | --- | --- | | PR checks | Every pull request event (including `pull_request_target` for fork visibility) | `pr-00-gate.yml` | Keep the default branch green by running the gating matrix before reviewers waste time. | | Maintenance & repo health | Daily/weekly schedules plus manual dispatch | Gate summary job in `pr-00-gate.yml`, `maint-46-post-ci.yml`, `maint-45-cosmetic-repair.yml`, `maint-62-integration-consumer.yml`, `maint-63-ensure-environments.yml`, `maint-65-sync-label-docs.yml`, `maint-66-monthly-audit.yml`, `health-4x-*.yml` | Scrub lingering CI debt, enforce branch protection, and surface drift before it breaks contributor workflows. | -| Issue / agents automation | Orchestrator dispatch (`workflow_dispatch`, `workflow_call`, `issues`), belt conveyor (`repository_dispatch`, `workflow_run`) | `agents-70-orchestrator.yml`, `agents-71-codex-belt-dispatcher.yml`, `agents-72-codex-belt-worker-dispatch.yml`, `agents-72-codex-belt-worker.yml`, `agents-73-codex-belt-conveyor.yml`, `agents-moderate-connector.yml`, `agents-autofix-dispatcher.yml`, `agents-autofix-loop.yml`, `agents-keepalive-loop.yml`, `agents-keepalive-sweep.yml`, `agents-keepalive-loop-reporter.yml`, `agents-keepalive-branch-sync.yml`, `agents-keepalive-dispatch-handler.yml`, `agents-74-pr-body-writer.yml`, `agents-63-*.yml`, `agents-64-pr-comment-commands.yml`, `agents-64-verify-agent-assignment.yml`, `agents-issue-optimizer.yml`, `agents-guard.yml` | Translate labelled issues into automated work while keeping the protected agents surface locked behind guardrails. | +| Issue / agents automation | Orchestrator dispatch (`workflow_dispatch`, `workflow_call`, `issues`), belt conveyor (`repository_dispatch`, `workflow_run`) | `agents-70-orchestrator.yml`, `agents-71-codex-belt-dispatcher.yml`, `agents-72-codex-belt-worker-dispatch.yml`, `agents-72-codex-belt-worker.yml`, `agents-73-codex-belt-conveyor.yml`, `agents-model-profile-trial.yml`, `agents-moderate-connector.yml`, `agents-autofix-dispatcher.yml`, `agents-autofix-loop.yml`, `agents-keepalive-loop.yml`, `agents-keepalive-sweep.yml`, `agents-keepalive-loop-reporter.yml`, `agents-keepalive-branch-sync.yml`, `agents-keepalive-dispatch-handler.yml`, `agents-74-pr-body-writer.yml`, `agents-63-*.yml`, `agents-64-pr-comment-commands.yml`, `agents-64-verify-agent-assignment.yml`, `agents-issue-optimizer.yml`, `agents-guard.yml` | Translate labelled issues into automated work while keeping the protected agents surface locked behind guardrails. | | Error checking, linting, and testing topology | Reusable fan-out invoked by Gate, Gate summary job, and manual triggers | `reusable-10-ci-python.yml`, `reusable-12-ci-docker.yml`, `reusable-13-cross-repo-smoke.yml`, `reusable-16-agents.yml`, `reusable-18-autofix.yml`, `reusable-20-pr-meta.yml`, `reusable-70-orchestrator-init.yml`, `reusable-70-orchestrator-main.yml`, `selftest-reusable-ci.yml` | Provide a single source of truth for lint/type/test/container jobs so every caller runs the same matrix with consistent tooling. | Keep this table handy when you are triaging automation: it confirms which workflows wake up on which events, the YAML files to inspect, and the safety purpose each bucket serves. @@ -662,6 +662,9 @@ Keep this table handy when you are triaging automation: it confirms which workfl `cross-repo-smoke.yml` plus `CROSS_REPO_SMOKE_*` repository variables. - **Reusable Agents** – `reusable-16-agents.yml` powers orchestrated dispatch. - **Reusable Autofix** – `reusable-18-autofix.yml` centralizes fixers for Gate summary job. +- **Reusable Model Profile Trial** – `reusable-model-profile-trial.yml` runs one + immutable read-only Sol/Terra/Luna plumbing arm and emits quarantine-only + identity telemetry; `agents-model-profile-trial.yml` is its manual shim. - **Selftest: Reusables** – `selftest-reusable-ci.yml` is the consolidated entry point. It runs nightly via cron (06:30 UTC) to rehearse the reusable matrix and accepts manual dispatches for summary/comment publication. Inputs: @@ -760,6 +763,7 @@ Keep this table handy when you are triaging automation: it confirms which workfl | **Agents Verifier** (`agents-verifier.yml`, agents bucket) | `pull_request` (`closed` → merged), `push` (`main`) | Build acceptance-context prompt (PR + linked issues), run Codex in verifier mode, and open a follow-up issue when the verdict is FAIL. | ⚪ Post-merge automation | [Agents verifier runs](https://github.com/stranske/Workflows/actions/workflows/agents-verifier.yml) | | **agents-weekly-metrics** (`agents-weekly-metrics.yml`, agents bucket) | `schedule` (weekly), `workflow_dispatch` | Aggregate agent metrics (keepalive, autofix, verifier) and generate markdown summary. | ⚪ Scheduled weekly | [Weekly metrics runs](https://github.com/stranske/Workflows/actions/workflows/agents-weekly-metrics.yml) | | **Agents 70 Orchestrator** (`agents-70-orchestrator.yml`, agents bucket) | `schedule` (`*/30 * * * *`), `workflow_dispatch` | Fan out consumer automation (readiness, diagnostics, keepalive sweep) and dispatch work; honours the `keepalive:paused` label and `keepalive_enabled` flag. | ⚪ Critical surface (triage immediately if red) | [Orchestrator runs](https://github.com/stranske/Workflows/actions/workflows/agents-70-orchestrator.yml) | +| **Agents Model Profile Trial** (`agents-model-profile-trial.yml`, agents bucket) | `workflow_dispatch` | Dispatch exactly one pinned, read-only Sol/Terra/Luna instrumentation arm with a frozen packet/source identity. | ⚪ Manual canary | [Model profile trial runs](https://github.com/stranske/Workflows/actions/workflows/agents-model-profile-trial.yml) | | **Agents 63 Issue Intake** (`agents-63-issue-intake.yml`, agents bucket) | `issues`, `workflow_call`, `workflow_dispatch` | Canonical front door for agent issue intake. Listens for `agent:codex` labels and services ChatGPT sync requests through the shared normalization pipeline. | ⚪ Critical surface (automation intake) | [Issue intake runs](https://github.com/stranske/Workflows/actions/workflows/agents-63-issue-intake.yml) | | **Agents 64 Verify Agent Assignment** (`agents-64-verify-agent-assignment.yml`, agents bucket) | `schedule`, `workflow_dispatch` | Audit orchestrated assignments and alert on drift. | ⚪ Scheduled | [Agents 64 audit history](https://github.com/stranske/Workflows/actions/workflows/agents-64-verify-agent-assignment.yml) | | **Agents Moderate Connector Comments** (`agents-moderate-connector.yml`, agents bucket) | `issue_comment` (`created`) | Guard connector-authored comments on PR threads using allow/deny lists and optional debug labelling. | ⚪ Event-driven | [Moderation workflow runs](https://github.com/stranske/Workflows/actions/workflows/agents-moderate-connector.yml) | @@ -769,6 +773,7 @@ Keep this table handy when you are triaging automation: it confirms which workfl | **Reusable Cross-Repo Smoke** (`reusable-13-cross-repo-smoke.yml`, error-checking bucket) | `workflow_call` | Check out a pinned dependency repo and run caller-provided cross-repo smoke commands. | ✅ When invoked | [Reusable cross-repo smoke runs](https://github.com/stranske/Workflows/actions/workflows/reusable-13-cross-repo-smoke.yml) | | **Reusable Agents** (`reusable-16-agents.yml`, error-checking bucket) | `workflow_call` | Power orchestrated dispatch. | ✅ When invoked | [Reusable Agents history](https://github.com/stranske/Workflows/actions/workflows/reusable-16-agents.yml) | | **Reusable Autofix** (`reusable-18-autofix.yml`, error-checking bucket) | `workflow_call` | Centralise formatter + fixer execution. | ✅ When invoked | [Reusable Autofix runs](https://github.com/stranske/Workflows/actions/workflows/reusable-18-autofix.yml) | +| **Reusable Model Profile Trial** (`reusable-model-profile-trial.yml`, agents bucket) | `workflow_call` | Execute one immutable read-only Codex trial arm and upload a strict quarantine artifact. | ✅ When invoked | [Reusable model profile trial runs](https://github.com/stranske/Workflows/actions/workflows/reusable-model-profile-trial.yml) | | **Selftest: Reusables** (`selftest-reusable-ci.yml`, error-checking bucket) | `schedule` (06:30 UTC), `workflow_dispatch` | Rehearse the reusable CI scenarios nightly and publish manual summaries or PR comments on demand. | ⚪ Scheduled/manual | [Self-test workflow history](https://github.com/stranske/Workflows/actions/workflows/selftest-reusable-ci.yml) | **PR body conflict guard.** `pr_body.md` is PR-specific. It must stay out of `main` and be ignored in consumer repos (`.gitignore` + `merge=ours` in `.gitattributes`). The Codex runner already `git reset`s `pr_body.md` before committing; if the file ever lands in `main`, rerun `maint-72-fix-pr-body-conflicts.yml` to delete it and reapply the ignore rule. diff --git a/scripts/validate_template_completeness.py b/scripts/validate_template_completeness.py index c56f405c9..6b2d6c02b 100755 --- a/scripts/validate_template_completeness.py +++ b/scripts/validate_template_completeness.py @@ -36,6 +36,7 @@ "health-76-codex-cli-freshness.yml", # Debug/testing workflows "agents-debug-issue-event.yml", + "agents-model-profile-trial.yml", # Workflows-owned remote trial transport # Internal dispatch handlers "agents-keepalive-branch-sync.yml", "agents-keepalive-dispatch-handler.yml", diff --git a/scripts/validate_workflow_yaml.py b/scripts/validate_workflow_yaml.py index 296b39f60..103a711a7 100755 --- a/scripts/validate_workflow_yaml.py +++ b/scripts/validate_workflow_yaml.py @@ -17,7 +17,10 @@ sys.exit(1) -LINE_LENGTH_EXEMPTIONS = ("stranske/Workflows/.github/actions/setup-api-client@",) +LINE_LENGTH_EXEMPTIONS = ( + "stranske/Workflows/.github/actions/setup-api-client@", + "stranske/Workflows/.github/workflows/reusable-model-profile-trial.yml@", +) def check_line_length(file_path: Path, max_length: int = 100) -> list[tuple[int, str]]: diff --git a/templates/consumer-repo/.github/agents/registry.yml b/templates/consumer-repo/.github/agents/registry.yml index 40c7972e7..a0e7a0f82 100644 --- a/templates/consumer-repo/.github/agents/registry.yml +++ b/templates/consumer-repo/.github/agents/registry.yml @@ -5,6 +5,23 @@ default_agent: codex # Shared keepalive marker prefix (agent-agnostic) keepalive_marker_prefix: agent-keepalive +# Dedicated instrumentation-only contract for the Sol/Terra/Luna plumbing +# canary. The reusable workflow ref is replaced with its exact first-commit +# SHA before the dispatch shim is merged. It intentionally cannot fall back, +# mutate source, invoke an evaluator, or claim provider-resolved identity. +model_profile_trial_contract: + mode: read-only + artifact_schema: workflows.model-profile-trial-result/v1 + identity_authority: workflows-read-only-trial-artifact/v1 + runner_ref: stranske/Workflows/.github/workflows/reusable-model-profile-trial.yml@9a76665fa59afbbd0464259f91371d5704e44b1b + cli_version: 0.144.1 + runtime_fallback_allowed: false + auxiliary_evaluator_allowed: false + provider_resolved_identity: unavailable + capacity_pool_mapping: + orchestrator: codex-subscription + workflows: codex-standard + execution_profiles: codex-default: agent: codex @@ -26,26 +43,35 @@ execution_profiles: agent: codex model: gpt-5.6-sol fallback_model: gpt-5.5 - runner: reusable-codex-run + runner: reusable-model-profile-trial + runner_ref: stranske/Workflows/.github/workflows/reusable-model-profile-trial.yml@9a76665fa59afbbd0464259f91371d5704e44b1b capacity_pool: codex-standard - safety: standard + safety: read-only lifecycle: trial + reasoning_effort: high + permission_mode: read-only codex-5.6-terra-high: agent: codex model: gpt-5.6-terra fallback_model: gpt-5.5 - runner: reusable-codex-run + runner: reusable-model-profile-trial + runner_ref: stranske/Workflows/.github/workflows/reusable-model-profile-trial.yml@9a76665fa59afbbd0464259f91371d5704e44b1b capacity_pool: codex-standard - safety: standard + safety: read-only lifecycle: trial + reasoning_effort: high + permission_mode: read-only codex-5.6-luna-high: agent: codex model: gpt-5.6-luna fallback_model: gpt-5.5 - runner: reusable-codex-run + runner: reusable-model-profile-trial + runner_ref: stranske/Workflows/.github/workflows/reusable-model-profile-trial.yml@9a76665fa59afbbd0464259f91371d5704e44b1b capacity_pool: codex-standard - safety: standard + safety: read-only lifecycle: trial + reasoning_effort: high + permission_mode: read-only agents: codex: diff --git a/tests/workflows/test_model_profile_trial_workflows.py b/tests/workflows/test_model_profile_trial_workflows.py new file mode 100644 index 000000000..889a45179 --- /dev/null +++ b/tests/workflows/test_model_profile_trial_workflows.py @@ -0,0 +1,113 @@ +from __future__ import annotations + +import re +from pathlib import Path + +import yaml + + +SHIM = Path(".github/workflows/agents-model-profile-trial.yml") +RUNNER = Path(".github/workflows/reusable-model-profile-trial.yml") +REGISTRY = Path(".github/agents/registry.yml") + + +def _workflow(path: Path): + data = yaml.safe_load(path.read_text(encoding="utf-8")) + # PyYAML 1.1 parses the unquoted GitHub Actions `on` key as True. + if True in data and "on" not in data: + data["on"] = data.pop(True) + return data + + +def test_dispatch_shim_is_single_arm_and_calls_only_pinned_reusable_runner(): + workflow = _workflow(SHIM) + assert set(workflow["on"]) == {"workflow_dispatch"} + inputs = workflow["on"]["workflow_dispatch"]["inputs"] + assert set(inputs) == { + "trial_id", + "request_id", + "request_hash", + "trial_run_id", + "profile_id", + "packet_hash", + "launch_ordinal", + "expected_source_sha", + } + assert all(value["required"] is True for value in inputs.values()) + assert inputs["profile_id"]["options"] == [ + "codex-5.6-sol-high", + "codex-5.6-terra-high", + "codex-5.6-luna-high", + ] + assert list(workflow["jobs"]) == ["trial"] + runner_ref = workflow["jobs"]["trial"]["uses"] + assert re.fullmatch( + r"stranske/Workflows/\.github/workflows/" + r"reusable-model-profile-trial\.yml@[0-9a-f]{40}", + runner_ref, + ) + assert workflow["permissions"] == {"contents": "read"} + assert "inherit" not in str(workflow["jobs"]["trial"].get("secrets")) + + +def test_reusable_runner_is_read_only_exact_cli_and_has_no_write_lane(): + workflow = _workflow(RUNNER) + assert set(workflow["on"]) == {"workflow_call"} + assert workflow["permissions"] == {"contents": "read"} + job = workflow["jobs"]["run-single-arm"] + assert job["permissions"] == {"contents": "read"} + source = RUNNER.read_text(encoding="utf-8") + assert '@openai/codex@0.144.1' in source + assert "--sandbox read-only" in source + assert "model_reasoning_effort=\"high\"" in source + assert "--ignore-user-config" in source + assert "persist-credentials: false" in source + assert "provider_resolved" not in source + forbidden = ( + "git commit", + "git push", + "gh pr", + "gh issue", + "create-pull-request", + "refresh-codex", + "OPENAI_API_KEY", + "CLAUDE_API", + ) + assert not [token for token in forbidden if token in source] + + +def test_runner_uploads_one_unique_attempt_and_enforces_source_integrity(): + workflow = _workflow(RUNNER) + steps = workflow["jobs"]["run-single-arm"]["steps"] + uploads = [step for step in steps if str(step.get("uses", "")).startswith("actions/upload-artifact@")] + assert len(uploads) == 1 + name = uploads[0]["with"]["name"] + assert "github.run_id" in name and "github.run_attempt" in name + source = RUNNER.read_text(encoding="utf-8") + assert "source-sha-before" in source + assert "source-sha-after" in source + assert "git status --porcelain --untracked-files=all" in source + assert "model_profile_trial_contract.py artifact" in source + + +def test_registry_trial_profiles_share_exact_pinned_read_only_contract(): + registry = yaml.safe_load(REGISTRY.read_text(encoding="utf-8")) + trial = registry["model_profile_trial_contract"] + assert trial["mode"] == "read-only" + assert trial["artifact_schema"] == "workflows.model-profile-trial-result/v1" + assert trial["identity_authority"] == "workflows-read-only-trial-artifact/v1" + assert trial["cli_version"] == "0.144.1" + assert trial["runtime_fallback_allowed"] is False + assert trial["auxiliary_evaluator_allowed"] is False + for profile_id in ( + "codex-5.6-sol-high", + "codex-5.6-terra-high", + "codex-5.6-luna-high", + ): + profile = registry["execution_profiles"][profile_id] + assert profile["runner"] == "reusable-model-profile-trial" + assert profile["runner_ref"] == trial["runner_ref"] + assert profile["capacity_pool"] == "codex-standard" + assert profile["reasoning_effort"] == "high" + assert profile["permission_mode"] == "read-only" + assert profile["safety"] == "read-only" diff --git a/tests/workflows/test_workflow_naming.py b/tests/workflows/test_workflow_naming.py index 528acdf22..8f355abe9 100644 --- a/tests/workflows/test_workflow_naming.py +++ b/tests/workflows/test_workflow_naming.py @@ -222,6 +222,7 @@ def test_workflow_display_names_are_unique(): EXPECTED_NAMES = { + "agents-model-profile-trial.yml": "Agents Model Profile Trial", "agents-autofix-loop.yml": "Agents Autofix Loop", "agents-autofix-dispatcher.yml": "Agents Autofix Dispatch", "agents-auto-label.yml": "Auto-Label Issues", @@ -319,6 +320,7 @@ def test_workflow_display_names_are_unique(): "reusable-cursor-run.yml": "Reusable Cursor Run", "reusable-gemini-run.yml": "Reusable Gemini Run", "reusable-codex-run.yml": "Reusable Codex Run", + "reusable-model-profile-trial.yml": "Reusable Model Profile Trial", "reusable-20-pr-meta.yml": "Reusable 20 PR Meta", "reusable-70-orchestrator-init.yml": "Agents 70 Init (Reusable)", "reusable-70-orchestrator-main.yml": "Agents 70 Main (Reusable)", From 292eb205b84edba2b081f57a9b130370036565ff Mon Sep 17 00:00:00 2001 From: Tim Stranske Date: Fri, 10 Jul 2026 10:51:48 -0500 Subject: [PATCH 3/9] Satisfy trial runner docs and formatting guards --- docs/ci/WORKFLOW_OUTPUTS.md | 1 + scripts/model_profile_trial_contract.py | 17 +++++++++-------- .../test_model_profile_trial_contract.py | 10 ++-------- .../test_model_profile_trial_workflows.py | 12 ++++++------ 4 files changed, 18 insertions(+), 22 deletions(-) diff --git a/docs/ci/WORKFLOW_OUTPUTS.md b/docs/ci/WORKFLOW_OUTPUTS.md index dd8cd8893..bf56fcb8e 100644 --- a/docs/ci/WORKFLOW_OUTPUTS.md +++ b/docs/ci/WORKFLOW_OUTPUTS.md @@ -166,6 +166,7 @@ The workflows below do not expose `workflow_call` outputs. They publish artifact - `reusable-agents-issue-bridge.yml` - `reusable-agents-pr-health.yml` - `reusable-agents-verifier.yml` +- `reusable-model-profile-trial.yml` ## Example usage in dependent jobs diff --git a/scripts/model_profile_trial_contract.py b/scripts/model_profile_trial_contract.py index e68d14871..40cde36b2 100644 --- a/scripts/model_profile_trial_contract.py +++ b/scripts/model_profile_trial_contract.py @@ -18,7 +18,6 @@ from pathlib import Path from typing import Any - ARTIFACT_SCHEMA = "workflows.model-profile-trial-result/v1" IDENTITY_AUTHORITY = "workflows-read-only-trial-artifact/v1" EXPECTED_CLI_VERSION = "0.144.1" @@ -30,8 +29,7 @@ SHA256_RE = re.compile(r"^sha256:[0-9a-f]{64}$") SOURCE_SHA_RE = re.compile(r"^[0-9a-f]{40}$") PINNED_RUNNER_RE = re.compile( - r"^stranske/Workflows/\.github/workflows/" - r"reusable-model-profile-trial\.yml@[0-9a-f]{40}$" + r"^stranske/Workflows/\.github/workflows/" r"reusable-model-profile-trial\.yml@[0-9a-f]{40}$" ) SAFE_ID_RE = re.compile(r"^[A-Za-z0-9._:/-]{1,200}$") @@ -127,8 +125,7 @@ def resolve_profile( for field, expected in expected_profile.items(): if profile.get(field) != expected: raise ContractError( - f"execution profile {profile_id} {field} mismatch: " - f"expected {expected!r}" + f"execution profile {profile_id} {field} mismatch: " f"expected {expected!r}" ) expected_contract = { @@ -152,7 +149,9 @@ def resolve_profile( models = model_registry.get("models") if not isinstance(models, list): raise ContractError("model registry missing models array") - matches = [row for row in models if isinstance(row, dict) and row.get("model_id") == expected_model] + matches = [ + row for row in models if isinstance(row, dict) and row.get("model_id") == expected_model + ] if len(matches) != 1: raise ContractError(f"model registry must contain one exact row for {expected_model}") model = matches[0] @@ -197,7 +196,9 @@ def extract_thread_id(session_stream: Path) -> str | None: return next(iter(values), None) -def extract_reported_identity(codex_home: Path, thread_id: str | None) -> tuple[str | None, str | None]: +def extract_reported_identity( + codex_home: Path, thread_id: str | None +) -> tuple[str | None, str | None]: """Read model and effort only from the matching persisted turn_context.""" if not thread_id: return None, None @@ -221,7 +222,7 @@ def extract_reported_identity(codex_home: Path, thread_id: str | None) -> tuple[ models.add(str(payload["model"])) effort = payload.get("effort") if not effort: - settings = ((payload.get("collaboration_mode") or {}).get("settings") or {}) + settings = (payload.get("collaboration_mode") or {}).get("settings") or {} effort = settings.get("reasoning_effort") if effort: efforts.add(str(effort)) diff --git a/tests/scripts/test_model_profile_trial_contract.py b/tests/scripts/test_model_profile_trial_contract.py index 216b9ee70..91e434691 100644 --- a/tests/scripts/test_model_profile_trial_contract.py +++ b/tests/scripts/test_model_profile_trial_contract.py @@ -7,11 +7,7 @@ from scripts import model_profile_trial_contract as contract - -PINNED_REF = ( - "stranske/Workflows/.github/workflows/" - "reusable-model-profile-trial.yml@" + "1" * 40 -) +PINNED_REF = "stranske/Workflows/.github/workflows/" "reusable-model-profile-trial.yml@" + "1" * 40 def _registries(): @@ -127,9 +123,7 @@ def test_resolve_profile_requires_exact_read_only_pinned_contract(): "identity_authority": contract.IDENTITY_AUTHORITY, } - registry["execution_profiles"]["codex-5.6-terra-high"]["permission_mode"] = ( - "workspace-write" - ) + registry["execution_profiles"]["codex-5.6-terra-high"]["permission_mode"] = "workspace-write" with pytest.raises(contract.ContractError, match="permission_mode mismatch"): contract.resolve_profile(registry, models, "codex-5.6-terra-high") diff --git a/tests/workflows/test_model_profile_trial_workflows.py b/tests/workflows/test_model_profile_trial_workflows.py index 889a45179..fddd47c08 100644 --- a/tests/workflows/test_model_profile_trial_workflows.py +++ b/tests/workflows/test_model_profile_trial_workflows.py @@ -5,7 +5,6 @@ import yaml - SHIM = Path(".github/workflows/agents-model-profile-trial.yml") RUNNER = Path(".github/workflows/reusable-model-profile-trial.yml") REGISTRY = Path(".github/agents/registry.yml") @@ -42,8 +41,7 @@ def test_dispatch_shim_is_single_arm_and_calls_only_pinned_reusable_runner(): assert list(workflow["jobs"]) == ["trial"] runner_ref = workflow["jobs"]["trial"]["uses"] assert re.fullmatch( - r"stranske/Workflows/\.github/workflows/" - r"reusable-model-profile-trial\.yml@[0-9a-f]{40}", + r"stranske/Workflows/\.github/workflows/" r"reusable-model-profile-trial\.yml@[0-9a-f]{40}", runner_ref, ) assert workflow["permissions"] == {"contents": "read"} @@ -57,9 +55,9 @@ def test_reusable_runner_is_read_only_exact_cli_and_has_no_write_lane(): job = workflow["jobs"]["run-single-arm"] assert job["permissions"] == {"contents": "read"} source = RUNNER.read_text(encoding="utf-8") - assert '@openai/codex@0.144.1' in source + assert "@openai/codex@0.144.1" in source assert "--sandbox read-only" in source - assert "model_reasoning_effort=\"high\"" in source + assert 'model_reasoning_effort="high"' in source assert "--ignore-user-config" in source assert "persist-credentials: false" in source assert "provider_resolved" not in source @@ -79,7 +77,9 @@ def test_reusable_runner_is_read_only_exact_cli_and_has_no_write_lane(): def test_runner_uploads_one_unique_attempt_and_enforces_source_integrity(): workflow = _workflow(RUNNER) steps = workflow["jobs"]["run-single-arm"]["steps"] - uploads = [step for step in steps if str(step.get("uses", "")).startswith("actions/upload-artifact@")] + uploads = [ + step for step in steps if str(step.get("uses", "")).startswith("actions/upload-artifact@") + ] assert len(uploads) == 1 name = uploads[0]["with"]["name"] assert "github.run_id" in name and "github.run_attempt" in name From d497362bc4b409895c3ddf7b5dd33e587a0414d2 Mon Sep 17 00:00:00 2001 From: Tim Stranske Date: Fri, 10 Jul 2026 10:53:03 -0500 Subject: [PATCH 4/9] Document exact Codex CLI install exception --- .github/workflows/reusable-model-profile-trial.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/reusable-model-profile-trial.yml b/.github/workflows/reusable-model-profile-trial.yml index f211ee6e6..76674b61b 100644 --- a/.github/workflows/reusable-model-profile-trial.yml +++ b/.github/workflows/reusable-model-profile-trial.yml @@ -129,6 +129,8 @@ jobs: id: cli run: | set -euo pipefail + # zizmor: ignore[adhoc-install] The subscription CLI has no supported lockfile runner; + # this canary pins one immutable npm version and fails unless codex reports it exactly. npm install -g "@openai/codex@0.144.1" cli_version="$(codex --version | tr '\n' ' ' | sed -e 's/[[:space:]]*$//')" if ! printf '%s' "$cli_version" | grep -Eq '(^|[[:space:]])0\.144\.1($|[[:space:]])'; then From eab411f83e0ca2596ba377e139e2894c8024fa84 Mon Sep 17 00:00:00 2001 From: Tim Stranske Date: Fri, 10 Jul 2026 10:53:23 -0500 Subject: [PATCH 5/9] Re-pin trial dispatch to reviewed runner --- .github/agents/registry.yml | 8 ++++---- .github/workflows/agents-model-profile-trial.yml | 2 +- templates/consumer-repo/.github/agents/registry.yml | 8 ++++---- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/agents/registry.yml b/.github/agents/registry.yml index a0e7a0f82..75533a5f5 100644 --- a/.github/agents/registry.yml +++ b/.github/agents/registry.yml @@ -13,7 +13,7 @@ model_profile_trial_contract: mode: read-only artifact_schema: workflows.model-profile-trial-result/v1 identity_authority: workflows-read-only-trial-artifact/v1 - runner_ref: stranske/Workflows/.github/workflows/reusable-model-profile-trial.yml@9a76665fa59afbbd0464259f91371d5704e44b1b + runner_ref: stranske/Workflows/.github/workflows/reusable-model-profile-trial.yml@d497362bc4b409895c3ddf7b5dd33e587a0414d2 cli_version: 0.144.1 runtime_fallback_allowed: false auxiliary_evaluator_allowed: false @@ -44,7 +44,7 @@ execution_profiles: model: gpt-5.6-sol fallback_model: gpt-5.5 runner: reusable-model-profile-trial - runner_ref: stranske/Workflows/.github/workflows/reusable-model-profile-trial.yml@9a76665fa59afbbd0464259f91371d5704e44b1b + runner_ref: stranske/Workflows/.github/workflows/reusable-model-profile-trial.yml@d497362bc4b409895c3ddf7b5dd33e587a0414d2 capacity_pool: codex-standard safety: read-only lifecycle: trial @@ -55,7 +55,7 @@ execution_profiles: model: gpt-5.6-terra fallback_model: gpt-5.5 runner: reusable-model-profile-trial - runner_ref: stranske/Workflows/.github/workflows/reusable-model-profile-trial.yml@9a76665fa59afbbd0464259f91371d5704e44b1b + runner_ref: stranske/Workflows/.github/workflows/reusable-model-profile-trial.yml@d497362bc4b409895c3ddf7b5dd33e587a0414d2 capacity_pool: codex-standard safety: read-only lifecycle: trial @@ -66,7 +66,7 @@ execution_profiles: model: gpt-5.6-luna fallback_model: gpt-5.5 runner: reusable-model-profile-trial - runner_ref: stranske/Workflows/.github/workflows/reusable-model-profile-trial.yml@9a76665fa59afbbd0464259f91371d5704e44b1b + runner_ref: stranske/Workflows/.github/workflows/reusable-model-profile-trial.yml@d497362bc4b409895c3ddf7b5dd33e587a0414d2 capacity_pool: codex-standard safety: read-only lifecycle: trial diff --git a/.github/workflows/agents-model-profile-trial.yml b/.github/workflows/agents-model-profile-trial.yml index 7f6a2997c..d59e78eeb 100644 --- a/.github/workflows/agents-model-profile-trial.yml +++ b/.github/workflows/agents-model-profile-trial.yml @@ -49,7 +49,7 @@ permissions: jobs: trial: - uses: stranske/Workflows/.github/workflows/reusable-model-profile-trial.yml@9a76665fa59afbbd0464259f91371d5704e44b1b + uses: stranske/Workflows/.github/workflows/reusable-model-profile-trial.yml@d497362bc4b409895c3ddf7b5dd33e587a0414d2 permissions: contents: read with: diff --git a/templates/consumer-repo/.github/agents/registry.yml b/templates/consumer-repo/.github/agents/registry.yml index a0e7a0f82..75533a5f5 100644 --- a/templates/consumer-repo/.github/agents/registry.yml +++ b/templates/consumer-repo/.github/agents/registry.yml @@ -13,7 +13,7 @@ model_profile_trial_contract: mode: read-only artifact_schema: workflows.model-profile-trial-result/v1 identity_authority: workflows-read-only-trial-artifact/v1 - runner_ref: stranske/Workflows/.github/workflows/reusable-model-profile-trial.yml@9a76665fa59afbbd0464259f91371d5704e44b1b + runner_ref: stranske/Workflows/.github/workflows/reusable-model-profile-trial.yml@d497362bc4b409895c3ddf7b5dd33e587a0414d2 cli_version: 0.144.1 runtime_fallback_allowed: false auxiliary_evaluator_allowed: false @@ -44,7 +44,7 @@ execution_profiles: model: gpt-5.6-sol fallback_model: gpt-5.5 runner: reusable-model-profile-trial - runner_ref: stranske/Workflows/.github/workflows/reusable-model-profile-trial.yml@9a76665fa59afbbd0464259f91371d5704e44b1b + runner_ref: stranske/Workflows/.github/workflows/reusable-model-profile-trial.yml@d497362bc4b409895c3ddf7b5dd33e587a0414d2 capacity_pool: codex-standard safety: read-only lifecycle: trial @@ -55,7 +55,7 @@ execution_profiles: model: gpt-5.6-terra fallback_model: gpt-5.5 runner: reusable-model-profile-trial - runner_ref: stranske/Workflows/.github/workflows/reusable-model-profile-trial.yml@9a76665fa59afbbd0464259f91371d5704e44b1b + runner_ref: stranske/Workflows/.github/workflows/reusable-model-profile-trial.yml@d497362bc4b409895c3ddf7b5dd33e587a0414d2 capacity_pool: codex-standard safety: read-only lifecycle: trial @@ -66,7 +66,7 @@ execution_profiles: model: gpt-5.6-luna fallback_model: gpt-5.5 runner: reusable-model-profile-trial - runner_ref: stranske/Workflows/.github/workflows/reusable-model-profile-trial.yml@9a76665fa59afbbd0464259f91371d5704e44b1b + runner_ref: stranske/Workflows/.github/workflows/reusable-model-profile-trial.yml@d497362bc4b409895c3ddf7b5dd33e587a0414d2 capacity_pool: codex-standard safety: read-only lifecycle: trial From 49cc9b470ab71a8dad9f16bbabd54019bfae3e3a Mon Sep 17 00:00:00 2001 From: Tim Stranske Date: Fri, 10 Jul 2026 10:54:58 -0500 Subject: [PATCH 6/9] Fix trial contract lint --- scripts/model_profile_trial_contract.py | 1 - tests/scripts/test_model_profile_trial_contract.py | 1 - 2 files changed, 2 deletions(-) diff --git a/scripts/model_profile_trial_contract.py b/scripts/model_profile_trial_contract.py index 40cde36b2..da8423e8b 100644 --- a/scripts/model_profile_trial_contract.py +++ b/scripts/model_profile_trial_contract.py @@ -11,7 +11,6 @@ from __future__ import annotations import argparse -import hashlib import json import os import re diff --git a/tests/scripts/test_model_profile_trial_contract.py b/tests/scripts/test_model_profile_trial_contract.py index 91e434691..b8833d896 100644 --- a/tests/scripts/test_model_profile_trial_contract.py +++ b/tests/scripts/test_model_profile_trial_contract.py @@ -4,7 +4,6 @@ from pathlib import Path import pytest - from scripts import model_profile_trial_contract as contract PINNED_REF = "stranske/Workflows/.github/workflows/" "reusable-model-profile-trial.yml@" + "1" * 40 From 822e323eefba3edb640e0bd9c922caec6fffee65 Mon Sep 17 00:00:00 2001 From: Tim Stranske Date: Fri, 10 Jul 2026 11:11:39 -0500 Subject: [PATCH 7/9] Harden immutable model profile trial boundary --- .../reusable-model-profile-trial.yml | 153 +++++++++++++--- scripts/model_profile_trial_contract.py | 168 +++++++++++++++++- 2 files changed, 287 insertions(+), 34 deletions(-) diff --git a/.github/workflows/reusable-model-profile-trial.yml b/.github/workflows/reusable-model-profile-trial.yml index 76674b61b..0d33c8e03 100644 --- a/.github/workflows/reusable-model-profile-trial.yml +++ b/.github/workflows/reusable-model-profile-trial.yml @@ -32,7 +32,11 @@ on: required: true type: number expected_source_sha: - description: Full immutable Workflows source SHA for this arm. + description: Full immutable Workflows main SHA for this arm. + required: true + type: string + runner_sha: + description: Immutable commit containing this reusable runner and contract helper. required: true type: string secrets: @@ -54,11 +58,53 @@ jobs: timeout-minutes: 30 permissions: contents: read + env: + PINNED_RUNNER_SHA: ${{ inputs.runner_sha }} + PINNED_RUNNER_REF: >- + stranske/Workflows/.github/workflows/reusable-model-profile-trial.yml@${{ inputs.runner_sha }} + ZERO_SOURCE_MANIFEST: sha256:0000000000000000000000000000000000000000000000000000000000000000 steps: - - name: Checkout exact source - uses: actions/checkout@v7 + - name: Checkout immutable trial runner + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + repository: stranske/Workflows + ref: ${{ inputs.runner_sha }} + path: runner-src + persist-credentials: false + + - name: Require pinned runner and current remote main + env: + EXPECTED_SOURCE_SHA: ${{ inputs.expected_source_sha }} + run: | + set -euo pipefail + if ! printf '%s' "$EXPECTED_SOURCE_SHA" | grep -Eq '^[0-9a-f]{40}$'; then + echo "::error::expected_source_sha must be a full lowercase Git SHA." + exit 1 + fi + runner_sha="$(git -C runner-src rev-parse HEAD)" + if ! printf '%s' "$PINNED_RUNNER_SHA" | grep -Eq '^[0-9a-f]{40}$'; then + echo "::error::runner_sha must be a full lowercase Git SHA." + exit 1 + fi + if [ "$runner_sha" != "$PINNED_RUNNER_SHA" ]; then + echo "::error::Runner checkout does not match its immutable runner commit." + exit 1 + fi + remote_main="$( + git ls-remote https://github.com/stranske/Workflows.git refs/heads/main | + awk 'NR == 1 { print $1 }' + )" + if [ -z "$remote_main" ] || [ "$remote_main" != "$EXPECTED_SOURCE_SHA" ]; then + echo "::error::expected_source_sha must equal current remote main before auth." + exit 1 + fi + + - name: Checkout exact trial source + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: + repository: stranske/Workflows ref: ${{ inputs.expected_source_sha }} + path: target-src persist-credentials: false - name: Set up Node @@ -67,7 +113,7 @@ jobs: node-version: "24" - name: Set up Python - uses: actions/setup-python@v6 + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 with: python-version: "3.14" @@ -84,13 +130,9 @@ jobs: LAUNCH_ORDINAL: ${{ inputs.launch_ordinal }} run: | set -euo pipefail - source_sha="$(git rev-parse HEAD)" + source_sha="$(git -C target-src rev-parse HEAD)" if [ "$source_sha" != "$EXPECTED_SOURCE_SHA" ]; then - echo "::error::Checkout SHA does not match expected_source_sha." - exit 1 - fi - if ! printf '%s' "$EXPECTED_SOURCE_SHA" | grep -Eq '^[0-9a-f]{40}$'; then - echo "::error::expected_source_sha must be a full lowercase Git SHA." + echo "::error::Target checkout SHA does not match expected_source_sha." exit 1 fi if ! printf '%s' "$REQUEST_HASH" | grep -Eq '^sha256:[0-9a-f]{64}$'; then @@ -112,19 +154,34 @@ jobs: fi done - # Ruby's standard-library YAML parser avoids an unpinned Python - # dependency solely to read the authoritative agent registry. + # The target registry is parsed as data. All executable helper code + # comes from the separately checked-out immutable runner commit. ruby -ryaml -rjson -e \ 'puts JSON.generate(YAML.safe_load(File.read(ARGV[0]), aliases: false))' \ - .github/agents/registry.yml > "$RUNNER_TEMP/agent-registry.json" - - python scripts/model_profile_trial_contract.py resolve \ + target-src/.github/agents/registry.yml > "$RUNNER_TEMP/agent-registry.json" + python runner-src/scripts/model_profile_trial_contract.py resolve \ --registry-json "$RUNNER_TEMP/agent-registry.json" \ - --model-registry config/model_registry.json \ + --model-registry target-src/config/model_registry.json \ --profile-id "$PROFILE_ID" \ --output "$RUNNER_TEMP/resolved-profile.json" + resolved_runner_ref="$( + python -c 'import json,sys; print(json.load(open(sys.argv[1]))["runner_ref"])' \ + "$RUNNER_TEMP/resolved-profile.json" + )" + if [ "$resolved_runner_ref" != "$PINNED_RUNNER_REF" ]; then + echo "::error::Target registry runner_ref does not match the executing runner." + exit 1 + fi echo "source-sha-before=$source_sha" >> "$GITHUB_OUTPUT" + - name: Snapshot bounded trial source before auth + id: manifest_before + run: | + set -euo pipefail + python runner-src/scripts/model_profile_trial_contract.py source-manifest \ + --root target-src \ + --output "$RUNNER_TEMP/source-manifest-before.json" + - name: Install exact Codex CLI id: cli run: | @@ -160,12 +217,12 @@ jobs: EXPECTED_SOURCE_SHA: ${{ inputs.expected_source_sha }} run: | set -euo pipefail + install -d "$RUNNER_TEMP/codex-workdir" { echo "You are one arm of a frozen read-only model-profile plumbing canary." - echo "Inspect this checkout only enough to confirm that it is readable." + echo "Do not use tools or inspect or execute the target checkout." echo "Do not edit, create, commit, push, comment, or refresh auth." echo "Do not invoke another evaluator." - echo "The source checkout must remain byte-for-byte clean." echo "Return one concise line containing all four exact values below:" echo "packet_hash=$PACKET_HASH" echo "source_sha=$EXPECTED_SOURCE_SHA" @@ -190,11 +247,12 @@ jobs: : > "$final_message" rm -f "$stderr_pipe" mkfifo "$stderr_pipe" - python scripts/model_profile_trial_contract.py bound-stream \ + python runner-src/scripts/model_profile_trial_contract.py bound-stream \ --output "$stderr_log" --max-bytes 65536 \ < "$stderr_pipe" & stderr_capture_pid=$! + cd "$RUNNER_TEMP/codex-workdir" codex exec \ --json \ --ignore-user-config \ @@ -220,12 +278,34 @@ jobs: SOURCE_SHA_BEFORE: ${{ steps.profile.outputs.source-sha-before }} EXPECTED_SOURCE_SHA: ${{ inputs.expected_source_sha }} run: | - set -euo pipefail - source_sha_after="$(git rev-parse HEAD)" + set -uo pipefail source_clean=true + source_sha_after="$EXPECTED_SOURCE_SHA" + if ! source_sha_after="$( + git -C target-src rev-parse HEAD 2>"$RUNNER_TEMP/git-head.err" + )"; then + echo "::error::Unable to read the target checkout HEAD." + sed -n '1,20p' "$RUNNER_TEMP/git-head.err" + source_clean=false + fi + if ! status_output="$( + git -C target-src status --porcelain --untracked-files=all \ + 2>"$RUNNER_TEMP/git-status.err" + )"; then + echo "::error::Unable to determine target checkout status." + sed -n '1,20p' "$RUNNER_TEMP/git-status.err" + source_clean=false + status_output="git-status-failed" + fi if [ "$SOURCE_SHA_BEFORE" != "$EXPECTED_SOURCE_SHA" ] || \ [ "$source_sha_after" != "$EXPECTED_SOURCE_SHA" ] || \ - [ -n "$(git status --porcelain --untracked-files=all)" ]; then + [ -n "$status_output" ]; then + source_clean=false + fi + if ! python runner-src/scripts/model_profile_trial_contract.py source-manifest \ + --root target-src \ + --output "$RUNNER_TEMP/source-manifest-after.json"; then + echo "::error::Unable to compute the bounded post-run source manifest." source_clean=false fi echo "source-sha-after=$source_sha_after" >> "$GITHUB_OUTPUT" @@ -241,9 +321,12 @@ jobs: ${{ steps.profile.outputs.source-sha-before || inputs.expected_source_sha }} SOURCE_SHA_AFTER: >- ${{ steps.source_after.outputs.source-sha-after || inputs.expected_source_sha }} + SOURCE_MANIFEST_BEFORE: >- + ${{ steps.manifest_before.outputs.aggregate-sha256 || env.ZERO_SOURCE_MANIFEST }} + SOURCE_MANIFEST_AFTER: >- + ${{ steps.source_after.outputs.aggregate-sha256 || env.ZERO_SOURCE_MANIFEST }} SOURCE_CLEAN: ${{ steps.source_after.outputs.source-clean || 'false' }} REQUESTED_MODEL: ${{ steps.profile.outputs.model }} - RUNNER_VERSION: ${{ steps.profile.outputs.runner-ref }} CLI_VERSION: ${{ steps.cli.outputs.cli-version || 'unknown' }} TRIAL_ID: ${{ inputs.trial_id }} REQUEST_ID: ${{ inputs.request_id }} @@ -253,9 +336,16 @@ jobs: LAUNCH_ORDINAL: ${{ inputs.launch_ordinal }} PACKET_HASH: ${{ inputs.packet_hash }} EXPECTED_SOURCE_SHA: ${{ inputs.expected_source_sha }} + GITHUB_REPOSITORY_VALUE: ${{ github.repository }} + GITHUB_WORKFLOW_REF_VALUE: ${{ github.workflow_ref }} + GITHUB_WORKFLOW_SHA_VALUE: ${{ github.workflow_sha }} + GITHUB_RUN_ID_VALUE: ${{ github.run_id }} + GITHUB_RUN_ATTEMPT_VALUE: ${{ github.run_attempt }} run: | set -euo pipefail - python scripts/model_profile_trial_contract.py artifact \ + artifact_name="model-profile-trial-${PROFILE_ID}-${GITHUB_RUN_ID_VALUE}" + artifact_name+="-${GITHUB_RUN_ATTEMPT_VALUE}-${LAUNCH_ORDINAL}" + python runner-src/scripts/model_profile_trial_contract.py artifact \ --trial-id "$TRIAL_ID" \ --request-id "$REQUEST_ID" \ --request-hash "$REQUEST_HASH" \ @@ -266,21 +356,30 @@ jobs: --expected-source-sha "$EXPECTED_SOURCE_SHA" \ --source-sha-before "$SOURCE_SHA_BEFORE" \ --source-sha-after "$SOURCE_SHA_AFTER" \ + --source-manifest-sha256-before "$SOURCE_MANIFEST_BEFORE" \ + --source-manifest-sha256-after "$SOURCE_MANIFEST_AFTER" \ --requested-model "$REQUESTED_MODEL" \ - --runner-version "$RUNNER_VERSION" \ + --requested-reasoning-effort "high" \ + --runner-version "$PINNED_RUNNER_REF" \ --cli-version "$CLI_VERSION" \ --session-stream "$RUNNER_TEMP/codex-trial-session.jsonl" \ --codex-home "$CODEX_HOME" \ --final-message "$RUNNER_TEMP/codex-trial-final.txt" \ --exit-code "$CODEX_EXIT_CODE" \ --source-clean "$SOURCE_CLEAN" \ + --github-repository "$GITHUB_REPOSITORY_VALUE" \ + --github-workflow-ref "$GITHUB_WORKFLOW_REF_VALUE" \ + --github-workflow-sha "$GITHUB_WORKFLOW_SHA_VALUE" \ + --github-run-id "$GITHUB_RUN_ID_VALUE" \ + --github-run-attempt "$GITHUB_RUN_ATTEMPT_VALUE" \ + --artifact-name "$artifact_name" \ --output "$RUNNER_TEMP/model-profile-trial-attempt.json" - name: Upload unique trial attempt if: ${{ always() && steps.artifact.outcome == 'success' }} - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: - name: model-trial-${{ github.run_id }}-${{ github.run_attempt }} + name: ${{ steps.artifact.outputs.artifact-name }} path: ${{ runner.temp }}/model-profile-trial-attempt.json if-no-files-found: error retention-days: 30 diff --git a/scripts/model_profile_trial_contract.py b/scripts/model_profile_trial_contract.py index da8423e8b..5dfac7ee8 100644 --- a/scripts/model_profile_trial_contract.py +++ b/scripts/model_profile_trial_contract.py @@ -11,15 +11,24 @@ from __future__ import annotations import argparse +import hashlib import json import os import re from pathlib import Path from typing import Any -ARTIFACT_SCHEMA = "workflows.model-profile-trial-result/v1" -IDENTITY_AUTHORITY = "workflows-read-only-trial-artifact/v1" +ARTIFACT_SCHEMA = "workflows.model-profile-trial-result/v2" +IDENTITY_AUTHORITY = "workflows-read-only-trial-artifact/v2" +COLLECTOR_IDENTITY_AUTHORITY = "github-actions-api/workflows-read-only-trial-artifact/v2" EXPECTED_CLI_VERSION = "0.144.1" +EXPECTED_REPOSITORY = "stranske/Workflows" +EXPECTED_WORKFLOW_REF = ( + "stranske/Workflows/.github/workflows/agents-model-profile-trial.yml@refs/heads/main" +) +MAX_SOURCE_FILES = 20_000 +MAX_SOURCE_BYTES = 200 * 1024 * 1024 +SOURCE_SKIP_DIRS = {".git", ".venv", "venv", "node_modules", "dist", "build", "__pycache__"} EXPECTED_PROFILES = { "codex-5.6-sol-high": "gpt-5.6-sol", "codex-5.6-terra-high": "gpt-5.6-terra", @@ -28,7 +37,8 @@ SHA256_RE = re.compile(r"^sha256:[0-9a-f]{64}$") SOURCE_SHA_RE = re.compile(r"^[0-9a-f]{40}$") PINNED_RUNNER_RE = re.compile( - r"^stranske/Workflows/\.github/workflows/" r"reusable-model-profile-trial\.yml@[0-9a-f]{40}$" + r"^stranske/Workflows/\.github/workflows/" + r"reusable-model-profile-trial\.yml@[0-9a-f]{40}$" ) SAFE_ID_RE = re.compile(r"^[A-Za-z0-9._:/-]{1,200}$") @@ -50,11 +60,25 @@ "provider_resolved_provider", "provider_resolved_model", "fallback_reason", + "identity_authority", + "operation_role", "runner_version", "cli_version", "thread_id", + "requested_reasoning_effort", + "reported_reasoning_effort", "source_sha_before", "source_sha_after", + "source_manifest_sha256_before", + "source_manifest_sha256_after", + "source_clean", + "exit_code", + "github_repository", + "github_workflow_ref", + "github_workflow_sha", + "github_run_id", + "github_run_attempt", + "artifact_name", } @@ -93,6 +117,62 @@ def _require_source_sha(label: str, value: str) -> str: return text +def _require_positive_int(label: str, value: int | str) -> int: + try: + parsed = int(value) + except (TypeError, ValueError) as exc: + raise ContractError(f"{label} must be a positive integer") from exc + if parsed <= 0: + raise ContractError(f"{label} must be a positive integer") + return parsed + + +def source_manifest(root: Path) -> dict[str, Any]: + """Hash a bounded checkout without following symlinks or reading Git metadata.""" + root = root.expanduser().resolve() + if not root.is_dir(): + raise ContractError(f"source manifest root is not a directory: {root}") + rows: list[dict[str, Any]] = [] + total_bytes = 0 + for current, dirs, files in os.walk(root, topdown=True, followlinks=False): + dirs[:] = sorted(name for name in dirs if name not in SOURCE_SKIP_DIRS) + base = Path(current) + for name in sorted(files): + path = base / name + rel = path.relative_to(root).as_posix() + if len(rows) >= MAX_SOURCE_FILES: + raise ContractError("source manifest file limit exceeded") + if path.is_symlink(): + target = os.readlink(path) + payload = target.encode("utf-8", errors="surrogateescape") + kind = "symlink" + elif path.is_file(): + size = path.stat().st_size + if total_bytes + size > MAX_SOURCE_BYTES: + raise ContractError("source manifest byte limit exceeded") + payload = path.read_bytes() + total_bytes += size + kind = "file" + else: + continue + rows.append( + { + "path": rel, + "kind": kind, + "bytes": len(payload), + "sha256": hashlib.sha256(payload).hexdigest(), + } + ) + encoded = json.dumps(rows, sort_keys=True, separators=(",", ":")).encode("utf-8") + return { + "schema": "workflows.source-manifest/v1", + "version": 1, + "file_count": len(rows), + "total_bytes": total_bytes, + "aggregate_sha256": "sha256:" + hashlib.sha256(encoded).hexdigest(), + } + + def resolve_profile( registry: dict[str, Any], model_registry: dict[str, Any], profile_id: str ) -> dict[str, Any]: @@ -131,6 +211,7 @@ def resolve_profile( "mode": "read-only", "artifact_schema": ARTIFACT_SCHEMA, "identity_authority": IDENTITY_AUTHORITY, + "collector_identity_authority": COLLECTOR_IDENTITY_AUTHORITY, "cli_version": EXPECTED_CLI_VERSION, "runtime_fallback_allowed": False, "auxiliary_evaluator_allowed": False, @@ -141,7 +222,7 @@ def resolve_profile( runner_ref = str(contract.get("runner_ref") or "") if not PINNED_RUNNER_RE.fullmatch(runner_ref): - raise ContractError("trial contract runner_ref is not an immutable reusable workflow ref") + raise ContractError("trial contract runner_ref is not this immutable reusable workflow") if profile.get("runner_ref") != runner_ref: raise ContractError(f"execution profile {profile_id} runner_ref drifted") @@ -247,7 +328,10 @@ def build_artifact( expected_source_sha: str, source_sha_before: str, source_sha_after: str, + source_manifest_sha256_before: str, + source_manifest_sha256_after: str, requested_model: str, + requested_reasoning_effort: str, runner_version: str, cli_version: str, session_stream: Path, @@ -255,6 +339,12 @@ def build_artifact( final_message: Path, exit_code: int, source_clean: bool, + github_repository: str, + github_workflow_ref: str, + github_workflow_sha: str, + github_run_id: int, + github_run_attempt: int, + artifact_name: str, ) -> dict[str, Any]: """Build one strict attempt artifact, including failed canaries.""" trial_id = _require_safe_id("trial_id", trial_id) @@ -266,12 +356,28 @@ def build_artifact( expected_source_sha = _require_source_sha("expected_source_sha", expected_source_sha) source_sha_before = _require_source_sha("source_sha_before", source_sha_before) source_sha_after = _require_source_sha("source_sha_after", source_sha_after) + source_manifest_sha256_before = _require_hash( + "source_manifest_sha256_before", source_manifest_sha256_before + ) + source_manifest_sha256_after = _require_hash( + "source_manifest_sha256_after", source_manifest_sha256_after + ) if profile_id not in EXPECTED_PROFILES or requested_model != EXPECTED_PROFILES[profile_id]: raise ContractError("requested model does not match the exact profile") if not 1 <= int(launch_ordinal) <= 3: raise ContractError("launch_ordinal must be between 1 and 3") - if not PINNED_RUNNER_RE.fullmatch(str(runner_version or "")): - raise ContractError("runner_version is not the pinned reusable trial workflow") + if not PINNED_RUNNER_RE.fullmatch(runner_version): + raise ContractError("runner_version is not an immutable reusable trial workflow") + if requested_reasoning_effort != "high": + raise ContractError("requested_reasoning_effort must be high") + if github_repository != EXPECTED_REPOSITORY: + raise ContractError("github_repository is not the authoritative Workflows repo") + github_workflow_sha = _require_source_sha("github_workflow_sha", github_workflow_sha) + github_run_id = _require_positive_int("github_run_id", github_run_id) + github_run_attempt = _require_positive_int("github_run_attempt", github_run_attempt) + if github_workflow_ref != EXPECTED_WORKFLOW_REF: + raise ContractError("github_workflow_ref is not the authoritative main-branch shim") + artifact_name = _require_safe_id("artifact_name", artifact_name) identity_parse_failed = False try: @@ -306,6 +412,8 @@ def build_artifact( failures.append("source_sha_changed") if not source_clean: failures.append("source_tree_changed") + if source_manifest_sha256_after != source_manifest_sha256_before: + failures.append("source_manifest_changed") if not acknowledged: failures.append("packet_not_acknowledged") if not thread_id: @@ -322,7 +430,7 @@ def build_artifact( fallback_reason = failures[0] if failures else None artifact = { "schema": ARTIFACT_SCHEMA, - "version": 1, + "version": 2, "trial_id": trial_id, "request_id": request_id, "request_hash": request_hash, @@ -338,11 +446,25 @@ def build_artifact( "provider_resolved_provider": None, "provider_resolved_model": None, "fallback_reason": fallback_reason, + "identity_authority": IDENTITY_AUTHORITY, + "operation_role": "worker", "runner_version": runner_version, "cli_version": cli_version, "thread_id": thread_id, + "requested_reasoning_effort": requested_reasoning_effort, + "reported_reasoning_effort": reported_effort, "source_sha_before": source_sha_before, "source_sha_after": source_sha_after, + "source_manifest_sha256_before": source_manifest_sha256_before, + "source_manifest_sha256_after": source_manifest_sha256_after, + "source_clean": bool(source_clean), + "exit_code": int(exit_code), + "github_repository": github_repository, + "github_workflow_ref": github_workflow_ref, + "github_workflow_sha": github_workflow_sha, + "github_run_id": github_run_id, + "github_run_attempt": github_run_attempt, + "artifact_name": artifact_name, } if set(artifact) != ARTIFACT_FIELDS: raise AssertionError("strict trial artifact schema drifted") @@ -379,7 +501,10 @@ def _artifact_command(args: argparse.Namespace) -> int: expected_source_sha=args.expected_source_sha, source_sha_before=args.source_sha_before, source_sha_after=args.source_sha_after, + source_manifest_sha256_before=args.source_manifest_sha256_before, + source_manifest_sha256_after=args.source_manifest_sha256_after, requested_model=args.requested_model, + requested_reasoning_effort=args.requested_reasoning_effort, runner_version=args.runner_version, cli_version=args.cli_version, session_stream=Path(args.session_stream), @@ -387,6 +512,12 @@ def _artifact_command(args: argparse.Namespace) -> int: final_message=Path(args.final_message), exit_code=args.exit_code, source_clean=args.source_clean == "true", + github_repository=args.github_repository, + github_workflow_ref=args.github_workflow_ref, + github_workflow_sha=args.github_workflow_sha, + github_run_id=args.github_run_id, + github_run_attempt=args.github_run_attempt, + artifact_name=args.artifact_name, ) output = Path(args.output) output.parent.mkdir(parents=True, exist_ok=True) @@ -397,6 +528,7 @@ def _artifact_command(args: argparse.Namespace) -> int: "fallback-reason": artifact["fallback_reason"] or "", "thread-id": artifact["thread_id"] or "", "reported-model": artifact["reported_model"] or "", + "artifact-name": artifact["artifact_name"], } ) return 0 @@ -419,6 +551,15 @@ def _bound_stream_command(args: argparse.Namespace) -> int: return 0 +def _source_manifest_command(args: argparse.Namespace) -> int: + manifest = source_manifest(Path(args.root)) + output = Path(args.output) + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8") + _write_github_output({"aggregate-sha256": manifest["aggregate_sha256"]}) + return 0 + + def _parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(description=__doc__) subparsers = parser.add_subparsers(dest="command", required=True) @@ -441,13 +582,22 @@ def _parser() -> argparse.ArgumentParser: "expected-source-sha", "source-sha-before", "source-sha-after", + "source-manifest-sha256-before", + "source-manifest-sha256-after", "requested-model", + "requested-reasoning-effort", "runner-version", "cli-version", "session-stream", "codex-home", "final-message", "source-clean", + "github-repository", + "github-workflow-ref", + "github-workflow-sha", + "github-run-id", + "github-run-attempt", + "artifact-name", "output", ): artifact.add_argument(f"--{name}", required=True) @@ -459,6 +609,10 @@ def _parser() -> argparse.ArgumentParser: bound_stream.add_argument("--output", required=True) bound_stream.add_argument("--max-bytes", type=int, default=65536) bound_stream.set_defaults(func=_bound_stream_command) + manifest = subparsers.add_parser("source-manifest") + manifest.add_argument("--root", required=True) + manifest.add_argument("--output", required=True) + manifest.set_defaults(func=_source_manifest_command) return parser From d405867413fb2cbecdebde2ecee8cc77f6a99d0f Mon Sep 17 00:00:00 2001 From: Tim Stranske Date: Fri, 10 Jul 2026 11:12:41 -0500 Subject: [PATCH 8/9] Close trial routing and provenance gaps --- .github/agents/registry.yml | 20 +++--- .../scripts/__tests__/agent-registry.test.js | 26 +++---- .../keepalive-model-profile-contract.test.js | 14 ++++ .github/scripts/agent_registry.js | 6 ++ .github/workflows/README.md | 2 +- .../workflows/agents-model-profile-trial.yml | 3 +- docs/WORKFLOW_GUIDE.md | 15 ++-- docs/ci/WORKFLOWS.md | 2 +- docs/ci/WORKFLOW_SYSTEM.md | 6 +- .../consumer-repo/.github/agents/registry.yml | 20 +++--- .../.github/scripts/agent_registry.js | 6 ++ .../test_model_profile_trial_contract.py | 71 ++++++++++++++++++- .../test_model_profile_trial_workflows.py | 55 ++++++++++++-- 13 files changed, 194 insertions(+), 52 deletions(-) diff --git a/.github/agents/registry.yml b/.github/agents/registry.yml index 75533a5f5..429bc3670 100644 --- a/.github/agents/registry.yml +++ b/.github/agents/registry.yml @@ -6,14 +6,16 @@ default_agent: codex keepalive_marker_prefix: agent-keepalive # Dedicated instrumentation-only contract for the Sol/Terra/Luna plumbing -# canary. The reusable workflow ref is replaced with its exact first-commit -# SHA before the dispatch shim is merged. It intentionally cannot fall back, -# mutate source, invoke an evaluator, or claim provider-resolved identity. +# canary. The reusable workflow ref is replaced with the exact commit containing +# this runner before merge. Trial profiles are rejected by ordinary agent and +# Keepalive execution. The lane cannot fall back, mutate source, invoke an +# evaluator, or claim provider-resolved identity. model_profile_trial_contract: mode: read-only - artifact_schema: workflows.model-profile-trial-result/v1 - identity_authority: workflows-read-only-trial-artifact/v1 - runner_ref: stranske/Workflows/.github/workflows/reusable-model-profile-trial.yml@d497362bc4b409895c3ddf7b5dd33e587a0414d2 + artifact_schema: workflows.model-profile-trial-result/v2 + identity_authority: workflows-read-only-trial-artifact/v2 + collector_identity_authority: github-actions-api/workflows-read-only-trial-artifact/v2 + runner_ref: stranske/Workflows/.github/workflows/reusable-model-profile-trial.yml@822e323eefba3edb640e0bd9c922caec6fffee65 cli_version: 0.144.1 runtime_fallback_allowed: false auxiliary_evaluator_allowed: false @@ -44,7 +46,7 @@ execution_profiles: model: gpt-5.6-sol fallback_model: gpt-5.5 runner: reusable-model-profile-trial - runner_ref: stranske/Workflows/.github/workflows/reusable-model-profile-trial.yml@d497362bc4b409895c3ddf7b5dd33e587a0414d2 + runner_ref: stranske/Workflows/.github/workflows/reusable-model-profile-trial.yml@822e323eefba3edb640e0bd9c922caec6fffee65 capacity_pool: codex-standard safety: read-only lifecycle: trial @@ -55,7 +57,7 @@ execution_profiles: model: gpt-5.6-terra fallback_model: gpt-5.5 runner: reusable-model-profile-trial - runner_ref: stranske/Workflows/.github/workflows/reusable-model-profile-trial.yml@d497362bc4b409895c3ddf7b5dd33e587a0414d2 + runner_ref: stranske/Workflows/.github/workflows/reusable-model-profile-trial.yml@822e323eefba3edb640e0bd9c922caec6fffee65 capacity_pool: codex-standard safety: read-only lifecycle: trial @@ -66,7 +68,7 @@ execution_profiles: model: gpt-5.6-luna fallback_model: gpt-5.5 runner: reusable-model-profile-trial - runner_ref: stranske/Workflows/.github/workflows/reusable-model-profile-trial.yml@d497362bc4b409895c3ddf7b5dd33e587a0414d2 + runner_ref: stranske/Workflows/.github/workflows/reusable-model-profile-trial.yml@822e323eefba3edb640e0bd9c922caec6fffee65 capacity_pool: codex-standard safety: read-only lifecycle: trial diff --git a/.github/scripts/__tests__/agent-registry.test.js b/.github/scripts/__tests__/agent-registry.test.js index a7fd30603..608f77866 100644 --- a/.github/scripts/__tests__/agent-registry.test.js +++ b/.github/scripts/__tests__/agent-registry.test.js @@ -227,22 +227,16 @@ test('resolveExecutionProfile returns registry-backed codex model contract', () assert.equal(profile.runner, 'reusable-codex-run'); }); -test('resolveExecutionProfile exposes the explicit Sol Terra Luna trial profiles', () => { - const expected = { - 'codex-5.6-sol-high': ['gpt-5.6-sol', 'gpt-5.5'], - 'codex-5.6-terra-high': ['gpt-5.6-terra', 'gpt-5.5'], - 'codex-5.6-luna-high': ['gpt-5.6-luna', 'gpt-5.5'], - }; - for (const [profileId, [model, fallback]] of Object.entries(expected)) { - const profile = resolveExecutionProfile(profileId, { registryPath: REGISTRY_PATH }); - assert.equal(profile.agent, 'codex'); - assert.equal(profile.model, model); - assert.equal(profile.fallback_model, fallback); - assert.equal(profile.runner, 'reusable-model-profile-trial'); - assert.equal(profile.capacity_pool, 'codex-standard'); - assert.equal(profile.lifecycle, 'trial'); - assert.equal(profile.reasoning_effort, 'high'); - assert.equal(profile.permission_mode, 'read-only'); +test('resolveExecutionProfile rejects trial profiles from ordinary agent execution', () => { + for (const profileId of [ + 'codex-5.6-sol-high', + 'codex-5.6-terra-high', + 'codex-5.6-luna-high', + ]) { + assert.throws( + () => resolveExecutionProfile(profileId, { registryPath: REGISTRY_PATH }), + new RegExp(`Execution profile ${profileId} has lifecycle trial`), + ); } }); diff --git a/.github/scripts/__tests__/keepalive-model-profile-contract.test.js b/.github/scripts/__tests__/keepalive-model-profile-contract.test.js index 6b4f81b53..298c8eeb1 100644 --- a/.github/scripts/__tests__/keepalive-model-profile-contract.test.js +++ b/.github/scripts/__tests__/keepalive-model-profile-contract.test.js @@ -44,6 +44,20 @@ test('workflow dispatch profile input is honored when PR body omits a profile', ); }); +test('ordinary keepalive execution rejects lifecycle trial profiles', () => { + const registry = readWorkflow('.github/scripts/agent_registry.js'); + assert.match( + registry, + /String\(profile\.lifecycle \|\| ''\)\.trim\(\) !== 'active'/, + 'expected the shared resolver to fail closed before Keepalive can use a trial profile', + ); + assert.match( + registry, + /ordinary agent execution accepts active profiles only/, + 'expected an explicit lifecycle rejection rather than a normal runner fallback', + ); +}); + test('profile validation is limited to codex execution actions', () => { const keepaliveLoop = readWorkflow('.github/scripts/keepalive_loop.js'); assert.match( diff --git a/.github/scripts/agent_registry.js b/.github/scripts/agent_registry.js index f0074226d..33fbfa9a8 100644 --- a/.github/scripts/agent_registry.js +++ b/.github/scripts/agent_registry.js @@ -262,6 +262,12 @@ function resolveExecutionProfile(profileId, options = {}) { ); } validateExecutionProfile(id, profile, registry); + if (String(profile.lifecycle || '').trim() !== 'active') { + throw new Error( + `Execution profile ${id} has lifecycle ${profile.lifecycle || '(empty)'}; ` + + 'ordinary agent execution accepts active profiles only', + ); + } return { id, ...profile, diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 023ccad7f..62dcbafa1 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -12,7 +12,7 @@ Core layers: - Minimal invariant CI (`pr-11-ci-smoke.yml`): lean push/PR workflow that installs the project once on Python 3.12, sanity-checks imports, and executes the invariant tests from Issue #3651 so regressions surface quickly. - Gate summary (`pr-00-gate.yml` post-CI jobs): integrated post-CI reporting that batches small hygiene fixes, posts Gate summaries, and manages trivial failure remediation using the composite autofix action. - Agents orchestration (`agents-70-orchestrator.yml` + `reusable-16-agents.yml`): single entry point for Codex readiness, bootstrap, diagnostics, and watchdog sweeps. Use the [Agent task issue template][agent-task-template] (auto-labels `agents` + `agent:codex`) to raise work for Codex; the issue bridge listens for `agent:codex` and hands issues to the orchestrator. Legacy consumer shims remain removed following Issue #2650. -- Model-profile trial (`agents-model-profile-trial.yml` + `reusable-model-profile-trial.yml`): Workflows-only manual transport for one immutable, read-only Sol/Terra/Luna instrumentation arm. It uploads quarantine-only identity telemetry and has no commit, push, comment, auth-refresh, evaluator, or provider-resolution path. +- Model-profile trial (`agents-model-profile-trial.yml` + `reusable-model-profile-trial.yml`): Workflows-only manual transport for one immutable, read-only Sol/Terra/Luna instrumentation arm. A separately pinned helper requires the target to equal current remote `main` before auth and uploads v2 quarantine telemetry with source-manifest and GitHub provenance. It has no commit, push, comment, auth-refresh, evaluator, provider-resolution, or ordinary Keepalive path. - PR metadata management (`agents-pr-meta.yml`): serializes Codex activation commands and PR body decoration through dedicated jobs that share a concurrency group keyed by PR number. This prevents marker thrash while keeping activation dispatch responsive. - Agents intake + orchestration (`agents-63-issue-intake.yml`, `agents-70-orchestrator.yml`, `reusable-16-agents.yml`): unified entry point for ChatGPT topic imports and Codex readiness/bridge sweeps. Use the [Agent task issue template][agent-task-template] (auto-labels `agents` + `agent:codex`) to raise work for Codex; the intake workflow handles both label-triggered bridges and manual dispatch while keeping parsing and bridge logic centralised. - Codex belt automation (`agents-71-codex-belt-dispatcher.yml`, `agents-72-codex-belt-worker.yml`, `agents-73-codex-belt-conveyor.yml`): hands-off conveyor for labelled issues—dispatcher selects `agent:codex` + `status:ready` issues and prepares a `codex/issue-*` branch, worker opens or refreshes the PR with labels/assignees, and conveyor merges after Gate success before re-queuing the dispatcher. diff --git a/.github/workflows/agents-model-profile-trial.yml b/.github/workflows/agents-model-profile-trial.yml index d59e78eeb..49f09a92a 100644 --- a/.github/workflows/agents-model-profile-trial.yml +++ b/.github/workflows/agents-model-profile-trial.yml @@ -49,7 +49,7 @@ permissions: jobs: trial: - uses: stranske/Workflows/.github/workflows/reusable-model-profile-trial.yml@d497362bc4b409895c3ddf7b5dd33e587a0414d2 + uses: stranske/Workflows/.github/workflows/reusable-model-profile-trial.yml@822e323eefba3edb640e0bd9c922caec6fffee65 permissions: contents: read with: @@ -61,5 +61,6 @@ jobs: packet_hash: ${{ inputs.packet_hash }} launch_ordinal: ${{ fromJSON(inputs.launch_ordinal) }} expected_source_sha: ${{ inputs.expected_source_sha }} + runner_sha: 822e323eefba3edb640e0bd9c922caec6fffee65 secrets: CODEX_AUTH_JSON: ${{ secrets.CODEX_AUTH_JSON }} diff --git a/docs/WORKFLOW_GUIDE.md b/docs/WORKFLOW_GUIDE.md index 6c4dcfed0..c0002d4b4 100644 --- a/docs/WORKFLOW_GUIDE.md +++ b/docs/WORKFLOW_GUIDE.md @@ -91,7 +91,8 @@ _Inline Gate helper_ instrumentation-only Sol/Terra/Luna plumbing trial. It forwards only frozen trial/request/profile/hash/source inputs and subscription auth to the immutable `reusable-model-profile-trial.yml` ref recorded in the registry. - It is not a keepalive, evaluator, merge, or learning lane. + It is not a keepalive, evaluator, merge, or learning lane; ordinary agent + and Keepalive resolution reject all `lifecycle: trial` profiles. - **Consumer default entry points** — New consumer repos should install the template-managed pair `agents-80-pr-event-hub.yml` + `agents-81-gate-followups.yml` (along with `agents-verifier.yml`, `pr-00-gate.yml`, `AGENTS.md`, and `CLAUDE.md`). Treat `agents-pr-meta-v4.yml` as Workflows-local infrastructure, not a default consumer setup file. - **`agents-63-issue-intake.yml`** — Canonical front door for Codex issues. It normalizes ChatGPT export blobs / topic lists, dedupes and validates the resulting queue, optionally re-formats the new issues via LangChain, and drives the label-enforced issue bridge when `agent_bridge` mode is selected. The workflow now relies solely on the installation token + the shared API client (no ad-hoc GitHub App token mints), so manual reruns and workflow_call invocations stay lightweight while still enforcing the single-agent label contract before dispatching work to the belt. - **`agents-64-verify-agent-assignment.yml`** — Workflow-call validator that enforces the single `agent:*` label contract and confirms the assignee belongs to the approved automation roster. It now runs entirely on the default workflow token with the shared retry helper (no bespoke GitHub App mint), so verification hooks stay lightweight for orchestrator and issue-bridge callers. @@ -158,11 +159,13 @@ _Inline Gate helper_ - **`reusable-bot-comment-handler.yml`** — Collects unresolved bot review comments, generates a per-agent prompt, and dispatches the appropriate runner. Prefers GitHub App client ID auth, records the selected App auth mode, keeps a warning-only legacy App ID fallback, and still falls back to `service_bot_pat` or `GITHUB_TOKEN` so consumer repos don’t have to configure extra secrets. - **`reusable-codex-run.yml`** — Codex execution wrapper that checks out the target PR branch, installs the pinned Codex CLI, runs the prompt, and pushes commits when the GitHub App token is available (otherwise drops to read-only mode using `GITHUB_TOKEN`). - **`reusable-model-profile-trial.yml`** — Dedicated read-only Codex trial - worker. It validates the registry profile and immutable source SHA, installs - exactly Codex CLI 0.144.1, requests high reasoning once with no fallback, - reads the CLI-reported model from the matching persisted `turn_context`, and - uploads one strict `workflows.model-profile-trial-result/v1` artifact. - Provider-resolved identity remains null and the artifact is quarantine-only. + worker. It executes an immutable helper checkout separately from the target, + requires the target SHA to equal current remote `main` before auth, installs + exactly Codex CLI 0.144.1, and requests high reasoning once with no fallback. + Its strict `workflows.model-profile-trial-result/v2` artifact retains requested + and CLI-reported reasoning, GitHub provenance, and bounded before/after source + manifests. Provider-resolved identity remains null and the artifact is + quarantine-only; the collector verifies it with a separate API authority. - **`reusable-claude-run.yml`** — Claude CLI wrapper for keepalive/autofix scenarios. It mints the Workflows GitHub App token when available so branch pushes can mirror Codex parity, reuses the shared setup-api-client checkout, hardens the Workflows scripts checkout (detecting blobless-clone ghost dirs and reinstalling @octokit deps), and exposes inputs for prompt files, sandbox/safety flags, runtime caps, and appendices. Use it anywhere Claude needs to run the same branch-update loop as Codex. - **`reusable-agents-issue-bridge.yml`** — Shared issue→PR bridge used by `agents-63-issue-intake.yml`; reads `.github/agents/registry.yml` to honor each agent’s branch prefix + assignee list, applies invite/create modes, and now relies solely on the shared token chain (service bot / owner PAT / default token) without minting extra App tokens. - **`reusable-agents-verifier.yml`** — Post-merge verifier reusable that waits for CI (tracking every configured workflow until each has both started and completed), builds PR context, runs checkbox/evaluate/compare modes via the agent verifier stack, and opens follow-up issues when acceptance criteria fail. It mints the Workflows GitHub App token up front so both the caller repo and the Workflows scripts checkout succeed for private/same-repo callers, then falls back to `GITHUB_TOKEN` automatically when App secrets are absent. diff --git a/docs/ci/WORKFLOWS.md b/docs/ci/WORKFLOWS.md index 5f3eeaf8c..6258d5373 100644 --- a/docs/ci/WORKFLOWS.md +++ b/docs/ci/WORKFLOWS.md @@ -100,7 +100,7 @@ The gate uses the shared `.github/scripts/detect-changes.js` helper to decide wh * [`reusable-pr-context.yml`](../../.github/workflows/reusable-pr-context.yml) fetches comprehensive PR context via a single GraphQL query (60-80% API reduction vs REST). Returns PR metadata, labels, files, reviews, comments, and CI status as job outputs for downstream consumption. * [`reusable-codex-run.yml`](../../.github/workflows/reusable-codex-run.yml) exposes a reusable Codex runner with prompt-file input, sandbox/safety defaults, artifact upload, and commit/push handling so keepalive, autofix, and verifier wrappers can share the same execution surface. -* [`reusable-model-profile-trial.yml`](../../.github/workflows/reusable-model-profile-trial.yml) is a separate immutable, read-only worker for one Sol/Terra/Luna instrumentation arm. It installs exactly Codex CLI 0.144.1, enforces high reasoning and the registry-backed `codex-standard` capacity mapping, and emits a strict quarantine artifact with requested/selected/CLI-reported identity plus source SHA before/after. It never commits, pushes, comments, refreshes auth, or invokes an evaluator; provider-resolved identity is explicitly null. +* [`reusable-model-profile-trial.yml`](../../.github/workflows/reusable-model-profile-trial.yml) is a separate immutable, read-only worker for one Sol/Terra/Luna instrumentation arm. A pinned helper checkout validates a target checkout that must equal current remote `main` before auth. It installs exactly Codex CLI 0.144.1, enforces high reasoning and the registry-backed `codex-standard` capacity mapping, and emits a strict v2 quarantine artifact with requested/CLI-reported identity and reasoning, GitHub provenance, and bounded source manifests before/after. It never commits, pushes, comments, refreshes auth, invokes an evaluator, or executes target-checkout code after auth; provider-resolved identity is explicitly null. Ordinary agent and Keepalive lanes reject these `lifecycle: trial` profiles. * Optional exported Orchestrator skill context for opener/closer Codex lanes is documented in [`ORCHESTRATOR_SKILL_CONTEXT.md`](ORCHESTRATOR_SKILL_CONTEXT.md). Repos opt in via `.github/orchestrator_skill.json`; remote runs receive exported instructions only, not local Orchestrator runtime access. * [`reusable-claude-run.yml`](../../.github/workflows/reusable-claude-run.yml) exposes a reusable Claude runner that builds a prompt from a file + optional appendix, runs the configured Claude CLI, and publishes the output and summary as artifacts. * [`reusable-cursor-run.yml`](../../.github/workflows/reusable-cursor-run.yml) exposes a reusable Cursor runner that builds a prompt from a file + optional appendix, runs the `cursor-agent` CLI headlessly (`-p --force --output-format text`, authenticated via `CURSOR_API_KEY`), and publishes the output and summary as artifacts. Keepalive routes `agent:cursor` PRs here. diff --git a/docs/ci/WORKFLOW_SYSTEM.md b/docs/ci/WORKFLOW_SYSTEM.md index 0bf624e3c..3e43c75b8 100644 --- a/docs/ci/WORKFLOW_SYSTEM.md +++ b/docs/ci/WORKFLOW_SYSTEM.md @@ -663,8 +663,10 @@ Keep this table handy when you are triaging automation: it confirms which workfl - **Reusable Agents** – `reusable-16-agents.yml` powers orchestrated dispatch. - **Reusable Autofix** – `reusable-18-autofix.yml` centralizes fixers for Gate summary job. - **Reusable Model Profile Trial** – `reusable-model-profile-trial.yml` runs one - immutable read-only Sol/Terra/Luna plumbing arm and emits quarantine-only - identity telemetry; `agents-model-profile-trial.yml` is its manual shim. + immutable read-only Sol/Terra/Luna plumbing arm from a separately pinned + helper checkout. It requires current remote `main` before auth and emits v2 + quarantine telemetry with GitHub provenance and bounded source manifests; + `agents-model-profile-trial.yml` is its manual shim. - **Selftest: Reusables** – `selftest-reusable-ci.yml` is the consolidated entry point. It runs nightly via cron (06:30 UTC) to rehearse the reusable matrix and accepts manual dispatches for summary/comment publication. Inputs: diff --git a/templates/consumer-repo/.github/agents/registry.yml b/templates/consumer-repo/.github/agents/registry.yml index 75533a5f5..429bc3670 100644 --- a/templates/consumer-repo/.github/agents/registry.yml +++ b/templates/consumer-repo/.github/agents/registry.yml @@ -6,14 +6,16 @@ default_agent: codex keepalive_marker_prefix: agent-keepalive # Dedicated instrumentation-only contract for the Sol/Terra/Luna plumbing -# canary. The reusable workflow ref is replaced with its exact first-commit -# SHA before the dispatch shim is merged. It intentionally cannot fall back, -# mutate source, invoke an evaluator, or claim provider-resolved identity. +# canary. The reusable workflow ref is replaced with the exact commit containing +# this runner before merge. Trial profiles are rejected by ordinary agent and +# Keepalive execution. The lane cannot fall back, mutate source, invoke an +# evaluator, or claim provider-resolved identity. model_profile_trial_contract: mode: read-only - artifact_schema: workflows.model-profile-trial-result/v1 - identity_authority: workflows-read-only-trial-artifact/v1 - runner_ref: stranske/Workflows/.github/workflows/reusable-model-profile-trial.yml@d497362bc4b409895c3ddf7b5dd33e587a0414d2 + artifact_schema: workflows.model-profile-trial-result/v2 + identity_authority: workflows-read-only-trial-artifact/v2 + collector_identity_authority: github-actions-api/workflows-read-only-trial-artifact/v2 + runner_ref: stranske/Workflows/.github/workflows/reusable-model-profile-trial.yml@822e323eefba3edb640e0bd9c922caec6fffee65 cli_version: 0.144.1 runtime_fallback_allowed: false auxiliary_evaluator_allowed: false @@ -44,7 +46,7 @@ execution_profiles: model: gpt-5.6-sol fallback_model: gpt-5.5 runner: reusable-model-profile-trial - runner_ref: stranske/Workflows/.github/workflows/reusable-model-profile-trial.yml@d497362bc4b409895c3ddf7b5dd33e587a0414d2 + runner_ref: stranske/Workflows/.github/workflows/reusable-model-profile-trial.yml@822e323eefba3edb640e0bd9c922caec6fffee65 capacity_pool: codex-standard safety: read-only lifecycle: trial @@ -55,7 +57,7 @@ execution_profiles: model: gpt-5.6-terra fallback_model: gpt-5.5 runner: reusable-model-profile-trial - runner_ref: stranske/Workflows/.github/workflows/reusable-model-profile-trial.yml@d497362bc4b409895c3ddf7b5dd33e587a0414d2 + runner_ref: stranske/Workflows/.github/workflows/reusable-model-profile-trial.yml@822e323eefba3edb640e0bd9c922caec6fffee65 capacity_pool: codex-standard safety: read-only lifecycle: trial @@ -66,7 +68,7 @@ execution_profiles: model: gpt-5.6-luna fallback_model: gpt-5.5 runner: reusable-model-profile-trial - runner_ref: stranske/Workflows/.github/workflows/reusable-model-profile-trial.yml@d497362bc4b409895c3ddf7b5dd33e587a0414d2 + runner_ref: stranske/Workflows/.github/workflows/reusable-model-profile-trial.yml@822e323eefba3edb640e0bd9c922caec6fffee65 capacity_pool: codex-standard safety: read-only lifecycle: trial diff --git a/templates/consumer-repo/.github/scripts/agent_registry.js b/templates/consumer-repo/.github/scripts/agent_registry.js index f0074226d..33fbfa9a8 100644 --- a/templates/consumer-repo/.github/scripts/agent_registry.js +++ b/templates/consumer-repo/.github/scripts/agent_registry.js @@ -262,6 +262,12 @@ function resolveExecutionProfile(profileId, options = {}) { ); } validateExecutionProfile(id, profile, registry); + if (String(profile.lifecycle || '').trim() !== 'active') { + throw new Error( + `Execution profile ${id} has lifecycle ${profile.lifecycle || '(empty)'}; ` + + 'ordinary agent execution accepts active profiles only', + ); + } return { id, ...profile, diff --git a/tests/scripts/test_model_profile_trial_contract.py b/tests/scripts/test_model_profile_trial_contract.py index b8833d896..0534a75b3 100644 --- a/tests/scripts/test_model_profile_trial_contract.py +++ b/tests/scripts/test_model_profile_trial_contract.py @@ -6,7 +6,10 @@ import pytest from scripts import model_profile_trial_contract as contract -PINNED_REF = "stranske/Workflows/.github/workflows/" "reusable-model-profile-trial.yml@" + "1" * 40 +PINNED_REF = ( + "stranske/Workflows/.github/workflows/reusable-model-profile-trial.yml@" + + ("1" * 40) +) def _registries(): @@ -15,6 +18,7 @@ def _registries(): "mode": "read-only", "artifact_schema": contract.ARTIFACT_SCHEMA, "identity_authority": contract.IDENTITY_AUTHORITY, + "collector_identity_authority": contract.COLLECTOR_IDENTITY_AUTHORITY, "runner_ref": PINNED_REF, "cli_version": contract.EXPECTED_CLI_VERSION, "runtime_fallback_allowed": False, @@ -96,7 +100,10 @@ def _artifact(tmp_path: Path, **overrides): "expected_source_sha": source_sha, "source_sha_before": source_sha, "source_sha_after": source_sha, + "source_manifest_sha256_before": "sha256:" + "d" * 64, + "source_manifest_sha256_after": "sha256:" + "d" * 64, "requested_model": "gpt-5.6-sol", + "requested_reasoning_effort": "high", "runner_version": PINNED_REF, "cli_version": "codex-cli 0.144.1", "session_stream": stream, @@ -104,6 +111,12 @@ def _artifact(tmp_path: Path, **overrides): "final_message": final_message, "exit_code": 0, "source_clean": True, + "github_repository": contract.EXPECTED_REPOSITORY, + "github_workflow_ref": contract.EXPECTED_WORKFLOW_REF, + "github_workflow_sha": source_sha, + "github_run_id": 12345, + "github_run_attempt": 2, + "artifact_name": "model-trial-12345-2", } values.update(overrides) return contract.build_artifact(**values) @@ -137,7 +150,12 @@ def test_success_artifact_uses_session_turn_context_and_keeps_provider_null(tmp_ assert artifact["fallback_reason"] is None assert artifact["thread_id"] == "019f-trial-thread" assert artifact["source_sha_before"] == artifact["source_sha_after"] + assert artifact["source_manifest_sha256_before"] == artifact["source_manifest_sha256_after"] assert artifact["launch_ordinal"] == 2 + assert artifact["version"] == 2 + assert artifact["requested_reasoning_effort"] == "high" + assert artifact["reported_reasoning_effort"] == "high" + assert artifact["github_workflow_ref"] == contract.EXPECTED_WORKFLOW_REF def test_artifact_fails_closed_on_model_effort_or_source_drift(tmp_path): @@ -153,6 +171,13 @@ def test_artifact_fails_closed_on_model_effort_or_source_drift(tmp_path): assert source["status"] == "failed" assert source["fallback_reason"] == "source_sha_changed" + manifest = _artifact( + tmp_path / "manifest", + source_manifest_sha256_after="sha256:" + "e" * 64, + ) + assert manifest["status"] == "failed" + assert manifest["fallback_reason"] == "source_manifest_changed" + def test_thread_identity_must_come_from_exact_matching_rollout(tmp_path): artifact = _artifact(tmp_path) @@ -180,7 +205,10 @@ def test_thread_identity_must_come_from_exact_matching_rollout(tmp_path): expected_source_sha=source_sha, source_sha_before=source_sha, source_sha_after=source_sha, + source_manifest_sha256_before="sha256:" + "d" * 64, + source_manifest_sha256_after="sha256:" + "d" * 64, requested_model="gpt-5.6-sol", + requested_reasoning_effort="high", runner_version=PINNED_REF, cli_version="codex-cli 0.144.1", session_stream=tmp_path / "stream.jsonl", @@ -188,6 +216,12 @@ def test_thread_identity_must_come_from_exact_matching_rollout(tmp_path): final_message=final_message, exit_code=0, source_clean=True, + github_repository=contract.EXPECTED_REPOSITORY, + github_workflow_ref=contract.EXPECTED_WORKFLOW_REF, + github_workflow_sha=source_sha, + github_run_id=12345, + github_run_attempt=1, + artifact_name="model-trial-12345-1", ) assert broken["reported_model"] is None assert broken["fallback_reason"] == "reported_model_missing" @@ -211,7 +245,10 @@ def test_failed_cli_with_malformed_stream_still_emits_failure_artifact(tmp_path) expected_source_sha=source_sha, source_sha_before=source_sha, source_sha_after=source_sha, + source_manifest_sha256_before="sha256:" + "d" * 64, + source_manifest_sha256_after="sha256:" + "d" * 64, requested_model="gpt-5.6-sol", + requested_reasoning_effort="high", runner_version=PINNED_REF, cli_version="codex-cli 0.144.1", session_stream=stream, @@ -219,7 +256,39 @@ def test_failed_cli_with_malformed_stream_still_emits_failure_artifact(tmp_path) final_message=final_message, exit_code=1, source_clean=True, + github_repository=contract.EXPECTED_REPOSITORY, + github_workflow_ref=contract.EXPECTED_WORKFLOW_REF, + github_workflow_sha=source_sha, + github_run_id=12345, + github_run_attempt=1, + artifact_name="model-trial-12345-1", ) assert artifact["status"] == "failed" assert artifact["fallback_reason"] == "codex_cli_failed" assert set(artifact) == contract.ARTIFACT_FIELDS + + +def test_artifact_rejects_non_authoritative_github_provenance(tmp_path): + with pytest.raises(contract.ContractError, match="github_workflow_ref"): + _artifact( + tmp_path, + github_workflow_ref=( + "stranske/Workflows/.github/workflows/agents-model-profile-trial.yml@refs/pull/1/merge" + ), + ) + + +def test_source_manifest_is_stable_and_detects_ignored_file_changes(tmp_path): + checkout = tmp_path / "checkout" + checkout.mkdir() + (checkout / "tracked.txt").write_text("tracked\n", encoding="utf-8") + ignored = checkout / ".trial-local" + ignored.write_text("before\n", encoding="utf-8") + + before = contract.source_manifest(checkout) + assert before["file_count"] == 2 + assert before == contract.source_manifest(checkout) + + ignored.write_text("after\n", encoding="utf-8") + after = contract.source_manifest(checkout) + assert before["aggregate_sha256"] != after["aggregate_sha256"] diff --git a/tests/workflows/test_model_profile_trial_workflows.py b/tests/workflows/test_model_profile_trial_workflows.py index fddd47c08..487bef01b 100644 --- a/tests/workflows/test_model_profile_trial_workflows.py +++ b/tests/workflows/test_model_profile_trial_workflows.py @@ -8,6 +8,7 @@ SHIM = Path(".github/workflows/agents-model-profile-trial.yml") RUNNER = Path(".github/workflows/reusable-model-profile-trial.yml") REGISTRY = Path(".github/agents/registry.yml") +TEMPLATE_REGISTRY = Path("templates/consumer-repo/.github/agents/registry.yml") def _workflow(path: Path): @@ -41,9 +42,13 @@ def test_dispatch_shim_is_single_arm_and_calls_only_pinned_reusable_runner(): assert list(workflow["jobs"]) == ["trial"] runner_ref = workflow["jobs"]["trial"]["uses"] assert re.fullmatch( - r"stranske/Workflows/\.github/workflows/" r"reusable-model-profile-trial\.yml@[0-9a-f]{40}", + r"stranske/Workflows/\.github/workflows/" + r"reusable-model-profile-trial\.yml@[0-9a-f]{40}", runner_ref, ) + runner_sha = workflow["jobs"]["trial"]["with"]["runner_sha"] + assert re.fullmatch(r"[0-9a-f]{40}", runner_sha) + assert runner_ref.endswith("@" + runner_sha) assert workflow["permissions"] == {"contents": "read"} assert "inherit" not in str(workflow["jobs"]["trial"].get("secrets")) @@ -60,6 +65,9 @@ def test_reusable_runner_is_read_only_exact_cli_and_has_no_write_lane(): assert 'model_reasoning_effort="high"' in source assert "--ignore-user-config" in source assert "persist-credentials: false" in source + assert "expected_source_sha must equal current remote main before auth" in source + assert "git ls-remote https://github.com/stranske/Workflows.git refs/heads/main" in source + assert "target-src/scripts/" not in source assert "provider_resolved" not in source forbidden = ( "git commit", @@ -81,21 +89,52 @@ def test_runner_uploads_one_unique_attempt_and_enforces_source_integrity(): step for step in steps if str(step.get("uses", "")).startswith("actions/upload-artifact@") ] assert len(uploads) == 1 - name = uploads[0]["with"]["name"] - assert "github.run_id" in name and "github.run_attempt" in name + assert uploads[0]["with"]["name"] == "${{ steps.artifact.outputs.artifact-name }}" source = RUNNER.read_text(encoding="utf-8") + assert 'artifact_name="model-profile-trial-${PROFILE_ID}-${GITHUB_RUN_ID_VALUE}"' in source + assert 'artifact_name+="-${GITHUB_RUN_ATTEMPT_VALUE}-${LAUNCH_ORDINAL}"' in source assert "source-sha-before" in source assert "source-sha-after" in source - assert "git status --porcelain --untracked-files=all" in source + assert "source-manifest-sha256-before" in source + assert "source-manifest-sha256-after" in source + assert "git -C target-src status --porcelain --untracked-files=all" in source + assert "Unable to determine target checkout status" in source assert "model_profile_trial_contract.py artifact" in source +def test_runner_uses_separate_pinned_helper_checkout_and_full_action_shas(): + workflow = _workflow(RUNNER) + steps = workflow["jobs"]["run-single-arm"]["steps"] + checkouts = [step for step in steps if str(step.get("uses", "")).startswith("actions/checkout@")] + assert len(checkouts) == 2 + assert checkouts[0]["with"] == { + "repository": "stranske/Workflows", + "ref": "${{ inputs.runner_sha }}", + "path": "runner-src", + "persist-credentials": False, + } + assert checkouts[1]["with"]["path"] == "target-src" + for step in steps: + uses = str(step.get("uses", "")) + if uses.startswith(("actions/checkout@", "actions/setup-python@", "actions/upload-artifact@")): + assert re.fullmatch(r"[^@]+@[0-9a-f]{40}", uses) + + source = RUNNER.read_text(encoding="utf-8") + auth_offset = source.index("Configure isolated Codex subscription auth") + assert "python target-src/" not in source[auth_offset:] + assert "runner-src/scripts/model_profile_trial_contract.py" in source[auth_offset:] + + def test_registry_trial_profiles_share_exact_pinned_read_only_contract(): registry = yaml.safe_load(REGISTRY.read_text(encoding="utf-8")) trial = registry["model_profile_trial_contract"] assert trial["mode"] == "read-only" - assert trial["artifact_schema"] == "workflows.model-profile-trial-result/v1" - assert trial["identity_authority"] == "workflows-read-only-trial-artifact/v1" + assert trial["artifact_schema"] == "workflows.model-profile-trial-result/v2" + assert trial["identity_authority"] == "workflows-read-only-trial-artifact/v2" + assert ( + trial["collector_identity_authority"] + == "github-actions-api/workflows-read-only-trial-artifact/v2" + ) assert trial["cli_version"] == "0.144.1" assert trial["runtime_fallback_allowed"] is False assert trial["auxiliary_evaluator_allowed"] is False @@ -111,3 +150,7 @@ def test_registry_trial_profiles_share_exact_pinned_read_only_contract(): assert profile["reasoning_effort"] == "high" assert profile["permission_mode"] == "read-only" assert profile["safety"] == "read-only" + + +def test_consumer_template_registry_matches_authoritative_registry(): + assert TEMPLATE_REGISTRY.read_text(encoding="utf-8") == REGISTRY.read_text(encoding="utf-8") From d20414f12434e5ebb9f44df9f6058b3697a18d10 Mon Sep 17 00:00:00 2001 From: Tim Stranske Date: Fri, 10 Jul 2026 11:14:40 -0500 Subject: [PATCH 9/9] Format hardened trial contract --- scripts/model_profile_trial_contract.py | 3 +-- tests/scripts/test_model_profile_trial_contract.py | 5 +---- tests/workflows/test_model_profile_trial_workflows.py | 11 +++++++---- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/scripts/model_profile_trial_contract.py b/scripts/model_profile_trial_contract.py index 5dfac7ee8..e82ab08fb 100644 --- a/scripts/model_profile_trial_contract.py +++ b/scripts/model_profile_trial_contract.py @@ -37,8 +37,7 @@ SHA256_RE = re.compile(r"^sha256:[0-9a-f]{64}$") SOURCE_SHA_RE = re.compile(r"^[0-9a-f]{40}$") PINNED_RUNNER_RE = re.compile( - r"^stranske/Workflows/\.github/workflows/" - r"reusable-model-profile-trial\.yml@[0-9a-f]{40}$" + r"^stranske/Workflows/\.github/workflows/" r"reusable-model-profile-trial\.yml@[0-9a-f]{40}$" ) SAFE_ID_RE = re.compile(r"^[A-Za-z0-9._:/-]{1,200}$") diff --git a/tests/scripts/test_model_profile_trial_contract.py b/tests/scripts/test_model_profile_trial_contract.py index 0534a75b3..ce09baa6a 100644 --- a/tests/scripts/test_model_profile_trial_contract.py +++ b/tests/scripts/test_model_profile_trial_contract.py @@ -6,10 +6,7 @@ import pytest from scripts import model_profile_trial_contract as contract -PINNED_REF = ( - "stranske/Workflows/.github/workflows/reusable-model-profile-trial.yml@" - + ("1" * 40) -) +PINNED_REF = "stranske/Workflows/.github/workflows/reusable-model-profile-trial.yml@" + ("1" * 40) def _registries(): diff --git a/tests/workflows/test_model_profile_trial_workflows.py b/tests/workflows/test_model_profile_trial_workflows.py index 487bef01b..f2bad5027 100644 --- a/tests/workflows/test_model_profile_trial_workflows.py +++ b/tests/workflows/test_model_profile_trial_workflows.py @@ -42,8 +42,7 @@ def test_dispatch_shim_is_single_arm_and_calls_only_pinned_reusable_runner(): assert list(workflow["jobs"]) == ["trial"] runner_ref = workflow["jobs"]["trial"]["uses"] assert re.fullmatch( - r"stranske/Workflows/\.github/workflows/" - r"reusable-model-profile-trial\.yml@[0-9a-f]{40}", + r"stranske/Workflows/\.github/workflows/" r"reusable-model-profile-trial\.yml@[0-9a-f]{40}", runner_ref, ) runner_sha = workflow["jobs"]["trial"]["with"]["runner_sha"] @@ -105,7 +104,9 @@ def test_runner_uploads_one_unique_attempt_and_enforces_source_integrity(): def test_runner_uses_separate_pinned_helper_checkout_and_full_action_shas(): workflow = _workflow(RUNNER) steps = workflow["jobs"]["run-single-arm"]["steps"] - checkouts = [step for step in steps if str(step.get("uses", "")).startswith("actions/checkout@")] + checkouts = [ + step for step in steps if str(step.get("uses", "")).startswith("actions/checkout@") + ] assert len(checkouts) == 2 assert checkouts[0]["with"] == { "repository": "stranske/Workflows", @@ -116,7 +117,9 @@ def test_runner_uses_separate_pinned_helper_checkout_and_full_action_shas(): assert checkouts[1]["with"]["path"] == "target-src" for step in steps: uses = str(step.get("uses", "")) - if uses.startswith(("actions/checkout@", "actions/setup-python@", "actions/upload-artifact@")): + if uses.startswith( + ("actions/checkout@", "actions/setup-python@", "actions/upload-artifact@") + ): assert re.fullmatch(r"[^@]+@[0-9a-f]{40}", uses) source = RUNNER.read_text(encoding="utf-8")