diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 038d8ebc80e38..74c8232fea230 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1034,6 +1034,27 @@ repos: pass_filenames: false require_serial: true verbose: true + - id: run-skill-eval-codex + name: Run skill-eval against AGENTS.md and cases (Codex runtime) + # Kept separate from run-skill-eval so only contributors who actually run + # the Codex arm build an env carrying the Codex CLI binary. promptfoo + # bundles its own @openai/codex-sdk, but that build ships a binary whose + # Developer ID certificate Apple has revoked — macOS kills it on exec — + # so a notarized version has to be installed explicitly. + entry: env AGENT_RUNTIME=codex ./dev/skill-evals/eval.py + language: node + language_version: '22.22.0' + additional_dependencies: + - 'promptfoo@0.121.17' + - '@openai/codex-sdk@0.144.6' + stages: ['manual'] + files: > + (?x) + ^AGENTS\.md$| + ^dev/skill-evals/ + pass_filenames: false + require_serial: true + verbose: true - id: view-skill-eval name: View skill-eval results in browser (manual) entry: env PROMPTFOO_CONFIG_DIR=.build/promptfoo promptfoo view diff --git a/dev/skill-evals/README.md b/dev/skill-evals/README.md index 57748b96c2e5c..e25333ff89a9f 100644 --- a/dev/skill-evals/README.md +++ b/dev/skill-evals/README.md @@ -23,6 +23,7 @@ - [Skill-Eval Harness](#skill-eval-harness) - [Prerequisites](#prerequisites) + - [Agent runtimes](#agent-runtimes) - [Usage](#usage) - [Cleanup](#cleanup) - [Adding cases](#adding-cases) @@ -48,12 +49,49 @@ regular-file CLAUDE.md would make every arm read identical guidance. ## Prerequisites -- **Authentication** (one of): +- **Claude authentication** (one of): - Claude Code session (`claude /login`) — Pro/Max subscription - `ANTHROPIC_API_KEY` environment variable — API credits +- **Codex authentication** (when using the Codex runtime): a Codex CLI + session (`codex login`) -That's it — prek provisions Node, promptfoo, and the Claude Agent SDK -automatically. +That's it — prek provisions Node, promptfoo, and the agent SDKs automatically. + +## Agent runtimes + +Claude is the default runtime. The `run-skill-eval-codex` hook runs the same +arms and cases through the official Codex SDK instead. The Codex provider uses +a fresh thread for every prompt, read-only sandboxing, disabled network access, +disabled session-history persistence, and the SDK's structured-output support. + +The two runtimes are separate hooks, not one hook with a flag, because the +Codex env carries a ~300 MB Codex CLI binary. Keeping it separate means prek +only builds that env for contributors who actually run the Codex arm. promptfoo +bundles its own older `@openai/codex-sdk`, but that build ships a binary whose +Developer ID certificate Apple has revoked — macOS kills it on exec and reports +it as malware — so the Codex hook installs a notarized version explicitly. + +Codex caps the project docs it reads at `project_doc_max_bytes` and truncates +past it, and the 32,768-byte default is smaller than `AGENTS.md`. The harness +raises that cap so the whole file reaches the model — otherwise guidance +appended at the end would be invisible to every arm and the eval would compare +identical prefixes. + +When `SKILL_NAME` is set, both runtimes verify skill usage with promptfoo's +`skill-used` assertion. The Codex SDK does not expose a first-class skill-use +event, so promptfoo infers usage from successful reads of the skill's +`SKILL.md` file. + +```bash +# Default: Claude Agent SDK +prek run run-skill-eval --hook-stage manual --all-files + +# Codex SDK, using its configured default model +prek run run-skill-eval-codex --hook-stage manual --all-files + +# Codex SDK with an explicit model +MODEL=gpt-5.4 prek run run-skill-eval-codex --hook-stage manual --all-files +``` ## Usage @@ -125,8 +163,9 @@ Use `output.should_create` directly in assertions. 1. Creates git worktrees — one with `main`'s AGENTS.md, one with your working tree version. Both are full repo checkouts. 2. Generates a [promptfoo](https://github.com/promptfoo/promptfoo) - config with `anthropic:claude-agent-sdk` provider and - `output_format: json_schema` for structured output. + config for the selected runtime. Claude uses promptfoo's + `anthropic:claude-agent-sdk` provider; Codex uses its `openai:codex-sdk` + provider. Both request schema-constrained structured output. 3. Runs each case against all arms in parallel. 4. Reports pass/fail diff. Worktrees cleaned up on exit. diff --git a/dev/skill-evals/eval.py b/dev/skill-evals/eval.py index 20eb766862a41..19184629994c4 100755 --- a/dev/skill-evals/eval.py +++ b/dev/skill-evals/eval.py @@ -26,12 +26,15 @@ Usage: prek run run-skill-eval --hook-stage manual --all-files + prek run run-skill-eval-codex --hook-stage manual --all-files -Env knobs: MODEL, SKILL_NAME, EVAL_REPEAT, EVAL_FULL (baseline arm). +Env knobs: AGENT_RUNTIME (set by the Codex hook), MODEL, SKILL_NAME, EVAL_REPEAT, +EVAL_FULL (baseline arm). Promptfoo flags like --filter* are argv-only — wire them as fixed entry args on a hook variant when needed. -Authentication: Claude Code session (claude /login) or ANTHROPIC_API_KEY. +Authentication: Claude Code session (claude /login), ANTHROPIC_API_KEY, +or a Codex CLI session (codex login), depending on the selected runtime. """ from __future__ import annotations @@ -45,6 +48,16 @@ from pathlib import Path PROMPTFOO_VERSION = "0.121.17" +SDK_PACKAGES = { + "claude": "@anthropic-ai/claude-agent-sdk", + "codex": "@openai/codex-sdk", +} +SUPPORTED_RUNTIMES = tuple(SDK_PACKAGES) + +# Codex prefix-truncates project docs past project_doc_max_bytes, and the 32,768-byte +# default is already smaller than AGENTS.md — guidance appended at the end would be +# invisible to every arm while the eval still recorded a hash as proof it ran. +CODEX_PROJECT_DOC_MAX_BYTES = 1_048_576 REPO_ROOT = Path(__file__).resolve().parent.parent.parent SCRIPT_DIR = Path(__file__).resolve().parent @@ -65,7 +78,7 @@ "type": "json_schema", "schema": { "type": "object", - "required": ["should_create", "rationale"], + "required": ["should_create", "type", "rationale"], "additionalProperties": False, "properties": { "should_create": {"type": "boolean"}, @@ -83,12 +96,20 @@ def run(cmd: list[str], **kwargs) -> subprocess.CompletedProcess[str]: return subprocess.run(cmd, capture_output=True, text=True, check=False, **kwargs) -def find_sdk_modules() -> Path: - """Locate the node_modules dir (in the prek env) that contains the Claude Agent SDK. +def find_sdk_modules(runtime: str) -> Path: + """Locate the prek node_modules dir containing the selected agent SDK. - promptfoo resolves the SDK from the eval config's directory, not from its - own install tree — the caller symlinks this dir next to the config. + promptfoo resolves providers from the eval config's directory. The caller + symlinks this directory next to the generated config. + + Only the hook's own install root counts, never promptfoo's nested + node_modules: promptfoo declares both SDKs as optional dependencies, so a + prek env holds two copies of each, and its bundled Codex build ships a + binary whose Developer ID certificate Apple has revoked — macOS kills it on + exec. Resolving by location rather than by version keeps the pins in + .pre-commit-config.yaml the single source of truth. """ + sdk_package = SDK_PACKAGES[runtime] promptfoo_bin = shutil.which("promptfoo") if promptfoo_bin: # PROMPTFOO_DISABLE_UPDATE avoids an outdated version banner on stdout when a newer @@ -102,12 +123,13 @@ def find_sdk_modules() -> Path: if pf_pkg.parent == pf_pkg: break pf_pkg = pf_pkg.parent - for candidate in (pf_pkg / "node_modules", pf_pkg.parent): - if (candidate / "@anthropic-ai" / "claude-agent-sdk").is_dir(): - return candidate + install_root = pf_pkg.parent + if install_root.joinpath(*sdk_package.split("/"), "package.json").is_file(): + return install_root + hook = "run-skill-eval-codex" if runtime == "codex" else "run-skill-eval" print( - "Error: promptfoo with the Claude Agent SDK not found. Run the eval via:\n" - " prek run run-skill-eval --hook-stage manual --all-files", + f"Error: promptfoo {PROMPTFOO_VERSION} with {sdk_package} not found. Run the eval via:\n" + f" prek run {hook} --hook-stage manual --all-files", file=sys.stderr, ) sys.exit(1) @@ -207,20 +229,56 @@ def remove_worktree(wt_dir: Path) -> None: run(["git", "-C", str(REPO_ROOT), "worktree", "remove", "--force", str(wt_dir)]) -def build_provider(label: str, working_dir: Path, model: str, skill_name: str | None = None) -> dict: +def build_provider(label: str, working_dir: Path, model: str | None, skill_name: str | None = None) -> dict: config: dict = { - "model": model, "apiKeyRequired": False, "setting_sources": ["project"], "append_allowed_tools": ["Read", "Grep", "Glob"], "working_dir": str(working_dir), "output_format": OUTPUT_FORMAT, } + if model: + config["model"] = model if skill_name: config["skills"] = [skill_name] return {"id": "anthropic:claude-agent-sdk", "label": label, "config": config} +def build_codex_provider( + label: str, working_dir: Path, model: str | None, detect_skill_usage: bool = False +) -> dict: + config = { + "approval_policy": "never", + "cli_config": { + "history": {"persistence": "none"}, + "project_doc_max_bytes": CODEX_PROJECT_DOC_MAX_BYTES, + }, + "network_access_enabled": False, + "output_schema": OUTPUT_FORMAT["schema"], + "sandbox_mode": "read-only", + "web_search_mode": "disabled", + "working_dir": str(working_dir), + } + if model: + config["model"] = model + if detect_skill_usage: + config["enable_streaming"] = True + return { + "id": "openai:codex-sdk", + "label": label, + "config": config, + "transform": "JSON.parse(output)", + } + + +def get_runtime() -> str: + runtime = os.environ.get("AGENT_RUNTIME", "claude").lower() + if runtime not in SUPPORTED_RUNTIMES: + choices = ", ".join(SUPPORTED_RUNTIMES) + raise ValueError(f"AGENT_RUNTIME must be one of: {choices}; got {runtime!r}") + return runtime + + def count_provider_errors(results_file: Path) -> int: """Count results whose provider call errored (as opposed to failing an assertion).""" try: @@ -231,9 +289,16 @@ def count_provider_errors(results_file: Path) -> int: def main() -> int: - sdk_modules = find_sdk_modules() + try: + runtime = get_runtime() + except ValueError as error: + print(f"Error: {error}", file=sys.stderr) + return 1 + sdk_modules = find_sdk_modules(runtime) - model = os.environ.get("MODEL", "claude-sonnet-4-6") + model = os.environ.get("MODEL") + if not model and runtime == "claude": + model = "claude-sonnet-4-6" skill_name = os.environ.get("SKILL_NAME") skill_src = None if skill_name: @@ -263,7 +328,8 @@ def main() -> int: return 1 base_branch = resolve_base_branch() - check_claude_md_symlink(base_branch) + if runtime == "claude": + check_claude_md_symlink(base_branch) # Hash before building arms so edits made mid-run aren't recorded as tested. guidance_hash = compute_guidance_hash(AGENTS_SRC, CASES_DIR) @@ -273,7 +339,7 @@ def main() -> int: worktrees: list[Path] = [] try: - # promptfoo resolves the Claude Agent SDK from the config directory + # promptfoo resolves the selected agent SDK from the config directory (work_dir / "node_modules").symlink_to(sdk_modules) # Extract main-branch AGENTS.md @@ -319,11 +385,16 @@ def main() -> int: arm_baseline = create_worktree(work_dir, "baseline", base_branch, None, worktrees) # Generate config (JSON — valid promptfoo config, keeps the script stdlib-only) - providers = [build_provider("main", arm_main, model, skill_name)] + def provider(label: str, arm: Path, selected_skill: str | None = None) -> dict: + if runtime == "codex": + return build_codex_provider(label, arm, model, detect_skill_usage=bool(selected_skill)) + return build_provider(label, arm, model, selected_skill) + + providers = [provider("main", arm_main, skill_name)] if arm_working: - providers.append(build_provider("working", arm_working, model, skill_name)) + providers.append(provider("working", arm_working, skill_name)) if full_mode and arm_baseline: - providers.append(build_provider("baseline", arm_baseline, model)) + providers.append(provider("baseline", arm_baseline)) default_test: dict = {"options": {"disableVarExpansion": True}} if skill_name: @@ -339,10 +410,13 @@ def main() -> int: config_path.write_text(json.dumps(config, indent=2)) # Report + model_label = model or "runtime default" if skill_name: - print(f"Mode: {len(providers)} arms, skill '{skill_name}', model: {model}") + print( + f"Mode: {len(providers)} arms, skill '{skill_name}', runtime: {runtime}, model: {model_label}" + ) else: - print(f"Mode: {len(providers)} arms, AGENTS.md only, model: {model}") + print(f"Mode: {len(providers)} arms, AGENTS.md only, runtime: {runtime}, model: {model_label}") print() print(f"Changes detected (vs {base_branch}):") diff --git a/scripts/tests/ci/prek/test_skill_eval.py b/scripts/tests/ci/prek/test_skill_eval.py new file mode 100644 index 0000000000000..1e1635a4fc936 --- /dev/null +++ b/scripts/tests/ci/prek/test_skill_eval.py @@ -0,0 +1,174 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import importlib.util +from pathlib import Path +from types import SimpleNamespace + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[4] +MODULE_PATH = REPO_ROOT / "dev" / "skill-evals" / "eval.py" + + +@pytest.fixture(scope="module") +def skill_eval_module(): + spec = importlib.util.spec_from_file_location("skill_eval", MODULE_PATH) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.mark.parametrize( + ("configured", "expected"), + [ + pytest.param(None, "claude", id="default"), + pytest.param("Claude", "claude", id="case-insensitive"), + pytest.param("codex", "codex", id="codex"), + ], +) +def test_get_runtime(monkeypatch, skill_eval_module, configured, expected): + if configured is None: + monkeypatch.delenv("AGENT_RUNTIME", raising=False) + else: + monkeypatch.setenv("AGENT_RUNTIME", configured) + + assert skill_eval_module.get_runtime() == expected + + +def test_get_runtime_rejects_unknown_runtime(monkeypatch, skill_eval_module): + monkeypatch.setenv("AGENT_RUNTIME", "unknown") + + with pytest.raises(ValueError, match="claude, codex"): + skill_eval_module.get_runtime() + + +def test_build_codex_provider(skill_eval_module, tmp_path): + working_dir = tmp_path / "worktree" + + provider = skill_eval_module.build_codex_provider( + "working", working_dir, "gpt-test", detect_skill_usage=True + ) + + assert provider == { + "id": "openai:codex-sdk", + "label": "working", + "config": { + "approval_policy": "never", + "cli_config": { + "history": {"persistence": "none"}, + "project_doc_max_bytes": skill_eval_module.CODEX_PROJECT_DOC_MAX_BYTES, + }, + "enable_streaming": True, + "model": "gpt-test", + "network_access_enabled": False, + "output_schema": skill_eval_module.OUTPUT_FORMAT["schema"], + "sandbox_mode": "read-only", + "web_search_mode": "disabled", + "working_dir": str(working_dir), + }, + "transform": "JSON.parse(output)", + } + + +def test_output_schema_requires_every_property(skill_eval_module): + schema = skill_eval_module.OUTPUT_FORMAT["schema"] + + assert set(schema["required"]) == set(schema["properties"]) + + +def test_build_codex_provider_uses_runtime_default_model(skill_eval_module, tmp_path): + provider = skill_eval_module.build_codex_provider("main", tmp_path / "worktree", None) + + assert "model" not in provider["config"] + assert "enable_streaming" not in provider["config"] + + +def test_build_claude_provider_keeps_existing_configuration(skill_eval_module, tmp_path): + provider = skill_eval_module.build_provider("main", tmp_path, "claude-test", "example-skill") + + assert provider["id"] == "anthropic:claude-agent-sdk" + assert provider["label"] == "main" + assert provider["config"] == { + "apiKeyRequired": False, + "append_allowed_tools": ["Read", "Grep", "Glob"], + "model": "claude-test", + "output_format": skill_eval_module.OUTPUT_FORMAT, + "setting_sources": ["project"], + "skills": ["example-skill"], + "working_dir": str(tmp_path), + } + + +@pytest.fixture +def promptfoo_layout(monkeypatch, skill_eval_module, tmp_path): + """Build a prek-style node env and return an SDK installer for either location. + + ``install_root`` mirrors a package listed in the hook's + additional_dependencies; ``bundled`` mirrors promptfoo's own optional + dependency, nested inside its package. + """ + install_root = tmp_path / "node_modules" + promptfoo = install_root / "promptfoo" + bundled = promptfoo / "node_modules" + promptfoo.mkdir(parents=True) + (promptfoo / "package.json").write_text("{}") + promptfoo_bin = promptfoo / "promptfoo" + promptfoo_bin.touch() + monkeypatch.setattr(skill_eval_module.shutil, "which", lambda _: str(promptfoo_bin)) + monkeypatch.setattr( + skill_eval_module, + "run", + lambda *args, **kwargs: SimpleNamespace(stdout=f"{skill_eval_module.PROMPTFOO_VERSION}\n"), + ) + + def install(location: Path, package: str) -> None: + sdk = location.joinpath(*package.split("/")) + sdk.mkdir(parents=True) + (sdk / "package.json").write_text('{"version": "1.2.3"}') + + return SimpleNamespace(install_root=install_root, bundled=bundled, install=install) + + +@pytest.mark.parametrize("runtime", ["claude", "codex"]) +def test_find_sdk_modules_resolves_the_hook_installed_copy(skill_eval_module, promptfoo_layout, runtime): + promptfoo_layout.install(promptfoo_layout.install_root, skill_eval_module.SDK_PACKAGES[runtime]) + + assert skill_eval_module.find_sdk_modules(runtime) == promptfoo_layout.install_root + + +@pytest.mark.parametrize("runtime", ["claude", "codex"]) +def test_find_sdk_modules_ignores_the_copy_bundled_with_promptfoo( + skill_eval_module, promptfoo_layout, runtime +): + promptfoo_layout.install(promptfoo_layout.bundled, skill_eval_module.SDK_PACKAGES[runtime]) + + with pytest.raises(SystemExit) as exit_info: + skill_eval_module.find_sdk_modules(runtime) + + assert exit_info.value.code == 1 + + +def test_find_sdk_modules_requires_the_selected_runtimes_sdk(skill_eval_module, promptfoo_layout): + promptfoo_layout.install(promptfoo_layout.install_root, skill_eval_module.SDK_PACKAGES["claude"]) + + with pytest.raises(SystemExit) as exit_info: + skill_eval_module.find_sdk_modules("codex") + + assert exit_info.value.code == 1