From dd123ea6c5dd3f274572537c317d0aec1ad65e31 Mon Sep 17 00:00:00 2001 From: stranske Date: Fri, 31 Jul 2026 04:50:22 -0500 Subject: [PATCH 1/3] fix(verifier): vendor the LLM model registry into .workflows-lib Compare-mode verification has been reporting "compare mode requires two cross-family verifier judges; available families: none" on every repo even though OPENAI_API_KEY and CLAUDE_API_STRANSKE are configured. The verifier sparse-checkout of stranske/Workflows vendored `tools` but not `config`, and tools/llm_registry.py resolves config/llm_slots.json and config/model_registry.json relative to that vendored tree, so both judge slots resolved to no model and the LLM evaluation never ran. agents-verify-to-issue-v2.yml vendors the same client for followup_issue_generator.py and had the same omission. --- .../workflows/agents-verify-to-issue-v2.yml | 1 + .../workflows/reusable-agents-verifier.yml | 1 + tests/workflows/test_workflow_llm_installs.py | 53 +++++++++++++++++++ 3 files changed, 55 insertions(+) diff --git a/.github/workflows/agents-verify-to-issue-v2.yml b/.github/workflows/agents-verify-to-issue-v2.yml index ab6f6c34a..1aefcee67 100644 --- a/.github/workflows/agents-verify-to-issue-v2.yml +++ b/.github/workflows/agents-verify-to-issue-v2.yml @@ -79,6 +79,7 @@ jobs: .github/scripts/github-api-with-retry.js .github/scripts/terminal_disposition.js .github/scripts/token_load_balancer.js + config scripts/langchain tools sparse-checkout-cone-mode: false diff --git a/.github/workflows/reusable-agents-verifier.yml b/.github/workflows/reusable-agents-verifier.yml index f3d3753da..fb6188182 100644 --- a/.github/workflows/reusable-agents-verifier.yml +++ b/.github/workflows/reusable-agents-verifier.yml @@ -125,6 +125,7 @@ jobs: .github/agents/registry.yml .github/scripts .github/codex/prompts + config scripts tools sparse-checkout-cone-mode: false diff --git a/tests/workflows/test_workflow_llm_installs.py b/tests/workflows/test_workflow_llm_installs.py index ea99d355c..727b83e71 100644 --- a/tests/workflows/test_workflow_llm_installs.py +++ b/tests/workflows/test_workflow_llm_installs.py @@ -15,6 +15,7 @@ AUTO_PILOT = WORKFLOWS_DIR / "agents-auto-pilot.yml" ISSUE_OPTIMIZER = WORKFLOWS_DIR / "agents-issue-optimizer.yml" VERIFIER = WORKFLOWS_DIR / "reusable-agents-verifier.yml" +VERIFY_TO_ISSUE = WORKFLOWS_DIR / "agents-verify-to-issue-v2.yml" REUSABLE_CODEX_RUN = WORKFLOWS_DIR / "reusable-codex-run.yml" REUSABLE_CLAUDE_RUN = WORKFLOWS_DIR / "reusable-claude-run.yml" NEEDS_HUMAN_COMMENT = Path("agents/codex-1447.md") @@ -26,6 +27,8 @@ "gpt-5.4": (0, 125, 0), } ACTIONS_CACHE_V6_REF = "actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9" +LLM_REGISTRY_MODULE = "tools/llm_registry.py" +LLM_CONFIG_PATHS = (Path("config/llm_slots.json"), Path("config/model_registry.json")) def _load_text(path: Path) -> str: @@ -90,6 +93,20 @@ def _iter_steps(workflow: dict) -> list[dict]: return steps +def _sparse_checkout_paths(step: dict) -> list[str]: + raw = str((step.get("with") or {}).get("sparse-checkout", "")) + return [line.strip() for line in raw.splitlines() if line.strip()] + + +def _workflows_library_checkout_steps(workflow: dict) -> list[dict]: + steps = [] + for step in _iter_steps(workflow): + with_block = step.get("with") or {} + if with_block.get("repository") == "stranske/Workflows" and "sparse-checkout" in with_block: + steps.append(step) + return steps + + def _find_step_by_name(workflow: dict, step_name: str) -> dict: for step in _iter_steps(workflow): if step.get("name") == step_name: @@ -241,6 +258,42 @@ def test_reusable_agents_verifier_pip_cache_is_configured() -> None: _assert_pip_cache(workflow, ".workflows-lib/tools/requirements-llm.txt", VERIFIER.name) +@pytest.mark.parametrize("workflow_path", [VERIFIER, VERIFY_TO_ISSUE]) +def test_llm_workflows_vendor_the_model_registry_config(workflow_path: Path) -> None: + workflow = _load_workflow(workflow_path) + checkouts = _workflows_library_checkout_steps(workflow) + assert checkouts, f"{workflow_path.name} must sparse-checkout stranske/Workflows" + + vendors_tools = False + for step in checkouts: + paths = _sparse_checkout_paths(step) + if "tools" not in paths: + continue + vendors_tools = True + missing = [entry for entry in ("config",) if entry not in paths] + assert not missing, ( + f"{workflow_path.name} vendors `tools` but not {missing}; " + f"{LLM_REGISTRY_MODULE} resolves {', '.join(str(p) for p in LLM_CONFIG_PATHS)} " + "relative to the vendored tree, so every judge slot resolves to no model and " + 'compare mode reports "available families: none".' + ) + + assert vendors_tools, f"{workflow_path.name} must vendor `tools` for the LLM client" + + +def test_bundled_llm_config_resolves_two_cross_family_judges() -> None: + for config_path in LLM_CONFIG_PATHS: + assert config_path.is_file(), f"Vendored LLM config {config_path} must exist" + + from tools.llm_registry import load_slot_config + + families = {slot.provider for slot in load_slot_config() if slot.model} + assert len(families) >= 2, ( + "Compare mode needs two cross-family judges; the bundled config resolved " + f"models for {sorted(families) or 'no provider'}." + ) + + def test_reusable_codex_run_persists_refreshed_auth_bundle_with_app_token() -> None: workflow = _load_workflow(REUSABLE_CODEX_RUN) step = _find_step_by_name(workflow, "Persist refreshed Codex auth secret") From 3f239a25930baf821064f5b290adb37e63969551 Mon Sep 17 00:00:00 2001 From: stranske Date: Fri, 31 Jul 2026 05:13:58 -0500 Subject: [PATCH 2/3] fix(verifier): vendor config into the new-PR verifier path too agents-verify-to-new-pr.yml runs the same scripts/langchain client as agents-verify-to-issue-v2.yml but sparse-checked out `tools` without `config`, so verify:create-new-pr still resolved no model on either judge slot and fell back to the non-LLM path while verify:create-issue got the fix. Replace the hand-maintained parametrize list with discovery over workflows that vendor `tools` and reference scripts/langchain, so a future workflow vendoring the client inherits the guard, and add a coverage test that fails if discovery stops seeing the three known verifier surfaces. Pin the bundled-config judge test to the vendored files with monkeypatch so an ambient LANGCHAIN_SLOT_CONFIG / LANGCHAIN_MODEL_REGISTRY_CONFIG override cannot make it pass. --- .github/workflows/agents-verify-to-new-pr.yml | 1 + tests/workflows/test_workflow_llm_installs.py | 50 ++++++++++++++++++- 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/.github/workflows/agents-verify-to-new-pr.yml b/.github/workflows/agents-verify-to-new-pr.yml index 3428d1c11..76b6d489d 100644 --- a/.github/workflows/agents-verify-to-new-pr.yml +++ b/.github/workflows/agents-verify-to-new-pr.yml @@ -78,6 +78,7 @@ jobs: .github/scripts/github-api-with-retry.js .github/scripts/terminal_disposition.js .github/scripts/token_load_balancer.js + config scripts/langchain tools sparse-checkout-cone-mode: false diff --git a/tests/workflows/test_workflow_llm_installs.py b/tests/workflows/test_workflow_llm_installs.py index 727b83e71..70966b9a4 100644 --- a/tests/workflows/test_workflow_llm_installs.py +++ b/tests/workflows/test_workflow_llm_installs.py @@ -29,6 +29,9 @@ ACTIONS_CACHE_V6_REF = "actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9" LLM_REGISTRY_MODULE = "tools/llm_registry.py" LLM_CONFIG_PATHS = (Path("config/llm_slots.json"), Path("config/model_registry.json")) +LANGCHAIN_ENTRYPOINT_DIR = "scripts/langchain" +VERIFY_TO_NEW_PR = WORKFLOWS_DIR / "agents-verify-to-new-pr.yml" +KNOWN_LLM_CLIENT_WORKFLOWS = (VERIFIER, VERIFY_TO_ISSUE, VERIFY_TO_NEW_PR) def _load_text(path: Path) -> str: @@ -107,6 +110,28 @@ def _workflows_library_checkout_steps(workflow: dict) -> list[dict]: return steps +def _discover_llm_client_workflows() -> list[Path]: + """Workflows that vendor `tools` and then run the LangChain client from it. + + Discovery rather than an explicit list: any future workflow that vendors the + client inherits the config-vendoring requirement automatically. + """ + matches = [] + for path in sorted(WORKFLOWS_DIR.glob("*.yml")) + sorted(WORKFLOWS_DIR.glob("*.yaml")): + text = path.read_text(encoding="utf-8") + if LANGCHAIN_ENTRYPOINT_DIR not in text: + continue + workflow = yaml.safe_load(text) + if not isinstance(workflow, dict): + continue + if any( + "tools" in _sparse_checkout_paths(step) + for step in _workflows_library_checkout_steps(workflow) + ): + matches.append(path) + return matches + + def _find_step_by_name(workflow: dict, step_name: str) -> dict: for step in _iter_steps(workflow): if step.get("name") == step_name: @@ -258,7 +283,18 @@ def test_reusable_agents_verifier_pip_cache_is_configured() -> None: _assert_pip_cache(workflow, ".workflows-lib/tools/requirements-llm.txt", VERIFIER.name) -@pytest.mark.parametrize("workflow_path", [VERIFIER, VERIFY_TO_ISSUE]) +def test_llm_client_workflow_discovery_covers_the_known_verifier_surfaces() -> None: + discovered = set(_discover_llm_client_workflows()) + missing = [path.name for path in KNOWN_LLM_CLIENT_WORKFLOWS if path not in discovered] + assert not missing, ( + f"Discovery no longer sees {missing}; the config-vendoring guard below would " + "silently stop covering them." + ) + + +@pytest.mark.parametrize( + "workflow_path", _discover_llm_client_workflows(), ids=lambda path: path.name +) def test_llm_workflows_vendor_the_model_registry_config(workflow_path: Path) -> None: workflow = _load_workflow(workflow_path) checkouts = _workflows_library_checkout_steps(workflow) @@ -281,12 +317,22 @@ def test_llm_workflows_vendor_the_model_registry_config(workflow_path: Path) -> assert vendors_tools, f"{workflow_path.name} must vendor `tools` for the LLM client" -def test_bundled_llm_config_resolves_two_cross_family_judges() -> None: +def test_bundled_llm_config_resolves_two_cross_family_judges( + monkeypatch: pytest.MonkeyPatch, +) -> None: + slots_path, registry_path = LLM_CONFIG_PATHS for config_path in LLM_CONFIG_PATHS: assert config_path.is_file(), f"Vendored LLM config {config_path} must exist" + from tools import llm_registry from tools.llm_registry import load_slot_config + # Pin resolution to the bundled files so an ambient override cannot make this pass. + monkeypatch.delenv(llm_registry.ENV_SLOT_CONFIG, raising=False) + monkeypatch.delenv(llm_registry.ENV_MODEL_REGISTRY_CONFIG, raising=False) + monkeypatch.setattr(llm_registry, "DEFAULT_SLOT_CONFIG_PATH", slots_path.resolve()) + monkeypatch.setattr(llm_registry, "DEFAULT_MODEL_REGISTRY_CONFIG_PATH", registry_path.resolve()) + families = {slot.provider for slot in load_slot_config() if slot.model} assert len(families) >= 2, ( "Compare mode needs two cross-family judges; the bundled config resolved " From 71e2b41c8b3f10a2b75d400a63a5e33ee6a4759f Mon Sep 17 00:00:00 2001 From: stranske Date: Fri, 31 Jul 2026 05:16:33 -0500 Subject: [PATCH 3/3] fix(templates): sync the new-PR verifier config vendoring to the consumer copy templates/consumer-repo/.github/workflows/agents-verify-to-new-pr.yml is a byte-parity copy (not allowlisted baseline drift), so the root-only edit tripped Health 74 Template Drift. Consumer repos run this workflow, so the copy needs the same `config` entry for verify:create-new-pr to resolve the judge slots there too. Verified with the exact failing command: python scripts/check_template_drift.py --allowlist config/template-drift-allowlist.txt -> unallowlisted drift 0 (was 1). --- .../consumer-repo/.github/workflows/agents-verify-to-new-pr.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/templates/consumer-repo/.github/workflows/agents-verify-to-new-pr.yml b/templates/consumer-repo/.github/workflows/agents-verify-to-new-pr.yml index 3428d1c11..76b6d489d 100644 --- a/templates/consumer-repo/.github/workflows/agents-verify-to-new-pr.yml +++ b/templates/consumer-repo/.github/workflows/agents-verify-to-new-pr.yml @@ -78,6 +78,7 @@ jobs: .github/scripts/github-api-with-retry.js .github/scripts/terminal_disposition.js .github/scripts/token_load_balancer.js + config scripts/langchain tools sparse-checkout-cone-mode: false