From 695396522963ec8ce8fe00a38e5abcaf99ee8712 Mon Sep 17 00:00:00 2001 From: Paulo Lacerda Date: Fri, 7 Aug 2026 17:41:52 -0300 Subject: [PATCH 1/2] fix(cicd): let the eval gate target the agent version the pipeline produced `agentops.yaml` pins a fully-qualified Foundry agent whose URL ends in a version segment (`.../agents/helpdeskbot/versions/11`). The generated dev, QA, and prod pipelines run `azd provision`, which publishes a new agent version, and then run the eval gate. Nothing between those two steps retargeted the pin, so the gate scored version 11 while the pipeline had just shipped version 12. The run went green on an artifact it had already replaced. A regression introduced by the deploy could not fail its own gate, and in `prod` that same shape gates a release. The seam for the fix already existed and was dead. `RunOptions.agent_override` sits at `orchestrator.py:56` and is consumed by all three execution backends (`_run_evaluation_local`, `_run_evaluation_cloud`, `_run_evaluation_azd`) via `classify_agent(options.agent_override or config.agent, config.protocol)`. Repo-wide, `agent_override` appeared in exactly those four lines: no CLI flag, no env var, no tests. Wiring it up is strictly cheaper and less risky than inventing a parallel mechanism. The reporter's first choice was for the deploy job to emit the resolved version as a job output. That is not implementable against the shipped templates. The only job that could observe a new version is `provision`, which runs `azd provision` and nothing else, and `azd` does not surface a Foundry agent version in any form AgentOps parses. Resolving "latest" at eval time was the second option and is equally unavailable: the Foundry SDK surface used in this repo is `client.agents.get_version(name, version)` and `client.agents.create_version(name, body=...)`, with no list-versions helper to build "latest" on. So this implements the consumption half of option 1, which is the half that has to exist regardless of who supplies the value. `agentops eval run` now takes `--agent`, falling back to `$AGENTOPS_AGENT`. A bare number replaces just the version segment of the configured target; a full agent reference replaces the target outright; empty or unset leaves the config untouched, so existing pipelines are unchanged. Every generated eval step forwards the variable: GitHub Actions emits `AGENTOPS_AGENT: ${{ env.AGENTOPS_AGENT || vars.AGENTOPS_AGENT }}` so a job-level env wins and a repo variable is the fallback, and Azure DevOps emits `AGENTOPS_AGENT: $(AGENTOPS_AGENT)`. Both cover placeholder, azd, and prompt-agent deploy modes across dev, QA, and prod. The official Foundry eval-runner branch is left alone because it does not shell out to `agentops eval run`. Azure DevOps leaves `$(NAME)` in the environment verbatim when the variable is undefined, unlike GitHub which substitutes an empty string. Without a guard that literal would be parsed as an agent expression, so an unexpanded CI token is treated as no override. That keeps ADO templates working without forcing every pipeline to declare a `variables:` default. `workflow analyze` now reports the pinned version as an `agent_version_pin` signal plus a warning naming `AGENTOPS_AGENT`, so the drift is visible before it reaches CI. The signal is restricted to `foundry_hosted` targets because prompt-agent deploys were never affected: `stage_prompt_agent_candidate` already writes `agentops.candidate.yaml` with a fresh `agent` value that the eval step consumes. Two corrections to the issue as filed. The templates order jobs `provision -> eval -> deploy` (azd) and `eval -> build -> deploy` (placeholder), not `provision -> deploy -> eval`; the substance holds because `azd provision` is what bumps the agent version. And the bug is narrower than reported, since prompt-agent mode was already immune. Fixes #388 Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: bcb9c0b6-d506-46dc-90d2-8120413166ee --- CHANGELOG.md | 27 ++++ src/agentops/cli/app.py | 42 +++++ src/agentops/core/agentops_config.py | 49 ++++++ src/agentops/services/cicd.py | 13 +- src/agentops/services/workflow_analysis.py | 28 +++- tests/unit/test_agentops_config.py | 43 +++++ tests/unit/test_cicd.py | 72 +++++++++ tests/unit/test_eval_agent_override.py | 173 +++++++++++++++++++++ tests/unit/test_workflow_analysis.py | 50 ++++++ 9 files changed, 493 insertions(+), 4 deletions(-) create mode 100644 tests/unit/test_eval_agent_override.py diff --git a/CHANGELOG.md b/CHANGELOG.md index e35fc78..2178290 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,33 @@ This format follows [Keep a Changelog](https://keepachangelog.com/) and adheres ## [Unreleased] +### Fixed +- **Generated workflows evaluated a stale agent version.** `agentops.yaml` pins + a fully-qualified Foundry agent that ends in a version segment + (`.../agents/helpdeskbot/versions/11`), and nothing in the generated dev, QA, + or prod pipelines ever retargeted it. `azd provision` published a new agent + version and the eval job that ran next still scored the previous one, so the + gate reported green on an artifact the pipeline had already replaced. A + regression introduced by the deploy could not fail its own quality gate, and + in `prod` that same shape gated a release. + + `RunOptions.agent_override` already existed in the orchestrator and was + already consumed by all three execution backends, but nothing ever set it. + `agentops eval run` now accepts `--agent`, falling back to the + `AGENTOPS_AGENT` environment variable, and every generated eval step on + GitHub Actions and Azure DevOps forwards that variable. A bare number + (`--agent 12`) replaces just the version segment of the configured target; + a full agent reference replaces the target outright. Unset means unchanged, + so existing pipelines behave exactly as before. `agentops.yaml` stays + declarative and CI never writes to tracked config. + + Azure DevOps leaves `$(NAME)` in the environment verbatim when a variable is + undefined, so an unexpanded token is treated as "no override" rather than as + an agent expression. `workflow analyze` now reports the pinned version as a + signal and a warning, so the drift is visible before it reaches CI. + Prompt-agent deploys were never affected: they already stage + `agentops.candidate.yaml` with a fresh `agent` value for the eval step. + ## [0.8.5] - 2026-08-07 ### Fixed diff --git a/src/agentops/cli/app.py b/src/agentops/cli/app.py index 3214819..b509ab0 100644 --- a/src/agentops/cli/app.py +++ b/src/agentops/cli/app.py @@ -2332,6 +2332,17 @@ def cmd_eval_run( report_format: Annotated[ str, typer.Option("--format", "-f", help="Report format: md, html, or all.") ] = "md", + agent: Annotated[ + str | None, + typer.Option( + "--agent", + help=( + "Override the agent target for this run. Accepts a full agent " + "reference or a bare Foundry version (e.g. 12) to replace the " + "version pinned in agentops.yaml. Falls back to $AGENTOPS_AGENT." + ), + ), + ] = None, explain: Annotated[str | None, typer.Argument(hidden=True)] = None, ) -> None: """Run an evaluation defined in agentops.yaml.""" @@ -2366,6 +2377,7 @@ def cmd_eval_run( config_path=config_path, output=output, baseline=baseline, + agent=agent, ) @@ -3335,12 +3347,25 @@ def _apply_http_redteam_defaults(target: dict[str, Any], cfg: AgentOpsConfig) -> target.setdefault("stream", cfg.stream.model_dump(exclude_none=True)) +def _is_unexpanded_ci_token(value: str) -> bool: + """Return True when *value* is an unexpanded CI variable reference.""" + candidate = value.strip() + return candidate.startswith("$(") or candidate.startswith("${{") + + def _run_flat_schema_eval( *, config_path: Path, output: Path | None, baseline: Path | None, + agent: str | None = None, ) -> None: + import os + + from agentops.core.agentops_config import ( + AGENT_OVERRIDE_ENV, + apply_agent_version_override, + ) from agentops.core.config_loader import load_agentops_config from agentops.pipeline.orchestrator import ( RunOptions, @@ -3357,6 +3382,22 @@ def _run_flat_schema_eval( ) raise typer.Exit(code=1) from exc + requested_agent = agent if agent is not None else os.environ.get(AGENT_OVERRIDE_ENV) + if requested_agent is not None and _is_unexpanded_ci_token(requested_agent): + # Azure DevOps leaves `$(NAME)` verbatim when a variable is undefined, + # and GitHub expressions can leak through the same way. Treat that as + # "no override" instead of trying to evaluate it as an agent target. + requested_agent = None + agent_override: str | None = None + if requested_agent is not None and requested_agent.strip(): + try: + agent_override = apply_agent_version_override(config_obj.agent, requested_agent) + except ValueError as exc: + typer.echo(f"{_cli_error('Error')}: {exc}", err=True) + raise typer.Exit(code=1) from exc + if agent_override != config_obj.agent: + typer.echo(f"{_cli_label('Agent override')}: {agent_override}") + use_default_layout = output is None if use_default_layout: output_dir: Path = _default_flat_output_dir(config_path) @@ -3369,6 +3410,7 @@ def _run_flat_schema_eval( output_dir=output_dir, baseline_path=baseline.resolve() if baseline else None, progress=lambda msg: typer.echo(msg), + agent_override=agent_override, ) try: diff --git a/src/agentops/core/agentops_config.py b/src/agentops/core/agentops_config.py index 0cb0d3a..14f813e 100644 --- a/src/agentops/core/agentops_config.py +++ b/src/agentops/core/agentops_config.py @@ -1271,6 +1271,55 @@ def _parse_hosted_agent_reference(url: str) -> tuple[Optional[str], Optional[str return (name or None), (version or None) +#: Environment variable that overrides the ``agent:`` target for a single run. +#: CI exports it so the eval gate scores the agent version the surrounding +#: deploy/provision step just produced instead of the version pinned in +#: ``agentops.yaml``. +AGENT_OVERRIDE_ENV = "AGENTOPS_AGENT" + +#: A bare Foundry agent version. Versions are numeric, so anything else in an +#: override is unambiguously a complete agent expression. +_BARE_AGENT_VERSION_RE = re.compile(r"\d+") + + +def apply_agent_version_override(agent: str, override: str) -> str: + """Resolve the effective ``agent`` expression for an overridden run. + + *override* is either a complete agent expression (a hosted endpoint URL, + ``name:version``, or ``model:``) or a bare numeric version such + as ``12``. A bare version is substituted into the version slot of *agent*, + so CI only has to carry the number Foundry just produced instead of + rebuilding the whole endpoint URL. + + An empty *override* leaves *agent* untouched. + """ + + base = agent.strip() + candidate = override.strip() + if not candidate: + return base + if not _BARE_AGENT_VERSION_RE.fullmatch(candidate): + return candidate + + match = _HOSTED_AGENT_REFERENCE_RE.search(base) + if match is not None: + start, end = match.span("version") + return base[:start] + candidate + base[end:] + + lowered = base.lower() + if not lowered.startswith(("http://", "https://", "model:")): + name, separator, _version = base.partition(":") + if separator and name.strip(): + return f"{name.strip()}:{candidate}" + + raise ValueError( + f"cannot apply agent version {candidate!r}: {base!r} has no version " + "segment to replace. Override with a full agent reference such as a " + "hosted endpoint URL ending in '/agents//versions/' or " + "':'." + ) + + def classify_agent( agent: str, protocol: Optional[Protocol] = None, diff --git a/src/agentops/services/cicd.py b/src/agentops/services/cicd.py index aa0b912..065c670 100644 --- a/src/agentops/services/cicd.py +++ b/src/agentops/services/cicd.py @@ -7,6 +7,7 @@ from pathlib import Path from typing import Dict, List, Mapping, Sequence, Tuple +from agentops.core.agentops_config import AGENT_OVERRIDE_ENV from agentops.pipeline.official_eval import ( AGENTOPS_CLOUD_RUNNER, AGENTOPS_LOCAL_RUNNER, @@ -447,6 +448,7 @@ def _github_eval_substitutions( AZURE_OPENAI_DEPLOYMENT: ${{{{ vars.AZURE_OPENAI_DEPLOYMENT }}}} AZURE_OPENAI_MODEL_NAME: ${{{{ vars.AZURE_OPENAI_MODEL_NAME }}}} APPLICATIONINSIGHTS_CONNECTION_STRING: ${{{{ secrets.APPLICATIONINSIGHTS_CONNECTION_STRING || vars.APPLICATIONINSIGHTS_CONNECTION_STRING }}}} + {AGENT_OVERRIDE_ENV}: ${{{{ env.{AGENT_OVERRIDE_ENV} || vars.{AGENT_OVERRIDE_ENV} }}}} run: | set +e agentops eval run --config "{config_path}" --output "{_CI_EVAL_OUTPUT}" @@ -498,6 +500,7 @@ def _github_eval_substitutions( AZURE_OPENAI_DEPLOYMENT: ${{{{ vars.AZURE_OPENAI_DEPLOYMENT }}}} AZURE_OPENAI_MODEL_NAME: ${{{{ vars.AZURE_OPENAI_MODEL_NAME }}}} APPLICATIONINSIGHTS_CONNECTION_STRING: ${{{{ secrets.APPLICATIONINSIGHTS_CONNECTION_STRING || vars.APPLICATIONINSIGHTS_CONNECTION_STRING }}}} + {AGENT_OVERRIDE_ENV}: ${{{{ env.{AGENT_OVERRIDE_ENV} || vars.{AGENT_OVERRIDE_ENV} }}}} run: | set +e agentops eval run --config "$AGENTOPS_CI_CONFIG" --output "{_CI_EVAL_OUTPUT}" @@ -601,6 +604,7 @@ def _github_eval_substitutions( AZURE_OPENAI_DEPLOYMENT: ${{{{ vars.AZURE_OPENAI_DEPLOYMENT }}}} AZURE_OPENAI_MODEL_NAME: ${{{{ vars.AZURE_OPENAI_MODEL_NAME }}}} APPLICATIONINSIGHTS_CONNECTION_STRING: ${{{{ secrets.APPLICATIONINSIGHTS_CONNECTION_STRING || vars.APPLICATIONINSIGHTS_CONNECTION_STRING }}}} + {AGENT_OVERRIDE_ENV}: ${{{{ env.{AGENT_OVERRIDE_ENV} || vars.{AGENT_OVERRIDE_ENV} }}}} run: | set +e {_github_baseline_autodetect_block(kind)} agentops eval run --config \"{config_path}\"{_baseline_arg_suffix(kind)} @@ -656,7 +660,8 @@ def _ado_eval_substitutions( AZURE_OPENAI_ENDPOINT: $(AZURE_OPENAI_ENDPOINT) AZURE_OPENAI_DEPLOYMENT: $(AZURE_OPENAI_DEPLOYMENT) AZURE_OPENAI_MODEL_NAME: $(AZURE_OPENAI_MODEL_NAME) - APPLICATIONINSIGHTS_CONNECTION_STRING: $(APPLICATIONINSIGHTS_CONNECTION_STRING)""", + APPLICATIONINSIGHTS_CONNECTION_STRING: $(APPLICATIONINSIGHTS_CONNECTION_STRING) + {AGENT_OVERRIDE_ENV}: $({AGENT_OVERRIDE_ENV})""", base_indent, ), "__EVAL_ARTIFACT_TARGET__": _CI_EVAL_OUTPUT, @@ -700,7 +705,8 @@ def _ado_eval_substitutions( AZURE_OPENAI_ENDPOINT: $(AZURE_OPENAI_ENDPOINT) AZURE_OPENAI_DEPLOYMENT: $(AZURE_OPENAI_DEPLOYMENT) AZURE_OPENAI_MODEL_NAME: $(AZURE_OPENAI_MODEL_NAME) - APPLICATIONINSIGHTS_CONNECTION_STRING: $(APPLICATIONINSIGHTS_CONNECTION_STRING)""", + APPLICATIONINSIGHTS_CONNECTION_STRING: $(APPLICATIONINSIGHTS_CONNECTION_STRING) + {AGENT_OVERRIDE_ENV}: $({AGENT_OVERRIDE_ENV})""", base_indent, ), "__EVAL_ARTIFACT_TARGET__": _CI_EVAL_OUTPUT, @@ -787,7 +793,8 @@ def _ado_eval_substitutions( AZURE_OPENAI_ENDPOINT: $(AZURE_OPENAI_ENDPOINT) AZURE_OPENAI_DEPLOYMENT: $(AZURE_OPENAI_DEPLOYMENT) AZURE_OPENAI_MODEL_NAME: $(AZURE_OPENAI_MODEL_NAME) - APPLICATIONINSIGHTS_CONNECTION_STRING: $(APPLICATIONINSIGHTS_CONNECTION_STRING)""", + APPLICATIONINSIGHTS_CONNECTION_STRING: $(APPLICATIONINSIGHTS_CONNECTION_STRING) + {AGENT_OVERRIDE_ENV}: $({AGENT_OVERRIDE_ENV})""", base_indent, ), "__EVAL_ARTIFACT_TARGET__": ".agentops/results/latest", diff --git a/src/agentops/services/workflow_analysis.py b/src/agentops/services/workflow_analysis.py index cd1e2d5..3230d0d 100644 --- a/src/agentops/services/workflow_analysis.py +++ b/src/agentops/services/workflow_analysis.py @@ -8,7 +8,7 @@ from pathlib import Path from typing import Any, Dict, Iterable, List, Optional, Sequence -from agentops.core.agentops_config import classify_agent +from agentops.core.agentops_config import AGENT_OVERRIDE_ENV, classify_agent from agentops.core.azd_eval import find_eval_yaml, load_eval_recipe, recipe_metric_names from agentops.pipeline.official_eval import ( AGENTOPS_CLOUD_RUNNER, @@ -181,6 +181,28 @@ def analyze_workflow_project(directory: Path) -> WorkflowAnalysis: confidence="medium", ) ) + pinned_version = agentops.get("pinned_agent_version") + if pinned_version: + signals.append( + WorkflowSignal( + "agent_version_pin", + "Pinned agent version", + ( + f"agentops.yaml pins agent version {pinned_version}. Generated " + "deploy pipelines publish a new Foundry version, so the eval gate " + "scores the pinned version rather than the one the run produced. " + f"Export {AGENT_OVERRIDE_ENV} (or pass `agentops eval run --agent`) " + "with the version the deploy step resolved so the gate follows the " + "deployed artifact." + ), + "agentops.yaml", + confidence="medium", + ) + ) + warnings.append( + f"agentops.yaml pins agent version {pinned_version}; set " + f"{AGENT_OVERRIDE_ENV} in the eval job so the gate scores the deployed version." + ) bicep_files = _find_files(root, "*.bicep") if bicep_files: @@ -786,6 +808,7 @@ def _signal_type(key: str) -> str: "azd_project": "Deploy mode", "prompt_file": "Prompt source", "prompt_agent_bootstrap_missing": "Prompt-agent bootstrap", + "agent_version_pin": "Eval target", "bicep_infra": "Infrastructure", "ailz_manifest": "Landing zone", "ailz_preflight": "Preflight", @@ -820,6 +843,9 @@ def _agentops_signal(root: Path) -> Dict[str, Any]: "prompt_agent": target.kind == "foundry_prompt", "prompt_file": prompt_file, "prompt_agent_bootstrap": bool(bootstrap), + "pinned_agent_version": ( + target.version if target.kind == "foundry_hosted" else None + ), "signal": WorkflowSignal( "agentops_config", "AgentOps config", diff --git a/tests/unit/test_agentops_config.py b/tests/unit/test_agentops_config.py index 66c47de..c243846 100644 --- a/tests/unit/test_agentops_config.py +++ b/tests/unit/test_agentops_config.py @@ -8,6 +8,7 @@ from pydantic import ValidationError from agentops.core.agentops_config import ( + AGENT_OVERRIDE_ENV, AgentOpsConfig, DatasetSyncConfig, ObservabilityConfig, @@ -15,6 +16,7 @@ RubricConfig, RubricDimensionConfig, Threshold, + apply_agent_version_override, classify_agent, ) @@ -122,6 +124,47 @@ def test_unrecognized_value(self) -> None: classify_agent("just-a-name") +# --------------------------------------------------------------------------- +# apply_agent_version_override +# --------------------------------------------------------------------------- + + +_HOSTED = ( + "https://acct.services.ai.azure.com/api/projects/proj/agents/helpdeskbot/versions/11" +) + + +class TestApplyAgentVersionOverride: + """Regression for #388: the eval gate must be able to retarget the version.""" + + def test_env_var_name_is_stable(self) -> None: + assert AGENT_OVERRIDE_ENV == "AGENTOPS_AGENT" + + def test_bare_version_replaces_hosted_url_version(self) -> None: + result = apply_agent_version_override(_HOSTED, "12") + assert result.endswith("/agents/helpdeskbot/versions/12") + assert classify_agent(result).version == "12" + + def test_bare_version_replaces_prompt_agent_version(self) -> None: + assert apply_agent_version_override("helpdeskbot:11", "12") == "helpdeskbot:12" + + def test_empty_override_keeps_configured_agent(self) -> None: + assert apply_agent_version_override(_HOSTED, "") == _HOSTED + assert apply_agent_version_override(_HOSTED, " ") == _HOSTED + + def test_full_reference_override_wins_outright(self) -> None: + other = "https://acct.services.ai.azure.com/api/projects/other/agents/bot/versions/3" + assert apply_agent_version_override(_HOSTED, other) == other + + def test_bare_version_without_version_slot_is_rejected(self) -> None: + with pytest.raises(ValueError, match="no version segment"): + apply_agent_version_override("https://plain.example.com/chat", "12") + + def test_bare_version_against_model_target_is_rejected(self) -> None: + with pytest.raises(ValueError, match="no version segment"): + apply_agent_version_override("model:gpt-4o-mini", "12") + + # --------------------------------------------------------------------------- # Threshold parser # --------------------------------------------------------------------------- diff --git a/tests/unit/test_cicd.py b/tests/unit/test_cicd.py index 3efe634..cb95794 100644 --- a/tests/unit/test_cicd.py +++ b/tests/unit/test_cicd.py @@ -1408,3 +1408,75 @@ def test_workflow_install_lines_fall_back_to_main_for_dev_installs(tmp_path: Pat content = path.read_text(encoding="utf-8") assert "__AGENTOPS_INSTALL_SPEC__" not in content assert " @ git+https://github.com/Azure/agentops.git@main" in content + + +# --------------------------------------------------------------------------- +# Agent version override (#388) +# --------------------------------------------------------------------------- + + +_HOSTED_AGENT_CONFIG = ( + "version: 1\n" + "agent: https://acct.services.ai.azure.com/api/projects/proj/agents/helpdeskbot/versions/11\n" + "dataset: data.jsonl\n" + "protocol: responses\n" +) + + +def _seed_hosted_project(root: Path, *, azure_yaml: bool = False) -> None: + (root / "agentops.yaml").write_text(_HOSTED_AGENT_CONFIG, encoding="utf-8") + (root / "data.jsonl").write_text( + '{"input": "Hello", "expected": "Hello!"}\n', encoding="utf-8" + ) + if azure_yaml: + (root / "azure.yaml").write_text("name: sample\n", encoding="utf-8") + + +def test_github_eval_steps_forward_agent_version_override(tmp_path: Path) -> None: + """Regression for #388: eval must be retargetable without editing agentops.yaml.""" + + _seed_hosted_project(tmp_path, azure_yaml=True) + generate_cicd_workflows(directory=tmp_path, kinds=["dev", "qa", "prod"], force=True) + + for rel in (_DEV_PATH, _QA_PATH, _PROD_PATH): + content = (tmp_path / rel).read_text(encoding="utf-8") + assert ( + "AGENTOPS_AGENT: ${{ env.AGENTOPS_AGENT || vars.AGENTOPS_AGENT }}" in content + ), f"{rel} does not forward the agent override to the eval step" + assert isinstance(_read_yaml(tmp_path / rel), dict) + + +def test_azure_devops_eval_steps_forward_agent_version_override(tmp_path: Path) -> None: + _seed_hosted_project(tmp_path, azure_yaml=True) + generate_cicd_workflows( + directory=tmp_path, + kinds=["dev", "qa", "prod"], + platform="azure-devops", + force=True, + ) + + for rel in (_ADO_DEV, _ADO_QA, _ADO_PROD): + content = (tmp_path / rel).read_text(encoding="utf-8") + assert "AGENTOPS_AGENT: $(AGENTOPS_AGENT)" in content, ( + f"{rel} does not forward the agent override to the eval step" + ) + assert isinstance(_read_yaml(tmp_path / rel), dict) + + +def test_placeholder_and_prompt_agent_modes_also_forward_the_override( + tmp_path: Path, +) -> None: + """The override seam must not be azd-only.""" + + for mode, needle in ( + ("placeholder", "AGENTOPS_AGENT: ${{ env.AGENTOPS_AGENT || vars.AGENTOPS_AGENT }}"), + ("prompt-agent", "AGENTOPS_AGENT: ${{ env.AGENTOPS_AGENT || vars.AGENTOPS_AGENT }}"), + ): + root = tmp_path / mode + root.mkdir() + _seed_hosted_project(root) + generate_cicd_workflows( + directory=root, kinds=["dev"], deploy_mode=mode, force=True + ) + content = (root / _DEV_PATH).read_text(encoding="utf-8") + assert needle in content, f"{mode} deploy mode drops the agent override" diff --git a/tests/unit/test_eval_agent_override.py b/tests/unit/test_eval_agent_override.py new file mode 100644 index 0000000..595e4c0 --- /dev/null +++ b/tests/unit/test_eval_agent_override.py @@ -0,0 +1,173 @@ +"""`agentops eval run` must be able to retarget the agent version (#388). + +``agentops.yaml`` pins a fully-qualified Foundry agent that includes a version +segment. Generated pipelines publish a new version and then evaluate, so +without an override the gate scores the previous version and a regression the +run just introduced cannot fail it. ``--agent`` and ``$AGENTOPS_AGENT`` give CI +a way to point the gate at the version it actually produced, without CI writing +to the tracked config. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +from typer.testing import CliRunner + +from agentops.cli.app import app +from agentops.core.results import RunResult, RunSummary, TargetInfo + +runner = CliRunner() + +_HOSTED = ( + "https://acct.services.ai.azure.com/api/projects/proj/agents/helpdeskbot/versions/11" +) + + +def _passing_result() -> RunResult: + return RunResult( + started_at="2026-06-01T00:00:00+00:00", + finished_at="2026-06-01T00:01:00+00:00", + duration_seconds=60.0, + target=TargetInfo(kind="foundry_hosted", raw=_HOSTED), + dataset_path="dataset.jsonl", + evaluators=[], + rows=[], + aggregate_metrics={}, + thresholds=[], + summary=RunSummary( + items_total=0, + items_passed_all=0, + items_pass_rate=1.0, + thresholds_total=0, + thresholds_passed=0, + threshold_pass_rate=1.0, + overall_passed=True, + ), + ) + + +def _write_hosted_config(tmp_path: Path) -> Path: + dataset = tmp_path / "dataset.jsonl" + dataset.write_text(json.dumps({"input": "hi", "expected": "hi"}), encoding="utf-8") + config = tmp_path / "agentops.yaml" + config.write_text( + json.dumps( + { + "version": 1, + "agent": _HOSTED, + "dataset": str(dataset), + "protocol": "responses", + } + ), + encoding="utf-8", + ) + return config + + +def _invoke(tmp_path: Path, monkeypatch, extra_args: list[str]) -> tuple[object, list]: + config = _write_hosted_config(tmp_path) + output = tmp_path / "out" + output.mkdir() + + seen: list = [] + + import agentops.pipeline.orchestrator as orch + + def fake_run(cfg, options=None): + seen.append(options) + return _passing_result() + + monkeypatch.setattr(orch, "run_evaluation", fake_run) + + result = runner.invoke( + app, + ["eval", "run", "--config", str(config), "--output", str(output), *extra_args], + ) + return result, seen + + +def test_agent_flag_replaces_pinned_version(tmp_path, monkeypatch) -> None: + result, seen = _invoke(tmp_path, monkeypatch, ["--agent", "12"]) + + assert result.exit_code == 0, result.output + assert seen and seen[0].agent_override is not None + assert seen[0].agent_override.endswith("/agents/helpdeskbot/versions/12") + assert "Agent override" in result.output + + +def test_env_var_replaces_pinned_version(tmp_path, monkeypatch) -> None: + monkeypatch.setenv("AGENTOPS_AGENT", "14") + result, seen = _invoke(tmp_path, monkeypatch, []) + + assert result.exit_code == 0, result.output + assert seen[0].agent_override.endswith("/agents/helpdeskbot/versions/14") + + +def test_flag_wins_over_env_var(tmp_path, monkeypatch) -> None: + monkeypatch.setenv("AGENTOPS_AGENT", "14") + result, seen = _invoke(tmp_path, monkeypatch, ["--agent", "12"]) + + assert result.exit_code == 0, result.output + assert seen[0].agent_override.endswith("/versions/12") + + +def test_no_override_leaves_the_pinned_agent_alone(tmp_path, monkeypatch) -> None: + monkeypatch.delenv("AGENTOPS_AGENT", raising=False) + result, seen = _invoke(tmp_path, monkeypatch, []) + + assert result.exit_code == 0, result.output + assert seen[0].agent_override is None + assert "Agent override" not in result.output + + +def test_unexpanded_ado_variable_is_ignored(tmp_path, monkeypatch) -> None: + """Azure DevOps leaves `$(NAME)` verbatim when the variable is undefined.""" + + monkeypatch.setenv("AGENTOPS_AGENT", "$(AGENTOPS_AGENT)") + result, seen = _invoke(tmp_path, monkeypatch, []) + + assert result.exit_code == 0, result.output + assert seen[0].agent_override is None + + +def test_full_agent_reference_override_is_used_verbatim(tmp_path, monkeypatch) -> None: + other = "https://acct.services.ai.azure.com/api/projects/other/agents/bot/versions/3" + result, seen = _invoke(tmp_path, monkeypatch, ["--agent", other]) + + assert result.exit_code == 0, result.output + assert seen[0].agent_override == other + + +def test_unusable_override_fails_loudly(tmp_path, monkeypatch) -> None: + dataset = tmp_path / "dataset.jsonl" + dataset.write_text(json.dumps({"input": "hi", "expected": "hi"}), encoding="utf-8") + config = tmp_path / "agentops.yaml" + config.write_text( + json.dumps({"version": 1, "agent": "model:gpt-4o", "dataset": str(dataset)}), + encoding="utf-8", + ) + output = tmp_path / "out" + output.mkdir() + + import agentops.pipeline.orchestrator as orch + + monkeypatch.setattr(orch, "run_evaluation", lambda *a, **k: _passing_result()) + + result = runner.invoke( + app, + [ + "eval", + "run", + "--config", + str(config), + "--output", + str(output), + "--agent", + "12", + ], + ) + + assert result.exit_code == 1, result.output + assert "no version segment" in result.output diff --git a/tests/unit/test_workflow_analysis.py b/tests/unit/test_workflow_analysis.py index b885545..873c851 100644 --- a/tests/unit/test_workflow_analysis.py +++ b/tests/unit/test_workflow_analysis.py @@ -462,3 +462,53 @@ def test_foundry_eval_rows_always_have_two_reasons_when_selected(tmp_path: Path) checked.add(analysis.recommended_eval_runner) assert checked == {AZD_EVAL_RUNNER, AGENTOPS_CLOUD_RUNNER} + + +# --------------------------------------------------------------------------- +# Pinned agent version drift (#388) +# --------------------------------------------------------------------------- + + +def test_analyze_flags_pinned_hosted_agent_version(tmp_path: Path) -> None: + """Regression for #388: the pin must be visible before it reaches CI.""" + + (tmp_path / "agentops.yaml").write_text( + "version: 1\n" + "agent: https://acct.services.ai.azure.com/api/projects/proj/agents/helpdeskbot/versions/11\n" + "dataset: data.jsonl\n" + "protocol: responses\n", + encoding="utf-8", + ) + (tmp_path / "data.jsonl").write_text( + json.dumps({"input": "Hello", "expected": "Hello!"}) + "\n", + encoding="utf-8", + ) + + analysis = analyze_workflow_project(tmp_path) + signal = next( + (s for s in analysis.signals if s.key == "agent_version_pin"), + None, + ) + + assert signal is not None, "analyze did not surface the pinned agent version" + assert "11" in signal.detail + assert "AGENTOPS_AGENT" in signal.detail + assert any("AGENTOPS_AGENT" in warning for warning in analysis.warnings) + assert "AGENTOPS_AGENT" in render_workflow_analysis(analysis, "text") + + +def test_analyze_does_not_flag_prompt_agent_pin(tmp_path: Path) -> None: + """Prompt-agent deploys already retarget eval via the candidate config.""" + + (tmp_path / "agentops.yaml").write_text( + "version: 1\nagent: quickstart-agent:2\ndataset: data.jsonl\n", + encoding="utf-8", + ) + (tmp_path / "data.jsonl").write_text( + json.dumps({"input": "Hello", "expected": "Hello!"}) + "\n", + encoding="utf-8", + ) + + analysis = analyze_workflow_project(tmp_path) + + assert not any(s.key == "agent_version_pin" for s in analysis.signals) From 74efdf4bd73e4738168f528d1c4de7d7cbc648dc Mon Sep 17 00:00:00 2001 From: Paulo Lacerda Date: Fri, 7 Aug 2026 17:57:49 -0300 Subject: [PATCH 2/2] refactor(eval): reframe the agent override as a seam, not a fix for #388 The previous commit on this branch claimed to fix stale-version evaluation. Review found that claim is wrong, and the reason is worth recording rather than quietly correcting. Nothing populates AGENTOPS_AGENT. A grep across src/ and templates/ finds the constant, the CLI consumer, the six injection sites, and nothing else. On GitHub Actions `${{ env.AGENTOPS_AGENT || vars.AGENTOPS_AGENT }}` resolves to an empty string; on Azure DevOps `$(AGENTOPS_AGENT)` arrives verbatim and the unexpanded-token guard strips it. Both paths fall back to the pin, so a user reproducing #388 sees behavior identical to before this branch. The injected expressions also cannot become correct by adding a producer later. They read same-job env scope, which cannot receive a value from a prior job or stage. That needs declared `jobs..outputs` plus `needs..outputs.*` on GitHub Actions, or explicit `stageDependencies` mapping on Azure DevOps. And the eval gate is structurally pre-deploy in every generated template, so if the deploy step is what publishes the new Foundry version then no value of AGENTOPS_AGENT can be correct at the time eval runs. Fixing #388 requires restructuring the templates, not assigning a variable. The override seam is still useful on its own, so it stays. It is now described as a feature: `agentops eval run --agent` and the AGENTOPS_AGENT fallback let a caller retarget a run without editing tracked config. The CHANGELOG entry moved from Fixed to Added and no longer claims to resolve stale-version evaluation. The `workflow analyze` warning was actively harmful and is gone. It fired on every correctly-pinned hosted project, including eval-only repos with no deploy pipeline at all, and it told users to export a value that no supported mechanism produces. The signal now fires only when the repo also has a generated deploy pipeline, which is the only place a deploy step could move the target underneath the gate, and its text describes the limitation instead of instructing an impossible action. Three cicd tests asserted only that a literal expression string appeared in the rendered YAML, which passes against a pipeline that does nothing. They now carry docstrings saying so, joined by a test that asserts no generated workflow declares job outputs or assigns AGENTOPS_AGENT, and by a parametrized CLI test that pins the actual runtime behavior: empty, whitespace, and unexpanded CI tokens all fall back to the configured agent. An empty value is deliberately not a hard failure, because every pipeline generated today passes exactly that and failing closed would break all of them. Also noted in the helper docstring: a non-numeric override is returned verbatim with no validation, so malformed input surfaces as a confusing classification failure rather than a clear message. Worth tightening when a producer lands. Unit suite: 1145 passed, 5 skipped. Refs #388 Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: bcb9c0b6-d506-46dc-90d2-8120413166ee --- CHANGELOG.md | 46 +++++++++++--------- src/agentops/core/agentops_config.py | 7 +++ src/agentops/services/workflow_analysis.py | 39 ++++++++++++----- tests/unit/test_cicd.py | 39 ++++++++++++++++- tests/unit/test_eval_agent_override.py | 30 +++++++++++++ tests/unit/test_workflow_analysis.py | 50 ++++++++++++++++------ 6 files changed, 166 insertions(+), 45 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2178290..19928b1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,32 +5,36 @@ This format follows [Keep a Changelog](https://keepachangelog.com/) and adheres ## [Unreleased] -### Fixed -- **Generated workflows evaluated a stale agent version.** `agentops.yaml` pins - a fully-qualified Foundry agent that ends in a version segment - (`.../agents/helpdeskbot/versions/11`), and nothing in the generated dev, QA, - or prod pipelines ever retargeted it. `azd provision` published a new agent - version and the eval job that ran next still scored the previous one, so the - gate reported green on an artifact the pipeline had already replaced. A - regression introduced by the deploy could not fail its own quality gate, and - in `prod` that same shape gated a release. +### Added +- **`agentops eval run` accepts an explicit agent target.** The eval target was + only ever read from `agentops.yaml`, so retargeting a run meant editing + tracked config. `--agent` now overrides it, falling back to the + `AGENTOPS_AGENT` environment variable when the flag is absent. A bare number + (`--agent 12`) replaces just the version segment of the configured target; a + full agent reference (endpoint URL, `name:version`, or `model:`) + replaces the target outright. Unset means unchanged, so every existing run + behaves exactly as before. `RunOptions.agent_override` already existed in the orchestrator and was - already consumed by all three execution backends, but nothing ever set it. - `agentops eval run` now accepts `--agent`, falling back to the - `AGENTOPS_AGENT` environment variable, and every generated eval step on - GitHub Actions and Azure DevOps forwards that variable. A bare number - (`--agent 12`) replaces just the version segment of the configured target; - a full agent reference replaces the target outright. Unset means unchanged, - so existing pipelines behave exactly as before. `agentops.yaml` stays - declarative and CI never writes to tracked config. + already consumed by all three execution backends, but nothing set it. This + connects the CLI to that seam and forwards `AGENTOPS_AGENT` into every + generated eval step on GitHub Actions and Azure DevOps, so a future pipeline + change can retarget the gate without rewriting `agentops.yaml` mid-run. + Nothing in the shipped pipelines assigns a value yet, and the injected + expressions read same-job scope, which cannot see a prior job's output. + Wiring a producer needs declared job outputs on GitHub Actions or + `stageDependencies` on Azure DevOps; see issue #388. Azure DevOps leaves `$(NAME)` in the environment verbatim when a variable is undefined, so an unexpanded token is treated as "no override" rather than as - an agent expression. `workflow analyze` now reports the pinned version as a - signal and a warning, so the drift is visible before it reaches CI. - Prompt-agent deploys were never affected: they already stage - `agentops.candidate.yaml` with a fresh `agent` value for the eval step. + an agent expression. An empty or unexpanded value falls back to the + configured agent instead of failing, because every pipeline generated today + passes exactly that. + + `agentops workflow analyze` reports the pinned agent version as a signal when + the repo also has a generated deploy pipeline, which is the only place a + later deploy step could move the target underneath the gate. Eval-only repos + see nothing. ## [0.8.5] - 2026-08-07 diff --git a/src/agentops/core/agentops_config.py b/src/agentops/core/agentops_config.py index 14f813e..6783011 100644 --- a/src/agentops/core/agentops_config.py +++ b/src/agentops/core/agentops_config.py @@ -1292,6 +1292,13 @@ def apply_agent_version_override(agent: str, override: str) -> str: rebuilding the whole endpoint URL. An empty *override* leaves *agent* untouched. + + Known limitation: a non-numeric *override* is returned verbatim with no + validation, so malformed input (``12abc``, a truncated endpoint URL) is + passed straight to :func:`classify_agent` and surfaces as a confusing + classification failure rather than a clear message about the override. + Worth tightening when the CI producer described in issue #388 lands and + starts generating these values programmatically. """ base = agent.strip() diff --git a/src/agentops/services/workflow_analysis.py b/src/agentops/services/workflow_analysis.py index 3230d0d..92dfa88 100644 --- a/src/agentops/services/workflow_analysis.py +++ b/src/agentops/services/workflow_analysis.py @@ -182,27 +182,25 @@ def analyze_workflow_project(directory: Path) -> WorkflowAnalysis: ) ) pinned_version = agentops.get("pinned_agent_version") - if pinned_version: + if pinned_version and _has_generated_deploy_pipeline(root): signals.append( WorkflowSignal( "agent_version_pin", "Pinned agent version", ( - f"agentops.yaml pins agent version {pinned_version}. Generated " - "deploy pipelines publish a new Foundry version, so the eval gate " - "scores the pinned version rather than the one the run produced. " - f"Export {AGENT_OVERRIDE_ENV} (or pass `agentops eval run --agent`) " - "with the version the deploy step resolved so the gate follows the " - "deployed artifact." + f"agentops.yaml pins agent version {pinned_version} and this " + "repo has a generated deploy pipeline. The eval gate runs " + "before deploy, so it scores the pinned version, which is " + "correct today. If a later change makes deploy publish a new " + "version that the gate should score instead, the eval step " + f"accepts {AGENT_OVERRIDE_ENV} (or `agentops eval run " + "--agent`) as an override. Nothing in the generated pipelines " + "sets that value yet; see issue #388." ), "agentops.yaml", confidence="medium", ) ) - warnings.append( - f"agentops.yaml pins agent version {pinned_version}; set " - f"{AGENT_OVERRIDE_ENV} in the eval job so the gate scores the deployed version." - ) bicep_files = _find_files(root, "*.bicep") if bicep_files: @@ -962,6 +960,25 @@ def _accelerator_hint(readme_lower: str) -> Optional[WorkflowSignal]: return None +def _has_generated_deploy_pipeline(root: Path) -> bool: + """True when the repo already has a generated AgentOps deploy pipeline. + + The pinned-version signal is only interesting where a deploy step could + move the agent underneath the gate. Eval-only repos have no such step, so + a correctly-pinned target there is just a correctly-pinned target. + """ + for directory, prefix in ( + (root / ".github" / "workflows", "agentops-deploy-"), + (root / ".azuredevops" / "pipelines", "agentops-deploy-"), + ): + if not directory.is_dir(): + continue + for path in directory.glob(f"{prefix}*.yml"): + if path.is_file(): + return True + return False + + def _existing_ci_signal(root: Path) -> Optional[WorkflowSignal]: github = root / ".github" / "workflows" ado = root / ".azuredevops" / "pipelines" diff --git a/tests/unit/test_cicd.py b/tests/unit/test_cicd.py index cb95794..927f15f 100644 --- a/tests/unit/test_cicd.py +++ b/tests/unit/test_cicd.py @@ -1433,7 +1433,15 @@ def _seed_hosted_project(root: Path, *, azure_yaml: bool = False) -> None: def test_github_eval_steps_forward_agent_version_override(tmp_path: Path) -> None: - """Regression for #388: eval must be retargetable without editing agentops.yaml.""" + """The seam is wired end to end (#388). + + This asserts only that the generated YAML *plumbs* the variable. It says + nothing about the value being populated: nothing in the shipped templates + sets AGENTOPS_AGENT today, so at runtime this expression resolves to an + empty string and eval falls back to the pin in agentops.yaml. See + test_empty_agent_override_falls_back_to_the_configured_agent for the + behavior users actually get, and #388 for what a producer would require. + """ _seed_hosted_project(tmp_path, azure_yaml=True) generate_cicd_workflows(directory=tmp_path, kinds=["dev", "qa", "prod"], force=True) @@ -1446,6 +1454,33 @@ def test_github_eval_steps_forward_agent_version_override(tmp_path: Path) -> Non assert isinstance(_read_yaml(tmp_path / rel), dict) +def test_no_generated_workflow_produces_a_value_for_the_override( + tmp_path: Path, +) -> None: + """Documents the gap in #388: the seam has a consumer but no producer. + + When a producer is built it will need `jobs..outputs` plus + `needs..outputs.*` on GitHub Actions (step-level `env` cannot read a + prior job's output), or `stageDependencies` on Azure DevOps. At that point + this test must be updated, not deleted. + """ + + _seed_hosted_project(tmp_path, azure_yaml=True) + generate_cicd_workflows(directory=tmp_path, kinds=["dev", "qa", "prod"], force=True) + + for rel in (_DEV_PATH, _QA_PATH, _PROD_PATH): + content = (tmp_path / rel).read_text(encoding="utf-8") + assert "outputs:" not in content, ( + f"{rel} declares job outputs; if it now emits an agent version, " + "update this test and the #388 notes" + ) + assert "needs.provision.outputs" not in content + assert "AGENTOPS_AGENT=" not in content, ( + f"{rel} assigns AGENTOPS_AGENT; the producer described in #388 " + "may now exist" + ) + + def test_azure_devops_eval_steps_forward_agent_version_override(tmp_path: Path) -> None: _seed_hosted_project(tmp_path, azure_yaml=True) generate_cicd_workflows( @@ -1460,6 +1495,8 @@ def test_azure_devops_eval_steps_forward_agent_version_override(tmp_path: Path) assert "AGENTOPS_AGENT: $(AGENTOPS_AGENT)" in content, ( f"{rel} does not forward the agent override to the eval step" ) + # Azure DevOps passes `$(NAME)` through verbatim when undefined, which + # the CLI treats as "no override". No pipeline declares it today. assert isinstance(_read_yaml(tmp_path / rel), dict) diff --git a/tests/unit/test_eval_agent_override.py b/tests/unit/test_eval_agent_override.py index 595e4c0..3a71cec 100644 --- a/tests/unit/test_eval_agent_override.py +++ b/tests/unit/test_eval_agent_override.py @@ -13,6 +13,7 @@ import json from pathlib import Path +import pytest from typer.testing import CliRunner from agentops.cli.app import app @@ -132,6 +133,35 @@ def test_unexpanded_ado_variable_is_ignored(tmp_path, monkeypatch) -> None: assert seen[0].agent_override is None +@pytest.mark.parametrize( + "ci_value", ["", " ", "$(AGENTOPS_AGENT)", "${{ env.AGENTOPS_AGENT }}"] +) +def test_generated_ci_today_evaluates_the_pin_because_nothing_sets_the_override( + tmp_path, monkeypatch, ci_value +) -> None: + """Documents the real behavior of every pipeline this tool generates (#388). + + No shipped workflow or pipeline assigns AGENTOPS_AGENT. GitHub Actions + resolves the injected expression to an empty string; Azure DevOps passes + `$(AGENTOPS_AGENT)` through untouched. Both must fall back to the pinned + target in agentops.yaml rather than fail, because failing closed would + break every generated pipeline in existence. + + When a producer is built (see #388), this test must be updated to assert + the new behavior. Do not delete it: the fallback still applies to repos + whose pipelines predate the producer. + """ + + monkeypatch.setenv("AGENTOPS_AGENT", ci_value) + result, seen = _invoke(tmp_path, monkeypatch, []) + + assert result.exit_code == 0, result.output + assert seen[0].agent_override is None, ( + f"AGENTOPS_AGENT={ci_value!r} was treated as a real override; " + "eval must fall back to the configured agent" + ) + + def test_full_agent_reference_override_is_used_verbatim(tmp_path, monkeypatch) -> None: other = "https://acct.services.ai.azure.com/api/projects/other/agents/bot/versions/3" result, seen = _invoke(tmp_path, monkeypatch, ["--agent", other]) diff --git a/tests/unit/test_workflow_analysis.py b/tests/unit/test_workflow_analysis.py index 873c851..91ded43 100644 --- a/tests/unit/test_workflow_analysis.py +++ b/tests/unit/test_workflow_analysis.py @@ -465,24 +465,38 @@ def test_foundry_eval_rows_always_have_two_reasons_when_selected(tmp_path: Path) # --------------------------------------------------------------------------- -# Pinned agent version drift (#388) +# Pinned agent version visibility (#388) # --------------------------------------------------------------------------- -def test_analyze_flags_pinned_hosted_agent_version(tmp_path: Path) -> None: - """Regression for #388: the pin must be visible before it reaches CI.""" +_PINNED_HOSTED_CONFIG = ( + "version: 1\n" + "agent: https://acct.services.ai.azure.com/api/projects/proj/agents/helpdeskbot/versions/11\n" + "dataset: data.jsonl\n" + "protocol: responses\n" +) - (tmp_path / "agentops.yaml").write_text( - "version: 1\n" - "agent: https://acct.services.ai.azure.com/api/projects/proj/agents/helpdeskbot/versions/11\n" - "dataset: data.jsonl\n" - "protocol: responses\n", - encoding="utf-8", - ) - (tmp_path / "data.jsonl").write_text( + +def _seed_pinned_hosted(root: Path, *, with_deploy_pipeline: bool) -> None: + (root / "agentops.yaml").write_text(_PINNED_HOSTED_CONFIG, encoding="utf-8") + (root / "data.jsonl").write_text( json.dumps({"input": "Hello", "expected": "Hello!"}) + "\n", encoding="utf-8", ) + if with_deploy_pipeline: + workflows = root / ".github" / "workflows" + workflows.mkdir(parents=True) + (workflows / "agentops-deploy-dev.yml").write_text( + "name: AgentOps Deploy (DEV)\n", encoding="utf-8" + ) + + +def test_analyze_reports_pinned_version_when_a_deploy_pipeline_exists( + tmp_path: Path, +) -> None: + """The pin is only worth reporting where a deploy step could move the target.""" + + _seed_pinned_hosted(tmp_path, with_deploy_pipeline=True) analysis = analyze_workflow_project(tmp_path) signal = next( @@ -493,10 +507,22 @@ def test_analyze_flags_pinned_hosted_agent_version(tmp_path: Path) -> None: assert signal is not None, "analyze did not surface the pinned agent version" assert "11" in signal.detail assert "AGENTOPS_AGENT" in signal.detail - assert any("AGENTOPS_AGENT" in warning for warning in analysis.warnings) + # The signal must not instruct an action that no shipped pipeline supports. + assert "#388" in signal.detail assert "AGENTOPS_AGENT" in render_workflow_analysis(analysis, "text") +def test_analyze_stays_quiet_about_pins_in_eval_only_repos(tmp_path: Path) -> None: + """No deploy pipeline means nothing can move the agent underneath the gate.""" + + _seed_pinned_hosted(tmp_path, with_deploy_pipeline=False) + + analysis = analyze_workflow_project(tmp_path) + + assert not any(s.key == "agent_version_pin" for s in analysis.signals) + assert not any("AGENTOPS_AGENT" in warning for warning in analysis.warnings) + + def test_analyze_does_not_flag_prompt_agent_pin(tmp_path: Path) -> None: """Prompt-agent deploys already retarget eval via the candidate config."""