From 32535e97f47b3210172768d868c88742deb75fe2 Mon Sep 17 00:00:00 2001 From: Paulo Lacerda Date: Fri, 29 May 2026 01:38:22 -0300 Subject: [PATCH] fix(cicd): upload cloud_output_items.json as CI artifact (#197) The cloud-eval parser fix in the previous release added a `0 usable scores` warning that points operators at `.agentops/results/latest/cloud_output_items.json` for triage. That file was being written by the orchestrator but never made it into the GitHub Actions / Azure DevOps artifact bundle, so the very people who needed it most (anyone hitting an unrecognized Foundry grader shape in CI) could not actually inspect it without re-running the eval locally. Add the raw dump to both `__EVAL_ARTIFACT_PATHS__` upload lists in `services/cicd.py`. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- CHANGELOG.md | 1 + src/agentops/services/cicd.py | 1442 +++++++++++++++++---------------- 2 files changed, 723 insertions(+), 720 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index df0497d..9c383d9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ This format follows [Keep a Changelog](https://keepachangelog.com/) and adheres succeeded. ### Added +- **`cloud_output_items.json` is now uploaded as a CI artifact.** Generated PR and deploy workflows (GitHub Actions and Azure DevOps) include `.agentops/results/latest/cloud_output_items.json` in the `agentops-*-results` artifact bundle alongside `results.json`, `report.md`, and `cloud_evaluation.json`. Pairs with the "0 usable scores" warning so operators can diagnose unrecognized Foundry grader shapes without re-running locally. - **`cloud_output_items.json` raw dump.** Every cloud eval run now writes the raw `output_items` it received from Foundry to `/cloud_output_items.json`, in addition to the parsed diff --git a/src/agentops/services/cicd.py b/src/agentops/services/cicd.py index 8180349..80d04bb 100644 --- a/src/agentops/services/cicd.py +++ b/src/agentops/services/cicd.py @@ -1,720 +1,722 @@ -"""CI/CD workflow generation service for `agentops workflow generate`.""" - -from __future__ import annotations - -from dataclasses import dataclass, field -from importlib.resources import files -from pathlib import Path -from typing import Dict, List, Mapping, Sequence, Tuple - -from agentops.pipeline.official_eval import ( - AGENTOPS_CLOUD_RUNNER, - AGENTOPS_LOCAL_RUNNER, - OFFICIAL_EVAL_ACTION_ENV, - OFFICIAL_EVAL_ADO_TASK_ENV, - OFFICIAL_EVAL_RUNNER, - official_eval_action_ref, - official_eval_ado_task_ref, -) -from agentops.services.workflow_analysis import ( - has_ailz_preflight, - recommended_deploy_mode, - recommended_eval_runner, -) - - -_TEMPLATE_PACKAGE = "agentops.templates" -_CLOUD_EVAL_CONFIG_NAME = ".agentops.cloud.yaml" -_CI_EVAL_OUTPUT = ".agentops/results/latest" - -# CI/CD platforms supported by ``agentops workflow generate``. -PLATFORMS: Tuple[str, ...] = ("github", "azure-devops") - -# Deployment template modes. ``placeholder`` keeps the stack-agnostic -# scaffold; ``azd`` delegates infrastructure and app deployment to Azure -# Developer CLI; ``prompt-agent`` creates/evaluates a candidate Foundry prompt -# agent version from a source-controlled prompt file. ``auto`` selects -# ``azd`` when the target repo has ``azure.yaml`` and ``prompt-agent`` when -# ``agentops.yaml`` targets a Foundry prompt agent. -DEPLOY_MODES: Tuple[str, ...] = ("auto", "placeholder", "azd", "prompt-agent") - -# Doctor gate severities supported by ``agentops workflow generate -# --doctor-gate``. The PR workflow template runs `agentops doctor -# --severity-fail `; ``critical`` (the default) blocks the PR on -# critical Doctor findings (including regression detection), ``warning`` -# blocks on warning or higher, and ``none`` keeps Doctor advisory. -DOCTOR_GATES: Tuple[str, ...] = ("critical", "warning", "none") -DEFAULT_DOCTOR_GATE: str = "critical" - -# Per-platform mapping of workflow kind -> (template path inside package, -# output path in repo). -# -# The default templates form a complete GenAIOps GitFlow scaffold: -# -# pr -> agentops-pr (PR gate; PRs to develop, release/**, main) -# dev -> agentops-deploy-dev (push to develop -> environment: dev) -# qa -> agentops-deploy-qa (push to release/** -> environment: qa) -# prod -> agentops-deploy-prod (push to main -> environment: production) -# -# A scheduled Doctor workflow is also available as an explicit optional kind: -# -# doctor -> agentops-doctor (scheduled Doctor + eval health check) -_TEMPLATES_BY_PLATFORM: Dict[str, Dict[str, Tuple[str, str]]] = { - "github": { - "pr": ("workflows/agentops-pr.yml", ".github/workflows/agentops-pr.yml"), - "dev": ("workflows/agentops-deploy-dev.yml", ".github/workflows/agentops-deploy-dev.yml"), - "qa": ("workflows/agentops-deploy-qa.yml", ".github/workflows/agentops-deploy-qa.yml"), - "prod": ("workflows/agentops-deploy-prod.yml", ".github/workflows/agentops-deploy-prod.yml"), - "doctor": ("workflows/agentops-watchdog.yml", ".github/workflows/agentops-doctor.yml"), - }, - "azure-devops": { - "pr": ( - "pipelines/azuredevops/agentops-pr.yml", - ".azuredevops/pipelines/agentops-pr.yml", - ), - "dev": ( - "pipelines/azuredevops/agentops-deploy-dev.yml", - ".azuredevops/pipelines/agentops-deploy-dev.yml", - ), - "qa": ( - "pipelines/azuredevops/agentops-deploy-qa.yml", - ".azuredevops/pipelines/agentops-deploy-qa.yml", - ), - "prod": ( - "pipelines/azuredevops/agentops-deploy-prod.yml", - ".azuredevops/pipelines/agentops-deploy-prod.yml", - ), - "doctor": ( - "pipelines/azuredevops/agentops-watchdog.yml", - ".azuredevops/pipelines/agentops-doctor.yml", - ), - }, -} - -_AZD_TEMPLATES_BY_PLATFORM: Dict[str, Dict[str, Tuple[str, str]]] = { - "github": { - "dev": ("workflows/agentops-deploy-dev-azd.yml", ".github/workflows/agentops-deploy-dev.yml"), - "qa": ("workflows/agentops-deploy-qa-azd.yml", ".github/workflows/agentops-deploy-qa.yml"), - "prod": ("workflows/agentops-deploy-prod-azd.yml", ".github/workflows/agentops-deploy-prod.yml"), - }, - "azure-devops": { - "dev": ( - "pipelines/azuredevops/agentops-deploy-dev-azd.yml", - ".azuredevops/pipelines/agentops-deploy-dev.yml", - ), - "qa": ( - "pipelines/azuredevops/agentops-deploy-qa-azd.yml", - ".azuredevops/pipelines/agentops-deploy-qa.yml", - ), - "prod": ( - "pipelines/azuredevops/agentops-deploy-prod-azd.yml", - ".azuredevops/pipelines/agentops-deploy-prod.yml", - ), - }, -} - -ALL_KINDS: tuple[str, ...] = ("pr", "dev", "qa", "prod", "doctor") -DEFAULT_KINDS: tuple[str, ...] = ("pr", "dev", "qa", "prod") -LEGACY_KIND_ALIASES: Mapping[str, str] = {"watchdog": "doctor"} - - -@dataclass -class CicdResult: - """Result of generating CI/CD workflow files.""" - - platform: str = "github" - deploy_mode: str = "placeholder" - eval_runner: str = AGENTOPS_LOCAL_RUNNER - doctor_gate: str = "critical" - kinds: List[str] = field(default_factory=list) - created_files: List[Path] = field(default_factory=list) - overwritten_files: List[Path] = field(default_factory=list) - skipped_files: List[Path] = field(default_factory=list) - - -def _write_template( - templates_root, - template_path: str, - output_path: Path, - force: bool, - result: CicdResult, - substitutions: Mapping[str, str] | None = None, -) -> None: - template_resource = templates_root.joinpath(template_path) - template_content = template_resource.read_text(encoding="utf-8") - for key, value in (substitutions or {}).items(): - template_content = template_content.replace(key, value) - - existed_before = output_path.exists() - - if existed_before and not force: - result.skipped_files.append(output_path) - return - - output_path.parent.mkdir(parents=True, exist_ok=True) - output_path.write_text(template_content, encoding="utf-8") - - if existed_before: - result.overwritten_files.append(output_path) - else: - result.created_files.append(output_path) - - -def _branch_block_github(*branches: str) -> str: - return "".join(f" - {branch}\n" for branch in branches).rstrip() - - -def _branch_block_ado(*branches: str) -> str: - return "".join(f" - {branch}\n" for branch in branches).rstrip() - - -_PROMPT_AGENT_VALUES: Dict[str, Dict[str, str]] = { - "pr": { - "__ENV_LABEL__": "PR", - "__ENV_KEY__": "pr", - # PR candidates are staged in the dev Foundry project so the - # gate evaluates the same target the deploy workflow will use. - # Sandbox is the author's playground only. - "__ENV_NAME__": "dev", - "__BRANCHES__": "", - "__EVAL_JOB_NAME__": "AgentOps eval (PR gate)", - }, - "dev": { - "__ENV_LABEL__": "DEV", - "__ENV_KEY__": "dev", - "__ENV_NAME__": "dev", - "__BRANCHES__": _branch_block_github("develop"), - "__EVAL_JOB_NAME__": "Eval candidate (gate)", - }, - "qa": { - "__ENV_LABEL__": "QA", - "__ENV_KEY__": "qa", - "__ENV_NAME__": "qa", - "__BRANCHES__": _branch_block_github('"release/**"'), - "__EVAL_JOB_NAME__": "Eval candidate (gate)", - }, - "prod": { - "__ENV_LABEL__": "PROD", - "__ENV_KEY__": "prod", - "__ENV_NAME__": "production", - "__BRANCHES__": _branch_block_github("main"), - "__EVAL_JOB_NAME__": "Safety eval candidate (gate)", - }, -} - -_PROMPT_AGENT_VALUES_ADO: Dict[str, Dict[str, str]] = { - "pr": { - "__ENV_LABEL__": "PR", - "__ENV_KEY__": "pr", - # PR candidates are staged in the dev Foundry project so the - # gate evaluates the same target the deploy pipeline will use. - "__ENV_NAME__": "dev", - "__BRANCHES__": "", - }, - "dev": { - "__ENV_LABEL__": "dev", - "__ENV_KEY__": "dev", - "__ENV_NAME__": "dev", - "__BRANCHES__": _branch_block_ado("develop"), - }, - "qa": { - "__ENV_LABEL__": "qa", - "__ENV_KEY__": "qa", - "__ENV_NAME__": "qa", - "__BRANCHES__": _branch_block_ado("release/*"), - }, - "prod": { - "__ENV_LABEL__": "production", - "__ENV_KEY__": "prod", - "__ENV_NAME__": "production", - "__BRANCHES__": _branch_block_ado("main"), - }, -} - -_PROMPT_AGENT_TEMPLATES_BY_PLATFORM: Dict[str, Dict[str, Tuple[str, str]]] = { - "github": { - "pr": ( - "workflows/agentops-pr-prompt-agent.yml", - ".github/workflows/agentops-pr.yml", - ), - "dev": ("workflows/agentops-deploy-prompt-agent.yml", ".github/workflows/agentops-deploy-dev.yml"), - "qa": ("workflows/agentops-deploy-prompt-agent.yml", ".github/workflows/agentops-deploy-qa.yml"), - "prod": ("workflows/agentops-deploy-prompt-agent.yml", ".github/workflows/agentops-deploy-prod.yml"), - }, - "azure-devops": { - "pr": ( - "pipelines/azuredevops/agentops-pr-prompt-agent.yml", - ".azuredevops/pipelines/agentops-pr.yml", - ), - "dev": ( - "pipelines/azuredevops/agentops-deploy-prompt-agent.yml", - ".azuredevops/pipelines/agentops-deploy-dev.yml", - ), - "qa": ( - "pipelines/azuredevops/agentops-deploy-prompt-agent.yml", - ".azuredevops/pipelines/agentops-deploy-qa.yml", - ), - "prod": ( - "pipelines/azuredevops/agentops-deploy-prompt-agent.yml", - ".azuredevops/pipelines/agentops-deploy-prod.yml", - ), - }, -} - - -def _eval_substitutions( - platform: str, - eval_runner: str, - config_path: str, - *, - ado_indent: int = 10, -) -> Mapping[str, str]: - if platform == "azure-devops": - return _ado_eval_substitutions(eval_runner, config_path, base_indent=ado_indent) - return _github_eval_substitutions(eval_runner, config_path) - - -def _github_eval_substitutions(eval_runner: str, config_path: str) -> Mapping[str, str]: - if eval_runner == AGENTOPS_CLOUD_RUNNER: - return { - "__EVAL_STEPS__": f""" - name: Prepare AgentOps cloud eval config - env: - AGENTOPS_SOURCE_CONFIG: "{config_path}" - run: | - python - <<'PY' - import os - from pathlib import Path - from agentops.utils.yaml import load_yaml, save_yaml - - source = Path(os.environ["AGENTOPS_SOURCE_CONFIG"]) - target = source.with_name("{_CLOUD_EVAL_CONFIG_NAME}") - data = load_yaml(source) - data["execution"] = "cloud" - data["publish"] = True - save_yaml(target, data) - with Path(os.environ["GITHUB_ENV"]).open("a", encoding="utf-8") as env_file: - env_file.write(f"AGENTOPS_CI_CONFIG={{target}}\\n") - print(f"Prepared AgentOps cloud eval config: {{target}}") - PY - - - name: Run AgentOps Foundry cloud eval - id: eval - env: - AZURE_AI_FOUNDRY_PROJECT_ENDPOINT: ${{{{ vars.AZURE_AI_FOUNDRY_PROJECT_ENDPOINT }}}} - AZURE_OPENAI_ENDPOINT: ${{{{ vars.AZURE_OPENAI_ENDPOINT }}}} - AZURE_OPENAI_DEPLOYMENT: ${{{{ vars.AZURE_OPENAI_DEPLOYMENT }}}} - APPLICATIONINSIGHTS_CONNECTION_STRING: ${{{{ secrets.APPLICATIONINSIGHTS_CONNECTION_STRING || vars.APPLICATIONINSIGHTS_CONNECTION_STRING }}}} - run: | - set +e - agentops eval run --config "$AGENTOPS_CI_CONFIG" --output "{_CI_EVAL_OUTPUT}" - ec=$? - echo "exit_code=$ec" >> "$GITHUB_OUTPUT" - if [ $ec -eq 0 ]; then - echo "result=pass" >> "$GITHUB_OUTPUT" - elif [ $ec -eq 2 ]; then - echo "result=threshold_failed" >> "$GITHUB_OUTPUT" - else - echo "result=error" >> "$GITHUB_OUTPUT" - fi - exit $ec""", - "__EVAL_ARTIFACT_PATHS__": f"""{_CI_EVAL_OUTPUT}/results.json - {_CI_EVAL_OUTPUT}/report.md - {_CI_EVAL_OUTPUT}/cloud_evaluation.json""", - } - if eval_runner == OFFICIAL_EVAL_RUNNER: - official_action = official_eval_action_ref() - return { - "__EVAL_STEPS__": f""" - name: Prepare official AI Agent Evaluation input - id: official_eval_input - env: - AZURE_OPENAI_DEPLOYMENT: ${{{{ vars.AZURE_OPENAI_DEPLOYMENT }}}} - {OFFICIAL_EVAL_ACTION_ENV}: {official_action} - run: | - python -m agentops.pipeline.official_eval prepare \\ - --config \"{config_path}\" \\ - --out \".agentops/official-eval/input.json\" \\ - --github-output \"$GITHUB_OUTPUT\" - - # Official runner for Foundry prompt agents. AgentOps keeps the prepared - # input/metadata as release evidence until the action exposes a stable - # machine-readable threshold artifact. - - name: Run official AI Agent Evaluation - id: official_eval - uses: {official_action} - with: - azure-ai-project-endpoint: ${{{{ vars.AZURE_AI_FOUNDRY_PROJECT_ENDPOINT }}}} - deployment-name: ${{{{ steps.official_eval_input.outputs.deployment_name }}}} - agent-ids: ${{{{ steps.official_eval_input.outputs.agent_ids }}}} - data-path: ${{{{ steps.official_eval_input.outputs.data_path }}}} - - - name: Record official eval result - if: always() - id: eval - run: | - mkdir -p .agentops/official-eval - outcome=\"${{{{ steps.official_eval.outcome }}}}\" - conclusion=\"${{{{ steps.official_eval.conclusion }}}}\" - if [ \"$outcome\" = \"success\" ]; then - echo \"exit_code=0\" >> \"$GITHUB_OUTPUT\" - echo \"result=official-ai-agent-evaluation\" >> \"$GITHUB_OUTPUT\" - else - echo \"exit_code=1\" >> \"$GITHUB_OUTPUT\" - echo \"result=official-ai-agent-evaluation-failed\" >> \"$GITHUB_OUTPUT\" - fi - python - <<'PY' - import json - from datetime import datetime, timezone - from pathlib import Path - - output = Path('.agentops/official-eval/result.json') - outcome = \"${{{{ steps.official_eval.outcome }}}}\" - conclusion = \"${{{{ steps.official_eval.conclusion }}}}\" - status = 'success' if outcome == 'success' else 'failed' if outcome else 'unknown' - payload = dict( - runner='{OFFICIAL_EVAL_RUNNER}', - system='github-actions', - action='{official_action}', - status=status, - outcome=outcome, - conclusion=conclusion, - agent_ids=\"${{{{ steps.official_eval_input.outputs.agent_ids }}}}\", - deployment_name=\"${{{{ steps.official_eval_input.outputs.deployment_name }}}}\", - data_path=\"${{{{ steps.official_eval_input.outputs.data_path }}}}\", - metadata_path=\"${{{{ steps.official_eval_input.outputs.metadata_path }}}}\", - machine_readable_thresholds=False, - recorded_at=datetime.now(timezone.utc).isoformat(), - ) - output.write_text(json.dumps(payload, indent=2) + '\\n', encoding='utf-8') - PY""", - "__EVAL_ARTIFACT_PATHS__": """.agentops/official-eval/input.json - .agentops/official-eval/metadata.json - .agentops/official-eval/result.json""", - } - return { - "__EVAL_STEPS__": f""" - name: Run AgentOps eval - id: eval - env: - AZURE_AI_FOUNDRY_PROJECT_ENDPOINT: ${{{{ vars.AZURE_AI_FOUNDRY_PROJECT_ENDPOINT }}}} - AZURE_OPENAI_ENDPOINT: ${{{{ vars.AZURE_OPENAI_ENDPOINT }}}} - AZURE_OPENAI_DEPLOYMENT: ${{{{ vars.AZURE_OPENAI_DEPLOYMENT }}}} - APPLICATIONINSIGHTS_CONNECTION_STRING: ${{{{ secrets.APPLICATIONINSIGHTS_CONNECTION_STRING || vars.APPLICATIONINSIGHTS_CONNECTION_STRING }}}} - run: | - set +e - agentops eval run --config \"{config_path}\" - ec=$? - echo \"exit_code=$ec\" >> \"$GITHUB_OUTPUT\" - if [ $ec -eq 0 ]; then - echo \"result=pass\" >> \"$GITHUB_OUTPUT\" - elif [ $ec -eq 2 ]; then - echo \"result=threshold_failed\" >> \"$GITHUB_OUTPUT\" - else - echo \"result=error\" >> \"$GITHUB_OUTPUT\" - fi - exit $ec""", - "__EVAL_ARTIFACT_PATHS__": """.agentops/results/latest/results.json - .agentops/results/latest/report.md - .agentops/results/latest/cloud_evaluation.json""", - } - - -def _ado_eval_substitutions( - eval_runner: str, - config_path: str, - *, - base_indent: int, -) -> Mapping[str, str]: - if eval_runner == AGENTOPS_CLOUD_RUNNER: - return { - "__EVAL_TASKS__": _indent_block( - f"""- bash: | - python - <<'PY' - import os - from pathlib import Path - from agentops.utils.yaml import load_yaml, save_yaml - - source = Path(os.environ["AGENTOPS_SOURCE_CONFIG"]) - target = source.with_name("{_CLOUD_EVAL_CONFIG_NAME}") - data = load_yaml(source) - data["execution"] = "cloud" - data["publish"] = True - save_yaml(target, data) - print(f"##vso[task.setvariable variable=AGENTOPS_CI_CONFIG]{{target}}") - print(f"Prepared AgentOps cloud eval config: {{target}}") - PY - displayName: Prepare AgentOps cloud eval config - env: - AGENTOPS_SOURCE_CONFIG: "{config_path}" - -- task: AzureCLI@2 - displayName: Run AgentOps Foundry cloud eval - inputs: - azureSubscription: $(AZURE_SERVICE_CONNECTION) - scriptType: bash - scriptLocation: inlineScript - inlineScript: | - set +e - agentops eval run --config "$(AGENTOPS_CI_CONFIG)" --output "{_CI_EVAL_OUTPUT}" - code=$? - echo "##vso[task.setvariable variable=AGENTOPS_EVAL_EXIT_CODE]$code" - exit $code - env: - AZURE_AI_FOUNDRY_PROJECT_ENDPOINT: $(AZURE_AI_FOUNDRY_PROJECT_ENDPOINT) - AZURE_OPENAI_ENDPOINT: $(AZURE_OPENAI_ENDPOINT) - AZURE_OPENAI_DEPLOYMENT: $(AZURE_OPENAI_DEPLOYMENT) - APPLICATIONINSIGHTS_CONNECTION_STRING: $(APPLICATIONINSIGHTS_CONNECTION_STRING)""", - base_indent, - ), - "__EVAL_ARTIFACT_TARGET__": _CI_EVAL_OUTPUT, - } - if eval_runner == OFFICIAL_EVAL_RUNNER: - official_task = official_eval_ado_task_ref() - return { - "__EVAL_TASKS__": _indent_block( - f"""- task: AzureCLI@2 - name: official_eval_input - displayName: Prepare official AI Agent Evaluation input - inputs: - azureSubscription: $(AZURE_SERVICE_CONNECTION) - scriptType: bash - scriptLocation: inlineScript - inlineScript: | - python -m agentops.pipeline.official_eval prepare \\ - --config \"{config_path}\" \\ - --out \".agentops/official-eval/input.json\" \\ - --ado-output - env: - AZURE_OPENAI_DEPLOYMENT: $(AZURE_OPENAI_DEPLOYMENT) - {OFFICIAL_EVAL_ADO_TASK_ENV}: {official_task} - -- task: {official_task} - displayName: Run official AI Agent Evaluation - inputs: - azure-ai-project-endpoint: $(AZURE_AI_FOUNDRY_PROJECT_ENDPOINT) - deployment-name: $(official_eval_input.officialDeploymentName) - data-path: $(official_eval_input.officialDataPath) - agent-ids: $(official_eval_input.officialAgentIds) - -- bash: | - mkdir -p .agentops/official-eval - python - <<'PY' - import json - from datetime import datetime, timezone - from pathlib import Path - - job_status = "$(Agent.JobStatus)" - normalized = job_status.strip().lower().replace("_", "").replace("-", "") - status = "success" if normalized == "succeeded" else "failed" if normalized in ("failed", "canceled", "cancelled") else "unknown" - payload = dict( - runner="{OFFICIAL_EVAL_RUNNER}", - system="azure-devops", - task="{official_task}", - status=status, - job_status=job_status, - agent_ids="$(official_eval_input.officialAgentIds)", - deployment_name="$(official_eval_input.officialDeploymentName)", - data_path="$(official_eval_input.officialDataPath)", - metadata_path="$(official_eval_input.officialMetadataPath)", - machine_readable_thresholds=False, - recorded_at=datetime.now(timezone.utc).isoformat(), - ) - Path(".agentops/official-eval/result.json").write_text( - json.dumps(payload, indent=2) + "\\n", - encoding="utf-8", - ) - PY - displayName: Record official eval result - condition: always()""", - base_indent, - ), - "__EVAL_ARTIFACT_TARGET__": ".agentops/official-eval", - } - return { - "__EVAL_TASKS__": _indent_block( - f"""- task: AzureCLI@2 - displayName: Run AgentOps eval - inputs: - azureSubscription: $(AZURE_SERVICE_CONNECTION) - scriptType: bash - scriptLocation: inlineScript - inlineScript: | - set +e - agentops eval run --config \"{config_path}\" - code=$? - echo \"##vso[task.setvariable variable=AGENTOPS_EVAL_EXIT_CODE]$code\" - exit $code - env: - AZURE_AI_FOUNDRY_PROJECT_ENDPOINT: $(AZURE_AI_FOUNDRY_PROJECT_ENDPOINT) - AZURE_OPENAI_ENDPOINT: $(AZURE_OPENAI_ENDPOINT) - AZURE_OPENAI_DEPLOYMENT: $(AZURE_OPENAI_DEPLOYMENT) - APPLICATIONINSIGHTS_CONNECTION_STRING: $(APPLICATIONINSIGHTS_CONNECTION_STRING)""", - base_indent, - ), - "__EVAL_ARTIFACT_TARGET__": ".agentops/results/latest", - } - - -def _indent_block(block: str, spaces: int) -> str: - prefix = " " * spaces - return "\n".join(prefix + line if line else "" for line in block.splitlines()) - - -def normalize_workflow_kind(kind: str) -> str: - """Return the canonical workflow kind, accepting legacy aliases.""" - return LEGACY_KIND_ALIASES.get(kind, kind) - - -def generate_cicd_workflows( - directory: Path, - force: bool = False, - kinds: Sequence[str] | None = None, - platform: str = "github", - deploy_mode: str = "auto", - doctor_gate: str = DEFAULT_DOCTOR_GATE, -) -> CicdResult: - """Generate AgentOps GitFlow CI/CD workflows. - - By default writes the release-path templates (``pr``, ``dev``, ``qa``, - ``prod``) for the requested *platform*. Pass *kinds* to opt into a subset, - including the optional scheduled ``doctor`` workflow. - - Args: - directory: Root directory of the consumer repository. - force: When True, overwrite existing workflow files. - kinds: Optional explicit list of workflow kinds. ``None`` means - "generate the default release-path templates". Unknown kinds are ignored. - platform: ``"github"`` (default) writes ``.github/workflows/*.yml`` - using GitHub Actions; ``"azure-devops"`` writes - ``.azuredevops/pipelines/*.yml`` using Azure DevOps Pipelines. - The conceptual workflows (PR gate + three deploy stages) are - identical across platforms. - deploy_mode: ``"placeholder"`` writes the stack-agnostic deploy - scaffold, ``"azd"`` writes Azure Developer CLI provision/deploy - workflows, ``"prompt-agent"`` writes Foundry prompt-agent - candidate/eval/deploy workflows, and ``"auto"`` selects - ``"azd"`` when ``azure.yaml`` exists or ``"prompt-agent"`` when - ``agentops.yaml`` targets a Foundry prompt agent. - doctor_gate: Severity floor for the PR-gate Doctor step. One of - ``"critical"`` (default), ``"warning"``, or ``"none"``. - ``critical`` blocks the PR on critical Doctor findings such as - regression drops; ``warning`` blocks on warning or higher; - ``none`` keeps Doctor advisory (pre-1.x behavior). Only the PR - template uses this value; deploy templates keep the - hardcoded ``critical`` gate. - - Returns: - CicdResult with platform and paths of created, overwritten, or - skipped files. - """ - if platform not in _TEMPLATES_BY_PLATFORM: - raise ValueError( - f"unknown platform {platform!r}; valid: {', '.join(PLATFORMS)}" - ) - if deploy_mode not in DEPLOY_MODES: - raise ValueError( - f"unknown deploy mode {deploy_mode!r}; valid: {', '.join(DEPLOY_MODES)}" - ) - if doctor_gate not in DOCTOR_GATES: - raise ValueError( - f"unknown doctor gate {doctor_gate!r}; valid: {', '.join(DOCTOR_GATES)}" - ) - - if kinds is None: - kinds = DEFAULT_KINDS - - directory = directory.resolve() - effective_deploy_mode = deploy_mode - if effective_deploy_mode == "auto": - effective_deploy_mode = recommended_deploy_mode(directory) - effective_eval_runner = recommended_eval_runner(directory) - - result = CicdResult( - platform=platform, - deploy_mode=effective_deploy_mode, - eval_runner=effective_eval_runner, - doctor_gate=doctor_gate, - ) - templates_root = files(_TEMPLATE_PACKAGE) - template_map = _TEMPLATES_BY_PLATFORM[platform] - azd_template_map = _AZD_TEMPLATES_BY_PLATFORM.get(platform, {}) - prompt_agent_template_map = _PROMPT_AGENT_TEMPLATES_BY_PLATFORM.get(platform, {}) - azd_substitutions = _azd_substitutions(platform, has_ailz_preflight(directory)) - - seen: set[str] = set() - for requested_kind in kinds: - kind = normalize_workflow_kind(requested_kind) - if kind in seen or kind not in template_map: - continue - seen.add(kind) - result.kinds.append(kind) - substitutions: dict[str, str] = {"__DOCTOR_GATE__": doctor_gate} - eval_config = ( - "${{ inputs.config || 'agentops.yaml' }}" - if platform == "github" and kind == "pr" - else "$(AGENTOPS_CONFIG)" - if platform == "azure-devops" - else "agentops.yaml" - ) - if effective_deploy_mode == "azd" and kind in azd_template_map: - template_path, output_rel = azd_template_map[kind] - substitutions.update(azd_substitutions) - elif effective_deploy_mode == "prompt-agent" and kind in prompt_agent_template_map: - template_path, output_rel = prompt_agent_template_map[kind] - eval_config = ".agentops/deployments/agentops.candidate.yaml" - prompt_values = ( - _PROMPT_AGENT_VALUES if platform == "github" else _PROMPT_AGENT_VALUES_ADO - )[kind] - substitutions.update(prompt_values) - else: - template_path, output_rel = template_map[kind] - ado_indent = ( - 10 - if kind == "pr" - or (effective_deploy_mode == "azd" and kind in azd_template_map) - else 16 - ) - substitutions.update( - _eval_substitutions( - platform, - effective_eval_runner, - eval_config, - ado_indent=ado_indent, - ) - ) - output_path = (directory / output_rel).resolve() - _write_template( - templates_root, - template_path, - output_path, - force, - result, - substitutions=substitutions, - ) - - return result - - -def _azd_substitutions(platform: str, ailz_preflight: bool) -> Mapping[str, str]: - if not ailz_preflight: - return {"__AILZ_PREFLIGHT_COMMAND__": ""} - if platform == "azure-devops": - return { - "__AILZ_PREFLIGHT_COMMAND__": ( - " echo \"Running AI Landing Zone preflight.\"\n" - " pwsh ./scripts/Invoke-PreflightChecks.ps1 -Strict" - ) - } - return { - "__AILZ_PREFLIGHT_COMMAND__": ( - " echo \"Running AI Landing Zone preflight.\"\n" - " pwsh ./scripts/Invoke-PreflightChecks.ps1 -Strict" - ) - } - - -def generate_cicd_workflow( - directory: Path, - force: bool = False, - platform: str = "github", -) -> CicdResult: - """Generate only the PR workflow template (legacy convenience).""" - return generate_cicd_workflows(directory, force=force, kinds=["pr"], platform=platform) +"""CI/CD workflow generation service for `agentops workflow generate`.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from importlib.resources import files +from pathlib import Path +from typing import Dict, List, Mapping, Sequence, Tuple + +from agentops.pipeline.official_eval import ( + AGENTOPS_CLOUD_RUNNER, + AGENTOPS_LOCAL_RUNNER, + OFFICIAL_EVAL_ACTION_ENV, + OFFICIAL_EVAL_ADO_TASK_ENV, + OFFICIAL_EVAL_RUNNER, + official_eval_action_ref, + official_eval_ado_task_ref, +) +from agentops.services.workflow_analysis import ( + has_ailz_preflight, + recommended_deploy_mode, + recommended_eval_runner, +) + + +_TEMPLATE_PACKAGE = "agentops.templates" +_CLOUD_EVAL_CONFIG_NAME = ".agentops.cloud.yaml" +_CI_EVAL_OUTPUT = ".agentops/results/latest" + +# CI/CD platforms supported by ``agentops workflow generate``. +PLATFORMS: Tuple[str, ...] = ("github", "azure-devops") + +# Deployment template modes. ``placeholder`` keeps the stack-agnostic +# scaffold; ``azd`` delegates infrastructure and app deployment to Azure +# Developer CLI; ``prompt-agent`` creates/evaluates a candidate Foundry prompt +# agent version from a source-controlled prompt file. ``auto`` selects +# ``azd`` when the target repo has ``azure.yaml`` and ``prompt-agent`` when +# ``agentops.yaml`` targets a Foundry prompt agent. +DEPLOY_MODES: Tuple[str, ...] = ("auto", "placeholder", "azd", "prompt-agent") + +# Doctor gate severities supported by ``agentops workflow generate +# --doctor-gate``. The PR workflow template runs `agentops doctor +# --severity-fail `; ``critical`` (the default) blocks the PR on +# critical Doctor findings (including regression detection), ``warning`` +# blocks on warning or higher, and ``none`` keeps Doctor advisory. +DOCTOR_GATES: Tuple[str, ...] = ("critical", "warning", "none") +DEFAULT_DOCTOR_GATE: str = "critical" + +# Per-platform mapping of workflow kind -> (template path inside package, +# output path in repo). +# +# The default templates form a complete GenAIOps GitFlow scaffold: +# +# pr -> agentops-pr (PR gate; PRs to develop, release/**, main) +# dev -> agentops-deploy-dev (push to develop -> environment: dev) +# qa -> agentops-deploy-qa (push to release/** -> environment: qa) +# prod -> agentops-deploy-prod (push to main -> environment: production) +# +# A scheduled Doctor workflow is also available as an explicit optional kind: +# +# doctor -> agentops-doctor (scheduled Doctor + eval health check) +_TEMPLATES_BY_PLATFORM: Dict[str, Dict[str, Tuple[str, str]]] = { + "github": { + "pr": ("workflows/agentops-pr.yml", ".github/workflows/agentops-pr.yml"), + "dev": ("workflows/agentops-deploy-dev.yml", ".github/workflows/agentops-deploy-dev.yml"), + "qa": ("workflows/agentops-deploy-qa.yml", ".github/workflows/agentops-deploy-qa.yml"), + "prod": ("workflows/agentops-deploy-prod.yml", ".github/workflows/agentops-deploy-prod.yml"), + "doctor": ("workflows/agentops-watchdog.yml", ".github/workflows/agentops-doctor.yml"), + }, + "azure-devops": { + "pr": ( + "pipelines/azuredevops/agentops-pr.yml", + ".azuredevops/pipelines/agentops-pr.yml", + ), + "dev": ( + "pipelines/azuredevops/agentops-deploy-dev.yml", + ".azuredevops/pipelines/agentops-deploy-dev.yml", + ), + "qa": ( + "pipelines/azuredevops/agentops-deploy-qa.yml", + ".azuredevops/pipelines/agentops-deploy-qa.yml", + ), + "prod": ( + "pipelines/azuredevops/agentops-deploy-prod.yml", + ".azuredevops/pipelines/agentops-deploy-prod.yml", + ), + "doctor": ( + "pipelines/azuredevops/agentops-watchdog.yml", + ".azuredevops/pipelines/agentops-doctor.yml", + ), + }, +} + +_AZD_TEMPLATES_BY_PLATFORM: Dict[str, Dict[str, Tuple[str, str]]] = { + "github": { + "dev": ("workflows/agentops-deploy-dev-azd.yml", ".github/workflows/agentops-deploy-dev.yml"), + "qa": ("workflows/agentops-deploy-qa-azd.yml", ".github/workflows/agentops-deploy-qa.yml"), + "prod": ("workflows/agentops-deploy-prod-azd.yml", ".github/workflows/agentops-deploy-prod.yml"), + }, + "azure-devops": { + "dev": ( + "pipelines/azuredevops/agentops-deploy-dev-azd.yml", + ".azuredevops/pipelines/agentops-deploy-dev.yml", + ), + "qa": ( + "pipelines/azuredevops/agentops-deploy-qa-azd.yml", + ".azuredevops/pipelines/agentops-deploy-qa.yml", + ), + "prod": ( + "pipelines/azuredevops/agentops-deploy-prod-azd.yml", + ".azuredevops/pipelines/agentops-deploy-prod.yml", + ), + }, +} + +ALL_KINDS: tuple[str, ...] = ("pr", "dev", "qa", "prod", "doctor") +DEFAULT_KINDS: tuple[str, ...] = ("pr", "dev", "qa", "prod") +LEGACY_KIND_ALIASES: Mapping[str, str] = {"watchdog": "doctor"} + + +@dataclass +class CicdResult: + """Result of generating CI/CD workflow files.""" + + platform: str = "github" + deploy_mode: str = "placeholder" + eval_runner: str = AGENTOPS_LOCAL_RUNNER + doctor_gate: str = "critical" + kinds: List[str] = field(default_factory=list) + created_files: List[Path] = field(default_factory=list) + overwritten_files: List[Path] = field(default_factory=list) + skipped_files: List[Path] = field(default_factory=list) + + +def _write_template( + templates_root, + template_path: str, + output_path: Path, + force: bool, + result: CicdResult, + substitutions: Mapping[str, str] | None = None, +) -> None: + template_resource = templates_root.joinpath(template_path) + template_content = template_resource.read_text(encoding="utf-8") + for key, value in (substitutions or {}).items(): + template_content = template_content.replace(key, value) + + existed_before = output_path.exists() + + if existed_before and not force: + result.skipped_files.append(output_path) + return + + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(template_content, encoding="utf-8") + + if existed_before: + result.overwritten_files.append(output_path) + else: + result.created_files.append(output_path) + + +def _branch_block_github(*branches: str) -> str: + return "".join(f" - {branch}\n" for branch in branches).rstrip() + + +def _branch_block_ado(*branches: str) -> str: + return "".join(f" - {branch}\n" for branch in branches).rstrip() + + +_PROMPT_AGENT_VALUES: Dict[str, Dict[str, str]] = { + "pr": { + "__ENV_LABEL__": "PR", + "__ENV_KEY__": "pr", + # PR candidates are staged in the dev Foundry project so the + # gate evaluates the same target the deploy workflow will use. + # Sandbox is the author's playground only. + "__ENV_NAME__": "dev", + "__BRANCHES__": "", + "__EVAL_JOB_NAME__": "AgentOps eval (PR gate)", + }, + "dev": { + "__ENV_LABEL__": "DEV", + "__ENV_KEY__": "dev", + "__ENV_NAME__": "dev", + "__BRANCHES__": _branch_block_github("develop"), + "__EVAL_JOB_NAME__": "Eval candidate (gate)", + }, + "qa": { + "__ENV_LABEL__": "QA", + "__ENV_KEY__": "qa", + "__ENV_NAME__": "qa", + "__BRANCHES__": _branch_block_github('"release/**"'), + "__EVAL_JOB_NAME__": "Eval candidate (gate)", + }, + "prod": { + "__ENV_LABEL__": "PROD", + "__ENV_KEY__": "prod", + "__ENV_NAME__": "production", + "__BRANCHES__": _branch_block_github("main"), + "__EVAL_JOB_NAME__": "Safety eval candidate (gate)", + }, +} + +_PROMPT_AGENT_VALUES_ADO: Dict[str, Dict[str, str]] = { + "pr": { + "__ENV_LABEL__": "PR", + "__ENV_KEY__": "pr", + # PR candidates are staged in the dev Foundry project so the + # gate evaluates the same target the deploy pipeline will use. + "__ENV_NAME__": "dev", + "__BRANCHES__": "", + }, + "dev": { + "__ENV_LABEL__": "dev", + "__ENV_KEY__": "dev", + "__ENV_NAME__": "dev", + "__BRANCHES__": _branch_block_ado("develop"), + }, + "qa": { + "__ENV_LABEL__": "qa", + "__ENV_KEY__": "qa", + "__ENV_NAME__": "qa", + "__BRANCHES__": _branch_block_ado("release/*"), + }, + "prod": { + "__ENV_LABEL__": "production", + "__ENV_KEY__": "prod", + "__ENV_NAME__": "production", + "__BRANCHES__": _branch_block_ado("main"), + }, +} + +_PROMPT_AGENT_TEMPLATES_BY_PLATFORM: Dict[str, Dict[str, Tuple[str, str]]] = { + "github": { + "pr": ( + "workflows/agentops-pr-prompt-agent.yml", + ".github/workflows/agentops-pr.yml", + ), + "dev": ("workflows/agentops-deploy-prompt-agent.yml", ".github/workflows/agentops-deploy-dev.yml"), + "qa": ("workflows/agentops-deploy-prompt-agent.yml", ".github/workflows/agentops-deploy-qa.yml"), + "prod": ("workflows/agentops-deploy-prompt-agent.yml", ".github/workflows/agentops-deploy-prod.yml"), + }, + "azure-devops": { + "pr": ( + "pipelines/azuredevops/agentops-pr-prompt-agent.yml", + ".azuredevops/pipelines/agentops-pr.yml", + ), + "dev": ( + "pipelines/azuredevops/agentops-deploy-prompt-agent.yml", + ".azuredevops/pipelines/agentops-deploy-dev.yml", + ), + "qa": ( + "pipelines/azuredevops/agentops-deploy-prompt-agent.yml", + ".azuredevops/pipelines/agentops-deploy-qa.yml", + ), + "prod": ( + "pipelines/azuredevops/agentops-deploy-prompt-agent.yml", + ".azuredevops/pipelines/agentops-deploy-prod.yml", + ), + }, +} + + +def _eval_substitutions( + platform: str, + eval_runner: str, + config_path: str, + *, + ado_indent: int = 10, +) -> Mapping[str, str]: + if platform == "azure-devops": + return _ado_eval_substitutions(eval_runner, config_path, base_indent=ado_indent) + return _github_eval_substitutions(eval_runner, config_path) + + +def _github_eval_substitutions(eval_runner: str, config_path: str) -> Mapping[str, str]: + if eval_runner == AGENTOPS_CLOUD_RUNNER: + return { + "__EVAL_STEPS__": f""" - name: Prepare AgentOps cloud eval config + env: + AGENTOPS_SOURCE_CONFIG: "{config_path}" + run: | + python - <<'PY' + import os + from pathlib import Path + from agentops.utils.yaml import load_yaml, save_yaml + + source = Path(os.environ["AGENTOPS_SOURCE_CONFIG"]) + target = source.with_name("{_CLOUD_EVAL_CONFIG_NAME}") + data = load_yaml(source) + data["execution"] = "cloud" + data["publish"] = True + save_yaml(target, data) + with Path(os.environ["GITHUB_ENV"]).open("a", encoding="utf-8") as env_file: + env_file.write(f"AGENTOPS_CI_CONFIG={{target}}\\n") + print(f"Prepared AgentOps cloud eval config: {{target}}") + PY + + - name: Run AgentOps Foundry cloud eval + id: eval + env: + AZURE_AI_FOUNDRY_PROJECT_ENDPOINT: ${{{{ vars.AZURE_AI_FOUNDRY_PROJECT_ENDPOINT }}}} + AZURE_OPENAI_ENDPOINT: ${{{{ vars.AZURE_OPENAI_ENDPOINT }}}} + AZURE_OPENAI_DEPLOYMENT: ${{{{ vars.AZURE_OPENAI_DEPLOYMENT }}}} + APPLICATIONINSIGHTS_CONNECTION_STRING: ${{{{ secrets.APPLICATIONINSIGHTS_CONNECTION_STRING || vars.APPLICATIONINSIGHTS_CONNECTION_STRING }}}} + run: | + set +e + agentops eval run --config "$AGENTOPS_CI_CONFIG" --output "{_CI_EVAL_OUTPUT}" + ec=$? + echo "exit_code=$ec" >> "$GITHUB_OUTPUT" + if [ $ec -eq 0 ]; then + echo "result=pass" >> "$GITHUB_OUTPUT" + elif [ $ec -eq 2 ]; then + echo "result=threshold_failed" >> "$GITHUB_OUTPUT" + else + echo "result=error" >> "$GITHUB_OUTPUT" + fi + exit $ec""", + "__EVAL_ARTIFACT_PATHS__": f"""{_CI_EVAL_OUTPUT}/results.json + {_CI_EVAL_OUTPUT}/report.md + {_CI_EVAL_OUTPUT}/cloud_evaluation.json + {_CI_EVAL_OUTPUT}/cloud_output_items.json""", + } + if eval_runner == OFFICIAL_EVAL_RUNNER: + official_action = official_eval_action_ref() + return { + "__EVAL_STEPS__": f""" - name: Prepare official AI Agent Evaluation input + id: official_eval_input + env: + AZURE_OPENAI_DEPLOYMENT: ${{{{ vars.AZURE_OPENAI_DEPLOYMENT }}}} + {OFFICIAL_EVAL_ACTION_ENV}: {official_action} + run: | + python -m agentops.pipeline.official_eval prepare \\ + --config \"{config_path}\" \\ + --out \".agentops/official-eval/input.json\" \\ + --github-output \"$GITHUB_OUTPUT\" + + # Official runner for Foundry prompt agents. AgentOps keeps the prepared + # input/metadata as release evidence until the action exposes a stable + # machine-readable threshold artifact. + - name: Run official AI Agent Evaluation + id: official_eval + uses: {official_action} + with: + azure-ai-project-endpoint: ${{{{ vars.AZURE_AI_FOUNDRY_PROJECT_ENDPOINT }}}} + deployment-name: ${{{{ steps.official_eval_input.outputs.deployment_name }}}} + agent-ids: ${{{{ steps.official_eval_input.outputs.agent_ids }}}} + data-path: ${{{{ steps.official_eval_input.outputs.data_path }}}} + + - name: Record official eval result + if: always() + id: eval + run: | + mkdir -p .agentops/official-eval + outcome=\"${{{{ steps.official_eval.outcome }}}}\" + conclusion=\"${{{{ steps.official_eval.conclusion }}}}\" + if [ \"$outcome\" = \"success\" ]; then + echo \"exit_code=0\" >> \"$GITHUB_OUTPUT\" + echo \"result=official-ai-agent-evaluation\" >> \"$GITHUB_OUTPUT\" + else + echo \"exit_code=1\" >> \"$GITHUB_OUTPUT\" + echo \"result=official-ai-agent-evaluation-failed\" >> \"$GITHUB_OUTPUT\" + fi + python - <<'PY' + import json + from datetime import datetime, timezone + from pathlib import Path + + output = Path('.agentops/official-eval/result.json') + outcome = \"${{{{ steps.official_eval.outcome }}}}\" + conclusion = \"${{{{ steps.official_eval.conclusion }}}}\" + status = 'success' if outcome == 'success' else 'failed' if outcome else 'unknown' + payload = dict( + runner='{OFFICIAL_EVAL_RUNNER}', + system='github-actions', + action='{official_action}', + status=status, + outcome=outcome, + conclusion=conclusion, + agent_ids=\"${{{{ steps.official_eval_input.outputs.agent_ids }}}}\", + deployment_name=\"${{{{ steps.official_eval_input.outputs.deployment_name }}}}\", + data_path=\"${{{{ steps.official_eval_input.outputs.data_path }}}}\", + metadata_path=\"${{{{ steps.official_eval_input.outputs.metadata_path }}}}\", + machine_readable_thresholds=False, + recorded_at=datetime.now(timezone.utc).isoformat(), + ) + output.write_text(json.dumps(payload, indent=2) + '\\n', encoding='utf-8') + PY""", + "__EVAL_ARTIFACT_PATHS__": """.agentops/official-eval/input.json + .agentops/official-eval/metadata.json + .agentops/official-eval/result.json""", + } + return { + "__EVAL_STEPS__": f""" - name: Run AgentOps eval + id: eval + env: + AZURE_AI_FOUNDRY_PROJECT_ENDPOINT: ${{{{ vars.AZURE_AI_FOUNDRY_PROJECT_ENDPOINT }}}} + AZURE_OPENAI_ENDPOINT: ${{{{ vars.AZURE_OPENAI_ENDPOINT }}}} + AZURE_OPENAI_DEPLOYMENT: ${{{{ vars.AZURE_OPENAI_DEPLOYMENT }}}} + APPLICATIONINSIGHTS_CONNECTION_STRING: ${{{{ secrets.APPLICATIONINSIGHTS_CONNECTION_STRING || vars.APPLICATIONINSIGHTS_CONNECTION_STRING }}}} + run: | + set +e + agentops eval run --config \"{config_path}\" + ec=$? + echo \"exit_code=$ec\" >> \"$GITHUB_OUTPUT\" + if [ $ec -eq 0 ]; then + echo \"result=pass\" >> \"$GITHUB_OUTPUT\" + elif [ $ec -eq 2 ]; then + echo \"result=threshold_failed\" >> \"$GITHUB_OUTPUT\" + else + echo \"result=error\" >> \"$GITHUB_OUTPUT\" + fi + exit $ec""", + "__EVAL_ARTIFACT_PATHS__": """.agentops/results/latest/results.json + .agentops/results/latest/report.md + .agentops/results/latest/cloud_evaluation.json + .agentops/results/latest/cloud_output_items.json""", + } + + +def _ado_eval_substitutions( + eval_runner: str, + config_path: str, + *, + base_indent: int, +) -> Mapping[str, str]: + if eval_runner == AGENTOPS_CLOUD_RUNNER: + return { + "__EVAL_TASKS__": _indent_block( + f"""- bash: | + python - <<'PY' + import os + from pathlib import Path + from agentops.utils.yaml import load_yaml, save_yaml + + source = Path(os.environ["AGENTOPS_SOURCE_CONFIG"]) + target = source.with_name("{_CLOUD_EVAL_CONFIG_NAME}") + data = load_yaml(source) + data["execution"] = "cloud" + data["publish"] = True + save_yaml(target, data) + print(f"##vso[task.setvariable variable=AGENTOPS_CI_CONFIG]{{target}}") + print(f"Prepared AgentOps cloud eval config: {{target}}") + PY + displayName: Prepare AgentOps cloud eval config + env: + AGENTOPS_SOURCE_CONFIG: "{config_path}" + +- task: AzureCLI@2 + displayName: Run AgentOps Foundry cloud eval + inputs: + azureSubscription: $(AZURE_SERVICE_CONNECTION) + scriptType: bash + scriptLocation: inlineScript + inlineScript: | + set +e + agentops eval run --config "$(AGENTOPS_CI_CONFIG)" --output "{_CI_EVAL_OUTPUT}" + code=$? + echo "##vso[task.setvariable variable=AGENTOPS_EVAL_EXIT_CODE]$code" + exit $code + env: + AZURE_AI_FOUNDRY_PROJECT_ENDPOINT: $(AZURE_AI_FOUNDRY_PROJECT_ENDPOINT) + AZURE_OPENAI_ENDPOINT: $(AZURE_OPENAI_ENDPOINT) + AZURE_OPENAI_DEPLOYMENT: $(AZURE_OPENAI_DEPLOYMENT) + APPLICATIONINSIGHTS_CONNECTION_STRING: $(APPLICATIONINSIGHTS_CONNECTION_STRING)""", + base_indent, + ), + "__EVAL_ARTIFACT_TARGET__": _CI_EVAL_OUTPUT, + } + if eval_runner == OFFICIAL_EVAL_RUNNER: + official_task = official_eval_ado_task_ref() + return { + "__EVAL_TASKS__": _indent_block( + f"""- task: AzureCLI@2 + name: official_eval_input + displayName: Prepare official AI Agent Evaluation input + inputs: + azureSubscription: $(AZURE_SERVICE_CONNECTION) + scriptType: bash + scriptLocation: inlineScript + inlineScript: | + python -m agentops.pipeline.official_eval prepare \\ + --config \"{config_path}\" \\ + --out \".agentops/official-eval/input.json\" \\ + --ado-output + env: + AZURE_OPENAI_DEPLOYMENT: $(AZURE_OPENAI_DEPLOYMENT) + {OFFICIAL_EVAL_ADO_TASK_ENV}: {official_task} + +- task: {official_task} + displayName: Run official AI Agent Evaluation + inputs: + azure-ai-project-endpoint: $(AZURE_AI_FOUNDRY_PROJECT_ENDPOINT) + deployment-name: $(official_eval_input.officialDeploymentName) + data-path: $(official_eval_input.officialDataPath) + agent-ids: $(official_eval_input.officialAgentIds) + +- bash: | + mkdir -p .agentops/official-eval + python - <<'PY' + import json + from datetime import datetime, timezone + from pathlib import Path + + job_status = "$(Agent.JobStatus)" + normalized = job_status.strip().lower().replace("_", "").replace("-", "") + status = "success" if normalized == "succeeded" else "failed" if normalized in ("failed", "canceled", "cancelled") else "unknown" + payload = dict( + runner="{OFFICIAL_EVAL_RUNNER}", + system="azure-devops", + task="{official_task}", + status=status, + job_status=job_status, + agent_ids="$(official_eval_input.officialAgentIds)", + deployment_name="$(official_eval_input.officialDeploymentName)", + data_path="$(official_eval_input.officialDataPath)", + metadata_path="$(official_eval_input.officialMetadataPath)", + machine_readable_thresholds=False, + recorded_at=datetime.now(timezone.utc).isoformat(), + ) + Path(".agentops/official-eval/result.json").write_text( + json.dumps(payload, indent=2) + "\\n", + encoding="utf-8", + ) + PY + displayName: Record official eval result + condition: always()""", + base_indent, + ), + "__EVAL_ARTIFACT_TARGET__": ".agentops/official-eval", + } + return { + "__EVAL_TASKS__": _indent_block( + f"""- task: AzureCLI@2 + displayName: Run AgentOps eval + inputs: + azureSubscription: $(AZURE_SERVICE_CONNECTION) + scriptType: bash + scriptLocation: inlineScript + inlineScript: | + set +e + agentops eval run --config \"{config_path}\" + code=$? + echo \"##vso[task.setvariable variable=AGENTOPS_EVAL_EXIT_CODE]$code\" + exit $code + env: + AZURE_AI_FOUNDRY_PROJECT_ENDPOINT: $(AZURE_AI_FOUNDRY_PROJECT_ENDPOINT) + AZURE_OPENAI_ENDPOINT: $(AZURE_OPENAI_ENDPOINT) + AZURE_OPENAI_DEPLOYMENT: $(AZURE_OPENAI_DEPLOYMENT) + APPLICATIONINSIGHTS_CONNECTION_STRING: $(APPLICATIONINSIGHTS_CONNECTION_STRING)""", + base_indent, + ), + "__EVAL_ARTIFACT_TARGET__": ".agentops/results/latest", + } + + +def _indent_block(block: str, spaces: int) -> str: + prefix = " " * spaces + return "\n".join(prefix + line if line else "" for line in block.splitlines()) + + +def normalize_workflow_kind(kind: str) -> str: + """Return the canonical workflow kind, accepting legacy aliases.""" + return LEGACY_KIND_ALIASES.get(kind, kind) + + +def generate_cicd_workflows( + directory: Path, + force: bool = False, + kinds: Sequence[str] | None = None, + platform: str = "github", + deploy_mode: str = "auto", + doctor_gate: str = DEFAULT_DOCTOR_GATE, +) -> CicdResult: + """Generate AgentOps GitFlow CI/CD workflows. + + By default writes the release-path templates (``pr``, ``dev``, ``qa``, + ``prod``) for the requested *platform*. Pass *kinds* to opt into a subset, + including the optional scheduled ``doctor`` workflow. + + Args: + directory: Root directory of the consumer repository. + force: When True, overwrite existing workflow files. + kinds: Optional explicit list of workflow kinds. ``None`` means + "generate the default release-path templates". Unknown kinds are ignored. + platform: ``"github"`` (default) writes ``.github/workflows/*.yml`` + using GitHub Actions; ``"azure-devops"`` writes + ``.azuredevops/pipelines/*.yml`` using Azure DevOps Pipelines. + The conceptual workflows (PR gate + three deploy stages) are + identical across platforms. + deploy_mode: ``"placeholder"`` writes the stack-agnostic deploy + scaffold, ``"azd"`` writes Azure Developer CLI provision/deploy + workflows, ``"prompt-agent"`` writes Foundry prompt-agent + candidate/eval/deploy workflows, and ``"auto"`` selects + ``"azd"`` when ``azure.yaml`` exists or ``"prompt-agent"`` when + ``agentops.yaml`` targets a Foundry prompt agent. + doctor_gate: Severity floor for the PR-gate Doctor step. One of + ``"critical"`` (default), ``"warning"``, or ``"none"``. + ``critical`` blocks the PR on critical Doctor findings such as + regression drops; ``warning`` blocks on warning or higher; + ``none`` keeps Doctor advisory (pre-1.x behavior). Only the PR + template uses this value; deploy templates keep the + hardcoded ``critical`` gate. + + Returns: + CicdResult with platform and paths of created, overwritten, or + skipped files. + """ + if platform not in _TEMPLATES_BY_PLATFORM: + raise ValueError( + f"unknown platform {platform!r}; valid: {', '.join(PLATFORMS)}" + ) + if deploy_mode not in DEPLOY_MODES: + raise ValueError( + f"unknown deploy mode {deploy_mode!r}; valid: {', '.join(DEPLOY_MODES)}" + ) + if doctor_gate not in DOCTOR_GATES: + raise ValueError( + f"unknown doctor gate {doctor_gate!r}; valid: {', '.join(DOCTOR_GATES)}" + ) + + if kinds is None: + kinds = DEFAULT_KINDS + + directory = directory.resolve() + effective_deploy_mode = deploy_mode + if effective_deploy_mode == "auto": + effective_deploy_mode = recommended_deploy_mode(directory) + effective_eval_runner = recommended_eval_runner(directory) + + result = CicdResult( + platform=platform, + deploy_mode=effective_deploy_mode, + eval_runner=effective_eval_runner, + doctor_gate=doctor_gate, + ) + templates_root = files(_TEMPLATE_PACKAGE) + template_map = _TEMPLATES_BY_PLATFORM[platform] + azd_template_map = _AZD_TEMPLATES_BY_PLATFORM.get(platform, {}) + prompt_agent_template_map = _PROMPT_AGENT_TEMPLATES_BY_PLATFORM.get(platform, {}) + azd_substitutions = _azd_substitutions(platform, has_ailz_preflight(directory)) + + seen: set[str] = set() + for requested_kind in kinds: + kind = normalize_workflow_kind(requested_kind) + if kind in seen or kind not in template_map: + continue + seen.add(kind) + result.kinds.append(kind) + substitutions: dict[str, str] = {"__DOCTOR_GATE__": doctor_gate} + eval_config = ( + "${{ inputs.config || 'agentops.yaml' }}" + if platform == "github" and kind == "pr" + else "$(AGENTOPS_CONFIG)" + if platform == "azure-devops" + else "agentops.yaml" + ) + if effective_deploy_mode == "azd" and kind in azd_template_map: + template_path, output_rel = azd_template_map[kind] + substitutions.update(azd_substitutions) + elif effective_deploy_mode == "prompt-agent" and kind in prompt_agent_template_map: + template_path, output_rel = prompt_agent_template_map[kind] + eval_config = ".agentops/deployments/agentops.candidate.yaml" + prompt_values = ( + _PROMPT_AGENT_VALUES if platform == "github" else _PROMPT_AGENT_VALUES_ADO + )[kind] + substitutions.update(prompt_values) + else: + template_path, output_rel = template_map[kind] + ado_indent = ( + 10 + if kind == "pr" + or (effective_deploy_mode == "azd" and kind in azd_template_map) + else 16 + ) + substitutions.update( + _eval_substitutions( + platform, + effective_eval_runner, + eval_config, + ado_indent=ado_indent, + ) + ) + output_path = (directory / output_rel).resolve() + _write_template( + templates_root, + template_path, + output_path, + force, + result, + substitutions=substitutions, + ) + + return result + + +def _azd_substitutions(platform: str, ailz_preflight: bool) -> Mapping[str, str]: + if not ailz_preflight: + return {"__AILZ_PREFLIGHT_COMMAND__": ""} + if platform == "azure-devops": + return { + "__AILZ_PREFLIGHT_COMMAND__": ( + " echo \"Running AI Landing Zone preflight.\"\n" + " pwsh ./scripts/Invoke-PreflightChecks.ps1 -Strict" + ) + } + return { + "__AILZ_PREFLIGHT_COMMAND__": ( + " echo \"Running AI Landing Zone preflight.\"\n" + " pwsh ./scripts/Invoke-PreflightChecks.ps1 -Strict" + ) + } + + +def generate_cicd_workflow( + directory: Path, + force: bool = False, + platform: str = "github", +) -> CicdResult: + """Generate only the PR workflow template (legacy convenience).""" + return generate_cicd_workflows(directory, force=force, kinds=["pr"], platform=platform)