From b5cdb2d4e9c09bc9297b40c7d81168e2ee911436 Mon Sep 17 00:00:00 2001 From: Paulo Lacerda Date: Mon, 1 Jun 2026 17:37:58 -0300 Subject: [PATCH 1/2] fix: grant eval RBAC to Foundry managed identities (#229) Cloud evaluations run server-side and some agent or grader calls authenticate as the managed identities on the AI Services account and child Foundry project, not only as the signed-in user. Granting Cognitive Services OpenAI User only to the user can still produce grader AuthenticationError warnings/failures even when every computable threshold passes. Update the prompt-agent, hosted-agent, and end-to-end tutorials plus the packaged agentops-eval skill to assign the data-plane role to all managed identities in the Foundry resource group as well as the user. Sync the plugin skill copy and document the fix in the changelog. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- CHANGELOG.md | 12 +++++++ docs/tutorial-end-to-end.md | 30 +++++++++++++---- docs/tutorial-hosted-agent-quickstart.md | 30 +++++++++++++---- docs/tutorial-prompt-agent-quickstart.md | 32 +++++++++++++++---- .../agentops/skills/agentops-eval/SKILL.md | 22 +++++++++++-- .../templates/skills/agentops-eval/SKILL.md | 22 +++++++++++-- 6 files changed, 121 insertions(+), 27 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f7d77d3..990702e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,18 @@ This format follows [Keep a Changelog](https://keepachangelog.com/) and adheres ## [Unreleased] +### Fixed +- **RBAC preflight now covers Foundry/Azure AI managed identities, not only + the signed-in user.** Cloud evaluations run server-side and some agent or + grader calls authenticate as the managed identities on the backing AI + Services account and child Foundry project. Granting `Cognitive Services + OpenAI User` only to the user still allowed intermittent grader + `AuthenticationError` failures and the v0.3.6 execution warning. The + prompt-agent, hosted-agent, and end-to-end tutorials plus the + `agentops-eval` skill now assign the same data-plane role to every managed + identity in the Foundry resource group, preventing the warning/failure path + before `agentops eval run`. + ## [0.3.6] - 2026-06-01 ### Changed diff --git a/docs/tutorial-end-to-end.md b/docs/tutorial-end-to-end.md index e338baf..71670ea 100644 --- a/docs/tutorial-end-to-end.md +++ b/docs/tutorial-end-to-end.md @@ -286,7 +286,7 @@ for creating agents, tools, tracing, evaluation, and red-team scans: https://github.com/Azure-Samples/microsoft-foundry-e2e-agent-observability-workshop/tree/2026-04-aie-europe ``` -### Grant your identity data-plane access to the AI Services account +### Grant data-plane access to your identity and Foundry managed identities Both options above (prompt agent and hosted HTTP agent) eventually drive an `agentops eval run` that calls chat-completions on the AI Services @@ -300,16 +300,32 @@ what causes the eval to fail later with `PermissionDenied` on `Microsoft.CognitiveServices/accounts/OpenAI/deployments/chat/ completions/action`. -Run the assignment once per resource group that hosts a Foundry account -you will evaluate against. Replace ``, -``, and `` with your own values (use -`az ad signed-in-user show --query id -o tsv` to get the object ID): +Run these assignments once per resource group that hosts a Foundry account +you will evaluate against. Cloud evaluations run server-side and some agent +or grader calls may authenticate as Foundry/Azure AI managed identities, not +only as your signed-in user. Assigning the role only to your user can still +leave graders failing with `AuthenticationError`. ```powershell +$subscriptionId = az account show --query id -o tsv +$resourceGroup = "" +$scope = "/subscriptions/$subscriptionId/resourceGroups/$resourceGroup" +$userObjectId = az ad signed-in-user show --query id -o tsv + az role assignment create ` - --assignee ` + --assignee $userObjectId ` --role "Cognitive Services OpenAI User" ` - --scope /subscriptions//resourceGroups/ + --scope $scope + +az resource list -g $resourceGroup ` + --query "[?identity.principalId!=null].identity.principalId" -o tsv | + ForEach-Object { + az role assignment create ` + --assignee-object-id $_ ` + --assignee-principal-type ServicePrincipal ` + --role "Cognitive Services OpenAI User" ` + --scope $scope + } ``` > **Give the assignment a few minutes to propagate.** Data-plane role diff --git a/docs/tutorial-hosted-agent-quickstart.md b/docs/tutorial-hosted-agent-quickstart.md index 188f076..3d9122b 100644 --- a/docs/tutorial-hosted-agent-quickstart.md +++ b/docs/tutorial-hosted-agent-quickstart.md @@ -310,7 +310,7 @@ If the deployed endpoint needs a bearer token: $env:HOSTED_AGENT_TOKEN = "" ``` -### Grant your identity data-plane access to the AI Services account +### Grant data-plane access to your identity and Foundry managed identities The local AI-assisted evaluators that AgentOps runs in step 8 call chat-completions on the AI Services account that backs your Foundry @@ -322,16 +322,32 @@ but `dataActions: []`. Skipping this once causes the eval to fail with `PermissionDenied` on `Microsoft.CognitiveServices/accounts/OpenAI/ deployments/chat/completions/action`. -Run the assignment once per resource group hosting a Foundry account -you will evaluate against (replace ``, -``, and `` with your values; get the -object ID with `az ad signed-in-user show --query id -o tsv`): +Run these assignments once per resource group hosting a Foundry account +you will evaluate against. Local AI-assisted evaluators use your identity, +while Foundry-hosted/server-side eval paths may use Azure AI managed +identities from the same resource group. Assigning only the user can still +leave server-side graders failing with `AuthenticationError`. ```powershell +$subscriptionId = az account show --query id -o tsv +$resourceGroup = "" +$scope = "/subscriptions/$subscriptionId/resourceGroups/$resourceGroup" +$userObjectId = az ad signed-in-user show --query id -o tsv + az role assignment create ` - --assignee ` + --assignee $userObjectId ` --role "Cognitive Services OpenAI User" ` - --scope /subscriptions//resourceGroups/ + --scope $scope + +az resource list -g $resourceGroup ` + --query "[?identity.principalId!=null].identity.principalId" -o tsv | + ForEach-Object { + az role assignment create ` + --assignee-object-id $_ ` + --assignee-principal-type ServicePrincipal ` + --role "Cognitive Services OpenAI User" ` + --scope $scope + } ``` > **Give the assignment a few minutes to propagate.** Data-plane role diff --git a/docs/tutorial-prompt-agent-quickstart.md b/docs/tutorial-prompt-agent-quickstart.md index b2843d7..48a0f8e 100644 --- a/docs/tutorial-prompt-agent-quickstart.md +++ b/docs/tutorial-prompt-agent-quickstart.md @@ -241,7 +241,7 @@ Show me the planned changes and the resulting endpoints before applying. If the skill is not available, use Path A. -### Grant your identity data-plane access to the AI Services account +### Grant data-plane access to your identity and Foundry managed identities Creating a project through the portal only assigns you `Foundry User` **at the project scope**. That role does not cover the OpenAI data-plane actions @@ -257,16 +257,34 @@ Skipping this step is what causes the eval grader to fail later with:: data action `Microsoft.CognitiveServices/accounts/OpenAI/deployments/ chat/completions/action` to perform `POST /openai/deployments/...` -Run the assignment once per resource group that hosts a Foundry account -you will evaluate against. Replace ``, ``, -and `` with your own values (you can get the object ID -with `az ad signed-in-user show --query id -o tsv`): +Run these assignments once per resource group that hosts a Foundry account +you will evaluate against. Cloud evaluations run server-side: the agent call +and graders may authenticate as Foundry/Azure AI managed identities, not only +as your signed-in user. Assigning the role only to your user can still leave +some graders failing with `AuthenticationError`. ```powershell +$subscriptionId = az account show --query id -o tsv +$resourceGroup = "" +$scope = "/subscriptions/$subscriptionId/resourceGroups/$resourceGroup" +$userObjectId = az ad signed-in-user show --query id -o tsv + +# User running local commands / creating cloud evals. az role assignment create ` - --assignee ` + --assignee $userObjectId ` --role "Cognitive Services OpenAI User" ` - --scope /subscriptions//resourceGroups/ + --scope $scope + +# Foundry/Azure AI managed identities used by server-side agent/evaluator calls. +az resource list -g $resourceGroup ` + --query "[?identity.principalId!=null].identity.principalId" -o tsv | + ForEach-Object { + az role assignment create ` + --assignee-object-id $_ ` + --assignee-principal-type ServicePrincipal ` + --role "Cognitive Services OpenAI User" ` + --scope $scope + } ``` Repeat the command with the `travel-agent-dev` resource group if the dev diff --git a/plugins/agentops/skills/agentops-eval/SKILL.md b/plugins/agentops/skills/agentops-eval/SKILL.md index b5b2701..d2d7d43 100644 --- a/plugins/agentops/skills/agentops-eval/SKILL.md +++ b/plugins/agentops/skills/agentops-eval/SKILL.md @@ -41,8 +41,12 @@ PermissionDenied … lacks the required data action 'Microsoft.CognitiveServices/accounts/OpenAI/deployments/chat/completions/action' ``` -Run this preflight before Step 1 - it is idempotent (Azure returns -`RoleAssignmentExists` if already granted) and takes ~5 seconds: +Run this preflight before Step 1. It must grant the role to the signed-in +user **and** to the Foundry/Azure AI managed identities in the resource +group. Cloud evaluations run server-side and some graders authenticate as +those managed identities, so assigning only the user can still produce +intermittent `AuthenticationError` grader failures. The commands are +idempotent (`RoleAssignmentExists` means the role was already granted): ```bash # 1. Resolve the AI Services account from agentops.yaml / .azure//.env @@ -55,11 +59,23 @@ SUB_ID=$(az account show --query id -o tsv) RG=$(az cognitiveservices account list --subscription "$SUB_ID" --query "[?name=='$ACCOUNT_NAME'].resourceGroup | [0]" -o tsv) OBJ_ID=$(az ad signed-in-user show --query id -o tsv) -# 3. Grant data-plane access at the RG scope (covers sandbox + future evals) +# 3. Grant the user data-plane access at RG scope. az role assignment create \ --assignee "$OBJ_ID" \ --role "Cognitive Services OpenAI User" \ --scope "/subscriptions/$SUB_ID/resourceGroups/$RG" + +# 4. Grant the same data-plane role to Foundry/Azure AI managed identities. +az resource list -g "$RG" \ + --query "[?identity.principalId!=null].identity.principalId" -o tsv | +while read -r PRINCIPAL_ID; do + [ -z "$PRINCIPAL_ID" ] && continue + az role assignment create \ + --assignee-object-id "$PRINCIPAL_ID" \ + --assignee-principal-type ServicePrincipal \ + --role "Cognitive Services OpenAI User" \ + --scope "/subscriptions/$SUB_ID/resourceGroups/$RG" +done ``` PowerShell equivalent: replace `$(...)` with the PowerShell variable diff --git a/src/agentops/templates/skills/agentops-eval/SKILL.md b/src/agentops/templates/skills/agentops-eval/SKILL.md index b5b2701..d2d7d43 100644 --- a/src/agentops/templates/skills/agentops-eval/SKILL.md +++ b/src/agentops/templates/skills/agentops-eval/SKILL.md @@ -41,8 +41,12 @@ PermissionDenied … lacks the required data action 'Microsoft.CognitiveServices/accounts/OpenAI/deployments/chat/completions/action' ``` -Run this preflight before Step 1 - it is idempotent (Azure returns -`RoleAssignmentExists` if already granted) and takes ~5 seconds: +Run this preflight before Step 1. It must grant the role to the signed-in +user **and** to the Foundry/Azure AI managed identities in the resource +group. Cloud evaluations run server-side and some graders authenticate as +those managed identities, so assigning only the user can still produce +intermittent `AuthenticationError` grader failures. The commands are +idempotent (`RoleAssignmentExists` means the role was already granted): ```bash # 1. Resolve the AI Services account from agentops.yaml / .azure//.env @@ -55,11 +59,23 @@ SUB_ID=$(az account show --query id -o tsv) RG=$(az cognitiveservices account list --subscription "$SUB_ID" --query "[?name=='$ACCOUNT_NAME'].resourceGroup | [0]" -o tsv) OBJ_ID=$(az ad signed-in-user show --query id -o tsv) -# 3. Grant data-plane access at the RG scope (covers sandbox + future evals) +# 3. Grant the user data-plane access at RG scope. az role assignment create \ --assignee "$OBJ_ID" \ --role "Cognitive Services OpenAI User" \ --scope "/subscriptions/$SUB_ID/resourceGroups/$RG" + +# 4. Grant the same data-plane role to Foundry/Azure AI managed identities. +az resource list -g "$RG" \ + --query "[?identity.principalId!=null].identity.principalId" -o tsv | +while read -r PRINCIPAL_ID; do + [ -z "$PRINCIPAL_ID" ] && continue + az role assignment create \ + --assignee-object-id "$PRINCIPAL_ID" \ + --assignee-principal-type ServicePrincipal \ + --role "Cognitive Services OpenAI User" \ + --scope "/subscriptions/$SUB_ID/resourceGroups/$RG" +done ``` PowerShell equivalent: replace `$(...)` with the PowerShell variable From af7a068eb392121caa37ec600a99071063d50503 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 1 Jun 2026 20:38:39 +0000 Subject: [PATCH 2/2] chore: prepare release 0.3.7 --- .claude-plugin/marketplace.json | 2 +- .github/plugin/marketplace.json | 2 +- CHANGELOG.md | 2 ++ plugins/agentops/package.json | 2 +- plugins/agentops/plugin.json | 2 +- 5 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index ae7fec2..8cb848a 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -13,7 +13,7 @@ "name": "agentops-accelerator", "source": "../../plugins/agentops", "description": "Copilot agent skills for running standardized evaluation workflows with AgentOps Toolkit and Microsoft Foundry agents.", - "version": "0.3.6", + "version": "0.3.7", "keywords": [ "agentops", "evaluation", diff --git a/.github/plugin/marketplace.json b/.github/plugin/marketplace.json index ae7fec2..8cb848a 100644 --- a/.github/plugin/marketplace.json +++ b/.github/plugin/marketplace.json @@ -13,7 +13,7 @@ "name": "agentops-accelerator", "source": "../../plugins/agentops", "description": "Copilot agent skills for running standardized evaluation workflows with AgentOps Toolkit and Microsoft Foundry agents.", - "version": "0.3.6", + "version": "0.3.7", "keywords": [ "agentops", "evaluation", diff --git a/CHANGELOG.md b/CHANGELOG.md index 990702e..85d2b25 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ This format follows [Keep a Changelog](https://keepachangelog.com/) and adheres ## [Unreleased] +## [0.3.7] - 2026-06-01 + ### Fixed - **RBAC preflight now covers Foundry/Azure AI managed identities, not only the signed-in user.** Cloud evaluations run server-side and some agent or diff --git a/plugins/agentops/package.json b/plugins/agentops/package.json index 9706810..94cd453 100644 --- a/plugins/agentops/package.json +++ b/plugins/agentops/package.json @@ -2,7 +2,7 @@ "name": "agentops-accelerator", "displayName": "AgentOps Accelerator — Skills for GitHub Copilot", "description": "Copilot agent skills for running standardized evaluation workflows with AgentOps Accelerator and Microsoft Foundry agents.", - "version": "0.3.6", + "version": "0.3.7", "publisher": "AgentOpsAccelerator", "icon": "icon.png", "license": "MIT", diff --git a/plugins/agentops/plugin.json b/plugins/agentops/plugin.json index 59bb9fa..9654852 100644 --- a/plugins/agentops/plugin.json +++ b/plugins/agentops/plugin.json @@ -1,7 +1,7 @@ { "name": "agentops-accelerator", "description": "Copilot agent skills for running standardized evaluation workflows with AgentOps Accelerator and Microsoft Foundry agents.", - "version": "0.3.6", + "version": "0.3.7", "author": { "name": "AgentOps Accelerator", "url": "https://github.com/Azure/agentops"