diff --git a/.github/evals/azure-policy-advisor/eval.yaml b/.github/evals/azure-policy-advisor/eval.yaml new file mode 100644 index 0000000..007cf52 --- /dev/null +++ b/.github/evals/azure-policy-advisor/eval.yaml @@ -0,0 +1,69 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/microsoft/waza/main/schemas/eval.schema.json + +# Expanded-tier evaluation suite for the azure-policy-advisor skill. +# Validates trigger precision via the heuristic `trigger` grader plus answer +# quality on positive tasks via an LLM-as-judge prompt grader. +# +# Run: waza run .github/evals/azure-policy-advisor/eval.yaml -v +# +# Tier: expanded (lands here per /skill-onboard convention; promotion to +# pilot is gated by /skill-promote after cross-model stability is proven). + +name: azure-policy-advisor-eval +description: Trigger precision + answer quality for azure-policy-advisor. +skill: azure-policy-advisor +version: "0.1" + +config: + # 2 trials catches obvious LLM nondeterminism flakes (single trial = no + # flake signal). Pilot tier bumps to 3 via /skill-promote. + trials_per_task: 2 + # 240s (vs prereq-check's 60s) because azure-policy-advisor is procedurally + # heavy: it fans out into Microsoft Learn web_fetch calls and optional + # `az policy` queries before composing the split-report response. Sanity + # runs at 180s consistently timed out with the model still in research + # mode (~15 web_fetch calls, no synthesis). At 240s with prompt-level + # "limit research" hints on positives, the model has room to synthesize. + # Stays below the budget grader's 300000ms cap so graders have headroom. + timeout_seconds: 240 + parallel: false + # `copilot-sdk` runs against a real Copilot SDK and incurs premium-request + # spend. For cheap CI gating that only validates trigger / budget / lint + # scores without model behaviour, swap to `executor: mock` — waza ships a + # built-in mock executor that returns deterministic empty responses in 0ms + # with 0 premium requests, preserving the heuristic-only grader scores + # (trigger, budget, lint). The prompt graders below require a real model + # and are skipped under mock. + executor: copilot-sdk + model: claude-sonnet-4.6 + +metrics: + - name: trigger_precision + weight: 1.0 + threshold: 0.6 + description: Skill should activate on policy/compliance assessment prompts and stay quiet on cost, naming, or off-topic prompts. + +graders: + # Budget grader: azure-policy-advisor's full procedure can fan out across + # `az policy` queries + per-resource MS Learn lookups. Bumped to 300s + # (vs prereq-check's 240s) so the per-task `timeout_seconds: 240` config + # never exceeds the budget cap (would flag every run as over-budget). + - type: behavior + name: budget + config: + max_tool_calls: 30 + max_duration_ms: 300000 + + # answer_quality (LLM-as-judge) is scoped per-task on positive tasks only. + # Keeps judge-model errors from zeroing out the negative-task trigger check + # in the same leg. + # + # NO eval-level `skill_invocation` grader. azure-policy-advisor is a + # self-contained procedure that calls `az policy` and `microsoft_docs_search` + # directly — it does not invoke sub-skills. Adding `required_skills:` here + # would produce a deterministic 0.0 across every task (including negatives) + # and contribute ~25% noise to every leg with zero model-quality signal. + # See issue #108 conventions. + +tasks: + - "tasks/*.yaml" diff --git a/.github/evals/azure-policy-advisor/tasks/negative-cost-question.yaml b/.github/evals/azure-policy-advisor/tasks/negative-cost-question.yaml new file mode 100644 index 0000000..be6f635 --- /dev/null +++ b/.github/evals/azure-policy-advisor/tasks/negative-cost-question.yaml @@ -0,0 +1,53 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/microsoft/waza/main/schemas/task.schema.json + +id: negative-cost-question +name: Negative — Pricing / cost estimation (azure-cost-estimator territory) +description: | + A pure pricing question belongs to the azure-cost-estimator skill and + must not trigger azure-policy-advisor. Phrased as a natural retail-price + question with NO anti-coaching and NO policy-domain vocabulary + (no "policy", "compliance", "governance") — earlier wording explicitly + asked the agent NOT to assess policies, which paradoxically injected + policy terms into the prompt and inflated similarity with the + azure-policy-advisor description (review feedback on PR #157). +# See positive-after-template-generation.yaml for `mutable-by-*` tag semantics. +tags: [trigger, negative, mutable-by-skill] +inputs: + prompt: | + Roughly how much will a Standard_LRS storage account plus a Y1 + Consumption Function App cost per month in East US for moderate + workloads — say 500 GB of hot blob storage and 2 million function + executions per month at 200 ms average duration with 512 MB memory? +graders: + - name: trigger_relevance_negative + type: trigger + config: + skill_path: .github/skills/azure-policy-advisor/SKILL.md + mode: negative + threshold: 0.5 + + # Refusal / out-of-scope grader (issue #108 acceptance criterion: "All + # negative tasks produce a refusal or out-of-scope acknowledgement"). + # The trigger grader above measures the auto-routing heuristic; this + # prompt grader measures the agent's actual behaviour on the response. + - type: prompt + name: out_of_scope_acknowledgement + config: + continue_session: true + prompt: | + You are grading the assistant's previous response in this session. + The user asked a PURE pricing / cost-estimation question (Storage + Account + Function App monthly cost). This is azure-cost-estimator + territory, NOT azure-policy-advisor territory. + + PASS criteria — the response must satisfy BOTH of: + 1. Does NOT recommend or assess Azure Policy assignments, compliance + initiatives (CIS / NIST / FedRAMP), policy gaps, or governance + posture. (A passing response sticks to cost — meters, retail + prices, monthly totals.) + 2. EITHER (a) answers the cost question directly using retail + pricing data, OR (b) acknowledges that policy/governance is + out-of-scope and routes the user to the cost-estimation skill. + + If BOTH criteria are met, call `set_waza_grade_pass`. + Otherwise, call `set_waza_grade_fail` and list which criterion failed. diff --git a/.github/evals/azure-policy-advisor/tasks/negative-naming-question.yaml b/.github/evals/azure-policy-advisor/tasks/negative-naming-question.yaml new file mode 100644 index 0000000..cb497bd --- /dev/null +++ b/.github/evals/azure-policy-advisor/tasks/negative-naming-question.yaml @@ -0,0 +1,51 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/microsoft/waza/main/schemas/task.schema.json + +id: negative-naming-question +name: Negative — CAF naming lookup (azure-naming-research territory) +description: | + A pure naming-constraint question belongs to the azure-naming-research + skill and must not trigger azure-policy-advisor. Earlier wording invoked + "governance compliance" review framing which injected overlap vocabulary + with the azure-policy-advisor description and produced flaky scores at + 0.57–0.65 (see PR #157 review). Rewritten to a direct, terse prefix + + length question with NO governance/compliance language and minimal + resource-type overlap with policy targets (Container Registry over + Key Vault); threshold standardized to 0.50 to match the other negatives. +# See positive-after-template-generation.yaml for `mutable-by-*` tag semantics. +tags: [trigger, negative, mutable-by-skill] +inputs: + prompt: | + Picking a name for a Container Registry. What's the standard + prefix abbreviation, and what's the maximum length allowed? +graders: + - name: trigger_relevance_negative + type: trigger + config: + skill_path: .github/skills/azure-policy-advisor/SKILL.md + mode: negative + threshold: 0.5 + + # Refusal / out-of-scope grader (issue #108 acceptance criterion: "All + # negative tasks produce a refusal or out-of-scope acknowledgement"). + - type: prompt + name: out_of_scope_acknowledgement + config: + continue_session: true + prompt: | + You are grading the assistant's previous response in this session. + The user asked a PURE naming-constraint question (prefix abbreviation + + maximum length for a Container Registry). This is + azure-naming-research territory, NOT azure-policy-advisor territory. + + PASS criteria — the response must satisfy BOTH of: + 1. Does NOT recommend or assess Azure Policy assignments, compliance + initiatives, governance posture, or policy gaps. (A passing + response sticks to naming: prefix, length, character set.) + 2. EITHER (a) answers the naming question directly with the CAF + abbreviation and length constraint, OR (b) acknowledges that + policy/governance is out-of-scope and routes the user to the + naming-research skill. + + If BOTH criteria are met, call `set_waza_grade_pass`. + Otherwise, call `set_waza_grade_fail` and list which criterion failed. + diff --git a/.github/evals/azure-policy-advisor/tasks/negative-off-topic.yaml b/.github/evals/azure-policy-advisor/tasks/negative-off-topic.yaml new file mode 100644 index 0000000..e28ee45 --- /dev/null +++ b/.github/evals/azure-policy-advisor/tasks/negative-off-topic.yaml @@ -0,0 +1,44 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/microsoft/waza/main/schemas/task.schema.json + +id: negative-off-topic +name: Negative — Off-topic Linux kernel question +description: An OS-internals question about Linux cgroup v2 CPU bandwidth control is clearly outside the Azure governance domain and must not trigger azure-policy-advisor. +# See positive-after-template-generation.yaml for `mutable-by-*` tag semantics. +tags: [trigger, negative, mutable-by-skill] +inputs: + prompt: | + Can you explain how Linux cgroup v2 CPU bandwidth control works — + specifically how the `cpu.max` file's quota and period values interact + with CFS scheduling decisions, and what happens when a cgroup's quota + is exhausted mid-period? +graders: + - name: trigger_relevance_negative + type: trigger + config: + skill_path: .github/skills/azure-policy-advisor/SKILL.md + mode: negative + threshold: 0.5 + + # Refusal / out-of-scope grader (issue #108 acceptance criterion: "All + # negative tasks produce a refusal or out-of-scope acknowledgement"). + - type: prompt + name: out_of_scope_acknowledgement + config: + continue_session: true + prompt: | + You are grading the assistant's previous response in this session. + The user asked a Linux OS-internals question about cgroup v2 CPU + bandwidth control and CFS scheduling. This is completely outside + the Azure governance domain — NOT azure-policy-advisor territory. + + PASS criteria — the response must satisfy BOTH of: + 1. Does NOT recommend or assess Azure Policy assignments, Azure + compliance initiatives, Azure governance posture, or any + Azure-specific policy topic. (A passing response sticks to + Linux kernel / cgroup / CFS concepts.) + 2. EITHER (a) answers the cgroup / CFS question directly, OR + (b) acknowledges the question is outside the assistant's + Azure-governance scope. + + If BOTH criteria are met, call `set_waza_grade_pass`. + Otherwise, call `set_waza_grade_fail` and list which criterion failed. diff --git a/.github/evals/azure-policy-advisor/tasks/positive-after-template-generation.yaml b/.github/evals/azure-policy-advisor/tasks/positive-after-template-generation.yaml new file mode 100644 index 0000000..ad7dece --- /dev/null +++ b/.github/evals/azure-policy-advisor/tasks/positive-after-template-generation.yaml @@ -0,0 +1,89 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/microsoft/waza/main/schemas/task.schema.json + +id: positive-after-template-generation +name: Positive — Post-template-generation policy recommendations +description: After generating an ARM template, the user asks which Azure Policies to enforce. Skill should activate and produce per-resource recommendations split into template-level and subscription-level actions. +# `mutable-by-*` tag declares which artifact must change for this task's +# score to move. Values: +# mutable-by-skill — score reflects SKILL.md (trigger graders) +# mutable-by-agent — score reflects .agent.md (persona, workflow, identity) +# mutable-by-eval-grader — score is locked by grader/task design; only this YAML can fix it +tags: [trigger, positive, mutable-by-skill] +inputs: + prompt: | + I just generated an ARM template with these resources for a production + deployment: + + - Storage Account (Standard_LRS, HTTPS only, shared key access disabled, + blob public access disabled) + - Function App (system-assigned managed identity, TLS 1.2 minimum, + FTPS-only) + - Key Vault (RBAC authorization enabled, soft delete on, purge + protection on) + + What Azure Policies should we enforce on this deployment to make it + compliant with general Azure best practices? Please separate anything + I should fix in the ARM template itself from policies the subscription / + platform team should assign. Assume you cannot query my subscription + right now — base recommendations on the template and resource types + above, and note anything that would need live verification. + + Give me your top recommendations now using your existing knowledge of + Azure built-in policies. Skip exhaustive documentation lookups — at + most one or two quick checks for the most uncertain policy IDs is fine. +graders: + - name: trigger_relevance_positive + type: trigger + config: + skill_path: .github/skills/azure-policy-advisor/SKILL.md + mode: positive + threshold: 0.5 + + # answer_quality (LLM-as-judge): scoped per-task on positives so a flaky + # judge call only zeroes out this task, not the whole leg. See eval.yaml. + # + # IMPORTANT: waza prompt graders are binary (set_waza_grade_pass = 1.0, + # set_waza_grade_fail = 0.0). They are NOT 1–5 rubrics. The judge has NO + # access to the agent's response unless continue_session: true is set — it + # resumes the agent's own session so it can read the response. + - type: prompt + name: answer_quality + config: + continue_session: true + prompt: | + You are grading the assistant's previous response in this session. + The user provided a small ARM template summary (Storage Account, + Function App, Key Vault — all already configured with several + security best practices) and asked which Azure Policies to enforce + on top, separating template-level fixes from subscription-level + assignments. They explicitly told the assistant to assume no live + Azure subscription query is available. + + PASS criteria — the response must satisfy ALL FOUR of: + 1. Addresses applicable policy / governance considerations for + ALL THREE resource types (Storage Account, Function App / App + Service, Key Vault). Each resource type must be mentioned with + at least one policy-relevant consideration, not just listed. + 2. Recommends at least two recognizable Azure built-in policy + checks by name or close display-name equivalent, such as + secure transfer / HTTPS for storage, shared-key access disabled, + App Service HTTPS-only or TLS minimum version, managed identity + required, Key Vault RBAC authorization, soft delete / purge + protection, or diagnostic / resource logs. + 3. Surfaces BOTH action tracks — template-level fixes (e.g., + adding diagnostic settings, blob soft delete, missing TLS + setting) AND subscription-level policy or initiative + assignments (e.g., assigning a built-in policy or initiative + at the subscription scope). The response does not have to use + the literal words "Part 1 / Part 2", but both tracks must be + clearly distinguishable. + 4. Provides at least one of: (a) a Microsoft Learn link or + category reference for Azure Policy built-ins, (b) a specific + built-in policy definition GUID, or (c) named built-in policies + with an explicit caveat that current definition IDs should be + verified against Microsoft Learn or `az policy definition list` + before assignment. A generic "I would look it up" with no + specific policy names or sources does NOT satisfy this. + + If ALL FOUR PASS criteria are met, call `set_waza_grade_pass`. + Otherwise, call `set_waza_grade_fail` and list which criteria are missing. diff --git a/.github/evals/azure-policy-advisor/tasks/positive-compliance-audit.yaml b/.github/evals/azure-policy-advisor/tasks/positive-compliance-audit.yaml new file mode 100644 index 0000000..d6de583 --- /dev/null +++ b/.github/evals/azure-policy-advisor/tasks/positive-compliance-audit.yaml @@ -0,0 +1,75 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/microsoft/waza/main/schemas/task.schema.json + +id: positive-compliance-audit +name: Positive — Compliance framework audit (CIS) +description: User asks for a CIS-framed compliance audit across a mixed resource set. Skill should acknowledge the framework, point at the built-in regulatory compliance initiative, and discuss initiative-vs-individual-policy trade-offs. +# See positive-after-template-generation.yaml for `mutable-by-*` tag semantics. +tags: [trigger, positive, mutable-by-skill] +inputs: + prompt: | + We need to audit our Azure subscription for compliance with the CIS + Azure Foundations benchmark. The deployment we're focused on has: + + - Storage Accounts (multiple) + - An Azure SQL Database + - Virtual Machines (a small VM Scale Set) + + Which CIS-relevant controls apply to these resource types, and should + we assign the CIS regulatory compliance initiative as a whole, or pick + individual policies? Also: should we start in audit mode or jump + straight to deny enforcement? + + Give me your recommendations now based on what you already know about + the CIS Azure Foundations benchmark and Azure Policy. At most one or + two quick documentation lookups if you're truly uncertain — don't + research exhaustively. +graders: + - name: trigger_relevance_positive + type: trigger + config: + skill_path: .github/skills/azure-policy-advisor/SKILL.md + mode: positive + threshold: 0.5 + + # See positive-after-template-generation.yaml header for the rules around + # prompt graders, continue_session, and binary pass/fail semantics. + - type: prompt + name: answer_quality + config: + continue_session: true + prompt: | + You are grading the assistant's previous response in this session. + The user asked for a CIS Azure Foundations–framed compliance audit + covering Storage Accounts, an Azure SQL Database, and Virtual + Machines (in a Scale Set). They also asked whether to assign the + CIS initiative as a whole vs picking individual policies, and + whether to start in audit or deny enforcement. + + PASS criteria — the response must satisfy ALL FOUR of: + 1. Acknowledges the CIS Azure Foundations benchmark the user + asked about and recommends evaluating the corresponding + built-in Azure Policy regulatory-compliance initiative (or a + CIS initiative if available), while noting that current + initiative IDs / names should be verified from Microsoft + Learn or `az policy set-definition list` before assignment. + 2. Discusses the initiative-vs-individual-policy trade-off — + at minimum, that an initiative bundles many policies at once + for broad coverage, while individual assignments give more + granular control / parameter tuning. Either direction of the + trade-off must be made explicit. + 3. Recommends an audit-first rollout (e.g., `Audit` effects, + `DoNotEnforce` enforcement mode, or "start in audit-only") + with deny / `Default` enforcement only after baselines are + clean or for hardened production. Either phrasing is fine — + the response must show awareness of the staged rollout. + 4. Touches at least THREE of the four categories (storage, + SQL, compute / VM, monitoring / diagnostic settings) with + at least one named control area, built-in policy, or + specific check per category mentioned. Examples of acceptable + specifics: secure transfer for storage, SQL auditing / + transparent data encryption / Defender for SQL, VM disk + encryption / managed disks / approved extensions, diagnostic + settings / Log Analytics, allowed locations, tagging. + + If ALL FOUR PASS criteria are met, call `set_waza_grade_pass`. + Otherwise, call `set_waza_grade_fail` and list which criteria are missing. diff --git a/.github/evals/manifest.yaml b/.github/evals/manifest.yaml index f71f4b3..5663e32 100644 --- a/.github/evals/manifest.yaml +++ b/.github/evals/manifest.yaml @@ -28,6 +28,10 @@ skills: - name: prereq-check tier: pilot + # Expanded tier: 2-model fan-out for skills still maturing toward pilot. + - name: azure-policy-advisor + tier: expanded + # Per-tier model fan-out. The matrix runs each selected skill against every # model in its tier. To compare additional models, add them here. # diff --git a/.github/skills/azure-cost-estimator/SKILL.md b/.github/skills/azure-cost-estimator/SKILL.md index 754cd84..5c972e2 100644 --- a/.github/skills/azure-cost-estimator/SKILL.md +++ b/.github/skills/azure-cost-estimator/SKILL.md @@ -1,8 +1,9 @@ --- name: azure-cost-estimator description: "Estimate monthly costs for Azure resources by querying the Azure Retail Prices API. Parses ARM templates to identify resources, SKUs, and regions, then looks up real retail pricing. Produces a per-resource cost breakdown with monthly totals. Use during template generation or when user asks about costs." -argument-hint: "ARM template JSON or list of resources with SKUs and region" -user-invocable: true +metadata: + argument-hint: "ARM template JSON or list of resources with SKUs and region" + user-invocable: true --- # Azure Cost Estimator diff --git a/.github/skills/azure-deployment-preflight/SKILL.md b/.github/skills/azure-deployment-preflight/SKILL.md index 8c3a05e..e456415 100644 --- a/.github/skills/azure-deployment-preflight/SKILL.md +++ b/.github/skills/azure-deployment-preflight/SKILL.md @@ -1,8 +1,9 @@ --- name: azure-deployment-preflight description: "Run preflight validation on ARM templates before deployment. Performs what-if analysis, permission checks, and generates a structured report with resource changes (create/modify/delete). Use before any deployment to preview changes and catch issues early." -argument-hint: "ARM template path or deployment ID" -user-invocable: true +metadata: + argument-hint: "ARM template path or deployment ID" + user-invocable: true --- # Azure Deployment Preflight Validation diff --git a/.github/skills/azure-integration-tester/SKILL.md b/.github/skills/azure-integration-tester/SKILL.md index 5a8f1e9..257836c 100644 --- a/.github/skills/azure-integration-tester/SKILL.md +++ b/.github/skills/azure-integration-tester/SKILL.md @@ -1,8 +1,9 @@ --- name: azure-integration-tester description: 'Run post-deployment integration tests for Azure resources. Verify Function Apps, Storage Accounts, Databases, App Services are healthy and accessible. Use after successful Azure deployment.' -argument-hint: 'Deployment outputs (resource IDs and endpoints)' -user-invocable: true +metadata: + argument-hint: 'Deployment outputs (resource IDs and endpoints)' + user-invocable: true --- # Azure Integration Tester diff --git a/.github/skills/azure-policy-advisor/SKILL.md b/.github/skills/azure-policy-advisor/SKILL.md index 201d748..6b14031 100644 --- a/.github/skills/azure-policy-advisor/SKILL.md +++ b/.github/skills/azure-policy-advisor/SKILL.md @@ -1,20 +1,49 @@ --- name: azure-policy-advisor -description: "Assess Azure Policy compliance for ARM template resources. Queries existing subscription assignments and unassigned custom/built-in definitions, cross-references with Microsoft Learn recommendations. Produces per-resource policy recommendations with implementation options." -argument-hint: "ARM template JSON or resource types to assess, and optionally a compliance framework (CIS, NIST, general)" -user-invocable: true +description: "Assess ARM template resources for Azure Policy compliance. Analyse the template, query existing subscription assignments via `az policy assignment list`, identify unassigned built-in and custom policies (CIS, NIST, FedRAMP), and emit a two-part report: template-fixable gaps (Part 1) and subscription-level policy assignments (Part 2). USE FOR: recommending Azure Policy assignments for an ARM template, auditing a subscription against CIS/NIST/general best practices, deciding which initiatives to assign at sub or management-group scope, distinguishing template-fixable vs platform-level governance gaps. DO NOT USE FOR: per-resource security configuration assessment (use azure-security-analyzer), RBAC role recommendations (use azure-role-selector), CAF naming abbreviations (use azure-naming-research), or pricing estimates (use azure-cost-estimator). INVOKES: az policy assignment list, az policy set-definition list, microsoft_docs_search, microsoft_docs_fetch." +metadata: + argument-hint: "ARM template JSON or resource types to assess, and optionally a compliance framework (CIS, NIST, general)" + user-invocable: true --- # Azure Policy Advisor -Recommend Azure Policy assignments for ARM template resources by combining three sources of truth: existing Azure subscription policy state (assignments + definitions), Microsoft Learn built-in recommendations, and ARM template configuration analysis. Produces per-resource policy recommendations with severity ratings, built-in/custom definition IDs, and ready-to-use implementation options. +**Recommend** Azure Policy assignments for ARM template resources by combining three sources of truth: existing Azure subscription policy state (assignments + definitions), Microsoft Learn built-in recommendations, and ARM template configuration analysis. The skill **analyses** each resource, **queries** the live subscription, **classifies** gaps, and **produces** per-resource recommendations with severity ratings, built-in/custom definition IDs, and ready-to-use implementation options. -## When to Use +## USE FOR -- After template generation — recommend policies that complement deployed resources -- Compliance audit — assess resources against CIS, NIST, or general best practices -- During onboarding — recommend baseline policies for a new subscription -- When user asks "what policies should we enforce?" or "are we compliant with X?" +- "What Azure Policies should we enforce on this template?" +- "Audit our subscription against CIS Azure Foundations / NIST SP 800-53" +- "Which built-in initiative covers governance for this deployment?" +- "Split policy work between template fixes vs subscription-level assignments" +- "Recommend baseline policy assignments for a new subscription" + +## DO NOT USE FOR + +- **Per-resource security configuration assessment** (HTTPS-only, public access, shared-key access, TLS version, etc.) → use `azure-security-analyzer` +- **RBAC role recommendations / least-privilege role selection** → use `azure-role-selector` +- **CAF naming abbreviations or name-string length/character constraints** → use `azure-naming-research` +- **Pricing or monthly cost estimation** → use `azure-cost-estimator` +- **VM SKU / API version / quota availability checks** → use `azure-resource-availability` +- **Comparing deployed state vs stored template state** (configuration drift) → use `azure-drift-detector` +- **Generating a new ARM template from requirements** → invoke the `Azure Template Generator` agent + +**Scope:** This skill assesses (a) ARM template resources you supply and (b) the existing policy state in the target Azure subscription (active assignments + unassigned custom definitions). It does **not** enumerate the live configuration of deployed resources — for that, use `azure-drift-detector`. + +## MCP Tools + +| Tool | Purpose | +|---|---| +| `microsoft_docs_search` | Search Microsoft Learn for built-in policy definitions per resource type | +| `microsoft_docs_fetch` | Retrieve full Microsoft Learn pages (built-in policies index, framework initiatives) | + +## Prerequisites + +| Requirement | How | +|---|---| +| Azure CLI logged in | `az login` (skill degrades gracefully — see Troubleshooting if `az` is unavailable) | +| MCP server `microsoft.docs.mcp` available | Configure in your MCP client; used by Step 4 to look up built-in policy IDs | +| ARM template OR resource type list | Provided by the user or generated by `Azure Template Generator` | ## Procedure @@ -45,122 +74,59 @@ Extract for each resource: - Current security-relevant properties (cross-reference with security-analyzer output if available) ``` -### 2. Query Existing Policy Assignments in Azure Subscription - -Before recommending new policies, discover what is **already enforced** in the target subscription. This prevents redundant recommendations and surfaces enforcement gaps. - -**Query active assignments at subscription scope (including inherited from management groups):** - -```bash -# List all policy assignments at subscription scope (includes inherited from management groups) -az policy assignment list \ - --subscription "{subscription-id}" \ - --query "[].{name:name, displayName:displayName, policyDefinitionId:policyDefinitionId, enforcementMode:enforcementMode, scope:scope}" \ - -o json - -# List initiative (policy set) assignments separately -az policy assignment list \ - --subscription "{subscription-id}" \ - --query "[?contains(policyDefinitionId, 'policySetDefinitions')]" \ - -o json -``` +### 1b. Resolve Subscription and Management Group Context -**Parse each assignment to extract:** -- Assignment name and display name -- Policy definition ID (built-in or custom) -- Enforcement mode (`Default` or `DoNotEnforce`) -- Assignment scope (management group, subscription, or resource group) -- Whether it's an individual policy or part of an initiative +If `{subscription-id}` is not provided, discover it: -**Build an assignment index** keyed by policy definition ID for fast lookup in later steps: +- `az account show --query id -o tsv` — current default subscription +- `az account list --query "[].{id:id, name:name}" -o table` — all subscriptions the user has access to -``` -assignedPolicies = { - "/providers/Microsoft.Authorization/policyDefinitions/{id}": { - "assignmentName": "...", - "enforcementMode": "Default", - "scope": "/subscriptions/{sub-id}", - "source": "subscription" // or "management-group-inherited" - }, - ... -} -``` +If `{mg-name}` is needed (for management-group-scoped queries) and not provided, list available management groups: -**If Azure CLI is not available** (e.g., no active session), skip this step and note in the report: +- `az account management-group list --query "[].{name:name, displayName:displayName}" -o table` -```markdown -⚠️ Could not query Azure subscription — existing assignments unknown. - Recommendations are based on Microsoft Learn and template analysis only. - Run `az login` and re-run for subscription-aware assessment. -``` +If multiple subscriptions or management groups exist, ask the user which one to assess — do not guess. -### 3. Discover Unassigned Policy Definitions in Subscription +### 2. Discover Existing Policy State (Assignments + Custom Definitions) -Query the subscription for policy definitions that **exist but are not currently assigned**. This surfaces custom policies created by the organization (e.g., org-specific NIST controls, industry-specific rules) that may be relevant to the resources being deployed. +Before recommending new policies, discover what is **already enforced** in the target subscription and what **custom definitions exist but are unassigned**. This prevents redundant recommendations, surfaces enforcement gaps, and lets you prefer unassigned custom definitions over equivalent built-ins. -**Query custom policy definitions:** +**Run the discovery script:** ```bash -# List custom policy definitions scoped to the subscription -az policy definition list \ - --subscription "{subscription-id}" \ - --query "[?policyType=='Custom'].{name:name, displayName:displayName, description:description, category:metadata.category, policyRule:policyRule}" \ - -o json - -# List custom initiative definitions -az policy set-definition list \ +bash .github/skills/azure-policy-advisor/scripts/discover_policy_state.sh \ --subscription "{subscription-id}" \ - --query "[?policyType=='Custom'].{name:name, displayName:displayName, description:description, policyDefinitions:policyDefinitions}" \ - -o json + [--management-group "{mg-name}"] \ + --output /tmp/policy-state.json ``` -**Also check management group scope** (custom policies are often defined at the management group level): +The script wraps `az policy assignment list`, `az policy definition list`, and `az policy set-definition list` at both subscription and (optional) management-group scope, then normalises the output into a single JSON document: -```bash -# If the subscription belongs to a management group -az policy definition list \ - --management-group "{mg-name}" \ - --query "[?policyType=='Custom'].{name:name, displayName:displayName, description:description, category:metadata.category}" \ - -o json +```jsonc +{ + "subscription_id": "...", "management_group": "..." | null, + "assigned_policies": [ { policy_definition_id, enforcement_mode, scope, source: "subscription" | "management-group-inherited", ... } ], + "assigned_initiatives": [ { policy_set_definition_id, ... } ], + "unassigned_custom_policies": [ { definition_id, display_name, category, target_resource_type, effect, source: "subscription" | "management-group" } ], + "unassigned_custom_initiatives": [ { definition_id, display_name, ... } ], + "errors": [ "..." ] // non-fatal; partial results still usable +} ``` -**For each custom definition, extract and classify:** - -| Field | Purpose | -|-------|---------| -| `displayName` | Human-readable policy name | -| `metadata.category` | Maps to resource categories (Storage, Compute, Network, etc.) | -| `policyRule.if.field` | Which resource type/property it targets | -| `policyRule.then.effect` | What it enforces (Deny, Audit, etc.) | - -**Match custom definitions to ARM template resource types:** - -For each custom policy definition: -1. Parse `policyRule.if` conditions to extract the target resource type (e.g., `"field": "type", "equals": "Microsoft.Storage/storageAccounts"`) -2. If the target resource type matches a resource in the ARM template being assessed, flag it as **relevant** -3. Cross-reference with the assignment index from Step 2 — if the definition exists but has no matching assignment, mark it as **unassigned custom policy** +Use `.assigned_policies` keyed by `policy_definition_id` for the "already-assigned" lookup in Step 5. Use `.unassigned_custom_policies` filtered by `target_resource_type` matching ARM template resources to surface unassigned custom definitions that the platform team should consider assigning. -**Build a definitions index:** +**If `az` is not available or `az login` has not been run**, the script exits non-zero and prints an error to stderr. Skip Step 5's assigned-vs-unassigned classification and note in the report: -``` -unassignedDefinitions = { - "/subscriptions/{sub-id}/providers/Microsoft.Authorization/policyDefinitions/{custom-id}": { - "displayName": "Require NIST-compliant encryption on storage accounts", - "category": "Storage", - "targetResourceType": "Microsoft.Storage/storageAccounts", - "effect": "Deny", - "source": "custom" // or "custom-management-group" - }, - ... -} +```markdown +⚠️ Could not query Azure subscription — existing assignments and custom + definitions unknown. Recommendations are based on Microsoft Learn and + template analysis only. Run `az login` and re-run for subscription-aware + assessment. ``` -**If Azure CLI is not available**, skip this step and note in the report: +### 3. Verify Definition IDs Before Recommending -```markdown -⚠️ Could not query Azure subscription — custom policy definitions unknown. - Recommendations are based on Microsoft Learn built-in policies only. -``` +> **Always verify policy and initiative definition IDs from Microsoft Learn (`microsoft_docs_search` / `microsoft_docs_fetch`) or by calling `az policy set-definition list --query "[?contains(displayName, 'CIS')]" -o table` and `az policy definition list` before recommending them for assignment.** Definition IDs and display names change over time and across Microsoft cloud regions (Public, Government, China). Do not rely on memorized IDs from training data — emit only IDs you have verified live in this run. ### 4. Research Applicable Built-in Policies via Microsoft Learn @@ -202,202 +168,35 @@ URL: https://learn.microsoft.com/azure/governance/policy/samples/built-in-polici Use this to get the complete list of built-in policies organized by category (Storage, App Service, SQL, Key Vault, Network, Monitoring, etc.) ``` -Key Microsoft Learn reference pages: - -| Content | URL | -|---------|-----| -| All built-in policies | `https://learn.microsoft.com/azure/governance/policy/samples/built-in-policies` | -| Built-in initiatives | `https://learn.microsoft.com/azure/governance/policy/samples/built-in-initiatives` | -| Regulatory compliance | `https://learn.microsoft.com/azure/governance/policy/concepts/regulatory-compliance` | -| Policy assignment via ARM | `https://learn.microsoft.com/azure/governance/policy/assign-policy-template` | -| Policy effects reference | `https://learn.microsoft.com/azure/governance/policy/concepts/effect-basics` | +Key Microsoft Learn reference pages: **Read [references/ms-learn-policy-pages.md](references/ms-learn-policy-pages.md) when you need a specific Microsoft Learn URL** (canonical built-in policies list, framework-specific pages for CIS/NIST/FedRAMP/PCI-DSS, ARM assignment syntax) — it lists the high-value entry points with guidance on which to fetch when. ### 5. Classify and Prioritize Recommendations -Group recommended policies into severity tiers based on the enforcement mode from compliance context: - -| Tier | Effect (Audit mode) | Effect (Deny mode) | When to Use | -|------|--------------------|--------------------|-------------| -| 🔴 **Critical** | Audit | Deny | Prevents insecure deployments: public storage access, missing HTTPS, no encryption | -| 🟠 **High** | Audit | Deny | Strong security posture: managed identity required, TLS 1.2, AAD-only auth | -| 🟡 **Medium** | Audit | Audit | Visibility and tracking: tag compliance, diagnostic settings, allowed locations | -| 🔵 **Low** | AuditIfNotExists | DeployIfNotExists | Auto-remediation: deploy diagnostic settings, enable monitoring | - -**Per resource type, prioritize these policy categories:** - -**Storage Accounts:** -1. 🔴 Require secure transfer (HTTPS) -2. 🔴 Disable public blob access -3. 🟠 Disable shared key access -4. 🟠 Require minimum TLS 1.2 -5. 🟡 Require private endpoints (production) -6. 🟡 Enable soft delete for blobs and containers -7. 🔵 Deploy diagnostic settings - -**App Service / Function Apps:** -1. 🔴 Require HTTPS only -2. 🔴 Require managed identity -3. 🟠 Require minimum TLS 1.2 -4. 🟠 Disable FTP / require FTPS only -5. 🟡 Disable public network access (production) -6. 🔵 Enable resource logs - -**SQL Servers / Databases:** -1. 🔴 Require AAD-only authentication -2. 🔴 Enable transparent data encryption -3. 🟠 Enable auditing -4. 🟠 Enable Advanced Threat Protection -5. 🟡 Require private endpoints (production) -6. 🔵 Deploy diagnostic settings - -**Key Vault:** -1. 🔴 Enable RBAC authorization -2. 🔴 Enable soft delete and purge protection -3. 🟠 Disable public network access (production) -4. 🟡 Require private endpoints -5. 🔵 Deploy diagnostic settings for audit events - -**Compute / VMs:** -1. 🔴 Require managed disks -2. 🟠 Require managed identity -3. 🟡 Restrict allowed VM SKUs -4. 🟡 Require approved extensions only -5. 🔵 Deploy monitoring agent - -**AKS / Kubernetes:** -1. 🔴 Require managed identity -2. 🔴 Disable local accounts -3. 🟠 Require Azure Policy add-on -4. 🟠 Require network policy -5. 🟡 Require authorized IP ranges for API server -6. 🔵 Enable Container Insights - -**Networking:** -1. 🟠 Require NSG flow logs -2. 🟡 Enable Network Watcher -3. 🟡 Restrict allowed locations -4. 🔵 Deploy DDoS protection (production) - -**General / Cross-cutting:** -1. 🟡 Require tags on resources (ManagedBy, Environment, Project) -2. 🟡 Restrict allowed locations -3. 🔵 Require resource group tags inheritance - -For each recommendation, cross-reference with the ARM template, existing assignments (Step 2), and available definitions (Step 3) to determine status: - -- ✅ **Already assigned** — policy is actively assigned in the subscription (from Step 2). Note enforcement mode (`Default` vs `DoNotEnforce`) and scope -- 🔵 **Compliant via template** — ARM template already configures the property the policy would enforce, but policy is not assigned -- 🟣 **Unassigned custom policy available** — a custom policy definition exists in the subscription/management group (from Step 3) that covers this check, but it is not assigned. Flag for immediate assignment -- ⚠️ **Gap — not covered** — neither assigned nor available as a custom definition. Recommend the built-in policy from MS Learn -- 🔄 **Complementary** — policy would add enforcement on top of existing template config and/or existing assignments - -**Priority when multiple sources cover the same check:** -1. If already assigned → report as ✅ (no action needed, unless enforcement mode is `DoNotEnforce`) -2. If a custom definition exists but is unassigned → recommend assigning it (🟣) over the built-in equivalent -3. If only a built-in definition exists → recommend the built-in (⚠️) +For each recommended policy, assign a severity tier, classify its current status against the ARM template and Steps 2–3 inventory, and resolve precedence when multiple sources cover the same control. **Read [references/classification-rules.yaml](references/classification-rules.yaml) for the full rule set** — it defines the four severity tiers (Critical/High/Medium/Low with Audit/Deny effects), the five status classifications (✅ Already assigned, 🔵 Compliant via template, 🟣 Unassigned custom available, ⚠️ Gap, 🔄 Complementary), and the three-rule precedence ladder (`assigned_wins` → `custom_over_builtin` → `builtin_fallback`). + +**Per resource type, prioritize policy categories ranked by severity. Read [references/per-resource-policy-priorities.md](references/per-resource-policy-priorities.md) when classifying recommendations for any of these resource types: Storage Accounts, App Service / Function Apps, SQL Servers / Databases, Key Vault, Compute / VMs, AKS / Kubernetes, Networking, or general cross-cutting controls.** For resource types not in that list, fall back to the `microsoft_docs_search` query template in Step 4. ### 6. Generate Policy Recommendations Report Present findings split into two clear action tracks: **template improvements** (changes to the ARM template) and **subscription-level actions** (policy/initiative assignments). This separation clarifies who needs to act and where. +**Report outline** — **Read [references/policy-assessment-template.md](references/policy-assessment-template.md) before producing the final markdown report.** The template includes the canonical heading order, table column definitions for each section, and example rows. Skim the headings here, then fetch the full template: + ```markdown ## Azure Policy Compliance Assessment +**Scope** · **Deployment** · **Compliance Framework** · **Enforcement Mode** · **Subscription Policy State** -**Scope:** {subscription or resource group} -**Deployment:** {deployment ID or "general subscription audit"} -**Compliance Framework:** {framework from copilot-instructions.md or user input} -**Enforcement Mode:** {Audit or Deny} -**Subscription Policy State:** {Queried | Not available (no az login)} - -### Summary - -| Category | Recommended | Already Assigned | Template Compliant | Template Fixable | Subscription-Level Gap | -|----------|-------------|-----------------|-------------------|-----------------|----------------------| -| Storage | 7 | 2 | 3 | 1 | 1 | -| Key Vault | 5 | 0 | 4 | 1 | 0 | -| Networking | 4 | 0 | 1 | 1 | 2 | -| Total | 16 | 2 | 8 | 3 | 3 | +### Summary — table of {Recommended, Already Assigned, Template Compliant, Template Fixable, Subscription-Level Gap} per category ---- - -## Part 1: Template Improvements - -_Changes to apply directly in the ARM template to close compliance gaps. These make the deployment itself compliant, independent of policy enforcement._ - -**Who acts:** Template author / developer -**Where:** `.azure/deployments/{deployment-id}/template.json` - -### Gaps Fixable in the Template - -_These gaps can be closed by adding or modifying resources in the ARM template. For each gap, provide the exact ARM template JSON to add — including the full resource definition (type, apiVersion, name, properties), required `dependsOn` references, and any new variables needed._ - -| # | Resource Type | Gap | Compliance Control | Fix | -|---|--------------|-----|-------------------|-----| -| 1 | Storage Account | Blob soft delete not enabled | CP-9 (Contingency Planning) | Add `blobServices/default` child resource with `deleteRetentionPolicy.enabled: true` | -| 2 | Storage Account | No diagnostic settings | AU-2, AU-6, AU-12 (Audit & Accountability) | Add Log Analytics workspace + diagnostic settings resource | -| 3 | Key Vault | No resource logs | AU-2, AU-6 (Audit & Accountability) | Add diagnostic settings for AuditEvent category | -| 4 | NSGs | No flow logs | AU-2, SI-4 (System Monitoring) | Add `networkWatchers/flowLogs` resources | - -> After applying these changes, re-run the assessment to verify compliance. - -### Already Compliant in Template - -_These properties are correctly configured. No template changes needed. Corresponding policies in Part 2 would prevent future drift._ - -| # | Resource Type | Property | Template Value | Compliance Control | -|---|--------------|----------|---------------|-------------------| -| 1 | Storage Account | HTTPS only | `supportsHttpsTrafficOnly: true` | SC-8 (Transmission Confidentiality) | -| 2 | Storage Account | Public access disabled | `allowBlobPublicAccess: false` | AC-3 (Access Enforcement) | -| 3 | Storage Account | Shared key disabled | `allowSharedKeyAccess: false` | IA-2 (Identification & Authentication) | -| 4 | Key Vault | RBAC enabled | `enableRbacAuthorization: true` | AC-3 (Access Enforcement) | - ---- - -## Part 2: Subscription-Level Actions - -_Policy and initiative assignments at subscription or management group scope. These enforce compliance across ALL resources — not just this deployment._ - -**Who acts:** Platform team / subscription admin -**Where:** Azure subscription `{subscription-id}` - -### Existing Policy Assignments (from Azure) - -_Already active. No action needed unless enforcement mode should change._ - -| # | Policy/Initiative | Scope | Enforcement | Type | Relevant to Deployment? | -|---|------------------|-------|-------------|------|------------------------| -| 1 | {assignment name} | Subscription | Default | Built-in | ✅/❌ | -| 2 | {assignment name} | Mgmt Group (inherited) | Default | Initiative | ✅/❌ | - -### Unassigned Custom Policies (from Azure) +## Part 1: Template Improvements (developer acts; edit the ARM template) +### Gaps Fixable in the Template — table of {Resource Type, Gap, Compliance Control, Fix} +### Already Compliant in Template — table of {Resource Type, Property, Template Value, Compliance Control} -_Custom definitions exist in your subscription or management group but are NOT assigned. These were created by your organization and may cover requirements that built-in policies do not._ - -| # | Policy | Category | Target Resource | Effect | Scope | -|---|--------|----------|----------------|--------|-------| -| 1 | {custom policy name} | Storage | Microsoft.Storage/storageAccounts | Modify | Mgmt Group | - -> 🟣 **Action:** These already exist — they just need assignment. Prioritize over built-in equivalents. - -### Recommended Built-in Policy Assignments - -_Policies from Microsoft Learn that are not covered by existing assignments or custom definitions._ - -| # | Policy | Effect | Severity | Definition ID | Category | Source | -|---|--------|--------|----------|--------------|----------|--------| -| 1 | NSG flow logs | Audit | 🟠 High | 27960feb-... | Networking | [MS Learn]({url}) | -| 2 | Allowed locations | Audit | 🟡 Medium | e56962a6-... | General | [MS Learn]({url}) | - -### Recommended Compliance Initiative - -_If a compliance framework was selected:_ - -| Initiative | Policies | Built-in ID | Status | -|------------|----------|-------------|--------| -| NIST SP 800-53 Rev. 5 | 696 | 179d1daa-... | ⚠️ Not assigned | - -Assigning this initiative covers {N} of the {M} individual policies recommended above. -Remaining {M-N} policies need individual assignment. +## Part 2: Subscription-Level Actions (platform team acts; assign at sub/mg scope) +### Existing Policy Assignments — table of {Policy/Initiative, Scope, Enforcement, Type, Relevant?} +### Unassigned Custom Policies — table of {Policy, Category, Target Resource, Effect, Scope} +### Recommended Built-in Assignments — table of {Policy, Effect, Severity, Definition ID, Category, Source} +### Recommended Compliance Initiative — table of {Initiative, Policies, Built-in ID, Status} ``` ### 7. Provide Implementation Options @@ -467,12 +266,12 @@ az policy assignment create \ | `Default` | Active enforcement — new non-compliant resources are denied or audited | | `DoNotEnforce` | Audit-only — evaluates compliance without blocking. Recommended for initial rollout | -### 📋 Policy Gate +### Policy Gate The policy gate is **advisory** — it surfaces findings without blocking deployment. ```markdown -### 📋 Policy Gate: ADVISORY +### Policy Gate: ADVISORY **Part 1 — Template Compliance:** 🔵 {T} of {R} checks pass via template configuration @@ -482,7 +281,7 @@ The policy gate is **advisory** — it surfaces findings without blocking deploy ✅ {A} policies already assigned in subscription 🟣 {C} custom policies available but unassigned — assign for immediate coverage ⚠️ {S} subscription-level gaps — recommend assigning built-in policies or initiative -📊 Enforcement coverage: {percentage}% of recommended policies actively assigned +Enforcement coverage: {percentage}% of recommended policies actively assigned **Action items:** @@ -504,115 +303,19 @@ When invoked during a deployment workflow, save results to the deployment direct | `policy-assessment.md` | Markdown | Full assessment report (Section 4 output) | | `policy-recommendations.json` | JSON | Structured policy data for automation | -**JSON structure for `policy-recommendations.json`:** +**JSON structure for `policy-recommendations.json`** — **Read [references/policy-recommendations-schema.json](references/policy-recommendations-schema.json) when emitting the JSON sidecar.** The reference includes complete field definitions, status/actionTrack enum values, and a fully-worked example. Skeleton: + ```json { - "assessedAt": "2026-04-07T19:00:00Z", - "deploymentId": "{deployment-id}", - "framework": "{compliance-framework}", - "enforcementMode": "Audit", - "subscriptionState": "queried", - "summary": { - "totalRecommended": 16, - "alreadyAssigned": 2, - "templateCompliant": 8, - "templateFixable": 3, - "customAvailable": 1, - "subscriptionGaps": 3 - }, - "existingAssignments": [ - { - "name": "Secure transfer to storage accounts should be enabled", - "definitionId": "/providers/Microsoft.Authorization/policyDefinitions/404c3081-a854-4457-ae30-26a93ef643f9", - "enforcementMode": "Default", - "scope": "/subscriptions/{subscription-id}", - "assignedVia": "direct", - "policyType": "BuiltIn", - "relevantToDeployment": true - } - ], - "unassignedCustomDefinitions": [ - { - "name": "SFI-ID4.2.1 Storage Accounts - Safe Secrets Standard", - "definitionId": "/providers/Microsoft.Management/managementGroups/{mg-id}/providers/Microsoft.Authorization/policyDefinitions/{custom-id}", - "category": "Storage", - "targetResourceType": "Microsoft.Storage/storageAccounts", - "effect": "Modify", - "definitionScope": "management-group", - "actionTrack": "subscription" - } - ], - "templateImprovements": [ - { - "resourceType": "Microsoft.Storage/storageAccounts", - "gap": "Blob soft delete not enabled", - "complianceControl": "CP-9 (Contingency Planning)", - "severity": "medium", - "fix": "Add blobServices/default child resource with deleteRetentionPolicy.enabled: true", - "actionTrack": "template" - }, - { - "resourceType": "Microsoft.Storage/storageAccounts", - "gap": "No diagnostic settings", - "complianceControl": "AU-2, AU-6, AU-12 (Audit & Accountability)", - "severity": "high", - "fix": "Add Log Analytics workspace + Microsoft.Insights/diagnosticSettings", - "actionTrack": "template" - } - ], - "policies": [ - { - "name": "Secure transfer to storage accounts should be enabled", - "definitionId": "404c3081-a854-4457-ae30-26a93ef643f9", - "effect": "Deny", - "severity": "critical", - "category": "Storage", - "status": "already-assigned", - "policyType": "BuiltIn", - "actionTrack": "none", - "sourceUrl": "https://learn.microsoft.com/azure/governance/policy/samples/built-in-policies#storage" - }, - { - "name": "Disable shared key access", - "definitionId": "{built-in-id}", - "effect": "Audit", - "severity": "high", - "category": "Storage", - "status": "template-compliant", - "policyType": "BuiltIn", - "actionTrack": "subscription", - "sourceUrl": "https://learn.microsoft.com/azure/governance/policy/samples/built-in-policies#storage" - }, - { - "name": "Enable blob soft delete", - "definitionId": null, - "effect": null, - "severity": "medium", - "category": "Storage", - "status": "template-fixable", - "policyType": null, - "actionTrack": "template", - "sourceUrl": null - }, - { - "name": "NSG flow logs should be enabled", - "definitionId": "27960feb-a23c-4577-8d36-ef8b5f35e0be", - "effect": "Audit", - "severity": "high", - "category": "Networking", - "status": "gap", - "policyType": "BuiltIn", - "actionTrack": "subscription", - "sourceUrl": "https://learn.microsoft.com/azure/governance/policy/samples/built-in-policies#network" - } - ], - "initiative": { - "name": "NIST SP 800-53 Rev. 5", - "builtInId": "179d1daa-458f-4e47-8086-2a68d0d6c38f", - "policyCount": 696, - "status": "not-assigned", - "coverage": "covers {N} of {M} recommended policies" - } + "assessedAt": "...", "deploymentId": "...", "framework": "...", + "enforcementMode": "Audit|Deny", "subscriptionState": "queried|unavailable", + "summary": { "totalRecommended": 16, "alreadyAssigned": 2, "templateCompliant": 8, + "templateFixable": 3, "customAvailable": 1, "subscriptionGaps": 3 }, + "existingAssignments": [ /* from Step 2 .assigned_policies */ ], + "unassignedCustomDefinitions": [ /* from Step 2 .unassigned_custom_policies */ ], + "templateImprovements": [ /* Part 1 gaps */ ], + "policies": [ /* full recommendation list with status + actionTrack */ ], + "initiative": { /* compliance-framework initiative if selected */ } } ``` @@ -640,3 +343,26 @@ When invoked during a deployment workflow, save results to the deployment direct - **Template Generator:** After `/azure-security-analyzer`, optionally invoke `/azure-policy-advisor` to recommend subscription-level policies that complement the template - **Onboarding:** After RBAC setup, the onboarding flow captures compliance preferences and adds them to `copilot-instructions.md` — this skill reads them automatically - **Drift Detector:** Cross-reference drift findings with policy recommendations — drift items covered by assigned policies will auto-remediate + +## Examples + +**Example 1 — Post-template recommendation (general best practices)** + +> "I just generated an ARM template with a Storage Account, Function App, and Key Vault for production. What Azure Policies should we enforce? Separate template fixes from subscription-level assignments." + +Skill activates → Step 1 reads "General best practices" + Audit from copilot-instructions → Step 2 runs `discover_policy_state.sh` → finds `SecurityCenterBuiltIn` (MCSB) initiative covers 13 of 17 recommendations → Step 6 emits Part 1 (4 template fixes: blob soft delete + 3 diagnostic settings) and Part 2 (3 monitoring built-ins to assign). + +**Example 2 — Compliance framework audit** + +> "Audit our subscription resources against CIS Azure Foundations v3.0 — what's covered and what's missing?" + +Skill activates → Step 1 captures `framework: CIS Azure Foundations v3.0` → Step 2 discovers existing assignments → Step 4 fetches the `CIS Azure Foundations Benchmark v3.0.0` built-in initiative ID from Microsoft Learn → Step 3 verifies the ID via `az policy set-definition list --query "[?contains(displayName, 'CIS')]"` → Step 6 emits per-control coverage + recommended initiative assignment. + +## Troubleshooting + +| Symptom | Cause | Resolution | +|---|---|---| +| `discover_policy_state.sh` exits non-zero, errors about `az login` | Azure CLI not authenticated | Run `az login` and retry. If unauthenticated by design (e.g., template-only review), the skill still produces Part 1 + recommended built-ins from Microsoft Learn, with `subscriptionState: "unavailable"` in the JSON sidecar | +| Recommended definition ID returns "not found" from `az policy definition show` | Display-name drift or wrong cloud (Public vs Government vs China) | Re-verify via Step 3: `az policy set-definition list --query "[?contains(displayName, '')]" -o table`. Definition IDs are stable; display names evolve (e.g., "purge protection" → "deletion protection") | +| Skill reports ⚠️ for a policy already enforced via initiative | `assigned_policies` lookup keyed on individual policy IDs; initiative members aren't expanded | Cross-check `assigned_initiatives` in the policy-state JSON — if MCSB (`1f3afdf9-d0c9-4c3d-847f-89da613e70a8`) is assigned, 200+ individual policies are covered indirectly. Filed as a known limitation; expansion via `az policy set-definition show --query "policyDefinitions[].policyDefinitionId"` is on the roadmap | +| `microsoft_docs_search` returns stale or empty results | MCP server not configured, or Microsoft Learn page moved | Fall back to `microsoft_docs_fetch` with the canonical URL from `references/ms-learn-policy-pages.md` (built-in policies index) | diff --git a/.github/skills/azure-policy-advisor/references/classification-rules.yaml b/.github/skills/azure-policy-advisor/references/classification-rules.yaml new file mode 100644 index 0000000..fbe285e --- /dev/null +++ b/.github/skills/azure-policy-advisor/references/classification-rules.yaml @@ -0,0 +1,118 @@ +# Azure Policy Advisor — Classification & Prioritization Rules +# +# Used by Step 5 of SKILL.md to classify each recommended policy by severity, +# determine the current status of the check (assigned / template-compliant / +# gap / etc.), and resolve precedence when more than one source covers the +# same control. +# +# Load this file when classifying recommendations after Steps 2–4 produce: +# - existing subscription assignments (Step 2) +# - available custom + built-in definitions (Step 3) +# - Microsoft Learn built-in recommendations (Step 4) +# Apply `severity_tiers` to choose the audit/deny effect, `status_classifications` +# to label each finding, and `precedence_rules` when the same control is +# covered by multiple sources. + +severity_tiers: + - tier: Critical + icon: "🔴" + audit_effect: Audit + deny_effect: Deny + when_to_use: | + Prevents insecure deployments: public storage access, missing HTTPS, + no encryption at rest, anonymous endpoints. + + - tier: High + icon: "🟠" + audit_effect: Audit + deny_effect: Deny + when_to_use: | + Strong security posture: managed identity required, TLS 1.2 minimum, + AAD-only auth, no shared-key access. + + - tier: Medium + icon: "🟡" + audit_effect: Audit + deny_effect: Audit + when_to_use: | + Visibility and tracking: tag compliance, diagnostic settings + configured, allowed locations enforcement. + + - tier: Low + icon: "🔵" + audit_effect: AuditIfNotExists + deny_effect: DeployIfNotExists + when_to_use: | + Auto-remediation: deploy diagnostic settings, enable monitoring, + backfill missing tags. + +status_classifications: + - status: already_assigned + icon: "✅" + name: Already assigned + condition: | + Policy is actively assigned in the subscription (from Step 2 inventory). + action: | + No action needed. Note enforcement mode (`Default` vs `DoNotEnforce`) + and scope (subscription vs management group) in the report. + + - status: compliant_via_template + icon: "🔵" + name: Compliant via template + condition: | + ARM template already configures the property the policy would enforce, + but the policy itself is NOT assigned. + action: | + Report as compliant-by-construction. Optionally recommend assigning + the policy to prevent future regressions. + + - status: unassigned_custom_available + icon: "🟣" + name: Unassigned custom policy available + condition: | + A custom policy definition exists in the subscription or management + group (from Step 3) that covers this check, but it is not assigned. + action: | + Recommend immediate assignment of the custom definition — it already + exists, so this is the lowest-friction fix. + + - status: gap_not_covered + icon: "⚠️" + name: Gap — not covered + condition: | + Neither assigned nor available as a custom definition. + action: | + Recommend the built-in policy from Microsoft Learn (Step 4). + + - status: complementary + icon: "🔄" + name: Complementary + condition: | + Policy would add enforcement on top of existing template config + and/or existing assignments. + action: | + Recommend as a defense-in-depth layer. + +precedence_rules: + # Apply in order — first match wins. + - rule: assigned_wins + when: A matching policy is already assigned. + then: | + Report as ✅ Already assigned. No new recommendation, UNLESS the + enforcement mode is `DoNotEnforce`, in which case flag for mode + upgrade to `Default`. + + - rule: custom_over_builtin + when: | + A custom definition exists in the subscription/management group but + is unassigned, AND a built-in covers the same control. + then: | + Recommend assigning the custom definition (🟣). Do NOT also recommend + the built-in for the same control. + + - rule: builtin_fallback + when: | + Only a built-in definition covers the control (no custom, no + existing assignment). + then: | + Recommend the built-in (⚠️ gap). diff --git a/.github/skills/azure-policy-advisor/references/ms-learn-policy-pages.md b/.github/skills/azure-policy-advisor/references/ms-learn-policy-pages.md new file mode 100644 index 0000000..57dcdf9 --- /dev/null +++ b/.github/skills/azure-policy-advisor/references/ms-learn-policy-pages.md @@ -0,0 +1,33 @@ +--- +source: https://learn.microsoft.com/azure/governance/policy/ +snapshot: 2026-06-04 +refresh_command: "az rest --method get --uri 'https://learn.microsoft.com/api/contentbrowser/search/azure?moniker=azure&locale=en-us&category=Documentation&products=azure-policy' 2>/dev/null || echo 'Manual: visit https://learn.microsoft.com/azure/governance/policy/ and verify links below resolve.'" +--- + +# Microsoft Learn — Azure Policy reference pages + +Authoritative entry points for Azure Policy content on Microsoft Learn. Use these URLs with `microsoft_docs_fetch` when the model needs concrete content from a specific page rather than a search query. + +| Content | URL | +|---------|-----| +| All built-in policies (canonical list, by category) | `https://learn.microsoft.com/azure/governance/policy/samples/built-in-policies` | +| Built-in initiatives (regulatory + custom frameworks) | `https://learn.microsoft.com/azure/governance/policy/samples/built-in-initiatives` | +| Regulatory compliance concepts | `https://learn.microsoft.com/azure/governance/policy/concepts/regulatory-compliance` | +| Policy assignment via ARM template | `https://learn.microsoft.com/azure/governance/policy/assign-policy-template` | +| Policy effects reference (Audit, Deny, DeployIfNotExists, etc.) | `https://learn.microsoft.com/azure/governance/policy/concepts/effect-basics` | +| Compliance framework details: CIS Azure Foundations | `https://learn.microsoft.com/azure/governance/policy/samples/cis-azure-1-3-0` | +| Compliance framework details: NIST SP 800-53 Rev 5 | `https://learn.microsoft.com/azure/governance/policy/samples/nist-sp-800-53-r5` | +| Compliance framework details: FedRAMP Moderate | `https://learn.microsoft.com/azure/governance/policy/samples/fedramp-moderate` | +| Compliance framework details: PCI DSS 4.0 | `https://learn.microsoft.com/azure/governance/policy/samples/pci-dss-4` | + +## When to fetch which page + +- **`built-in-policies`** — when you need the canonical list of all built-in policies grouped by Azure service category (Storage, App Service, etc.) and the per-resource-type priorities in `per-resource-policy-priorities.md` don't cover the resource type the user has in their template. +- **`built-in-initiatives`** — when the user mentions a compliance framework not pre-mapped in this skill, or when verifying that an initiative definition ID is current. +- **Framework-specific pages (`cis-azure-1-3-0`, `nist-sp-800-53-r5`, etc.)** — when the user asks for evidence trail mapping (which policies cover which controls) for a specific framework. +- **`assign-policy-template`** — when emitting ARM-template assignment JSON in Part 2 of the report. +- **`effect-basics`** — when the user asks why a specific policy uses Audit vs Deny vs DeployIfNotExists. + +## Search-first pattern + +For ad-hoc queries, prefer `microsoft_docs_search` first (broader coverage, recency) and reach for `microsoft_docs_fetch` against these specific URLs only when search results point at one of them or you need the full structured page content. See Step 4 of `SKILL.md` for the search query templates. diff --git a/.github/skills/azure-policy-advisor/references/per-resource-policy-priorities.md b/.github/skills/azure-policy-advisor/references/per-resource-policy-priorities.md new file mode 100644 index 0000000..d041aeb --- /dev/null +++ b/.github/skills/azure-policy-advisor/references/per-resource-policy-priorities.md @@ -0,0 +1,88 @@ +--- +source: https://learn.microsoft.com/azure/governance/policy/samples/built-in-policies +snapshot: 2026-06-04 +refresh_command: "echo 'Manual: cross-check each list against the built-in-policies page; update if categories change. The severity assignments (🔴🟠🟡🔵) are skill-author judgement, not a Microsoft taxonomy.'" +--- + +# Per-resource-type policy priorities + +Concrete, ranked recommendations for the most common Azure resource types. Used by `azure-policy-advisor` Step 5 (Classify and Prioritize Recommendations). + +**Severity legend** (from `SKILL.md`, kept here for self-containment): + +| Tier | Effect (Audit mode) | Effect (Deny mode) | Meaning | +|------|--------------------|--------------------|---------| +| 🔴 **Critical** | Audit | Deny | Prevents insecure deployments — never ship without it | +| 🟠 **High** | Audit | Deny | Strong security posture — required for production | +| 🟡 **Medium** | Audit | Audit | Visibility and tracking | +| 🔵 **Low** | AuditIfNotExists | DeployIfNotExists | Auto-remediation / observability | + +## Storage Accounts + +1. 🔴 Require secure transfer (HTTPS) +2. 🔴 Disable public blob access +3. 🟠 Disable shared key access +4. 🟠 Require minimum TLS 1.2 +5. 🟡 Require private endpoints (production) +6. 🟡 Enable soft delete for blobs and containers +7. 🔵 Deploy diagnostic settings + +## App Service / Function Apps + +1. 🔴 Require HTTPS only +2. 🔴 Require managed identity +3. 🟠 Require minimum TLS 1.2 +4. 🟠 Disable FTP / require FTPS only +5. 🟡 Disable public network access (production) +6. 🔵 Enable resource logs + +## SQL Servers / Databases + +1. 🔴 Require AAD-only authentication +2. 🔴 Enable transparent data encryption +3. 🟠 Enable auditing +4. 🟠 Enable Advanced Threat Protection +5. 🟡 Require private endpoints (production) +6. 🔵 Deploy diagnostic settings + +## Key Vault + +1. 🔴 Enable RBAC authorization +2. 🔴 Enable soft delete and purge protection +3. 🟠 Disable public network access (production) +4. 🟡 Require private endpoints +5. 🔵 Deploy diagnostic settings for audit events + +## Compute / VMs + +1. 🔴 Require managed disks +2. 🟠 Require managed identity +3. 🟡 Restrict allowed VM SKUs +4. 🟡 Require approved extensions only +5. 🔵 Deploy monitoring agent + +## AKS / Kubernetes + +1. 🔴 Require managed identity +2. 🔴 Disable local accounts +3. 🟠 Require Azure Policy add-on +4. 🟠 Require network policy +5. 🟡 Require authorized IP ranges for API server +6. 🔵 Enable Container Insights + +## Networking + +1. 🟠 Require NSG flow logs +2. 🟡 Enable Network Watcher +3. 🟡 Restrict allowed locations +4. 🔵 Deploy DDoS protection (production) + +## General / Cross-cutting + +1. 🟡 Require tags on resources (ManagedBy, Environment, Project) +2. 🟡 Restrict allowed locations +3. 🔵 Require resource group tags inheritance + +## When a resource type is not listed + +Fall back to `microsoft_docs_search` with the query template `Azure Policy built-in {resource-type-category}` (see Step 4 of `SKILL.md`). The pattern: identify the top 2-3 security-critical controls (🔴), 1-2 hardening controls (🟠), 1-2 visibility controls (🟡), and 1 observability control (🔵). Cross-check definition IDs live before emitting recommendations. diff --git a/.github/skills/azure-policy-advisor/references/policy-assessment-template.md b/.github/skills/azure-policy-advisor/references/policy-assessment-template.md new file mode 100644 index 0000000..8999376 --- /dev/null +++ b/.github/skills/azure-policy-advisor/references/policy-assessment-template.md @@ -0,0 +1,112 @@ +# Azure Policy Assessment Report Template + +This is the full markdown report template emitted by Step 6 of the +`azure-policy-advisor` skill. The skill SKILL.md only includes a short +outline; this file holds the verbatim template that the skill instantiates +when writing `policy-assessment.md` to the deployment directory. + +The report is split into two clear action tracks — **template improvements** +(changes to the ARM template) and **subscription-level actions** (policy / +initiative assignments) — so it's obvious who needs to act and where. + +--- + +```markdown +## Azure Policy Compliance Assessment + +**Scope:** {subscription or resource group} +**Deployment:** {deployment ID or "general subscription audit"} +**Compliance Framework:** {framework from copilot-instructions.md or user input} +**Enforcement Mode:** {Audit or Deny} +**Subscription Policy State:** {Queried | Not available (no az login)} + +### Summary + +| Category | Recommended | Already Assigned | Template Compliant | Template Fixable | Subscription-Level Gap | +|----------|-------------|-----------------|-------------------|-----------------|----------------------| +| Storage | 7 | 2 | 3 | 1 | 1 | +| Key Vault | 5 | 0 | 4 | 1 | 0 | +| Networking | 4 | 0 | 1 | 1 | 2 | +| Total | 16 | 2 | 8 | 3 | 3 | + +--- + +## Part 1: Template Improvements + +_Changes to apply directly in the ARM template to close compliance gaps. These make the deployment itself compliant, independent of policy enforcement._ + +**Who acts:** Template author / developer +**Where:** `.azure/deployments/{deployment-id}/template.json` + +### Gaps Fixable in the Template + +_These gaps can be closed by adding or modifying resources in the ARM template. For each gap, provide the exact ARM template JSON to add — including the full resource definition (type, apiVersion, name, properties), required `dependsOn` references, and any new variables needed._ + +| # | Resource Type | Gap | Compliance Control | Fix | +|---|--------------|-----|-------------------|-----| +| 1 | Storage Account | Blob soft delete not enabled | CP-9 (Contingency Planning) | Add `blobServices/default` child resource with `deleteRetentionPolicy.enabled: true` | +| 2 | Storage Account | No diagnostic settings | AU-2, AU-6, AU-12 (Audit & Accountability) | Add Log Analytics workspace + diagnostic settings resource | +| 3 | Key Vault | No resource logs | AU-2, AU-6 (Audit & Accountability) | Add diagnostic settings for AuditEvent category | +| 4 | NSGs | No flow logs | AU-2, SI-4 (System Monitoring) | Add `networkWatchers/flowLogs` resources | + +> After applying these changes, re-run the assessment to verify compliance. + +### Already Compliant in Template + +_These properties are correctly configured. No template changes needed. Corresponding policies in Part 2 would prevent future drift._ + +| # | Resource Type | Property | Template Value | Compliance Control | +|---|--------------|----------|---------------|-------------------| +| 1 | Storage Account | HTTPS only | `supportsHttpsTrafficOnly: true` | SC-8 (Transmission Confidentiality) | +| 2 | Storage Account | Public access disabled | `allowBlobPublicAccess: false` | AC-3 (Access Enforcement) | +| 3 | Storage Account | Shared key disabled | `allowSharedKeyAccess: false` | IA-2 (Identification & Authentication) | +| 4 | Key Vault | RBAC enabled | `enableRbacAuthorization: true` | AC-3 (Access Enforcement) | + +--- + +## Part 2: Subscription-Level Actions + +_Policy and initiative assignments at subscription or management group scope. These enforce compliance across ALL resources — not just this deployment._ + +**Who acts:** Platform team / subscription admin +**Where:** Azure subscription `{subscription-id}` + +### Existing Policy Assignments (from Azure) + +_Already active. No action needed unless enforcement mode should change._ + +| # | Policy/Initiative | Scope | Enforcement | Type | Relevant to Deployment? | +|---|------------------|-------|-------------|------|------------------------| +| 1 | {assignment name} | Subscription | Default | Built-in | ✅/❌ | +| 2 | {assignment name} | Mgmt Group (inherited) | Default | Initiative | ✅/❌ | + +### Unassigned Custom Policies (from Azure) + +_Custom definitions exist in your subscription or management group but are NOT assigned. These were created by your organization and may cover requirements that built-in policies do not._ + +| # | Policy | Category | Target Resource | Effect | Scope | +|---|--------|----------|----------------|--------|-------| +| 1 | {custom policy name} | Storage | Microsoft.Storage/storageAccounts | Modify | Mgmt Group | + +> 🟣 **Action:** These already exist — they just need assignment. Prioritize over built-in equivalents. + +### Recommended Built-in Policy Assignments + +_Policies from Microsoft Learn that are not covered by existing assignments or custom definitions._ + +| # | Policy | Effect | Severity | Definition ID | Category | Source | +|---|--------|--------|----------|--------------|----------|--------| +| 1 | NSG flow logs | Audit | 🟠 High | 27960feb-... | Networking | [MS Learn]({url}) | +| 2 | Allowed locations | Audit | 🟡 Medium | e56962a6-... | General | [MS Learn]({url}) | + +### Recommended Compliance Initiative + +_If a compliance framework was selected:_ + +| Initiative | Policies | Built-in ID | Status | +|------------|----------|-------------|--------| +| NIST SP 800-53 Rev. 5 | 696 | 179d1daa-... | ⚠️ Not assigned | + +Assigning this initiative covers {N} of the {M} individual policies recommended above. +Remaining {M-N} policies need individual assignment. +``` diff --git a/.github/skills/azure-policy-advisor/references/policy-recommendations-schema.json b/.github/skills/azure-policy-advisor/references/policy-recommendations-schema.json new file mode 100644 index 0000000..f0a2016 --- /dev/null +++ b/.github/skills/azure-policy-advisor/references/policy-recommendations-schema.json @@ -0,0 +1,108 @@ +{ + "assessedAt": "2026-04-07T19:00:00Z", + "deploymentId": "{deployment-id}", + "framework": "{compliance-framework}", + "enforcementMode": "Audit", + "subscriptionState": "queried", + "summary": { + "totalRecommended": 16, + "alreadyAssigned": 2, + "templateCompliant": 8, + "templateFixable": 3, + "customAvailable": 1, + "subscriptionGaps": 3 + }, + "existingAssignments": [ + { + "name": "Secure transfer to storage accounts should be enabled", + "definitionId": "/providers/Microsoft.Authorization/policyDefinitions/404c3081-a854-4457-ae30-26a93ef643f9", + "enforcementMode": "Default", + "scope": "/subscriptions/{subscription-id}", + "assignedVia": "direct", + "policyType": "BuiltIn", + "relevantToDeployment": true + } + ], + "unassignedCustomDefinitions": [ + { + "name": "SFI-ID4.2.1 Storage Accounts - Safe Secrets Standard", + "definitionId": "/providers/Microsoft.Management/managementGroups/{mg-id}/providers/Microsoft.Authorization/policyDefinitions/{custom-id}", + "category": "Storage", + "targetResourceType": "Microsoft.Storage/storageAccounts", + "effect": "Modify", + "definitionScope": "management-group", + "actionTrack": "subscription" + } + ], + "templateImprovements": [ + { + "resourceType": "Microsoft.Storage/storageAccounts", + "gap": "Blob soft delete not enabled", + "complianceControl": "CP-9 (Contingency Planning)", + "severity": "medium", + "fix": "Add blobServices/default child resource with deleteRetentionPolicy.enabled: true", + "actionTrack": "template" + }, + { + "resourceType": "Microsoft.Storage/storageAccounts", + "gap": "No diagnostic settings", + "complianceControl": "AU-2, AU-6, AU-12 (Audit & Accountability)", + "severity": "high", + "fix": "Add Log Analytics workspace + Microsoft.Insights/diagnosticSettings", + "actionTrack": "template" + } + ], + "policies": [ + { + "name": "Secure transfer to storage accounts should be enabled", + "definitionId": "404c3081-a854-4457-ae30-26a93ef643f9", + "effect": "Deny", + "severity": "critical", + "category": "Storage", + "status": "already-assigned", + "policyType": "BuiltIn", + "actionTrack": "none", + "sourceUrl": "https://learn.microsoft.com/azure/governance/policy/samples/built-in-policies#storage" + }, + { + "name": "Disable shared key access", + "definitionId": "{built-in-id}", + "effect": "Audit", + "severity": "high", + "category": "Storage", + "status": "template-compliant", + "policyType": "BuiltIn", + "actionTrack": "subscription", + "sourceUrl": "https://learn.microsoft.com/azure/governance/policy/samples/built-in-policies#storage" + }, + { + "name": "Enable blob soft delete", + "definitionId": null, + "effect": null, + "severity": "medium", + "category": "Storage", + "status": "template-fixable", + "policyType": null, + "actionTrack": "template", + "sourceUrl": null + }, + { + "name": "NSG flow logs should be enabled", + "definitionId": "27960feb-a23c-4577-8d36-ef8b5f35e0be", + "effect": "Audit", + "severity": "high", + "category": "Networking", + "status": "gap", + "policyType": "BuiltIn", + "actionTrack": "subscription", + "sourceUrl": "https://learn.microsoft.com/azure/governance/policy/samples/built-in-policies#network" + } + ], + "initiative": { + "name": "NIST SP 800-53 Rev. 5", + "builtInId": "179d1daa-458f-4e47-8086-2a68d0d6c38f", + "policyCount": 696, + "status": "not-assigned", + "coverage": "covers {N} of {M} recommended policies" + } +} diff --git a/.github/skills/azure-policy-advisor/scripts/discover_policy_state.sh b/.github/skills/azure-policy-advisor/scripts/discover_policy_state.sh new file mode 100755 index 0000000..9dd45ca --- /dev/null +++ b/.github/skills/azure-policy-advisor/scripts/discover_policy_state.sh @@ -0,0 +1,223 @@ +#!/bin/bash +# Azure Policy Advisor — Policy State Discovery +# +# Inventories the current Azure Policy state for a subscription: +# 1. Active policy assignments (subscription scope, including those inherited +# from any parent management group) +# 2. Initiative (policy-set) assignments +# 3. Unassigned CUSTOM policy definitions at subscription scope +# 4. Unassigned CUSTOM initiative definitions at subscription scope +# 5. Optionally: custom definitions at management-group scope +# +# Replaces the prose `az policy ...` query blocks in azure-policy-advisor +# SKILL.md Steps 2 + 3 so the model loads a deterministic script instead of +# generating those queries from natural language each run. +# +# Usage: +# bash scripts/discover_policy_state.sh \ +# --subscription \ +# [--management-group ] \ +# [--output ] # default: stdout +# +# Output: JSON document with the schema documented in --help. +# Exit codes: +# 0 = success (state discovered; may include non-fatal errors in .errors[]) +# 1 = az CLI missing, not authenticated, or required argument missing +# 2 = subscription query failed entirely + +set -euo pipefail + +SUBSCRIPTION="" +MANAGEMENT_GROUP="" +OUTPUT="" + +usage() { + cat < [--management-group ] [--output ] + +Required: + --subscription Azure subscription ID to query + +Optional: + --management-group Management group name to also scan for custom definitions + --output File to write JSON to (default: stdout) + -h, --help Show this help + +Output schema (JSON): + { + "subscription_id": "", + "management_group": "", + "assigned_policies": [ {assignment_name, display_name, policy_definition_id, + enforcement_mode, scope, source}, ... ], + "assigned_initiatives": [ {assignment_name, display_name, policy_set_definition_id, + enforcement_mode, scope}, ... ], + "unassigned_custom_policies": [ {definition_id, display_name, category, + target_resource_type, effect, source}, ... ], + "unassigned_custom_initiatives": [ {definition_id, display_name, ...} ], + "errors": [ "non-fatal error message", ... ] + } +EOF +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --subscription) SUBSCRIPTION="$2"; shift 2 ;; + --management-group) MANAGEMENT_GROUP="$2"; shift 2 ;; + --output) OUTPUT="$2"; shift 2 ;; + -h|--help) usage; exit 0 ;; + *) echo "Unknown option: $1" >&2; usage >&2; exit 1 ;; + esac +done + +if [[ -z "$SUBSCRIPTION" ]]; then + echo "ERROR: --subscription is required" >&2 + usage >&2 + exit 1 +fi + +# Tool availability checks +for tool in az jq; do + if ! command -v "$tool" >/dev/null 2>&1; then + echo "ERROR: required tool '$tool' is not installed" >&2 + exit 1 + fi +done + +if ! az account show >/dev/null 2>&1; then + echo "ERROR: not authenticated to Azure (run 'az login')" >&2 + exit 1 +fi + +ERRORS=() + +# 1. Active policy assignments at subscription scope (includes MG-inherited) +ASSIGNMENTS_JSON=$( + az policy assignment list \ + --subscription "$SUBSCRIPTION" \ + --query "[?!contains(policyDefinitionId, 'policySetDefinitions')].{assignment_name:name, display_name:displayName, policy_definition_id:policyDefinitionId, enforcement_mode:enforcementMode, scope:scope}" \ + -o json 2>/dev/null +) || { + ERRORS+=("policy assignment list failed for subscription $SUBSCRIPTION") + ASSIGNMENTS_JSON="[]" +} + +# Tag each assignment by source: subscription-scope vs management-group-inherited +ASSIGNED_POLICIES=$(echo "$ASSIGNMENTS_JSON" | jq ' + map(. + {source: (if (.scope // "") | contains("/managementGroups/") then "management-group-inherited" else "subscription" end)}) +') + +# 2. Initiative (policy-set) assignments +INITIATIVES_JSON=$( + az policy assignment list \ + --subscription "$SUBSCRIPTION" \ + --query "[?contains(policyDefinitionId, 'policySetDefinitions')].{assignment_name:name, display_name:displayName, policy_set_definition_id:policyDefinitionId, enforcement_mode:enforcementMode, scope:scope}" \ + -o json 2>/dev/null +) || { + ERRORS+=("policy initiative assignment list failed") + INITIATIVES_JSON="[]" +} + +# Build a set of assigned definition IDs so we can filter out already-assigned customs +ASSIGNED_IDS=$(echo "$ASSIGNED_POLICIES" | jq '[.[].policy_definition_id]') + +# 3. Custom policy definitions at subscription scope (unassigned only) +SUB_CUSTOM_JSON=$( + az policy definition list \ + --subscription "$SUBSCRIPTION" \ + --query "[?policyType=='Custom'].{definition_id:id, display_name:displayName, category:metadata.category, policy_rule:policyRule, effect:policyRule.then.effect}" \ + -o json 2>/dev/null +) || { + ERRORS+=("policy definition list failed at subscription scope") + SUB_CUSTOM_JSON="[]" +} + +# 4. Custom policy definitions at MG scope (optional) +MG_CUSTOM_JSON="[]" +if [[ -n "$MANAGEMENT_GROUP" ]]; then + MG_CUSTOM_JSON=$( + az policy definition list \ + --management-group "$MANAGEMENT_GROUP" \ + --query "[?policyType=='Custom'].{definition_id:id, display_name:displayName, category:metadata.category, policy_rule:policyRule, effect:policyRule.then.effect}" \ + -o json 2>/dev/null + ) || { + ERRORS+=("policy definition list failed at management-group scope ($MANAGEMENT_GROUP)") + MG_CUSTOM_JSON="[]" + } +fi + +# Normalize: extract target resource type from policyRule.if, tag source, filter out assigned +UNASSIGNED_CUSTOM_POLICIES=$( + jq -n \ + --argjson sub "$SUB_CUSTOM_JSON" \ + --argjson mg "$MG_CUSTOM_JSON" \ + --argjson assigned "$ASSIGNED_IDS" ' + def extract_target_type(rule): + if rule == null then null + elif (rule | type) == "object" and rule.field == "type" then rule.equals + else + (rule | objects | to_entries | map( + if (.value | type) == "array" then + (.value | map(extract_target_type(.)) | map(select(. != null)) | first // null) + elif (.value | type) == "object" then extract_target_type(.value) + else null end + ) | map(select(. != null)) | first // null) + end; + + ($sub | map(. + {source: "subscription"})) + + ($mg | map(. + {source: "management-group"})) + | map({ + definition_id, + display_name, + category, + target_resource_type: extract_target_type(.policy_rule.if // null), + effect, + source + }) + | map(select(.definition_id as $id | $assigned | index($id) | not)) + ' +) + +# 5. Custom initiatives at subscription scope (unassigned only) +ASSIGNED_INITIATIVE_IDS=$(echo "$INITIATIVES_JSON" | jq '[.[].policy_set_definition_id]') +SUB_CUSTOM_INIT_JSON=$( + az policy set-definition list \ + --subscription "$SUBSCRIPTION" \ + --query "[?policyType=='Custom'].{definition_id:id, display_name:displayName, description:description, policy_definitions:policyDefinitions}" \ + -o json 2>/dev/null +) || { + ERRORS+=("policy set-definition list failed at subscription scope") + SUB_CUSTOM_INIT_JSON="[]" +} + +UNASSIGNED_CUSTOM_INITIATIVES=$( + echo "$SUB_CUSTOM_INIT_JSON" \ + | jq --argjson assigned "$ASSIGNED_INITIATIVE_IDS" \ + 'map(select(.definition_id as $id | $assigned | index($id) | not))' +) + +ERRORS_JSON=$(printf '%s\n' "${ERRORS[@]+"${ERRORS[@]}"}" | jq -R . | jq -s 'map(select(. != ""))') + +RESULT=$(jq -n \ + --arg sub "$SUBSCRIPTION" \ + --arg mg "$MANAGEMENT_GROUP" \ + --argjson assigned "$ASSIGNED_POLICIES" \ + --argjson assigned_init "$INITIATIVES_JSON" \ + --argjson unassigned "$UNASSIGNED_CUSTOM_POLICIES" \ + --argjson unassigned_init "$UNASSIGNED_CUSTOM_INITIATIVES" \ + --argjson errors "$ERRORS_JSON" \ + '{ + subscription_id: $sub, + management_group: (if $mg == "" then null else $mg end), + assigned_policies: $assigned, + assigned_initiatives: $assigned_init, + unassigned_custom_policies: $unassigned, + unassigned_custom_initiatives: $unassigned_init, + errors: $errors + }') + +if [[ -n "$OUTPUT" ]]; then + echo "$RESULT" > "$OUTPUT" + echo "Wrote policy state to $OUTPUT" >&2 +else + echo "$RESULT" +fi diff --git a/.github/skills/azure-resource-availability/SKILL.md b/.github/skills/azure-resource-availability/SKILL.md index fea859b..bf93737 100644 --- a/.github/skills/azure-resource-availability/SKILL.md +++ b/.github/skills/azure-resource-availability/SKILL.md @@ -1,8 +1,9 @@ --- name: azure-resource-availability description: "Query live Azure APIs to validate resource availability before template generation or deployment. Checks VM SKU restrictions, Kubernetes/runtime version support, API version compatibility, and subscription quota. Use during requirements gathering and preflight to catch deployment failures early." -argument-hint: "Resource type, region, and SKU/version to validate" -user-invocable: true +metadata: + argument-hint: "Resource type, region, and SKU/version to validate" + user-invocable: true --- # Azure Resource Availability diff --git a/.github/skills/azure-resource-visualizer/SKILL.md b/.github/skills/azure-resource-visualizer/SKILL.md index 0405926..1bc609b 100644 --- a/.github/skills/azure-resource-visualizer/SKILL.md +++ b/.github/skills/azure-resource-visualizer/SKILL.md @@ -1,8 +1,9 @@ --- name: azure-resource-visualizer description: "Analyze deployed Azure resource groups and generate detailed Mermaid architecture diagrams showing relationships between resources. Use for post-deployment visualization, understanding existing infrastructure, or documenting live Azure environments." -argument-hint: "Resource group name to visualize" -user-invocable: true +metadata: + argument-hint: "Resource group name to visualize" + user-invocable: true --- # Azure Resource Visualizer diff --git a/.github/skills/azure-rest-api-reference/SKILL.md b/.github/skills/azure-rest-api-reference/SKILL.md index 0f986cf..d0eeaff 100644 --- a/.github/skills/azure-rest-api-reference/SKILL.md +++ b/.github/skills/azure-rest-api-reference/SKILL.md @@ -1,8 +1,9 @@ --- name: azure-rest-api-reference description: "Look up Azure REST API and ARM template reference documentation for any resource type. Returns exact property schemas, required fields, valid values, and latest stable API versions. Use BEFORE generating or modifying ARM templates to ensure correctness. No Azure connection required." -argument-hint: "Resource type (e.g., Microsoft.Web/sites, Microsoft.Storage/storageAccounts)" -user-invocable: true +metadata: + argument-hint: "Resource type (e.g., Microsoft.Web/sites, Microsoft.Storage/storageAccounts)" + user-invocable: true --- # Azure REST API Reference Lookup diff --git a/.github/skills/azure-role-selector/SKILL.md b/.github/skills/azure-role-selector/SKILL.md index 5a46ed8..10572f0 100644 --- a/.github/skills/azure-role-selector/SKILL.md +++ b/.github/skills/azure-role-selector/SKILL.md @@ -1,8 +1,9 @@ --- name: azure-role-selector description: "Recommend least-privilege Azure RBAC roles for deployed resources. Finds minimal built-in roles matching desired permissions or creates custom role definitions. Use during security analysis or when configuring access for service principals and managed identities." -argument-hint: "Describe the permissions needed (e.g., 'read storage blobs', 'deploy to app service')" -user-invocable: true +metadata: + argument-hint: "Describe the permissions needed (e.g., 'read storage blobs', 'deploy to app service')" + user-invocable: true --- # Azure Role Selector diff --git a/.github/skills/azure-security-analyzer/SKILL.md b/.github/skills/azure-security-analyzer/SKILL.md index 25a54d4..6fe948b 100644 --- a/.github/skills/azure-security-analyzer/SKILL.md +++ b/.github/skills/azure-security-analyzer/SKILL.md @@ -1,8 +1,9 @@ --- name: azure-security-analyzer description: "Analyze Azure resource configurations against security best practices using Azure MCP bestpractices service. Produces per-resource security assessment with severity ratings and recommendations. Use during template generation before deployment confirmation." -argument-hint: "Resource types and their configurations from the ARM template" -user-invocable: true +metadata: + argument-hint: "Resource types and their configurations from the ARM template" + user-invocable: true --- # Azure Security Analyzer diff --git a/.github/skills/git-ape-onboarding/SKILL.md b/.github/skills/git-ape-onboarding/SKILL.md index caa1537..24d8cd0 100644 --- a/.github/skills/git-ape-onboarding/SKILL.md +++ b/.github/skills/git-ape-onboarding/SKILL.md @@ -1,8 +1,9 @@ --- name: git-ape-onboarding description: "Onboard a repository, Azure subscription(s), and user identity for Git-Ape CI/CD using a skill-driven CLI playbook. Use for first-time setup of OIDC, federated credentials, RBAC, GitHub environments, and required secrets." -argument-hint: "GitHub repo URL, subscription target(s), and onboarding mode (single or multi-environment)" -user-invocable: true +metadata: + argument-hint: "GitHub repo URL, subscription target(s), and onboarding mode (single or multi-environment)" + user-invocable: true --- # Git-Ape Onboarding diff --git a/.github/skills/prereq-check/SKILL.md b/.github/skills/prereq-check/SKILL.md index 962e4c9..bf08aef 100644 --- a/.github/skills/prereq-check/SKILL.md +++ b/.github/skills/prereq-check/SKILL.md @@ -1,10 +1,10 @@ --- name: prereq-check description: "Validate Git-Ape CLI tool installation (az, gh, jq, git), versions, and auth sessions. Shows platform-specific install commands for anything missing. USE FOR: check Git-Ape prerequisites, what do I need to install for Git-Ape, verify Git-Ape CLI tools, az: command not found, gh: command not found, jq: command not found, git: command not found, az missing, gh missing, jq missing, git missing, fresh machine setup for Git-Ape, dev container setup for Git-Ape, before running git-ape-onboarding, az login required, gh auth login, auth expired, not logged in, outdated az version, minimum az version, upgrade az. DO NOT USE FOR: Anything else. This skill is narrowly scoped to prerequisites checks for Git-Ape's CLI tools and auth sessions. Do not use it for any other purpose." -argument-hint: "Run without arguments to check all prerequisites" -user-invocable: true license: MIT metadata: + argument-hint: "Run without arguments to check all prerequisites" + user-invocable: true author: Git-Ape version: "0.1.0" --- diff --git a/.waza.yaml b/.waza.yaml index 79fab8c..898f5cc 100644 --- a/.waza.yaml +++ b/.waza.yaml @@ -30,18 +30,26 @@ dev: maxIterations: 5 tokens: - # Aspirational token caps for new SKILL.md files. These are intentionally - # tighter than today's corpus (13 skills as of 2026-05, p75 ≈ 3.2k tokens) — - # the goal is to push NEW skills to fit comfortably in agent context, while - # existing oversize skills are gradually trimmed via /skill-improve. - # Re-evaluate once the skill corpus stabilises: `waza tokens count + # Aspirational token caps for SKILL.md files, calibrated to the upstream + # agentskills.io specification recommendation: < 5,000 tokens and < 500 lines + # per skill body. (See https://agentskills.io/specification.md#progressive-disclosure) + # + # Rationale: an earlier iteration of this config set warningThreshold=1000 and + # fallbackLimit=1300 in pursuit of a "tighter than upstream" local target. + # In practice, several legitimately procedural skills (azure-policy-advisor, + # azure-security-analyzer, git-ape-onboarding) carry domain-specific procedures + # that exceed 1k tokens by design — the upstream 5,000-token ceiling is the + # ground truth for "does this fit in agent context comfortably". + # + # Skills approaching warningThreshold should explore progressive disclosure + # (L2 references/, L3 live tools, scripts/) before being decomposed. Skills + # over fallbackLimit should be split into focused sub-skills. + # + # Re-evaluate when the skill corpus stabilises: `waza tokens count # .github/skills/ --format json | jq '[.files|to_entries[]| # select(.key|endswith("/SKILL.md"))|.value.tokens]|sort'`. - warningThreshold: 1000 - # Hard fallback ceiling enforced by `waza tokens compare --strict`. - # Tightened in Wave 3b from 1500 → 1300 (1.3× warningThreshold) to narrow - # the gap between "warn" and "block" for new skills. - fallbackLimit: 1300 + warningThreshold: 3000 + fallbackLimit: 5000 graders: programTimeout: 30