diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 5813bea..e46b76a 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -220,7 +220,7 @@ flowchart TB Stewards do **not** review all Detection logic. They own catalog integrity and vocabulary discipline. -Deprecation: `deprecated` + `replacedBy` + N−1 MINOR support window; mandatory→recommended needs RFC. +Deprecation: `deprecated` + `replacedBy` + N−1 MINOR support window; mandatory→recommended needs RFC. **Pre-release exception** (no tagged versions yet): see [`id-gaps.md`](packages/aprf-engine/rules/_index/id-gaps.md) and [APRF-RFC-0002](rfcs/0002-incident-readiness-mandatory-to-recommended.md). --- diff --git a/CHANGELOG.md b/CHANGELOG.md index bd17b07..ea73db9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ Format follows [Keep a Changelog](https://keepachangelog.com/). Versioning is Se ## [Unreleased] ### Changed +- Rewrote **incident-readiness** **INC-M1**–**INC-M2** and **INC-R1**/**INC-R3**; demoted **INC-M3 → INC-R2** and **INC-M4 → INC-R4** per [APRF-RFC-0002](rfcs/0002-incident-readiness-mandatory-to-recommended.md) (regulated 56→55, tier3-only 17→16; pre-release ID-removal exception in `id-gaps.md`); hybrid collectors; synced `aprf-spec.json`. - Rewrote **change management** from template stub to hybrid Check + collectors; cleared `technologies` filter; synced `aprf-spec.json`. - Rewrote **prompt-engineering** from template stub to hybrid Check + collectors; cleared `technologies` filter; synced `aprf-spec.json`. - Rewrote **model-governance** from template stub to hybrid Check + collector; cleared cloud-only `technologies`; synced `aprf-spec.json`. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 50d4e6a..d0fecf5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -18,7 +18,7 @@ This repository is the **normative public home** for APRF: 1. Read [ARCHITECTURE.md](ARCHITECTURE.md) and [rfcs/0000-template.md](rfcs/0000-template.md). 2. For substantive normative changes (new Pillars, gate semantics, ID renumbers), open an RFC first. 3. For Check edits: follow the schema in `packages/aprf-engine/rules/_schema/rule.schema.json`. -4. Never reuse a published Check ID — deprecate with `replacedBy` instead. +4. Never reuse a published Check ID — deprecate with `replacedBy` instead. Before the first tagged release, M→R remaps may omit deprecated stubs only when documented in an RFC and [`id-gaps.md`](packages/aprf-engine/rules/_index/id-gaps.md). ## Local checks diff --git a/README.md b/README.md index 10ac20d..ba907bf 100644 --- a/README.md +++ b/README.md @@ -131,7 +131,7 @@ Platform-specific **detections** (scanners, collectors) stay in product/plugin r 5. If the generated catalog changed, commit `packages/aprf-engine/src/generated/catalog.ts`. 6. Open a PR — CI will re-validate and fail on catalog drift. -Deprecate with `status: deprecated`, `replacedBy`, and `deprecationNote` — never reuse IDs. Numbering gaps are intentional — see [`packages/aprf-engine/rules/_index/id-gaps.md`](packages/aprf-engine/rules/_index/id-gaps.md). +Deprecate with `status: deprecated`, `replacedBy`, and `deprecationNote` — never reuse IDs. Numbering gaps are intentional — see [`packages/aprf-engine/rules/_index/id-gaps.md`](packages/aprf-engine/rules/_index/id-gaps.md). **Pre-release exception:** before the first tagged version, M→R remaps may remove IDs when an RFC and `id-gaps.md` record the change (see [APRF-RFC-0002](rfcs/0002-incident-readiness-mandatory-to-recommended.md)). ## Continuous integration diff --git a/package.json b/package.json index c24ad6d..f5599f7 100644 --- a/package.json +++ b/package.json @@ -67,6 +67,12 @@ "aprf:ai-canary-progressive-delivery": "npx tsx skills/aprf-auditor/collectors/runner.ts --plugins ai-canary-progressive-delivery", "aprf:env-parity-model-tool-catalog": "npx tsx skills/aprf-auditor/collectors/runner.ts --plugins env-parity-model-tool-catalog", "aprf:embedding-index-migration": "npx tsx skills/aprf-auditor/collectors/runner.ts --plugins embedding-index-migration", + "aprf:incident-playbooks": "npx tsx skills/aprf-auditor/collectors/runner.ts --plugins incident-playbooks", + "aprf:ai-containment-drill": "npx tsx skills/aprf-auditor/collectors/runner.ts --plugins ai-containment-drill", + "aprf:post-incident-aprf-actions": "npx tsx skills/aprf-auditor/collectors/runner.ts --plugins post-incident-aprf-actions", + "aprf:ai-incident-tabletop": "npx tsx skills/aprf-auditor/collectors/runner.ts --plugins ai-incident-tabletop", + "aprf:ai-safety-quality-alerts": "npx tsx skills/aprf-auditor/collectors/runner.ts --plugins ai-safety-quality-alerts", + "aprf:ai-customer-notification-criteria": "npx tsx skills/aprf-auditor/collectors/runner.ts --plugins ai-customer-notification-criteria", "aprf:cost-alerts": "npx tsx skills/aprf-auditor/collectors/runner.ts --plugins ai-cost-alerts", "aprf:retry-amplification": "npx tsx skills/aprf-auditor/collectors/runner.ts --plugins ai-retry-amplification", "aprf:prompt-cache": "npx tsx skills/aprf-auditor/collectors/runner.ts --plugins ai-prompt-cache", @@ -111,7 +117,7 @@ "aprf:report-html": "npx tsx skills/aprf-auditor/scripts/render-html-report.ts", "aprf:verify-html": "npx tsx skills/aprf-auditor/scripts/verify-html-report.ts", "test:unit": "npm run test:unit -w @stackrail-io/aprf-engine && npm run test:unit -w @stackrail-io/aprf-framework-definition", - "test:auditor-skill": "npx tsx skills/aprf-auditor/tests/validate-fixture.ts && npx tsx skills/aprf-auditor/tests/collectors-smoke.ts && npx tsx skills/aprf-auditor/tests/http-auth-probe-smoke.ts && npx tsx skills/aprf-auditor/tests/mcp-s2s-inventory-smoke.ts && npx tsx skills/aprf-auditor/tests/authz-entry-tests-smoke.ts && npx tsx skills/aprf-auditor/tests/cross-tenant-tests-smoke.ts && npx tsx skills/aprf-auditor/tests/secrets-hygiene-smoke.ts && npx tsx skills/aprf-auditor/tests/secret-redaction-smoke.ts && npx tsx skills/aprf-auditor/tests/injection-policy-gate-smoke.ts && npx tsx skills/aprf-auditor/tests/agent-charter-inventory-smoke.ts && npx tsx skills/aprf-auditor/tests/agent-loop-limits-smoke.ts && npx tsx skills/aprf-auditor/tests/agent-kill-switch-smoke.ts && npx tsx skills/aprf-auditor/tests/agent-goal-policy-smoke.ts && npx tsx skills/aprf-auditor/tests/agent-sandbox-sim-smoke.ts && npx tsx skills/aprf-auditor/tests/agent-raci-ownership-smoke.ts && npx tsx skills/aprf-auditor/tests/human-approval-smoke.ts && npx tsx skills/aprf-auditor/tests/ai-spend-limits-smoke.ts && npx tsx skills/aprf-auditor/tests/context-budget-smoke.ts && npx tsx skills/aprf-auditor/tests/context-source-acl-smoke.ts && npx tsx skills/aprf-auditor/tests/context-sensitive-inclusion-smoke.ts && npx tsx skills/aprf-auditor/tests/context-budget-monitoring-smoke.ts && npx tsx skills/aprf-auditor/tests/context-compaction-evals-smoke.ts && npx tsx skills/aprf-auditor/tests/context-structured-blocks-smoke.ts && npx tsx skills/aprf-auditor/tests/eval-suite-ci-smoke.ts && npx tsx skills/aprf-auditor/tests/eval-release-gates-smoke.ts && npx tsx skills/aprf-auditor/tests/eval-online-signals-smoke.ts && npx tsx skills/aprf-auditor/tests/eval-shadow-cutover-smoke.ts && npx tsx skills/aprf-auditor/tests/eval-track-catalog-smoke.ts && npx tsx skills/aprf-auditor/tests/eval-human-review-smoke.ts && npx tsx skills/aprf-auditor/tests/model-pin-config-smoke.ts && npx tsx skills/aprf-auditor/tests/model-inventory-smoke.ts && npx tsx skills/aprf-auditor/tests/model-promotion-eval-smoke.ts && npx tsx skills/aprf-auditor/tests/model-deprecation-sunset-smoke.ts && npx tsx skills/aprf-auditor/tests/model-capability-allowlist-smoke.ts && npx tsx skills/aprf-auditor/tests/model-license-provenance-smoke.ts && npx tsx skills/aprf-auditor/tests/prompt-version-registry-smoke.ts && npx tsx skills/aprf-auditor/tests/prompt-change-review-eval-smoke.ts && npx tsx skills/aprf-auditor/tests/prompt-rollback-smoke.ts && npx tsx skills/aprf-auditor/tests/prompt-template-hygiene-smoke.ts && npx tsx skills/aprf-auditor/tests/prompt-lint-ci-smoke.ts && npx tsx skills/aprf-auditor/tests/prompt-ab-shadow-eval-smoke.ts && npx tsx skills/aprf-auditor/tests/prompt-model-version-retention-smoke.ts && npx tsx skills/aprf-auditor/tests/rollback-runbook-smoke.ts && npx tsx skills/aprf-auditor/tests/rollback-drill-smoke.ts && npx tsx skills/aprf-auditor/tests/quality-slo-auto-rollback-smoke.ts && npx tsx skills/aprf-auditor/tests/one-click-ai-rollback-smoke.ts && npx tsx skills/aprf-auditor/tests/agent-behavior-feature-flags-smoke.ts && npx tsx skills/aprf-auditor/tests/ai-artifact-promotion-path-smoke.ts && npx tsx skills/aprf-auditor/tests/ai-artifact-change-records-smoke.ts && npx tsx skills/aprf-auditor/tests/ai-config-as-code-smoke.ts && npx tsx skills/aprf-auditor/tests/ai-canary-progressive-delivery-smoke.ts && npx tsx skills/aprf-auditor/tests/env-parity-model-tool-catalog-smoke.ts && npx tsx skills/aprf-auditor/tests/embedding-index-migration-smoke.ts && npx tsx skills/aprf-auditor/tests/ai-cost-alerts-smoke.ts && npx tsx skills/aprf-auditor/tests/ai-retry-amplification-smoke.ts && npx tsx skills/aprf-auditor/tests/ai-prompt-cache-smoke.ts && npx tsx skills/aprf-auditor/tests/ai-model-routing-smoke.ts && npx tsx skills/aprf-auditor/tests/ai-finops-unit-economics-smoke.ts && npx tsx skills/aprf-auditor/tests/platform-golden-path-smoke.ts && npx tsx skills/aprf-auditor/tests/platform-ai-pipeline-gates-smoke.ts && npx tsx skills/aprf-auditor/tests/platform-ownership-support-smoke.ts && npx tsx skills/aprf-auditor/tests/platform-scaffolding-templates-smoke.ts && npx tsx skills/aprf-auditor/tests/platform-inner-loop-evals-smoke.ts && npx tsx skills/aprf-auditor/tests/platform-dx-metrics-smoke.ts && npx tsx skills/aprf-auditor/tests/rag-corpus-governance-smoke.ts && npx tsx skills/aprf-auditor/tests/dataset-provenance-governance-smoke.ts && npx tsx skills/aprf-auditor/tests/feedback-promotion-governance-smoke.ts && npx tsx skills/aprf-auditor/tests/corpus-freshness-metrics-smoke.ts && npx tsx skills/aprf-auditor/tests/train-serve-skew-monitor-smoke.ts && npx tsx skills/aprf-auditor/tests/dataset-cards-registry-smoke.ts && npx tsx skills/aprf-auditor/tests/model-payload-classification-smoke.ts && npx tsx skills/aprf-auditor/tests/model-payload-redaction-smoke.ts && npx tsx skills/aprf-auditor/tests/vendor-model-terms-smoke.ts && npx tsx skills/aprf-auditor/tests/ai-deletion-export-smoke.ts && npx tsx skills/aprf-auditor/tests/ai-residency-routing-smoke.ts && npx tsx skills/aprf-auditor/tests/ai-dpia-smoke.ts && npx tsx skills/aprf-auditor/tests/memory-isolation-smoke.ts && npx tsx skills/aprf-auditor/tests/memory-retention-smoke.ts && npx tsx skills/aprf-auditor/tests/memory-write-policy-smoke.ts && npx tsx skills/aprf-auditor/tests/memory-integrity-smoke.ts && npx tsx skills/aprf-auditor/tests/memory-poisoning-evals-smoke.ts && npx tsx skills/aprf-auditor/tests/memory-promotion-architecture-smoke.ts && npx tsx skills/aprf-auditor/tests/ai-obligations-register-smoke.ts && npx tsx skills/aprf-auditor/tests/ai-control-evidence-matrix-smoke.ts && npx tsx skills/aprf-auditor/tests/ai-control-plane-audit-logs-smoke.ts && npx tsx skills/aprf-auditor/tests/ai-control-testing-smoke.ts && npx tsx skills/aprf-auditor/tests/ai-trust-documentation-smoke.ts && npx tsx skills/aprf-auditor/tests/ai-independent-assessment-smoke.ts && npx tsx skills/aprf-auditor/tests/ai-acceptable-use-policy-smoke.ts && npx tsx skills/aprf-auditor/tests/ai-domain-ownership-smoke.ts && npx tsx skills/aprf-auditor/tests/ai-risk-acceptance-smoke.ts && npx tsx skills/aprf-auditor/tests/ai-leadership-review-smoke.ts && npx tsx skills/aprf-auditor/tests/ai-improvement-backlog-smoke.ts && npx tsx skills/aprf-auditor/tests/ai-org-aprf-sampling-smoke.ts && npx tsx skills/aprf-auditor/tests/a2a-peer-auth-smoke.ts && npx tsx skills/aprf-auditor/tests/render-html-smoke.ts", + "test:auditor-skill": "npx tsx skills/aprf-auditor/tests/validate-fixture.ts && npx tsx skills/aprf-auditor/tests/collectors-smoke.ts && npx tsx skills/aprf-auditor/tests/http-auth-probe-smoke.ts && npx tsx skills/aprf-auditor/tests/mcp-s2s-inventory-smoke.ts && npx tsx skills/aprf-auditor/tests/authz-entry-tests-smoke.ts && npx tsx skills/aprf-auditor/tests/cross-tenant-tests-smoke.ts && npx tsx skills/aprf-auditor/tests/secrets-hygiene-smoke.ts && npx tsx skills/aprf-auditor/tests/secret-redaction-smoke.ts && npx tsx skills/aprf-auditor/tests/injection-policy-gate-smoke.ts && npx tsx skills/aprf-auditor/tests/agent-charter-inventory-smoke.ts && npx tsx skills/aprf-auditor/tests/agent-loop-limits-smoke.ts && npx tsx skills/aprf-auditor/tests/agent-kill-switch-smoke.ts && npx tsx skills/aprf-auditor/tests/agent-goal-policy-smoke.ts && npx tsx skills/aprf-auditor/tests/agent-sandbox-sim-smoke.ts && npx tsx skills/aprf-auditor/tests/agent-raci-ownership-smoke.ts && npx tsx skills/aprf-auditor/tests/human-approval-smoke.ts && npx tsx skills/aprf-auditor/tests/ai-spend-limits-smoke.ts && npx tsx skills/aprf-auditor/tests/context-budget-smoke.ts && npx tsx skills/aprf-auditor/tests/context-source-acl-smoke.ts && npx tsx skills/aprf-auditor/tests/context-sensitive-inclusion-smoke.ts && npx tsx skills/aprf-auditor/tests/context-budget-monitoring-smoke.ts && npx tsx skills/aprf-auditor/tests/context-compaction-evals-smoke.ts && npx tsx skills/aprf-auditor/tests/context-structured-blocks-smoke.ts && npx tsx skills/aprf-auditor/tests/eval-suite-ci-smoke.ts && npx tsx skills/aprf-auditor/tests/eval-release-gates-smoke.ts && npx tsx skills/aprf-auditor/tests/eval-online-signals-smoke.ts && npx tsx skills/aprf-auditor/tests/eval-shadow-cutover-smoke.ts && npx tsx skills/aprf-auditor/tests/eval-track-catalog-smoke.ts && npx tsx skills/aprf-auditor/tests/eval-human-review-smoke.ts && npx tsx skills/aprf-auditor/tests/model-pin-config-smoke.ts && npx tsx skills/aprf-auditor/tests/model-inventory-smoke.ts && npx tsx skills/aprf-auditor/tests/model-promotion-eval-smoke.ts && npx tsx skills/aprf-auditor/tests/model-deprecation-sunset-smoke.ts && npx tsx skills/aprf-auditor/tests/model-capability-allowlist-smoke.ts && npx tsx skills/aprf-auditor/tests/model-license-provenance-smoke.ts && npx tsx skills/aprf-auditor/tests/prompt-version-registry-smoke.ts && npx tsx skills/aprf-auditor/tests/prompt-change-review-eval-smoke.ts && npx tsx skills/aprf-auditor/tests/prompt-rollback-smoke.ts && npx tsx skills/aprf-auditor/tests/prompt-template-hygiene-smoke.ts && npx tsx skills/aprf-auditor/tests/prompt-lint-ci-smoke.ts && npx tsx skills/aprf-auditor/tests/prompt-ab-shadow-eval-smoke.ts && npx tsx skills/aprf-auditor/tests/prompt-model-version-retention-smoke.ts && npx tsx skills/aprf-auditor/tests/rollback-runbook-smoke.ts && npx tsx skills/aprf-auditor/tests/rollback-drill-smoke.ts && npx tsx skills/aprf-auditor/tests/quality-slo-auto-rollback-smoke.ts && npx tsx skills/aprf-auditor/tests/one-click-ai-rollback-smoke.ts && npx tsx skills/aprf-auditor/tests/agent-behavior-feature-flags-smoke.ts && npx tsx skills/aprf-auditor/tests/ai-artifact-promotion-path-smoke.ts && npx tsx skills/aprf-auditor/tests/ai-artifact-change-records-smoke.ts && npx tsx skills/aprf-auditor/tests/ai-config-as-code-smoke.ts && npx tsx skills/aprf-auditor/tests/ai-canary-progressive-delivery-smoke.ts && npx tsx skills/aprf-auditor/tests/env-parity-model-tool-catalog-smoke.ts && npx tsx skills/aprf-auditor/tests/embedding-index-migration-smoke.ts && npx tsx skills/aprf-auditor/tests/incident-playbooks-smoke.ts && npx tsx skills/aprf-auditor/tests/ai-containment-drill-smoke.ts && npx tsx skills/aprf-auditor/tests/post-incident-aprf-actions-smoke.ts && npx tsx skills/aprf-auditor/tests/ai-incident-tabletop-smoke.ts && npx tsx skills/aprf-auditor/tests/ai-safety-quality-alerts-smoke.ts && npx tsx skills/aprf-auditor/tests/ai-customer-notification-criteria-smoke.ts && npx tsx skills/aprf-auditor/tests/ai-cost-alerts-smoke.ts && npx tsx skills/aprf-auditor/tests/ai-retry-amplification-smoke.ts && npx tsx skills/aprf-auditor/tests/ai-prompt-cache-smoke.ts && npx tsx skills/aprf-auditor/tests/ai-model-routing-smoke.ts && npx tsx skills/aprf-auditor/tests/ai-finops-unit-economics-smoke.ts && npx tsx skills/aprf-auditor/tests/platform-golden-path-smoke.ts && npx tsx skills/aprf-auditor/tests/platform-ai-pipeline-gates-smoke.ts && npx tsx skills/aprf-auditor/tests/platform-ownership-support-smoke.ts && npx tsx skills/aprf-auditor/tests/platform-scaffolding-templates-smoke.ts && npx tsx skills/aprf-auditor/tests/platform-inner-loop-evals-smoke.ts && npx tsx skills/aprf-auditor/tests/platform-dx-metrics-smoke.ts && npx tsx skills/aprf-auditor/tests/rag-corpus-governance-smoke.ts && npx tsx skills/aprf-auditor/tests/dataset-provenance-governance-smoke.ts && npx tsx skills/aprf-auditor/tests/feedback-promotion-governance-smoke.ts && npx tsx skills/aprf-auditor/tests/corpus-freshness-metrics-smoke.ts && npx tsx skills/aprf-auditor/tests/train-serve-skew-monitor-smoke.ts && npx tsx skills/aprf-auditor/tests/dataset-cards-registry-smoke.ts && npx tsx skills/aprf-auditor/tests/model-payload-classification-smoke.ts && npx tsx skills/aprf-auditor/tests/model-payload-redaction-smoke.ts && npx tsx skills/aprf-auditor/tests/vendor-model-terms-smoke.ts && npx tsx skills/aprf-auditor/tests/ai-deletion-export-smoke.ts && npx tsx skills/aprf-auditor/tests/ai-residency-routing-smoke.ts && npx tsx skills/aprf-auditor/tests/ai-dpia-smoke.ts && npx tsx skills/aprf-auditor/tests/memory-isolation-smoke.ts && npx tsx skills/aprf-auditor/tests/memory-retention-smoke.ts && npx tsx skills/aprf-auditor/tests/memory-write-policy-smoke.ts && npx tsx skills/aprf-auditor/tests/memory-integrity-smoke.ts && npx tsx skills/aprf-auditor/tests/memory-poisoning-evals-smoke.ts && npx tsx skills/aprf-auditor/tests/memory-promotion-architecture-smoke.ts && npx tsx skills/aprf-auditor/tests/ai-obligations-register-smoke.ts && npx tsx skills/aprf-auditor/tests/ai-control-evidence-matrix-smoke.ts && npx tsx skills/aprf-auditor/tests/ai-control-plane-audit-logs-smoke.ts && npx tsx skills/aprf-auditor/tests/ai-control-testing-smoke.ts && npx tsx skills/aprf-auditor/tests/ai-trust-documentation-smoke.ts && npx tsx skills/aprf-auditor/tests/ai-independent-assessment-smoke.ts && npx tsx skills/aprf-auditor/tests/ai-acceptable-use-policy-smoke.ts && npx tsx skills/aprf-auditor/tests/ai-domain-ownership-smoke.ts && npx tsx skills/aprf-auditor/tests/ai-risk-acceptance-smoke.ts && npx tsx skills/aprf-auditor/tests/ai-leadership-review-smoke.ts && npx tsx skills/aprf-auditor/tests/ai-improvement-backlog-smoke.ts && npx tsx skills/aprf-auditor/tests/ai-org-aprf-sampling-smoke.ts && npx tsx skills/aprf-auditor/tests/a2a-peer-auth-smoke.ts && npx tsx skills/aprf-auditor/tests/render-html-smoke.ts", "build": "npm run build -w @stackrail-io/aprf-engine && npm run build -w @stackrail-io/aprf-framework-definition", "validate": "npm run aprf:validate && npm run aprf:catalog && npm run aprf:integrity && npm run test:unit && npm run test:auditor-skill", "publish:packages": "npm run build && npm publish -w @stackrail-io/aprf-framework-definition --access public && npm publish -w @stackrail-io/aprf-engine --access public" diff --git a/packages/aprf-engine/rules/_index/id-gaps.md b/packages/aprf-engine/rules/_index/id-gaps.md new file mode 100644 index 0000000..31120b6 --- /dev/null +++ b/packages/aprf-engine/rules/_index/id-gaps.md @@ -0,0 +1,19 @@ +# Check ID gaps + +Numbering gaps in Check IDs are **intentional**. Do not fill gaps by reusing retired IDs. + +## Policy + +- After a **tagged release**, never delete a published Check ID. Deprecate with `status: deprecated`, `replacedBy`, and `deprecationNote` (N−1 MINOR support window). See `CONTRIBUTING.md` and `ARCHITECTURE.md`. +- **Mandatory → recommended** always needs an RFC (`ARCHITECTURE.md`). + +## Pre-release exception (before first tagged version) + +APRF `0.10.x` is a working draft; **no release versions have been tagged yet**. For this window only, stewards may remove a Check ID when remapping M→R **if** an RFC records the remapping and this file lists the gap. + +| Retired ID | Replaced by | RFC | Notes | +| --- | --- | --- | --- | +| `INC-M3` | `INC-R2` | [APRF-RFC-0002](../../../rfcs/0002-incident-readiness-mandatory-to-recommended.md) | Post-incident APRF-pillar actions; was never on Core/Regulated | +| `INC-M4` | `INC-R4` | [APRF-RFC-0002](../../../rfcs/0002-incident-readiness-mandatory-to-recommended.md) | AI-focused tabletop ≤180d; removed from Regulated mandatories | + +**After the first tagged release**, this exception closes. New demotions must keep deprecated stubs (pattern: `INF-R1` → `SCI-R1`). diff --git a/packages/aprf-engine/rules/by-domain/reliability/incident-readiness/INC-M1.yaml b/packages/aprf-engine/rules/by-domain/reliability/incident-readiness/INC-M1.yaml index a11ad68..f56a877 100644 --- a/packages/aprf-engine/rules/by-domain/reliability/incident-readiness/INC-M1.yaml +++ b/packages/aprf-engine/rules/by-domain/reliability/incident-readiness/INC-M1.yaml @@ -1,39 +1,56 @@ id: "INC-M1" category: "incident-readiness" title: "AI-specific incident playbooks must exist for abuse, leakage, bad actions, and provider outage" -description: "AI-specific incident playbooks shall exist for abuse, leakage, bad actions, and provider outage" -whyItMatters: "AI-specific incident playbooks shall exist for abuse, leakage, bad actions, and - provider outage Failing this leaves a production gap against: Four playbooks present (abuse, - leakage, bad actions, provider outage), each with owner and review date ≤ 12 months" +description: > + Production AI systems must maintain incident playbooks for abuse, data + leakage, bad agent/tool actions, and provider outage—each with a named owner + and a review date within the last 12 months. +whyItMatters: > + Generic outage runbooks do not cover AI-specific failure modes. Without + owned, recently reviewed playbooks for abuse, leakage, bad actions, and + provider loss, on-call improvises under pressure and containment is slow or + incomplete. severity: "high" weight: 3 gate: "mandatory" -passCondition: "Four playbooks present (abuse, leakage, bad actions, provider outage), each with - owner and review date ≤ 12 months" +passCondition: > + Four playbooks present (abuse, leakage, bad actions, provider outage), each + with owner and review date ≤12 months (playbook evidence measuredAt ≤90 + days). evidenceRequired: - - "Playbook set covering the four scenarios with owners" + - "Playbook set covering abuse, leakage, bad actions, and provider outage with owners" + - "Review dates ≤12 months for each of the four playbooks" detection: capability: "hybrid" detectors: - id: "repo-incident-playbooks" - params: {} + params: + hint: > + Discover AI-specific incident playbooks for abuse, leakage, bad + actions, and provider outage, plus owner and review-date coverage. - id: "manual-attest" params: - hint: "Playbook set covering the four scenarios with owners" -manualVerification: "For this Check (AI-specific incident playbooks must exist for abuse, leakage, - bad actions, and provider outage): inspect current evidence for [Playbook set covering the four - scenarios with owners] and confirm the pass condition holds — Four playbooks present (abuse, - leakage, bad actions, provider outage), each with owner and review date ≤ 12 months" -falsePositiveGuidance: "(Incident Readiness): when automation and attestation disagree, prefer the - stricter outcome until reconciled. Waive only with owner, expiry, and which signal covers the - gap." + hint: > + If automation cannot prove coverage, attest all four playbooks exist + with owners and review dates ≤12 months (measuredAt ≤90 days). +manualVerification: > + 1) Confirm a production AI system is in scope. If none, score NOT_APPLICABLE. + 2) Confirm playbooks exist for abuse, leakage, bad actions, and provider + outage. 3) Confirm each has a named owner. 4) Confirm each was reviewed + within the last 12 months. 5) PASS only if all four + owners + fresh reviews + hold with measuredAt ≤90 days. INC-M2 containment drills alone do not prove + the four playbooks. Generic SRE incident docs that omit AI scenarios do not + satisfy. A single combined doc may pass only if it clearly covers all four + scenarios with owners and review dates. +falsePositiveGuidance: > + Do not pass placeholder stubs without owners or review dates. Do not pass + reviews older than 12 months. Do not pass a provider-outage playbook that only + covers non-AI infrastructure. Named exceptions need owner and expiry ≤90 days. recommendedFixes: - - "Implement and operationalize: AI-specific incident playbooks must exist for abuse, leakage, bad - actions, and provider outage" - - "Retain evidence artifacts required by this Check, starting with: Playbook set covering the four - scenarios with owners" - - "Wire or verify detectors declared on this Check so automation matches the pass condition" - - "Block release (or open a time-boxed waiver with owner and expiry) until this Check passes" + - "Author and own playbooks for abuse, leakage, bad actions, and provider outage" + - "Review each playbook on a ≤12-month cadence and retain dates" + - "Retain evidence under imports/incident-playbooks/" + - "Time-box gaps with owner and expiry ≤90 days" references: - title: "Google SRE — Managing Incidents" url: "https://sre.google/sre-book/managing-incidents/" @@ -41,17 +58,20 @@ references: url: "https://www.nist.gov/itl/ai-risk-management-framework" relatedRules: - "INC-M2" - - "INC-M3" - - "INC-M4" + - "INC-R2" + - "INC-R4" - "INC-R1" - "INC-R3" + - "AGN-M3" + - "CHG-M2" tags: - "incident-readiness" - "mandatory" - "hybrid" + - "playbooks" + - "ai-incidents" applicability: - technologies: - - "cicd" + technologies: [] minCriticality: 2 requiredFromLevel: 3 status: "active" diff --git a/packages/aprf-engine/rules/by-domain/reliability/incident-readiness/INC-M2.yaml b/packages/aprf-engine/rules/by-domain/reliability/incident-readiness/INC-M2.yaml index e984728..882ee82 100644 --- a/packages/aprf-engine/rules/by-domain/reliability/incident-readiness/INC-M2.yaml +++ b/packages/aprf-engine/rules/by-domain/reliability/incident-readiness/INC-M2.yaml @@ -1,40 +1,57 @@ id: "INC-M2" category: "incident-readiness" -title: "On-call must be able to execute containment: pause agents, disable tools, roll back - prompts/models" -description: "On-call shall be able to execute containment: pause agents, disable tools, roll back - prompts/models" -whyItMatters: "On-call shall be able to execute containment: pause agents, disable tools, roll back - prompts/models Failing this leaves a production gap against: Drill in last 90 days successfully - demonstrated pause agents, disable tools, and roll back prompt/model within documented time - budgets" +title: "On-call must be able to execute containment: pause agents, disable tools, roll back prompts/models" +description: > + On-call must be able to pause agents, disable tools, and roll back prompts or + models, proven by a drill in the last 90 days completed within documented + time budgets. +whyItMatters: > + Playbooks alone do not stop a live AI incident. On-call needs practiced + containment—pause, disable tools, roll back—within known time budgets so + blast radius shrinks before customers absorb more damage. severity: "high" weight: 3 gate: "mandatory" -passCondition: "Drill in last 90 days successfully demonstrated pause agents, disable tools, and - roll back prompt/model within documented time budgets" +passCondition: > + Drill in last 90 days successfully demonstrated pause agents, disable tools, + and roll back prompt/model within documented time budgets (drill evidence + measuredAt ≤90 days). evidenceRequired: - - "Containment runbook + drill record exercising pause, disable, and rollback" + - "Containment runbook covering pause agents, disable tools, and prompt/model rollback" + - "Drill record ≤90 days showing all three actions within documented time budgets" detection: - capability: "manual" + capability: "hybrid" detectors: + - id: "repo-ai-containment-drill" + params: + hint: > + Discover containment runbooks and drill evidence for pause agents, + disable tools, and prompt/model rollback within time budgets. - id: "manual-attest" params: - hint: "Containment runbook + drill record exercising pause, disable, and rollback" -manualVerification: "For this Check (On-call must be able to execute containment: pause agents, - disable tools, roll back prompts/models): inspect current evidence for [Containment runbook + - drill record exercising pause, disable, and rollback] and confirm the pass condition holds — Drill - in last 90 days successfully demonstrated pause agents, disable tools, and roll back prompt/model - within documented time budgets" -falsePositiveGuidance: "(Incident Readiness): re-verify against a current artifact for this specific - Check , not a sibling control. Document named exceptions with owner and expiry." + hint: > + If automation cannot prove coverage, attest a drill ≤90 days + demonstrated pause agents, disable tools, and prompt/model rollback + within documented time budgets (measuredAt ≤90 days). +manualVerification: > + 1) Confirm production agents, tools, or prompt/model release units exist. If + none, score NOT_APPLICABLE. 2) Confirm a containment runbook covers pause + agents, disable tools, and prompt/model rollback with documented time + budgets. 3) Confirm a drill in the last 90 days successfully exercised all + three within those budgets. 4) PASS only if drill + all three actions + time + budgets hold with measuredAt ≤90 days. INC-M1 playbooks alone do not prove + executed containment. AGN kill-switch Checks alone do not prove tool disable + and prompt/model rollback. CHG rollback Checks alone do not prove pause + + disable. +falsePositiveGuidance: > + Do not pass tabletop discussion without executing pause/disable/rollback. Do + not pass a drill older than 90 days. Do not pass actions that missed + documented time budgets. Named exceptions need owner and expiry ≤90 days. recommendedFixes: - - "Implement and operationalize: On-call must be able to execute containment: pause agents, - disable tools, roll back prompts/models" - - "Retain evidence artifacts required by this Check, starting with: Containment runbook + drill - record exercising pause, disable, and rollback" - - "Schedule recurring manual verification for this Check with a named owner and retained report" - - "Block release (or open a time-boxed waiver with owner and expiry) until this Check passes" + - "Document containment steps for pause, disable tools, and prompt/model rollback with time budgets" + - "Run and retain a ≤90-day drill exercising all three within budgets" + - "Retain evidence under imports/ai-containment-drill/" + - "Time-box gaps with owner and expiry ≤90 days" references: - title: "Google SRE — Managing Incidents" url: "https://sre.google/sre-book/managing-incidents/" @@ -42,17 +59,22 @@ references: url: "https://www.nist.gov/itl/ai-risk-management-framework" relatedRules: - "INC-M1" - - "INC-M3" - - "INC-M4" + - "INC-R2" + - "INC-R4" - "INC-R1" - "INC-R3" + - "AGN-M3" + - "CHG-M2" + - "CHG-M3" + - "PRM-M3" tags: - "incident-readiness" - "mandatory" - - "manual" + - "hybrid" + - "containment" + - "drill" applicability: - technologies: - - "cicd" + technologies: [] minCriticality: 2 requiredFromLevel: 3 status: "active" diff --git a/packages/aprf-engine/rules/by-domain/reliability/incident-readiness/INC-M3.yaml b/packages/aprf-engine/rules/by-domain/reliability/incident-readiness/INC-M3.yaml deleted file mode 100644 index 1f72155..0000000 --- a/packages/aprf-engine/rules/by-domain/reliability/incident-readiness/INC-M3.yaml +++ /dev/null @@ -1,52 +0,0 @@ -id: "INC-M3" -category: "incident-readiness" -title: "Post-incident reviews must produce tracked actions against APRF pillars" -description: "Post-incident reviews shall produce tracked actions against APRF pillars" -whyItMatters: "Post-incident reviews shall produce tracked actions against APRF pillars Failing this - leaves a production gap against: 100% of SEV-eligible AI incidents in last 90 days have a review - with ≥1 tracked action mapped to an APRF pillar or explicit “no action” rationale" -severity: "high" -weight: 3 -gate: "mandatory" -passCondition: "100% of SEV-eligible AI incidents in last 90 days have a review with ≥1 tracked - action mapped to an APRF pillar or explicit “no action” rationale" -evidenceRequired: - - "Post-incident review template + sample reviews with linked actions" -detection: - capability: "manual" - detectors: [] -manualVerification: "For this Check (Post-incident reviews must produce tracked actions against APRF - pillars): inspect current evidence for [Post-incident review template + sample reviews with linked - actions] and confirm the pass condition holds — 100% of SEV-eligible AI incidents in last 90 days - have a review with ≥1 tracked action mapped to an APRF pillar or explicit “no action” rationale" -falsePositiveGuidance: "(Incident Readiness): re-verify against a current artifact for this specific - Check , not a sibling control. Document named exceptions with owner and expiry." -recommendedFixes: - - "Implement and operationalize: Post-incident reviews must produce tracked actions against APRF - pillars" - - "Retain evidence artifacts required by this Check, starting with: Post-incident review template - + sample reviews with linked actions" - - "Schedule recurring manual verification for this Check with a named owner and retained report" - - "Block release (or open a time-boxed waiver with owner and expiry) until this Check passes" -references: - - title: "Google SRE — Managing Incidents" - url: "https://sre.google/sre-book/managing-incidents/" - - title: "NIST AI RMF — Manage" - url: "https://www.nist.gov/itl/ai-risk-management-framework" -relatedRules: - - "INC-M1" - - "INC-M2" - - "INC-M4" - - "INC-R1" - - "INC-R3" -tags: - - "incident-readiness" - - "mandatory" - - "manual" -applicability: - technologies: - - "cicd" - minCriticality: 2 - requiredFromLevel: 3 -status: "active" -introducedIn: "0.10.0" diff --git a/packages/aprf-engine/rules/by-domain/reliability/incident-readiness/INC-M4.yaml b/packages/aprf-engine/rules/by-domain/reliability/incident-readiness/INC-M4.yaml deleted file mode 100644 index f4c9eee..0000000 --- a/packages/aprf-engine/rules/by-domain/reliability/incident-readiness/INC-M4.yaml +++ /dev/null @@ -1,53 +0,0 @@ -id: "INC-M4" -category: "incident-readiness" -title: "Regular tabletop exercises must cover AI-specific incidents" -description: "Regular tabletop exercises shall cover AI-specific incidents" -whyItMatters: "Regular tabletop exercises shall cover AI-specific incidents Failing this leaves a - production gap against: PASS if an AI-focused tabletop completed ≤180 days with retained actions - and owners" -severity: "high" -weight: 3 -gate: "mandatory" -passCondition: "PASS if an AI-focused tabletop completed ≤180 days with retained actions and owners" -evidenceRequired: - - "Tabletop plan + dated after-action report for an AI incident scenario" -detection: - capability: "manual" - detectors: - - id: "manual-attest" - params: - hint: "Tabletop plan + dated after-action report for an AI incident scenario" -manualVerification: "For this Check (Regular tabletop exercises must cover AI-specific incidents): - inspect current evidence for [Tabletop plan + dated after-action report for an AI incident - scenario] and confirm the pass condition holds — PASS if an AI-focused tabletop completed ≤180 - days with retained actions and owners" -falsePositiveGuidance: "(Incident Readiness): re-verify against a current artifact for this specific - Check , not a sibling control. Document named exceptions with owner and expiry." -recommendedFixes: - - "Implement and operationalize: Regular tabletop exercises must cover AI-specific incidents" - - "Retain evidence artifacts required by this Check, starting with: Tabletop plan + dated - after-action report for an AI incident scenario" - - "Schedule recurring manual verification for this Check with a named owner and retained report" - - "Block release (or open a time-boxed waiver with owner and expiry) until this Check passes" -references: - - title: "Google SRE — Managing Incidents" - url: "https://sre.google/sre-book/managing-incidents/" - - title: "NIST AI RMF — Manage" - url: "https://www.nist.gov/itl/ai-risk-management-framework" -relatedRules: - - "INC-M1" - - "INC-M2" - - "INC-M3" - - "INC-R1" - - "INC-R3" -tags: - - "incident-readiness" - - "mandatory" - - "manual" -applicability: - technologies: - - "cicd" - minCriticality: 3 - requiredFromLevel: 4 -status: "active" -introducedIn: "0.10.0" diff --git a/packages/aprf-engine/rules/by-domain/reliability/incident-readiness/INC-R1.yaml b/packages/aprf-engine/rules/by-domain/reliability/incident-readiness/INC-R1.yaml index 8baf594..23beef2 100644 --- a/packages/aprf-engine/rules/by-domain/reliability/incident-readiness/INC-R1.yaml +++ b/packages/aprf-engine/rules/by-domain/reliability/incident-readiness/INC-R1.yaml @@ -1,59 +1,76 @@ id: "INC-R1" category: "incident-readiness" -title: "Production systems should have page-worthy alerts for safety and quality signals, not only infra" -description: "Page-worthy alerts for safety and quality signals, not only infra" -whyItMatters: "Page-worthy alerts for safety and quality signals, not only infra Failing this leaves - a production gap against: At least two non-infra signals (e.g. refusal-rate spike, eval-score - drop, toxicity/jailbreak hit rate) page an on-call; each has a documented threshold and owner; - policy reviewed ≤90 days ago" +title: "Production systems should page on safety and quality signals, not only infra" +description: > + On-call should receive page-worthy alerts for at least two non-infra AI + safety/quality signals, each with a documented threshold and owner, and a + policy reviewed within 90 days. +whyItMatters: > + Infra pages alone miss AI failure modes—refusal spikes, eval drops, + toxicity/jailbreak hits—until customers or regulators notice. Paging with + owned thresholds turns those signals into incidents before blast radius grows. severity: "high" weight: 3 gate: "recommended" -passCondition: "At least two non-infra signals (e.g. refusal-rate spike, eval-score drop, - toxicity/jailbreak hit rate) page an on-call; each has a documented threshold and owner; policy - reviewed ≤90 days ago" +passCondition: > + At least two non-infra signals (e.g. refusal-rate spike, eval-score drop, + toxicity/jailbreak hit rate) page an on-call; each has a documented threshold + and owner; policy reviewed ≤90 days ago (alert evidence measuredAt ≤90 days). evidenceRequired: - - "On-call alert policy export listing safety/quality pages + last 90 days of triggered incidents - (or drill tickets)" + - "On-call alert policy export listing safety/quality pages" + - "Last 90 days of triggered incidents or drill tickets for those pages" detection: - capability: "manual" + capability: "hybrid" detectors: + - id: "repo-ai-safety-quality-alerts" + params: + hint: > + Discover on-call alert policies for non-infra AI safety/quality + signals with thresholds and owners. - id: "manual-attest" params: - hint: "On-call alert policy export listing safety/quality pages + last 90 days of triggered - incidents (or drill tickets)" -manualVerification: "For this Check (Page-worthy alerts for safety and quality signals, not only - infra): inspect current evidence for [On-call alert policy export listing safety/quality pages + - last 90 days of triggered incidents (or drill tickets)] and confirm the pass condition holds — At - least two non-infra signals (e.g. refusal-rate spike, eval-score drop, toxicity/jailbreak hit - rate) page an on-call; each has a documented threshold and owner; policy reviewed ≤90 days ago" -falsePositiveGuidance: "(Incident Readiness): re-verify against a current artifact for this specific - Check , not a sibling control. Document named exceptions with owner and expiry." + hint: > + If automation cannot prove coverage, attest ≥2 non-infra safety/ + quality signals page on-call, each with threshold and owner, and + policy reviewed ≤90 days (measuredAt ≤90 days). +manualVerification: > + 1) Confirm a production AI system is in scope. If none, score NOT_APPLICABLE. + 2) Confirm an on-call alert policy lists ≥2 non-infra safety/quality signals + that page (e.g. refusal-rate spike, eval-score drop, toxicity/jailbreak hit + rate). 3) Confirm each signal has a documented threshold and owner. 4) + Confirm policy review ≤90 days ago (or triggered incidents/drill tickets in + last 90 days). 5) PASS only if all hold with measuredAt ≤90 days. Infra-only + CPU/memory/latency pages do not satisfy. COST spend alerts alone do not + satisfy. OBS dashboards without paging do not satisfy. +falsePositiveGuidance: > + Do not pass dashboards that do not page. Do not pass a single signal. Do not + pass thresholds without owners. Do not pass policy older than 90 days. Named + exceptions need owner and expiry ≤90 days. recommendedFixes: - - "Implement and operationalize: this Check: Page-worthy alerts for safety and quality signals, - not only infra" - - "Retain evidence artifacts required by this Check, starting with: On-call alert policy export - listing safety/quality pages + last 90 days of triggered incidents (or drill tickets)" - - "Schedule recurring manual verification for this Check with a named owner and retained report" - - "Block release (or open a time-boxed waiver with owner and expiry) until this Check passes" + - "Define ≥2 non-infra AI safety/quality paging signals with thresholds and owners" + - "Wire those signals to on-call and retain policy export + recent pages/drills" + - "Retain evidence under imports/ai-safety-quality-alerts/" + - "Time-box gaps with owner and expiry ≤90 days" references: - - title: "Google SRE — Managing Incidents" - url: "https://sre.google/sre-book/managing-incidents/" + - title: "Google SRE — Monitoring Distributed Systems" + url: "https://sre.google/sre-book/monitoring-distributed-systems/" - title: "NIST AI RMF — Manage" url: "https://www.nist.gov/itl/ai-risk-management-framework" relatedRules: - "INC-M1" - "INC-M2" - - "INC-M3" - - "INC-M4" + - "INC-R2" - "INC-R3" + - "INC-R4" + - "OBS-M1" tags: - "incident-readiness" - "recommended" - - "manual" + - "hybrid" + - "paging" + - "safety-quality" applicability: - technologies: - - "cicd" + technologies: [] minCriticality: 2 requiredFromLevel: 4 status: "active" diff --git a/packages/aprf-engine/rules/by-domain/reliability/incident-readiness/INC-R2.yaml b/packages/aprf-engine/rules/by-domain/reliability/incident-readiness/INC-R2.yaml new file mode 100644 index 0000000..1b25255 --- /dev/null +++ b/packages/aprf-engine/rules/by-domain/reliability/incident-readiness/INC-R2.yaml @@ -0,0 +1,80 @@ +id: "INC-R2" +category: "incident-readiness" +title: "Post-incident reviews should produce tracked actions against APRF pillars" +description: > + SEV-eligible AI incidents should receive a post-incident review with at least + one tracked action mapped to an APRF pillar—or an explicit no-action + rationale. +whyItMatters: > + Reviews without pillar-mapped actions do not harden the system. Tracking + remediation (or documenting why none is needed) turns incidents into APRF + control improvements instead of one-off war stories. +severity: "high" +weight: 3 +gate: "recommended" +passCondition: > + 100% of SEV-eligible AI incidents in last 90 days have a review with ≥1 + tracked action mapped to an APRF pillar or explicit “no action” rationale + (review evidence measuredAt ≤90 days). +evidenceRequired: + - "Post-incident review template requiring APRF pillar mapping" + - "Last-90-day coverage: SEV-eligible AI reviews with tracked actions or no-action rationale" +detection: + capability: "hybrid" + detectors: + - id: "repo-post-incident-aprf-actions" + params: + hint: > + Discover post-incident review templates and coverage showing + APRF-pillar-mapped actions (or no-action rationale). + - id: "manual-attest" + params: + hint: > + If automation cannot prove coverage, attest 100% of SEV-eligible AI + incidents in the last 90 days have a review with ≥1 tracked + APRF-pillar action or explicit no-action rationale (measuredAt ≤90 + days). Score N/A if sevEligibleIncidentCount=0. +manualVerification: > + 1) Confirm SEV-eligible AI incidents exist in the last 90 days. If none, + score NOT_APPLICABLE with retained attestation. 2) Confirm a post-incident + review template requires APRF pillar mapping (or equivalent). 3) Confirm + 100% of those incidents have a review with ≥1 tracked action mapped to an + APRF pillar or an explicit no-action rationale. 4) PASS only if coverage + holds with measuredAt ≤90 days. INC-M1/M2 playbooks and containment alone + do not prove pillar-mapped actions. Generic postmortems without APRF pillar + linkage do not satisfy. +falsePositiveGuidance: > + Do not pass reviews that list actions without owners or tickets. Do not pass + “no action” without a written rationale. Do not pass coverage older than 90 + days. Named exceptions need owner and expiry ≤90 days. +recommendedFixes: + - "Require APRF pillar mapping (or no-action rationale) on every SEV-eligible AI review" + - "Track actions to tickets/owners and retain last-90-day coverage" + - "Retain evidence under imports/post-incident-aprf-actions/" + - "Time-box gaps with owner and expiry ≤90 days" +references: + - title: "APRF-RFC-0002 — Demote INC-M3 / INC-M4 to recommended" + url: "https://github.com/stackrail-io/APRF/blob/main/rfcs/0002-incident-readiness-mandatory-to-recommended.md" + - title: "Google SRE — Managing Incidents" + url: "https://sre.google/sre-book/managing-incidents/" + - title: "NIST AI RMF — Manage" + url: "https://www.nist.gov/itl/ai-risk-management-framework" +relatedRules: + - "INC-M1" + - "INC-M2" + - "INC-R4" + - "INC-R1" + - "INC-R3" + - "ORG-R4" +tags: + - "incident-readiness" + - "recommended" + - "hybrid" + - "post-incident" + - "aprf-actions" +applicability: + technologies: [] + minCriticality: 2 + requiredFromLevel: 3 +status: "active" +introducedIn: "0.10.0" diff --git a/packages/aprf-engine/rules/by-domain/reliability/incident-readiness/INC-R3.yaml b/packages/aprf-engine/rules/by-domain/reliability/incident-readiness/INC-R3.yaml index 771883c..4763fd4 100644 --- a/packages/aprf-engine/rules/by-domain/reliability/incident-readiness/INC-R3.yaml +++ b/packages/aprf-engine/rules/by-domain/reliability/incident-readiness/INC-R3.yaml @@ -1,37 +1,57 @@ id: "INC-R3" category: "incident-readiness" -title: "Production systems should have customer notification criteria for AI-related events" -description: "Customer notification criteria for AI-related events" -whyItMatters: "Customer notification criteria for AI-related events Failing this leaves a production - gap against: Criteria map event types (safety incident, widespread quality fail, data exposure) to - notify / no-notify; last drill or incident ≤12 months followed the criteria with timestamps" +title: "Production systems should define customer notification criteria for AI-related events" +description: > + Teams should maintain criteria that map AI event types to notify / no-notify + decisions, and show a drill or real notification ≤12 months that followed + those criteria with timestamps. +whyItMatters: > + Without clear notify/no-notify rules for safety incidents, widespread quality + failures, and data exposure, teams either over-notify customers or stay silent + when disclosure is required—both erode trust and invite regulatory risk. severity: "high" weight: 3 gate: "recommended" -passCondition: "Criteria map event types (safety incident, widespread quality fail, data exposure) - to notify / no-notify; last drill or incident ≤12 months followed the criteria with timestamps" +passCondition: > + Criteria map event types (safety incident, widespread quality fail, data + exposure) to notify / no-notify; last drill or incident ≤12 months followed + the criteria with timestamps (notification evidence measuredAt ≤90 days). evidenceRequired: - - "Customer notification criteria for AI-related events + last drill or real notification sample" + - "Customer notification criteria for AI-related events" + - "Last drill or real notification sample ≤12 months with timestamps" detection: - capability: "manual" + capability: "hybrid" detectors: + - id: "repo-ai-customer-notification-criteria" + params: + hint: > + Discover AI customer notification criteria and drill/incident samples + that followed notify/no-notify decisions with timestamps. - id: "manual-attest" params: - hint: "Customer notification criteria for AI-related events + last drill or real notification - sample" -manualVerification: "For this Check (Customer notification criteria for AI-related events): inspect - current evidence for [Customer notification criteria for AI-related events + last drill or real - notification sample] and confirm the pass condition holds — Criteria map event types (safety - incident, widespread quality fail, data exposure) to notify / no-notify; last drill or incident - ≤12 months followed the criteria with timestamps" -falsePositiveGuidance: "(Incident Readiness): re-verify against a current artifact for this specific - Check , not a sibling control. Document named exceptions with owner and expiry." + hint: > + If automation cannot prove coverage, attest criteria map safety, + quality-fail, and data-exposure events to notify/no-notify, and a + drill or incident ≤12 months followed them with timestamps + (measuredAt ≤90 days). +manualVerification: > + 1) Confirm a customer-facing or externally disclosed AI system is in scope. + If none, score NOT_APPLICABLE. 2) Confirm criteria map event types (safety + incident, widespread quality fail, data exposure) to notify / no-notify. 3) + Confirm a drill or real notification ≤12 months followed those criteria with + timestamps. 4) PASS only if criteria + followed sample hold with measuredAt + ≤90 days. INC-R2 internal post-incident reviews alone do not prove customer + notification. Generic PR/comms playbooks without AI event-type mapping do not + satisfy. +falsePositiveGuidance: > + Do not pass criteria without notify/no-notify decisions. Do not pass a sample + older than 12 months. Do not pass a sample without timestamps. Named + exceptions need owner and expiry ≤90 days. recommendedFixes: - - "Implement and operationalize: this Check: Customer notification criteria for AI-related events" - - "Retain evidence artifacts required by this Check, starting with: Customer notification criteria - for AI-related events + last drill or real notification sample" - - "Schedule recurring manual verification for this Check with a named owner and retained report" - - "Block release (or open a time-boxed waiver with owner and expiry) until this Check passes" + - "Publish AI event-type → notify/no-notify criteria covering safety, quality fail, and data exposure" + - "Run a notification drill (or retain a real sample) ≤12 months with timestamps" + - "Retain evidence under imports/ai-customer-notification-criteria/" + - "Time-box gaps with owner and expiry ≤90 days" references: - title: "Google SRE — Managing Incidents" url: "https://sre.google/sre-book/managing-incidents/" @@ -40,16 +60,18 @@ references: relatedRules: - "INC-M1" - "INC-M2" - - "INC-M3" - - "INC-M4" - "INC-R1" + - "INC-R2" + - "INC-R4" + - "PRI-M1" tags: - "incident-readiness" - "recommended" - - "manual" + - "hybrid" + - "customer-notification" + - "disclosure" applicability: - technologies: - - "cicd" + technologies: [] minCriticality: 2 requiredFromLevel: 4 status: "active" diff --git a/packages/aprf-engine/rules/by-domain/reliability/incident-readiness/INC-R4.yaml b/packages/aprf-engine/rules/by-domain/reliability/incident-readiness/INC-R4.yaml new file mode 100644 index 0000000..f85dc4c --- /dev/null +++ b/packages/aprf-engine/rules/by-domain/reliability/incident-readiness/INC-R4.yaml @@ -0,0 +1,76 @@ +id: "INC-R4" +category: "incident-readiness" +title: "Production systems should run regular tabletop exercises covering AI-specific incidents" +description: > + Teams should complete an AI-focused incident tabletop at least every 180 days + and retain an after-action report with actions and owners. +whyItMatters: > + Containment drills prove mechanics; tabletops prove decision-making under + AI-specific scenarios. Without a recent exercise and owned follow-ups, + on-call discovers coordination gaps only during a real SEV. +severity: "high" +weight: 3 +gate: "recommended" +passCondition: > + An AI-focused tabletop completed ≤180 days with retained actions and owners + (tabletop evidence measuredAt ≤90 days). +evidenceRequired: + - "Tabletop plan for an AI-specific incident scenario" + - "Dated after-action report ≤180 days with retained actions and owners" +detection: + capability: "hybrid" + detectors: + - id: "repo-ai-incident-tabletop" + params: + hint: > + Discover AI-focused tabletop plans and after-action reports with + retained actions and owners. + - id: "manual-attest" + params: + hint: > + If automation cannot prove coverage, attest an AI-focused tabletop + completed ≤180 days with retained actions and owners (measuredAt ≤90 + days). +manualVerification: > + 1) Confirm a production AI system is in scope. If none, score NOT_APPLICABLE. + 2) Confirm a tabletop plan covers an AI-specific incident scenario. 3) + Confirm a dated after-action report shows completion ≤180 days ago. 4) + Confirm retained actions have owners. 5) PASS only if plan + completion + + owned actions hold with measuredAt ≤90 days. INC-M2 containment drills alone + do not prove tabletop decision exercises. INC-R2 post-incident reviews alone + do not prove proactive tabletops. Generic non-AI tabletops do not satisfy. +falsePositiveGuidance: > + Do not pass a slide deck without an after-action report. Do not pass actions + without owners. Do not pass a tabletop older than 180 days. Named exceptions + need owner and expiry ≤90 days. +recommendedFixes: + - "Schedule AI-specific tabletops on a ≤180-day cadence" + - "Retain after-action reports with owned actions" + - "Retain evidence under imports/ai-incident-tabletop/" + - "Time-box gaps with owner and expiry ≤90 days" +references: + - title: "APRF-RFC-0002 — Demote INC-M3 / INC-M4 to recommended" + url: "https://github.com/stackrail-io/APRF/blob/main/rfcs/0002-incident-readiness-mandatory-to-recommended.md" + - title: "Google SRE — Managing Incidents" + url: "https://sre.google/sre-book/managing-incidents/" + - title: "NIST AI RMF — Manage" + url: "https://www.nist.gov/itl/ai-risk-management-framework" +relatedRules: + - "INC-M1" + - "INC-M2" + - "INC-R1" + - "INC-R2" + - "INC-R3" + - "REL-M7" +tags: + - "incident-readiness" + - "recommended" + - "hybrid" + - "tabletop" + - "ai-incidents" +applicability: + technologies: [] + minCriticality: 3 + requiredFromLevel: 4 +status: "active" +introducedIn: "0.10.0" diff --git a/packages/aprf-engine/src/detectors/catalog-ids.ts b/packages/aprf-engine/src/detectors/catalog-ids.ts index 5c0c13e..d156112 100644 --- a/packages/aprf-engine/src/detectors/catalog-ids.ts +++ b/packages/aprf-engine/src/detectors/catalog-ids.ts @@ -80,6 +80,11 @@ export const CATALOG_DETECTOR_IDS = [ "mcp-no-unrestricted-network", "repo-policy-as-code", "repo-incident-playbooks", + "repo-ai-containment-drill", + "repo-post-incident-aprf-actions", + "repo-ai-incident-tabletop", + "repo-ai-safety-quality-alerts", + "repo-ai-customer-notification-criteria", "repo-sbom-config", "repo-scaffolding-templates", "repo-otel-config", diff --git a/packages/aprf-engine/src/generated/catalog.ts b/packages/aprf-engine/src/generated/catalog.ts index 26446fd..74c5674 100644 --- a/packages/aprf-engine/src/generated/catalog.ts +++ b/packages/aprf-engine/src/generated/catalog.ts @@ -6,7 +6,7 @@ import type { GeneratedCatalog } from "../catalog-types.js"; export const GENERATED_CATALOG: GeneratedCatalog = { - "generatedAt": "sha256:97b7754b721e5fc7db654ea33460c2a220d1dc6ee9b33b2cea9880e2c64ee37c", + "generatedAt": "sha256:0901f1e9378603cfef2af7d7ba02aafc847c15a5aad2fc9a52e5bb2b1932830d", "ruleCount": 177, "domains": [ { @@ -6324,37 +6324,40 @@ export const GENERATED_CATALOG: GeneratedCatalog = { "id": "INC-M1", "category": "incident-readiness", "title": "AI-specific incident playbooks must exist for abuse, leakage, bad actions, and provider outage", - "description": "AI-specific incident playbooks shall exist for abuse, leakage, bad actions, and provider outage", - "whyItMatters": "AI-specific incident playbooks shall exist for abuse, leakage, bad actions, and provider outage Failing this leaves a production gap against: Four playbooks present (abuse, leakage, bad actions, provider outage), each with owner and review date ≤ 12 months", + "description": "Production AI systems must maintain incident playbooks for abuse, data leakage, bad agent/tool actions, and provider outage—each with a named owner and a review date within the last 12 months.\n", + "whyItMatters": "Generic outage runbooks do not cover AI-specific failure modes. Without owned, recently reviewed playbooks for abuse, leakage, bad actions, and provider loss, on-call improvises under pressure and containment is slow or incomplete.\n", "severity": "high", "weight": 3, "gate": "mandatory", - "passCondition": "Four playbooks present (abuse, leakage, bad actions, provider outage), each with owner and review date ≤ 12 months", + "passCondition": "Four playbooks present (abuse, leakage, bad actions, provider outage), each with owner and review date ≤12 months (playbook evidence measuredAt ≤90 days).\n", "evidenceRequired": [ - "Playbook set covering the four scenarios with owners" + "Playbook set covering abuse, leakage, bad actions, and provider outage with owners", + "Review dates ≤12 months for each of the four playbooks" ], "detection": { "capability": "hybrid", "detectors": [ { "id": "repo-incident-playbooks", - "params": {} + "params": { + "hint": "Discover AI-specific incident playbooks for abuse, leakage, bad actions, and provider outage, plus owner and review-date coverage.\n" + } }, { "id": "manual-attest", "params": { - "hint": "Playbook set covering the four scenarios with owners" + "hint": "If automation cannot prove coverage, attest all four playbooks exist with owners and review dates ≤12 months (measuredAt ≤90 days).\n" } } ] }, - "manualVerification": "For this Check (AI-specific incident playbooks must exist for abuse, leakage, bad actions, and provider outage): inspect current evidence for [Playbook set covering the four scenarios with owners] and confirm the pass condition holds — Four playbooks present (abuse, leakage, bad actions, provider outage), each with owner and review date ≤ 12 months", - "falsePositiveGuidance": "(Incident Readiness): when automation and attestation disagree, prefer the stricter outcome until reconciled. Waive only with owner, expiry, and which signal covers the gap.", + "manualVerification": "1) Confirm a production AI system is in scope. If none, score NOT_APPLICABLE. 2) Confirm playbooks exist for abuse, leakage, bad actions, and provider outage. 3) Confirm each has a named owner. 4) Confirm each was reviewed within the last 12 months. 5) PASS only if all four + owners + fresh reviews hold with measuredAt ≤90 days. INC-M2 containment drills alone do not prove the four playbooks. Generic SRE incident docs that omit AI scenarios do not satisfy. A single combined doc may pass only if it clearly covers all four scenarios with owners and review dates.\n", + "falsePositiveGuidance": "Do not pass placeholder stubs without owners or review dates. Do not pass reviews older than 12 months. Do not pass a provider-outage playbook that only covers non-AI infrastructure. Named exceptions need owner and expiry ≤90 days.\n", "recommendedFixes": [ - "Implement and operationalize: AI-specific incident playbooks must exist for abuse, leakage, bad actions, and provider outage", - "Retain evidence artifacts required by this Check, starting with: Playbook set covering the four scenarios with owners", - "Wire or verify detectors declared on this Check so automation matches the pass condition", - "Block release (or open a time-boxed waiver with owner and expiry) until this Check passes" + "Author and own playbooks for abuse, leakage, bad actions, and provider outage", + "Review each playbook on a ≤12-month cadence and retain dates", + "Retain evidence under imports/incident-playbooks/", + "Time-box gaps with owner and expiry ≤90 days" ], "references": [ { @@ -6368,20 +6371,22 @@ export const GENERATED_CATALOG: GeneratedCatalog = { ], "relatedRules": [ "INC-M2", - "INC-M3", - "INC-M4", + "INC-R2", + "INC-R4", "INC-R1", - "INC-R3" + "INC-R3", + "AGN-M3", + "CHG-M2" ], "tags": [ "incident-readiness", "mandatory", - "hybrid" + "hybrid", + "playbooks", + "ai-incidents" ], "applicability": { - "technologies": [ - "cicd" - ], + "technologies": [], "minCriticality": 2, "requiredFromLevel": 3 }, @@ -6392,33 +6397,40 @@ export const GENERATED_CATALOG: GeneratedCatalog = { "id": "INC-M2", "category": "incident-readiness", "title": "On-call must be able to execute containment: pause agents, disable tools, roll back prompts/models", - "description": "On-call shall be able to execute containment: pause agents, disable tools, roll back prompts/models", - "whyItMatters": "On-call shall be able to execute containment: pause agents, disable tools, roll back prompts/models Failing this leaves a production gap against: Drill in last 90 days successfully demonstrated pause agents, disable tools, and roll back prompt/model within documented time budgets", + "description": "On-call must be able to pause agents, disable tools, and roll back prompts or models, proven by a drill in the last 90 days completed within documented time budgets.\n", + "whyItMatters": "Playbooks alone do not stop a live AI incident. On-call needs practiced containment—pause, disable tools, roll back—within known time budgets so blast radius shrinks before customers absorb more damage.\n", "severity": "high", "weight": 3, "gate": "mandatory", - "passCondition": "Drill in last 90 days successfully demonstrated pause agents, disable tools, and roll back prompt/model within documented time budgets", + "passCondition": "Drill in last 90 days successfully demonstrated pause agents, disable tools, and roll back prompt/model within documented time budgets (drill evidence measuredAt ≤90 days).\n", "evidenceRequired": [ - "Containment runbook + drill record exercising pause, disable, and rollback" + "Containment runbook covering pause agents, disable tools, and prompt/model rollback", + "Drill record ≤90 days showing all three actions within documented time budgets" ], "detection": { - "capability": "manual", + "capability": "hybrid", "detectors": [ + { + "id": "repo-ai-containment-drill", + "params": { + "hint": "Discover containment runbooks and drill evidence for pause agents, disable tools, and prompt/model rollback within time budgets.\n" + } + }, { "id": "manual-attest", "params": { - "hint": "Containment runbook + drill record exercising pause, disable, and rollback" + "hint": "If automation cannot prove coverage, attest a drill ≤90 days demonstrated pause agents, disable tools, and prompt/model rollback within documented time budgets (measuredAt ≤90 days).\n" } } ] }, - "manualVerification": "For this Check (On-call must be able to execute containment: pause agents, disable tools, roll back prompts/models): inspect current evidence for [Containment runbook + drill record exercising pause, disable, and rollback] and confirm the pass condition holds — Drill in last 90 days successfully demonstrated pause agents, disable tools, and roll back prompt/model within documented time budgets", - "falsePositiveGuidance": "(Incident Readiness): re-verify against a current artifact for this specific Check , not a sibling control. Document named exceptions with owner and expiry.", + "manualVerification": "1) Confirm production agents, tools, or prompt/model release units exist. If none, score NOT_APPLICABLE. 2) Confirm a containment runbook covers pause agents, disable tools, and prompt/model rollback with documented time budgets. 3) Confirm a drill in the last 90 days successfully exercised all three within those budgets. 4) PASS only if drill + all three actions + time budgets hold with measuredAt ≤90 days. INC-M1 playbooks alone do not prove executed containment. AGN kill-switch Checks alone do not prove tool disable and prompt/model rollback. CHG rollback Checks alone do not prove pause + disable.\n", + "falsePositiveGuidance": "Do not pass tabletop discussion without executing pause/disable/rollback. Do not pass a drill older than 90 days. Do not pass actions that missed documented time budgets. Named exceptions need owner and expiry ≤90 days.\n", "recommendedFixes": [ - "Implement and operationalize: On-call must be able to execute containment: pause agents, disable tools, roll back prompts/models", - "Retain evidence artifacts required by this Check, starting with: Containment runbook + drill record exercising pause, disable, and rollback", - "Schedule recurring manual verification for this Check with a named owner and retained report", - "Block release (or open a time-boxed waiver with owner and expiry) until this Check passes" + "Document containment steps for pause, disable tools, and prompt/model rollback with time budgets", + "Run and retain a ≤90-day drill exercising all three within budgets", + "Retain evidence under imports/ai-containment-drill/", + "Time-box gaps with owner and expiry ≤90 days" ], "references": [ { @@ -6432,20 +6444,24 @@ export const GENERATED_CATALOG: GeneratedCatalog = { ], "relatedRules": [ "INC-M1", - "INC-M3", - "INC-M4", + "INC-R2", + "INC-R4", "INC-R1", - "INC-R3" + "INC-R3", + "AGN-M3", + "CHG-M2", + "CHG-M3", + "PRM-M3" ], "tags": [ "incident-readiness", "mandatory", - "manual" + "hybrid", + "containment", + "drill" ], "applicability": { - "technologies": [ - "cicd" - ], + "technologies": [], "minCriticality": 2, "requiredFromLevel": 3 }, @@ -6453,34 +6469,48 @@ export const GENERATED_CATALOG: GeneratedCatalog = { "introducedIn": "0.10.0" }, { - "id": "INC-M3", + "id": "INC-R1", "category": "incident-readiness", - "title": "Post-incident reviews must produce tracked actions against APRF pillars", - "description": "Post-incident reviews shall produce tracked actions against APRF pillars", - "whyItMatters": "Post-incident reviews shall produce tracked actions against APRF pillars Failing this leaves a production gap against: 100% of SEV-eligible AI incidents in last 90 days have a review with ≥1 tracked action mapped to an APRF pillar or explicit “no action” rationale", + "title": "Production systems should page on safety and quality signals, not only infra", + "description": "On-call should receive page-worthy alerts for at least two non-infra AI safety/quality signals, each with a documented threshold and owner, and a policy reviewed within 90 days.\n", + "whyItMatters": "Infra pages alone miss AI failure modes—refusal spikes, eval drops, toxicity/jailbreak hits—until customers or regulators notice. Paging with owned thresholds turns those signals into incidents before blast radius grows.\n", "severity": "high", "weight": 3, - "gate": "mandatory", - "passCondition": "100% of SEV-eligible AI incidents in last 90 days have a review with ≥1 tracked action mapped to an APRF pillar or explicit “no action” rationale", + "gate": "recommended", + "passCondition": "At least two non-infra signals (e.g. refusal-rate spike, eval-score drop, toxicity/jailbreak hit rate) page an on-call; each has a documented threshold and owner; policy reviewed ≤90 days ago (alert evidence measuredAt ≤90 days).\n", "evidenceRequired": [ - "Post-incident review template + sample reviews with linked actions" + "On-call alert policy export listing safety/quality pages", + "Last 90 days of triggered incidents or drill tickets for those pages" ], "detection": { - "capability": "manual", - "detectors": [] + "capability": "hybrid", + "detectors": [ + { + "id": "repo-ai-safety-quality-alerts", + "params": { + "hint": "Discover on-call alert policies for non-infra AI safety/quality signals with thresholds and owners.\n" + } + }, + { + "id": "manual-attest", + "params": { + "hint": "If automation cannot prove coverage, attest ≥2 non-infra safety/ quality signals page on-call, each with threshold and owner, and policy reviewed ≤90 days (measuredAt ≤90 days).\n" + } + } + ] }, - "manualVerification": "For this Check (Post-incident reviews must produce tracked actions against APRF pillars): inspect current evidence for [Post-incident review template + sample reviews with linked actions] and confirm the pass condition holds — 100% of SEV-eligible AI incidents in last 90 days have a review with ≥1 tracked action mapped to an APRF pillar or explicit “no action” rationale", - "falsePositiveGuidance": "(Incident Readiness): re-verify against a current artifact for this specific Check , not a sibling control. Document named exceptions with owner and expiry.", + "manualVerification": "1) Confirm a production AI system is in scope. If none, score NOT_APPLICABLE. 2) Confirm an on-call alert policy lists ≥2 non-infra safety/quality signals that page (e.g. refusal-rate spike, eval-score drop, toxicity/jailbreak hit rate). 3) Confirm each signal has a documented threshold and owner. 4) Confirm policy review ≤90 days ago (or triggered incidents/drill tickets in last 90 days). 5) PASS only if all hold with measuredAt ≤90 days. Infra-only CPU/memory/latency pages do not satisfy. COST spend alerts alone do not satisfy. OBS dashboards without paging do not satisfy.\n", + "falsePositiveGuidance": "Do not pass dashboards that do not page. Do not pass a single signal. Do not pass thresholds without owners. Do not pass policy older than 90 days. Named exceptions need owner and expiry ≤90 days.\n", "recommendedFixes": [ - "Implement and operationalize: Post-incident reviews must produce tracked actions against APRF pillars", - "Retain evidence artifacts required by this Check, starting with: Post-incident review template + sample reviews with linked actions", - "Schedule recurring manual verification for this Check with a named owner and retained report", - "Block release (or open a time-boxed waiver with owner and expiry) until this Check passes" + "Define ≥2 non-infra AI safety/quality paging signals with thresholds and owners", + "Wire those signals to on-call and retain policy export + recent pages/drills", + "Retain evidence under imports/ai-safety-quality-alerts/", + "Time-box gaps with owner and expiry ≤90 days" ], "references": [ { - "title": "Google SRE — Managing Incidents", - "url": "https://sre.google/sre-book/managing-incidents/" + "title": "Google SRE — Monitoring Distributed Systems", + "url": "https://sre.google/sre-book/monitoring-distributed-systems/" }, { "title": "NIST AI RMF — Manage", @@ -6490,58 +6520,70 @@ export const GENERATED_CATALOG: GeneratedCatalog = { "relatedRules": [ "INC-M1", "INC-M2", - "INC-M4", - "INC-R1", - "INC-R3" + "INC-R2", + "INC-R3", + "INC-R4", + "OBS-M1" ], "tags": [ "incident-readiness", - "mandatory", - "manual" + "recommended", + "hybrid", + "paging", + "safety-quality" ], "applicability": { - "technologies": [ - "cicd" - ], + "technologies": [], "minCriticality": 2, - "requiredFromLevel": 3 + "requiredFromLevel": 4 }, "status": "active", "introducedIn": "0.10.0" }, { - "id": "INC-M4", + "id": "INC-R2", "category": "incident-readiness", - "title": "Regular tabletop exercises must cover AI-specific incidents", - "description": "Regular tabletop exercises shall cover AI-specific incidents", - "whyItMatters": "Regular tabletop exercises shall cover AI-specific incidents Failing this leaves a production gap against: PASS if an AI-focused tabletop completed ≤180 days with retained actions and owners", + "title": "Post-incident reviews should produce tracked actions against APRF pillars", + "description": "SEV-eligible AI incidents should receive a post-incident review with at least one tracked action mapped to an APRF pillar—or an explicit no-action rationale.\n", + "whyItMatters": "Reviews without pillar-mapped actions do not harden the system. Tracking remediation (or documenting why none is needed) turns incidents into APRF control improvements instead of one-off war stories.\n", "severity": "high", "weight": 3, - "gate": "mandatory", - "passCondition": "PASS if an AI-focused tabletop completed ≤180 days with retained actions and owners", + "gate": "recommended", + "passCondition": "100% of SEV-eligible AI incidents in last 90 days have a review with ≥1 tracked action mapped to an APRF pillar or explicit “no action” rationale (review evidence measuredAt ≤90 days).\n", "evidenceRequired": [ - "Tabletop plan + dated after-action report for an AI incident scenario" + "Post-incident review template requiring APRF pillar mapping", + "Last-90-day coverage: SEV-eligible AI reviews with tracked actions or no-action rationale" ], "detection": { - "capability": "manual", + "capability": "hybrid", "detectors": [ + { + "id": "repo-post-incident-aprf-actions", + "params": { + "hint": "Discover post-incident review templates and coverage showing APRF-pillar-mapped actions (or no-action rationale).\n" + } + }, { "id": "manual-attest", "params": { - "hint": "Tabletop plan + dated after-action report for an AI incident scenario" + "hint": "If automation cannot prove coverage, attest 100% of SEV-eligible AI incidents in the last 90 days have a review with ≥1 tracked APRF-pillar action or explicit no-action rationale (measuredAt ≤90 days). Score N/A if sevEligibleIncidentCount=0.\n" } } ] }, - "manualVerification": "For this Check (Regular tabletop exercises must cover AI-specific incidents): inspect current evidence for [Tabletop plan + dated after-action report for an AI incident scenario] and confirm the pass condition holds — PASS if an AI-focused tabletop completed ≤180 days with retained actions and owners", - "falsePositiveGuidance": "(Incident Readiness): re-verify against a current artifact for this specific Check , not a sibling control. Document named exceptions with owner and expiry.", + "manualVerification": "1) Confirm SEV-eligible AI incidents exist in the last 90 days. If none, score NOT_APPLICABLE with retained attestation. 2) Confirm a post-incident review template requires APRF pillar mapping (or equivalent). 3) Confirm 100% of those incidents have a review with ≥1 tracked action mapped to an APRF pillar or an explicit no-action rationale. 4) PASS only if coverage holds with measuredAt ≤90 days. INC-M1/M2 playbooks and containment alone do not prove pillar-mapped actions. Generic postmortems without APRF pillar linkage do not satisfy.\n", + "falsePositiveGuidance": "Do not pass reviews that list actions without owners or tickets. Do not pass “no action” without a written rationale. Do not pass coverage older than 90 days. Named exceptions need owner and expiry ≤90 days.\n", "recommendedFixes": [ - "Implement and operationalize: Regular tabletop exercises must cover AI-specific incidents", - "Retain evidence artifacts required by this Check, starting with: Tabletop plan + dated after-action report for an AI incident scenario", - "Schedule recurring manual verification for this Check with a named owner and retained report", - "Block release (or open a time-boxed waiver with owner and expiry) until this Check passes" + "Require APRF pillar mapping (or no-action rationale) on every SEV-eligible AI review", + "Track actions to tickets/owners and retain last-90-day coverage", + "Retain evidence under imports/post-incident-aprf-actions/", + "Time-box gaps with owner and expiry ≤90 days" ], "references": [ + { + "title": "APRF-RFC-0002 — Demote INC-M3 / INC-M4 to recommended", + "url": "https://github.com/stackrail-io/APRF/blob/main/rfcs/0002-incident-readiness-mandatory-to-recommended.md" + }, { "title": "Google SRE — Managing Incidents", "url": "https://sre.google/sre-book/managing-incidents/" @@ -6554,56 +6596,64 @@ export const GENERATED_CATALOG: GeneratedCatalog = { "relatedRules": [ "INC-M1", "INC-M2", - "INC-M3", + "INC-R4", "INC-R1", - "INC-R3" + "INC-R3", + "ORG-R4" ], "tags": [ "incident-readiness", - "mandatory", - "manual" + "recommended", + "hybrid", + "post-incident", + "aprf-actions" ], "applicability": { - "technologies": [ - "cicd" - ], - "minCriticality": 3, - "requiredFromLevel": 4 + "technologies": [], + "minCriticality": 2, + "requiredFromLevel": 3 }, "status": "active", "introducedIn": "0.10.0" }, { - "id": "INC-R1", + "id": "INC-R3", "category": "incident-readiness", - "title": "Production systems should have page-worthy alerts for safety and quality signals, not only infra", - "description": "Page-worthy alerts for safety and quality signals, not only infra", - "whyItMatters": "Page-worthy alerts for safety and quality signals, not only infra Failing this leaves a production gap against: At least two non-infra signals (e.g. refusal-rate spike, eval-score drop, toxicity/jailbreak hit rate) page an on-call; each has a documented threshold and owner; policy reviewed ≤90 days ago", + "title": "Production systems should define customer notification criteria for AI-related events", + "description": "Teams should maintain criteria that map AI event types to notify / no-notify decisions, and show a drill or real notification ≤12 months that followed those criteria with timestamps.\n", + "whyItMatters": "Without clear notify/no-notify rules for safety incidents, widespread quality failures, and data exposure, teams either over-notify customers or stay silent when disclosure is required—both erode trust and invite regulatory risk.\n", "severity": "high", "weight": 3, "gate": "recommended", - "passCondition": "At least two non-infra signals (e.g. refusal-rate spike, eval-score drop, toxicity/jailbreak hit rate) page an on-call; each has a documented threshold and owner; policy reviewed ≤90 days ago", + "passCondition": "Criteria map event types (safety incident, widespread quality fail, data exposure) to notify / no-notify; last drill or incident ≤12 months followed the criteria with timestamps (notification evidence measuredAt ≤90 days).\n", "evidenceRequired": [ - "On-call alert policy export listing safety/quality pages + last 90 days of triggered incidents (or drill tickets)" + "Customer notification criteria for AI-related events", + "Last drill or real notification sample ≤12 months with timestamps" ], "detection": { - "capability": "manual", + "capability": "hybrid", "detectors": [ + { + "id": "repo-ai-customer-notification-criteria", + "params": { + "hint": "Discover AI customer notification criteria and drill/incident samples that followed notify/no-notify decisions with timestamps.\n" + } + }, { "id": "manual-attest", "params": { - "hint": "On-call alert policy export listing safety/quality pages + last 90 days of triggered incidents (or drill tickets)" + "hint": "If automation cannot prove coverage, attest criteria map safety, quality-fail, and data-exposure events to notify/no-notify, and a drill or incident ≤12 months followed them with timestamps (measuredAt ≤90 days).\n" } } ] }, - "manualVerification": "For this Check (Page-worthy alerts for safety and quality signals, not only infra): inspect current evidence for [On-call alert policy export listing safety/quality pages + last 90 days of triggered incidents (or drill tickets)] and confirm the pass condition holds — At least two non-infra signals (e.g. refusal-rate spike, eval-score drop, toxicity/jailbreak hit rate) page an on-call; each has a documented threshold and owner; policy reviewed ≤90 days ago", - "falsePositiveGuidance": "(Incident Readiness): re-verify against a current artifact for this specific Check , not a sibling control. Document named exceptions with owner and expiry.", + "manualVerification": "1) Confirm a customer-facing or externally disclosed AI system is in scope. If none, score NOT_APPLICABLE. 2) Confirm criteria map event types (safety incident, widespread quality fail, data exposure) to notify / no-notify. 3) Confirm a drill or real notification ≤12 months followed those criteria with timestamps. 4) PASS only if criteria + followed sample hold with measuredAt ≤90 days. INC-R2 internal post-incident reviews alone do not prove customer notification. Generic PR/comms playbooks without AI event-type mapping do not satisfy.\n", + "falsePositiveGuidance": "Do not pass criteria without notify/no-notify decisions. Do not pass a sample older than 12 months. Do not pass a sample without timestamps. Named exceptions need owner and expiry ≤90 days.\n", "recommendedFixes": [ - "Implement and operationalize: this Check: Page-worthy alerts for safety and quality signals, not only infra", - "Retain evidence artifacts required by this Check, starting with: On-call alert policy export listing safety/quality pages + last 90 days of triggered incidents (or drill tickets)", - "Schedule recurring manual verification for this Check with a named owner and retained report", - "Block release (or open a time-boxed waiver with owner and expiry) until this Check passes" + "Publish AI event-type → notify/no-notify criteria covering safety, quality fail, and data exposure", + "Run a notification drill (or retain a real sample) ≤12 months with timestamps", + "Retain evidence under imports/ai-customer-notification-criteria/", + "Time-box gaps with owner and expiry ≤90 days" ], "references": [ { @@ -6618,19 +6668,20 @@ export const GENERATED_CATALOG: GeneratedCatalog = { "relatedRules": [ "INC-M1", "INC-M2", - "INC-M3", - "INC-M4", - "INC-R3" + "INC-R1", + "INC-R2", + "INC-R4", + "PRI-M1" ], "tags": [ "incident-readiness", "recommended", - "manual" + "hybrid", + "customer-notification", + "disclosure" ], "applicability": { - "technologies": [ - "cicd" - ], + "technologies": [], "minCriticality": 2, "requiredFromLevel": 4 }, @@ -6638,38 +6689,49 @@ export const GENERATED_CATALOG: GeneratedCatalog = { "introducedIn": "0.10.0" }, { - "id": "INC-R3", + "id": "INC-R4", "category": "incident-readiness", - "title": "Production systems should have customer notification criteria for AI-related events", - "description": "Customer notification criteria for AI-related events", - "whyItMatters": "Customer notification criteria for AI-related events Failing this leaves a production gap against: Criteria map event types (safety incident, widespread quality fail, data exposure) to notify / no-notify; last drill or incident ≤12 months followed the criteria with timestamps", + "title": "Production systems should run regular tabletop exercises covering AI-specific incidents", + "description": "Teams should complete an AI-focused incident tabletop at least every 180 days and retain an after-action report with actions and owners.\n", + "whyItMatters": "Containment drills prove mechanics; tabletops prove decision-making under AI-specific scenarios. Without a recent exercise and owned follow-ups, on-call discovers coordination gaps only during a real SEV.\n", "severity": "high", "weight": 3, "gate": "recommended", - "passCondition": "Criteria map event types (safety incident, widespread quality fail, data exposure) to notify / no-notify; last drill or incident ≤12 months followed the criteria with timestamps", + "passCondition": "An AI-focused tabletop completed ≤180 days with retained actions and owners (tabletop evidence measuredAt ≤90 days).\n", "evidenceRequired": [ - "Customer notification criteria for AI-related events + last drill or real notification sample" + "Tabletop plan for an AI-specific incident scenario", + "Dated after-action report ≤180 days with retained actions and owners" ], "detection": { - "capability": "manual", + "capability": "hybrid", "detectors": [ + { + "id": "repo-ai-incident-tabletop", + "params": { + "hint": "Discover AI-focused tabletop plans and after-action reports with retained actions and owners.\n" + } + }, { "id": "manual-attest", "params": { - "hint": "Customer notification criteria for AI-related events + last drill or real notification sample" + "hint": "If automation cannot prove coverage, attest an AI-focused tabletop completed ≤180 days with retained actions and owners (measuredAt ≤90 days).\n" } } ] }, - "manualVerification": "For this Check (Customer notification criteria for AI-related events): inspect current evidence for [Customer notification criteria for AI-related events + last drill or real notification sample] and confirm the pass condition holds — Criteria map event types (safety incident, widespread quality fail, data exposure) to notify / no-notify; last drill or incident ≤12 months followed the criteria with timestamps", - "falsePositiveGuidance": "(Incident Readiness): re-verify against a current artifact for this specific Check , not a sibling control. Document named exceptions with owner and expiry.", + "manualVerification": "1) Confirm a production AI system is in scope. If none, score NOT_APPLICABLE. 2) Confirm a tabletop plan covers an AI-specific incident scenario. 3) Confirm a dated after-action report shows completion ≤180 days ago. 4) Confirm retained actions have owners. 5) PASS only if plan + completion + owned actions hold with measuredAt ≤90 days. INC-M2 containment drills alone do not prove tabletop decision exercises. INC-R2 post-incident reviews alone do not prove proactive tabletops. Generic non-AI tabletops do not satisfy.\n", + "falsePositiveGuidance": "Do not pass a slide deck without an after-action report. Do not pass actions without owners. Do not pass a tabletop older than 180 days. Named exceptions need owner and expiry ≤90 days.\n", "recommendedFixes": [ - "Implement and operationalize: this Check: Customer notification criteria for AI-related events", - "Retain evidence artifacts required by this Check, starting with: Customer notification criteria for AI-related events + last drill or real notification sample", - "Schedule recurring manual verification for this Check with a named owner and retained report", - "Block release (or open a time-boxed waiver with owner and expiry) until this Check passes" + "Schedule AI-specific tabletops on a ≤180-day cadence", + "Retain after-action reports with owned actions", + "Retain evidence under imports/ai-incident-tabletop/", + "Time-box gaps with owner and expiry ≤90 days" ], "references": [ + { + "title": "APRF-RFC-0002 — Demote INC-M3 / INC-M4 to recommended", + "url": "https://github.com/stackrail-io/APRF/blob/main/rfcs/0002-incident-readiness-mandatory-to-recommended.md" + }, { "title": "Google SRE — Managing Incidents", "url": "https://sre.google/sre-book/managing-incidents/" @@ -6682,20 +6744,21 @@ export const GENERATED_CATALOG: GeneratedCatalog = { "relatedRules": [ "INC-M1", "INC-M2", - "INC-M3", - "INC-M4", - "INC-R1" + "INC-R1", + "INC-R2", + "INC-R3", + "REL-M7" ], "tags": [ "incident-readiness", "recommended", - "manual" + "hybrid", + "tabletop", + "ai-incidents" ], "applicability": { - "technologies": [ - "cicd" - ], - "minCriticality": 2, + "technologies": [], + "minCriticality": 3, "requiredFromLevel": 4 }, "status": "active", diff --git a/packages/framework-definition/scripts/self-test.ts b/packages/framework-definition/scripts/self-test.ts index 572c902..233a56a 100644 --- a/packages/framework-definition/scripts/self-test.ts +++ b/packages/framework-definition/scripts/self-test.ts @@ -23,14 +23,14 @@ assert(PROFILE_ID_CORE === "aprf-profile-core", "core id"); assert(PROFILE_CORE.targetCriticality === 2, "tier 2"); assert(PROFILE_CORE.mandatoryCheckIds.length === 39, "core has 39 mandatories"); assert( - PROFILE_REGULATED.mandatoryCheckIds.length === 56, - "regulated has 56 mandatories", + PROFILE_REGULATED.mandatoryCheckIds.length === 55, + "regulated has 55 mandatories", ); assert( PROFILE_REGULATED.mandatoryCheckIds.includes("AUTHN-M1"), "regulated includes core", ); -assert(getTier3OnlyMandatoryIds().length === 17, "tier3-only count"); +assert(getTier3OnlyMandatoryIds().length === 16, "tier3-only count"); assert( resolveCheckApplicability("AUTHN-M1", PROFILE_CORE) === "mandatory", diff --git a/packages/framework-definition/src/profiles.ts b/packages/framework-definition/src/profiles.ts index 5283661..2ff59bf 100644 --- a/packages/framework-definition/src/profiles.ts +++ b/packages/framework-definition/src/profiles.ts @@ -96,7 +96,6 @@ export const PROFILE_REGULATED: AprfProfile = { "HUM-M4", "REL-M7", "REL-M8", - "INC-M4", ], rationale: [ "Includes every Core gate — regulated systems must still clear production minimums.", diff --git a/rfcs/0002-incident-readiness-mandatory-to-recommended.md b/rfcs/0002-incident-readiness-mandatory-to-recommended.md new file mode 100644 index 0000000..919eedd --- /dev/null +++ b/rfcs/0002-incident-readiness-mandatory-to-recommended.md @@ -0,0 +1,81 @@ +# APRF-RFC-0002: Demote incident-readiness INC-M3 and INC-M4 to recommended + +| Field | Value | +| --- | --- | +| Status | accepted | +| Author(s) | StackRail (working-draft publisher) | +| Created | 2026-08-01 | +| SemVer impact | MINOR | +| Implements | PR [#14](https://github.com/stackrail-io/APRF/pull/14) (`feat/incident-readiness-hybrid`) | + +## Problem + +Two incident-readiness Checks were published as **mandatory** while the framework is still a pre-release working draft (`0.10.0`, no tagged release versions yet): + +| Former ID | Title (obligation) | Why mandatory was too strong | +| --- | --- | --- | +| **INC-M3** | Post-incident reviews must produce tracked actions against APRF pillars | Valuable learning loop, but not a production-blocking gate for Core/Regulated (was never in those profiles). Treating 100% SEV coverage as mandatory over-weights process maturity vs containment. | +| **INC-M4** | Regular tabletop exercises must cover AI-specific incidents | Tabletop cadence is important for Regulated maturity, but forcing it as a Tier-3 mandatory before adopters have stable playbooks/containment (INC-M1/M2) creates false fails. | + +Assessors and profile consumers need a clear, citable rationale for the gate change and for how IDs were remapped. + +## Proposal + +1. **Demote** the obligations from mandatory (`must`) to recommended (`should`): + - **INC-M3 → INC-R2** — post-incident reviews should produce tracked APRF-pillar actions (or explicit no-action rationale); hybrid detection via `repo-post-incident-aprf-actions`. + - **INC-M4 → INC-R4** — production systems should run AI-focused tabletops ≤180 days with retained owned actions; hybrid detection via `repo-ai-incident-tabletop`. +2. **Keep** INC-M1 (playbooks) and INC-M2 (containment drill) as the incident-readiness **mandatory** Core gates. +3. **Update** the Regulated profile: remove **INC-M4** from `mandatoryCheckIds` (56→55; tier3-only 17→16). INC-M3 was never on Core/Regulated, so demotion does not change those counts. +4. **Rewrite** sibling recommended Checks INC-R1 / INC-R3 from stubs to hybrid collectors in the same change set (editorial + measurable, not gate demotions). + +Normative pass conditions remain measurable (coverage %, ages, owners, measuredAt ≤90d attest freshness). + +## Alternatives considered + +- **Keep mandatory; soften passCondition** — rejected; still fails Regulated assessments for teams with strong containment but immature tabletops/PIR process. +- **Demote in place (same IDs, change `gate` only)** — rejected for M→R semantics clarity; recommended Checks use the `*-R*` namespace by convention. +- **Deprecate INC-M3/M4 stubs with `replacedBy` and retain files** — preferred after first tagged release; see Compatibility for the intentional pre-release exception. + +## Compatibility + +### Gate / profile impact + +| Change | Impact | +| --- | --- | +| INC-M3 removed; INC-R2 added | Additive recommended Check; no Core/Regulated mandatory delta | +| INC-M4 removed; INC-R4 added | Regulated mandatories −1; recommended surface +1 | +| Catalog size | Net same incident-readiness Check count (M3/M4 → R2/R4) | + +SemVer: **MINOR** — profile gate set and Check IDs changed while still on working-draft `0.10.x` before any tagged release. + +### Intentional exception — no deprecated stubs for INC-M3 / INC-M4 + +Per `ARCHITECTURE.md` / `CONTRIBUTING.md`, published Check IDs are normally retained as `status: deprecated` with `replacedBy` for an N−1 MINOR window. + +**Exception (pre-release only):** APRF has **not** shipped any tagged release versions yet. Retaining empty deprecated mandatories would create noise for early adopters without protecting a published baseline. Therefore INC-M3 and INC-M4 YAML were **removed** and replaced by INC-R2 / INC-R4 rather than left as deprecated stubs. + +This exception is recorded in [`packages/aprf-engine/rules/_index/id-gaps.md`](../packages/aprf-engine/rules/_index/id-gaps.md). **After the first tagged release**, future mandatory→recommended moves must use deprecate+`replacedBy` (or an explicit new RFC amending this exception). + +## Security considerations + +No weakening of containment or playbook mandatories (INC-M1/M2). Demoting tabletops and post-incident APRF-action mapping reduces false Regulated fails; residual risk is delayed learning loops, mitigated by keeping both as high-severity recommended Checks with hybrid collectors. + +## Open questions + +- Should Regulated re-include an AI tabletop Check as mandatory once Level-5 maturity guidance is ratified? +- After `v1.0.0`, should a one-time migration note in release notes list INC-M3/M4 → INC-R2/R4 for any private forks that copied pre-release IDs? + +## Checklist + +- [x] Problem and affected parties +- [x] Proposed change stated +- [x] SemVer impact justified +- [x] Compatibility / deprecation plan (incl. intentional pre-release exception) +- [x] Checks remain measurable +- [x] Crosswalk impact noted (N/A — informative crosswalks unchanged) +- [x] Security / safety considered +- [x] Open questions listed + +--- + +Comment window: 14 days from `Created` (working-draft quorum applies). Interim contact: see `/aprf/rfc/`. diff --git a/scripts/check-integrity.ts b/scripts/check-integrity.ts index bbcdc77..c20b21e 100644 --- a/scripts/check-integrity.ts +++ b/scripts/check-integrity.ts @@ -83,10 +83,10 @@ assert( `Core profile expected 39, got ${PROFILE_CORE.mandatoryCheckIds.length}`, ); assert( - PROFILE_REGULATED.mandatoryCheckIds.length === 56, - `Regulated profile expected 56, got ${PROFILE_REGULATED.mandatoryCheckIds.length}`, + PROFILE_REGULATED.mandatoryCheckIds.length === 55, + `Regulated profile expected 55, got ${PROFILE_REGULATED.mandatoryCheckIds.length}`, ); -assert(getTier3OnlyMandatoryIds().length === 17, "tier3-only count"); +assert(getTier3OnlyMandatoryIds().length === 16, "tier3-only count"); for (const id of PROFILE_REGULATED.mandatoryCheckIds) { assert(catalogIds.has(id), `profile Check missing from YAML catalog: ${id}`); diff --git a/skills/aprf-auditor/collectors/README.md b/skills/aprf-auditor/collectors/README.md index 00ed4e7..f81c2d2 100644 --- a/skills/aprf-auditor/collectors/README.md +++ b/skills/aprf-auditor/collectors/README.md @@ -99,6 +99,12 @@ Live mode is **opt-in**. Default collectors only read the local repo + `imports/ | **ai-canary-progressive-delivery** | `ai-canary-progressive-delivery.ts` | canary / progressive delivery for high-traffic AI (DEP-R1) | `imports/ai-canary-progressive-delivery/` | — | | **env-parity-model-tool-catalog** | `env-parity-model-tool-catalog.ts` | prod vs staging parity for model pins + tool catalogs (DEP-R2) | `imports/env-parity-model-tool-catalog/` | — | | **embedding-index-migration** | `embedding-index-migration.ts` | automated embedding/index version migration (DEP-R3) | `imports/embedding-index-migration/` | — | +| **incident-playbooks** | `incident-playbooks.ts` | AI-specific incident playbooks for four scenarios (INC-M1) | `imports/incident-playbooks/` | — | +| **ai-containment-drill** | `ai-containment-drill.ts` | pause / disable tools / rollback containment drill (INC-M2) | `imports/ai-containment-drill/` | — | +| **post-incident-aprf-actions** | `post-incident-aprf-actions.ts` | post-incident APRF-pillar tracked actions (INC-R2) | `imports/post-incident-aprf-actions/` | — | +| **ai-incident-tabletop** | `ai-incident-tabletop.ts` | AI-focused incident tabletop ≤180 days (INC-R4) | `imports/ai-incident-tabletop/` | — | +| **ai-safety-quality-alerts** | `ai-safety-quality-alerts.ts` | safety/quality on-call paging ≥2 signals (INC-R1) | `imports/ai-safety-quality-alerts/` | — | +| **ai-customer-notification-criteria** | `ai-customer-notification-criteria.ts` | customer notify/no-notify criteria + sample ≤12m (INC-R3) | `imports/ai-customer-notification-criteria/` | — | | **ai-cost-alerts** | `ai-cost-alerts.ts` | budget-burn + anomaly alerts | `imports/ai-cost-alerts/` | — | | **ai-retry-amplification** | `ai-retry-amplification.ts` | retry/loop cost bounds | `imports/ai-retry-amplification/` | — | | **ai-prompt-cache** | `ai-prompt-cache.ts` | prompt cache + hit-rate | `imports/ai-prompt-cache/` | — | diff --git a/skills/aprf-auditor/collectors/ai-containment-drill.ts b/skills/aprf-auditor/collectors/ai-containment-drill.ts new file mode 100644 index 0000000..952ffc8 --- /dev/null +++ b/skills/aprf-auditor/collectors/ai-containment-drill.ts @@ -0,0 +1,436 @@ +/** + * ai-containment-drill — INC-M2 / repo-ai-containment-drill. + * + * Discovers containment runbooks + drills for pause agents, disable tools, + * and prompt/model rollback. Import pauseAgentsDemonstrated + + * disableToolsDemonstrated + rollbackPromptOrModelDemonstrated + + * withinDocumentedTimeBudgets under imports/ai-containment-drill/ + * to unlock PASS (measuredAt ≤90d). + */ +import { writeFileSync } from "node:fs"; +import { join, basename } from "node:path"; +import type { + Collector, + CollectorContext, + CollectorResult, + EvidenceNode, +} from "./types.ts"; +import { + ensureDir, + listImportFiles, + readText, + redact, + rel, + walkFiles, +} from "./lib/fs.ts"; +import { + asBool, + measuredAtFresh, + parseMeasuredAt, +} from "./lib/import-attest.ts"; + +const PLUGIN_ID = "ai-containment-drill"; +const RELATED = ["INC-M2"] as const; +const DETECTOR_ID = "repo-ai-containment-drill"; +const IMPORT_MAX_AGE_DAYS = 90; + +const SKIP_DIR_HINT = + /(^|[/\\])(node_modules|\.git|dist|build|coverage|\.venv|venv|__pycache__|vendor)([/\\]|$)/i; + +const CONTAINMENT_RE = + /\b(containment|incident[\s_-]*contain|kill[\s_-]*switch|emergency[\s_-]*stop)\b/i; + +const PAUSE_RE = + /\b(pause[\s_-]*agent|halt[\s_-]*agent|stop[\s_-]*agent|freeze[\s_-]*agent|agent[\s_-]*pause)\b/i; + +const DISABLE_TOOLS_RE = + /\b(disable[\s_-]*tool|tool[\s_-]*disable|revoke[\s_-]*tool|tool[\s_-]*kill|disable[\s_-]*mcp)\b/i; + +const ROLLBACK_RE = + /\b(roll[\s_-]*back[\s_-]*(prompt|model)|prompt[\s_-]*rollback|model[\s_-]*rollback|revert[\s_-]*(prompt|model[\s_-]*pin))\b/i; + +const DRILL_RE = + /\b(drill|tabletop|game[\s_-]*day|exercis(?:e|ed)|containment[\s_-]*test)\b/i; + +const TIME_BUDGET_RE = + /\b(time[\s_-]*budget|within[\s_-]*\d+\s*(min|minute|m|sec|s)|mttr|time[\s_-]*to[\s_-]*contain|sla[\s_-]*contain)\b/i; + +export interface AiContainmentDrillReport { + schemaVersion: "0.2.0"; + pluginId: typeof PLUGIN_ID; + detectorId: typeof DETECTOR_ID; + relatedCheckIds: string[]; + assessedAt: string; + signals: { + pause: { found: boolean; refs: string[] }; + disableTools: { found: boolean; refs: string[] }; + rollback: { found: boolean; refs: string[] }; + drillOrBudget: { found: boolean; refs: string[] }; + }; + importedResults: { + found: boolean; + pauseAgentsDemonstrated: boolean | null; + disableToolsDemonstrated: boolean | null; + rollbackPromptOrModelDemonstrated: boolean | null; + withinDocumentedTimeBudgets: boolean | null; + ageDays: number | null; + measuredAt: string | null; + sources: string[]; + }; + summary: { + containmentSignalsPresent: boolean; + actionSignalCount: number; + incM2Satisfied: boolean | null; + statusHint: + | "pass" + | "partial" + | "fail" + | "not_demonstrated" + | "not_applicable"; + }; + notes: string[]; +} + +function importDir(ctx: CollectorContext): string { + return join(ctx.outputDir, "imports", PLUGIN_ID); +} + +function isSkippable(path: string): boolean { + return SKIP_DIR_HINT.test(path); +} + +function asNum(v: unknown): number | null { + return typeof v === "number" && Number.isFinite(v) ? v : null; +} + +function collectRefs( + targetPath: string, + maxFiles: number, + match: (path: string, text: string) => boolean, + limit = 16, +): string[] { + const refs: string[] = []; + const files = walkFiles(targetPath, { + maxFiles: Math.max(maxFiles, 5000), + extensions: [ + ".md", + ".txt", + ".yml", + ".yaml", + ".json", + ".sh", + ".ts", + ".py", + ], + }); + for (const f of files) { + const r = rel(targetPath, f); + if (isSkippable(r)) continue; + const text = readText(f, 80_000) || ""; + if (match(r, text)) refs.push(r); + if (refs.length >= limit) break; + } + return [...new Set(refs)]; +} + +function loadImported( + ctx: CollectorContext, +): AiContainmentDrillReport["importedResults"] { + const sources: string[] = []; + let pauseAgentsDemonstrated: boolean | null = null; + let disableToolsDemonstrated: boolean | null = null; + let rollbackPromptOrModelDemonstrated: boolean | null = null; + let withinDocumentedTimeBudgets: boolean | null = null; + let ageDays: number | null = null; + let measuredAt: string | null = null; + + for (const f of listImportFiles(ctx.outputDir, PLUGIN_ID)) { + if (/ai-containment-drill-report\.json$/i.test(f)) continue; + const text = readText(f); + if (!text) continue; + try { + const data = JSON.parse(text) as Record; + sources.push(basename(f)); + measuredAt = parseMeasuredAt(data) ?? measuredAt; + ageDays = asNum(data.ageDays) ?? asNum(data.age_days) ?? ageDays; + pauseAgentsDemonstrated = + asBool(data.pauseAgentsDemonstrated) ?? + asBool(data.pause_agents_demonstrated) ?? + asBool(data.pauseDemonstrated) ?? + pauseAgentsDemonstrated; + disableToolsDemonstrated = + asBool(data.disableToolsDemonstrated) ?? + asBool(data.disable_tools_demonstrated) ?? + asBool(data.disableDemonstrated) ?? + disableToolsDemonstrated; + rollbackPromptOrModelDemonstrated = + asBool(data.rollbackPromptOrModelDemonstrated) ?? + asBool(data.rollback_prompt_or_model_demonstrated) ?? + asBool(data.rollbackDemonstrated) ?? + rollbackPromptOrModelDemonstrated; + withinDocumentedTimeBudgets = + asBool(data.withinDocumentedTimeBudgets) ?? + asBool(data.within_documented_time_budgets) ?? + asBool(data.withinTimeBudgets) ?? + withinDocumentedTimeBudgets; + + if (asBool(data.containmentDrillPassedLast90Days) === true) { + pauseAgentsDemonstrated = pauseAgentsDemonstrated ?? true; + disableToolsDemonstrated = disableToolsDemonstrated ?? true; + rollbackPromptOrModelDemonstrated = + rollbackPromptOrModelDemonstrated ?? true; + withinDocumentedTimeBudgets = withinDocumentedTimeBudgets ?? true; + } + } catch { + /* skip */ + } + } + + return { + found: sources.length > 0, + pauseAgentsDemonstrated, + disableToolsDemonstrated, + rollbackPromptOrModelDemonstrated, + withinDocumentedTimeBudgets, + ageDays, + measuredAt, + sources, + }; +} + +export function buildAiContainmentDrillReport(opts: { + assessedAt: string; + pause: { found: boolean; refs: string[] }; + disableTools: { found: boolean; refs: string[] }; + rollback: { found: boolean; refs: string[] }; + drillOrBudget: { found: boolean; refs: string[] }; + imported: AiContainmentDrillReport["importedResults"]; +}): AiContainmentDrillReport { + const notes: string[] = []; + const actionSignalCount = [ + opts.pause.found, + opts.disableTools.found, + opts.rollback.found, + ].filter(Boolean).length; + const containmentSignalsPresent = + actionSignalCount > 0 || + opts.drillOrBudget.found || + CONTAINMENT_RE.test( + [...opts.pause.refs, ...opts.disableTools.refs, ...opts.rollback.refs].join( + " ", + ), + ); + + if (!containmentSignalsPresent && !opts.imported.found) { + notes.push( + "No containment signals — INC-M2 may be NOT_APPLICABLE if no production agents/tools/prompt-model units are in scope.", + ); + } + if (opts.pause.found) { + notes.push(`Pause refs: ${opts.pause.refs.slice(0, 3).join(", ")}`); + } + if (opts.disableTools.found) { + notes.push( + `Disable-tools refs: ${opts.disableTools.refs.slice(0, 3).join(", ")}`, + ); + } + if (opts.rollback.found) { + notes.push(`Rollback refs: ${opts.rollback.refs.slice(0, 3).join(", ")}`); + } + if (opts.imported.found) { + notes.push( + `Imported: ${opts.imported.sources.join(", ")} (pause=${opts.imported.pauseAgentsDemonstrated}, disable=${opts.imported.disableToolsDemonstrated}, rollback=${opts.imported.rollbackPromptOrModelDemonstrated}, withinBudget=${opts.imported.withinDocumentedTimeBudgets})`, + ); + } else if (containmentSignalsPresent) { + notes.push( + "Containment signals alone are PARTIAL — import pauseAgentsDemonstrated=true + disableToolsDemonstrated=true + rollbackPromptOrModelDemonstrated=true + withinDocumentedTimeBudgets=true (measuredAt ≤90d) under imports/ai-containment-drill/ to PASS.", + ); + } + + const ageOk = + opts.imported.ageDays === null || + opts.imported.ageDays <= IMPORT_MAX_AGE_DAYS; + const pauseOk = opts.imported.pauseAgentsDemonstrated === true; + const disableOk = opts.imported.disableToolsDemonstrated === true; + const rollbackOk = opts.imported.rollbackPromptOrModelDemonstrated === true; + const budgetOk = opts.imported.withinDocumentedTimeBudgets === true; + const importFresh = measuredAtFresh(opts.imported.measuredAt); + + let statusHint: AiContainmentDrillReport["summary"]["statusHint"]; + let incM2Satisfied: boolean | null = null; + + const explicitFail = + opts.imported.found && + (opts.imported.pauseAgentsDemonstrated === false || + opts.imported.disableToolsDemonstrated === false || + opts.imported.rollbackPromptOrModelDemonstrated === false || + opts.imported.withinDocumentedTimeBudgets === false || + (opts.imported.ageDays !== null && + opts.imported.ageDays > IMPORT_MAX_AGE_DAYS)); + + if (!containmentSignalsPresent && !opts.imported.found) { + statusHint = "not_applicable"; + incM2Satisfied = null; + } else if (explicitFail) { + statusHint = "fail"; + incM2Satisfied = false; + notes.push( + "Imported evidence shows missing pause/disable/rollback demonstration, missed time budgets, or evidence older than 90 days — INC-M2 fail.", + ); + } else if ( + (containmentSignalsPresent || opts.imported.found) && + pauseOk && + disableOk && + rollbackOk && + budgetOk && + ageOk && + importFresh && + opts.imported.found + ) { + statusHint = "pass"; + incM2Satisfied = true; + } else if (containmentSignalsPresent || opts.imported.found) { + statusHint = "partial"; + incM2Satisfied = false; + if (opts.imported.found && !pauseOk) { + notes.push("Import must show pauseAgentsDemonstrated=true."); + } + if (opts.imported.found && !disableOk) { + notes.push("Import must show disableToolsDemonstrated=true."); + } + if (opts.imported.found && !rollbackOk) { + notes.push("Import must show rollbackPromptOrModelDemonstrated=true."); + } + if (opts.imported.found && !budgetOk) { + notes.push("Import must show withinDocumentedTimeBudgets=true."); + } + if (opts.imported.found && !importFresh) { + notes.push( + "Import missing fresh measuredAt (≤90 days) — required to unlock INC-M2 PASS.", + ); + } + } else { + statusHint = "not_demonstrated"; + incM2Satisfied = null; + } + + return { + schemaVersion: "0.2.0", + pluginId: PLUGIN_ID, + detectorId: DETECTOR_ID, + relatedCheckIds: [...RELATED], + assessedAt: opts.assessedAt, + signals: { + pause: opts.pause, + disableTools: opts.disableTools, + rollback: opts.rollback, + drillOrBudget: opts.drillOrBudget, + }, + importedResults: opts.imported, + summary: { + containmentSignalsPresent, + actionSignalCount, + incM2Satisfied, + statusHint, + }, + notes, + }; +} + +export const aiContainmentDrillCollector: Collector = { + id: PLUGIN_ID, + async collect(ctx: CollectorContext): Promise { + const maxFiles = ctx.maxFiles ?? 8000; + + const pause = collectRefs( + ctx.targetPath, + maxFiles, + (path, text) => PAUSE_RE.test(path) || PAUSE_RE.test(text), + 8, + ); + const disableTools = collectRefs( + ctx.targetPath, + maxFiles, + (path, text) => + DISABLE_TOOLS_RE.test(path) || DISABLE_TOOLS_RE.test(text), + 8, + ); + const rollback = collectRefs( + ctx.targetPath, + maxFiles, + (path, text) => ROLLBACK_RE.test(path) || ROLLBACK_RE.test(text), + 8, + ); + const drillOrBudget = collectRefs( + ctx.targetPath, + maxFiles, + (path, text) => + ((CONTAINMENT_RE.test(path) || + CONTAINMENT_RE.test(text) || + PAUSE_RE.test(text) || + DISABLE_TOOLS_RE.test(text) || + ROLLBACK_RE.test(text)) && + (DRILL_RE.test(text) || TIME_BUDGET_RE.test(text))) || + DRILL_RE.test(path), + 8, + ); + + const imported = loadImported(ctx); + const report = buildAiContainmentDrillReport({ + assessedAt: ctx.assessedAt.toISOString(), + pause: { found: pause.length > 0, refs: pause }, + disableTools: { found: disableTools.length > 0, refs: disableTools }, + rollback: { found: rollback.length > 0, refs: rollback }, + drillOrBudget: { found: drillOrBudget.length > 0, refs: drillOrBudget }, + imported, + }); + + ensureDir(importDir(ctx)); + writeFileSync( + join(importDir(ctx), "ai-containment-drill-report.json"), + JSON.stringify(report, null, 2) + "\n", + "utf8", + ); + + const nodes: EvidenceNode[] = [ + { + id: `${PLUGIN_ID}:report`, + class: "ci", + ref: `imports/${PLUGIN_ID}/ai-containment-drill-report.json`, + pluginId: PLUGIN_ID, + signals: [ + "ai-containment-drill", + "inc-m2", + DETECTOR_ID, + ...(report.summary.incM2Satisfied ? ["inc-m2-satisfied"] : []), + ], + excerpt: redact(report.notes.slice(0, 3).join(" | ").slice(0, 400)), + relatedCheckIds: [...RELATED], + }, + ]; + for (const r of [ + ...new Set([ + ...report.signals.pause.refs, + ...report.signals.disableTools.refs, + ...report.signals.rollback.refs, + ...report.signals.drillOrBudget.refs, + ]), + ].slice(0, 8)) { + nodes.push({ + id: `${PLUGIN_ID}:ref:${r}`, + class: "ci", + ref: r, + pluginId: PLUGIN_ID, + signals: ["ai-containment-drill-ref"], + relatedCheckIds: [...RELATED], + }); + } + + return { + pluginId: PLUGIN_ID, + status: "ran", + detail: `INC-M2 status=${report.summary.statusHint} actions=${report.summary.actionSignalCount}/3 satisfied=${report.summary.incM2Satisfied}; report=imports/${PLUGIN_ID}/ai-containment-drill-report.json`, + nodes, + }; + }, +}; diff --git a/skills/aprf-auditor/collectors/ai-customer-notification-criteria.ts b/skills/aprf-auditor/collectors/ai-customer-notification-criteria.ts new file mode 100644 index 0000000..a3a7341 --- /dev/null +++ b/skills/aprf-auditor/collectors/ai-customer-notification-criteria.ts @@ -0,0 +1,383 @@ +/** + * ai-customer-notification-criteria — INC-R3 / + * repo-ai-customer-notification-criteria. + * + * Discovers AI customer notification criteria + drill/incident samples. + * Import criteriaMapEventTypesToNotifyDecision + + * lastDrillOrIncidentFollowedCriteriaWithin12Months + timestampsPresent + * under imports/ai-customer-notification-criteria/ to unlock PASS + * (measuredAt ≤90d). + */ +import { writeFileSync } from "node:fs"; +import { join, basename } from "node:path"; +import type { + Collector, + CollectorContext, + CollectorResult, + EvidenceNode, +} from "./types.ts"; +import { + ensureDir, + listImportFiles, + readText, + redact, + rel, + walkFiles, +} from "./lib/fs.ts"; +import { + asBool, + measuredAtFresh, + parseMeasuredAt, +} from "./lib/import-attest.ts"; + +const PLUGIN_ID = "ai-customer-notification-criteria"; +const RELATED = ["INC-R3"] as const; +const DETECTOR_ID = "repo-ai-customer-notification-criteria"; +const IMPORT_MAX_AGE_DAYS = 90; +const FOLLOWED_MAX_AGE_DAYS = 365; + +const SKIP_DIR_HINT = + /(^|[/\\])(node_modules|\.git|dist|build|coverage|\.venv|venv|__pycache__|vendor)([/\\]|$)/i; + +const CRITERIA_RE = + /\b(customer[\s_-]*notif|user[\s_-]*notif|notify[\s_-]*customer|disclosure[\s_-]*criter|notification[\s_-]*criter|notify[\s_-]*\/[\s_-]*no[\s_-]*notify|no[\s_-]*notify)\b/i; + +const EVENT_TYPE_RE = + /\b(safety[\s_-]*incident|quality[\s_-]*fail|data[\s_-]*exposure|data[\s_-]*breach|widespread[\s_-]*quality|ai[\s_-]*incident)\b/i; + +const SAMPLE_RE = + /\b(notification[\s_-]*drill|comms[\s_-]*drill|customer[\s_-]*comms|status[\s_-]*page|notification[\s_-]*sample|followed[\s_-]*criter)\b/i; + +const TIMESTAMP_RE = + /\b(timestamp|notified[\s_-]*at|sent[\s_-]*at|decision[\s_-]*at|\d{4}-\d{2}-\d{2})\b/i; + +export interface AiCustomerNotificationCriteriaReport { + schemaVersion: "0.2.0"; + pluginId: typeof PLUGIN_ID; + detectorId: typeof DETECTOR_ID; + relatedCheckIds: string[]; + assessedAt: string; + signals: { + criteria: { found: boolean; refs: string[] }; + sample: { found: boolean; refs: string[] }; + }; + importedResults: { + found: boolean; + criteriaMapEventTypesToNotifyDecision: boolean | null; + lastDrillOrIncidentFollowedCriteriaWithin12Months: boolean | null; + lastDrillOrIncidentAgeDays: number | null; + timestampsPresent: boolean | null; + ageDays: number | null; + measuredAt: string | null; + sources: string[]; + }; + summary: { + notificationSignalsPresent: boolean; + incR3Satisfied: boolean | null; + statusHint: + | "pass" + | "partial" + | "fail" + | "not_demonstrated" + | "not_applicable"; + }; + notes: string[]; +} + +function importDir(ctx: CollectorContext): string { + return join(ctx.outputDir, "imports", PLUGIN_ID); +} + +function isSkippable(path: string): boolean { + return SKIP_DIR_HINT.test(path); +} + +function asNum(v: unknown): number | null { + return typeof v === "number" && Number.isFinite(v) ? v : null; +} + +function collectRefs( + targetPath: string, + maxFiles: number, + match: (path: string, text: string) => boolean, + limit = 16, +): string[] { + const refs: string[] = []; + const files = walkFiles(targetPath, { + maxFiles: Math.max(maxFiles, 5000), + extensions: [".md", ".txt", ".yml", ".yaml", ".json", ".html"], + }); + for (const f of files) { + const r = rel(targetPath, f); + if (isSkippable(r)) continue; + const text = readText(f, 80_000) || ""; + if (match(r, text)) refs.push(r); + if (refs.length >= limit) break; + } + return [...new Set(refs)]; +} + +function loadImported( + ctx: CollectorContext, +): AiCustomerNotificationCriteriaReport["importedResults"] { + const sources: string[] = []; + let criteriaMapEventTypesToNotifyDecision: boolean | null = null; + let lastDrillOrIncidentFollowedCriteriaWithin12Months: boolean | null = null; + let lastDrillOrIncidentAgeDays: number | null = null; + let timestampsPresent: boolean | null = null; + let ageDays: number | null = null; + let measuredAt: string | null = null; + + for (const f of listImportFiles(ctx.outputDir, PLUGIN_ID)) { + if (/ai-customer-notification-criteria-report\.json$/i.test(f)) continue; + const text = readText(f); + if (!text) continue; + try { + const data = JSON.parse(text) as Record; + sources.push(basename(f)); + measuredAt = parseMeasuredAt(data) ?? measuredAt; + ageDays = asNum(data.ageDays) ?? asNum(data.age_days) ?? ageDays; + lastDrillOrIncidentAgeDays = + asNum(data.lastDrillOrIncidentAgeDays) ?? + asNum(data.last_drill_or_incident_age_days) ?? + lastDrillOrIncidentAgeDays; + criteriaMapEventTypesToNotifyDecision = + asBool(data.criteriaMapEventTypesToNotifyDecision) ?? + asBool(data.criteria_map_event_types_to_notify_decision) ?? + asBool(data.criteriaMapPresent) ?? + criteriaMapEventTypesToNotifyDecision; + lastDrillOrIncidentFollowedCriteriaWithin12Months = + asBool(data.lastDrillOrIncidentFollowedCriteriaWithin12Months) ?? + asBool(data.last_drill_or_incident_followed_criteria_within_12_months) ?? + asBool(data.followedCriteriaWithin12Months) ?? + lastDrillOrIncidentFollowedCriteriaWithin12Months; + timestampsPresent = + asBool(data.timestampsPresent) ?? + asBool(data.timestamps_present) ?? + timestampsPresent; + + if (lastDrillOrIncidentAgeDays !== null) { + lastDrillOrIncidentFollowedCriteriaWithin12Months = + lastDrillOrIncidentFollowedCriteriaWithin12Months ?? + lastDrillOrIncidentAgeDays <= FOLLOWED_MAX_AGE_DAYS; + } + } catch { + /* skip */ + } + } + + return { + found: sources.length > 0, + criteriaMapEventTypesToNotifyDecision, + lastDrillOrIncidentFollowedCriteriaWithin12Months, + lastDrillOrIncidentAgeDays, + timestampsPresent, + ageDays, + measuredAt, + sources, + }; +} + +export function buildAiCustomerNotificationCriteriaReport(opts: { + assessedAt: string; + criteria: { found: boolean; refs: string[] }; + sample: { found: boolean; refs: string[] }; + imported: AiCustomerNotificationCriteriaReport["importedResults"]; +}): AiCustomerNotificationCriteriaReport { + const notes: string[] = []; + const notificationSignalsPresent = + opts.criteria.found || opts.sample.found; + + if (!notificationSignalsPresent && !opts.imported.found) { + notes.push( + "No AI customer-notification signals — INC-R3 may be NOT_APPLICABLE if no customer-facing AI system is in scope.", + ); + } + if (opts.criteria.found) { + notes.push(`Criteria refs: ${opts.criteria.refs.slice(0, 4).join(", ")}`); + } + if (opts.sample.found) { + notes.push(`Sample refs: ${opts.sample.refs.slice(0, 3).join(", ")}`); + } + if (opts.imported.found) { + notes.push( + `Imported: ${opts.imported.sources.join(", ")} (criteriaMap=${opts.imported.criteriaMapEventTypesToNotifyDecision}, followed12m=${opts.imported.lastDrillOrIncidentFollowedCriteriaWithin12Months}, age=${opts.imported.lastDrillOrIncidentAgeDays}, timestamps=${opts.imported.timestampsPresent})`, + ); + } else if (notificationSignalsPresent) { + notes.push( + "Notification signals alone are PARTIAL — import criteriaMapEventTypesToNotifyDecision=true + lastDrillOrIncidentFollowedCriteriaWithin12Months=true (measuredAt ≤90d) under imports/ai-customer-notification-criteria/ to PASS.", + ); + } + + const ageOk = + opts.imported.ageDays === null || + opts.imported.ageDays <= IMPORT_MAX_AGE_DAYS; + const criteriaOk = + opts.imported.criteriaMapEventTypesToNotifyDecision === true; + const followedOk = + opts.imported.lastDrillOrIncidentFollowedCriteriaWithin12Months === true || + (opts.imported.lastDrillOrIncidentAgeDays !== null && + opts.imported.lastDrillOrIncidentAgeDays <= FOLLOWED_MAX_AGE_DAYS); + const timestampsOk = opts.imported.timestampsPresent === true; + const importFresh = measuredAtFresh(opts.imported.measuredAt); + + let statusHint: AiCustomerNotificationCriteriaReport["summary"]["statusHint"]; + let incR3Satisfied: boolean | null = null; + + const explicitFail = + opts.imported.found && + (opts.imported.criteriaMapEventTypesToNotifyDecision === false || + opts.imported.lastDrillOrIncidentFollowedCriteriaWithin12Months === + false || + (typeof opts.imported.lastDrillOrIncidentAgeDays === "number" && + opts.imported.lastDrillOrIncidentAgeDays > FOLLOWED_MAX_AGE_DAYS) || + opts.imported.timestampsPresent === false || + (opts.imported.ageDays !== null && + opts.imported.ageDays > IMPORT_MAX_AGE_DAYS)); + + if (!notificationSignalsPresent && !opts.imported.found) { + statusHint = "not_applicable"; + incR3Satisfied = null; + } else if (explicitFail) { + statusHint = "fail"; + incR3Satisfied = false; + notes.push( + "Imported evidence shows missing criteria map, stale/unfollowed sample (>12 months), missing timestamps, or evidence older than 90 days — INC-R3 fail.", + ); + } else if ( + (notificationSignalsPresent || opts.imported.found) && + criteriaOk && + followedOk && + timestampsOk && + ageOk && + importFresh && + opts.imported.found + ) { + statusHint = "pass"; + incR3Satisfied = true; + } else if (notificationSignalsPresent || opts.imported.found) { + statusHint = "partial"; + incR3Satisfied = false; + if (opts.imported.found && !criteriaOk) { + notes.push( + "Import must show criteriaMapEventTypesToNotifyDecision=true.", + ); + } + if (opts.imported.found && !followedOk) { + notes.push( + "Import must show lastDrillOrIncidentFollowedCriteriaWithin12Months=true (or lastDrillOrIncidentAgeDays≤365).", + ); + } + if (opts.imported.found && !timestampsOk) { + notes.push("Import must show timestampsPresent=true."); + } + if (opts.imported.found && !importFresh) { + notes.push( + "Import missing fresh measuredAt (≤90 days) — required to unlock INC-R3 PASS.", + ); + } + } else { + statusHint = "not_demonstrated"; + incR3Satisfied = null; + } + + return { + schemaVersion: "0.2.0", + pluginId: PLUGIN_ID, + detectorId: DETECTOR_ID, + relatedCheckIds: [...RELATED], + assessedAt: opts.assessedAt, + signals: { + criteria: opts.criteria, + sample: opts.sample, + }, + importedResults: opts.imported, + summary: { + notificationSignalsPresent, + incR3Satisfied, + statusHint, + }, + notes, + }; +} + +export const aiCustomerNotificationCriteriaCollector: Collector = { + id: PLUGIN_ID, + async collect(ctx: CollectorContext): Promise { + const maxFiles = ctx.maxFiles ?? 8000; + + const criteria = collectRefs( + ctx.targetPath, + maxFiles, + (path, text) => + (CRITERIA_RE.test(path) || CRITERIA_RE.test(text)) && + (EVENT_TYPE_RE.test(path + text) || + /ai|llm|safety|exposure/i.test(path + text)), + 10, + ); + const sample = collectRefs( + ctx.targetPath, + maxFiles, + (path, text) => + (SAMPLE_RE.test(path) || SAMPLE_RE.test(text)) && + (CRITERIA_RE.test(path + text) || + TIMESTAMP_RE.test(text) || + EVENT_TYPE_RE.test(text)), + 10, + ); + + const imported = loadImported(ctx); + const report = buildAiCustomerNotificationCriteriaReport({ + assessedAt: ctx.assessedAt.toISOString(), + criteria: { found: criteria.length > 0, refs: criteria }, + sample: { found: sample.length > 0, refs: sample }, + imported, + }); + + ensureDir(importDir(ctx)); + writeFileSync( + join(importDir(ctx), "ai-customer-notification-criteria-report.json"), + JSON.stringify(report, null, 2) + "\n", + "utf8", + ); + + const nodes: EvidenceNode[] = [ + { + id: `${PLUGIN_ID}:report`, + class: "ci", + ref: `imports/${PLUGIN_ID}/ai-customer-notification-criteria-report.json`, + pluginId: PLUGIN_ID, + signals: [ + "ai-customer-notification-criteria", + "inc-r3", + DETECTOR_ID, + ...(report.summary.incR3Satisfied ? ["inc-r3-satisfied"] : []), + ], + excerpt: redact(report.notes.slice(0, 3).join(" | ").slice(0, 400)), + relatedCheckIds: [...RELATED], + }, + ]; + for (const r of [ + ...new Set([ + ...report.signals.criteria.refs, + ...report.signals.sample.refs, + ]), + ].slice(0, 6)) { + nodes.push({ + id: `${PLUGIN_ID}:ref:${r}`, + class: "ci", + ref: r, + pluginId: PLUGIN_ID, + signals: ["ai-customer-notification-criteria-ref"], + relatedCheckIds: [...RELATED], + }); + } + + return { + pluginId: PLUGIN_ID, + status: "ran", + detail: `INC-R3 status=${report.summary.statusHint} signals=${report.summary.notificationSignalsPresent} satisfied=${report.summary.incR3Satisfied}; report=imports/${PLUGIN_ID}/ai-customer-notification-criteria-report.json`, + nodes, + }; + }, +}; diff --git a/skills/aprf-auditor/collectors/ai-incident-tabletop.ts b/skills/aprf-auditor/collectors/ai-incident-tabletop.ts new file mode 100644 index 0000000..15b39a8 --- /dev/null +++ b/skills/aprf-auditor/collectors/ai-incident-tabletop.ts @@ -0,0 +1,363 @@ +/** + * ai-incident-tabletop — INC-R4 / repo-ai-incident-tabletop. + * + * Discovers AI-focused incident tabletops + after-action reports. + * Import aiFocusedTabletopCompletedWithin180Days + + * retainedActionsWithOwners under imports/ai-incident-tabletop/ + * to unlock PASS (measuredAt ≤90d). + */ +import { writeFileSync } from "node:fs"; +import { join, basename } from "node:path"; +import type { + Collector, + CollectorContext, + CollectorResult, + EvidenceNode, +} from "./types.ts"; +import { + ensureDir, + listImportFiles, + readText, + redact, + rel, + walkFiles, +} from "./lib/fs.ts"; +import { + asBool, + measuredAtFresh, + parseMeasuredAt, +} from "./lib/import-attest.ts"; + +const PLUGIN_ID = "ai-incident-tabletop"; +const RELATED = ["INC-R4"] as const; +const DETECTOR_ID = "repo-ai-incident-tabletop"; +const IMPORT_MAX_AGE_DAYS = 90; +const TABLETOP_MAX_AGE_DAYS = 180; + +const SKIP_DIR_HINT = + /(^|[/\\])(node_modules|\.git|dist|build|coverage|\.venv|venv|__pycache__|vendor)([/\\]|$)/i; + +const TABLETOP_RE = + /\b(table[\s_-]*top|tabletop|war[\s_-]*game|game[\s_-]*day|scenario[\s_-]*exercise)\b/i; + +const AI_SCENARIO_RE = + /\b(ai[\s_-]*incident|prompt[\s_-]*injection|model[\s_-]*outage|agent[\s_-]*abuse|data[\s_-]*leakage|jailbreak|llm[\s_-]*incident)\b/i; + +const AFTER_ACTION_RE = + /\b(after[\s_-]*action|aar|retained[\s_-]*action|action[\s_-]*item|follow[\s_-]*up|owner)\b/i; + +export interface AiIncidentTabletopReport { + schemaVersion: "0.2.0"; + pluginId: typeof PLUGIN_ID; + detectorId: typeof DETECTOR_ID; + relatedCheckIds: string[]; + assessedAt: string; + signals: { + tabletop: { found: boolean; refs: string[] }; + afterAction: { found: boolean; refs: string[] }; + }; + importedResults: { + found: boolean; + aiFocusedTabletopCompletedWithin180Days: boolean | null; + tabletopAgeDays: number | null; + retainedActionsWithOwners: boolean | null; + ageDays: number | null; + measuredAt: string | null; + sources: string[]; + }; + summary: { + tabletopSignalsPresent: boolean; + incR4Satisfied: boolean | null; + statusHint: + | "pass" + | "partial" + | "fail" + | "not_demonstrated" + | "not_applicable"; + }; + notes: string[]; +} + +function importDir(ctx: CollectorContext): string { + return join(ctx.outputDir, "imports", PLUGIN_ID); +} + +function isSkippable(path: string): boolean { + return SKIP_DIR_HINT.test(path); +} + +function asNum(v: unknown): number | null { + return typeof v === "number" && Number.isFinite(v) ? v : null; +} + +function collectRefs( + targetPath: string, + maxFiles: number, + match: (path: string, text: string) => boolean, + limit = 16, +): string[] { + const refs: string[] = []; + const files = walkFiles(targetPath, { + maxFiles: Math.max(maxFiles, 5000), + extensions: [".md", ".txt", ".yml", ".yaml", ".json", ".html", ".pptx"], + }); + for (const f of files) { + const r = rel(targetPath, f); + if (isSkippable(r)) continue; + const text = readText(f, 80_000) || ""; + if (match(r, text)) refs.push(r); + if (refs.length >= limit) break; + } + return [...new Set(refs)]; +} + +function loadImported( + ctx: CollectorContext, +): AiIncidentTabletopReport["importedResults"] { + const sources: string[] = []; + let aiFocusedTabletopCompletedWithin180Days: boolean | null = null; + let tabletopAgeDays: number | null = null; + let retainedActionsWithOwners: boolean | null = null; + let ageDays: number | null = null; + let measuredAt: string | null = null; + + for (const f of listImportFiles(ctx.outputDir, PLUGIN_ID)) { + if (/ai-incident-tabletop-report\.json$/i.test(f)) continue; + const text = readText(f); + if (!text) continue; + try { + const data = JSON.parse(text) as Record; + sources.push(basename(f)); + measuredAt = parseMeasuredAt(data) ?? measuredAt; + ageDays = asNum(data.ageDays) ?? asNum(data.age_days) ?? ageDays; + tabletopAgeDays = + asNum(data.tabletopAgeDays) ?? + asNum(data.tabletop_age_days) ?? + tabletopAgeDays; + aiFocusedTabletopCompletedWithin180Days = + asBool(data.aiFocusedTabletopCompletedWithin180Days) ?? + asBool(data.ai_focused_tabletop_completed_within_180_days) ?? + asBool(data.tabletopCompleted) ?? + aiFocusedTabletopCompletedWithin180Days; + retainedActionsWithOwners = + asBool(data.retainedActionsWithOwners) ?? + asBool(data.retained_actions_with_owners) ?? + asBool(data.actionsHaveOwners) ?? + retainedActionsWithOwners; + + if (tabletopAgeDays !== null) { + aiFocusedTabletopCompletedWithin180Days = + aiFocusedTabletopCompletedWithin180Days ?? + tabletopAgeDays <= TABLETOP_MAX_AGE_DAYS; + } + } catch { + /* skip */ + } + } + + return { + found: sources.length > 0, + aiFocusedTabletopCompletedWithin180Days, + tabletopAgeDays, + retainedActionsWithOwners, + ageDays, + measuredAt, + sources, + }; +} + +export function buildAiIncidentTabletopReport(opts: { + assessedAt: string; + tabletop: { found: boolean; refs: string[] }; + afterAction: { found: boolean; refs: string[] }; + imported: AiIncidentTabletopReport["importedResults"]; +}): AiIncidentTabletopReport { + const notes: string[] = []; + const tabletopSignalsPresent = + opts.tabletop.found || opts.afterAction.found; + + if (!tabletopSignalsPresent && !opts.imported.found) { + notes.push( + "No AI tabletop signals — INC-R4 may be NOT_APPLICABLE if no production AI system is in scope.", + ); + } + if (opts.tabletop.found) { + notes.push(`Tabletop refs: ${opts.tabletop.refs.slice(0, 4).join(", ")}`); + } + if (opts.afterAction.found) { + notes.push( + `After-action refs: ${opts.afterAction.refs.slice(0, 3).join(", ")}`, + ); + } + if (opts.imported.found) { + notes.push( + `Imported: ${opts.imported.sources.join(", ")} (completed180d=${opts.imported.aiFocusedTabletopCompletedWithin180Days}, age=${opts.imported.tabletopAgeDays}, actionsOwned=${opts.imported.retainedActionsWithOwners})`, + ); + } else if (tabletopSignalsPresent) { + notes.push( + "Tabletop signals alone are PARTIAL — import aiFocusedTabletopCompletedWithin180Days=true + retainedActionsWithOwners=true (measuredAt ≤90d) under imports/ai-incident-tabletop/ to PASS.", + ); + } + + const ageOk = + opts.imported.ageDays === null || + opts.imported.ageDays <= IMPORT_MAX_AGE_DAYS; + const completedOk = + opts.imported.aiFocusedTabletopCompletedWithin180Days === true || + (opts.imported.tabletopAgeDays !== null && + opts.imported.tabletopAgeDays <= TABLETOP_MAX_AGE_DAYS); + const actionsOk = opts.imported.retainedActionsWithOwners === true; + const importFresh = measuredAtFresh(opts.imported.measuredAt); + + let statusHint: AiIncidentTabletopReport["summary"]["statusHint"]; + let incR4Satisfied: boolean | null = null; + + const explicitFail = + opts.imported.found && + (opts.imported.aiFocusedTabletopCompletedWithin180Days === false || + (typeof opts.imported.tabletopAgeDays === "number" && + opts.imported.tabletopAgeDays > TABLETOP_MAX_AGE_DAYS) || + opts.imported.retainedActionsWithOwners === false || + (opts.imported.ageDays !== null && + opts.imported.ageDays > IMPORT_MAX_AGE_DAYS)); + + if (!tabletopSignalsPresent && !opts.imported.found) { + statusHint = "not_applicable"; + incR4Satisfied = null; + } else if (explicitFail) { + statusHint = "fail"; + incR4Satisfied = false; + notes.push( + "Imported evidence shows missing/stale AI tabletop (>180d), actions without owners, or evidence older than 90 days — INC-R4 fail.", + ); + } else if ( + (tabletopSignalsPresent || opts.imported.found) && + completedOk && + actionsOk && + ageOk && + importFresh && + opts.imported.found + ) { + statusHint = "pass"; + incR4Satisfied = true; + } else if (tabletopSignalsPresent || opts.imported.found) { + statusHint = "partial"; + incR4Satisfied = false; + if (opts.imported.found && !completedOk) { + notes.push( + "Import must show aiFocusedTabletopCompletedWithin180Days=true (or tabletopAgeDays≤180).", + ); + } + if (opts.imported.found && !actionsOk) { + notes.push("Import must show retainedActionsWithOwners=true."); + } + if (opts.imported.found && !importFresh) { + notes.push( + "Import missing fresh measuredAt (≤90 days) — required to unlock INC-R4 PASS.", + ); + } + } else { + statusHint = "not_demonstrated"; + incR4Satisfied = null; + } + + return { + schemaVersion: "0.2.0", + pluginId: PLUGIN_ID, + detectorId: DETECTOR_ID, + relatedCheckIds: [...RELATED], + assessedAt: opts.assessedAt, + signals: { + tabletop: opts.tabletop, + afterAction: opts.afterAction, + }, + importedResults: opts.imported, + summary: { + tabletopSignalsPresent, + incR4Satisfied, + statusHint, + }, + notes, + }; +} + +export const aiIncidentTabletopCollector: Collector = { + id: PLUGIN_ID, + async collect(ctx: CollectorContext): Promise { + const maxFiles = ctx.maxFiles ?? 8000; + + const tabletop = collectRefs( + ctx.targetPath, + maxFiles, + (path, text) => + (TABLETOP_RE.test(path) || TABLETOP_RE.test(text)) && + (AI_SCENARIO_RE.test(path) || + AI_SCENARIO_RE.test(text) || + /ai|llm|agent|prompt/i.test(path + text)), + 10, + ); + const afterAction = collectRefs( + ctx.targetPath, + maxFiles, + (path, text) => + AFTER_ACTION_RE.test(path) || + ((TABLETOP_RE.test(path) || TABLETOP_RE.test(text)) && + AFTER_ACTION_RE.test(text)), + 10, + ); + + const imported = loadImported(ctx); + const report = buildAiIncidentTabletopReport({ + assessedAt: ctx.assessedAt.toISOString(), + tabletop: { found: tabletop.length > 0, refs: tabletop }, + afterAction: { found: afterAction.length > 0, refs: afterAction }, + imported, + }); + + ensureDir(importDir(ctx)); + writeFileSync( + join(importDir(ctx), "ai-incident-tabletop-report.json"), + JSON.stringify(report, null, 2) + "\n", + "utf8", + ); + + const nodes: EvidenceNode[] = [ + { + id: `${PLUGIN_ID}:report`, + class: "ci", + ref: `imports/${PLUGIN_ID}/ai-incident-tabletop-report.json`, + pluginId: PLUGIN_ID, + signals: [ + "ai-incident-tabletop", + "inc-r4", + DETECTOR_ID, + ...(report.summary.incR4Satisfied ? ["inc-r4-satisfied"] : []), + ], + excerpt: redact(report.notes.slice(0, 3).join(" | ").slice(0, 400)), + relatedCheckIds: [...RELATED], + }, + ]; + for (const r of [ + ...new Set([ + ...report.signals.tabletop.refs, + ...report.signals.afterAction.refs, + ]), + ].slice(0, 6)) { + nodes.push({ + id: `${PLUGIN_ID}:ref:${r}`, + class: "ci", + ref: r, + pluginId: PLUGIN_ID, + signals: ["ai-incident-tabletop-ref"], + relatedCheckIds: [...RELATED], + }); + } + + return { + pluginId: PLUGIN_ID, + status: "ran", + detail: `INC-R4 status=${report.summary.statusHint} tabletop=${report.summary.tabletopSignalsPresent} satisfied=${report.summary.incR4Satisfied}; report=imports/${PLUGIN_ID}/ai-incident-tabletop-report.json`, + nodes, + }; + }, +}; diff --git a/skills/aprf-auditor/collectors/ai-safety-quality-alerts.ts b/skills/aprf-auditor/collectors/ai-safety-quality-alerts.ts new file mode 100644 index 0000000..2a6fbd1 --- /dev/null +++ b/skills/aprf-auditor/collectors/ai-safety-quality-alerts.ts @@ -0,0 +1,375 @@ +/** + * ai-safety-quality-alerts — INC-R1 / repo-ai-safety-quality-alerts. + * + * Discovers on-call paging for non-infra AI safety/quality signals. + * Import atLeastTwoNonInfraPagingSignals + eachSignalHasThresholdAndOwner + + * policyReviewedWithin90Days under imports/ai-safety-quality-alerts/ + * to unlock PASS (measuredAt ≤90d). + */ +import { writeFileSync } from "node:fs"; +import { join, basename } from "node:path"; +import type { + Collector, + CollectorContext, + CollectorResult, + EvidenceNode, +} from "./types.ts"; +import { + ensureDir, + listImportFiles, + readText, + redact, + rel, + walkFiles, +} from "./lib/fs.ts"; +import { + asBool, + measuredAtFresh, + parseMeasuredAt, +} from "./lib/import-attest.ts"; + +const PLUGIN_ID = "ai-safety-quality-alerts"; +const RELATED = ["INC-R1"] as const; +const DETECTOR_ID = "repo-ai-safety-quality-alerts"; +const IMPORT_MAX_AGE_DAYS = 90; + +const SKIP_DIR_HINT = + /(^|[/\\])(node_modules|\.git|dist|build|coverage|\.venv|venv|__pycache__|vendor)([/\\]|$)/i; + +const SAFETY_QUALITY_RE = + /\b(refusal[\s_-]*rate|eval[\s_-]*score|toxicity|jailbreak|hallucinat|safety[\s_-]*signal|quality[\s_-]*signal|prompt[\s_-]*injection[\s_-]*rate|guardrail[\s_-]*hit|content[\s_-]*filter)\b/i; + +const PAGING_RE = + /\b(page|paging|pagerduty|opsgenie|on[\s_-]*call|alert[\s_-]*policy|notification[\s_-]*policy|escalate|sev[\s_-]*page)\b/i; + +const THRESHOLD_OWNER_RE = + /\b(threshold|owner|runbook|alert[\s_-]*owner|paging[\s_-]*policy)\b/i; + +export interface AiSafetyQualityAlertsReport { + schemaVersion: "0.2.0"; + pluginId: typeof PLUGIN_ID; + detectorId: typeof DETECTOR_ID; + relatedCheckIds: string[]; + assessedAt: string; + signals: { + safetyQuality: { found: boolean; refs: string[] }; + paging: { found: boolean; refs: string[] }; + }; + importedResults: { + found: boolean; + atLeastTwoNonInfraPagingSignals: boolean | null; + nonInfraPagingSignalCount: number | null; + eachSignalHasThresholdAndOwner: boolean | null; + policyReviewedWithin90Days: boolean | null; + ageDays: number | null; + measuredAt: string | null; + sources: string[]; + }; + summary: { + alertSignalsPresent: boolean; + incR1Satisfied: boolean | null; + statusHint: + | "pass" + | "partial" + | "fail" + | "not_demonstrated" + | "not_applicable"; + }; + notes: string[]; +} + +function importDir(ctx: CollectorContext): string { + return join(ctx.outputDir, "imports", PLUGIN_ID); +} + +function isSkippable(path: string): boolean { + return SKIP_DIR_HINT.test(path); +} + +function asNum(v: unknown): number | null { + return typeof v === "number" && Number.isFinite(v) ? v : null; +} + +function collectRefs( + targetPath: string, + maxFiles: number, + match: (path: string, text: string) => boolean, + limit = 16, +): string[] { + const refs: string[] = []; + const files = walkFiles(targetPath, { + maxFiles: Math.max(maxFiles, 5000), + extensions: [".md", ".txt", ".yml", ".yaml", ".json", ".tf", ".hcl"], + }); + for (const f of files) { + const r = rel(targetPath, f); + if (isSkippable(r)) continue; + const text = readText(f, 80_000) || ""; + if (match(r, text)) refs.push(r); + if (refs.length >= limit) break; + } + return [...new Set(refs)]; +} + +function loadImported( + ctx: CollectorContext, +): AiSafetyQualityAlertsReport["importedResults"] { + const sources: string[] = []; + let atLeastTwoNonInfraPagingSignals: boolean | null = null; + let nonInfraPagingSignalCount: number | null = null; + let eachSignalHasThresholdAndOwner: boolean | null = null; + let policyReviewedWithin90Days: boolean | null = null; + let ageDays: number | null = null; + let measuredAt: string | null = null; + + for (const f of listImportFiles(ctx.outputDir, PLUGIN_ID)) { + if (/ai-safety-quality-alerts-report\.json$/i.test(f)) continue; + const text = readText(f); + if (!text) continue; + try { + const data = JSON.parse(text) as Record; + sources.push(basename(f)); + measuredAt = parseMeasuredAt(data) ?? measuredAt; + ageDays = asNum(data.ageDays) ?? asNum(data.age_days) ?? ageDays; + nonInfraPagingSignalCount = + asNum(data.nonInfraPagingSignalCount) ?? + asNum(data.non_infra_paging_signal_count) ?? + nonInfraPagingSignalCount; + atLeastTwoNonInfraPagingSignals = + asBool(data.atLeastTwoNonInfraPagingSignals) ?? + asBool(data.at_least_two_non_infra_paging_signals) ?? + atLeastTwoNonInfraPagingSignals; + eachSignalHasThresholdAndOwner = + asBool(data.eachSignalHasThresholdAndOwner) ?? + asBool(data.each_signal_has_threshold_and_owner) ?? + asBool(data.allSignalsHaveThresholdAndOwner) ?? + eachSignalHasThresholdAndOwner; + policyReviewedWithin90Days = + asBool(data.policyReviewedWithin90Days) ?? + asBool(data.policy_reviewed_within_90_days) ?? + policyReviewedWithin90Days; + + if (nonInfraPagingSignalCount !== null) { + atLeastTwoNonInfraPagingSignals = + atLeastTwoNonInfraPagingSignals ?? nonInfraPagingSignalCount >= 2; + } + } catch { + /* skip */ + } + } + + return { + found: sources.length > 0, + atLeastTwoNonInfraPagingSignals, + nonInfraPagingSignalCount, + eachSignalHasThresholdAndOwner, + policyReviewedWithin90Days, + ageDays, + measuredAt, + sources, + }; +} + +export function buildAiSafetyQualityAlertsReport(opts: { + assessedAt: string; + safetyQuality: { found: boolean; refs: string[] }; + paging: { found: boolean; refs: string[] }; + imported: AiSafetyQualityAlertsReport["importedResults"]; +}): AiSafetyQualityAlertsReport { + const notes: string[] = []; + const alertSignalsPresent = + opts.safetyQuality.found || opts.paging.found; + + if (!alertSignalsPresent && !opts.imported.found) { + notes.push( + "No AI safety/quality paging signals — INC-R1 may be NOT_APPLICABLE if no production AI system is in scope.", + ); + } + if (opts.safetyQuality.found) { + notes.push( + `Safety/quality refs: ${opts.safetyQuality.refs.slice(0, 4).join(", ")}`, + ); + } + if (opts.paging.found) { + notes.push(`Paging refs: ${opts.paging.refs.slice(0, 3).join(", ")}`); + } + if (opts.imported.found) { + notes.push( + `Imported: ${opts.imported.sources.join(", ")} (twoSignals=${opts.imported.atLeastTwoNonInfraPagingSignals}, count=${opts.imported.nonInfraPagingSignalCount}, thresholdOwner=${opts.imported.eachSignalHasThresholdAndOwner}, reviewed=${opts.imported.policyReviewedWithin90Days})`, + ); + } else if (alertSignalsPresent) { + notes.push( + "Alert signals alone are PARTIAL — import atLeastTwoNonInfraPagingSignals=true + eachSignalHasThresholdAndOwner=true + policyReviewedWithin90Days=true (measuredAt ≤90d) under imports/ai-safety-quality-alerts/ to PASS.", + ); + } + + const ageOk = + opts.imported.ageDays === null || + opts.imported.ageDays <= IMPORT_MAX_AGE_DAYS; + const twoSignalsOk = + opts.imported.atLeastTwoNonInfraPagingSignals === true || + (opts.imported.nonInfraPagingSignalCount !== null && + opts.imported.nonInfraPagingSignalCount >= 2); + const thresholdOwnerOk = + opts.imported.eachSignalHasThresholdAndOwner === true; + const reviewedOk = opts.imported.policyReviewedWithin90Days === true; + const importFresh = measuredAtFresh(opts.imported.measuredAt); + + let statusHint: AiSafetyQualityAlertsReport["summary"]["statusHint"]; + let incR1Satisfied: boolean | null = null; + + const explicitFail = + opts.imported.found && + (opts.imported.atLeastTwoNonInfraPagingSignals === false || + (typeof opts.imported.nonInfraPagingSignalCount === "number" && + opts.imported.nonInfraPagingSignalCount < 2) || + opts.imported.eachSignalHasThresholdAndOwner === false || + opts.imported.policyReviewedWithin90Days === false || + (opts.imported.ageDays !== null && + opts.imported.ageDays > IMPORT_MAX_AGE_DAYS)); + + if (!alertSignalsPresent && !opts.imported.found) { + statusHint = "not_applicable"; + incR1Satisfied = null; + } else if (explicitFail) { + statusHint = "fail"; + incR1Satisfied = false; + notes.push( + "Imported evidence shows <2 non-infra paging signals, missing threshold/owner, stale policy, or evidence older than 90 days — INC-R1 fail.", + ); + } else if ( + (alertSignalsPresent || opts.imported.found) && + twoSignalsOk && + thresholdOwnerOk && + reviewedOk && + ageOk && + importFresh && + opts.imported.found + ) { + statusHint = "pass"; + incR1Satisfied = true; + } else if (alertSignalsPresent || opts.imported.found) { + statusHint = "partial"; + incR1Satisfied = false; + if (opts.imported.found && !twoSignalsOk) { + notes.push( + "Import must show atLeastTwoNonInfraPagingSignals=true (or nonInfraPagingSignalCount≥2).", + ); + } + if (opts.imported.found && !thresholdOwnerOk) { + notes.push("Import must show eachSignalHasThresholdAndOwner=true."); + } + if (opts.imported.found && !reviewedOk) { + notes.push("Import must show policyReviewedWithin90Days=true."); + } + if (opts.imported.found && !importFresh) { + notes.push( + "Import missing fresh measuredAt (≤90 days) — required to unlock INC-R1 PASS.", + ); + } + } else { + statusHint = "not_demonstrated"; + incR1Satisfied = null; + } + + return { + schemaVersion: "0.2.0", + pluginId: PLUGIN_ID, + detectorId: DETECTOR_ID, + relatedCheckIds: [...RELATED], + assessedAt: opts.assessedAt, + signals: { + safetyQuality: opts.safetyQuality, + paging: opts.paging, + }, + importedResults: opts.imported, + summary: { + alertSignalsPresent, + incR1Satisfied, + statusHint, + }, + notes, + }; +} + +export const aiSafetyQualityAlertsCollector: Collector = { + id: PLUGIN_ID, + async collect(ctx: CollectorContext): Promise { + const maxFiles = ctx.maxFiles ?? 8000; + + const safetyQuality = collectRefs( + ctx.targetPath, + maxFiles, + (path, text) => + SAFETY_QUALITY_RE.test(path) || SAFETY_QUALITY_RE.test(text), + 10, + ); + const paging = collectRefs( + ctx.targetPath, + maxFiles, + (path, text) => + (PAGING_RE.test(path) || PAGING_RE.test(text)) && + (SAFETY_QUALITY_RE.test(path + text) || + THRESHOLD_OWNER_RE.test(text) || + /alert|pager|on[_-]?call/i.test(path)), + 10, + ); + + const imported = loadImported(ctx); + const report = buildAiSafetyQualityAlertsReport({ + assessedAt: ctx.assessedAt.toISOString(), + safetyQuality: { + found: safetyQuality.length > 0, + refs: safetyQuality, + }, + paging: { found: paging.length > 0, refs: paging }, + imported, + }); + + ensureDir(importDir(ctx)); + writeFileSync( + join(importDir(ctx), "ai-safety-quality-alerts-report.json"), + JSON.stringify(report, null, 2) + "\n", + "utf8", + ); + + const nodes: EvidenceNode[] = [ + { + id: `${PLUGIN_ID}:report`, + class: "ci", + ref: `imports/${PLUGIN_ID}/ai-safety-quality-alerts-report.json`, + pluginId: PLUGIN_ID, + signals: [ + "ai-safety-quality-alerts", + "inc-r1", + DETECTOR_ID, + ...(report.summary.incR1Satisfied ? ["inc-r1-satisfied"] : []), + ], + excerpt: redact(report.notes.slice(0, 3).join(" | ").slice(0, 400)), + relatedCheckIds: [...RELATED], + }, + ]; + for (const r of [ + ...new Set([ + ...report.signals.safetyQuality.refs, + ...report.signals.paging.refs, + ]), + ].slice(0, 6)) { + nodes.push({ + id: `${PLUGIN_ID}:ref:${r}`, + class: "ci", + ref: r, + pluginId: PLUGIN_ID, + signals: ["ai-safety-quality-alerts-ref"], + relatedCheckIds: [...RELATED], + }); + } + + return { + pluginId: PLUGIN_ID, + status: "ran", + detail: `INC-R1 status=${report.summary.statusHint} signals=${report.summary.alertSignalsPresent} satisfied=${report.summary.incR1Satisfied}; report=imports/${PLUGIN_ID}/ai-safety-quality-alerts-report.json`, + nodes, + }; + }, +}; diff --git a/skills/aprf-auditor/collectors/incident-playbooks.ts b/skills/aprf-auditor/collectors/incident-playbooks.ts new file mode 100644 index 0000000..9f5f5fc --- /dev/null +++ b/skills/aprf-auditor/collectors/incident-playbooks.ts @@ -0,0 +1,482 @@ +/** + * incident-playbooks — INC-M1 / repo-incident-playbooks. + * + * Discovers AI-specific incident playbooks for abuse, leakage, bad actions, + * and provider outage. Import fourPlaybooksPresent + allPlaybooksHaveOwner + + * allPlaybooksReviewedWithin12Months under imports/incident-playbooks/ + * to unlock PASS (measuredAt ≤90d). + */ +import { writeFileSync } from "node:fs"; +import { join, basename } from "node:path"; +import type { + Collector, + CollectorContext, + CollectorResult, + EvidenceNode, +} from "./types.ts"; +import { + ensureDir, + listImportFiles, + readText, + redact, + rel, + walkFiles, +} from "./lib/fs.ts"; +import { + asBool, + measuredAtFresh, + parseMeasuredAt, +} from "./lib/import-attest.ts"; + +const PLUGIN_ID = "incident-playbooks"; +const RELATED = ["INC-M1"] as const; +const DETECTOR_ID = "repo-incident-playbooks"; +const IMPORT_MAX_AGE_DAYS = 90; + +const SKIP_DIR_HINT = + /(^|[/\\])(node_modules|\.git|dist|build|coverage|\.venv|venv|__pycache__|vendor)([/\\]|$)/i; + +const PLAYBOOK_RE = + /\b(playbook|runbook|incident[\s_-]*response|ir[\s_-]*plan)\b/i; + +const ABUSE_RE = + /\b(abuse|prompt[\s_-]*injection|jailbreak|adversarial[\s_-]*use|misuse)\b/i; + +const LEAKAGE_RE = + /\b(leak(?:age)?|data[\s_-]*exfil|pii[\s_-]*expos|secret[\s_-]*expos|confidential[\s_-]*disclos)\b/i; + +const BAD_ACTIONS_RE = + /\b(bad[\s_-]*action|harmful[\s_-]*action|unsafe[\s_-]*tool|rogue[\s_-]*agent|unintended[\s_-]*action|tool[\s_-]*misuse)\b/i; + +const PROVIDER_OUTAGE_RE = + /\b(provider[\s_-]*outage|model[\s_-]*outage|llm[\s_-]*outage|api[\s_-]*outage|vendor[\s_-]*outage|openai[\s_-]*down|bedrock[\s_-]*outage)\b/i; + +const OWNER_RE = + /\b(owner|owned[\s_-]*by|playbook[\s_-]*owner|on[\s_-]*call[\s_-]*owner|responsible)\b/i; + +const REVIEW_RE = + /\b(reviewed?[\s_-]*(on|at|date)|last[\s_-]*review|review[\s_-]*date|next[\s_-]*review)\b/i; + +export interface IncidentPlaybooksReport { + schemaVersion: "0.2.0"; + pluginId: typeof PLUGIN_ID; + detectorId: typeof DETECTOR_ID; + relatedCheckIds: string[]; + assessedAt: string; + signals: { + abuse: { found: boolean; refs: string[] }; + leakage: { found: boolean; refs: string[] }; + badActions: { found: boolean; refs: string[] }; + providerOutage: { found: boolean; refs: string[] }; + ownerOrReview: { found: boolean; refs: string[] }; + }; + importedResults: { + found: boolean; + fourPlaybooksPresent: boolean | null; + allPlaybooksHaveOwner: boolean | null; + allPlaybooksReviewedWithin12Months: boolean | null; + missingPlaybookCount: number | null; + playbooksMissingOwner: number | null; + playbooksWithStaleReview: number | null; + ageDays: number | null; + measuredAt: string | null; + sources: string[]; + }; + summary: { + playbookSignalsPresent: boolean; + scenarioCoverageCount: number; + incM1Satisfied: boolean | null; + statusHint: + | "pass" + | "partial" + | "fail" + | "not_demonstrated" + | "not_applicable"; + }; + notes: string[]; +} + +function importDir(ctx: CollectorContext): string { + return join(ctx.outputDir, "imports", PLUGIN_ID); +} + +function isSkippable(path: string): boolean { + return SKIP_DIR_HINT.test(path); +} + +function asNum(v: unknown): number | null { + return typeof v === "number" && Number.isFinite(v) ? v : null; +} + +function collectRefs( + targetPath: string, + maxFiles: number, + match: (path: string, text: string) => boolean, + limit = 16, +): string[] { + const refs: string[] = []; + const files = walkFiles(targetPath, { + maxFiles: Math.max(maxFiles, 5000), + extensions: [ + ".md", + ".txt", + ".yml", + ".yaml", + ".json", + ".html", + ".rst", + ], + }); + for (const f of files) { + const r = rel(targetPath, f); + if (isSkippable(r)) continue; + const text = readText(f, 80_000) || ""; + if (match(r, text)) refs.push(r); + if (refs.length >= limit) break; + } + return [...new Set(refs)]; +} + +function scenarioRefs( + targetPath: string, + maxFiles: number, + scenarioRe: RegExp, +): string[] { + return collectRefs( + targetPath, + maxFiles, + (path, text) => + (PLAYBOOK_RE.test(path) || PLAYBOOK_RE.test(text)) && + (scenarioRe.test(path) || scenarioRe.test(text)), + 8, + ); +} + +function loadImported( + ctx: CollectorContext, +): IncidentPlaybooksReport["importedResults"] { + const sources: string[] = []; + let fourPlaybooksPresent: boolean | null = null; + let allPlaybooksHaveOwner: boolean | null = null; + let allPlaybooksReviewedWithin12Months: boolean | null = null; + let missingPlaybookCount: number | null = null; + let playbooksMissingOwner: number | null = null; + let playbooksWithStaleReview: number | null = null; + let ageDays: number | null = null; + let measuredAt: string | null = null; + + for (const f of listImportFiles(ctx.outputDir, PLUGIN_ID)) { + if (/incident-playbooks-report\.json$/i.test(f)) continue; + const text = readText(f); + if (!text) continue; + try { + const data = JSON.parse(text) as Record; + sources.push(basename(f)); + measuredAt = parseMeasuredAt(data) ?? measuredAt; + ageDays = asNum(data.ageDays) ?? asNum(data.age_days) ?? ageDays; + fourPlaybooksPresent = + asBool(data.fourPlaybooksPresent) ?? + asBool(data.four_playbooks_present) ?? + fourPlaybooksPresent; + allPlaybooksHaveOwner = + asBool(data.allPlaybooksHaveOwner) ?? + asBool(data.all_playbooks_have_owner) ?? + allPlaybooksHaveOwner; + allPlaybooksReviewedWithin12Months = + asBool(data.allPlaybooksReviewedWithin12Months) ?? + asBool(data.all_playbooks_reviewed_within_12_months) ?? + allPlaybooksReviewedWithin12Months; + missingPlaybookCount = + asNum(data.missingPlaybookCount) ?? + asNum(data.missing_playbook_count) ?? + missingPlaybookCount; + playbooksMissingOwner = + asNum(data.playbooksMissingOwner) ?? + asNum(data.playbooks_missing_owner) ?? + playbooksMissingOwner; + playbooksWithStaleReview = + asNum(data.playbooksWithStaleReview) ?? + asNum(data.playbooks_with_stale_review) ?? + playbooksWithStaleReview; + + const coverage = asNum(data.playbookCoverageCount) ?? + asNum(data.playbook_coverage_count); + if (coverage !== null) { + fourPlaybooksPresent = fourPlaybooksPresent ?? coverage >= 4; + missingPlaybookCount = + missingPlaybookCount ?? Math.max(0, 4 - coverage); + } + if (missingPlaybookCount !== null) { + fourPlaybooksPresent = + fourPlaybooksPresent ?? missingPlaybookCount === 0; + } + if (playbooksMissingOwner !== null) { + allPlaybooksHaveOwner = + allPlaybooksHaveOwner ?? playbooksMissingOwner === 0; + } + if (playbooksWithStaleReview !== null) { + allPlaybooksReviewedWithin12Months = + allPlaybooksReviewedWithin12Months ?? + playbooksWithStaleReview === 0; + } + } catch { + /* skip */ + } + } + + return { + found: sources.length > 0, + fourPlaybooksPresent, + allPlaybooksHaveOwner, + allPlaybooksReviewedWithin12Months, + missingPlaybookCount, + playbooksMissingOwner, + playbooksWithStaleReview, + ageDays, + measuredAt, + sources, + }; +} + +export function buildIncidentPlaybooksReport(opts: { + assessedAt: string; + abuse: { found: boolean; refs: string[] }; + leakage: { found: boolean; refs: string[] }; + badActions: { found: boolean; refs: string[] }; + providerOutage: { found: boolean; refs: string[] }; + ownerOrReview: { found: boolean; refs: string[] }; + imported: IncidentPlaybooksReport["importedResults"]; +}): IncidentPlaybooksReport { + const notes: string[] = []; + const scenarioCoverageCount = [ + opts.abuse.found, + opts.leakage.found, + opts.badActions.found, + opts.providerOutage.found, + ].filter(Boolean).length; + const playbookSignalsPresent = + scenarioCoverageCount > 0 || opts.ownerOrReview.found; + + if (!playbookSignalsPresent && !opts.imported.found) { + notes.push( + "No AI incident-playbook signals — INC-M1 may be NOT_APPLICABLE if no production AI system is in scope.", + ); + } + if (opts.abuse.found) { + notes.push(`Abuse refs: ${opts.abuse.refs.slice(0, 3).join(", ")}`); + } + if (opts.leakage.found) { + notes.push(`Leakage refs: ${opts.leakage.refs.slice(0, 3).join(", ")}`); + } + if (opts.badActions.found) { + notes.push( + `Bad-actions refs: ${opts.badActions.refs.slice(0, 3).join(", ")}`, + ); + } + if (opts.providerOutage.found) { + notes.push( + `Provider-outage refs: ${opts.providerOutage.refs.slice(0, 3).join(", ")}`, + ); + } + if (opts.imported.found) { + notes.push( + `Imported: ${opts.imported.sources.join(", ")} (four=${opts.imported.fourPlaybooksPresent}, owners=${opts.imported.allPlaybooksHaveOwner}, reviewed12m=${opts.imported.allPlaybooksReviewedWithin12Months})`, + ); + } else if (playbookSignalsPresent) { + notes.push( + "Playbook signals alone are PARTIAL — import fourPlaybooksPresent=true + allPlaybooksHaveOwner=true + allPlaybooksReviewedWithin12Months=true (measuredAt ≤90d) under imports/incident-playbooks/ to PASS.", + ); + } + + const ageOk = + opts.imported.ageDays === null || + opts.imported.ageDays <= IMPORT_MAX_AGE_DAYS; + const fourOk = + opts.imported.fourPlaybooksPresent === true || + opts.imported.missingPlaybookCount === 0; + const ownersOk = + opts.imported.allPlaybooksHaveOwner === true || + opts.imported.playbooksMissingOwner === 0; + const reviewOk = + opts.imported.allPlaybooksReviewedWithin12Months === true || + opts.imported.playbooksWithStaleReview === 0; + const importFresh = measuredAtFresh(opts.imported.measuredAt); + + let statusHint: IncidentPlaybooksReport["summary"]["statusHint"]; + let incM1Satisfied: boolean | null = null; + + const explicitFail = + opts.imported.found && + (opts.imported.fourPlaybooksPresent === false || + opts.imported.allPlaybooksHaveOwner === false || + opts.imported.allPlaybooksReviewedWithin12Months === false || + (typeof opts.imported.missingPlaybookCount === "number" && + opts.imported.missingPlaybookCount > 0) || + (typeof opts.imported.playbooksMissingOwner === "number" && + opts.imported.playbooksMissingOwner > 0) || + (typeof opts.imported.playbooksWithStaleReview === "number" && + opts.imported.playbooksWithStaleReview > 0) || + (opts.imported.ageDays !== null && + opts.imported.ageDays > IMPORT_MAX_AGE_DAYS)); + + if (!playbookSignalsPresent && !opts.imported.found) { + statusHint = "not_applicable"; + incM1Satisfied = null; + } else if (explicitFail) { + statusHint = "fail"; + incM1Satisfied = false; + notes.push( + "Imported evidence shows missing playbooks, missing owners, stale reviews (>12 months), or evidence older than 90 days — INC-M1 fail.", + ); + } else if ( + (playbookSignalsPresent || opts.imported.found) && + fourOk && + ownersOk && + reviewOk && + ageOk && + importFresh && + opts.imported.found + ) { + statusHint = "pass"; + incM1Satisfied = true; + } else if (playbookSignalsPresent || opts.imported.found) { + statusHint = "partial"; + incM1Satisfied = false; + if (opts.imported.found && !fourOk) { + notes.push( + "Import must show fourPlaybooksPresent=true (or missingPlaybookCount=0).", + ); + } + if (opts.imported.found && !ownersOk) { + notes.push( + "Import must show allPlaybooksHaveOwner=true (or playbooksMissingOwner=0).", + ); + } + if (opts.imported.found && !reviewOk) { + notes.push( + "Import must show allPlaybooksReviewedWithin12Months=true (or playbooksWithStaleReview=0).", + ); + } + if (opts.imported.found && !importFresh) { + notes.push( + "Import missing fresh measuredAt (≤90 days) — required to unlock INC-M1 PASS.", + ); + } + } else { + statusHint = "not_demonstrated"; + incM1Satisfied = null; + } + + return { + schemaVersion: "0.2.0", + pluginId: PLUGIN_ID, + detectorId: DETECTOR_ID, + relatedCheckIds: [...RELATED], + assessedAt: opts.assessedAt, + signals: { + abuse: opts.abuse, + leakage: opts.leakage, + badActions: opts.badActions, + providerOutage: opts.providerOutage, + ownerOrReview: opts.ownerOrReview, + }, + importedResults: opts.imported, + summary: { + playbookSignalsPresent, + scenarioCoverageCount, + incM1Satisfied, + statusHint, + }, + notes, + }; +} + +export const incidentPlaybooksCollector: Collector = { + id: PLUGIN_ID, + async collect(ctx: CollectorContext): Promise { + const maxFiles = ctx.maxFiles ?? 8000; + + const abuse = scenarioRefs(ctx.targetPath, maxFiles, ABUSE_RE); + const leakage = scenarioRefs(ctx.targetPath, maxFiles, LEAKAGE_RE); + const badActions = scenarioRefs(ctx.targetPath, maxFiles, BAD_ACTIONS_RE); + const providerOutage = scenarioRefs( + ctx.targetPath, + maxFiles, + PROVIDER_OUTAGE_RE, + ); + const ownerOrReview = collectRefs( + ctx.targetPath, + maxFiles, + (path, text) => + (PLAYBOOK_RE.test(path) || PLAYBOOK_RE.test(text)) && + (OWNER_RE.test(text) || REVIEW_RE.test(text)), + 8, + ); + + const imported = loadImported(ctx); + const report = buildIncidentPlaybooksReport({ + assessedAt: ctx.assessedAt.toISOString(), + abuse: { found: abuse.length > 0, refs: abuse }, + leakage: { found: leakage.length > 0, refs: leakage }, + badActions: { found: badActions.length > 0, refs: badActions }, + providerOutage: { + found: providerOutage.length > 0, + refs: providerOutage, + }, + ownerOrReview: { + found: ownerOrReview.length > 0, + refs: ownerOrReview, + }, + imported, + }); + + ensureDir(importDir(ctx)); + writeFileSync( + join(importDir(ctx), "incident-playbooks-report.json"), + JSON.stringify(report, null, 2) + "\n", + "utf8", + ); + + const nodes: EvidenceNode[] = [ + { + id: `${PLUGIN_ID}:report`, + class: "ci", + ref: `imports/${PLUGIN_ID}/incident-playbooks-report.json`, + pluginId: PLUGIN_ID, + signals: [ + "incident-playbooks", + "inc-m1", + DETECTOR_ID, + ...(report.summary.incM1Satisfied ? ["inc-m1-satisfied"] : []), + ], + excerpt: redact(report.notes.slice(0, 3).join(" | ").slice(0, 400)), + relatedCheckIds: [...RELATED], + }, + ]; + for (const r of [ + ...new Set([ + ...report.signals.abuse.refs, + ...report.signals.leakage.refs, + ...report.signals.badActions.refs, + ...report.signals.providerOutage.refs, + ...report.signals.ownerOrReview.refs, + ]), + ].slice(0, 8)) { + nodes.push({ + id: `${PLUGIN_ID}:ref:${r}`, + class: "ci", + ref: r, + pluginId: PLUGIN_ID, + signals: ["incident-playbooks-ref"], + relatedCheckIds: [...RELATED], + }); + } + + return { + pluginId: PLUGIN_ID, + status: "ran", + detail: `INC-M1 status=${report.summary.statusHint} scenarios=${report.summary.scenarioCoverageCount}/4 satisfied=${report.summary.incM1Satisfied}; report=imports/${PLUGIN_ID}/incident-playbooks-report.json`, + nodes, + }; + }, +}; diff --git a/skills/aprf-auditor/collectors/index.ts b/skills/aprf-auditor/collectors/index.ts index 6da17f3..2ee07c6 100644 --- a/skills/aprf-auditor/collectors/index.ts +++ b/skills/aprf-auditor/collectors/index.ts @@ -62,6 +62,12 @@ import { aiConfigAsCodeCollector } from "./ai-config-as-code.ts"; import { aiCanaryProgressiveDeliveryCollector } from "./ai-canary-progressive-delivery.ts"; import { envParityModelToolCatalogCollector } from "./env-parity-model-tool-catalog.ts"; import { embeddingIndexMigrationCollector } from "./embedding-index-migration.ts"; +import { incidentPlaybooksCollector } from "./incident-playbooks.ts"; +import { aiContainmentDrillCollector } from "./ai-containment-drill.ts"; +import { postIncidentAprfActionsCollector } from "./post-incident-aprf-actions.ts"; +import { aiIncidentTabletopCollector } from "./ai-incident-tabletop.ts"; +import { aiSafetyQualityAlertsCollector } from "./ai-safety-quality-alerts.ts"; +import { aiCustomerNotificationCriteriaCollector } from "./ai-customer-notification-criteria.ts"; import { aiCostAlertsCollector } from "./ai-cost-alerts.ts"; import { aiRetryAmplificationCollector } from "./ai-retry-amplification.ts"; import { aiPromptCacheCollector } from "./ai-prompt-cache.ts"; @@ -172,6 +178,12 @@ export const COLLECTORS: Collector[] = [ aiCanaryProgressiveDeliveryCollector, envParityModelToolCatalogCollector, embeddingIndexMigrationCollector, + incidentPlaybooksCollector, + aiContainmentDrillCollector, + postIncidentAprfActionsCollector, + aiIncidentTabletopCollector, + aiSafetyQualityAlertsCollector, + aiCustomerNotificationCriteriaCollector, aiCostAlertsCollector, aiRetryAmplificationCollector, aiPromptCacheCollector, diff --git a/skills/aprf-auditor/collectors/post-incident-aprf-actions.ts b/skills/aprf-auditor/collectors/post-incident-aprf-actions.ts new file mode 100644 index 0000000..2111c4f --- /dev/null +++ b/skills/aprf-auditor/collectors/post-incident-aprf-actions.ts @@ -0,0 +1,380 @@ +/** + * post-incident-aprf-actions — INC-R2 / repo-post-incident-aprf-actions. + * + * Discovers post-incident reviews with APRF-pillar-mapped actions. + * Import reviewsWithTrackedActionOrRationalePct=100 (or + * reviewsMissingTrackedActionOrRationale=0) with sevEligibleIncidentCount>0 + * under imports/post-incident-aprf-actions/ to unlock PASS (measuredAt ≤90d). + * N/A when sevEligibleIncidentCount=0. + */ +import { writeFileSync } from "node:fs"; +import { join, basename } from "node:path"; +import type { + Collector, + CollectorContext, + CollectorResult, + EvidenceNode, +} from "./types.ts"; +import { + ensureDir, + listImportFiles, + readText, + redact, + rel, + walkFiles, +} from "./lib/fs.ts"; +import { + asBool, + measuredAtFresh, + parseMeasuredAt, +} from "./lib/import-attest.ts"; + +const PLUGIN_ID = "post-incident-aprf-actions"; +const RELATED = ["INC-R2"] as const; +const DETECTOR_ID = "repo-post-incident-aprf-actions"; +const IMPORT_MAX_AGE_DAYS = 90; + +const SKIP_DIR_HINT = + /(^|[/\\])(node_modules|\.git|dist|build|coverage|\.venv|venv|__pycache__|vendor)([/\\]|$)/i; + +const POSTMORTEM_RE = + /\b(post[\s_-]*incident|postmortem|post[\s_-]*mortem|after[\s_-]*action|incident[\s_-]*review|pir)\b/i; + +const APRF_ACTION_RE = + /\b(aprf[\s_-]*pillar|pillar[\s_-]*mapped|tracked[\s_-]*action|remediation[\s_-]*ticket|no[\s_-]*action[\s_-]*rationale)\b/i; + +const SEV_RE = + /\b(sev[\s_-]*\d|severity|sev[\s_-]*eligible|p[0-3][\s_-]*incident)\b/i; + +export interface PostIncidentAprfActionsReport { + schemaVersion: "0.2.0"; + pluginId: typeof PLUGIN_ID; + detectorId: typeof DETECTOR_ID; + relatedCheckIds: string[]; + assessedAt: string; + signals: { + postmortem: { found: boolean; refs: string[] }; + aprfAction: { found: boolean; refs: string[] }; + }; + importedResults: { + found: boolean; + sevEligibleIncidentCount: number | null; + reviewsWithTrackedActionOrRationalePct: number | null; + reviewsMissingTrackedActionOrRationale: number | null; + ageDays: number | null; + measuredAt: string | null; + sources: string[]; + }; + summary: { + reviewSignalsPresent: boolean; + incR2Satisfied: boolean | null; + statusHint: + | "pass" + | "partial" + | "fail" + | "not_demonstrated" + | "not_applicable"; + }; + notes: string[]; +} + +function importDir(ctx: CollectorContext): string { + return join(ctx.outputDir, "imports", PLUGIN_ID); +} + +function isSkippable(path: string): boolean { + return SKIP_DIR_HINT.test(path); +} + +function asNum(v: unknown): number | null { + return typeof v === "number" && Number.isFinite(v) ? v : null; +} + +function collectRefs( + targetPath: string, + maxFiles: number, + match: (path: string, text: string) => boolean, + limit = 16, +): string[] { + const refs: string[] = []; + const files = walkFiles(targetPath, { + maxFiles: Math.max(maxFiles, 5000), + extensions: [".md", ".txt", ".yml", ".yaml", ".json", ".html"], + }); + for (const f of files) { + const r = rel(targetPath, f); + if (isSkippable(r)) continue; + const text = readText(f, 80_000) || ""; + if (match(r, text)) refs.push(r); + if (refs.length >= limit) break; + } + return [...new Set(refs)]; +} + +function loadImported( + ctx: CollectorContext, +): PostIncidentAprfActionsReport["importedResults"] { + const sources: string[] = []; + let sevEligibleIncidentCount: number | null = null; + let reviewsWithTrackedActionOrRationalePct: number | null = null; + let reviewsMissingTrackedActionOrRationale: number | null = null; + let ageDays: number | null = null; + let measuredAt: string | null = null; + + for (const f of listImportFiles(ctx.outputDir, PLUGIN_ID)) { + if (/post-incident-aprf-actions-report\.json$/i.test(f)) continue; + const text = readText(f); + if (!text) continue; + try { + const data = JSON.parse(text) as Record; + sources.push(basename(f)); + measuredAt = parseMeasuredAt(data) ?? measuredAt; + ageDays = asNum(data.ageDays) ?? asNum(data.age_days) ?? ageDays; + sevEligibleIncidentCount = + asNum(data.sevEligibleIncidentCount) ?? + asNum(data.sev_eligible_incident_count) ?? + sevEligibleIncidentCount; + reviewsWithTrackedActionOrRationalePct = + asNum(data.reviewsWithTrackedActionOrRationalePct) ?? + asNum(data.reviews_with_tracked_action_or_rationale_pct) ?? + asNum(data.coveragePct) ?? + reviewsWithTrackedActionOrRationalePct; + reviewsMissingTrackedActionOrRationale = + asNum(data.reviewsMissingTrackedActionOrRationale) ?? + asNum(data.reviews_missing_tracked_action_or_rationale) ?? + asNum(data.missingCount) ?? + reviewsMissingTrackedActionOrRationale; + + if (asBool(data.allSevReviewsHaveTrackedActionOrRationale) === true) { + reviewsWithTrackedActionOrRationalePct = + reviewsWithTrackedActionOrRationalePct ?? 100; + reviewsMissingTrackedActionOrRationale = + reviewsMissingTrackedActionOrRationale ?? 0; + } + } catch { + /* skip */ + } + } + + return { + found: sources.length > 0, + sevEligibleIncidentCount, + reviewsWithTrackedActionOrRationalePct, + reviewsMissingTrackedActionOrRationale, + ageDays, + measuredAt, + sources, + }; +} + +export function buildPostIncidentAprfActionsReport(opts: { + assessedAt: string; + postmortem: { found: boolean; refs: string[] }; + aprfAction: { found: boolean; refs: string[] }; + imported: PostIncidentAprfActionsReport["importedResults"]; +}): PostIncidentAprfActionsReport { + const notes: string[] = []; + const reviewSignalsPresent = + opts.postmortem.found || opts.aprfAction.found; + + if (!reviewSignalsPresent && !opts.imported.found) { + notes.push( + "No post-incident / APRF-action signals — INC-R2 may be NOT_APPLICABLE if no SEV-eligible AI incidents occur.", + ); + } + if (opts.postmortem.found) { + notes.push( + `Postmortem refs: ${opts.postmortem.refs.slice(0, 4).join(", ")}`, + ); + } + if (opts.aprfAction.found) { + notes.push( + `APRF-action refs: ${opts.aprfAction.refs.slice(0, 3).join(", ")}`, + ); + } + if (opts.imported.found) { + notes.push( + `Imported: ${opts.imported.sources.join(", ")} (sevCount=${opts.imported.sevEligibleIncidentCount}, coveragePct=${opts.imported.reviewsWithTrackedActionOrRationalePct}, missing=${opts.imported.reviewsMissingTrackedActionOrRationale})`, + ); + } else if (reviewSignalsPresent) { + notes.push( + "Review signals alone are PARTIAL — import reviewsWithTrackedActionOrRationalePct=100 (or reviewsMissingTrackedActionOrRationale=0) (measuredAt ≤90d) under imports/post-incident-aprf-actions/ to PASS.", + ); + } + + const ageOk = + opts.imported.ageDays === null || + opts.imported.ageDays <= IMPORT_MAX_AGE_DAYS; + const coverageOk = + opts.imported.reviewsWithTrackedActionOrRationalePct === 100 || + opts.imported.reviewsMissingTrackedActionOrRationale === 0; + const importFresh = measuredAtFresh(opts.imported.measuredAt); + const noSev = + opts.imported.found && opts.imported.sevEligibleIncidentCount === 0; + const sevCountOk = + typeof opts.imported.sevEligibleIncidentCount === "number" && + opts.imported.sevEligibleIncidentCount > 0; + + let statusHint: PostIncidentAprfActionsReport["summary"]["statusHint"]; + let incR2Satisfied: boolean | null = null; + + const explicitFail = + opts.imported.found && + !noSev && + ((typeof opts.imported.reviewsWithTrackedActionOrRationalePct === + "number" && + opts.imported.reviewsWithTrackedActionOrRationalePct < 100) || + (typeof opts.imported.reviewsMissingTrackedActionOrRationale === + "number" && + opts.imported.reviewsMissingTrackedActionOrRationale > 0) || + (opts.imported.ageDays !== null && + opts.imported.ageDays > IMPORT_MAX_AGE_DAYS)); + + if (noSev) { + statusHint = "not_applicable"; + incR2Satisfied = null; + notes.push( + "sevEligibleIncidentCount=0 — INC-R2 NOT_APPLICABLE (no SEV-eligible AI incidents in window).", + ); + } else if (!reviewSignalsPresent && !opts.imported.found) { + statusHint = "not_applicable"; + incR2Satisfied = null; + } else if (explicitFail) { + statusHint = "fail"; + incR2Satisfied = false; + notes.push( + "Imported evidence shows incomplete APRF-action coverage or evidence older than 90 days — INC-R2 fail.", + ); + } else if ( + (reviewSignalsPresent || opts.imported.found) && + sevCountOk && + coverageOk && + ageOk && + importFresh && + opts.imported.found + ) { + statusHint = "pass"; + incR2Satisfied = true; + } else if (reviewSignalsPresent || opts.imported.found) { + statusHint = "partial"; + incR2Satisfied = false; + if (opts.imported.found && !sevCountOk) { + notes.push( + "Import must show sevEligibleIncidentCount>0 to unlock PASS (use 0 for NOT_APPLICABLE).", + ); + } + if (opts.imported.found && !coverageOk) { + notes.push( + "Import must show reviewsWithTrackedActionOrRationalePct=100 or reviewsMissingTrackedActionOrRationale=0.", + ); + } + if (opts.imported.found && !importFresh) { + notes.push( + "Import missing fresh measuredAt (≤90 days) — required to unlock INC-R2 PASS.", + ); + } + } else { + statusHint = "not_demonstrated"; + incR2Satisfied = null; + } + + return { + schemaVersion: "0.2.0", + pluginId: PLUGIN_ID, + detectorId: DETECTOR_ID, + relatedCheckIds: [...RELATED], + assessedAt: opts.assessedAt, + signals: { + postmortem: opts.postmortem, + aprfAction: opts.aprfAction, + }, + importedResults: opts.imported, + summary: { + reviewSignalsPresent, + incR2Satisfied, + statusHint, + }, + notes, + }; +} + +export const postIncidentAprfActionsCollector: Collector = { + id: PLUGIN_ID, + async collect(ctx: CollectorContext): Promise { + const maxFiles = ctx.maxFiles ?? 8000; + + const postmortem = collectRefs( + ctx.targetPath, + maxFiles, + (path, text) => + POSTMORTEM_RE.test(path) || + POSTMORTEM_RE.test(text) || + (SEV_RE.test(text) && /incident/i.test(path + text)), + 10, + ); + const aprfAction = collectRefs( + ctx.targetPath, + maxFiles, + (path, text) => + APRF_ACTION_RE.test(path) || + APRF_ACTION_RE.test(text) || + ((POSTMORTEM_RE.test(path) || POSTMORTEM_RE.test(text)) && + /aprf|pillar|tracked[\s_-]*action/i.test(text)), + 10, + ); + + const imported = loadImported(ctx); + const report = buildPostIncidentAprfActionsReport({ + assessedAt: ctx.assessedAt.toISOString(), + postmortem: { found: postmortem.length > 0, refs: postmortem }, + aprfAction: { found: aprfAction.length > 0, refs: aprfAction }, + imported, + }); + + ensureDir(importDir(ctx)); + writeFileSync( + join(importDir(ctx), "post-incident-aprf-actions-report.json"), + JSON.stringify(report, null, 2) + "\n", + "utf8", + ); + + const nodes: EvidenceNode[] = [ + { + id: `${PLUGIN_ID}:report`, + class: "ci", + ref: `imports/${PLUGIN_ID}/post-incident-aprf-actions-report.json`, + pluginId: PLUGIN_ID, + signals: [ + "post-incident-aprf-actions", + "inc-r2", + DETECTOR_ID, + ...(report.summary.incR2Satisfied ? ["inc-r2-satisfied"] : []), + ], + excerpt: redact(report.notes.slice(0, 3).join(" | ").slice(0, 400)), + relatedCheckIds: [...RELATED], + }, + ]; + for (const r of [ + ...new Set([ + ...report.signals.postmortem.refs, + ...report.signals.aprfAction.refs, + ]), + ].slice(0, 6)) { + nodes.push({ + id: `${PLUGIN_ID}:ref:${r}`, + class: "ci", + ref: r, + pluginId: PLUGIN_ID, + signals: ["post-incident-aprf-actions-ref"], + relatedCheckIds: [...RELATED], + }); + } + + return { + pluginId: PLUGIN_ID, + status: "ran", + detail: `INC-R2 status=${report.summary.statusHint} reviews=${report.summary.reviewSignalsPresent} satisfied=${report.summary.incR2Satisfied}; report=imports/${PLUGIN_ID}/post-incident-aprf-actions-report.json`, + nodes, + }; + }, +}; diff --git a/skills/aprf-auditor/collectors/runner.ts b/skills/aprf-auditor/collectors/runner.ts index a68bbf4..394be25 100644 --- a/skills/aprf-auditor/collectors/runner.ts +++ b/skills/aprf-auditor/collectors/runner.ts @@ -266,6 +266,30 @@ DEP-R3 automated embedding/index version migration: npm run aprf:embedding-index-migration -- --target --out /aprf-assessment # PASS needs automatedMigrationWithValidationGates + lastUpgradeWithin12Months + lastUpgradeSucceededWithoutDualWriteGaps under imports/embedding-index-migration/ +INC-M1 AI-specific incident playbooks (abuse/leakage/bad actions/provider outage): + npm run aprf:incident-playbooks -- --target --out /aprf-assessment + # PASS needs fourPlaybooksPresent + allPlaybooksHaveOwner + allPlaybooksReviewedWithin12Months under imports/incident-playbooks/ + +INC-M2 AI containment drill (pause / disable tools / rollback): + npm run aprf:ai-containment-drill -- --target --out /aprf-assessment + # PASS needs pauseAgentsDemonstrated + disableToolsDemonstrated + rollbackPromptOrModelDemonstrated + withinDocumentedTimeBudgets under imports/ai-containment-drill/ + +INC-R2 post-incident reviews with APRF-pillar tracked actions: + npm run aprf:post-incident-aprf-actions -- --target --out /aprf-assessment + # PASS needs sevEligibleIncidentCount>0 + reviewsWithTrackedActionOrRationalePct=100 (or reviewsMissingTrackedActionOrRationale=0) under imports/post-incident-aprf-actions/ + +INC-R4 AI-focused incident tabletop ≤180 days: + npm run aprf:ai-incident-tabletop -- --target --out /aprf-assessment + # PASS needs aiFocusedTabletopCompletedWithin180Days + retainedActionsWithOwners under imports/ai-incident-tabletop/ + +INC-R1 AI safety/quality on-call paging (≥2 non-infra signals): + npm run aprf:ai-safety-quality-alerts -- --target --out /aprf-assessment + # PASS needs atLeastTwoNonInfraPagingSignals + eachSignalHasThresholdAndOwner + policyReviewedWithin90Days under imports/ai-safety-quality-alerts/ + +INC-R3 AI customer notification criteria (notify/no-notify + followed sample ≤12m): + npm run aprf:ai-customer-notification-criteria -- --target --out /aprf-assessment + # PASS needs criteriaMapEventTypesToNotifyDecision + lastDrillOrIncidentFollowedCriteriaWithin12Months + timestampsPresent under imports/ai-customer-notification-criteria/ + COST-M2 AI cost budget-burn / anomaly alerts: npm run aprf:cost-alerts -- --target --out /aprf-assessment # PASS needs notify proof under imports/ai-cost-alerts/ diff --git a/skills/aprf-auditor/evidence-map.yaml b/skills/aprf-auditor/evidence-map.yaml index 63ff0b0..d18a34c 100644 --- a/skills/aprf-auditor/evidence-map.yaml +++ b/skills/aprf-auditor/evidence-map.yaml @@ -251,7 +251,9 @@ categoryDefaults: - "**/*incident*" - "**/oncall*" - "**/pager*" - signals: [runbook, severity, postmortem] + - "**/*playbook*" + plugins: [incident-playbooks, ai-containment-drill, post-incident-aprf-actions, ai-incident-tabletop, ai-safety-quality-alerts, ai-customer-notification-criteria, repo-filesystem] + signals: [runbook, severity, postmortem, playbook, abuse, leakage, bad actions, provider outage, containment, pause agent, disable tools, rollback, APRF pillar action, tabletop, safety quality paging, refusal-rate, eval-score, toxicity, jailbreak, customer notification, notify/no-notify, disclosure] cost-optimization: search: - "**/*budget*" @@ -1746,6 +1748,105 @@ checks: PASS if embedding-index-migration statusHint=pass (automatedMigrationWithValidationGates + lastUpgradeWithin12Months + lastUpgradeSucceededWithoutDualWriteGaps + measuredAt ≤90d). PARTIAL if migration signals without complete import. NOT_APPLICABLE when no embedding/index signals or noUpgradeInWindowAttested. Distinct from DEP-M3 and DEP-R2. + INC-M1: + requiredEvidence: + - "Playbook set covering abuse, leakage, bad actions, and provider outage with owners" + - "fourPlaybooksPresent + allPlaybooksHaveOwner + allPlaybooksReviewedWithin12Months under imports/incident-playbooks/" + searchExtra: + - "**/*playbook*" + - "**/*runbook*" + - "**/*incident*" + - "**/*abuse*" + - "**/*leak*" + - "**/*provider*outage*" + plugins: [incident-playbooks, repo-filesystem] + naWhen: "No production AI system in scope" + evaluation: | + PASS if incident-playbooks statusHint=pass (fourPlaybooksPresent + allPlaybooksHaveOwner + allPlaybooksReviewedWithin12Months + measuredAt ≤90d). + PARTIAL if playbook signals without complete import. + NOT_APPLICABLE when no playbook signals. Distinct from INC-M2 containment drills. + INC-M2: + requiredEvidence: + - "Containment runbook covering pause agents, disable tools, and prompt/model rollback" + - "pauseAgentsDemonstrated + disableToolsDemonstrated + rollbackPromptOrModelDemonstrated + withinDocumentedTimeBudgets under imports/ai-containment-drill/" + searchExtra: + - "**/*containment*" + - "**/*pause*agent*" + - "**/*disable*tool*" + - "**/*prompt*rollback*" + - "**/*model*rollback*" + - "**/*drill*" + plugins: [ai-containment-drill, repo-filesystem] + naWhen: "No production agents/tools/prompt-model units" + evaluation: | + PASS if ai-containment-drill statusHint=pass (pause + disable tools + rollback within time budgets + measuredAt ≤90d). + PARTIAL if containment signals without complete import. + NOT_APPLICABLE when no containment signals. Distinct from INC-M1 playbooks and CHG/AGN rollback/kill-switch alone. + INC-R2: + requiredEvidence: + - "Post-incident review template requiring APRF pillar mapping" + - "sevEligibleIncidentCount>0 + reviewsWithTrackedActionOrRationalePct=100 under imports/post-incident-aprf-actions/" + searchExtra: + - "**/*postmortem*" + - "**/*post-incident*" + - "**/*after*action*" + - "**/*pir*" + - "**/*aprf*pillar*" + plugins: [post-incident-aprf-actions, repo-filesystem] + naWhen: "No SEV-eligible AI incidents (sevEligibleIncidentCount=0)" + evaluation: | + PASS if post-incident-aprf-actions statusHint=pass (sevEligibleIncidentCount>0 + reviewsWithTrackedActionOrRationalePct=100 or reviewsMissingTrackedActionOrRationale=0 + measuredAt ≤90d). + PARTIAL if review signals without complete import (including omitted sevEligibleIncidentCount). + NOT_APPLICABLE when sevEligibleIncidentCount=0 or no review signals. Distinct from INC-M1/M2. + INC-R1: + requiredEvidence: + - "On-call alert policy export listing safety/quality pages" + - "atLeastTwoNonInfraPagingSignals + eachSignalHasThresholdAndOwner + policyReviewedWithin90Days under imports/ai-safety-quality-alerts/" + searchExtra: + - "**/*refusal*" + - "**/*toxicity*" + - "**/*jailbreak*" + - "**/*eval*score*" + - "**/*pager*" + - "**/*on*call*alert*" + plugins: [ai-safety-quality-alerts, repo-filesystem] + naWhen: "No production AI system in scope" + evaluation: | + PASS if ai-safety-quality-alerts statusHint=pass (≥2 non-infra paging signals + threshold/owner + policy reviewed ≤90d + measuredAt ≤90d). + PARTIAL if alert signals without complete import. + NOT_APPLICABLE when no alert signals. Distinct from infra-only OBS pages and COST spend alerts. + INC-R3: + requiredEvidence: + - "Customer notification criteria for AI-related events" + - "criteriaMapEventTypesToNotifyDecision + lastDrillOrIncidentFollowedCriteriaWithin12Months + timestampsPresent under imports/ai-customer-notification-criteria/" + searchExtra: + - "**/*customer*notif*" + - "**/*disclosure*" + - "**/*notify*" + - "**/*comms*drill*" + - "**/*status*page*" + plugins: [ai-customer-notification-criteria, repo-filesystem] + naWhen: "No customer-facing / externally disclosed AI system in scope" + evaluation: | + PASS if ai-customer-notification-criteria statusHint=pass (criteria map + followed sample ≤12 months + timestampsPresent=true + measuredAt ≤90d). + PARTIAL if notification signals without complete import. + NOT_APPLICABLE when no notification signals. Distinct from INC-R2 internal post-incident reviews and generic PR playbooks without AI event-type mapping. + INC-R4: + requiredEvidence: + - "Tabletop plan for an AI-specific incident scenario" + - "aiFocusedTabletopCompletedWithin180Days + retainedActionsWithOwners under imports/ai-incident-tabletop/" + searchExtra: + - "**/*tabletop*" + - "**/*table-top*" + - "**/*war*game*" + - "**/*game*day*" + - "**/*after*action*" + plugins: [ai-incident-tabletop, repo-filesystem] + naWhen: "No production AI system in scope" + evaluation: | + PASS if ai-incident-tabletop statusHint=pass (aiFocusedTabletopCompletedWithin180Days or tabletopAgeDays≤180 + retainedActionsWithOwners + measuredAt ≤90d). + PARTIAL if tabletop signals without complete import. + NOT_APPLICABLE when no tabletop signals. Distinct from INC-M2 containment drills and INC-R2 post-incident reviews. # Global fail-closed patterns (support FAIL with high confidence when matched in scope) globalFailSignals: diff --git a/skills/aprf-auditor/plugins/ai-containment-drill.yaml b/skills/aprf-auditor/plugins/ai-containment-drill.yaml new file mode 100644 index 0000000..46e7191 --- /dev/null +++ b/skills/aprf-auditor/plugins/ai-containment-drill.yaml @@ -0,0 +1,26 @@ +id: ai-containment-drill +title: AI containment drill — pause, disable tools, rollback (INC-M2) +version: "0.2.0" +executor: collectors/ai-containment-drill.ts +evidenceClass: ci +alsoEmits: [] +detectorIds: [repo-ai-containment-drill] +inputs: + - "Target repo (containment runbooks + drill records)" + - "Optional suite: aprf-assessment/imports/ai-containment-drill/*.{json}" +collect: + - "Detect pause/disable/rollback containment + drill/time-budget signals" + - "Ingest coverage; write ai-containment-drill-report.json" +emits: + nodeClass: ci + refPattern: "aprf-assessment/imports/ai-containment-drill/*" + signals: [ai-containment-drill, inc-m2, repo-ai-containment-drill] +mapsToChecks: [INC-M2] +whenUnavailable: > + Containment signals alone are PARTIAL. INC-M2 PASS needs + pauseAgentsDemonstrated, disableToolsDemonstrated, + rollbackPromptOrModelDemonstrated, withinDocumentedTimeBudgets, + measuredAt ≤90 days. +live: + required: false + note: Offline containment heuristics + optional imported JSON. diff --git a/skills/aprf-auditor/plugins/ai-customer-notification-criteria.yaml b/skills/aprf-auditor/plugins/ai-customer-notification-criteria.yaml new file mode 100644 index 0000000..609f1a0 --- /dev/null +++ b/skills/aprf-auditor/plugins/ai-customer-notification-criteria.yaml @@ -0,0 +1,31 @@ +id: ai-customer-notification-criteria +title: AI customer notification criteria (INC-R3) +version: "0.2.0" +executor: collectors/ai-customer-notification-criteria.ts +evidenceClass: ci +alsoEmits: [] +detectorIds: [repo-ai-customer-notification-criteria] +inputs: + - "Target repo (customer notification criteria + drill/incident samples)" + - "Optional suite: aprf-assessment/imports/ai-customer-notification-criteria/*.{json}" +collect: + - "Detect AI notify/no-notify criteria + followed samples" + - "Ingest coverage; write ai-customer-notification-criteria-report.json" +emits: + nodeClass: ci + refPattern: "aprf-assessment/imports/ai-customer-notification-criteria/*" + signals: + [ + ai-customer-notification-criteria, + inc-r3, + repo-ai-customer-notification-criteria, + ] +mapsToChecks: [INC-R3] +whenUnavailable: > + Notification signals alone are PARTIAL. INC-R3 PASS needs + criteriaMapEventTypesToNotifyDecision, + lastDrillOrIncidentFollowedCriteriaWithin12Months, + timestampsPresent=true, measuredAt ≤90 days. +live: + required: false + note: Offline notification heuristics + optional imported JSON. diff --git a/skills/aprf-auditor/plugins/ai-incident-tabletop.yaml b/skills/aprf-auditor/plugins/ai-incident-tabletop.yaml new file mode 100644 index 0000000..50d2225 --- /dev/null +++ b/skills/aprf-auditor/plugins/ai-incident-tabletop.yaml @@ -0,0 +1,25 @@ +id: ai-incident-tabletop +title: AI-focused incident tabletop ≤180 days (INC-R4) +version: "0.2.0" +executor: collectors/ai-incident-tabletop.ts +evidenceClass: ci +alsoEmits: [] +detectorIds: [repo-ai-incident-tabletop] +inputs: + - "Target repo (AI tabletop plans + after-action reports)" + - "Optional suite: aprf-assessment/imports/ai-incident-tabletop/*.{json}" +collect: + - "Detect AI-focused tabletop + after-action signals" + - "Ingest coverage; write ai-incident-tabletop-report.json" +emits: + nodeClass: ci + refPattern: "aprf-assessment/imports/ai-incident-tabletop/*" + signals: [ai-incident-tabletop, inc-r4, repo-ai-incident-tabletop] +mapsToChecks: [INC-R4] +whenUnavailable: > + Tabletop signals alone are PARTIAL. INC-R4 PASS needs + aiFocusedTabletopCompletedWithin180Days, retainedActionsWithOwners, + measuredAt ≤90 days. +live: + required: false + note: Offline tabletop heuristics + optional imported JSON. diff --git a/skills/aprf-auditor/plugins/ai-safety-quality-alerts.yaml b/skills/aprf-auditor/plugins/ai-safety-quality-alerts.yaml new file mode 100644 index 0000000..f1a1197 --- /dev/null +++ b/skills/aprf-auditor/plugins/ai-safety-quality-alerts.yaml @@ -0,0 +1,25 @@ +id: ai-safety-quality-alerts +title: AI safety/quality on-call paging alerts (INC-R1) +version: "0.2.0" +executor: collectors/ai-safety-quality-alerts.ts +evidenceClass: ci +alsoEmits: [] +detectorIds: [repo-ai-safety-quality-alerts] +inputs: + - "Target repo (alert policies for safety/quality signals)" + - "Optional suite: aprf-assessment/imports/ai-safety-quality-alerts/*.{json}" +collect: + - "Detect non-infra AI safety/quality paging signals" + - "Ingest coverage; write ai-safety-quality-alerts-report.json" +emits: + nodeClass: ci + refPattern: "aprf-assessment/imports/ai-safety-quality-alerts/*" + signals: [ai-safety-quality-alerts, inc-r1, repo-ai-safety-quality-alerts] +mapsToChecks: [INC-R1] +whenUnavailable: > + Alert signals alone are PARTIAL. INC-R1 PASS needs + atLeastTwoNonInfraPagingSignals, eachSignalHasThresholdAndOwner, + policyReviewedWithin90Days, measuredAt ≤90 days. +live: + required: false + note: Offline alert heuristics + optional imported JSON. diff --git a/skills/aprf-auditor/plugins/incident-playbooks.yaml b/skills/aprf-auditor/plugins/incident-playbooks.yaml new file mode 100644 index 0000000..04068d0 --- /dev/null +++ b/skills/aprf-auditor/plugins/incident-playbooks.yaml @@ -0,0 +1,25 @@ +id: incident-playbooks +title: AI-specific incident playbooks for four scenarios (INC-M1) +version: "0.2.0" +executor: collectors/incident-playbooks.ts +evidenceClass: ci +alsoEmits: [] +detectorIds: [repo-incident-playbooks] +inputs: + - "Target repo (AI incident playbooks for abuse/leakage/bad actions/provider outage)" + - "Optional suite: aprf-assessment/imports/incident-playbooks/*.{json}" +collect: + - "Detect playbooks covering the four AI incident scenarios" + - "Ingest owner/review coverage; write incident-playbooks-report.json" +emits: + nodeClass: ci + refPattern: "aprf-assessment/imports/incident-playbooks/*" + signals: [incident-playbooks, inc-m1, repo-incident-playbooks] +mapsToChecks: [INC-M1] +whenUnavailable: > + Playbook signals alone are PARTIAL. INC-M1 PASS needs + fourPlaybooksPresent, allPlaybooksHaveOwner, + allPlaybooksReviewedWithin12Months, measuredAt ≤90 days. +live: + required: false + note: Offline playbook heuristics + optional imported JSON. diff --git a/skills/aprf-auditor/plugins/post-incident-aprf-actions.yaml b/skills/aprf-auditor/plugins/post-incident-aprf-actions.yaml new file mode 100644 index 0000000..8438699 --- /dev/null +++ b/skills/aprf-auditor/plugins/post-incident-aprf-actions.yaml @@ -0,0 +1,27 @@ +id: post-incident-aprf-actions +title: Post-incident reviews with APRF-pillar tracked actions (INC-R2) +version: "0.2.0" +executor: collectors/post-incident-aprf-actions.ts +evidenceClass: ci +alsoEmits: [] +detectorIds: [repo-post-incident-aprf-actions] +inputs: + - "Target repo (post-incident / postmortem templates and reviews)" + - "Optional suite: aprf-assessment/imports/post-incident-aprf-actions/*.{json}" +collect: + - "Detect post-incident reviews + APRF pillar action mapping" + - "Ingest coverage; write post-incident-aprf-actions-report.json" +emits: + nodeClass: ci + refPattern: "aprf-assessment/imports/post-incident-aprf-actions/*" + signals: [post-incident-aprf-actions, inc-r2, repo-post-incident-aprf-actions] +mapsToChecks: [INC-R2] +whenUnavailable: > + Review signals alone are PARTIAL. INC-R2 PASS needs + sevEligibleIncidentCount>0 and + reviewsWithTrackedActionOrRationalePct=100 (or + reviewsMissingTrackedActionOrRationale=0), measuredAt ≤90 days. + N/A when sevEligibleIncidentCount=0. +live: + required: false + note: Offline post-incident heuristics + optional imported JSON. diff --git a/skills/aprf-auditor/tests/ai-containment-drill-smoke.ts b/skills/aprf-auditor/tests/ai-containment-drill-smoke.ts new file mode 100644 index 0000000..37f0838 --- /dev/null +++ b/skills/aprf-auditor/tests/ai-containment-drill-smoke.ts @@ -0,0 +1,125 @@ +/** + * Smoke: ai-containment-drill needs pause + disable + rollback within budgets for PASS. + */ +import { + mkdtempSync, + mkdirSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + aiContainmentDrillCollector, + type AiContainmentDrillReport, +} from "../collectors/ai-containment-drill.ts"; +import type { CollectorContext } from "../collectors/types.ts"; + +async function run( + target: string, + outDir: string, +): Promise { + await aiContainmentDrillCollector.collect({ + targetPath: target, + outputDir: outDir, + assessedAt: new Date(), + gitCommit: undefined, + live: false, + maxFiles: 2000, + } as CollectorContext); + return JSON.parse( + readFileSync( + join( + outDir, + "imports", + "ai-containment-drill", + "ai-containment-drill-report.json", + ), + "utf8", + ), + ); +} + +async function main() { + const root = mkdtempSync(join(tmpdir(), "aprf-inc-m2-")); + try { + const t1 = join(root, "t1"); + mkdirSync(join(t1, "docs"), { recursive: true }); + writeFileSync( + join(t1, "docs", "containment.md"), + "Containment: pause agent, disable tools, roll back prompt within time budget\n", + ); + const r1 = await run(t1, join(root, "o1")); + if ( + r1.summary.statusHint !== "partial" || + r1.summary.incM2Satisfied !== false + ) { + throw new Error(`partial expected: ${JSON.stringify(r1.summary)}`); + } + + const t2 = join(root, "t2"); + mkdirSync(join(t2, "ops"), { recursive: true }); + writeFileSync( + join(t2, "ops", "containment-drill.md"), + [ + "# Containment drill", + "pause agent within 5 min", + "disable tool / revoke tool access", + "roll back prompt and model pin", + "time budget documented", + ].join("\n"), + ); + const out2 = join(root, "o2"); + mkdirSync(join(out2, "imports", "ai-containment-drill"), { + recursive: true, + }); + writeFileSync( + join(out2, "imports", "ai-containment-drill", "coverage.json"), + JSON.stringify({ + measuredAt: new Date().toISOString(), + pauseAgentsDemonstrated: true, + disableToolsDemonstrated: true, + rollbackPromptOrModelDemonstrated: true, + withinDocumentedTimeBudgets: true, + }), + ); + const r2 = await run(t2, out2); + if (r2.summary.statusHint !== "pass" || r2.summary.incM2Satisfied !== true) { + throw new Error(`pass expected: ${JSON.stringify(r2.summary)}`); + } + + const t3 = join(root, "t3"); + mkdirSync(join(t3, "ops"), { recursive: true }); + writeFileSync( + join(t3, "ops", "kill-switch.md"), + "pause agent emergency stop drill\n", + ); + const out3 = join(root, "o3"); + mkdirSync(join(out3, "imports", "ai-containment-drill"), { + recursive: true, + }); + writeFileSync( + join(out3, "imports", "ai-containment-drill", "coverage.json"), + JSON.stringify({ + measuredAt: new Date().toISOString(), + pauseAgentsDemonstrated: true, + disableToolsDemonstrated: true, + rollbackPromptOrModelDemonstrated: true, + withinDocumentedTimeBudgets: false, + }), + ); + const r3 = await run(t3, out3); + if (r3.summary.statusHint !== "fail" || r3.summary.incM2Satisfied !== false) { + throw new Error(`fail expected: ${JSON.stringify(r3.summary)}`); + } + + console.log("ai-containment-drill smoke OK"); + } finally { + rmSync(root, { recursive: true, force: true }); + } +} +main().catch((e) => { + console.error(e); + process.exit(1); +}); diff --git a/skills/aprf-auditor/tests/ai-customer-notification-criteria-smoke.ts b/skills/aprf-auditor/tests/ai-customer-notification-criteria-smoke.ts new file mode 100644 index 0000000..a793891 --- /dev/null +++ b/skills/aprf-auditor/tests/ai-customer-notification-criteria-smoke.ts @@ -0,0 +1,160 @@ +/** + * Smoke: ai-customer-notification-criteria needs criteria map + followed sample ≤12m for PASS. + */ +import { + mkdtempSync, + mkdirSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + aiCustomerNotificationCriteriaCollector, + type AiCustomerNotificationCriteriaReport, +} from "../collectors/ai-customer-notification-criteria.ts"; +import type { CollectorContext } from "../collectors/types.ts"; + +async function run( + target: string, + outDir: string, +): Promise { + await aiCustomerNotificationCriteriaCollector.collect({ + targetPath: target, + outputDir: outDir, + assessedAt: new Date(), + gitCommit: undefined, + live: false, + maxFiles: 2000, + } as CollectorContext); + return JSON.parse( + readFileSync( + join( + outDir, + "imports", + "ai-customer-notification-criteria", + "ai-customer-notification-criteria-report.json", + ), + "utf8", + ), + ); +} + +async function main() { + const root = mkdtempSync(join(tmpdir(), "aprf-inc-r3-")); + try { + const t1 = join(root, "t1"); + mkdirSync(join(t1, "docs"), { recursive: true }); + writeFileSync( + join(t1, "docs", "customer-notification-criteria.md"), + "AI customer notification criteria: safety incident → notify; quality fail → no-notify\n", + ); + const r1 = await run(t1, join(root, "o1")); + if ( + r1.summary.statusHint !== "partial" || + r1.summary.incR3Satisfied !== false + ) { + throw new Error(`partial expected: ${JSON.stringify(r1.summary)}`); + } + + const t2 = join(root, "t2"); + mkdirSync(join(t2, "comms"), { recursive: true }); + writeFileSync( + join(t2, "comms", "notification-drill.md"), + "Customer notification drill for data exposure followed notify criteria with timestamps 2026-01-15\n", + ); + const out2 = join(root, "o2"); + mkdirSync(join(out2, "imports", "ai-customer-notification-criteria"), { + recursive: true, + }); + writeFileSync( + join( + out2, + "imports", + "ai-customer-notification-criteria", + "coverage.json", + ), + JSON.stringify({ + measuredAt: new Date().toISOString(), + criteriaMapEventTypesToNotifyDecision: true, + lastDrillOrIncidentFollowedCriteriaWithin12Months: true, + timestampsPresent: true, + }), + ); + const r2 = await run(t2, out2); + if (r2.summary.statusHint !== "pass" || r2.summary.incR3Satisfied !== true) { + throw new Error(`pass expected: ${JSON.stringify(r2.summary)}`); + } + + const t3 = join(root, "t3"); + mkdirSync(join(t3, "docs"), { recursive: true }); + writeFileSync( + join(t3, "docs", "disclosure-criteria.md"), + "Notify/no-notify criteria for AI safety incident and widespread quality fail\n", + ); + const out3 = join(root, "o3"); + mkdirSync(join(out3, "imports", "ai-customer-notification-criteria"), { + recursive: true, + }); + writeFileSync( + join( + out3, + "imports", + "ai-customer-notification-criteria", + "coverage.json", + ), + JSON.stringify({ + measuredAt: new Date().toISOString(), + criteriaMapEventTypesToNotifyDecision: true, + lastDrillOrIncidentAgeDays: 400, + timestampsPresent: true, + }), + ); + const r3 = await run(t3, out3); + if (r3.summary.statusHint !== "fail" || r3.summary.incR3Satisfied !== false) { + throw new Error(`fail expected: ${JSON.stringify(r3.summary)}`); + } + + const t4 = join(root, "t4"); + mkdirSync(join(t4, "docs"), { recursive: true }); + writeFileSync( + join(t4, "docs", "customer-notification-criteria.md"), + "AI customer notification criteria map with notify/no-notify for safety incident\n", + ); + const out4 = join(root, "o4"); + mkdirSync(join(out4, "imports", "ai-customer-notification-criteria"), { + recursive: true, + }); + writeFileSync( + join( + out4, + "imports", + "ai-customer-notification-criteria", + "coverage.json", + ), + JSON.stringify({ + measuredAt: new Date().toISOString(), + criteriaMapEventTypesToNotifyDecision: true, + lastDrillOrIncidentFollowedCriteriaWithin12Months: true, + }), + ); + const r4 = await run(t4, out4); + if ( + r4.summary.statusHint !== "partial" || + r4.summary.incR3Satisfied !== false + ) { + throw new Error( + `partial without timestamps expected: ${JSON.stringify(r4.summary)}`, + ); + } + + console.log("ai-customer-notification-criteria smoke OK"); + } finally { + rmSync(root, { recursive: true, force: true }); + } +} +main().catch((e) => { + console.error(e); + process.exit(1); +}); diff --git a/skills/aprf-auditor/tests/ai-incident-tabletop-smoke.ts b/skills/aprf-auditor/tests/ai-incident-tabletop-smoke.ts new file mode 100644 index 0000000..5078b1e --- /dev/null +++ b/skills/aprf-auditor/tests/ai-incident-tabletop-smoke.ts @@ -0,0 +1,115 @@ +/** + * Smoke: ai-incident-tabletop needs completion ≤180d + owned actions for PASS. + */ +import { + mkdtempSync, + mkdirSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + aiIncidentTabletopCollector, + type AiIncidentTabletopReport, +} from "../collectors/ai-incident-tabletop.ts"; +import type { CollectorContext } from "../collectors/types.ts"; + +async function run( + target: string, + outDir: string, +): Promise { + await aiIncidentTabletopCollector.collect({ + targetPath: target, + outputDir: outDir, + assessedAt: new Date(), + gitCommit: undefined, + live: false, + maxFiles: 2000, + } as CollectorContext); + return JSON.parse( + readFileSync( + join( + outDir, + "imports", + "ai-incident-tabletop", + "ai-incident-tabletop-report.json", + ), + "utf8", + ), + ); +} + +async function main() { + const root = mkdtempSync(join(tmpdir(), "aprf-inc-r4-")); + try { + const t1 = join(root, "t1"); + mkdirSync(join(t1, "docs"), { recursive: true }); + writeFileSync( + join(t1, "docs", "ai-tabletop.md"), + "AI incident tabletop for prompt injection scenario\n", + ); + const r1 = await run(t1, join(root, "o1")); + if ( + r1.summary.statusHint !== "partial" || + r1.summary.incR4Satisfied !== false + ) { + throw new Error(`partial expected: ${JSON.stringify(r1.summary)}`); + } + + const t2 = join(root, "t2"); + mkdirSync(join(t2, "exercises"), { recursive: true }); + writeFileSync( + join(t2, "exercises", "tabletop-aar.md"), + "Tabletop after-action report: LLM outage scenario; retained action with owner\n", + ); + const out2 = join(root, "o2"); + mkdirSync(join(out2, "imports", "ai-incident-tabletop"), { + recursive: true, + }); + writeFileSync( + join(out2, "imports", "ai-incident-tabletop", "coverage.json"), + JSON.stringify({ + measuredAt: new Date().toISOString(), + aiFocusedTabletopCompletedWithin180Days: true, + retainedActionsWithOwners: true, + }), + ); + const r2 = await run(t2, out2); + if (r2.summary.statusHint !== "pass" || r2.summary.incR4Satisfied !== true) { + throw new Error(`pass expected: ${JSON.stringify(r2.summary)}`); + } + + const t3 = join(root, "t3"); + mkdirSync(join(t3, "docs"), { recursive: true }); + writeFileSync( + join(t3, "docs", "war-game.md"), + "Game day tabletop for agent abuse AI incident\n", + ); + const out3 = join(root, "o3"); + mkdirSync(join(out3, "imports", "ai-incident-tabletop"), { + recursive: true, + }); + writeFileSync( + join(out3, "imports", "ai-incident-tabletop", "coverage.json"), + JSON.stringify({ + measuredAt: new Date().toISOString(), + tabletopAgeDays: 200, + retainedActionsWithOwners: true, + }), + ); + const r3 = await run(t3, out3); + if (r3.summary.statusHint !== "fail" || r3.summary.incR4Satisfied !== false) { + throw new Error(`fail expected: ${JSON.stringify(r3.summary)}`); + } + + console.log("ai-incident-tabletop smoke OK"); + } finally { + rmSync(root, { recursive: true, force: true }); + } +} +main().catch((e) => { + console.error(e); + process.exit(1); +}); diff --git a/skills/aprf-auditor/tests/ai-safety-quality-alerts-smoke.ts b/skills/aprf-auditor/tests/ai-safety-quality-alerts-smoke.ts new file mode 100644 index 0000000..91d50ce --- /dev/null +++ b/skills/aprf-auditor/tests/ai-safety-quality-alerts-smoke.ts @@ -0,0 +1,117 @@ +/** + * Smoke: ai-safety-quality-alerts needs ≥2 paging signals + thresholds/owners + review for PASS. + */ +import { + mkdtempSync, + mkdirSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + aiSafetyQualityAlertsCollector, + type AiSafetyQualityAlertsReport, +} from "../collectors/ai-safety-quality-alerts.ts"; +import type { CollectorContext } from "../collectors/types.ts"; + +async function run( + target: string, + outDir: string, +): Promise { + await aiSafetyQualityAlertsCollector.collect({ + targetPath: target, + outputDir: outDir, + assessedAt: new Date(), + gitCommit: undefined, + live: false, + maxFiles: 2000, + } as CollectorContext); + return JSON.parse( + readFileSync( + join( + outDir, + "imports", + "ai-safety-quality-alerts", + "ai-safety-quality-alerts-report.json", + ), + "utf8", + ), + ); +} + +async function main() { + const root = mkdtempSync(join(tmpdir(), "aprf-inc-r1-")); + try { + const t1 = join(root, "t1"); + mkdirSync(join(t1, "ops"), { recursive: true }); + writeFileSync( + join(t1, "ops", "refusal-rate-alert.md"), + "PagerDuty alert for refusal-rate spike\n", + ); + const r1 = await run(t1, join(root, "o1")); + if ( + r1.summary.statusHint !== "partial" || + r1.summary.incR1Satisfied !== false + ) { + throw new Error(`partial expected: ${JSON.stringify(r1.summary)}`); + } + + const t2 = join(root, "t2"); + mkdirSync(join(t2, "alerting"), { recursive: true }); + writeFileSync( + join(t2, "alerting", "safety-quality-paging.yaml"), + "on-call pages for toxicity and eval-score drop with threshold and owner\n", + ); + const out2 = join(root, "o2"); + mkdirSync(join(out2, "imports", "ai-safety-quality-alerts"), { + recursive: true, + }); + writeFileSync( + join(out2, "imports", "ai-safety-quality-alerts", "coverage.json"), + JSON.stringify({ + measuredAt: new Date().toISOString(), + atLeastTwoNonInfraPagingSignals: true, + eachSignalHasThresholdAndOwner: true, + policyReviewedWithin90Days: true, + }), + ); + const r2 = await run(t2, out2); + if (r2.summary.statusHint !== "pass" || r2.summary.incR1Satisfied !== true) { + throw new Error(`pass expected: ${JSON.stringify(r2.summary)}`); + } + + const t3 = join(root, "t3"); + mkdirSync(join(t3, "docs"), { recursive: true }); + writeFileSync( + join(t3, "docs", "jailbreak-alert.md"), + "On-call paging policy for jailbreak hit rate\n", + ); + const out3 = join(root, "o3"); + mkdirSync(join(out3, "imports", "ai-safety-quality-alerts"), { + recursive: true, + }); + writeFileSync( + join(out3, "imports", "ai-safety-quality-alerts", "coverage.json"), + JSON.stringify({ + measuredAt: new Date().toISOString(), + nonInfraPagingSignalCount: 1, + eachSignalHasThresholdAndOwner: true, + policyReviewedWithin90Days: true, + }), + ); + const r3 = await run(t3, out3); + if (r3.summary.statusHint !== "fail" || r3.summary.incR1Satisfied !== false) { + throw new Error(`fail expected: ${JSON.stringify(r3.summary)}`); + } + + console.log("ai-safety-quality-alerts smoke OK"); + } finally { + rmSync(root, { recursive: true, force: true }); + } +} +main().catch((e) => { + console.error(e); + process.exit(1); +}); diff --git a/skills/aprf-auditor/tests/incident-playbooks-smoke.ts b/skills/aprf-auditor/tests/incident-playbooks-smoke.ts new file mode 100644 index 0000000..9b060f0 --- /dev/null +++ b/skills/aprf-auditor/tests/incident-playbooks-smoke.ts @@ -0,0 +1,125 @@ +/** + * Smoke: incident-playbooks needs four scenarios + owners + reviews ≤12m for PASS. + */ +import { + mkdtempSync, + mkdirSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + incidentPlaybooksCollector, + type IncidentPlaybooksReport, +} from "../collectors/incident-playbooks.ts"; +import type { CollectorContext } from "../collectors/types.ts"; + +async function run( + target: string, + outDir: string, +): Promise { + await incidentPlaybooksCollector.collect({ + targetPath: target, + outputDir: outDir, + assessedAt: new Date(), + gitCommit: undefined, + live: false, + maxFiles: 2000, + } as CollectorContext); + return JSON.parse( + readFileSync( + join( + outDir, + "imports", + "incident-playbooks", + "incident-playbooks-report.json", + ), + "utf8", + ), + ); +} + +async function main() { + const root = mkdtempSync(join(tmpdir(), "aprf-inc-m1-")); + try { + const t1 = join(root, "t1"); + mkdirSync(join(t1, "docs", "incidents"), { recursive: true }); + writeFileSync( + join(t1, "docs", "incidents", "abuse-playbook.md"), + "# Abuse playbook\nRespond to prompt injection and misuse.\n", + ); + writeFileSync( + join(t1, "docs", "incidents", "leakage-runbook.md"), + "# Leakage runbook\nContain PII exposure and data exfil.\n", + ); + const r1 = await run(t1, join(root, "o1")); + if ( + r1.summary.statusHint !== "partial" || + r1.summary.incM1Satisfied !== false + ) { + throw new Error(`partial expected: ${JSON.stringify(r1.summary)}`); + } + + const t2 = join(root, "t2"); + mkdirSync(join(t2, "docs", "incidents"), { recursive: true }); + writeFileSync( + join(t2, "docs", "incidents", "ai-incidents.md"), + [ + "# AI incident playbooks", + "Owner: sre-ai@example.com", + "Last review date: 2026-01-15", + "## Abuse / prompt injection", + "## Leakage / PII exposure", + "## Bad actions / unsafe tool misuse", + "## Provider outage / LLM API outage", + ].join("\n"), + ); + const out2 = join(root, "o2"); + mkdirSync(join(out2, "imports", "incident-playbooks"), { recursive: true }); + writeFileSync( + join(out2, "imports", "incident-playbooks", "coverage.json"), + JSON.stringify({ + measuredAt: new Date().toISOString(), + fourPlaybooksPresent: true, + allPlaybooksHaveOwner: true, + allPlaybooksReviewedWithin12Months: true, + }), + ); + const r2 = await run(t2, out2); + if (r2.summary.statusHint !== "pass" || r2.summary.incM1Satisfied !== true) { + throw new Error(`pass expected: ${JSON.stringify(r2.summary)}`); + } + + const t3 = join(root, "t3"); + mkdirSync(join(t3, "docs"), { recursive: true }); + writeFileSync( + join(t3, "docs", "provider-outage-playbook.md"), + "Provider outage playbook for model API outage\n", + ); + const out3 = join(root, "o3"); + mkdirSync(join(out3, "imports", "incident-playbooks"), { recursive: true }); + writeFileSync( + join(out3, "imports", "incident-playbooks", "coverage.json"), + JSON.stringify({ + measuredAt: new Date().toISOString(), + fourPlaybooksPresent: true, + allPlaybooksHaveOwner: true, + allPlaybooksReviewedWithin12Months: false, + }), + ); + const r3 = await run(t3, out3); + if (r3.summary.statusHint !== "fail" || r3.summary.incM1Satisfied !== false) { + throw new Error(`fail expected: ${JSON.stringify(r3.summary)}`); + } + + console.log("incident-playbooks smoke OK"); + } finally { + rmSync(root, { recursive: true, force: true }); + } +} +main().catch((e) => { + console.error(e); + process.exit(1); +}); diff --git a/skills/aprf-auditor/tests/post-incident-aprf-actions-smoke.ts b/skills/aprf-auditor/tests/post-incident-aprf-actions-smoke.ts new file mode 100644 index 0000000..290af13 --- /dev/null +++ b/skills/aprf-auditor/tests/post-incident-aprf-actions-smoke.ts @@ -0,0 +1,160 @@ +/** + * Smoke: post-incident-aprf-actions needs 100% coverage (or N/A when sev=0). + */ +import { + mkdtempSync, + mkdirSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + postIncidentAprfActionsCollector, + type PostIncidentAprfActionsReport, +} from "../collectors/post-incident-aprf-actions.ts"; +import type { CollectorContext } from "../collectors/types.ts"; + +async function run( + target: string, + outDir: string, +): Promise { + await postIncidentAprfActionsCollector.collect({ + targetPath: target, + outputDir: outDir, + assessedAt: new Date(), + gitCommit: undefined, + live: false, + maxFiles: 2000, + } as CollectorContext); + return JSON.parse( + readFileSync( + join( + outDir, + "imports", + "post-incident-aprf-actions", + "post-incident-aprf-actions-report.json", + ), + "utf8", + ), + ); +} + +async function main() { + const root = mkdtempSync(join(tmpdir(), "aprf-inc-r2-")); + try { + const t1 = join(root, "t1"); + mkdirSync(join(t1, "docs"), { recursive: true }); + writeFileSync( + join(t1, "docs", "postmortem-template.md"), + "Post-incident review with APRF pillar tracked action mapping\n", + ); + const r1 = await run(t1, join(root, "o1")); + if ( + r1.summary.statusHint !== "partial" || + r1.summary.incR2Satisfied !== false + ) { + throw new Error(`partial expected: ${JSON.stringify(r1.summary)}`); + } + + const t2 = join(root, "t2"); + mkdirSync(join(t2, "incidents"), { recursive: true }); + writeFileSync( + join(t2, "incidents", "pir.md"), + "SEV-2 postmortem: tracked action mapped to APRF pillar; no-action rationale optional\n", + ); + const out2 = join(root, "o2"); + mkdirSync(join(out2, "imports", "post-incident-aprf-actions"), { + recursive: true, + }); + writeFileSync( + join(out2, "imports", "post-incident-aprf-actions", "coverage.json"), + JSON.stringify({ + measuredAt: new Date().toISOString(), + sevEligibleIncidentCount: 2, + reviewsWithTrackedActionOrRationalePct: 100, + }), + ); + const r2 = await run(t2, out2); + if (r2.summary.statusHint !== "pass" || r2.summary.incR2Satisfied !== true) { + throw new Error(`pass expected: ${JSON.stringify(r2.summary)}`); + } + + const t3 = join(root, "t3"); + mkdirSync(join(t3, "docs"), { recursive: true }); + writeFileSync( + join(t3, "docs", "after-action.md"), + "After-action report for AI incident\n", + ); + const out3 = join(root, "o3"); + mkdirSync(join(out3, "imports", "post-incident-aprf-actions"), { + recursive: true, + }); + writeFileSync( + join(out3, "imports", "post-incident-aprf-actions", "coverage.json"), + JSON.stringify({ + measuredAt: new Date().toISOString(), + sevEligibleIncidentCount: 1, + reviewsMissingTrackedActionOrRationale: 1, + }), + ); + const r3 = await run(t3, out3); + if (r3.summary.statusHint !== "fail" || r3.summary.incR2Satisfied !== false) { + throw new Error(`fail expected: ${JSON.stringify(r3.summary)}`); + } + + const t4 = join(root, "t4"); + mkdirSync(t4, { recursive: true }); + const out4 = join(root, "o4"); + mkdirSync(join(out4, "imports", "post-incident-aprf-actions"), { + recursive: true, + }); + writeFileSync( + join(out4, "imports", "post-incident-aprf-actions", "coverage.json"), + JSON.stringify({ + measuredAt: new Date().toISOString(), + sevEligibleIncidentCount: 0, + }), + ); + const r4 = await run(t4, out4); + if (r4.summary.statusHint !== "not_applicable") { + throw new Error(`na expected: ${JSON.stringify(r4.summary)}`); + } + + const t5 = join(root, "t5"); + mkdirSync(join(t5, "docs"), { recursive: true }); + writeFileSync( + join(t5, "docs", "postmortem.md"), + "Post-incident review with APRF pillar action\n", + ); + const out5 = join(root, "o5"); + mkdirSync(join(out5, "imports", "post-incident-aprf-actions"), { + recursive: true, + }); + writeFileSync( + join(out5, "imports", "post-incident-aprf-actions", "coverage.json"), + JSON.stringify({ + measuredAt: new Date().toISOString(), + reviewsWithTrackedActionOrRationalePct: 100, + }), + ); + const r5 = await run(t5, out5); + if ( + r5.summary.statusHint !== "partial" || + r5.summary.incR2Satisfied !== false + ) { + throw new Error( + `partial without sev count expected: ${JSON.stringify(r5.summary)}`, + ); + } + + console.log("post-incident-aprf-actions smoke OK"); + } finally { + rmSync(root, { recursive: true, force: true }); + } +} +main().catch((e) => { + console.error(e); + process.exit(1); +}); diff --git a/spec/aprf-spec.json b/spec/aprf-spec.json index 7b8907d..0638b56 100644 --- a/spec/aprf-spec.json +++ b/spec/aprf-spec.json @@ -273,6 +273,18 @@ "slug": "0001-working-draft-rfc-process", "href": "/aprf/rfc/0001-working-draft-rfc-process/", "markdownPath": "/aprf/rfc/0001-working-draft-rfc-process.md" + }, + { + "id": "APRF-RFC-0002", + "number": 2, + "title": "Demote incident-readiness INC-M3 and INC-M4 to recommended", + "status": "accepted", + "created": "2026-08-01", + "semverImpact": "MINOR", + "summary": "Demotes INC-M3→INC-R2 and INC-M4→INC-R4; documents pre-release exception for removing IDs without deprecated stubs before the first tagged version.", + "slug": "0002-incident-readiness-mandatory-to-recommended", + "href": "/aprf/rfc/0002-incident-readiness-mandatory-to-recommended/", + "markdownPath": "/aprf/rfc/0002-incident-readiness-mandatory-to-recommended.md" } ], "scope": { @@ -621,8 +633,7 @@ "EVL-M4", "HUM-M4", "REL-M7", - "REL-M8", - "INC-M4" + "REL-M8" ], "rationale": [ "Includes every Core gate — regulated systems must still clear production minimums.", @@ -4653,59 +4664,59 @@ "mandatoryChecks": [ { "id": "INC-M1", - "requirement": "AI-specific incident playbooks shall exist for abuse, leakage, bad actions, and provider outage", - "artifact": "Playbook set covering the four scenarios with owners", - "passCondition": "Four playbooks present (abuse, leakage, bad actions, provider outage), each with owner and review date ≤ 12 months", + "requirement": "Production AI systems must maintain incident playbooks for abuse, data leakage, bad agent/tool actions, and provider outage—each with a named owner and a review date within the last 12 months.", + "artifact": "Playbook set covering abuse, leakage, bad actions, and provider outage with owners + Review dates ≤12 months for each of the four playbooks", + "passCondition": "Four playbooks present (abuse, leakage, bad actions, provider outage), each with owner and review date ≤12 months (playbook evidence measuredAt ≤90 days).", "method": "hybrid", "requiredFromLevel": 3, "minCriticality": 2 }, { "id": "INC-M2", - "requirement": "On-call shall be able to execute containment: pause agents, disable tools, roll back prompts/models", - "artifact": "Containment runbook + drill record exercising pause, disable, and rollback", - "passCondition": "Drill in last 90 days successfully demonstrated pause agents, disable tools, and roll back prompt/model within documented time budgets", - "method": "manual", - "requiredFromLevel": 3, - "minCriticality": 2 - }, - { - "id": "INC-M3", - "requirement": "Post-incident reviews shall produce tracked actions against APRF pillars", - "artifact": "Post-incident review template + sample reviews with linked actions", - "passCondition": "100% of SEV-eligible AI incidents in last 90 days have a review with ≥1 tracked action mapped to an APRF pillar or explicit “no action” rationale", - "method": "manual", + "requirement": "On-call must be able to pause agents, disable tools, and roll back prompts or models, proven by a drill in the last 90 days completed within documented time budgets.", + "artifact": "Containment runbook covering pause agents, disable tools, and prompt/model rollback + Drill record ≤90 days showing all three actions within documented time budgets", + "passCondition": "Drill in last 90 days successfully demonstrated pause agents, disable tools, and roll back prompt/model within documented time budgets (drill evidence measuredAt ≤90 days).", + "method": "hybrid", "requiredFromLevel": 3, "minCriticality": 2 - }, - { - "id": "INC-M4", - "requirement": "Regular tabletop exercises shall cover AI-specific incidents", - "artifact": "Tabletop plan + dated after-action report for an AI incident scenario", - "passCondition": "PASS if an AI-focused tabletop completed ≤180 days with retained actions and owners", - "method": "manual", - "requiredFromLevel": 4, - "minCriticality": 3 } ], "recommendedChecks": [ { "id": "INC-R1", - "requirement": "Page-worthy alerts for safety and quality signals, not only infra", - "artifact": "On-call alert policy export listing safety/quality pages + last 90 days of triggered incidents (or drill tickets)", - "passCondition": "At least two non-infra signals (e.g. refusal-rate spike, eval-score drop, toxicity/jailbreak hit rate) page an on-call; each has a documented threshold and owner; policy reviewed ≤90 days ago", - "method": "manual", + "requirement": "On-call should receive page-worthy alerts for at least two non-infra AI safety/quality signals, each with a documented threshold and owner, and a policy reviewed within 90 days.", + "artifact": "On-call alert policy export listing safety/quality pages + Last 90 days of triggered incidents or drill tickets for those pages", + "passCondition": "At least two non-infra signals (e.g. refusal-rate spike, eval-score drop, toxicity/jailbreak hit rate) page an on-call; each has a documented threshold and owner; policy reviewed ≤90 days ago (alert evidence measuredAt ≤90 days).", + "method": "hybrid", "requiredFromLevel": 4, "minCriticality": 2 }, + { + "id": "INC-R2", + "requirement": "SEV-eligible AI incidents should receive a post-incident review with at least one tracked action mapped to an APRF pillar—or an explicit no-action rationale.", + "artifact": "Post-incident review template requiring APRF pillar mapping + Last-90-day coverage: SEV-eligible AI reviews with tracked actions or no-action rationale", + "passCondition": "100% of SEV-eligible AI incidents in last 90 days have a review with ≥1 tracked action mapped to an APRF pillar or explicit “no action” rationale (review evidence measuredAt ≤90 days).", + "method": "hybrid", + "requiredFromLevel": 3, + "minCriticality": 2 + }, { "id": "INC-R3", - "requirement": "Customer notification criteria for AI-related events", - "artifact": "Customer notification criteria for AI-related events + last drill or real notification sample", - "passCondition": "Criteria map event types (safety incident, widespread quality fail, data exposure) to notify / no-notify; last drill or incident ≤12 months followed the criteria with timestamps", - "method": "manual", + "requirement": "Teams should maintain criteria that map AI event types to notify / no-notify decisions, and show a drill or real notification ≤12 months that followed those criteria with timestamps.", + "artifact": "Customer notification criteria for AI-related events + Last drill or real notification sample ≤12 months with timestamps", + "passCondition": "Criteria map event types (safety incident, widespread quality fail, data exposure) to notify / no-notify; last drill or incident ≤12 months followed the criteria with timestamps (notification evidence measuredAt ≤90 days).", + "method": "hybrid", "requiredFromLevel": 4, "minCriticality": 2 + }, + { + "id": "INC-R4", + "requirement": "Teams should complete an AI-focused incident tabletop at least every 180 days and retain an after-action report with actions and owners.", + "artifact": "Tabletop plan for an AI-specific incident scenario + Dated after-action report ≤180 days with retained actions and owners", + "passCondition": "An AI-focused tabletop completed ≤180 days with retained actions and owners (tabletop evidence measuredAt ≤90 days).", + "method": "hybrid", + "requiredFromLevel": 4, + "minCriticality": 3 } ], "evidenceRequired": [