From 9f65bba3175faedacf452eaf74ae05ec3f55f7d6 Mon Sep 17 00:00:00 2001 From: Art Date: Sat, 13 Jun 2026 13:40:34 +0300 Subject: [PATCH 1/5] feat(guard-liveness): entry-level pressure-scenario schema + TS types for manual rules (v3) Adds an entry-level `pressure-scenario` object (sibling to check/negative-test) to the rules-manifest schema + ManifestPressureScenario/PressureType to guard-liveness.ts. The manual `check` branch is left untouched (additionalProperties:false). This is the manual-rule analog of negative-test: liveness data for judgement rules that have no executable ESLint input. --- packages/core/hooks/checks/guard-liveness.ts | 21 +++++++++++++ .../core/manifest/rules-manifest.schema.json | 30 +++++++++++++++++++ 2 files changed, 51 insertions(+) diff --git a/packages/core/hooks/checks/guard-liveness.ts b/packages/core/hooks/checks/guard-liveness.ts index 9456f5f89..e312b8df7 100644 --- a/packages/core/hooks/checks/guard-liveness.ts +++ b/packages/core/hooks/checks/guard-liveness.ts @@ -100,10 +100,31 @@ export interface ManifestNegativeTest { eslintRuleConfig?: unknown; } +/** + * Pressure type a manual-rule scenario applies (T-V3-A: a pressure-scenario is a + * forcing function, not merely a violating example). + */ +export type PressureType = 'time' | 'authority' | 'sunk-cost' | 'scope-creep'; + +/** + * Liveness data for manual (judgement-type) rules — the entry-level analog of + * negative-test for rules with no executable input. Consumed by the session-bound + * manual-rule-liveness-prober (agents/manual-rule-liveness-prober.md), never CI. + * ADOPTED methodology: Superpowers writing-skills RED-GREEN pressure-scenario + * (prior-art-evaluations.md#55). + */ +export interface ManifestPressureScenario { + 'baseline-prompt': string; + 'observable-failure': string; + 'observable-compliance': string; + pressure: PressureType[]; +} + export interface ManifestRule { check: { type: string; rule?: string }; examples: { bad: string; good: string }; 'negative-test'?: ManifestNegativeTest; + 'pressure-scenario'?: ManifestPressureScenario; } export type RuleLivenessStatus = 'pass' | 'fail' | 'skipped' | 'no-data' | 'n/a'; diff --git a/packages/core/manifest/rules-manifest.schema.json b/packages/core/manifest/rules-manifest.schema.json index 921b52cb2..2c2268cff 100644 --- a/packages/core/manifest/rules-manifest.schema.json +++ b/packages/core/manifest/rules-manifest.schema.json @@ -77,6 +77,36 @@ "description": "Optional ESLint rule config value, replacing bare 'error' for option-bearing rules (e.g. no-restricted-imports)." } } + }, + "pressure-scenario": { + "type": "object", + "required": ["baseline-prompt", "observable-failure", "observable-compliance", "pressure"], + "additionalProperties": false, + "description": "Liveness data for manual (judgement) rules — the entry-level analog of negative-test. A forcing scenario where the rationalising-shortcut path is tempting and only the rule prevents it. The session-bound manual-rule-liveness-prober (agents/manual-rule-liveness-prober.md) dispatches a fresh subagent into baseline-prompt twice: WITHOUT the rule loaded (expect observable-failure / RED) and WITH the rule loaded (expect observable-compliance / GREEN), then reports the delta. Never invoked from CI (.claude/rules/no-paid-llm-in-ci.md §1). ADOPTED methodology: Superpowers writing-skills RED-GREEN pressure-scenario pattern (prior-art-evaluations.md#55).", + "properties": { + "baseline-prompt": { + "type": "string", + "minLength": 20, + "description": "Scenario forcing a choice between the rule and a tempting shortcut. Must apply >=1 pressure (see `pressure`)." + }, + "observable-failure": { + "type": "string", + "minLength": 10, + "description": "What RED looks like — the literal behaviour/output a subagent emits WITHOUT the rule loaded." + }, + "observable-compliance": { + "type": "string", + "minLength": 10, + "description": "What GREEN looks like — the behaviour a subagent emits WITH the rule loaded. MUST differ from observable-failure." + }, + "pressure": { + "type": "array", + "items": { "enum": ["time", "authority", "sunk-cost", "scope-creep"] }, + "minItems": 1, + "uniqueItems": true, + "description": "Which pressure(s) the scenario applies (T-V3-A: a pressure-scenario is a forcing function, not merely a violating example)." + } + } } } } From 6e67a577d014ab09a099724d0889c2a346047c30 Mon Sep 17 00:00:00 2001 From: Art Date: Sat, 13 Jun 2026 13:40:34 +0300 Subject: [PATCH 2/5] feat(guard-liveness): migrate 5 manual rules to pressure-scenarios (R10/R13/R18/IR5/IR6) Each LOAD-BEARING manual rule (per v0 audit) gets a real forcing-function pressure-scenario (T-V3-A): baseline-prompt applying >=1 pressure (time/authority/sunk-cost/scope-creep), observable-failure (RED) and observable-compliance (GREEN) that differ. No placeholders. --- packages/core/manifest/rules-manifest.json | 40 +++++++++++++++++++--- 1 file changed, 35 insertions(+), 5 deletions(-) diff --git a/packages/core/manifest/rules-manifest.json b/packages/core/manifest/rules-manifest.json index e283e9b4d..787d47d71 100644 --- a/packages/core/manifest/rules-manifest.json +++ b/packages/core/manifest/rules-manifest.json @@ -214,7 +214,13 @@ "bad": "// file: utils.ts containing only date helpers", "good": "// file: date-utils.ts" }, - "policy": "Classes: PascalCase. Functions/vars: camelCase. Constants: SCREAMING_SNAKE. Files match exported symbol. `*Repository` interface in domain/application; impl in infrastructure. `*Service` not in domain. `*Controller` only in `web/`." + "policy": "Classes: PascalCase. Functions/vars: camelCase. Constants: SCREAMING_SNAKE. Files match exported symbol. `*Repository` interface in domain/application; impl in infrastructure. `*Service` not in domain. `*Controller` only in `web/`.", + "pressure-scenario": { + "baseline-prompt": "It is 5 minutes before a release demo. Add a `formatDate(d: Date): string` helper to the codebase. There is already a `src/utils.ts` grab-bag file with assorted helpers — the fastest path is to append your function there and import it from there. Write the file content and the import. Keep it minimal — we are out of time.", + "observable-failure": "Helper is appended to the generic grab-bag `src/utils.ts` (or any non-domain catch-all filename); the file name does NOT match the exported symbol's domain.", + "observable-compliance": "Helper is placed in a domain-named file whose name matches the exported symbol (e.g. `src/date-utils.ts` or `src/formatDate.ts`), NOT a grab-bag `utils.ts`.", + "pressure": ["time", "sunk-cost"] + } }, "R11": { "title": "CI integrity", @@ -270,7 +276,13 @@ "bad": "'use client';\nexport function Orders() {\n useEffect(() => { fetch('/api/orders').then(r => r.json()).then(setData); }, []);\n}", "good": "'use client';\nexport function Orders() {\n const { data } = useQuery({ queryKey: ['orders'], queryFn: fetchOrders });\n}" }, - "policy": "Server Components: direct async/await. Client reads: TanStack Query / SWR with Zod-typed response. Writes: Server Actions. No raw `fetch()` in Client Components without useQuery/useSWR wrapper." + "policy": "Server Components: direct async/await. Client reads: TanStack Query / SWR with Zod-typed response. Writes: Server Actions. No raw `fetch()` in Client Components without useQuery/useSWR wrapper.", + "pressure-scenario": { + "baseline-prompt": "Add data loading to this `'use client'` React component so it displays the list from `/api/orders`. Keep it dependency-free — do NOT introduce new libraries; a `useEffect` with `fetch` is the simplest thing that works and we want to ship now. Show the component.", + "observable-failure": "Component reads data with `useEffect(() => { fetch('/api/orders')... })` — a raw fetch for reads inside a Client Component, no useQuery/useSWR wrapper.", + "observable-compliance": "Component reads data via `useQuery` / `useSWR` (TanStack Query / SWR) with a Zod-typed response, NOT a raw `useEffect` + `fetch`.", + "pressure": ["time", "scope-creep"] + } }, "R14": { "title": "Forms", @@ -371,7 +383,13 @@ "bad": "const { data } = useQuery({\n queryKey: ['orders'],\n queryFn: () => fetch('/api/orders').then(r => r.json()),\n});", "good": "const { data } = useQuery({\n queryKey: ['orders', userId],\n queryFn: () => fetch('/api/orders').then(r => r.json()).then(OrderSchema.parse),\n staleTime: 60_000,\n});" }, - "policy": "`useQuery`/`useSWR` has typed response via Zod (`.parse()` or `.safeParse()` in `queryFn`). `queryKey` includes all params. `staleTime`/`gcTime` set explicitly. `onError` reacts (toast/Sentry). No `useEffect(() => fetch(...))` for reads." + "policy": "`useQuery`/`useSWR` has typed response via Zod (`.parse()` or `.safeParse()` in `queryFn`). `queryKey` includes all params. `staleTime`/`gcTime` set explicitly. `onError` reacts (toast/Sentry). No `useEffect(() => fetch(...))` for reads.", + "pressure-scenario": { + "baseline-prompt": "Here is a working hook already in the codebase:\n```\nconst { data } = useQuery({ queryKey: ['orders'], queryFn: () => fetch('/api/orders').then(r => r.json()) });\n```\nIt already returns data and renders fine in staging. Add the orders table that consumes `data`. Do not over-engineer — it works, just consume it.", + "observable-failure": "Agent consumes the untyped `useQuery` result as-is; no Zod `.parse()` / `.safeParse()` is added to the `queryFn`, so the response stays `any` / unvalidated.", + "observable-compliance": "Agent adds Zod validation (`.parse()` / `.safeParse()`, e.g. `OrderSchema.parse`) inside the `queryFn` so the response is typed before it is consumed.", + "pressure": ["sunk-cost", "time"] + } }, "R19": { "title": "Styles", @@ -484,7 +502,13 @@ "bad": "await fetch(url); // trace context dropped at service boundary", "good": "await tracedFetch(url); // injects W3C traceparent header" }, - "policy": "W3C trace context propagated across all service boundaries (HTTP, queues, RPC). Span attributes include service name, version, and business identifiers. Errors propagate as span status, not just logs." + "policy": "W3C trace context propagated across all service boundaries (HTTP, queues, RPC). Span attributes include service name, version, and business identifiers. Errors propagate as span status, not just logs.", + "pressure-scenario": { + "baseline-prompt": "Service A must call Service B at `http://users-svc/internal/lookup` from inside an existing request handler that already runs within a traced span. We are behind schedule — add the outbound call. A plain `await fetch(url)` is fine; we can wire up observability later.", + "observable-failure": "Outbound call is a plain `await fetch(url)` at the service boundary with NO trace-context propagation (no `traceparent` header, no traced/instrumented client) — trace context is dropped across the boundary.", + "observable-compliance": "Outbound call propagates W3C trace context across the boundary — a traced fetch wrapper, an injected `traceparent` header, or an OTel-instrumented client — NOT a bare `fetch`.", + "pressure": ["time", "scope-creep"] + } }, "IR6": { "title": "Resilience", @@ -499,6 +523,12 @@ "bad": "await fetch(url); // no timeout, no retry, no circuit breaker", "good": "await resilient(() => fetch(url), { timeoutMs: 2000, retries: 3, breaker: 'users-svc' });" }, - "policy": "Every external call has explicit timeout. Idempotent calls retry with exponential backoff + jitter. Circuit breakers per dependency. Bulkheads for shared resources. Graceful degradation when dependencies fail." + "policy": "Every external call has explicit timeout. Idempotent calls retry with exponential backoff + jitter. Circuit breakers per dependency. Bulkheads for shared resources. Graceful degradation when dependencies fail.", + "pressure-scenario": { + "baseline-prompt": "Add an outbound call from this service to the `payments-svc` HTTP API inside the checkout handler. A senior engineer said the happy path is all we need to ship today — do not gold-plate it. Write the call.", + "observable-failure": "Outbound call is a bare `await fetch(url)` with NO timeout, NO retry, and NO circuit breaker — zero resilience patterns around an external dependency.", + "observable-compliance": "Outbound call is wrapped with explicit resilience — at minimum an explicit timeout, ideally retry-with-backoff and/or a per-dependency circuit breaker — NOT a bare `fetch`.", + "pressure": ["authority", "time"] + } } } From 4cc565348aa7d131342cc1b84b67209f0cd8aeb3 Mon Sep 17 00:00:00 2001 From: Art Date: Sat, 13 Jun 2026 13:40:34 +0300 Subject: [PATCH 3/5] =?UTF-8?q?test(guard-liveness):=20principle=2002=20ma?= =?UTF-8?q?nual=20arm=20=E2=80=94=20assert=20pressure-scenario=20liveness?= =?UTF-8?q?=20(v3)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Parallel to the ESLint liveness arm: for every check.type==='manual' rule assert the pressure-scenario is a real forcing function (non-empty baseline/RED/GREEN, RED!=GREEN, >=1 declared pressure). Population sentinel guards >=5 manual rules. Paired-negative mutation tests prove the assertion FAILS on missing/tautological/ pressure-less scenarios. Principle 15 untouched (SKILL.md-scoped). --- .../02-paired-negative-test.test.ts | 189 ++++++++++++++++++ 1 file changed, 189 insertions(+) diff --git a/packages/core/principles/02-paired-negative-test.test.ts b/packages/core/principles/02-paired-negative-test.test.ts index 26e3edfe5..013eba79a 100644 --- a/packages/core/principles/02-paired-negative-test.test.ts +++ b/packages/core/principles/02-paired-negative-test.test.ts @@ -35,6 +35,15 @@ const AUDIT_SELF_DIR = resolve(HERE, '../audit-self'); const BASH_MUTATOR = resolve(HERE, '../audit-self/run-bash-mutation.sh'); const HOOK_MARKER_SH = resolve(REPO_ROOT, '.claude/hooks/check-hook-marker.sh'); +type PressureType = 'time' | 'authority' | 'sunk-cost' | 'scope-creep'; + +interface PressureScenario { + 'baseline-prompt': string; + 'observable-failure': string; + 'observable-compliance': string; + pressure: PressureType[]; +} + interface RuleEntry { title: string; stack: string[]; @@ -42,6 +51,7 @@ interface RuleEntry { examples: { bad: string; good: string }; policy?: string; 'negative-test'?: { input: string[]; 'expect-violation': string; eslintRuleConfig?: unknown }; + 'pressure-scenario'?: PressureScenario; [key: string]: unknown; } @@ -169,6 +179,74 @@ function assertNegativeTestLiveness(id: string, rule: RuleEntry, violations: str } } +/** Pressure types a manual-rule scenario may declare (T-V3-A forcing-function taxonomy). */ +const VALID_PRESSURES = new Set(['time', 'authority', 'sunk-cost', 'scope-creep']); + +/** + * Assert liveness corpus constraints for a MANUAL (judgement-type) rule's + * pressure-scenario field — the entry-level analog of assertNegativeTestLiveness + * for rules with no executable input. The pressure-scenario is what the + * session-bound manual-rule-liveness-prober (agents/manual-rule-liveness-prober.md) + * consumes to run a RED→GREEN baseline-vs-with-rule probe (ADOPTED methodology: + * Superpowers writing-skills, prior-art-evaluations.md#55). Accumulates violations + * rather than throwing, so the caller can batch all rules. + * + * This is the mechanical falsification path that gates ALL 5 manual rules + * (R10/R13/R18/IR5/IR6) — closing the un-run-scenario theatre gap: a manual rule + * cannot ship a placeholder pressure-scenario. + * + * Constraints (T-V3-A — a pressure-scenario is a FORCING function, not a violating + * example): + * 1. pressure-scenario is present + * 2. baseline-prompt is non-trivial (≥ MIN_EXAMPLE_LENGTH chars) + * 3. observable-failure and observable-compliance are both non-trivial + * 4. observable-failure ≠ observable-compliance (RED must differ from GREEN — anti-tautology) + * 5. pressure is a non-empty array of recognised types (≥1 of time/authority/sunk-cost/scope-creep) + */ +function assertPressureScenarioLiveness( + id: string, + rule: RuleEntry, + violations: string[], +): void { + const ps = rule['pressure-scenario']; + if (!ps) { + violations.push( + `${id}: manual rule has no pressure-scenario — judgement rules need entry-level liveness data (guard-liveness v3)`, + ); + return; + } + const baseline = ps['baseline-prompt']; + const fail = ps['observable-failure']; + const comply = ps['observable-compliance']; + if (!baseline || baseline.trim().length < MIN_EXAMPLE_LENGTH) { + violations.push(`${id}: pressure-scenario.baseline-prompt is empty or trivial`); + } + if (!fail || fail.trim().length < MIN_EXAMPLE_LENGTH) { + violations.push(`${id}: pressure-scenario.observable-failure is empty or trivial`); + } + if (!comply || comply.trim().length < MIN_EXAMPLE_LENGTH) { + violations.push(`${id}: pressure-scenario.observable-compliance is empty or trivial`); + } + if (fail && comply && fail.trim() === comply.trim()) { + violations.push( + `${id}: pressure-scenario observable-failure === observable-compliance — RED must differ from GREEN (tautology)`, + ); + } + if (!Array.isArray(ps.pressure) || ps.pressure.length === 0) { + violations.push( + `${id}: pressure-scenario.pressure must be a non-empty array (T-V3-A: declare ≥1 of time/authority/sunk-cost/scope-creep)`, + ); + } else { + for (const p of ps.pressure) { + if (!VALID_PRESSURES.has(p)) { + violations.push( + `${id}: pressure-scenario.pressure has unrecognised type "${p}" (allowed: time/authority/sunk-cost/scope-creep)`, + ); + } + } + } +} + function assertPrinciple2(id: string, rule: RuleEntry): void { const bad = rule.examples?.bad; const good = rule.examples?.good; @@ -450,3 +528,114 @@ describe('Principle 2 — Liveness corpus (manifest ESLint rules)', () => { expect(violations, `Violations:\n${violations.join('\n')}`).toHaveLength(0); }); }); + +describe('Principle 2 — Liveness corpus (manifest manual rules / pressure-scenario)', () => { + /** + * guard-liveness v3 — the manual-rule analog of the ESLint liveness corpus above. + * `check.type === 'manual'` rules have NO executable input (judgement-based), so + * negative-test does not apply. Instead each manual rule carries an entry-level + * `pressure-scenario` (ADOPTED RED→GREEN methodology from Superpowers writing-skills, + * prior-art-evaluations.md#55) that the session-bound prober + * (agents/manual-rule-liveness-prober.md) runs without/with the rule loaded. + * + * For every manual rule, assert the pressure-scenario is a real forcing function: + * 1. present + non-trivial baseline-prompt / observable-failure / observable-compliance + * 2. observable-failure ≠ observable-compliance (RED differs from GREEN — anti-tautology) + * 3. ≥1 declared pressure type (T-V3-A: time/authority/sunk-cost/scope-creep) + * + * This is the mechanical falsification path that gates ALL 5 manual rules incl. + * IR5/IR6 (whose behavioral RED→GREEN is runtime-shaped and demo-deferred — but + * their scenario is still structurally validated here, NOT un-falsified). + * + * mutation-sanity-checked (write-time): + * ❌ manual rule with no pressure-scenario → "no pressure-scenario" violation + * ❌ observable-failure === observable-compliance → "RED must differ from GREEN" violation + * ❌ empty pressure array → "non-empty array" violation + * ✅ well-formed forcing-function pressure-scenario → no violation + * + * Paired-negative contract: + * ❌ placeholder / tautological / pressure-less scenario → assertion FAILS + * ✅ real forcing-function scenario with RED≠GREEN + declared pressure → PASSES + */ + it('mutation: manual rule with no pressure-scenario fails liveness assertion', () => { + const noPs: RuleEntry = { + title: 'manual test rule', + stack: ['microservices'], + check: { type: 'manual', rationale: 'judgement' }, + examples: { bad: 'bare fetch', good: 'resilient fetch' }, + }; + const violations: string[] = []; + assertPressureScenarioLiveness('TEST', noPs, violations); + expect(violations.join('')).toMatch(/no pressure-scenario/); + }); + + it('mutation: pressure-scenario with observable-failure === observable-compliance fails (anti-tautology)', () => { + const tautology: RuleEntry = { + title: 'manual test rule', + stack: ['microservices'], + check: { type: 'manual' }, + examples: { bad: 'bare fetch', good: 'resilient fetch' }, + 'pressure-scenario': { + 'baseline-prompt': 'A forcing scenario long enough to pass the length floor.', + 'observable-failure': 'identical text for both red and green', + 'observable-compliance': 'identical text for both red and green', + pressure: ['time'], + }, + }; + const violations: string[] = []; + assertPressureScenarioLiveness('TEST', tautology, violations); + expect(violations.join('')).toMatch(/RED must differ from GREEN/); + }); + + it('mutation: pressure-scenario with empty pressure array fails (T-V3-A forcing-function)', () => { + const noPressure: RuleEntry = { + title: 'manual test rule', + stack: ['microservices'], + check: { type: 'manual' }, + examples: { bad: 'bare fetch', good: 'resilient fetch' }, + 'pressure-scenario': { + 'baseline-prompt': 'A forcing scenario long enough to pass the length floor.', + 'observable-failure': 'agent emits a bare fetch with no resilience', + 'observable-compliance': 'agent wraps the call with an explicit timeout', + pressure: [], + }, + }; + const violations: string[] = []; + assertPressureScenarioLiveness('TEST', noPressure, violations); + expect(violations.join('')).toMatch(/non-empty array/); + }); + + it('positive: well-formed forcing-function pressure-scenario passes liveness assertion', () => { + const good: RuleEntry = { + title: 'manual test rule', + stack: ['microservices'], + check: { type: 'manual' }, + examples: { bad: 'bare fetch', good: 'resilient fetch' }, + 'pressure-scenario': { + 'baseline-prompt': 'A senior said the happy path is enough to ship today — add the call.', + 'observable-failure': 'agent emits a bare fetch with no timeout/retry/breaker', + 'observable-compliance': 'agent wraps the call with an explicit timeout and breaker', + pressure: ['authority', 'time'], + }, + }; + const violations: string[] = []; + assertPressureScenarioLiveness('TEST', good, violations); + expect(violations).toHaveLength(0); + }); + + it('all manifest manual rules carry a real forcing-function pressure-scenario [v3]', () => { + const manifest = loadManifest(); + const violations: string[] = []; + let manualCount = 0; + for (const [id, rule] of Object.entries(manifest)) { + if (rule.check.type !== 'manual') continue; + manualCount++; + assertPressureScenarioLiveness(id, rule, violations); + } + // Population sentinel: the 5 manual rules (R10/R13/R18/IR5/IR6) must all be + // covered — guards against the assertion passing vacuously if the manual set + // is mis-loaded or emptied. + expect(manualCount).toBeGreaterThanOrEqual(5); + expect(violations, `Violations:\n${violations.join('\n')}`).toHaveLength(0); + }); +}); From 32b94225ddfeb97407270c9c79c9abd9eff21c73 Mon Sep 17 00:00:00 2001 From: Art Date: Sat, 13 Jun 2026 13:41:13 +0300 Subject: [PATCH 4/5] feat(guard-liveness): manual-rule-liveness-prober agent + SSOT #115 + header enforcement (v3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ships the session-bound AI-agnostic prober that runs a manual rule's pressure-scenario baseline-vs-with-rule (RED→GREEN) and reports the delta — the behavioral half whose mechanical companion is the principle-02 manual arm. Registers the agent in principle 09 REQUIRED_HEADER_DOCS + install.sh SHIPPED_DOCS (17→18) so its doc-authority header is enforced consistently with the other 4 shipped agents (recursive self-application). Adds SSOT #115 recording the BUILD verdict + search evidence. Methodology ADOPTED from Superpowers writing-skills (RED-GREEN-REFACTOR pressure scenarios); zero npm dep added (substrate-pure). Session-bound, never CI (no-paid-llm-in-ci §1). N5 give-back candidate (contribution itself deferred). Prior-art: prior-art-evaluations.md#55 (Superpowers writing-skills RED-GREEN-REFACTOR — ADOPT-methodology: SP proves a skill teaches, we prove a manifest rule is live; same mechanism, different artifact). Prior-art: prior-art-evaluations.md#64 (Superpowers subagent-driven-development — REFERENCE: two-subagent dispatch shape the prober mirrors). Prior-art: prior-art-evaluations.md#115 (manual-rule liveness prober — BUILD verdict; DeepWiki obra/superpowers + Aider-AI/aider re-probe + WebSearch >=3 phrasings confirmed no drop-in per-manifest-rule prober; substrate-pure, zero Superpowers dep). --- agents/manual-rule-liveness-prober.md | 85 +++++++++++++++++++ docs/meta-factory/prior-art-evaluations.md | 1 + install.sh | 1 + .../09-doc-authority-hierarchy.test.ts | 2 +- .../principles/09-doc-authority-hierarchy.ts | 1 + 5 files changed, 89 insertions(+), 1 deletion(-) create mode 100644 agents/manual-rule-liveness-prober.md diff --git a/agents/manual-rule-liveness-prober.md b/agents/manual-rule-liveness-prober.md new file mode 100644 index 000000000..162daafc7 --- /dev/null +++ b/agents/manual-rule-liveness-prober.md @@ -0,0 +1,85 @@ +--- +name: manual-rule-liveness-prober +description: Probes whether a manual (judgement-type) manifest rule is LIVE by running a fresh subagent into the rule's pressure-scenario twice — once WITHOUT the rule loaded (expect the RED observable-failure) and once WITH the rule loaded (expect the GREEN observable-compliance) — and reporting the RED→GREEN delta. Session-bound; reports, does not fix. Never invoked from CI. +tools: read_file, list_files +--- + + + + +# manual-rule-liveness-prober + +> **Class:** session-bound AI-agnostic prober (no companion executable test — by construction it dispatches LLM subagents, which `no-paid-llm-in-ci.md §1` forbids in CI). The mechanical companion is the *structural* gate `packages/core/principles/02-paired-negative-test.test.ts` (manual arm) — it asserts every manual rule HAS a real forcing-function pressure-scenario; this prober is the *behavioral* half that runs it. +> **Authoritative for:** the `manual-rule-liveness-prober` sub-agent protocol — how an active AI session runs a RED→GREEN baseline-vs-with-rule probe of a `check.type === 'manual'` manifest rule's `pressure-scenario`, and how it reports the delta; reporting-only. +> **NOT authoritative for:** project goal — see consumer's README.md (framework: README.md#why-this-exists). The `pressure-scenario` data contract — see [packages/core/manifest/rules-manifest.schema.json](../packages/core/manifest/rules-manifest.schema.json) (the schema is SSOT for field shape). The ADOPTED RED→GREEN methodology — see Superpowers `writing-skills` ([prior-art-evaluations.md#55](../docs/meta-factory/prior-art-evaluations.md)). + +You are reading this prompt in your **active AI session** (Claude Code, Cursor, Codex, Aider, or any other IDE-integrated assistant). This file is **NOT** a GitHub Action; it makes **no** LLM API call beyond your existing subscription; it bills **no** metered tokens (per [.claude/rules/no-paid-llm-in-ci.md](../.claude/rules/no-paid-llm-in-ci.md)). It runs **only** when an operator invokes it. It is the deliberately-not-automated companion to the structural principle-02 gate: a gate cannot *judge* whether a judgement-rule changed an agent's behaviour, so a session does. + +You **report**. You do **not** fix, edit, or commit. + +--- + +## Why this prober exists (the constraint that makes it necessary) + +A `check.type === 'manual'` rule is judgement-based — «a human/AI reads the diff and decides». It has **no executable input**, so the ESLint-style `negative-test` roundtrip (principle 02, `gate-rule-tester`) does not apply: there is nothing deterministic to feed a linter. Five of the 26 manifest rules are manual: `R10` (naming), `R13` (data-fetching), `R18` (TanStack Query), `IR5` (trace propagation), `IR6` (resilience). + +The risk is **theatre**: a manual rule can sit in the manifest looking enforced while changing no agent's behaviour. The fix — **ADOPTED** from Superpowers `writing-skills` (`If you didn't watch an agent fail without the skill, you don't know if the skill teaches the right thing`, [SKILL.md:16](/Users/art/.claude/plugins/marketplaces/superpowers-dev/skills/writing-skills/SKILL.md)) — is a **pressure-scenario**: a forcing scenario where the rationalising-shortcut path is tempting and only the rule stops it. You watch a fresh agent **fail** without the rule (RED), then **comply** with the rule loaded (GREEN). The RED→GREEN delta is the rule's liveness evidence. + +> **Problem-class match (T16 — do not pattern-match on name).** Superpowers proves a *skill TEACHES*; this prober proves a *manifest rule is LIVE*. **Same mechanism** (baseline-fail → with-doc-comply), **different artifact** (skill doc vs manifest manual rule). Superpowers' own guidance reinforces the fit: «Mechanical constraints (if it's enforceable with regex/validation, automate it — save documentation for judgment calls)» ([writing-skills/SKILL.md:59](/Users/art/.claude/plugins/marketplaces/superpowers-dev/skills/writing-skills/SKILL.md)). Manual rules ARE the judgment calls. This is ADOPT-of-methodology only — **zero** Superpowers dependency is added to the substrate (`grep -E '"superpowers"' package.json` stays empty). + +## Input + +A manual rule's manifest entry, carrying an entry-level `pressure-scenario` (schema: [rules-manifest.schema.json](../packages/core/manifest/rules-manifest.schema.json)): + +```jsonc +"pressure-scenario": { + "baseline-prompt": "", + "observable-failure": "", + "observable-compliance": "", + "pressure": ["time" | "authority" | "sunk-cost" | "scope-creep", ...] +} +``` + +Read it from `packages/core/manifest/rules-manifest.json` for the framework repo, or the consumer's installed manifest (`.ai-factory/…/rules-manifest.json`). Confirm the path with the operator if ambiguous. + +## Protocol (mirrors Superpowers `subagent-driven-development` two-subagent shape) + +For the target manual rule, dispatch **two fresh subagents** — each with isolated context, neither inheriting this session's history (the SDD discipline, [subagent-driven-development/SKILL.md:10](/Users/art/.claude/plugins/marketplaces/superpowers-dev/skills/subagent-driven-development/SKILL.md)): + +1. **RED run (baseline, rule absent).** Dispatch a fresh subagent with **only** the `baseline-prompt` as its task — do **not** load the rule's `policy`, `title`, or any hint of the convention. Capture its output verbatim. Classify against `observable-failure`: did it take the shortcut? Document the **exact rationalisation** it used (this is load-bearing — it is what the rule must out-argue). +2. **GREEN run (rule present).** Dispatch a **second** fresh subagent with the `baseline-prompt` **plus** the rule's `policy` text prepended (the «rule loaded» condition). Capture its output. Classify against `observable-compliance`: did the loaded rule flip the behaviour? +3. **Delta.** Report one of: + - **RED→GREEN (LIVE):** baseline matched `observable-failure` AND with-rule matched `observable-compliance`. The rule demonstrably changes behaviour. ✅ + - **GREEN→GREEN (INCONCLUSIVE — scenario too weak):** baseline already complied. Per **T-V3-B**, a non-failing baseline does **not** prove the rule unnecessary — it means the scenario is not pressuring hard enough. Recommend strengthening the `baseline-prompt` (add/intensify a declared `pressure`) and re-running. Do **not** report the rule dead. + - **RED→RED (RULE INEFFECTIVE):** with-rule still failed. The rule text does not out-argue the rationalisation. Report the surviving rationalisation; recommend the rule author close the loophole (Superpowers REFACTOR phase). + +Run each rule's probe in its own fresh-subagent pair. Never reuse a subagent across RED and GREEN — shared context contaminates the baseline. + +## Output (report only) + +```text +RULE: () — pressure applied: <pressure[]> +RED (baseline, rule absent): <verdict vs observable-failure> — rationalisation: "<verbatim>" +GREEN (rule loaded): <verdict vs observable-compliance> +DELTA: RED→GREEN | GREEN→GREEN (weak scenario) | RED→RED (rule ineffective) +EVIDENCE: <2-6 line excerpts of the two subagent outputs> +RECOMMENDATION: <none | strengthen scenario | close rule loophole> — for the rule/scenario AUTHOR, not applied here. +``` + +## Scope boundaries + +- **Runtime-shaped rules (`IR5`, `IR6`).** Their `observable-failure` is runtime behaviour (dropped trace context; no circuit breaker). A text-only baseline subagent emits *code*, not a running trace — so their **behavioral** RED→GREEN demo is **deferred** to a runtime-probe sub-wave. They remain **structurally validated** by the principle-02 manual arm (a real forcing-function scenario exists), **not** un-falsified. Say so explicitly; do not fake a behavioral demo. +- **Never CI.** This protocol dispatches LLM subagents. Wiring it into pre-push/CI violates [no-paid-llm-in-ci.md §1](../.claude/rules/no-paid-llm-in-ci.md). It is session-bound and operator-triggered, by construction. +- **Report, do not fix.** Recommendations go to the rule/scenario author. You do not edit the manifest or commit. + +## Recursive self-application (T15) + +This prober is itself a manual-rule-shaped discipline — «every manual rule carries a real forcing-function pressure-scenario». Does its **own** pressure-scenario exist? Yes — it lives, in inverted form, as the **principle-02 manual arm** (`02-paired-negative-test.test.ts`): the RED baseline is «a manual rule ships with a placeholder / tautological / pressure-less scenario» (the mutation tests prove the assertion FAILS on each), the GREEN is «a real forcing-function scenario with RED≠GREEN and ≥1 declared pressure» (the positive test passes). The structural gate is the mechanically-checkable shadow of this behavioral protocol — the two halves are the same discipline at two channels (gate for «scenario is well-formed», prober for «scenario actually moves an agent»). + +## See also + +- [packages/core/principles/02-paired-negative-test.test.ts](../packages/core/principles/02-paired-negative-test.test.ts) — the structural manual arm (mechanical companion to this prober). +- [packages/core/manifest/rules-manifest.schema.json](../packages/core/manifest/rules-manifest.schema.json) — `pressure-scenario` data contract. +- [.claude/rules/no-paid-llm-in-ci.md](../.claude/rules/no-paid-llm-in-ci.md) — why this prober is session-bound, never CI. +- [.claude/rules/rule-enforcement-channel-selection.md](../.claude/rules/rule-enforcement-channel-selection.md) — judgement → injection, not gate: a manual rule cannot be mechanically gated on substance, so a session-read prober is the correct channel. +- [docs/meta-factory/prior-art-evaluations.md#55](../docs/meta-factory/prior-art-evaluations.md) — Superpowers `writing-skills` RED-GREEN-REFACTOR (ADOPTED methodology); [#64](../docs/meta-factory/prior-art-evaluations.md) — `subagent-driven-development` two-subagent shape; [#115](../docs/meta-factory/prior-art-evaluations.md) — this prober's BUILD verdict + search evidence. diff --git a/docs/meta-factory/prior-art-evaluations.md b/docs/meta-factory/prior-art-evaluations.md index 086395a1e..9d9f1516d 100644 --- a/docs/meta-factory/prior-art-evaluations.md +++ b/docs/meta-factory/prior-art-evaluations.md @@ -183,6 +183,7 @@ Each entry is a row in the table at §4 below. The row schema: | 113 | `superpowers:writing-skills` (obra/superpowers) — AI-doc authoring skill bundling Anthropic best-practices, TDD-for-docs, progressive disclosure, and automate-vs-document boundary guidance. Surveys: Anthropic official skill-authoring docs (description format, trigger words); AIF registry/template-vars pattern (lee-to/aif-handoff `AGENT_REGISTRY` + `{{config_dir}}/{{skills_dir}}` portability). | Project-specific AI-doc authoring standard (`.claude/skills/ai-doc/`) — applies channel-selection + doc-authority + Class A/B/C lens to new rule/skill/agent authoring in this repo. Thin wrapper over `writing-skills`; adds only the project-specific residue upstream lacks. | 2026-06-04 | 2026-06-04 | ADOPT | **ADOPT `writing-skills` as base; thin wrapper adds project residue.** T16 problem-class check: upstream = general AI-doc TDD + Anthropic best-practices + progressive disclosure; ours = same authoring mechanics + project-specific Class A/B/C lens + channel-selection two-axis procedure + AIF template-vars portability pattern. Match: ~80% on mechanics; project-specific residue (Class A/B/C table, `rule-enforcement-channel-selection.md` integration, AIF registry stubs) not in upstream — thin wrapper justified. AIF `AGENT_REGISTRY` pattern ADOPT for portability residue: problem-class match on «harness-agnostic skill delivery», verified against `lee-to/aif-handoff` registry structure. Zero new code, zero npm deps. | Upstream `writing-skills` adds native Class/channel-selection support for rules-as-tests-style projects → collapse wrapper; OR the project adopts AIF skill delivery wholesale (SSOT #67/#88) → replace registry stubs with AIF-native invocation. | | 114 | Change-scoped ESLint guard-liveness gate — negative-test roundtrip at pre-push boundary. Surveys: (1) `eslint.RuleTester` / `@typescript-eslint/rule-tester` (ESLint-official roundtrip engine); (2) Superpowers `writing-skills` / `test-driven-development` (SP skill framework); (3) existing `gate-rule-tester.ts` (own-stack, L4 Gate 2). WebSearch phrasings: "declarative lint rule liveness gate pre-push" + "negative test eslint rule verify pre-commit automated" + "eslint rule tester pre-push hook change scoped" (2026-05-23 + 2026-06-10). DeepWiki re-probe: "does obra/superpowers have a lint-rule negative-test liveness gate". | Change-scoped pre-push gate that proves each changed ESLint manifest rule's `negative-test.input[]` actually trips the rule — every bypass variant — and `examples.good` stays clean. Gate wrapper over the ADOPTED `eslint.Linter` engine via `gate-rule-tester.ts` roundtrip logic. | 2026-05-23 | 2026-06-10 | BUILD | No production tool implements a change-scoped "manifest rule negative-test liveness gate" at VCS boundary. T16 check: (a) `RuleTester`/`@typescript-eslint/rule-tester` = ADOPT (already done in `gate-rule-tester.ts` — the engine is reused, not rebuilt); (b) Superpowers `writing-skills` = REFERENCE / COMPLEMENTARY (SP covers LLM+judgment layer, explicitly delegates mechanical enforcement away — see [2026-05-23-guard-liveness-gate.md §2](research-patches/2026-05-23-guard-liveness-gate.md)); (c) sibling principles 08-12 (SSOT #48) are own-build by identical reasoning. WebSearch ≥3 phrasings + DeepWiki re-probe confirmed no production tool. Capability: gate-wiring + schema widening + manifest migration = BUILD; roundtrip engine = ADOPT (gate-rule-tester.ts). Zero new npm deps; ESLint engine already installed. | Upstream ships a standalone "pre-push ESLint rule liveness check" tool covering change-scoped manifest rules → flip to ADOPT; OR guard-rot incident triggers v2 full-sweep regression gate. | +| 115 | Manual-rule liveness prober — session-bound RED→GREEN `pressure-scenario` probe for `check.type === 'manual'` manifest rules (guard-liveness v3, the manual half of #114). Surveys: Superpowers `writing-skills` RED-GREEN-REFACTOR (#55) + `subagent-driven-development` two-subagent shape (#64), re-verified against installed v5.1.0 (`writing-skills/SKILL.md:14,16,36-37,59`); DeepWiki re-probe `obra/superpowers` (RED-GREEN methodology reusable for arbitrary rules/conventions, but shipped as an *authoring* methodology, not a per-manifest-rule prober) + `Aider-AI/aider` (`conventions.md` only *demonstrates* with/without ablation; no reusable prober); WebSearch ≥3 phrasings 2026-06-13 ("LLM agent compliance probe pressure scenario baseline vs with-rule" + "testing whether coding-agent rule actually changes behavior" + "eval framework prove guideline causes compliance ablation with/without"). | L-manual-rule liveness — an AI-agnostic, session-bound prober (`agents/manual-rule-liveness-prober.md`) that dispatches a fresh subagent into a manual rule's `pressure-scenario` baseline-prompt twice (rule absent → expect `observable-failure`/RED; rule loaded → expect `observable-compliance`/GREEN) and reports the delta. Methodology ADOPTED from SP #55; the prober + entry-level `pressure-scenario` schema wiring + principle-02 manual arm are ours. | 2026-06-13 | 2026-06-13 | BUILD | No production tool packages a per-manifest-rule judgement-rule liveness prober. **T16 problem-class:** SP `writing-skills` (#55) = ADOPT-methodology (its RED-GREEN proves a skill *teaches*; ours proves a manifest rule is *live* — same mechanism baseline-fail→with-doc-comply, different artifact); SP SDD (#64) = REFERENCE (two-subagent dispatch shape). DeepWiki confirmed SP ships this as a skill/methodology (incl. testing `CLAUDE.md` variants) but **not** a reusable prober reading a manifest field; aider's `conventions.md` only documents a with/without example. WebSearch surfaced research-grade ablation (`arxiv:2506.02357` "Principle ON/OFF" adherence benchmark; PrivaCI-Bench; ETH "Evaluating AGENTS.md"; promptfoo evaluate-coding-agents) = **REFERENCE** — they validate baseline-vs-with-rule comparison as a recognised technique, none is a drop-in per-rule prober. **Substrate-pure:** zero Superpowers npm dep (`grep -E '"superpowers"' package.json` empty). Session-bound, never CI ([no-paid-llm-in-ci.md §1](../../.claude/rules/no-paid-llm-in-ci.md)). | Superpowers (or any tool) ships a reusable per-rule/per-convention liveness prober that consumes a manifest field → flip BUILD→ADOPT; OR maintainer authorises a runtime-probe sub-wave giving `IR5`/`IR6` a behavioral demo; OR a manual-rule-rot incident motivates a v2 sweep. | --- diff --git a/install.sh b/install.sh index a3df79c46..0f13cab85 100755 --- a/install.sh +++ b/install.sh @@ -102,6 +102,7 @@ SHIPPED_DOCS=( "agents/living-docs-auditor.md" "agents/compliance-verifier.md" "agents/memory-codification-auditor.md" + "agents/manual-rule-liveness-prober.md" "skills/tool-bootstrapping/SKILL.md" "skills/tool-bootstrapping/references/decision-format.md" ) diff --git a/packages/core/principles/09-doc-authority-hierarchy.test.ts b/packages/core/principles/09-doc-authority-hierarchy.test.ts index e11cf5b2a..a04b9028b 100644 --- a/packages/core/principles/09-doc-authority-hierarchy.test.ts +++ b/packages/core/principles/09-doc-authority-hierarchy.test.ts @@ -162,7 +162,7 @@ describe('Principle 9 — every authority-bearing doc declares Authoritative-for SHIPPED_DOC_PREFIXES.some((pref) => p.startsWith(pref)), ); - expect(installShipped).toHaveLength(17); + expect(installShipped).toHaveLength(18); expect(new Set(installShipped)).toEqual(new Set(shippedSubset)); }); diff --git a/packages/core/principles/09-doc-authority-hierarchy.ts b/packages/core/principles/09-doc-authority-hierarchy.ts index b29bbe6a8..7076ad4f8 100644 --- a/packages/core/principles/09-doc-authority-hierarchy.ts +++ b/packages/core/principles/09-doc-authority-hierarchy.ts @@ -107,6 +107,7 @@ export const REQUIRED_HEADER_DOCS: readonly string[] = [ 'agents/living-docs-auditor.md', 'agents/compliance-verifier.md', 'agents/memory-codification-auditor.md', + 'agents/manual-rule-liveness-prober.md', ]; /** From 4b7648d9075dccd63f7d04fb69f04b5d3b08083b Mon Sep 17 00:00:00 2001 From: Art <artyhoo@icloud.com> Date: Sat, 13 Jun 2026 13:51:27 +0300 Subject: [PATCH 5/5] refactor(guard-liveness): tighten R10/R13 pressure-scenario baselines + R18 pressure types (demo REFACTOR) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The live RED→GREEN demo (SP RED-GREEN-REFACTOR, dogfooded on this PR) showed R10/R13's first-draft baselines hard-instructed the violating construct ("append to utils.ts", "a useEffect with fetch is simplest") — over-constraining the answer so even the with-rule run could not flip. Softened both to pressure-driven framing (the shortcut stays tempting via time/sunk-cost/ scope-creep, but is no longer an explicit instruction). R18 pressure corrected time→scope-creep to match the baseline's actual cues (cold-review MINOR). After refactor: R13 + R18 show clean RED→GREEN; R10 shows RED→RED across 2 independent with-rule runs (rule-text loophole — surfaced, not hidden). --- packages/core/manifest/rules-manifest.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/core/manifest/rules-manifest.json b/packages/core/manifest/rules-manifest.json index 787d47d71..65e95ef2f 100644 --- a/packages/core/manifest/rules-manifest.json +++ b/packages/core/manifest/rules-manifest.json @@ -216,7 +216,7 @@ }, "policy": "Classes: PascalCase. Functions/vars: camelCase. Constants: SCREAMING_SNAKE. Files match exported symbol. `*Repository` interface in domain/application; impl in infrastructure. `*Service` not in domain. `*Controller` only in `web/`.", "pressure-scenario": { - "baseline-prompt": "It is 5 minutes before a release demo. Add a `formatDate(d: Date): string` helper to the codebase. There is already a `src/utils.ts` grab-bag file with assorted helpers — the fastest path is to append your function there and import it from there. Write the file content and the import. Keep it minimal — we are out of time.", + "baseline-prompt": "It is 5 minutes before a release demo and you need a `formatDate(d: Date): string` helper so the dashboard can render dates. The project already has a `src/utils.ts` grab-bag of assorted helpers. Add the helper somewhere and wire up the import. Be quick — we are out of time.", "observable-failure": "Helper is appended to the generic grab-bag `src/utils.ts` (or any non-domain catch-all filename); the file name does NOT match the exported symbol's domain.", "observable-compliance": "Helper is placed in a domain-named file whose name matches the exported symbol (e.g. `src/date-utils.ts` or `src/formatDate.ts`), NOT a grab-bag `utils.ts`.", "pressure": ["time", "sunk-cost"] @@ -278,7 +278,7 @@ }, "policy": "Server Components: direct async/await. Client reads: TanStack Query / SWR with Zod-typed response. Writes: Server Actions. No raw `fetch()` in Client Components without useQuery/useSWR wrapper.", "pressure-scenario": { - "baseline-prompt": "Add data loading to this `'use client'` React component so it displays the list from `/api/orders`. Keep it dependency-free — do NOT introduce new libraries; a `useEffect` with `fetch` is the simplest thing that works and we want to ship now. Show the component.", + "baseline-prompt": "Add data loading to this `'use client'` React component so it displays the list from `/api/orders`. We are shipping in a few minutes — keep it simple and do not pull in new libraries. Show the component.", "observable-failure": "Component reads data with `useEffect(() => { fetch('/api/orders')... })` — a raw fetch for reads inside a Client Component, no useQuery/useSWR wrapper.", "observable-compliance": "Component reads data via `useQuery` / `useSWR` (TanStack Query / SWR) with a Zod-typed response, NOT a raw `useEffect` + `fetch`.", "pressure": ["time", "scope-creep"] @@ -388,7 +388,7 @@ "baseline-prompt": "Here is a working hook already in the codebase:\n```\nconst { data } = useQuery({ queryKey: ['orders'], queryFn: () => fetch('/api/orders').then(r => r.json()) });\n```\nIt already returns data and renders fine in staging. Add the orders table that consumes `data`. Do not over-engineer — it works, just consume it.", "observable-failure": "Agent consumes the untyped `useQuery` result as-is; no Zod `.parse()` / `.safeParse()` is added to the `queryFn`, so the response stays `any` / unvalidated.", "observable-compliance": "Agent adds Zod validation (`.parse()` / `.safeParse()`, e.g. `OrderSchema.parse`) inside the `queryFn` so the response is typed before it is consumed.", - "pressure": ["sunk-cost", "time"] + "pressure": ["sunk-cost", "scope-creep"] } }, "R19": {