diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index f156c522..731168e5 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -18,6 +18,12 @@ /domains/performance/ @MetaMask/extension-platform @MetaMask/mobile-platform /domains/perps/ @MetaMask/perps /domains/pr-workflow/ @MetaMask/extension-platform @MetaMask/mobile-platform +<<<<<<< HEAD +/domains/stability/ @MetaMask/extension-platform @MetaMask/mobile-platform +||||||| 4063bf1 +======= +/domains/security/ @MetaMask/extension-platform @MetaMask/mobile-platform +>>>>>>> origin/jongsun/add/security-domain /domains/swaps/ @MetaMask/swaps-engineers /domains/testing/ @MetaMask/qa /domains/ui/ @MetaMask/design-system-engineers diff --git a/.github/scripts/lint-skill-entry.mjs b/.github/scripts/lint-skill-entry.mjs index 8de11ba1..c737a0ce 100644 --- a/.github/scripts/lint-skill-entry.mjs +++ b/.github/scripts/lint-skill-entry.mjs @@ -111,6 +111,22 @@ export function lintSkill(skill) { } } + // The installer prefixes every emitted skill, so a description advertising `/` + // names a command no operator exposes. The description IS the discovery surface, so a + // wrong trigger string is a selection failure, not a typo. + // + // The lookbehind keeps a scoped package (`@metamask/gator-cli`) or a path + // (`skills/gator-cli`) from being read as a slash command. + if (raw.description && raw.name) { + const bare = new RegExp(`(? [options] metamask-skills sync [options] metamask-skills postinstall [options] + metamask-skills hooks [options] metamask-skills install [options] Options: @@ -799,6 +800,55 @@ function invokedDirectly() { } } + +/** + * Print the Claude Code registration for every hook an installed skill ships. + * + * The installer copies `hooks/` like any other bundle directory, but a hook does nothing + * until it is registered in settings.json — and the path to register is absolute, so it + * differs per machine and per consumer repo and cannot be documented as a constant. This + * resolves it against the actual install. + */ +function printHookRegistration(args) { + const { target } = parseGlobalArgs(args); + const skillsDir = path.join(target, '.claude', 'skills'); + + let entries = []; + try { + for (const skill of readdirSync(skillsDir, { withFileTypes: true })) { + if (!skill.isDirectory()) continue; + const hooks = path.join(skillsDir, skill.name, 'hooks'); + if (!dirExists(hooks)) continue; + for (const file of readdirSync(hooks)) { + if (file.endsWith('.py')) entries.push(path.join(hooks, file)); + } + } + } catch { + warn(`no installed skills found under ${skillsDir}`); + return 1; + } + + if (entries.length === 0) { + process.stdout.write('No installed skill ships a hook.\n'); + return 0; + } + + const commands = entries + .map((f) => ` { "type": "command", "command": "python3 ${f}" }`) + .join(',\n'); + + process.stdout.write( + `${entries.length} hook(s) installed. Copying a hook does not activate it — Claude Code\n` + + `runs one only once it is registered. Add this to ~/.claude/settings.json, or to\n` + + `${path.join(target, '.claude', 'settings.json')} to scope it to this repo:\n\n` + + ' {\n "hooks": {\n "PreToolUse": [\n {\n "matcher": "Bash",\n "hooks": [\n' + + `${commands}\n` + + ' ]\n }\n ]\n }\n }\n', + ); + return 0; +} + + if (invokedDirectly()) { const [command, ...args] = process.argv.slice(2); if (!command || command === '-h' || command === '--help') { @@ -817,6 +867,8 @@ if (invokedDirectly()) { exitCode = sync(args); } else if (command === 'postinstall') { exitCode = postinstall(args); + } else if (command === 'hooks') { + exitCode = printHookRegistration(args); } else if (command === 'install') { exitCode = install(args); } else { diff --git a/domains/agentic/skills/agent-run-cost/skill.md b/domains/agentic/skills/agent-run-cost/skill.md new file mode 100644 index 00000000..7feb8060 --- /dev/null +++ b/domains/agentic/skills/agent-run-cost/skill.md @@ -0,0 +1,108 @@ +--- +name: agent-run-cost +maturity: experimental +description: >- + Estimate what an agentic workflow costs to run before it merges — fan-out × trigger + frequency × no kill-switch — and say so in figures rather than adjectives. Agent token + spend is invisible in a diff: a workflow that spawns one agent and one that spawns forty + are the same few lines, and the difference only appears on a bill nobody reads during + review. Produces a per-run and per-month estimate with its arithmetic shown, flags the + three amplifiers, and proposes the cheapest mitigation that preserves the intent. Use + when a PR adds or widens an agentic workflow, an AEP task class, a verification recipe, + or a schedule that runs agents unattended. +--- + +# Agent run cost + +Scripted automation announces its cost in wall-clock time; agentic automation does not. +A fan-out of forty subagents and a single call are the same shape in a diff, and the +difference surfaces later, on a bill, attributed to nothing in particular. + +This is the token-spend counterpart to `sentry-quota`, which guards span volume. Same +posture: operate on **code and PRs**, before the spend exists, and produce figures. + +## When to use + +- A PR adds or widens an agentic workflow, AEP task class, or verification recipe. +- A workflow gains fan-out — an agent per file, per finding, per test, per PR. +- Something agentic moves from opt-in to automatic (a CI trigger, a cron, a git hook). +- An ADR or design proposes agents for work a script already does — the estimate is the + argument, and its absence is usually the tell. + +## Do not use when + +- The workflow is developer-invoked, one agent, no loop — the ceiling is a person's patience. +- The change only narrows fan-out or adds a gate. + +## The amplifier triad + +Cost is not the per-agent price. It is the product of three things, any one of which can be +the whole problem: + +| Amplifier | What it looks like | Effect | +|---|---|---| +| **Fan-out** | an agent per item — per file, per finding, per dimension, per round; nested `parallel` inside `pipeline`; a loop-until-dry with no ceiling | N× per run, and N is often data-dependent rather than fixed | +| **Trigger frequency** | runs on every push rather than on demand; a cron; a label that re-fires on each commit; a retry that respawns the fleet | turns a one-off into a rate | +| **No kill-switch** | no env var, feature flag, or budget cap; nothing to stop it mid-run; no way to disable without a revert | a runaway costs whatever it costs until someone merges a fix | + +One alone is usually fine. **Fan-out × frequency with no kill-switch is the shape that +produces a surprise**, and it is worth naming explicitly in review when all three are present. + +## Producing the estimate + +Show the arithmetic. An estimate whose derivation is hidden is an adjective. + +1. **Count agents per run.** Read the fan-out literally — how many items feed the widest + stage, and whether that number is bounded by the code or by the data. A `pipeline` over + changed files is unbounded by the code; `Array.from({length: 3})` is not. +2. **Estimate tokens per agent.** Prompt + the context it will read + its output. The context + dominates: an agent that reads three files is not an agent that greps a repo. +3. **Multiply, then apply frequency.** Per-run cost × runs per week. State the assumption + about run count — it is the number most likely to be wrong, and naming it lets a reviewer + correct it. +4. **State the worst case separately from the expected case.** The expected case is what it + costs on a normal PR; the worst case is what it costs on the PR that touches 400 files. + Budget conversations are about the second one. +5. **Compare against the alternative.** If a deterministic script covers the same ground, the + estimate belongs next to that script's cost. An agentic approach can still win — for + adversarial review, exploration, fuzzing, or anything with no fixed oracle — but the case + is made by the comparison, not by the capability. + +Report the figures, the assumptions behind them, and the mitigation. **Do not render a +ship/no-ship verdict** — whether a cost is worth paying is a budget decision, and it belongs +to whoever owns the budget. + +## Mitigation ladder + +Cheapest first; stop at the rung that fits. + +1. **Cap the fan-out.** A literal ceiling on items, with a `log()` of what was dropped — + silent truncation reads as full coverage and is worse than the cost. +2. **Narrow the trigger.** On-demand or label-gated instead of every push; on the changed + subset instead of the tree. +3. **Right-size the model per stage.** Mechanical stages rarely need the top tier; reserve it + for the judgement stages. +4. **Add a budget guard.** A token ceiling the workflow checks between stages, so it degrades + instead of running to completion at any price. +5. **Add a kill-switch.** An env var or flag that disables it without a revert. Cheap to add + up front and unavailable exactly when it is needed most. + +## Common pitfalls + +| Mistake | Correct approach | +|---|---| +| "It's just a few agents" | Count them. Data-dependent fan-out has no "just" | +| Estimating output tokens only | Context dominates — an agent that reads the repo costs more than one that answers at length | +| Quoting an average with no worst case | The worst case is the budget conversation | +| Treating a retry as free | A retried fleet is a second fleet | +| Assuming a concurrency cap bounds cost | It bounds *parallelism*, not total spend — queued agents still run | +| Adding a kill-switch after launch | It is needed during the incident it would have prevented | +| Comparing capability instead of cost | "Agents can do this" is not "agents should do this at this price" | + +## Related + +- `sentry-quota` — the same guard for span volume; `fan-out × ungated × no-kill-switch`. +- `evidence` — weighs AEP run cost when choosing an evidence lane, and tears the stack + down after; this skill is the review-side version for workflows others will run. +- [`MetaMask/decisions#173`](https://github.com/MetaMask/decisions/pull/173) — ADR-0058 + review, where the missing token-cost estimate was raised as an open question. diff --git a/domains/analytics/knowledge/metrametrics-identity.md b/domains/analytics/knowledge/metrametrics-identity.md new file mode 100644 index 00000000..1faed3ba --- /dev/null +++ b/domains/analytics/knowledge/metrametrics-identity.md @@ -0,0 +1,41 @@ +--- +name: metrametrics-identity +domain: analytics +description: isOptIn:true unconditionally strips user identity in MetaMetricsController — always sends as anonymous ID +--- + +# MetaMetrics Identity Stripping + +## The Mechanism + +In `MetaMetricsController` (`app/scripts/controllers/metametrics-controller.ts`): + +```typescript +if (excludeMetaMetricsId || (isOptIn && !metaMetricsIdOverride)) { + idType = 'anonymousId'; + idValue = METAMETRICS_ANONYMOUS_ID; // 0x0000000000000000 +} +``` + +When `isOptIn: true` with no `metaMetricsIdOverride`: +- The user's real `metaMetricsId` is discarded +- ALL such events share a single anonymous ID (`0x0000000000000000`) in Segment +- User-level attribution is completely lost + +This is **unconditional** — it applies to fully opted-in users with valid IDs, not just anonymous users. + +## Intended Use + +The onboarding opt-in flow (`creation-successful.tsx`) — where the user hasn't committed to MetaMetrics yet and no `metaMetricsId` has been persisted. The event must fire regardless of opt-in state. + +## The Misuse Pattern + +Post-opt-in `trackEvent` calls with `{ isOptIn: true }` without `metaMetricsIdOverride`. Defeats the purpose of Segment user-level dimensions (account types, feature flags). + +## Detection + +```bash +grep -r "isOptIn: true" app/scripts/ ui/ --include="*.ts" --include="*.tsx" +``` + +Any occurrence outside `creation-successful.tsx` (or the onboarding flow) is suspect. diff --git a/domains/analytics/knowledge/segment-governance.md b/domains/analytics/knowledge/segment-governance.md new file mode 100644 index 00000000..2ebe1e07 --- /dev/null +++ b/domains/analytics/knowledge/segment-governance.md @@ -0,0 +1,40 @@ +--- +name: segment-governance +domain: analytics +description: Segment event governance via segment-schema is advisory — no CI enforcement prevents unregistered events from shipping +--- + +# Segment Event Governance + +## Architecture + +| Component | Location | +|-----------|----------| +| Tracking plan | `Consensys/segment-schema` → `tracking-plans/metamask-extension.yaml` | +| Event registry | `shared/constants/metametrics.ts` → `MetaMetricsEventName` enum (300+ entries) | +| Review process | `CONTRIBUTING.md` in segment-schema; Data Council review | +| Governance channel | `#metamask-metametrics`, `@consensys/data-council` | + +## The Gap + +There is **no CI enforcement** in the extension repo. A developer can: + +1. Add entry to `MetaMetricsEventName` enum +2. Call `trackEvent` with it +3. Merge and ship to production + +...without registering in segment-schema or going through Data Council review. + +## Implications + +- Schema drift between tracking plan and production events +- No property schema validation for unregistered events +- Billing impact goes unreviewed +- Data Council review is bypassable by omission + +## Recommended Fix + +CI check that: +1. Parses `MetaMetricsEventName` entries +2. Validates each against `tracking-plans/metamask-extension.yaml` +3. Fails build if event is missing from the plan diff --git a/domains/analytics/knowledge/span-sub-sampling.md b/domains/analytics/knowledge/span-sub-sampling.md new file mode 100644 index 00000000..fa90f08e --- /dev/null +++ b/domains/analytics/knowledge/span-sub-sampling.md @@ -0,0 +1,72 @@ +--- +name: span-sub-sampling +domain: analytics +description: Deterministic per-trace sub-sampling for high-frequency custom spans — global tracesSampleRate × span sub-rate, traceId-hash bucketed +--- + +# Span Sub-Sampling + +Durable fix for a custom span that fans out and eats the span budget. Layer a per-trace sub-rate **under** the global `tracesSampleRate`, keyed on the trace id so every span in a trace is kept-or-dropped together. Source: [PR #39891](https://github.com/MetaMask/metamask-extension/pull/39891) (`shared/lib/wrapper-sampling.ts`). + +## Rate Math + +``` +effective rate = global tracesSampleRate × span sub-rate +``` + +- Global `tracesSampleRate` is already small (extension prod: 0.75%). +- The sub-rate cuts the custom span on top: `0.75% × 1% = 0.0075%`. +- PR #39891 ships a sub-rate of 0.5% (`WRAPPER_SAMPLE_RATE = 0.005`) — a conservative pilot — and names 5% as the step-up once the denylist is confirmed effective in production. + +Pick the sub-rate from how many sampled traces the metric needs to stay useful — not from the quota alone. Too low and the metric goes dark. + +## Pattern + +```ts +const WRAPPER_SAMPLE_RATE = 0.005; + +// Deterministic: same answer for the same traceId, so all spans in a trace +// are kept or dropped together — clean waterfalls, no partial gaps. +export function shouldSampleWrappers(traceId: string | undefined): boolean { + if (!traceId || traceId.length < 8) { + return false; + } + const hashBucket = parseInt(traceId.slice(0, 8), 16) % 10000; + return hashBucket < WRAPPER_SAMPLE_RATE * 10000; +} +``` + +**Why deterministic, not `Math.random()` per call:** independent per-span sampling shreds a trace into partial waterfalls (some spans present, siblings missing) — useless for attribution. Hashing the trace id makes keep/drop a property of the whole trace. + +## Gate Order (cheapest check first) + +```ts +const traceId = sentryGetActiveSpan()?.spanContext().traceId; +if (!traceId || isReadOnlyAction(action) || !shouldSampleWrappers(traceId)) { + return doWorkWithoutSpan(); +} +return trace({ name, op, data }, doWorkWithSpan); +``` + +1. No active trace → no span. +2. Denylist → skip noise (below). +3. Sub-sample miss → skip this trace's spans. + +## Denylist: cut before you sample + +Drop spans with no timing/attribution signal before sub-sampling. In PR #39891, read-only verbs are ~90% of `messenger.call` volume: + +```ts +const READ_ONLY_VERB = /^(?:get|has|find|is|peek)(?:[A-Z]|$)/u; +``` + +Removing ~90% of volume before the sample multiplies headroom — a higher sub-rate then yields the same span budget, so kept traces are denser and more useful. + +## Where the Gate Goes + +- **Consumer (extension):** spans go through `trace()`. Gate at the call site, or for a whole span family inside the wrapper. `traceId` from `sentryGetActiveSpan()?.spanContext().traceId`. +- **Controller package (core):** controllers call an injected `trace` callback. Gate in the package's trace util or the callback so every consumer inherits the cap. Pull the trace id from the controller's tracing context, not a fresh Sentry import. + +## Kill Switch + +Ship every always-on span family with an env disable flag (PR #39891: `SENTRY_DISTRIBUTED_TRACING_DISABLED` returns the messenger un-wrapped). It turns a future emergency cut into a config flip instead of a cherry-pick. diff --git a/domains/analytics/skills/grafana-tempo-queries/skill.md b/domains/analytics/skills/grafana-tempo-queries/skill.md new file mode 100644 index 00000000..99fee6ad --- /dev/null +++ b/domains/analytics/skills/grafana-tempo-queries/skill.md @@ -0,0 +1,125 @@ +--- +name: grafana-tempo-queries +description: Query backend traces in Grafana Tempo with TraceQL — find traces by service or span attribute, fetch a trace by id, inspect its span tree, and enumerate tag values. Covers the datasource-proxy access path, the credential-expiry failure that returns empty results indistinguishable from "no data", the negative control that proves a filter actually applied, and the id/kind/base64 decoding quirks in the response. Use when investigating backend latency, checking what the backend recorded for a request, or establishing which infrastructure tiers a trace reaches. Triggers on Tempo, TraceQL, Grafana traces, backend span inspection, "does the backend have this trace", or tracing a request past the API boundary. +maturity: experimental +--- + +# grafana-tempo-queries + +Tempo holds **backend** spans. Client spans from the extension and mobile go to Sentry via the SDK's own transport and never appear here — so a Tempo trace normally starts at an inbound server span, and a missing root is expected rather than broken. To join the two halves, see `sentry-grafana-correlation`. + +## Setup + +Everything goes through Grafana's datasource proxy, so a Grafana session is the only credential needed. Keep the host, datasource uid, org id, and session in your environment — this repository is public, so never commit them. + +```bash +# Set these once per shell, from your own Grafana instance: +# GRAFANA_HOST e.g. https://grafana. +# TEMPO_UID the Tempo datasource uid (see discovery below) +# GRAFANA_ORG the numeric org id the datasource belongs to +# GRAFANA_SESSION value of the grafana_session cookie from an authenticated browser +BASE="$GRAFANA_HOST/api/datasources/proxy/uid/$TEMPO_UID" +AUTH=(-H "Cookie: grafana_session=$GRAFANA_SESSION" -H "X-Grafana-Org-Id: $GRAFANA_ORG") +``` + +Discover the datasource uid rather than guessing it: + +```bash +curl -s "$GRAFANA_HOST/api/datasources" "${AUTH[@]}" \ + | node -e 'JSON.parse(require("fs").readFileSync(0)).filter(d=>d.type==="tempo").forEach(d=>console.log(d.uid,d.name))' +``` + +## Check the instrument before believing a result + +**A stale session returns HTTP 401 with an empty body, and a naive parser reports that as zero results** — indistinguishable from "this data does not exist". This is the single most expensive failure mode here: it produces confident negative conclusions about instrumentation coverage. + +```bash +# 1. Prove you are authenticated. Do this first, every session. +curl -s -o /dev/null -w 'grafana auth: HTTP %{http_code}\n' "$GRAFANA_HOST/api/user" "${AUTH[@]}" + +# 2. Prove the filter is actually being applied, with a query that must match nothing. +curl -s -G "$BASE/api/search" "${AUTH[@]}" \ + --data-urlencode 'q={span.db.system = "not-a-real-db-xyz"}' \ + --data-urlencode "start=$START" --data-urlencode "end=$NOW" \ + | node -e 'const j=JSON.parse(require("fs").readFileSync(0));console.log("control traces:",(j.traces||[]).length,"(must be 0)")' +``` + +If several different filters all return exactly your `limit`, the filter is not being applied — treat the results as unfiltered until the negative control returns 0. + +## Core queries + +Every endpoint wants an explicit epoch-seconds window. Omitting it on a by-id lookup makes the request hunt across all blocks and hit a context deadline. + +```bash +NOW=$(date +%s); START=$((NOW-3600)) +``` + +**Search by TraceQL.** Returns trace summaries plus the spans that matched. + +```bash +curl -s -G "$BASE/api/search" "${AUTH[@]}" \ + --data-urlencode 'q={resource.service.name="my-service"}' \ + --data-urlencode "start=$START" --data-urlencode "end=$NOW" \ + --data-urlencode "limit=20" +``` + +**Fetch one trace in full** (OTLP JSON: resource batches → scope spans → spans). + +```bash +curl -s "$BASE/api/traces/$TRACE_ID?start=$START&end=$NOW" "${AUTH[@]}" +``` + +**Enumerate values for a tag** — useful for inventorying what a fleet emits. Expect a `502` on high-cardinality tags; fall back to inspecting individual traces rather than concluding the tag is unused. + +```bash +curl -s -G "$BASE/api/v2/search/tag/span.db.system/values" "${AUTH[@]}" \ + --data-urlencode "start=$START" --data-urlencode "end=$NOW" +``` + +## TraceQL patterns worth knowing + +| Goal | Query | +| --- | --- | +| One service | `{resource.service.name="svc-name"}` | +| Several services | `{resource.service.name=~"(svc-a|svc-b)-prd"}` | +| Attribute present at all | `{span.db.system != nil}` | +| Span kind | `{kind=server}`, `{kind=client}` | +| Slow spans | `{duration > 1s}` | +| **Two conditions anywhere in the same trace** | `{resource.service.name="svc-a"} && {span.db.system != nil}` | + +The last one is the important one: `&&` between two brace groups is a **trace-level** conjunction, not a single-span filter. It answers "does a request into this service reach a database at all", which is how you map how deep a trace goes without reading traces one at a time. + +## Reading the response + +- **Span and trace ids are base64**, not hex. Decode before comparing them to anything from a header or from Sentry: `Buffer.from(id,"base64").toString("hex")`. +- **`kind` is a string** (`SPAN_KIND_SERVER`, `SPAN_KIND_CLIENT`, `SPAN_KIND_INTERNAL`), not the numeric enum. Filtering on `sp.kind === 2` silently matches nothing. +- **Search results drop leading zeros from trace ids.** A 31-character id is a 32-character id with a leading zero; zero-pad before using it anywhere else, or the lookup fails for a reason that looks like absence. +- **`rootServiceName: ""`** means the trace's root is not in Tempo. For client-originated requests that is the normal case — the root is a client span living in Sentry — and it is the marker for finding them. +- Resource attributes carry deployment context (`service.name`, kubernetes pod/namespace/cluster, region); span attributes carry the request (`http.*`, `net.*`, `db.*`). + +## Deep links for sharing + +A link is more useful than a pasted id. Build a Grafana Explore URL with the query pre-filled: + +```bash +node -e ' +const left={datasource:process.env.TEMPO_UID, + queries:[{refId:"A",datasource:{type:"tempo",uid:process.env.TEMPO_UID},queryType:"traceql",query:process.argv[1]}], + range:{from:"now-6h",to:"now"}}; +console.log(`${process.env.GRAFANA_HOST}/explore?orgId=${process.env.GRAFANA_ORG}&left=${encodeURIComponent(JSON.stringify(left))}`); +' '' +``` + +Prefer an absolute `from`/`to` when the link needs to outlive the event; a relative window slides off it and the reader opens an empty result. + +## Failure modes + +| Symptom | Cause | Response | +| --- | --- | --- | +| All queries return 0 | Session expired (401, empty body) | Check `/api/user` first | +| Every filter returns exactly `limit` | Filter not applied | Run the negative control | +| By-id lookup times out | No time window | Pass `start`/`end` | +| Tag-values returns 502 | High cardinality | Inspect traces directly | +| Id from search not found elsewhere | Leading zeros stripped | Zero-pad to 32 chars | +| Kind filter matches nothing | Comparing to a number | Compare to `SPAN_KIND_*` | +| Trace has no root | Root is a client span | Expected; see `sentry-grafana-correlation` | diff --git a/domains/analytics/skills/instrumentation/repos/metamask-extension.md b/domains/analytics/skills/instrumentation/repos/metamask-extension.md new file mode 100644 index 00000000..d88ddaec --- /dev/null +++ b/domains/analytics/skills/instrumentation/repos/metamask-extension.md @@ -0,0 +1,61 @@ +--- +repo: metamask-extension +parent: instrumentation +--- + +## Key Files + +| Content | Path | +|---------|------| +| Sentry trace wrapper | `shared/lib/trace.ts` | +| Trace name enum | `shared/lib/trace.ts` → `TraceName` | +| MetaMetrics controller | `app/scripts/controllers/metametrics-controller.ts` | +| Event enum | `shared/constants/metametrics.ts` → `MetaMetricsEventName` | +| Sentry setup + sample rate | `app/scripts/lib/setupSentry.js` → `getTracesSampleRate()` | +| Segment tracking plan | `Consensys/segment-schema` → `tracking-plans/metamask-extension.yaml` | + +## Cross-Process Context (UI → Background) + +The extension has two Sentry hubs — one in the UI process and one in the background service worker. A trace starting in UI and continuing in background requires explicit context propagation across the RPC boundary: + +```typescript +// Serialize at UI call site +const context: SerializedTraceContext = { + _name: TraceName.MyOperation, + _traceId: span.spanContext().traceId, + _spanId: span.spanContext().spanId, +} + +// Background receives context, creates child span +trace({ name: TraceName.MyOperation, parentContext: context }, async () => { ... }) +``` + +Without propagation: Sentry shows two disconnected operations. With propagation: complete tree from user action to RPC call. + +## Sentry Sample Rate + +```bash +grep -n "tracesSampleRate" app/scripts/lib/setupSentry.js +# Verify current value before calculating — it has changed between releases +``` + +## Sentry Traces Explorer Query (Volume Estimation) + +``` +Environment: production | Time range: 30 days | Mode: aggregate +Query: span.op:http.client span.description:*{endpoint}* +Group by: span.description, transaction +Sort: -count(span.duration) +``` + +## Detect `isOptIn` Misuse + +```bash +grep -rn "isOptIn: true" app/scripts/ ui/ --include="*.ts" --include="*.tsx" +# Any occurrence outside the onboarding opt-in flow is suspect +``` + +## Data Council Contact + +- Slack: `#metamask-metametrics` +- Team: `@consensys/data-council` diff --git a/domains/analytics/skills/instrumentation/skill.md b/domains/analytics/skills/instrumentation/skill.md new file mode 100644 index 00000000..987ef8f3 --- /dev/null +++ b/domains/analytics/skills/instrumentation/skill.md @@ -0,0 +1,89 @@ +--- +maturity: experimental +name: instrumentation +description: Create and update Sentry spans, MetaMetrics events, and Segment events — methodology, policies, common pitfalls +--- + +# Analytics Instrumentation + +## When To Use + +- Adding or modifying a MetaMetrics (Segment) event +- Adding or modifying a Sentry performance span +- Estimating event or span volume from production data +- Auditing existing instrumentation for correctness + +--- + +## Do Not Use When + +- Adding local debug logging with no telemetry destination +- Investigating an existing Sentry error report (use `sentry-mcp-queries`) +- Internal feature flag evaluation not surfaced as an analytics event + +--- + +## Sentry Spans + +### Creating a Span + +1. **Register a named trace entry** in the repo's trace name enum before writing any span code. Unnamed spans are invisible in Sentry filters. +2. **Use the repo's `trace()` wrapper**, not raw `Sentry.startSpan()`. Wrappers handle cross-process context propagation, active-span inheritance, and consistent tag injection. +3. **Inherit parent automatically** — when no `parentContext` is provided, the wrapper inherits from `Sentry.getActiveSpan()`, making the new span a child of the active parent (e.g., a `pageload` span). + +### Updating a Span + +- Adding a tag: no governance required +- Renaming a trace name enum entry: grep all callsites; update enum and references atomically +- Changing an `op` value: breaks saved queries and dashboards — coordinate with whoever owns them + +--- + +## MetaMetrics / Segment Events + +### Creating an Event + +1. **Check the event name enum** — event may already exist under a different phrasing. +2. **Check the segment tracking plan** — event may be registered under a different name than the enum key. +3. **Add to the enum**, then implement the `trackEvent` call. +4. **Do NOT use `isOptIn: true` outside the onboarding opt-in flow.** It strips user identity unconditionally for all users, not just non-opted-in ones (see Reference Knowledge: metrametrics-identity). +5. **Open a data governance review** before merging. There is usually no CI enforcement on schema registration — this step is easy to skip (see Reference Knowledge: segment-governance). +6. **Register in the team's segment tracking plan** before shipping. + +### Updating an Event + +- Adding a property: requires governance review and schema update +- Renaming an event: deprecate old + add new in tracking plan; coordinate on migration window +- Removing an event: confirm no active dashboards depend on it before removing + +--- + +## Volume Estimation via Sentry + +When direct Segment access is unavailable, estimate from Sentry production span data: + +1. **Find a correlated HTTP endpoint** — one that fires 1:1 with the event. +2. **Query Sentry Traces Explorer** (aggregate mode): + ``` + span.op:http.client span.description:*{endpoint}* + ``` +3. **Extrapolate:** + ``` + estimated_actual = sampled_count × (1 / tracesSampleRate) + ``` +4. **Interpret as upper bound** — endpoint may have callers outside the event path. + +Caveats: sample population is MetaMetrics opted-in users only; verify the current `tracesSampleRate` before calculating (it changes between releases). For longer-range (30D+) or release-over-release queries, the sampled count is **not** comparable at face value — older releases are downsampled / retention-truncated and `.0` releases are sample-thin; see `sentry-mcp-queries` (Longer-Range Queries and Percentile Fidelity) and the `performance-attribution` skill. + +--- + +## Common Pitfalls + +| Mistake | Correct Approach | +|---------|-----------------| +| `isOptIn: true` on post-onboarding events | Strips user identity for all users; only valid in onboarding flow | +| Ship event without tracking-plan registration | No CI gate — add governance review explicitly to PR checklist | +| Raw `Sentry.startSpan()` instead of the repo's `trace()` wrapper | Use the wrapper — handles cross-process context and active-span inheritance | +| New span with no trace name enum entry | Register enum entry first; unnamed spans are invisible in Sentry filters | +| Multiply sampled count by `tracesSampleRate` | Multiply by inverse: `sampled × (1 / rate)` | +| Treat Sentry estimates as exact counts | Probabilistic sample — state sample size and confidence | diff --git a/domains/analytics/skills/performance-attribution/repos/metamask-extension.md b/domains/analytics/skills/performance-attribution/repos/metamask-extension.md new file mode 100644 index 00000000..4d0ab9ef --- /dev/null +++ b/domains/analytics/skills/performance-attribution/repos/metamask-extension.md @@ -0,0 +1,96 @@ +--- +repo: metamask-extension +parent: performance-attribution +--- + +## Source & Project + +Primary source is Sentry **Trace Explorer** (not Dashboard 219877): + +- Project `metamask` (ID `273505`), `environment:production` +- Mode `Aggregates`, **Group By** `release`, **Visualize** `p75(span.duration)` and `p95(span.duration)` +- Time `90d` (primary). Dashboard 219877 (30d) is legacy/context only. + +## Key Transactions + +| Transaction | What it measures | +|---|---| +| `UI Startup` | Extension click → interactive UI | +| `/home.html` | Home page render | +| `Asset Details` | Token/NFT detail view render | +| `/notification.html` | dApp confirmation popup (approvals/signatures) — high-frequency for power users, compounds with usage | + +## Query Template + +``` +is_transaction:true environment:production transaction:"UI Startup" (release:metamask-extension@13.11.2 OR release:metamask-extension@13.12.2 OR release:metamask-extension@13.13.1 OR release:metamask-extension@13.14.2 OR release:metamask-extension@13.15.0) +``` + +Swap the `transaction:"…"` value per metric; keep `statsPeriod=90d`. + +## Version Selection — Highest-Sample Patch Per Minor + +Anchor each minor line on its highest-sample patch, never the `.0`: + +| Minor | Patch used | Rationale | +|---|---|---| +| 13.11 | 13.11.2 | Highest sample count | +| 13.12 | 13.12.2 | Highest sample count | +| 13.13 | 13.13.1 | Highest sample count | +| 13.14 | 13.14.2 | Highest sample count | +| 13.15 | 13.15.0 | Current release | + +`.0` releases have **10–100× fewer samples** — never anchor a percentile on a `.0` when a higher patch exists in the same minor line. + +## 90d vs 30d — Empirical + +30d baselines ran **~2× higher** than 90d for the same metric (e.g. UI Startup p75 `9.39s → 3.47s` at 30d vs `4.40s → 3.34s` at 90d). Cause unconfirmed — residual-user population and/or sampling of residual traffic; **not** confirmed "power users" (no cohort segmentation). Report 90d; cite 30d only for context. Note: Sentry share links may render 30d in the UI even when the report figure is 90d — verify `statsPeriod=90d`. + +## Hot-Path Files + +| Path | Why it matters | +|---|---| +| `babel.config.js` | Build-time transforms (e.g. React Compiler) — broad scope | +| `ui/selectors/*.js` | Redux selectors — run on every state change | +| `ui/hooks/*.ts` | Hooks — component lifecycle | +| `ui/components/` | Virtualization / render patterns | +| `package.json` | Dependency runtime behavior + core-package bumps | + +## Analysis Commands + +```bash +git log v13.X.X..v13.Y.Y --oneline --no-merges | wc -l # commit count between releases +git diff v13.X.X..v13.Y.Y --stat -- ui/selectors babel.config.js # file-level change summary +git diff v13.X.X..v13.Y.Y -- # detailed diff for one file +git log v13.X.X..v13.Y.Y --oneline -- # commits touching specific paths +``` + +## Core Packages to Monitor + +App-repo diffs miss work shipped as version bumps. Diff `package.json`, then read each package CHANGELOG: + +| Package | Performance relevance | +|---|---| +| `@metamask/assets-controllers` | Token detection, balance fetching, NFT metadata | +| `@metamask/transaction-controller` | Transaction state size, history storage | +| `@metamask/network-controller` | RPC call handling, retry logic | + +```bash +git diff v13.X.X..v13.Y.Y -- package.json | grep -E "@metamask/(assets-controllers|transaction-controller|network-controller)" +``` + +Example findings: + +- `@metamask/transaction-controller` v62.8.0 — deprecated `history` / `sendFlowHistory` from `TransactionMeta` → significant state-size reduction for power users (consumed in extension [#38665](https://github.com/MetaMask/metamask-extension/pull/38665)). +- `@metamask/assets-controllers` v94.0.0 ([core #7408](https://github.com/MetaMask/core/pull/7408)) — Account API v2 → v4 for token detection → fewer RPC calls, delegated detection. + +## Worked Example: v13.11 → v13.15 (90d) + +| Metric | p75 (typical) | p95 (tail) | +|---|---|---| +| UI Startup | 4.40s → 3.34s (-24%) | 15.65s → 9.11s (**-42%**, -6.5s) | +| /home.html | 1.69s → 1.19s (-30%) | 4.96s → 3.24s (-35%) | +| Asset Details | 100ms → 47ms (**-53%**) | 287ms → 94ms (**-67%**) | +| /notification.html | 1.36s → 1.05s (-23%) | 4.30s → 4.71s (+9%, **high variance — inconclusive**) | + +Most UI Startup and /home.html gains landed in 13.12 (p95 UI Startup -40% in one release); Asset Details improved across 13.14 → 13.15. Treat the per-release header deltas as measured totals and attribute individual code changes as likely contributors only. diff --git a/domains/analytics/skills/performance-attribution/skill.md b/domains/analytics/skills/performance-attribution/skill.md new file mode 100644 index 00000000..379dac6d --- /dev/null +++ b/domains/analytics/skills/performance-attribution/skill.md @@ -0,0 +1,90 @@ +--- +maturity: experimental +name: performance-attribution +description: Attribute release-over-release p75/p95 performance movements to specific code changes via black-box diff analysis +--- + +# Performance Attribution + +Pair a **measured** percentile movement (from Sentry Trace Explorer) with **black-box code-diff analysis** to produce confidence-rated attributions: what changed across releases, how much it moved, and why. + +## When To Use + +- Explaining a confirmed p75/p95 latency change across releases +- Building a per-release attribution catalogue (change → confidence → metric) +- Auditing whether a "performance initiative" actually moved a metric +- Attributing movement that spans the app repo **and** `@metamask/*` core-package bumps + +## Do Not Use When + +- The metric movement isn't yet confirmed reliable — run query hygiene first (see `sentry-mcp-queries`: filter superseded/low-sample releases, normalize, verify stored sample size) +- You need proof of causation — this yields *likely contributors*, not isolated causes (see Limitations) +- Pre-merge perf review of a single PR — there is no production metric to attribute yet + +## Step 1 — Get the Measurement Right First + +Attribution is only as good as the metric. Lock these down before touching code: + +- **Percentile.** p75 = typical user (more stable signal). p95 = slowest 5% — *assumed* large-wallet/power users, but **not cohort-verified** (also slow hardware / poor network). Prioritize p95 when the optimization targets data size (memoization, virtualization) that disproportionately helps the tail; trust p75 as the more reliable number. +- **Time window.** Use the **longer (90d) window as primary** — it includes traffic from when older releases were actively used, so the population is representative and comparable across releases. A 30d window over-weights residual users still lingering on old versions → inflated baselines and bigger-looking deltas between *different* populations. Report 90d; cite 30d only as context. +- **Version selection.** Per minor line, anchor on the **highest-sample patch**, never the `.0`. `.0` releases have 10–100× fewer samples and are rollout-biased. See `sentry-mcp-queries` → *Filtering Unreliable Releases* and *Longer-Range (30D+) Queries and Percentile Fidelity*. + +## Step 2 — Black-Box Code Analysis + +Assess impact on **code content + execution frequency alone**. Deliberately ignore commit messages, PR titles/descriptions, claimed impact, and epic/initiative goals — they bias the read. Base it on: diff content, file location (→ execution frequency), algorithmic complexity, and memoization patterns. + +Hot-path categories — a change here can move a render metric: + +| Category | Why it matters | +|---|---| +| Build config | Build-time transforms (e.g. React Compiler) apply broadly | +| Selectors | Run on every state change — hottest path | +| Hooks | Affect component lifecycle / re-render frequency | +| Components | Virtualization & render patterns | +| Dependencies | Version bumps change runtime behavior | + +Pattern catalogue — what to grep for: + +| Pattern | Signal | Confidence | +|---|---|---| +| `createDeepEqualSelector` → `createSelector` + `EMPTY_ARRAY` sentinel | Removes per-change deep compares | HIGH if foundational selector | +| Identity-function selector `(foo => foo)` → real transform | Broken memoization fixed | HIGH if many consumers | +| In-place mutation `.sort()/.reverse()/.splice()` → spread copy | Mutation had broken all downstream memoization | HIGH | +| Build-plugin addition with broad scope | Build-time optimization | HIGH if scope = all `ui/` | +| O(n) string parse → O(1) lookup | Algorithmic reduction | MEDIUM — depends on call frequency | + +## Step 3 — Score Confidence + +1. **Mechanism** — how does this reduce work? (fewer re-renders / less allocation / better caching) +2. **Frequency** — is the path hot? (selector per state change = hot) +3. **Scope** — how many components/files does it touch? +4. **Match** — does it target what the metric measures? + +| Confidence | Criteria | +|---|---| +| HIGH | Clear mechanism + hot path + timing matches the metric move | +| MEDIUM | Mechanism clear, frequency or scope uncertain | +| LOW | Indirect or infrastructure-only | + +## Step 4 — Don't Forget Core Packages + +App-repo diffs miss work shipped as `@metamask/*` version bumps — it surfaces only as a `package.json` change. For each bump between releases, read the package CHANGELOG "Changed"/"Fixed" sections for state-size reduction, caching, fewer RPC calls, batching, or data-structure/field deprecations. (Commands and the packages to watch live in the repo file.) + +## Reading / Writing an Attribution Catalogue + +- Release-header totals = the **measured** improvement for the whole release +- Table rows = **likely contributors**, not isolated causes +- "High confidence" = mechanism + timing + population align +- Always keep an **Unattributed** section for movement no change explains +- Flag high-variance metrics (a noisy confirmation-popup p95) as inconclusive, not as wins + +## Limitations + +- **Correlation, not causation** — change + improvement in the same release does not prove the change caused it +- **Release totals, not isolated impact** — a "-44%" reflects the entire release, not one change +- **Production variance** — user hardware and network are uncontrolled +- **Code analysis, not runtime profiling** — based on structure, not measured execution paths +- **p95 cohort is assumed, not verified** — no power-user segmentation +- **Window choice changes the baseline** — always state which window a number came from + +For more precise attribution: per-optimization feature flags / A-B tests, CI synthetic benchmarks, and verified user-cohort segmentation. diff --git a/domains/analytics/skills/sentry-grafana-correlation/skill.md b/domains/analytics/skills/sentry-grafana-correlation/skill.md new file mode 100644 index 00000000..0ff7135a --- /dev/null +++ b/domains/analytics/skills/sentry-grafana-correlation/skill.md @@ -0,0 +1,100 @@ +--- +name: sentry-grafana-correlation +description: Join one trace across Sentry and Grafana Tempo by trace id to see the whole client-to-backend path, and diagnose why a half is missing. Covers the split-store model (client spans reach Sentry through the SDK and survive only head sampling; backend spans reach Tempo through tail sampling and Sentry through environment routing), the classification of both-halves / client-only / backend-only outcomes with the sampling and routing rule that causes each, and the id-padding, time-window, and query-syntax traps that make a present trace look absent. Use when a trace looks truncated, a backend span has no parent, per-hop latency needs attributing across the seam, or you need to know which store should hold a given span. Triggers on cross-stack trace, orphaned span, trace id lookup, client-backend correlation, split waterfall, or "where did the rest of the trace go". +maturity: experimental +--- + +# sentry-grafana-correlation + +One request produces spans in two stores, joined only by `trace_id`. Reading a trace end to end means querying both and knowing which absences are expected. + +Prerequisite: `grafana-tempo-queries` for the Tempo side, `sentry-mcp-queries` for richer Sentry work. + +## The model — what lands where, and why a half goes missing + +| Span | Reaches | Gated by | +| --- | --- | --- | +| Client (`pageload`, `navigation`, `http.client`, custom) | Sentry, via the SDK transport | the client's `tracesSampleRate` head decision | +| Backend (`http.server`, internal, db, messaging) | Tempo, via the collector | collector tail-sampling policy | +| Backend, additionally | Sentry, if the collector forwards it | an environment attribute on the span matching a routing policy | + +Three consequences drive every diagnosis below: + +- **The client's sampled flag and the client's own retention are separate decisions.** The propagated `traceparent` flag tells the backend whether to record; the client's head sampling decides whether the client span is kept. When the flag says record and head sampling drops the client span, the backend records a span whose parent was never stored anywhere — an orphan. This is the normal case at low client sample rates, not an anomaly. +- **A `-00` (not-sampled) flag can suppress the backend span entirely**, because a parent-respecting sampler delegates to "never record" for an unsampled remote parent. No backend span is created at all — different from one being dropped later. +- **Backend spans only reach Sentry if their environment attribute matches a routing policy.** A service that expresses environment under a different attribute name matches nothing and is silently absent from Sentry while still present in Tempo. + +## Setup + +Keep organisation slugs, project ids, and hosts in your environment — do not commit them. + +```bash +# SENTRY_ORG, SENTRY_PROJECT_ID (numeric), SENTRY_AUTH_TOKEN +# plus the grafana-tempo-queries variables for the Tempo side +``` + +## Query the Sentry half + +```bash +curl -fsS -G "https://sentry.io/api/0/organizations/$SENTRY_ORG/events/" \ + -H "Authorization: Bearer $SENTRY_AUTH_TOKEN" \ + --data-urlencode "dataset=spans" \ + --data-urlencode "field=span.op" --data-urlencode "field=span_id" \ + --data-urlencode "field=parent_span" --data-urlencode "field=span.description" \ + --data-urlencode "field=timestamp" \ + --data-urlencode "query=trace:$TRACE_ID" \ + --data-urlencode "project=$SENTRY_PROJECT_ID" \ + --data-urlencode "statsPeriod=24h" \ + --data-urlencode "sort=-timestamp" +``` + +Two syntax traps that produce misleading emptiness: + +- **Any field you sort on must also be selected.** Sorting by `-timestamp` without requesting `timestamp` returns `400 orderby must also be in the selected columns or groupby` — and a script that swallows errors reports it as no results. +- **`has:parent_span` is not valid**; request `parent_span` as a field and filter client-side. + +Use `project=-1` to search every project at once when you do not yet know which one should hold the span — that is how you tell "in the wrong project" apart from "absent". + +## Procedure — Tempo to Sentry + +Use when you have a backend trace and want its client context. + +1. Find client-originated backend traces: search your services in Tempo, then keep the results whose `rootServiceName` reports the root was never received. Those reference a client parent that is not in Tempo. +2. **Zero-pad each trace id to 32 characters** before querying Sentry. Tempo search strips leading zeros, and an unpadded id returns nothing for a reason that looks like absence. +3. Query Sentry for `trace:`, first in the client's project, then with `project=-1`. +4. Classify with the table below. + +## Procedure — Sentry to Tempo + +Use when a Sentry trace looks truncated at the network boundary. + +1. Take the trace id from the Sentry trace view. +2. Look for a matching `http.server` span in Sentry itself first — if the collector forwards backend spans for that environment, both halves may already be in one place and no cross-store hop is needed. +3. Otherwise fetch the trace from Tempo by id, with a time window that brackets the client span's timestamp. +4. If Tempo has nothing, the backend either never recorded it (a `-00` flag), or its trace fell outside the tail-sampling policy. + +## Classification + +| What you find | Meaning | Where to look next | +| --- | --- | --- | +| Client and backend spans, backend parented on the client's request span | Healthy join; per-hop latency is attributable | — | +| Client and backend spans, backend parented on an enclosing operation root | Propagation is attaching the wrong parent, so the backend span sits beside its caller instead of beneath it | The client's header-injection path | +| Backend spans only, client parent referenced but nowhere | Orphan: the flag instructed recording, head sampling discarded the client span | Client sample rate, or decoupling the flag from head sampling | +| Client spans only, no backend span anywhere | Either no header was propagated to that host, or the flag was `-00` so the backend never created a span | Propagation targets, then the flag | +| Backend in Tempo but not in Sentry when it should be | Environment attribute does not match a forwarding policy | The service's environment tagging | +| Nothing in either store | Head-sampled out end to end | Expected at low sample rates | + +## Checking whether a backend span nests correctly + +The parent identity, not the picture, is what determines nesting. Take the backend `http.server` span's `parentSpanId` (hex-decode it from Tempo's base64), then look that id up among the client's spans in Sentry: + +- Resolves to an `http.client` span whose description matches the same URL → correctly nested beneath the request that caused it. +- Resolves to a transaction root or custom operation span → the backend span is a sibling of its caller; hop latency cannot be read off the waterfall. +- Resolves to nothing in either store → orphan. + +## Traps + +- **Time windows differ per store.** Tempo retention is typically much shorter than Sentry's, so an older trace legitimately exists in one and not the other. Confirm the window before concluding a half is missing. +- **Verify credentials on both sides first.** An expired Grafana session and an out-of-scope Sentry token both present as empty results, which reads as a real finding about instrumentation. +- **A relative time window on a shared link expires.** Pin absolute ranges when the link needs to outlive the incident. +- **One sampling decision can be shared across a long-lived trace id.** If a client reuses a trace id across many operations, the proportion of spans marked sampled will not match the nominal client rate; do not read that ratio as an effective sample rate. diff --git a/domains/analytics/skills/sentry-mcp-queries/repos/metamask-extension.md b/domains/analytics/skills/sentry-mcp-queries/repos/metamask-extension.md new file mode 100644 index 00000000..efe70acd --- /dev/null +++ b/domains/analytics/skills/sentry-mcp-queries/repos/metamask-extension.md @@ -0,0 +1,60 @@ +--- +repo: metamask-extension +parent: sentry-mcp-queries +--- + +## Organization and Projects + +``` +mcp__sentry__find_organizations → confirm org slug +mcp__sentry__find_projects → metamask-extension (Chrome/MV3 + Firefox/MV2) +``` + +## Standard Filter Set for Extension Errors + +``` +environment:production +installType:normal +``` + +Then add `dist:mv3` or `dist:mv2` to isolate by manifest. + +## Sample Rate + +Production `tracesSampleRate` = `0.0075` (0.75%) → multiplier ≈ 133× + +```bash +# Verify current value before using +grep "tracesSampleRate" app/scripts/lib/setupSentry.js +``` + +## Volume Estimation — Worked Example + +`AssetsFirstInitFetchCompleted` correlates 1:1 with `accounts.api.cx.metamask.io/v1/supportedNetworks` (fires once per init) — **not** `/v4/multiaccount/balances` (fires per account): + +``` +/v1/supportedNetworks: 2.6M sampled (30d) × 133 ≈ 346M event fires / month +/v4/multiaccount/balances: ~26M sampled × 133 ≈ 3.5B balance API calls / month (per-account — NOT the event rate) +``` + +Lesson: pick the once-per-event endpoint or you over-count by the fan-out factor. + +## Common Issue Searches + +| What you're looking for | Query | +|---|---| +| Background connection errors | `is:unresolved background connection` | +| MV3-only errors | `is:unresolved dist:mv3` | +| Errors spiking in recent release | `is:unresolved times_seen:>100` | +| Performance issues | `issue.category:performance` | + +## Tag: `dist` Values + +| Value | Meaning | +|-------|---------| +| `mv3` | Chrome (Manifest V3 — service worker) | +| `mv2` | Firefox (Manifest V2 — background page) | + +## Seer Analysis Notes + +Seer has access to the Sentry issue, stack traces, and recent events. It does not have access to the codebase. Validate its hypothesis against the actual handler chain in the source — especially for keepalive, lifecycle, and concurrency conclusions. diff --git a/domains/analytics/skills/sentry-mcp-queries/skill.md b/domains/analytics/skills/sentry-mcp-queries/skill.md new file mode 100644 index 00000000..8a134576 --- /dev/null +++ b/domains/analytics/skills/sentry-mcp-queries/skill.md @@ -0,0 +1,129 @@ +--- +maturity: experimental +name: sentry-mcp-queries +description: Query Sentry via MCP — error triage, tag distribution, volume estimation, replay retrieval +--- + +# Sentry MCP Queries + +## When To Use + +- Investigating a production error before attributing root cause +- Checking dist (MV3 vs MV2) error distribution +- Estimating event or span volume from production data +- Comparing error rates release-over-release for regression detection +- Retrieving session replay or profiling data + +## Do Not Use When + +- The error reproduces locally with a full stack trace +- Reading product analytics (Segment events, not Sentry errors/spans) +- Pre-merge investigation — Sentry data is post-merge only + +## Setup + +Run once per session: + +``` +mcp__sentry__whoami → confirm auth +mcp__sentry__find_organizations → org slug +mcp__sentry__find_projects → project slug(s) +``` + +All subsequent tools require `organization_slug` and usually `project_slug`. Slug mismatch causes silent empty results. + +## Workflow: Error Triage + +1. `mcp__sentry__search_issues` — find by title, fingerprint, or keyword +2. `mcp__sentry__get_issue_tag_values` — check `dist` distribution **before** attributing root cause +3. If 99%+ one dist → platform lifecycle root cause (see `extension-errors-debugging`) +4. `mcp__sentry__search_issue_events` — individual events for stack trace detail +5. `mcp__sentry__analyze_issue_with_seer` — AI-assisted hypothesis (validate against code) + +## Workflow: Volume Estimation + +Segment event volume is invisible from Sentry, but a correlated `http.client` span is not. Anchor estimation on an HTTP endpoint the event's controller calls **1:1** with the event firing. + +1. Identify the correlated endpoint — the one that fires **once per event**, not per sub-call (e.g. a per-init call, not a per-account call). Picking a per-sub-call endpoint over-counts. +2. `mcp__sentry__search_events` aggregate mode, filter `span.op:http.client` + endpoint +3. Read sampled span count +4. Extrapolate: `estimated = sampled × (1 / tracesSampleRate)` +5. Treat as an **upper bound** — the endpoint may have callers beyond the event path. Sample population = MetaMetrics-opted-in users only (Sentry opt-in is tied to MetaMetrics). Sample rate changes — verify the current value. + +## Workflow: Release Comparison + +Compare error rates or metrics across releases for regression detection: + +1. `mcp__sentry__find_releases` — list releases sorted by date +2. **Filter out unreliable releases** (see below) before comparing +3. `mcp__sentry__search_events` with `release:12.5.0` for baseline +4. `mcp__sentry__search_events` with `release:12.6.0` for comparison +5. **Normalize by sessions or users** — raw counts conflate traffic changes with error rate changes: + ``` + rate = events / sessions_for_that_release + ``` +6. Report delta against baseline with sample-size caveat + +## Filtering Unreliable Releases + +Patch releases have uneven adoption — comparing raw counts against them produces false signal. Skip a release before comparing if: + +| Filter | Threshold | Reason | +|---|---|---| +| Age since publish | < 48–72h | Browser auto-update rollout still ramping (Chrome/Firefox/Edge) | +| Session count | < ~50% of previous stable release | Sample too small for meaningful rates | +| Stored span count | < ~few hundred for p75, < ~few thousand for p95+ | Tail percentiles are computed over the *stored* sample — extrapolated counts hide how few events back them | +| Superseded patch | a higher patch in the same `X.Y.*` line exists **and** the active window (`first_seen`→`last_seen`) is short | Hotfixed-past releases collect few spans, biased to early-updaters during the rollout/migration window | +| Release stage | `dev`, `canary`, `nightly` | Non-production build — different error profile | +| Environment | not `production` | Development / staging noise | +| Manifest split | compare only within same `dist` | MV3 and MV2 populations have different error distributions | + +**Rule of thumb:** use the newest release that has ≥ 3 days of production adoption **and** session volume comparable to the previous stable release. Everything in between is hotfix noise — skip it for regression comparisons unless investigating that specific patch. + +## Longer-Range (30D+) Queries and Percentile Fidelity + +Widening the window past ~30 days to gain sample size trades it back for **fidelity loss on older releases**. Three effects compound: + +- **Sample-rate drift** — `tracesSampleRate` changes between releases, so absolute span counts across a 30D+ window mix different capture rates. Normalize each release by *its own* sample rate (or by sessions/users), never a single global rate. +- **Extrapolation hides thin samples** — span datasets report sample-rate-weighted (extrapolated) counts. A release with 40 stored spans at 0.75% extrapolates to ~5,300 — a real-looking number backed by 40 events. Always check the **stored** sample count, not the extrapolated total, before trusting a release. +- **Retention downsampling** — spans near the retention boundary are partially evicted, so an old release's count is truncated, not representative. Treat the oldest releases in a 30D+ window as lower bounds only. + +**For p75+ analysis** (any tail percentile — p75/p90/p95/p99), sample size *and* quality both matter: + +- **Size** — percentiles are computed over stored events. p50 stabilizes in the low hundreds; p75 needs more; p95/p99 need thousands of stored spans. Below that, a handful of outliers move the number — don't report a tail percentile you can't back with stored count. +- **Quality** — rollout-window spans (first-launch, cold cache, state migration) skew the tail high. A superseded patch release's spans are disproportionately these, so its p75+ reads worse than its steady state would. + +**Resolving the size-vs-fidelity tension:** when a single release lacks the sample to support p75+, **collapse the patch chain** — aggregate `release:X.Y.*` across the minor line, or compare against the last *widely-adopted* patch — rather than extending the window into aged, downsampled, sample-rate-drifted territory. Reach for sample size *across adjacent stable patches inside the retention-safe window*, not by going further back in time. Use a longer (90d) window as the **primary, comparable-across-releases** source for p75/p95 and a 30d window only as **secondary context** — 30d over-weights the users still lingering on old versions and inflates baselines. + +For attributing a confirmed p75/p95 movement to specific code changes, see the `performance-attribution` skill. + +## Workflow: Replay and Profile + +1. `mcp__sentry__search_issue_events` — find an event ID with replay/profile +2. `mcp__sentry__get_replay_details` / `mcp__sentry__get_profile_details` for that event ID + +## Tag Filters + +| Tag | Values | Use | +|-----|--------|-----| +| `dist` | `mv3`, `mv2` | Isolate by manifest version | +| `environment` | `production`, `staging` | Exclude non-prod noise | +| `installType` | `normal`, `development`, `sideload`, `admin` | Exclude developer-loaded builds | + +**Do not conflate `environment` and `installType`** — a production build can have `installType:development` if loaded unpacked. + +## Common Pitfalls + +| Mistake | Correct Approach | +|---------|-----------------| +| Attribute root cause before checking `dist` distribution | Check tag values first — 99%+ MV3 → lifecycle, not app logic | +| Use raw sampled count as event volume | Multiply by `1 / tracesSampleRate` | +| Filter `environment:development` for dev builds | Filter `installType:normal` — environment ≠ install method | +| Skip `whoami` and guess org slug | Slug mismatch causes silent empty results | +| Treat Seer analysis as ground truth | Use as hypothesis to validate against code/traces | +| Compare raw event counts across releases | Normalize by sessions — traffic changes masquerade as regressions | +| Include a <48h-old release in a regression comparison | Wait for rollout; auto-update adoption takes 2–7 days | +| Treat every patch release as a comparison point | Most patches have low adoption — compare to the last *widely-adopted* release | +| Trust a release's p95 because its (extrapolated) span count looks large | Check the *stored* sample — p75+ needs hundreds-to-thousands of stored events to be stable | +| Compare span counts across a 30D+ window at face value | Normalize per-release sample rate; older releases are downsampled / retention-truncated | +| Anchor a percentile on a `.0` release | `.0` releases have 10–100× fewer samples — use the highest-sample patch in the minor line | diff --git a/domains/analytics/skills/sentry-quota/repos/metamask-extension.md b/domains/analytics/skills/sentry-quota/repos/metamask-extension.md new file mode 100644 index 00000000..d3d2b571 --- /dev/null +++ b/domains/analytics/skills/sentry-quota/repos/metamask-extension.md @@ -0,0 +1,50 @@ +--- +repo: metamask-extension +parent: sentry-quota +--- + +## File Paths + +| Path | Role | +|---|---| +| `shared/lib/trace.ts` | `TraceName` / `TraceOperation` enums = the custom-span registry; `trace({ name, op, data }, cb)` API | +| `shared/lib/wrapper-sampling.ts` | `shouldSampleWrappers(traceId)` — the Tier-2 deterministic sub-sample gate | +| `shared/lib/messenger-tracing.ts` | `wrapMessengerWithTracing` + `isReadOnlyAction` read-only denylist (~90% volume cut before sampling) | +| `app/scripts/lib/createMetaRPCHandler.ts` | `rpc.handler` span — gated behind `shouldSampleWrappers` | +| `app/scripts/lib/setupSentry.js` | global `tracesSampleRate` (`0.0075` = 0.75%) | + +Core controller instrumentation lives in the **`MetaMask/core`** monorepo: per-package `TraceName` in `packages//src/**/{constants/traces,utils/trace}.ts` (e.g. `bridge-controller/src/constants/traces.ts`). Controllers don't import Sentry — they call an injected `trace` callback (`traceAsControllerCallback` in the extension). + +## Commands + +```bash +EXT= +CORE= + +# Span registries (the inventory) +rg -n 'enum TraceName' "$EXT/shared/lib/trace.ts" +rg -n -g '**/{traces,trace}.ts' 'enum TraceName' "$CORE/packages" + +# Locate a culprit's emit site +rg -n '|TraceName.' "$EXT" "$CORE/packages" + +# All span creation sites — then read each enclosing scope for loop/poller (fan-out) +rg -n 'trace\(' "$EXT/app" "$EXT/shared" "$CORE/packages//src" + +# Gate present before the span? (absence = always-on) +rg -n 'shouldSample|tracesSampleRate|hashBucket|Math.random' + +# Kill-switch present? +rg -n 'SENTRY_[A-Z_]*DISABLED' "$EXT" "$CORE/packages//src" + +# PR review — added instrumentation lines only +gh pr diff --repo MetaMask/metamask-extension \ + | rg '^\+' | rg 'TraceName|trace\(|shouldSampleWrappers|SENTRY_.*DISABLED|op:' +``` + +## Architectural Notes + +- **Gate location differs by repo.** Extension spans go through `trace()` — gate at the call site or in the wrapper. Core controller spans go through the injected callback — gate in the package's trace util or the callback so every consumer (extension, mobile) inherits the cap. +- **`BackgroundRpc` / `MessengerCall`** (the `TraceName` tail) are the already-gated wrapper spans from [PR #39891](https://github.com/MetaMask/metamask-extension/pull/39891) — the reference implementation of the Tier-2 sub-sample pattern and the `SENTRY_DISTRIBUTED_TRACING_DISABLED` kill-switch. +- **Tier-0 fix path is a core PR + a patch on the extension release branch.** Controller instrumentation originates in `MetaMask/core`; the release branch is where the cherry-pick lands. The sev-1 blocker goes on the in-flight release milestone — e.g. [issue #43211](https://github.com/MetaMask/metamask-extension/issues/43211) ("Assets Controller Sentry Instrumentation exceeding quota"). +- **Spotting the culprit first:** `sentry-mcp-queries` → Volume Estimation (`span.op` aggregate × `1 / tracesSampleRate`) ranks span contributors; this skill takes over once you have the offending span name. diff --git a/domains/analytics/skills/sentry-quota/skill.md b/domains/analytics/skills/sentry-quota/skill.md new file mode 100644 index 00000000..8ef9e4a5 --- /dev/null +++ b/domains/analytics/skills/sentry-quota/skill.md @@ -0,0 +1,87 @@ +--- +maturity: experimental +name: sentry-quota +description: Catch quota-risky Sentry span instrumentation in code and PRs — fan-out × ungated × no-kill-switch — before it blows the span budget +--- + +# Sentry Span Quota Guard + +Find and fix custom Sentry span instrumentation that blows the project span budget. Operates on **code and PRs**, not Sentry dashboards — you spot the culprit in Sentry (`sentry-mcp-queries`), this skill fixes it in code. + +## When To Use + +- A PR adds custom Sentry spans (`trace()` calls / `TraceName` entries) — review it before merge. +- A custom span/transaction dominates span volume in Sentry — locate where it's emitted and fix it. +- Auditing controllers/UI for always-on, fan-out-prone instrumentation. +- A custom span is the top span-count contributor and must be cut fast (release blocker). + +## Do Not Use When + +- Reading the live span counts themselves — that's `sentry-mcp-queries` (Volume Estimation). +- Product-analytics events (Segment / `trackEvent`) — that's `instrumentation` + `segment-governance`. +- The span is already behind a per-trace sample gate **and** a kill-switch — already mitigated. + +## Breach Triad + +A custom span is a quota risk when these stack. The first three together are the breach profile. + +| Signal | Static signature | Why it blows quota | +|---|---|---| +| **Fan-out** | span created in a loop / `.map` / `.forEach` / per-asset / per-account / per-chain / poller | N spans per trace, not 1 | +| **Always-on** | no `tracesSampleRate` sub-rate, no hash gate before the span | every qualifying call emits | +| **No kill-switch** | not guarded by an env flag | disabling needs a release, not a config flip | +| Hot path | data-source / update-pipeline / network callback, not a discrete user action | high call frequency | + +Low fan-out + discrete user action + already gated = fine. Don't flag healthy spans. + +**The subtlest fan-out has no visible loop: a memoized selector.** A `trace` passed into a memoized selector (`createSelector` / `reselect`, or any function called from `useSelector`) fires on every input change by reference. If the selector also iterates entities, it is fan-out × recompute-frequency. Its volume tracks internal state-churn, not user action, so no user-facing metric predicts it — you cannot capacity-plan it. Treat any `trace` reaching a selector as fan-out. + +## Workflow + +### PR review (pre-merge gate) +1. `gh pr diff ` — scan **added** lines for three things, not two: new `TraceName` entries, new `trace(` call sites, **and a `trace`/trace-callback passed as an *argument*** into a call (`fn(…, trace)`). The third is the one reviews miss — a caller wiring up a function's optional `trace?` param adds instrumentation with no `trace(` site and no `TraceName` entry. +2. Score each against the breach triad: is the enclosing scope a loop, poller, **or selector**? is there a gate? a kill-switch? +3. Block if a new always-on span has no gate — require a sub-sample gate (`span-sub-sampling`) before merge. Cheaper than a post-ship cherry-pick. +4. If the diff adds no `trace(` sites, no `TraceName` entries, **and no `trace` argument passed into a call** → "no new instrumentation", stop. + +> **Instrumentation is not always added by an instrumentation PR.** The costliest spans arrive incidentally — a caller passes a `trace` argument into an existing function during an unrelated change (a bug fix, a refactor), so the PR's stated purpose gives no signal to review it for quota. Do not gate this scan on the PR *looking* like instrumentation. And accept the limit: a `trace` argument buried in a bug-fix diff will slip a human reviewer, which is why the runtime backstops (per-name volume alerting, the per-name sampler budget below) exist. This skill lowers the rate; it does not eliminate the class. + +### Locate (incident) +1. Grep the span name / `TraceName.X` across the consuming repo **and** the controller package source. +2. Open the call site; read the enclosing scope for the fan-out verdict (loop/poller?). +3. **No grep hits ≠ safe** — the culprit may be on a release ref not checked out. Verify the package version / `gh pr checkout` the shipping ref before concluding clean. + +### Audit +1. Sweep the span registries (`TraceName` enums) + `trace(` call sites. +2. Rank by breach triad — surface ungated × hot-path × fan-out first. + +### Mitigate +Pick the lowest tier that stops the bleed. + +## Mitigation Ladder + +| Tier | When | Action | +|---|---|---| +| **0 — Immediate** | a span fans out and is actively breaching on the live release | disable the `trace()` call at source (or env-guard it) + **cherry-pick to the release branch** + file a sev-1 release blocker on the in-flight release milestone | +| **1 — Release containment** | spike concentrated in an old, already-patched release with lingering users | Sentry **inbound filter** dropping `release:` spans + force-update. The only dashboard action. Filters target a whole release, not one span — don't filter a release you still want data from | +| **2 — Durable** | the span is justified long-term but ungated | deterministic `traceId`-hash sub-sample gate before the span (`span-sub-sampling`) | +| **3 — Wrong tool** | the metric needs full fidelity; sampling loses the signal | move the metric off trace spans — they are the wrong substrate for always-on high-cardinality metrics. Segment is the usual target, but it has its own ungoverned billing gap (`segment-governance`), so it is not a free lunch | + +Tier 0 + 1 stop the bleed now; Tier 2 is the follow-up so the metric returns. + +**Prevent the next one, not just this one.** Every tier above requires *naming* the offender first, so a new one runs unbounded until someone catches it. A per-transaction-name budget in the sampler — sample the first N of a name per session, then decay — bounds *any* name with no advance knowledge of which will misbehave. It is the only control that acts before the offender is named, and the only one that catches instrumentation added incidentally rather than deliberately. + +## Common Pitfalls + +| Mistake | Correct approach | +|---|---| +| Per-call random sampling (`Math.random()` per span) | Deterministic `traceId`-hash bucket — all spans in a trace kept-or-dropped together, clean waterfalls | +| Gate the span in Sentry config | Gate at the call site; for an injected-callback controller span, gate in the callback so every consumer inherits the cap | +| Inbound-filter a release you still need data from | Filters drop the whole release — fix in code (Tier 0/2) instead | +| "No grep hits, so it's safe" | The culprit may be on a release ref not checked out — verify the version/ref | +| Disable the span on `main` only | Cherry-pick to the active release branch — `main` alone leaves the live release breaching | +| Treat "move to Segment" as free | Segment events ship without CI governance or billing review (`segment-governance`) | +| Ship new always-on instrumentation with no kill-switch | Add an env disable flag on day one — turns a future cut into a config flip, not a cherry-pick | +| An optional `trace?` param passes review because it emits nothing | It is a dormant fan-out — it detonates when any caller supplies the argument. Remove the *param*, not just the argument, so one line can't re-arm it. | +| Disable one entry point of a multi-path change | One change can reach the backend by more than one path (a controller callback *and* a selector param). Audit every entry point it added, not just the one that fired. | +| Filter a release before its successor is fixed | The filter redirects users onto the next build; if that carries the same span, volume only moves. Filter a release only once the build users update to is clean. | diff --git a/domains/coding/skills/distinguishing-observation/skill.md b/domains/coding/skills/distinguishing-observation/skill.md new file mode 100644 index 00000000..fd11ab7b --- /dev/null +++ b/domains/coding/skills/distinguishing-observation/skill.md @@ -0,0 +1,106 @@ +--- +name: distinguishing-observation +description: Enumerate every mechanism that could produce a symptom, then design the observation that separates them — instead of instrumenting the one mechanism you already suspect. An observation your favourite hypothesis predicts, and the alternatives predict too, costs a debugging cycle and buys nothing. Use when a bug has more than one plausible cause, when you are about to add a log line to confirm a suspicion, when a fix landed and the symptom did not move, when every result so far "is consistent with" the theory you started with, or when the symptom looks impossible given your model of the system. Ranks observations by how much they split the candidate set rather than by how easy they are to collect, requires a per-candidate prediction written before looking, and records survivors as not-yet-distinguished rather than ruled out. +--- + +# /distinguishing-observation + +Given a symptom, the instinct is to instrument the mechanism you already suspect. That produces +evidence consistent with your hypothesis — and equally consistent with three others you never +wrote down. + +This is the diagnostic mirror of hypothesis-first validation. There, you fix the hypothesis +before seeing what it will be compared against. Here, you fix the *candidate set* before +choosing what to measure, because the value of an observation is a property of the whole set and +cannot be judged against one member of it. + +## The information is in the split + +A confirming observation feels like progress and usually is not. If four mechanisms could produce +this symptom and your log line fires under all four, you have learned that the code ran. You +already knew that; the symptom told you. + +The observation worth making is the one whose outcome you cannot predict, because the candidates +disagree about it. That is the only kind that costs a cycle and returns a cycle's worth of +information. Debugging that never converges is almost always a sequence of observations each of +which was compatible with everything. + +## The discipline + +1. **List the candidates before instrumenting.** Three to six mechanisms that could produce this + symptom. A list of one is not a list, it is a conclusion — and you will spend the next hour + collecting support for it. + +2. **For each pair, write what differs.** Not what you believe about each; what the world would + look like differently. If two candidates predict identical observations *everywhere*, they are + not distinguishable by observation at all, and you need either a different pair or a different + axis — often a level lower, where the two mechanisms stop coinciding. + +3. **Rank observations by how much they split the field**, not by how easy they are to collect. + The best observation halves the candidate set. The worst confirms the favourite. Cheapness is + worth something, but a cheap observation with no discriminating power is not cheap, it is free + and worthless. + +4. **Predict before you look.** Write down what each candidate predicts for the observation you + are about to make, then make it. Doing this after the fact is how every result becomes + consistent with the hypothesis you started with — the prediction is elastic until it is + written down, and reading the output first sets it. + +5. **A candidate that survives is not eliminated.** Say "not distinguished by this observation", + never "ruled out". The observation constrained what it constrained. This wording is not + pedantry: when the bug comes back in three weeks, a list of things "ruled out" is a list you + will not revisit, and the real mechanism is usually on it. + +## Pairs that look identical from outside + +These shapes recur, and knowing them saves the cycle you would spend rediscovering that your +evidence does not separate them. In each case the fix is to add the separating signal *before* +continuing — which is routinely faster than more reading. + +| indistinguishable pair | why the evidence coincides | what separates them | +|---|---|---| +| an error swallowed by a `catch` vs. a code path never reached | both produce no output, no error, and no trace | count entries to the `try`, not exits from the `catch`: entered-and-never-completed is the first, never-entered is the second | +| a cache hit vs. a correct recomputation | the returned value is the same value | poison the entry with a marker only a hit could return, or count invocations of the compute function | +| a retry that succeeded vs. a call that never failed | both end in one success log | log the attempt number, not the outcome — success on attempt 1 and success on attempt 3 are different worlds | +| a timing-dependent bug vs. a state-dependent bug | both reproduce "sometimes" | hold one axis fixed: a fresh process per run under varying load isolates timing; repeated runs in one process isolate accumulated state | +| the wrong value vs. the right value from the wrong source | the assertion fails the same way | print provenance alongside the value — which module, which config, which build | +| a change that had no effect vs. a change that never shipped | the symptom is unmoved either way | verify delivery first (hash, timestamp, a deliberate marker in the artifact); an undelivered treatment reads exactly like a null result | + +The last row generalises: **before concluding that a mechanism does not matter, prove the +mechanism was present.** Otherwise "no effect" and "not applied" are the same measurement. + +## The anti-pattern: the observation that always fires + +The tell is a log line you added, that printed, and that made you feel confirmed. Ask what would +have had to appear instead for you to abandon the hypothesis. If the answer is "nothing" — if +every candidate on your list predicts this exact output — the observation had no capacity to +discriminate and the confidence it produced is manufactured. + +This is why step 4 is ordered where it is. A prediction table written first makes an +always-fires observation obvious before you spend the cycle: the column is identical all the way +down, and you go find a different one. + +## When the candidate set is empty + +Sometimes you enumerate and get nothing: the symptom is impossible given your model of the +system. That is not a dead end, it is the most informative result available, because it means the +model is wrong and you now know it. + +Switch the question from "which mechanism did this" to **"what would have to be true for this to +happen at all"**, and enumerate *those*. The answers are usually assumptions you did not know you +were making — the built artifact is not the source you are editing, two copies of the module are +loaded, the process you are reading logs from is not the process serving the request, the +environment differs from the one you configured. Each is checkable, and one of them is the bug. + +## Related + +- [`flaky-test-detection`](../flaky-test-detection/skill.md) — the timing-vs-state pair applied + to one domain, where "reproduces sometimes" is the starting symptom rather than a row in a table +- [`falsifiers-first`](../../../pr-workflow/skills/falsifiers-first/skill.md) — the same sealing + discipline pointed at a change instead of a symptom: fix the hypotheses before seeing what they + will be compared against +- [`silent-failure`](../../../pr-workflow/skills/silent-failure/skill.md) — supplies the first + row of the table as a subject in its own right, and asks whether a mechanism announces its own + failure at all +- [`evidence`](../../../pr-workflow/skills/evidence/skill.md) — the runners that collect the + chosen observation and attach the prediction made before it diff --git a/domains/coding/skills/observability-gap/skill.md b/domains/coding/skills/observability-gap/skill.md new file mode 100644 index 00000000..35e5d1b1 --- /dev/null +++ b/domains/coding/skills/observability-gap/skill.md @@ -0,0 +1,114 @@ +--- +name: observability-gap +description: Before debugging a path, establish what signal already exists on it — logs, metrics, error reporting, test coverage, user-visible state — and treat the blanks in that inventory as the first finding. A bug you cannot see is a bug you will fix by guessing, so once reading has stopped narrowing the search, the productive move is to install signal rather than read further. Separates absent signal from suppressed signal — filtered by level, sample rate, a feature flag, or an error-swallowing wrapper — because they are different problems with different fixes, and the suppressed one is both more common and more expensive. Use when a bug reproduces but its cause is invisible, when a report arrives with no trace attached, when reading code has stopped eliminating candidates, or when instrumentation appears to exist and the environment where the bug happens is emitting none of it. +--- + +# /observability-gap + +The first question about a bug is not "where is it". It is **what would have told me**. + +A path you cannot see is a path you will fix by guessing, and a guess that happens to make the +symptom go away is indistinguishable from a fix until it comes back. The opening move on an +unobservable path is usually to make it observable — not because instrumentation is virtuous, +but because every subsequent step is cheaper once the path reports on itself. + +## Inventory the signal before the code + +For the path under investigation, write the list before reading further: + +| signal | what to check | a blank here means | +|---|---|---| +| logs | is anything written on this path, at what level | the path runs and leaves no trace | +| metrics / traces | is the operation counted, timed, spanned | you cannot tell how often, or whether it is getting worse | +| error reporting | does a failure here reach Sentry or equivalent | failures are counted by users, not by you | +| tests | does anything execute this path at all | you cannot reproduce without the full system | +| user-visible state | does the UI or the API response differ when this goes wrong | the only detector is a human noticing | + +The list is not the deliverable. **The blanks are the finding**, and a path with five blanks is +not a hard bug, it is an unobservable one — a different problem with a different first move. + +## Absent is not suppressed + +No log line, and a log line nobody sees, look identical from where you are sitting. They are +not the same problem: + +- **Absent** — the code never emits. The fix is to write the emission, and it lands in the diff. +- **Suppressed** — the code emits and something eats it: a level filter, a sample rate, a + feature flag or env gate, a transport pointed at a sink nobody reads, or a `catch` that + consumes the error before anything can report it. The fix is usually a config change, often + one line, sometimes in a repo you do not own. + +Suppressed is the more common case and by far the more frustrating, because the codebase reads +as instrumented. Grep found the log line. The line is there. It is just not reaching you, and +every minute spent explaining why the code "should" be logging is spent on the wrong question. +Establish which of the two you have before proposing anything. + +## Read until it stops narrowing, then install + +Reading has a point of diminishing returns and it is easy to blow past, because reading feels +like progress in a way that writing a log line does not. + +The tell is mechanical: **two consecutive passes over the same files that eliminate no +candidate**. At that point more reading is not going to produce the answer, and the cheapest +remaining move is to add signal and run it again. One log line at the right boundary routinely +settles a question that an hour of reading left open, because it reports what actually +happened rather than what the code permits to happen. + +## Instrument the boundary, not the suspect + +Put signal at the **edges of the subsystem**, not on the line you suspect. + +A boundary tells you whether the problem is inside or outside, which halves the search +regardless of whether your hypothesis was right. Signal on your favourite line tells you about +that line only, and only in the case where you had already guessed correctly — which is the +case where you needed the least help. Instrument in and out first; narrow after the halving. + +## The gaps worth naming + +| class | why it costs you | +|---|---| +| a failure path with no error reporting | the failure is real and the count is zero | +| an async boundary that loses context | the error surfaces detached from its cause, pointing at the awaiting frame instead of the failing one | +| a conditional whose branch is not recorded | you cannot tell which way it went, so both explanations survive | +| state mutated with no trace of the mutator | you can see the wrong value and not who wrote it | +| a third-party call whose failure mode is a default return | a degraded dependency is indistinguishable from an empty result | +| instrumentation that is off in the environment with the bug | the signal appears to exist | + +The last one is the most expensive in this table, and the reason is in the phrasing: the others +announce themselves as gaps once you look, and this one does not. You find the log line, you +assume the path is covered, and you spend the afternoon reasoning about why the covered path +produced no output. + +### The environment check + +Confirm the signal is on **in the environment where the bug happens**, not in the one where you +are reading the code. A metric emitted only in production and a log emitted only in development +are both silence exactly where you need them — and each looks like working instrumentation from +the other side. + +Concretely, for each signal you are counting on: which env vars, flags, log levels, sample +rates and build modes gate it, and what are their values *on the machine that reproduced the +bug*. If you cannot answer that, you do not know that the signal exists there; you know it +exists in the source. + +## Keeping what you added + +A signal you add to find a bug is a signal the next person needs. Decide deliberately before +removing it, and default to keeping it: the path was hard to debug **because** it was +unobservable, and reverting the instrumentation restores precisely that condition for whoever +arrives next. + +Reasons to remove are real but specific — a per-iteration log in a hot loop, output containing +user data, a metric whose cardinality is unbounded. "It was only for debugging" is not one of +them. If the volume is the problem, lower the level or gate it behind a sample rate rather than +deleting it, so the next person can turn it back on instead of rediscovering the gap. + +## Related + +- [`silent-failure`](../../../pr-workflow/skills/silent-failure/skill.md) — the review-facing + sibling. Same property, opposite end: it asks whether a mechanism would announce its own + failure, this one starts from a failure that already happened and nobody saw +- [`falsifiers-first`](../../../pr-workflow/skills/falsifiers-first/skill.md) — once the path + reports on itself, hypotheses about it become testable rather than arguable +- [`flaky-test-detection`](../flaky-test-detection/skill.md) — the same gap inside a suite, + where the missing signal is what the test observed on the run that failed diff --git a/domains/performance/knowledge/effect-antipatterns.md b/domains/performance/knowledge/effect-antipatterns.md new file mode 100644 index 00000000..2b24006d --- /dev/null +++ b/domains/performance/knowledge/effect-antipatterns.md @@ -0,0 +1,149 @@ +--- +name: effect-antipatterns +domain: performance +description: The React `useEffect` patterns that cause unnecessary renders, memory leaks, or race conditions — the canonical, platform-agnostic taxonomy that per-repo effect references instantiate +--- + +# Effect Anti-Patterns + +**This file is the single source for the pattern taxonomy.** Per-repo references — such as +the `mm-hook-dependency-arrays` and `mm-useeffect-antipatterns` references shipped with the +`performance` skill — name these patterns rather than redefining them, and add what only +they can: verified instances with `file:line`, repo-specific lint gaps, and fix recipes. + +Two halves, and they fail differently. Patterns 1–2 are about **when an effect re-runs** +(the dependency side). Patterns 3–5 are about **what happens inside and after it** (the +lifecycle side). + +## 1. Unstable dependency identity + +A dependency array is supposed to be a cheap identity check. Anything that produces a new +value every render defeats it — and usually signals an unstable reference upstream. + +```typescript +// ❌ serializes on EVERY render just to build the dep key +useEffect(() => { doSomething(config) }, [JSON.stringify(config)]) + +// ❌ new object every render → effect runs every render (or loops forever) +useEffect(() => { ... }, [{ id: user.id }]) + +// ✅ stabilize the reference upstream, then depend on it directly +const stableConfig = useMemo(() => derive(a, b), [a, b]) +useEffect(() => { doSomething(stableConfig) }, [stableConfig]) + +// ✅ or depend on the primitives +useEffect(() => { ... }, [user.id]) +``` + +Stabilizing the source beats hashing it. If you genuinely cannot, a primitive key computed +**once** (`useMemo(() => xs.join(','), [xs])`) still beats a per-render `JSON.stringify`. + +Detection: grep for `JSON.stringify` inside a dependency array, and for inline `{`/`[` +literals in the dep position. + +## 2. Wrong dependencies + +```typescript +// ❌ empty deps but reads state → stale closure, value frozen at first render +const onPress = useCallback(() => doThing(count), []) + +// ❌ empty deps and reads nothing → this was never a hook, hoist it out +const config = useMemo(() => ({ a: 1, b: 2 }), []) +``` + +**Fix:** include what you read; or if there is genuinely nothing to read, move the constant +outside the component. Where `react-hooks/exhaustive-deps` is not enabled, this is not +caught automatically and must be reviewed by hand. + +## 3. Derived state via effect + setState + +If a value is computable from props/state/store, compute it during render. State plus an +effect is for *synchronizing with something external*, not for derivation. + +```typescript +// ❌ two render passes per change: render → effect → setState → render again +const [visible, setVisible] = useState([]) +useEffect(() => { setVisible(items.filter((t) => !t.hidden)) }, [items]) + +// ✅ derive during render — one pass, no state to drift out of sync +const visible = useMemo(() => items.filter((t) => !t.hidden), [items]) +``` + +The React docs call this out directly: +[You Might Not Need an Effect](https://react.dev/learn/you-might-not-need-an-effect). + +### 3a. Cascading effect chains + +The same mistake compounded: effect A sets state, which triggers effect B, which sets +state, which triggers effect C. Each link is a full extra render pass *and* a window where +the UI shows an inconsistent intermediate combination. + +**Fix:** collapse the chain into render-time derivation — one `useMemo` per step, or one +for the lot. + +## 4. Missing timer cleanup + +Every `setInterval` and recurring `setTimeout` started in an effect must be cleared in its +cleanup. Otherwise the timer outlives unmount, fires against dead state, and leaks in +proportion to how often the component mounts. + +```typescript +// ❌ BROKEN: timer leaks after unmount +useEffect(() => { setInterval(poll, 1000) }, []) + +// ✅ FIXED +useEffect(() => { + const id = setInterval(poll, 1000) + return () => clearInterval(id) +}, []) +``` + +## 5. Uncancelled async work + +Async work started in an effect can resolve *after* unmount — or after the input changed, +letting a stale response overwrite a newer one. + +```typescript +// ❌ fetch races unmount; stale data can win +useEffect(() => { fetchMeta(address).then(setMeta) }, [address]) + +// ✅ cancelled flag — cheapest, works for any promise +useEffect(() => { + let cancelled = false + fetchMeta(address).then((m) => { if (!cancelled) setMeta(m) }) + return () => { cancelled = true } +}, [address]) + +// ✅ AbortController — also cancels the request itself +useEffect(() => { + const ctrl = new AbortController() + fetch(url, { signal: ctrl.signal }) + .then((r) => setData(r)) + .catch((e) => { if (e.name !== 'AbortError') throw e }) + return () => ctrl.abort() +}, [url]) +``` + +Pick one and apply it consistently. + +## Why these matter + +- **Renders.** Derived-state effects double every render in the affected subtree, and + chains multiply it. +- **Memory.** Uncleared timers and subscriptions leak proportional to mount count. +- **Correctness.** Uncancelled async work produces "state update on an unmounted + component" warnings and, worse, races where an older response overwrites a newer one. + +## Don't over-correct + +- Don't add `useMemo`/`useCallback` everywhere — only where profiling shows wasted work, or + where a memoized child depends on the reference. Compilers handle many cases on opted-in + paths. +- A `JSON.stringify` on a cold path with a small object is acceptable. Prioritize hot render + paths. + +## Related + +- `render-cascade` — how effect-driven re-renders propagate through the component graph. +- `selector-antipatterns` — the store-side counterpart; an unstable selector result is a + common source of the unstable dependency in pattern 1. diff --git a/domains/performance/knowledge/metrics-pipeline-design.md b/domains/performance/knowledge/metrics-pipeline-design.md new file mode 100644 index 00000000..03821ef5 --- /dev/null +++ b/domains/performance/knowledge/metrics-pipeline-design.md @@ -0,0 +1,67 @@ +--- +name: metrics-pipeline-design +domain: performance +description: Four-layer metric pipeline architecture for E2E benchmarks, with domain-specific statistical bounds and split reporting paths. +--- + +# Metrics Pipeline Design + +Architecture for adding metric types to an E2E benchmark suite. Separates collection, running, statistics, and reporting into independent layers. + +## Architecture + +``` +Collector → Runner → Statistics → Reporter +``` + +| Layer | Responsibility | +|-------|----------------| +| **Collector** | Extract raw metric from browser/extension per iteration | +| **Runner** | Per-iteration capture + aggregation orchestration | +| **Statistics** | Domain-specific filtering, outlier detection, percentiles | +| **Reporter** | Per-run spans (for quality gate comparison) + aggregated structured logs (for dashboards) | + +Flow files call the collector and return snapshots alongside timers. No flow file does statistics or reporting. + +## Adding a New Metric Type + +1. **Create collector** — function returning typed snapshot with nullable fields for unobserved metrics +2. **Define types** — per-run snapshot, aggregated (reuse `TimerStatistics` for numeric fields), summary +3. **Add domain-specific bounds** — each numeric field gets `{ min, max, allowZero }` +4. **Wire into runner** — collect alongside timers, call aggregation +5. **Add reporter** — per-run spans with `setMeasurement`, aggregated summary as structured log + +## Domain-Specific Statistical Bounds + +Generic timer bounds (1ms–120s, zero=invalid) silently discard valid data from other domains. + +```typescript +// WRONG: CLS values (0–1) all rejected by min=1ms floor +const result = filterBySanityChecks(clsValues); // → empty array + +// RIGHT: per-metric bounds +const BOUNDS = { + inp: { min: 1, max: 30_000, allowZero: false }, // ms + lcp: { min: 1, max: 60_000, allowZero: false }, // ms + cls: { min: 0, max: 10, allowZero: true }, // unitless ratio +}; +``` + +**Rule:** When adding a new metric type, verify whether existing `filterBySanityChecks` assumptions (ms units, zero=invalid) hold. If not, define metric-specific bounds. + +`allowZero` is the critical distinction: CLS=0 means perfect stability (valid); timer=0ms means measurement error (invalid). + +## Split Reporting Path + +| Data | Mechanism | Rationale | +|------|-----------|-----------| +| Aggregated statistics (mean, p75, p95) | Structured log | Low cardinality, dashboard-friendly | +| Per-run snapshots | Sentry spans + `setMeasurement` | Preserves granularity, enables quality gate comparison via Mann-Whitney U | + +`tracesSampleRate: 1.0` required in CI so all per-run spans are captured. + +## SDK Isolation Pattern + +When CI benchmark scripts run in Node but the extension uses a browser SDK (e.g. `@sentry/node` vs `@sentry/browser`): these never share a process. The package manager resolves separate versions per dependency tree. No compatibility issue — they are fully isolated under different lockfile entries. + +Risk: a shared module accidentally importing from the wrong SDK at bundle time. Mitigation: keep the CI SDK as a devDependency excluded from extension builds. diff --git a/domains/performance/knowledge/render-cascade.md b/domains/performance/knowledge/render-cascade.md new file mode 100644 index 00000000..e872081f --- /dev/null +++ b/domains/performance/knowledge/render-cascade.md @@ -0,0 +1,68 @@ +--- +name: render-cascade +domain: performance +description: React+Redux render cascade failure mode — single state change triggers multiple re-render cycles +--- + +# Render Cascade + +Single state change → broken selector returns new reference → `useSelector` detects "change" → parent re-renders all children → children trigger more selectors → cycle repeats 5+ times before stabilizing. + +## Cost Scaling + +| Factor | Impact | +|--------|--------| +| Component tree depth | Each level multiplies re-renders | +| User data size | O(n) selectors × n items = O(n²) operations | +| State update frequency | Background polling compounds the problem | + +Power users (large datasets, many accounts/tokens/transactions) are disproportionately affected. + +## Root Causes + +| Cause | Pattern | Fix | +|-------|---------|-----| +| Plain function selector | `export function get...` | Wrap in `createSelector` | +| Identity function selector | Transform in input, identity in result | Move transform to result function | +| Unnecessary deep equality | `createDeepEqualSelector` on stable Immer inputs | Use `createSelector` | +| O(n) lookup | `.find()` in selector | Normalize state to map; use direct access | +| Chained transforms (unmemoized) | Multiple `.map`/`.filter` in plain function | Single `createSelector` with all transforms | +| Context provider instability | `` inline | `useMemo` the value | +| Props recreation | `useParams()` passed directly as prop | `useMemo` the props object | + +## Selector Creator Decision Tree + +``` +Is INPUT unstable (not from Immer/Redux)? +├── YES → createDeepEqualSelector +└── NO → Is OUTPUT unstable (new array/object from transform)? + ├── YES → createResultEqualSelector (or createShallowResultSelector) + └── NO → createSelector +``` + +## Fix Order — Root Selectors First + +Selectors form a dependency graph. When a root selector returns an unstable reference, the cost cascades: + +- recomputations: **O(m)** — all m dependent selectors recompute +- cascade depth: **O(log m)** — propagates through the tree +- re-renders: **O(m × k)** — each selector triggers k subscribers + +**Fixing downstream selectors is ineffective until the upstream root is stable** — a fixed `getActiveAccount` still receives a new input every render if `getAccounts` is broken. + +``` +getAccountsObject (stable) + └─ getAccounts (broken: returns new array) + ├─ getActiveAccount ├─ getAccountCount └─ getAccountNames … +``` + +Triage the dependency graph top-down; fix roots first. + +## Why Cascade Breaks All Other Optimizations + +| Optimization | Without Cascade Fix | With Cascade Fix | +|---|---|---| +| Virtualization | Parent still re-renders all | Works as intended | +| `React.memo` | Parent defeats it | Works as intended | +| React Compiler | Can't cross file boundaries | Complements selectors | +| `useMemo`/`useCallback` | Recreated on parent render | Stable references | diff --git a/domains/performance/knowledge/selector-antipatterns.md b/domains/performance/knowledge/selector-antipatterns.md new file mode 100644 index 00000000..df65609f --- /dev/null +++ b/domains/performance/knowledge/selector-antipatterns.md @@ -0,0 +1,163 @@ +--- +name: selector-antipatterns +domain: performance +description: The Redux selector patterns that break memoization and cause render cascades — the canonical, platform-agnostic taxonomy that per-repo selector references instantiate +--- + +# Selector Anti-Patterns + +**This file is the single source for the pattern taxonomy.** Per-repo references — such as +the `mm-selector-memoization` reference shipped with the `performance` skill — name these +patterns rather than redefining them, and add what only they can: the codebase's own +selector-creator utilities, verified instances with `file:line`, and fix recipes. + +Every pattern below has the same failure shape: `useSelector` returns a **new reference** +when the underlying data did not change, so every consumer re-renders. One broken selector +near the root of the graph cascades through everything downstream, and the cost scales +superlinearly with user data. + +## 1. Unmemoized selector + +A plain function that allocates. No memoization at all — a new reference on every call. + +```typescript +// ❌ BROKEN +export function getPendingApprovals(state) { + return Object.values(state.pendingApprovals ?? {}); +} + +// ✅ FIXED +const getPendingApprovalsObject = (state) => state.pendingApprovals ?? {}; +export const getPendingApprovals = createSelector( + getPendingApprovalsObject, + (approvals) => Object.values(approvals), +); +``` + +Detection: grep exported `function get…` in the selectors directory. + +## 2. Identity / passthrough result + +The transform happens in the **input** and the result function returns its input unchanged, +so the cache can never hit. A plain `createSelector` only helps when its *inputs* are +reference-stable; controller-state slices usually are not. + +```typescript +// ❌ BROKEN: Object.values() in the INPUT creates a new array each call +export const getAccounts = createSelector( + (state) => Object.values(state.accounts), + (accounts) => accounts, // identity — cache never hits +); + +// ✅ FIXED: stable input, transform in the OUTPUT +export const getAccounts = createSelector( + (state) => state.accounts, // stable structural reference + (accounts) => Object.values(accounts), +); +``` + +Detection: the reselect/Jest warning `"result function returned its own inputs"`. + +## 3. New collection allocated in the result function + +Even a correctly-shaped `createSelector` returns a new reference whenever it recomputes — +and if its inputs are unstable, that is every dispatch. + +```typescript +// ❌ new array/Set/Map/object every call → always "changed" +(accounts) => Object.values(accounts).sort(...) +(transactions) => new Set(transactions.flatMap(...)) +(items) => items.filter(...) +(state) => state.swapsTransactions ?? {} // a fresh {} on every nullish hit +``` + +**Fix:** a deep-equal selector creator (returns the *cached* reference when data is +unchanged), a stable module-level constant for the empty case, or a result-equality check. + +## 4. Mutation in the result function + +```typescript +// ❌ mutates the input array AND returns a corrupting reference +createSelector([getItems], (items) => { items.sort(cmp); return items; }) +``` + +**Fix:** copy first — `[...items].sort(cmp)`. + +## 5. Over-broad input + +`state => state`, or a large slice, as an input selector forces recomputation on **any** +state change anywhere. Narrow the input to the smallest slice that actually feeds the +result. + +## 6. Unnecessary deep equality + +Deep-equal creators cost O(n) per comparison. Reaching for one when the input is already +reference-stable pays that cost for nothing — and deep-comparing a large slice on every +dispatch can be worse than the re-render it prevents. + +```typescript +// ❌ UNNECESSARY: this slice is already reference-stable +const getAccounts = createDeepEqualSelector( + (state) => state.accounts, + (accounts) => transformAccounts(accounts), +); +``` + +Prefer **narrowing the input** over deep-equalizing a giant object. + +## 7. O(n) lookups over unnormalized state + +`.find()` over `Object.values()` is O(n). With n items × m selectors per state change that +is O(n×m) on every dispatch. + +```typescript +// ❌ BROKEN +export const getAccountByAddress = (state, address) => + Object.values(state.accounts).find((a) => a.address === address); + +// ✅ FIXED: normalized state, O(1) access +export const getAccountByAddress = (state, address) => state.accounts[address]; +``` + +## 8. Chained unmemoized transforms + +Each transform allocates. Several in sequence means several new references per call. + +```typescript +// ❌ BROKEN: 3 new arrays per call +export function getSortedItems(state) { + const items = Object.values(state.items); // array 1 + const filtered = items.filter(isVisible); // array 2 + return filtered.sort(byDate); // array 3 +} + +// ✅ FIXED: single memoized output +export const getSortedItems = createSelector( + (state) => state.items, + getFilterCriteria, + (items, criteria) => + Object.values(items).filter((i) => matchesCriteria(i, criteria)).sort(byDate), +); +``` + +## Selector creator decision tree + +``` +Is the INPUT unstable (a fresh object/array every dispatch)? +├── YES → deep-equal selector creator (but prefer narrowing the input first) +└── NO → Is the OUTPUT unstable (a new array/object from the transform)? + ├── YES → result-equality selector creator + └── NO → plain createSelector +``` + +## Don't over-correct + +- A selector returning a **primitive** is fine even if it filters internally — the consumer + memoizes on the primitive value. Wasteful allocation, not a re-render bug. +- Memoization is not free. Prefer narrowing inputs over adding comparison work. + +## Related + +- `render-cascade` — what one broken root selector does to the component graph downstream. +- Per-repo instances: the `mm-selector-memoization` reference documents a codebase's own + selector creators, its verified broken selectors, and the fix recipe for each. diff --git a/domains/performance/knowledge/web-vitals-attribution-import.md b/domains/performance/knowledge/web-vitals-attribution-import.md new file mode 100644 index 00000000..98c0067b --- /dev/null +++ b/domains/performance/knowledge/web-vitals-attribution-import.md @@ -0,0 +1,19 @@ +--- +name: web-vitals-attribution-import +domain: performance +description: web-vitals/attribution is a module import path, not a separate package — no meaningful bundle cost, gives the symptom→cause link +--- + +# Web Vitals Attribution Import + +`web-vitals/attribution` is a **module import path**, not a separate package. The attribution build: +- Provides which script/element caused each metric +- Does **not** meaningfully increase production bundle size (tree-shaking applies) + +Don't skip it for "bundle size" reasons — that's a misread. + +## Why it matters +Attribution is the symptom→cause link: +- INP spike of 500ms +- Attribution: `eventTarget: '#confirm-swap-button'`, `eventType: 'click'` +- Combined with tracing → identifies the controller that blocked diff --git a/domains/performance/knowledge/web-vitals-production-vs-benchmarks.md b/domains/performance/knowledge/web-vitals-production-vs-benchmarks.md new file mode 100644 index 00000000..b6bfdf63 --- /dev/null +++ b/domains/performance/knowledge/web-vitals-production-vs-benchmarks.md @@ -0,0 +1,21 @@ +--- +name: web-vitals-production-vs-benchmarks +domain: performance +description: Web Vitals need different collection in production (web-vitals lib) vs benchmarks (PerformanceObserver); no TBT in prod +--- + +# Web Vitals — Production vs Benchmarks + +Collection differs by environment due to timing constraints. + +## Production — `web-vitals` library +- Reports on `visibilitychange` / `pagehide` +- Handles browser quirks, bfcache, session windowing +- Attribution build shows which element/script caused the metric +- Metrics: **INP, LCP, CLS** (not TBT) + +**Why no TBT in production:** TBT is cumulative and unbounded — it grows indefinitely over an open-ended session. INP is per-interaction → meaningful for real users. TBT fits bounded flows (benchmarks), not sessions. + +## Benchmarks — direct `PerformanceObserver` +- Query on demand (not dependent on page hide) +- Fits an existing `collectMetrics()` pattern diff --git a/domains/performance/knowledge/web-vitals-runtime-metrics.md b/domains/performance/knowledge/web-vitals-runtime-metrics.md new file mode 100644 index 00000000..e05d7c55 --- /dev/null +++ b/domains/performance/knowledge/web-vitals-runtime-metrics.md @@ -0,0 +1,22 @@ +--- +name: web-vitals-runtime-metrics +domain: performance +description: Core Web Vitals (INP, TBT) are runtime responsiveness metrics, not just page-load — high-value for extension UX gates +--- + +# Web Vitals as Runtime Metrics + +Core Web Vitals (INP, TBT) measure **runtime responsiveness**, not just page load. For a browser extension this distinction is critical. + +- **Page load is less relevant** — the popup opens fast; there's no traditional navigation. +- **Runtime interactions matter** — every button click, form submit, confirmation. INP and TBT measure responsiveness during interactions → high-value for extension UX quality gates. + +## Orthogonal to distributed tracing + +| | Web Vitals | Distributed Tracing | +|---|---|---| +| Question | "How did the user perceive it?" | "Which controller caused it?" | +| Scope | user perception | operation attribution | +| Granularity | per-interaction aggregate | per-operation breakdown | + +Use both — perception (web vitals) + attribution (tracing) — not one instead of the other. diff --git a/domains/performance/skills/data-analysis/skill.md b/domains/performance/skills/data-analysis/skill.md new file mode 100644 index 00000000..4566045d --- /dev/null +++ b/domains/performance/skills/data-analysis/skill.md @@ -0,0 +1,164 @@ +--- +maturity: experimental +name: data-analysis +description: Structured approach for analyzing metrics, attributing changes, and communicating findings — five phases (collection → filtering → curation → questioning → synthesis), confidence assignment, audience-appropriate artifacts +--- + +# Data Analysis Skill + +Structured approach for analyzing metrics, attributing changes, and communicating findings. + +--- + +## When to Use + +- Performance analysis from production metrics +- Attribution of improvements/regressions to code changes +- Creating executive summaries or stakeholder communications +- Any analysis requiring correlation of changes to measured outcomes + +--- + +## Quick Reference + +### Five Phases + +``` +Collection → Filtering → Curation → Questioning → Synthesis +``` + +| Phase | Key Question | Output | +| ----------- | --------------------------- | ----------------------------------- | +| Collection | What are we measuring? | Baseline, scope, change list | +| Filtering | What's signal vs. noise? | Categorized changes with confidence | +| Curation | What correlates with what? | Attribution table | +| Questioning | Do we KNOW or BELIEVE this? | Validated claims with caveats | +| Synthesis | Who needs to know what? | Audience-appropriate artifacts | + +### Confidence Assignment + +| Level | Use When | +| ---------- | ---------------------------------------------------------------- | +| **High** | Clear mechanism + timing alignment + targets measured population | +| **Medium** | Plausible mechanism but confounded by other changes | +| **Low** | Speculative or enabling-only | + +### Attribution Table Template + +| Change | Evidence | Release | Metric | Confidence | Notes | +| ------------- | ----------- | --------- | ----------------- | ------------ | --------------------- | +| [Description] | [PR/commit] | [version] | [affected metric] | High/Med/Low | [mechanism or caveat] | + +--- + +## Process + +### 1. Collection + +```markdown +**Metrics:** [What are you measuring?] +**Population:** [Who? All users, p75, specific cohort?] +**Period:** [Measurement window - release tags or dates] +**Source:** [APM, logs, synthetic benchmarks?] +**Baseline:** [Starting values with methodology] +``` + +Enumerate ALL changes in scope: + +- Code changes (PRs, commits) +- Config changes +- External factors (traffic, user growth, infrastructure) + +### 2. Filtering + +Categorize each change: + +- **Direct:** Clear causal path to measured metric +- **Indirect:** Enabling infrastructure (value materializes later) +- **Unknown:** In scope but mechanism unclear +- **Noise:** Unlikely to affect measured metrics + +### 3. Curation + +Build attribution table: + +1. Map changes to metric movements by release +2. Note co-landed changes (shared attribution) +3. Flag anomalies (improvement without cause, unexplained regression) +4. Separate measured vs. post-cutoff work + +### 4. Questioning + +Challenge every attribution: + +- [ ] "Do we KNOW this, or do we BELIEVE this?" +- [ ] "What would need to be true for this to be wrong?" +- [ ] "Are there alternative explanations?" + +Document what's missing: + +- [ ] Unexplained improvements +- [ ] Unexplained regressions +- [ ] Work that SHOULD have helped but didn't +- [ ] Metrics you wish you had + +### 5. Synthesis + +Create audience-appropriate artifacts: + +| Artifact | Audience | Focus | +| --------------------- | --------------- | ------------------------------------- | +| Executive Summary | Leadership | Hard data, key wins, team recognition | +| Attribution Catalogue | Engineering | Detailed per-change analysis | +| Methodology Doc | Future analysts | Process, assumptions, data sources | +| Communication Post | Stakeholders | Exciting but honest, caveats visible | + +--- + +## Communication Template + +```markdown +**[Metric]: [Before] → [After] ([Change %])** + +Population: [Who this measures] +Caveat: [Key limitation] +What's NOT included: [Equally interesting gaps] + +Notable contributors: + +- [Change 1] — [mechanism] +- [Change 2] — [mechanism] + +Bottom line: [One sentence impact statement] +``` + +--- + +## Anti-Patterns + +| Don't | Do Instead | +| ---------------------------------------- | ------------------------------------------------ | +| Claim causation from correlation | "Correlates with" or "plausible contributor" | +| Attribute release total to single change | Note multiple changes, unknown isolated impact | +| Bury caveats in footnotes | Caveats are part of the story | +| Use superlatives without data | Let numbers speak | +| Hide uncertainty | Use qualifiers: "likely," "plausible," "unknown" | + +--- + +## Checklist + +Before finalizing: + +- [ ] Measurement methodology documented +- [ ] Baseline values recorded with source +- [ ] All changes in scope enumerated +- [ ] Confidence levels assigned with justification +- [ ] Unexplained anomalies noted +- [ ] Limitations explicitly stated +- [ ] What's NOT included documented +- [ ] Uncertainty reflected in language +- [ ] Links/references for all claims +- [ ] Multiple artifacts for different audiences + +--- diff --git a/domains/performance/skills/effect-antipattern-scan/repos/metamask-extension.md b/domains/performance/skills/effect-antipattern-scan/repos/metamask-extension.md new file mode 100644 index 00000000..ee5fb3b2 --- /dev/null +++ b/domains/performance/skills/effect-antipattern-scan/repos/metamask-extension.md @@ -0,0 +1,27 @@ +--- +repo: metamask-extension +parent: effect-antipattern-scan +--- + +## Paths + +- Component sources: [`ui/`](https://github.com/MetaMask/metamask-extension/tree/main/ui) +- Shared hooks: [`ui/hooks/`](https://github.com/MetaMask/metamask-extension/tree/main/ui/hooks) + +## Commands + +```bash +# Pattern 1: JSON.stringify in deps +grep -rnE 'useEffect\([^)]*\[.*JSON\.stringify' ui/ --include="*.ts" --include="*.tsx" + +# Pattern 3: setInterval / setTimeout +grep -rnE 'setInterval|setTimeout' ui/ --include="*.ts" --include="*.tsx" + +# Pattern 4: fetch inside useEffect (manual review required for context) +grep -rn 'fetch(' ui/ --include="*.ts" --include="*.tsx" +``` + +## Reference Docs + +- [Frontend Performance Optimization Guidelines](https://github.com/MetaMask/contributor-docs/pull/159) (contributor-docs PR #159) +- [You Might Not Need an Effect](https://react.dev/learn/you-might-not-need-an-effect) diff --git a/domains/performance/skills/effect-antipattern-scan/repos/metamask-mobile.md b/domains/performance/skills/effect-antipattern-scan/repos/metamask-mobile.md new file mode 100644 index 00000000..b005c307 --- /dev/null +++ b/domains/performance/skills/effect-antipattern-scan/repos/metamask-mobile.md @@ -0,0 +1,31 @@ +--- +repo: metamask-mobile +parent: effect-antipattern-scan +--- + +## Paths + +- Component sources: [`app/`](https://github.com/MetaMask/metamask-mobile/tree/main/app) +- Shared hooks: [`app/component-library/hooks/`](https://github.com/MetaMask/metamask-mobile/tree/main/app/component-library/hooks) + +## Commands + +```bash +# Pattern 1: JSON.stringify in deps +grep -rnE 'useEffect\([^)]*\[.*JSON\.stringify' app/ --include="*.ts" --include="*.tsx" + +# Pattern 3: setInterval / setTimeout +grep -rnE 'setInterval|setTimeout' app/ --include="*.ts" --include="*.tsx" + +# Pattern 4: fetch inside useEffect (manual review required for context) +grep -rn 'fetch(' app/ --include="*.ts" --include="*.tsx" +``` + +## Differences from Extension + +- Prefer `AbortController` for all new async effects. No shared `useIsMounted` hook exists. +- React Native's `fetch` behaves identically to browser `fetch` for cancellation purposes. + +## Reference Docs + +- [You Might Not Need an Effect](https://react.dev/learn/you-might-not-need-an-effect) diff --git a/domains/performance/skills/effect-antipattern-scan/skill.md b/domains/performance/skills/effect-antipattern-scan/skill.md new file mode 100644 index 00000000..f890fcc8 --- /dev/null +++ b/domains/performance/skills/effect-antipattern-scan/skill.md @@ -0,0 +1,55 @@ +--- +maturity: experimental +name: effect-antipattern-scan +description: Review PR diffs that add or modify `useEffect` for the systemic React effect antipatterns +--- + +# Effect Anti-Pattern Review + +**Scope:** Pre-merge review of PRs that add or modify `useEffect` calls. The workflow is a grep-driven checklist against the patterns catalogued in the **`effect-antipatterns`** knowledge file, which is the single source for their definitions and fixes (installed alongside this skill under `knowledge/`). + +Applies to both `metamask-extension` and `metamask-mobile`. See overlays for repo-specific paths. + +## When To Use + +- Reviewing a PR that adds or modifies a `useEffect` call +- Reviewing a PR that adds `setInterval`, `setTimeout`, `fetch`, or `addEventListener` inside a component +- Investigating a "Can't perform a React state update on an unmounted component" warning + +## Do Not Use When + +- Reviewing selector or render-cascade issues (use [`selector-antipattern-scan`](../selector-antipattern-scan/skill.md)) +- Reviewing non-React code (background scripts, workers, test utilities) +- Reviewing an effect that is intentionally one-shot with no async work or timers (check patterns below anyway, but most do not apply) + +## Workflow + +1. **List changed files with `useEffect`.** `git diff --name-only origin/main...HEAD | xargs grep -l 'useEffect'` +2. **Run the [grep checklist](#grep-checklist)** against the changed files. +3. **For each hit, map to a pattern** in `effect-antipatterns` and apply the fix from the knowledge file. +4. **Block on unstable dependency identity.** `JSON.stringify` in a dependency array is always broken. Do not merge. +5. **Block on a timer without cleanup.** Any `setInterval` / `setTimeout` without a matching `clearInterval` / `clearTimeout` in the cleanup function is blocking. +6. **Require cancellation for async effects.** Any `fetch` / network call inside `useEffect` must use `AbortController`. + +## Grep Checklist + +| Pattern (`effect-antipatterns` §) | Detection | +|---|---| +| §1 Unstable dependency identity | `grep -rnE 'useEffect.*\[.*JSON\.stringify' `, plus inline `{`/`[` literals in the dep position | +| §2 Wrong dependencies | Hand review — empty deps that read state (stale closure), or deps that read nothing | +| §3 Derived state via effect + setState | Hand review — `useEffect` that calls `setX` from other state/props; §3a for chains of them | +| §4 Missing timer cleanup | `grep -rnE 'setInterval\|setTimeout' ` then check each effect returns a cleanup | +| §5 Uncancelled async work | `grep -rnB2 -A10 'fetch\(' ` within `useEffect` blocks | + +See the repo overlay for the concrete `` path. + +## Common Pitfalls + +| Mistake | Correct approach | +|---|---| +| Accept `JSON.stringify` in deps because "the effect needs to rerun when X changes" | Destructure to primitives or `useMemo` the object — never stringify | +| Accept a state-mirror effect because "the computation is expensive" | Use `useMemo` for expensive derivations. Effects are for side effects, not state derivation | +| Let `setInterval` ship without cleanup because "the component rarely unmounts" | Cleanup is non-negotiable — unmount frequency doesn't matter, correctness does | +| Treat "can't perform state update on unmounted component" as a cosmetic warning | It is a data race. An old response can overwrite a new one | +| Add a lint rule disable on `react-hooks/exhaustive-deps` | Almost always wrong. Destructure or memoize instead | +| Refactor toward `useEffect` + `setState` because it "feels like state" | You probably do not need an effect. See [You Might Not Need an Effect](https://react.dev/learn/you-might-not-need-an-effect) | diff --git a/domains/performance/skills/extension-profiling/skill.md b/domains/performance/skills/extension-profiling/skill.md new file mode 100644 index 00000000..c2a80272 --- /dev/null +++ b/domains/performance/skills/extension-profiling/skill.md @@ -0,0 +1,65 @@ +--- +maturity: experimental +name: extension-profiling +description: Compare browser extension performance between branches using WDYR, React DevTools Profiler, and E2E benchmarks with statistical rigor. +--- + +# Browser Extension Profiling + +Methodology for profiling and comparing extension performance across branches or commits. + +## When To Use + +- Validating that a refactor reduces unnecessary re-renders (needs before/after comparison) +- Establishing baseline metrics for a performance initiative +- Investigating a reported UI slowdown in the extension + +## Do Not Use When + +- Single-run comparisons — statistical significance requires ≥10 runs per scenario +- The change touches only non-render paths (background scripts, network with no UI impact) +- Target behavior is server-side latency, not UI rendering + +## Workflow + +1. **Build both branches** with `yarn build:test` on the same machine and Chrome version + +2. **WDYR profiling** (unnecessary re-render counts) + ```bash + ENABLE_WHY_DID_YOU_RENDER=true yarn start + ``` + Flags to watch: + - `different objects that are equal by value` → object recreation + - `different functions with the same name` → callback recreation + - `props object itself changed but values equal` → parent cascade + +3. **React DevTools Profiler** for flame graphs and commit timings + ```bash + yarn devtools:react + ``` + +4. **E2E benchmarks** for scenario durations + ```bash + yarn test:e2e:benchmark + ``` + +5. **Collect ≥10 runs** per scenario. Discard top/bottom 10%. Report mean, median, stddev, p75, p95. + +6. **Statistical threshold:** Cohen's d > 0.5 for a meaningful difference. + +## Common Pitfalls + +| Mistake | Correct Approach | +|---------|-----------------| +| Running branches on different machines or Chrome versions | Same machine, same Chrome, no other apps running | +| Pooling all runs including noisy late-session ones | Compute per-round stats first; report cleanest signal with explicit round attribution | +| Reporting absolute re-render counts without scenario context | Normalize per-action; cascade fixes show multiplied impact at root | +| Skipping cache and state reset between runs | Clear browser cache, reset extension state for each run | + +## Pre-Profiling Checklist + +- [ ] Both branches built with `yarn build:test` +- [ ] Same machine, same Chrome version +- [ ] No other tabs or applications running +- [ ] WDYR enabled: `ENABLE_WHY_DID_YOU_RENDER=true` +- [ ] Cache and extension state cleared between runs diff --git a/domains/performance/skills/performance/references/mm-audit-playbook.md b/domains/performance/skills/performance/references/mm-audit-playbook.md index b3f302c7..ff756989 100644 --- a/domains/performance/skills/performance/references/mm-audit-playbook.md +++ b/domains/performance/skills/performance/references/mm-audit-playbook.md @@ -12,6 +12,7 @@ For reviewing a PR/diff or auditing a file, component, or feature. Output: findi - **Targeted** (single file / component / small diff): read the files and report concrete findings with `file:line`. - **Broad** (whole feature / repo): run the grep sweeps below and triage hits; don't read everything. +- **Audit wave / program** (scheduled audit of a surface or division): per-surface audits miss mechanism-level patterns that live in *shared* infrastructure (`app/selectors`, shared hooks, the store) — run the cross-cutting sweeps below over the shared dirs **once per wave**, not once per team, and route findings to surface owners. Attach quantified acceptance criteria up front (template in [mm-planning.md](mm-planning.md)). If the surface ships on both platforms, cross-check the sibling platform's audit findings for the same surface before fresh discovery — the React/Redux mechanism patterns recur across extension and mobile. Always: **measure before asserting impact** where feasible, and respect the guardrails at the bottom (don't over-flag). @@ -42,8 +43,10 @@ Read the call sites: is a data hook running for tabs/pages/items that aren't vis grep -rn "createSelector(" app/selectors --include="*.ts" | grep -v createDeepEqualSelector grep -rn "=> .*\.\(map\|filter\|sort\|reverse\)\|new Set\|new Map\|Object\.\(values\|keys\|entries\)\|?? {}\|?? \[\]" app/selectors --include="*.ts" grep -rn "\.sort(\|\.reverse(\|\.push(\|\.splice(" app/selectors --include="*.ts" # mutation +grep -rn "(_state\|(_," app/selectors --include="*.ts" # parameterized selectors — single-entry cache → mm-state-normalization.md +grep -rnE "export (function|const) (get|select)[A-Z][A-Za-z]* = \(state|export function (get|select)" app/selectors --include="*.ts" # plain unmemoized function selectors (no createSelector at all) ``` -Check each result function for: identity/passthrough, new collection without deep-equal, mutation, `state=>state` input. +Check each result function for: identity/passthrough, new collection without deep-equal, mutation, `state=>state` input. If one broken selector has **many consumers**, switch to the cascade playbook — map the dependency tree to closure and plan the fix order *before* fixing anything: [mm-selector-cascade.md](mm-selector-cascade.md). ### Redux / useSelector → [mm-redux-antipatterns.md](mm-redux-antipatterns.md) ```bash @@ -57,11 +60,18 @@ grep -rn "dispatch(" app --include="*.ts" --include="*.tsx" | grep -v ".test." | grep -rn "Provider value={{" app --include="*.tsx" | grep -v ".test." ``` -### Hooks → [mm-hook-dependency-arrays.md](mm-hook-dependency-arrays.md) +### Hooks → [mm-hook-dependency-arrays.md](mm-hook-dependency-arrays.md) / [mm-useeffect-antipatterns.md](mm-useeffect-antipatterns.md) ```bash grep -rn "\[JSON.stringify\|, JSON.stringify" app --include="*.ts" --include="*.tsx" | grep -v ".test." +grep -rn -A6 "useEffect(" app --include="*.ts" --include="*.tsx" | grep -E "fetch\(|\.then\(" | grep -v "signal\|cancelled\|abort" | grep -v ".test." # async effects without cancellation ``` -(`exhaustive-deps` is NOT linted in this repo — check effect deps by hand.) +(`exhaustive-deps` is NOT linted in this repo — check effect deps by hand.) For effect-body problems — derived state via useEffect+setState, effect chains, post-unmount setState — use the read pass in [mm-useeffect-antipatterns.md](mm-useeffect-antipatterns.md). + +### React Compiler coverage → [mm-react-compiler-error-triage.md](mm-react-compiler-error-triage.md) +```bash +grep -rn "use no memo" app --include="*.ts" --include="*.tsx" # opt-outs: each needs a reason + TODO +``` +For a re-render-heavy screen, confirm the components are actually **compiled** (`Memo ✨` in DevTools) before suggesting manual memoization — they may be sitting in the error/unsupported bucket. ### Animations → [mm-layout-animations.md](mm-layout-animations.md) ```bash @@ -97,6 +107,7 @@ Grep finds *syntactic* patterns. The highest-impact re-render bugs are *data-flo - **Render-phase side effects / setState** — any `setState(...)`, `dispatch(...)`, or `trackEvent(...)` in a render body (not inside `useEffect`/`useCallback`)? Triggers extra render passes. - **O(n²) reduce-with-spread** — `reduce((acc, x) => ({ ...acc, ... }), {})` rebuilt every render. - **Per-item subscription hooks** — trace each into its manager; shared subscription = fine, per-subscriber whole-dataset snapshot = bug. → [mm-streaming-realtime.md](mm-streaming-realtime.md) +- **Deep-equal selector inputs** — for every `createDeepEqualSelector`, read its *input selectors*: an input function that allocates a fresh composite per call (object spreads of controller state, other selectors' results collected into a new object) forces the deep compare to run over the whole composite on every check — and no result-function grep catches it. `grep -rn -B3 "createDeepEqualSelector(" app/selectors` lists the sites; read each first argument. → [mm-selector-cascade.md](mm-selector-cascade.md) (proactive mode) Confirm any hit with the Profiler ("why did this render?") before asserting — see [mm-tools.md](mm-tools.md). @@ -107,6 +118,8 @@ Confirm any hit with the Profiler ("why did this render?") before asserting — - [ ] No real-time / high-frequency data dispatched to Redux - [ ] `Context.Provider value` is memoized (not an inline object) - [ ] No `JSON.stringify` in a hot dependency array +- [ ] Async effects guard against post-unmount / stale setState (cancelled flag or `AbortController`) +- [ ] No new parameterized selector (single-entry cache) on a list/hot path — use a lookup-map selector instead - [ ] Layout animations use Reanimated v3, not `Animated` + `useNativeDriver:false` - [ ] Growable lists use FlashList with stable keys (+ `getItemType` if mixed) - [ ] New event listeners / timers / subscriptions have cleanup diff --git a/domains/performance/skills/performance/references/mm-hook-dependency-arrays.md b/domains/performance/skills/performance/references/mm-hook-dependency-arrays.md index 5733ff81..117cc1dd 100644 --- a/domains/performance/skills/performance/references/mm-hook-dependency-arrays.md +++ b/domains/performance/skills/performance/references/mm-hook-dependency-arrays.md @@ -8,6 +8,8 @@ tags: useEffect, useMemo, useCallback, dependencies, JSON.stringify Dependency arrays decide when `useEffect`/`useMemo`/`useCallback` re-run. The most common MetaMask problem is **`JSON.stringify` inside a dependency array** — it runs a synchronous serialization on every render just to compute the dependency key, which is both expensive and a sign the upstream reference is unstable. +> **Scope.** This file is the *dependency* half of effect performance, instantiated for this codebase. The platform-agnostic taxonomy — unstable dependency identity, wrong dependencies, derived state via effect, cascading effect chains, missing cleanup, uncancelled async — lives in the **`effect-antipatterns`** knowledge file, installed alongside this skill under `knowledge/`. Read that for the general shape; read this for the verified instances and the repo's lint gaps. + ## Pattern — `JSON.stringify` in deps ```ts @@ -83,5 +85,6 @@ For each hit, ask: *does this dependency change identity every render?* If yes, ## Related +- [mm-useeffect-antipatterns.md](mm-useeffect-antipatterns.md) — the effect-body side: derived state, effect chains, unmount-safe async, cleanup - [js-react-compiler.md](js-react-compiler.md) / [mm-react-compiler.md](mm-react-compiler.md) — automatic memoization on opted-in paths - [js-concurrent-react.md](js-concurrent-react.md) — defer expensive derived work diff --git a/domains/performance/skills/performance/references/mm-planning.md b/domains/performance/skills/performance/references/mm-planning.md index ca23c644..3a610685 100644 --- a/domains/performance/skills/performance/references/mm-planning.md +++ b/domains/performance/skills/performance/references/mm-planning.md @@ -20,9 +20,9 @@ The cheapest performance fix is the one you make before writing code. Catch arch | Risk | Trigger question | Default mitigation | |---|---|---| | Real-time / WebSocket data | Updates faster than once per user action? | Never put it in Redux. Local state / shared value / direct UI update. Manage subscribe/unsubscribe by visibility + app foreground/background; avoid double-subscribe. See [mm-redux-antipatterns.md](mm-redux-antipatterns.md). | -| Unbounded data | Can the list/dataset grow without ceiling? | Paginate + virtualize from day one; plan server-side filtering. | +| Unbounded data | Can the list/dataset grow without ceiling? | Paginate + virtualize from day one; plan server-side filtering. Never persist unbounded data via redux-persist — use a dedicated storage layer. | | Large lists | >~50 items now, infinite later? | FlashList v2 with stable keys + `getItemType`; no heavy work per item. [js-lists-flatlist-flashlist.md](js-lists-flatlist-flashlist.md) | -| New selector / derived state | Adding `createSelector`? | Decide memoization + equality up front; never identity/mutation. [mm-selector-memoization.md](mm-selector-memoization.md) | +| New selector / derived state | Adding `createSelector`? | Decide memoization + equality up front; never identity/mutation. [mm-selector-memoization.md](mm-selector-memoization.md). Frequent keyed lookups? Decide the lookup shape now (keyed index vs O(n) scan) — [mm-state-normalization.md](mm-state-normalization.md) | | Heavy computation | Big transforms, sorts, regex on large input? | Server offload, or memoize, or defer with `useDeferredValue`. | | Crypto | Hashing/signing/derivation in hot path? | `react-native-quick-crypto` (already installed); keep off the JS thread. | | New npm dependency | Adds to `package.json`? | Check size (Expo Atlas / bundlephobia); avoid main-package/barrel imports; reuse existing libs (we already have dayjs, luxon, lodash). [bundle-library-size.md](bundle-library-size.md) | @@ -34,7 +34,7 @@ The cheapest performance fix is the one you make before writing code. Catch arch ## System-design checklist -- **State shape:** new Redux slice for real-time data? → flag. New selector? → memoization + equality decided now. +- **State shape:** new Redux slice for real-time data? → flag. New selector? → memoization + equality decided now. Frequent lookups by key? → plan a `byId`/`byAddress` index ([mm-state-normalization.md](mm-state-normalization.md)). - **Subscription lifecycle:** diagram subscribe/unsubscribe tied to mount/unmount + foreground/background; no double-subscribe; cleanup guaranteed. - **List strategy:** ScrollView only for <20 fixed items; FlashList for anything that can grow; no `.map()` in JSX for growable lists. - **Data flow:** minimize how many components subscribe to a frequently-updating selector. diff --git a/domains/performance/skills/performance/references/mm-react-compiler-error-triage.md b/domains/performance/skills/performance/references/mm-react-compiler-error-triage.md new file mode 100644 index 00000000..68db17f8 --- /dev/null +++ b/domains/performance/skills/performance/references/mm-react-compiler-error-triage.md @@ -0,0 +1,90 @@ +--- +title: React Compiler Error Triage & Coverage Accounting (MetaMask) +impact: HIGH +tags: react-compiler, panicThreshold, error-triage, coverage, babel, build +--- + +# Skill: React Compiler Error Triage & Coverage Accounting + +The React Compiler **fails open**: when it can't compile a component, it silently skips it and ships the unoptimized original. The build stays green, DevTools shows no warning — you just don't get the memoization. Once the compiler is enabled broadly (metamask-mobile#31171 enabled v1.0.0 app-wide), the question stops being "is it on?" and becomes **"what is it actually compiling, and which of its errors are worth fixing?"** This file is the triage playbook. Extension PR metamask-extension#38007 is the reference implementation. + +## The `panicThreshold` ladder + +`panicThreshold` controls when a compiler diagnostic fails the build instead of silently skipping the file: + +| Setting | Build fails on | Use for | +|---|---|---| +| `'none'` (default) | never — every failed file is **silently skipped** | production builds, always | +| `'critical_errors'` | only critical errors (compiler-internal invariant violations) | CI / debug builds, first ratchet target | +| `'all_errors'` | every diagnostic, including unsupported syntax | CI / debug builds, end-state ratchet | + +**The ratchet strategy** (extension roadmap, MetaMask-planning#6552 → #6553): keep production at `'none'` permanently; aim for a *non-production* build that passes at `'critical_errors'`, then at `'all_errors'`. Each ratchet step turns a class of silent skips into a visible, fixable error list. Never enable a non-`'none'` threshold in a release build — one un-compilable file would block the release for an optimization that is optional by design. + +## Triage: unsupported syntax vs. legitimate errors + +Compiler diagnostics are **not one bucket**. The logger event's `category` field separates them, and the distinction decides whether you act: + +- **`category === 'Todo'` → "unsupported."** Syntax or a pattern the compiler *itself* has not implemented yet. There is **no actionable fix on our side** — rewriting working code to appease an unimplemented compiler path is wasted effort and churn. Count these separately, leave the code alone, and re-check after compiler upgrades. +- **Any other category (e.g. `InvalidReact`, `InvalidJS`) → legitimate, actionable.** A real Rules-of-React violation in our code (mutation during render, conditional hooks, side effects in render). Fixing it both unlocks compilation *and* removes a latent correctness bug. + +A healthcheck that doesn't make this split is noise: the `Todo` count swamps the actionable list and the team learns to ignore the output. The extension's verbose run at enablement (metamask-extension#38007) is the canonical illustration — of 7,308 files processed: 253 compiled, **31 actionable errors**, **7,024 unsupported** (`Todo`). Without the split that reads as ~7,000 hopeless errors; with it, the team's backlog is 31 files and the rest is the compiler's to burn down across upgrades. The extension's webpack wrapper makes the split in ~10 lines: + +```ts +// adapted from metamask-extension development/webpack/utils/loaders/reactCompilerLoaderWrapper.ts +// (mobile equivalent: pass a `logger` in babel-plugin-react-compiler options) +logger: { + logEvent(filename, event) { + switch (event.kind) { + case 'CompileSuccess': record(filename, 'compiled'); break; + case 'CompileSkip': record(filename, 'skipped'); break; + case 'CompileError': { + const category = event.detail?.options?.category ?? event.detail?.category; + // 'Todo' = not yet supported by the compiler — no actionable fix on our side + record(filename, category === 'Todo' ? 'unsupported' : 'error'); + break; + } + } + }, +} +``` + +The extension exposes this as `yarn webpack --reactCompilerVerbose` (per-file ✅/⏭️/🔍/❌ output + summary stats) and `--reactCompilerDebug={all|critical|none}` (maps to `panicThreshold: '_errors'`). On mobile the same taxonomy is available through the Babel plugin's `logger` option or `eslint-plugin-react-compiler` (the lint rule runs the same analysis the compiler does). + +## Coverage accounting + +Track four buckets — **compiled / skipped / errors / unsupported** — at file and component granularity, with **worst-status-wins per file** (`error > unsupported > skipped > compiled`): a file with five compiled components and one error is an *error file*, otherwise mixed files inflate the compiled count and the number lies to you. + +What the buckets tell you: + +- **compiled** — your real optimization coverage. "The compiler is enabled" claims nothing; this number does. +- **errors** — the actionable backlog. Each is a Rules-of-React fix. +- **unsupported** — the compiler's backlog, not yours. Trend it across compiler upgrades. +- **skipped** — intentional exclusions: test/story files, `'use no memo'` directives, and **class components** (never compiled — metamask-mobile#30919 counted 53 at full enablement; migration to function components is the only way to move them into the compiled bucket). + +## Staged adoption roadmap + +The extension's sequence (epic MetaMask-planning#6549) generalizes to any repo: + +1. **Lint clean:** update `eslint-plugin-react-hooks` / `eslint-plugin-react-compiler` to latest; fix violations — these are exactly what the compiler will refuse to compile. +2. **Audit opt-outs:** every `'use no memo'` carries a reason + TODO; the count only goes down. `grep -rn "use no memo" app --include="*.ts*"`. +3. **Ratchet `critical_errors`:** non-prod build passes; fix what surfaces. +4. **Ratchet `all_errors`:** remaining actionable errors fixed; what's left is the `Todo` (unsupported) set, which you wait out. + +## Verify + +- Per component: `Memo ✨` badge in React DevTools (see [js-profile-react.md](js-profile-react.md)). +- Per repo: the compiled-files count from the logger stats rises (or at least doesn't silently fall) release over release — silent coverage regressions are the failure mode this file exists to catch. +- After a compiler version bump: re-run the verbose build and diff the `unsupported` list — `Todo`s that became `compiled` are free wins; new `error`s are regressions to triage. + +## Don't over-correct + +- **Never "fix" a `Todo`.** Rewriting working code around an unimplemented compiler feature is churn with no perf evidence; the next compiler release may compile it as-is. +- Don't gate releases on compiler errors (`panicThreshold` stays `'none'` in production builds). +- Don't treat `skipped` as a problem — tests, stories, and deliberate opt-outs belong there. The smell is *unexplained* `'use no memo'` directives, not the bucket itself. +- A component without `Memo ✨` is not automatically a bug to chase — check the buckets first; it may be `unsupported`. + +## Related + +- [mm-react-compiler.md](mm-react-compiler.md) — enabling the compiler in this repo (Babel config, Metro cache, ESLint healthcheck) +- [js-react-compiler.md](js-react-compiler.md) — how the compiler transforms code; Rules-of-React background +- [mm-selector-cascade.md](mm-selector-cascade.md) — what the compiler **cannot** fix: unstable values crossing file boundaries (selectors, imported hooks) diff --git a/domains/performance/skills/performance/references/mm-react-compiler.md b/domains/performance/skills/performance/references/mm-react-compiler.md index e95d70d6..4f987cfb 100644 --- a/domains/performance/skills/performance/references/mm-react-compiler.md +++ b/domains/performance/skills/performance/references/mm-react-compiler.md @@ -51,6 +51,8 @@ React Compiler auto-memoizes components, callbacks, and computed values at build On opted-in paths you can gradually drop hand-written `useMemo`/`useCallback`/`React.memo` once the compiler is verified working — but do it deliberately and re-measure. Off opted-in paths, manual memoization still matters. +**Exception — effect dependencies.** Keep any `useMemo`/`useCallback` whose output is used as a `useEffect` dependency, here or in a consumer: the compiler's memoization is not guaranteed to match the manual strategy, and a mismatch causes over/under-firing of effects or infinite loops — a correctness change, not a perf tweak. Official guidance is to leave existing manual memoization in place and only omit it in *new* code ([reactwg/react-compiler#16](https://github.com/reactwg/react-compiler/discussions/16)). + ## What breaks compilation (it will skip the component) - Mutating props or state during render. @@ -68,4 +70,5 @@ Fix the ESLint `react-compiler` warnings on a path before/after opting it in. ## Related - [js-react-compiler.md](js-react-compiler.md) — upstream reference on how the compiler transforms code +- [mm-react-compiler-error-triage.md](mm-react-compiler-error-triage.md) — triaging compiler errors (`Todo`/unsupported vs actionable), `panicThreshold` ratcheting, and measuring real coverage - [mm-selector-memoization.md](mm-selector-memoization.md) — fix data-layer re-renders the compiler can't diff --git a/domains/performance/skills/performance/references/mm-redux-antipatterns.md b/domains/performance/skills/performance/references/mm-redux-antipatterns.md index a830bc35..965811a7 100644 --- a/domains/performance/skills/performance/references/mm-redux-antipatterns.md +++ b/domains/performance/skills/performance/references/mm-redux-antipatterns.md @@ -38,6 +38,8 @@ const browserTabs = useSelector((state: any) => state.browser.tabs); // also dro **Why it's wrong:** an inline accessor returning an array/object hands a fresh reference to the consumer whenever that slice changes (and defeats reuse/memoization across the app). For derived data it's worse — `useSelector(s => s.items.filter(...))` allocates every render. +**Perf-triage note:** an inline accessor returning a **primitive or stable field** (`s => s.settings.basicFunctionalityEnabled`) is reuse/type debt, not a re-render bug — the new arrow function per render is irrelevant; only the result's identity matters. Flag it for cleanup, not as a perf finding. + **Fix:** create a named selector in `app/selectors/`: ```ts // selectors/browser.ts @@ -86,5 +88,7 @@ grep -rn "dispatch(" app --include="*.ts" --include="*.tsx" | grep -v ".test." \ ## Related - [mm-selector-memoization.md](mm-selector-memoization.md) — the upstream fix for Pattern 1 +- [mm-selector-cascade.md](mm-selector-cascade.md) — repairing the whole dependency graph and removing accumulated `isEqual` band-aids after the root fix +- [mm-state-normalization.md](mm-state-normalization.md) — consolidating many `useSelector` calls into one view selector - [mm-context-performance.md](mm-context-performance.md) — the Context equivalent of over-broad subscriptions - [js-profile-react.md](js-profile-react.md) — confirm the re-render reduction diff --git a/domains/performance/skills/performance/references/mm-selector-cascade.md b/domains/performance/skills/performance/references/mm-selector-cascade.md new file mode 100644 index 00000000..5892e599 --- /dev/null +++ b/domains/performance/skills/performance/references/mm-selector-cascade.md @@ -0,0 +1,122 @@ +--- +title: Selector Dependency Cascades — Blast Radius & Repair (MetaMask) +impact: CRITICAL +tags: reselect, cascade, dependency-graph, isEqual, structural-sharing, react-compiler +--- + +# Skill: Selector Dependency Cascades + +> **Scope.** What a broken root selector does to the component graph is defined generically +> in the **`render-cascade`** knowledge file, and the selector patterns that cause it in +> **`selector-antipatterns`** — both installed alongside this skill under `knowledge/`. +> This file is the MetaMask Mobile instance: the real dependency graph, its blast radius, +> and the repair order. + +[mm-selector-memoization.md](mm-selector-memoization.md) catalogues the broken-selector *patterns*. This file is about what happens **downstream of one broken root selector** — and how to repair the whole graph instead of patching its leaves. Reference case: extension PR metamask-extension#37147, where a single identity output selector (`getInternalAccounts`) was recomputing through **15 direct + 35+ transitive consumer selectors into 50+ components on every dispatch** — every 5-second balance poll, every keystroke in the send flow. + +## Anatomy of a cascade + +```ts +// ❌ the root: identity output selector — memoizes nothing, new "result" every dispatch +export const getInternalAccounts = createSelector( + (state) => state.engine.internalAccounts.accounts, + (accounts) => accounts, // output === input: the cache can never hit meaningfully +); +``` + +Every consumer selector that takes the root as an input now sees a "changed" input on every dispatch, recomputes, and — because most result functions allocate (`.filter()`, `.map()`, `Object.values()`) — emits its *own* fresh reference, propagating the invalidation one layer further. Three layers down, nobody remembers the root; they see "my selector keeps firing" and reach for local fixes: + +```ts +// ❌ the band-aids that accumulate downstream of a broken root +const accounts = useSelector(getAccountsByScope, isEqual); // deep compare per dispatch +export const getX = createDeepEqualSelector(getInternalAccounts, …); // deep compare per dispatch +export const getMemoizedAccounts = createSelector(getInternalAccounts, (a) => a); // does nothing +``` + +Each band-aid suppresses the re-render for one consumer while *adding* an O(n) deep comparison on every dispatch — and the cascade cost scales superlinearly with power-user data (see [mm-power-user-scenario.md](mm-power-user-scenario.md)). + +A live cascade also **nullifies every optimization downstream of it**: `React.memo` children re-render anyway (their props are fresh refs), virtualized rows churn, compiler-memoized components re-render (the unstable value crosses the file boundary), and `useMemo`s recompute. Fix the cascade before evaluating any other optimization on the screen — and re-measure them after. + +## Step 1 — Traverse the dependency tree exhaustively before fixing + +The repair PR's evidence (and its review) should enumerate the graph **to closure** — every selector reachable from the suspect, not just its immediate neighborhood — the way #37147 did: + +1. **Direct consumers:** every selector that lists the suspect as an input. `grep -rn "getInternalAccounts" app/selectors --include="*.ts"` +2. **Transitive consumers:** repeat for each direct consumer until the frontier adds no new selectors. Don't stop at a fixed depth — cascades often have **more than one broken root**, and a partial map produces a wrong fix order. +3. **Component consumers:** `useSelector` call sites of anything in the graph. +4. **Recomputation count:** instrument with `selector.recomputations()` (reselect) or a `console.count` in the result function across a few dispatches (a balance poll is a convenient metronome). +5. **WDYR pass — already wired in this repo:** `wdyr.js` at the repo root tracks `useSelector` hook diffs. Run `ENABLE_WHY_DID_YOU_RENDER=true yarn start`, reproduce one dispatch, and every consumer logging *same values, different reference* is a node in the cascade — the live counterpart of the static map above. + +A before/after table — *recomputations per dispatch, re-renders per poll cycle, on the same interaction* — is what distinguishes a verified cascade fix from a speculative refactor. + +**Proactive mode — find the big trees without waiting for a symptom.** Rank roots by blast radius first (grep each selector's name across `app/` for consumer-file counts; appearances inside other selectors' input arrays give direct dependents), then for each large root verify its **input reference-stability**, not just its result function. The pattern that defeats every result-function grep: an input *function* that builds a fresh composite per call — spreading controller states and collecting other selectors' results into a new object. It looks disciplined, passes all pattern sweeps, and silently downgrades a `createDeepEqualSelector` into a whole-composite deep compare on **every check**. Verified instance: `getStateForAssetSelector` feeding `selectAssetsBySelectedAccountGroup` (`app/selectors/assets/assets-list.ts:107`) — the root of the asset-surface tree (15+ dependent selectors, including the per-row `selectAsset`), deep-comparing effectively the entire asset state per consumer per flush. + +## Step 2 — Plan the memoization fix order from the map: roots first + +Write down the fix order before writing any fix. The order is **topological** — roots, then their descendants, layer by layer: + +- A descendant fix can't be *verified* while any of its inputs is still unstable: its output identity keeps changing for upstream reasons, so the before/after numbers measure the wrong thing. +- Most descendant "problems" stop being fixes once the roots are stable — they reclassify from "add memoization here" to "remove the band-aid here" (Step 4). The plan is what tells you which is which *in advance*, instead of memoizing selectors that were only recomputing because of their inputs. +- If the map surfaced multiple roots sharing consumers, fix them together — otherwise the shared consumers keep re-rendering and the first root's win never shows up in the numbers. + +## Step 3 — Fix the root, not the 50 consumers + +Memoizing consumers one by one is whack-a-mole: each fix adds comparison cost and the graph keeps re-deriving from a poisoned root. Trace **upward** (who are my inputs? are *they* stable?) until you hit the selector whose output identity changes without its data changing — that's the root. Fix its memoization there (patterns + recipes in [mm-selector-memoization.md](mm-selector-memoization.md)). + +**Know your reference-stability contract first.** What the correct fix looks like depends on whether your store gives you stable references for unchanged data: + +- With **Immer-based reducers** (Redux Toolkit), structural sharing guarantees `state.a.b` keeps its reference **iff** nothing under that path changed. Under that contract, a plain `createSelector` over a *narrow* input is already correct, and deep-equal selectors are pure overhead. +- Where state is replaced wholesale on sync (documented for this repo's controller-state slices in [mm-selector-memoization.md](mm-selector-memoization.md)), input references break even when data didn't change, and `createDeepEqualSelector` at the *root* is the pragmatic tool. + +Establish which contract a slice actually follows (log `prev === next` for the input across two unrelated dispatches) before choosing — the answer differs per slice, and assuming the wrong contract either reintroduces the cascade or buys deep-compares you don't need. + +Then match the tool to **which side is unstable**: an unstable *input* (slice replaced wholesale on sync) calls for a deep-equal **input** compare (`createDeepEqualSelector`); a stable input with an unstable *output* (the result function allocates a fresh collection) calls for a `resultEqualityCheck`, so an unchanged result returns the cached ref. Deep-equalizing inputs to paper over an allocating result function runs the wrong comparison on every dispatch. + +Verified mechanism for this repo (`app/core/redux/slices/engine`): `UPDATE_BG_STATE` replaces only the **changed controller's key** with `Engine.state[key]`, and BaseController v2 state is Immer-produced — so an unchanged controller keeps its reference across flushes, and unchanged paths *within* a changed controller are structurally shared. Plain accessors into controller state are stable by construction; deep-equal is only warranted where a selector's *inputs* genuinely churn. And remember a deep-equal selector is output-**stable** but pays its compare per check, scaled by input size — over a power-user transaction history that is an O(n) deep compare per consumer per flush. + +## Step 4 — Sweep the graph and *remove* the band-aids + +This is the step most fixes skip. After the root is stable, every downstream `isEqual`, `createDeepEqualSelector`-wrapping-a-now-stable-input, and `getMemoized*` duplicate is dead weight: it still runs its deep comparison on every dispatch, and it **masks regressions** — if the root breaks again, the band-aids hide it until the app is slow everywhere again. + +```bash +# downstream band-aid sweep, scoped to the fixed graph +grep -rn "useSelector(.*isEqual)" app --include="*.tsx" | grep -v ".test." +grep -rn "createDeepEqualSelector" app/selectors --include="*.ts" +grep -rn "getMemoized\|selectMemoized" app/selectors --include="*.ts" +``` + +For each hit that consumes the fixed root (directly or transitively): remove the equality argument / downgrade to plain `createSelector`, and re-verify the consumer doesn't re-render on unrelated dispatches. #37147 deleted the band-aids in the same PR as the root fix — that's the model. + +## What the React Compiler can and cannot do here + +The compiler memoizes **within a file**. A `useSelector` result, an imported hook's return value, or an external context value is opaque to it — if the selector hands back a fresh reference, the compiled component still re-renders, and any derivation from it still recomputes (extension audit ticket MetaMask-planning#6661): + +```tsx +const tokens = useSelector(selectTokens); // compiler cannot see/stabilize this +const rows = tokens.map(toRow); // ❌ recomputes every render even when compiled +const rows = useMemo(() => tokens.map(toRow), [tokens]); // ✅ still needed +``` + +Rule of thumb: values that **cross a file boundary** (Redux selectors, imported hooks/functions, external context) keep their manual `useMemo`/`useCallback`; same-file props/state derivations can lean on the compiler. Fix the selector graph first — automatic memoization downstream of an unstable root optimizes nothing. + +## Don't over-correct + +- Not every busy selector is a cascade root — a selector returning a **primitive** breaks the chain at that point regardless of recomputation (allocation waste ≠ re-render bug). +- A *global* top-level cascade (an unstable value in a root provider/HOC re-rendering the whole tree on every state change) is a pattern the extension audit found at app root — worth **ruling out** with one profiler pass ("why did this render?" on a top-level component during an unrelated dispatch), but don't assume it exists here; verify before restructuring providers. See [mm-context-performance.md](mm-context-performance.md) for the provider-value mechanics. +- Don't add `useMemo` around every `useSelector` read preemptively — only where a non-primitive result feeds a derivation or a memoized child (see guardrails in [mm-hook-dependency-arrays.md](mm-hook-dependency-arrays.md)). + +## Verify + +1. Root selector returns the **same reference** across two unrelated dispatches (the contract test from [mm-selector-memoization.md](mm-selector-memoization.md)). +2. Recomputation counts on direct + transitive consumers drop to ~0 on unrelated dispatches. +3. Profiler on a top consumer (account list, send flow): the re-render cascade is gone during a balance poll. +4. The band-aid greps above return no hits inside the repaired graph. +5. Lock the win in CI: add a Reassure `*.perf-test.tsx` on a top consumer so the cascade can't silently return. + +## Related + +- [mm-selector-memoization.md](mm-selector-memoization.md) — the root-selector patterns and fix recipes +- [mm-redux-antipatterns.md](mm-redux-antipatterns.md) — `useSelector(x, isEqual)` as symptom; per-consumer view +- [mm-unstable-hook-return.md](mm-unstable-hook-return.md) — the same cascade shape, with a hook as the root +- [mm-state-normalization.md](mm-state-normalization.md) — state shape that prevents cascade-prone selectors +- [mm-react-compiler-error-triage.md](mm-react-compiler-error-triage.md) — confirming what the compiler actually covers diff --git a/domains/performance/skills/performance/references/mm-selector-memoization.md b/domains/performance/skills/performance/references/mm-selector-memoization.md index b9e117db..414c48b0 100644 --- a/domains/performance/skills/performance/references/mm-selector-memoization.md +++ b/domains/performance/skills/performance/references/mm-selector-memoization.md @@ -13,38 +13,22 @@ Broken or absent memoization in widely-used selectors is the single highest-impa - `createSelector` from `reselect` — reference-equality on inputs. - **`createDeepEqualSelector`** from `app/selectors/util.ts` — `createSelectorCreator(lruMemoize, deepEqual)`. Recomputes only when inputs are **deeply** equal-or-not. Use this when an input selector returns a fresh object/array on every dispatch (very common with controller state slices). -## The four deadly patterns +## The patterns, and what they look like here -### 1. Identity / passthrough in a plain `createSelector` -```ts -// ❌ Does nothing — output is the input, but the input ref changes every dispatch -export const selectX = createSelector(selectControllerState, (s) => s.things); -``` -A plain `createSelector` only helps if its **inputs** are reference-stable. Controller-state slices usually are not. Result: recomputes + new ref every dispatch. - -**Fix:** use `createDeepEqualSelector`, or narrow the input to the smallest stable slice. +The pattern taxonomy itself lives in the **`selector-antipatterns`** knowledge file, +installed alongside this skill under `knowledge/`. It is the single source — read it for the +full definition, the worked before/after of each, and the selector-creator decision tree. +This section maps each pattern onto *this* codebase. -### 2. New collection in the result function -```ts -// ❌ new array/Set/Map/object every call → always "changed" -(accounts) => Object.values(accounts).sort(...) -(transactions) => new Set(transactions.flatMap(...)) -(items) => items.filter(...) -(state) => state.swapsTransactions ?? {} // new {} when nullish -``` -Even a correct `createSelector` produces a new reference whenever it recomputes; if the inputs aren't stable, that's every dispatch. - -**Fix:** `createDeepEqualSelector` (deep-compares so it returns the *cached* ref when data is unchanged), or a stable module-level constant for the empty case, or a `resultEqualityCheck`. - -### 3. Mutation in the result function -```ts -// ❌ mutates the input array AND returns a new-but-corrupting ref -createSelector([getItems], (items) => { items.sort(cmp); return items; }) -``` -**Fix:** copy first — `[...items].sort(cmp)`. +| Pattern | How it shows up in Mobile | Fix here | +|---|---|---| +| **Identity / passthrough result** | `createSelector(selectControllerState, (s) => s.things)` — controller-state slices are not reference-stable, so it recomputes and returns a new ref every dispatch | `createDeepEqualSelector`, or narrow the input to the smallest stable slice | +| **New collection in the result function** | `Object.values(...).sort(...)`, `new Set(...flatMap(...))`, `items.filter(...)`, `state.swapsTransactions ?? {}` | `createDeepEqualSelector`, a stable module-level constant for the empty case, or a `resultEqualityCheck` | +| **Mutation in the result function** | `createSelector([getItems], (items) => { items.sort(cmp); return items; })` | copy first — `[...items].sort(cmp)` | +| **Over-broad input** | `state => state`, or a whole controller slice, as an input selector | narrow the input | +| **Unnecessary deep equality** | reaching for `createDeepEqualSelector` on an already-stable slice | plain `createSelector`; see *Don't over-correct* below | -### 4. `state => state` (or a huge slice) as an input selector -Forces recomputation on **any** state change anywhere. Narrow the input. +The two that dominate the verified instances below are the first two. ## Verified MetaMask instances @@ -113,4 +97,6 @@ Escalate severity by one level if the selector is imported in **10+ files**. ## Related - [mm-redux-antipatterns.md](mm-redux-antipatterns.md) — `useSelector(x, isEqual)` is the *symptom* of a broken selector; fix the selector, then remove the `isEqual`. +- [mm-selector-cascade.md](mm-selector-cascade.md) — graph-level view: blast radius of one broken root, and sweeping out downstream band-aids after the fix. +- [mm-state-normalization.md](mm-state-normalization.md) — state/selector *shape*: O(1) lookups, parameterized-selector cache thrashing, view-selector consolidation. - [js-profile-react.md](js-profile-react.md) — prove the re-render reduction. diff --git a/domains/performance/skills/performance/references/mm-state-normalization.md b/domains/performance/skills/performance/references/mm-state-normalization.md new file mode 100644 index 00000000..0bcaa57c --- /dev/null +++ b/domains/performance/skills/performance/references/mm-state-normalization.md @@ -0,0 +1,136 @@ +--- +title: State Normalization & Selector Shape (MetaMask) +impact: HIGH +tags: redux, normalization, selectors, O(1)-lookups, cache-thrashing, useSelector +--- + +# Skill: State Normalization & Selector Shape + +> **Scope.** The generic form of the O(n)-lookup problem is `selector-antipatterns` §7, +> in the knowledge file installed alongside this skill under `knowledge/`. This file is the +> MetaMask Mobile instance, plus the parameterized-selector cache-thrashing and +> view-selector consolidation work that is specific to this store's shape. + +Selector *memoization* fixes when things recompute; state and selector **shape** fixes how much each recomputation costs and how many subscriptions fire. The patterns here come from the extension performance audit (MetaMask-planning#6580, #6484), where linear scans and reshaping selectors multiplied across power-user data: with 1,000 tokens, 27 `.find()`-based lookups per render is 27,000 comparisons — per render. + +## Pattern — O(n) scans where the state shape should provide O(1) lookups + +```ts +// ❌ linear scan through all accounts on every call +export const getAccountByAddress = createSelector( + selectAccounts, + (_, address) => address, + (accounts, address) => + Object.values(accounts).find((a) => a.address.toLowerCase() === address.toLowerCase()), +); +``` + +When lookups by some key are frequent, **index the state once** instead of scanning per consumer: + +```ts +// ✅ build the index once per data change; lookups are O(1) key access +export const selectAccountsByAddress = createSelector(selectAccounts, (accounts) => + Object.fromEntries(Object.values(accounts).map((a) => [a.address.toLowerCase(), a])), +); +// consumers key into the memoized index — no scan, no per-arg selector cache to bust +const account = useSelector(selectAccountsByAddress)[address.toLowerCase()]; +``` + +Normalized shape (`byId` / `byAddress` maps + an `ids` array for order) is the same idea applied at the reducer level — the index is maintained on write instead of derived on read. + +## Pattern — parameterized selector cache thrashing + +`createSelector` has a **single-entry cache**. A parameterized selector called with different arguments from different components busts that one cache slot on every call: + +```ts +// ❌ each component's call evicts the previous component's result +const a1 = useSelector((s) => getAccountByAddress(s, addr1)); // miss +const a2 = useSelector((s) => getAccountByAddress(s, addr2)); // miss, evicts addr1 +const a3 = useSelector((s) => getAccountByAddress(s, addr3)); // miss, evicts addr2 — and so on every render cycle +``` + +In a list rendering N rows, the "memoized" selector recomputes N times per render, forever. **Check the memoizer before flagging:** this codebase already uses `weakMapMemoize` for some parameterized selectors (e.g. `selectNetworkConfigurationByChainId`), which caches per-argument and doesn't thrash — but only for *stable* arguments. A fresh **object literal** argument per call (`selectAsset(state, { address, chainId, isStaked })`) defeats `weakMapMemoize` too: every call is a new WeakMap key. Fixes, in order of preference: + +1. **Lookup-map selector** (above): select the whole memoized index once; key into it. Sidesteps per-arg caching entirely. +2. **Per-instance selector**: a factory (`makeSelectAccountByAddress()`) instantiated in the component with `useMemo`, so each call site owns its own cache slot. +3. **Bigger cache**: reselect's `lruMemoize` with `maxSize: N` — last resort; sizing is a guess that goes stale. + +```bash +# parameterized selectors: second input selector reads the argument, not state +grep -rn "(_, \|(_state" app/selectors --include="*.ts" +``` + +## Pattern — selectors that reorganize nested state + +```ts +// ❌ inverts { account → chain → tokens } into { chain → account → tokens } on every recompute +export const getTokensByChain = createSelector(selectAllTokens, (byAccount) => { + const byChain = {}; + for (const [account, chains] of Object.entries(byAccount)) + for (const [chainId, tokens] of Object.entries(chains)) + (byChain[chainId] ??= {})[account] = tokens; + return byChain; +}); +``` + +A full restructure allocates a new tree every recomputation — expensive to build, and every consumer sees a fresh reference. If two access patterns are both hot, **store both shapes** (maintain the second index in the reducer on write) or normalize so both reads are key lookups. A reshaping selector is acceptable only for cold paths. + +## Pattern — deep property access instead of composed input selectors + +```ts +// ❌ re-derives the full path; recomputes when ANY ancestor changes; nothing is reusable +export const getGroupName = (state, walletId, groupId) => + state.engine.accountTree.wallets[walletId]?.groups[groupId]?.metadata?.name; +``` + +Compose granular selectors at each level (`selectWallets` → `selectWalletById` → `selectGroupById` → …). Each layer memoizes independently, intermediate results are reusable by other selectors, and a change to one wallet no longer recomputes selectors reading a different one. This is also what keeps inputs *narrow* — the prerequisite for the memoization patterns in [mm-selector-memoization.md](mm-selector-memoization.md). + +## Pattern — many useSelector calls where one view selector should exist + +```tsx +// ❌ 11 store subscriptions; each runs on every dispatch; component re-checks 11 results +const quotes = useSelector(getQuotes); +const currency = useSelector(getCurrentCurrency); +const gasFee = useSelector(getGasFee); +// … ×8 more +``` + +Each `useSelector` is an independent store subscription with its own equality check per store notification. **Check the dispatch cadence before flagging count alone:** in this codebase, controller state changes batch into a 250ms flush (`app/core/Batcher`, `EngineService`'s `updateBatcher`) and dispatch inside `unstable_batchedUpdates`, so checks run at most a few times per second and React renders once per flush — N cheap accessor reads are *not* a problem. The actionable findings inside a high-count component are the **expensive** selectors (cost paid on every check) and the **unstable-ref** selectors (a re-render per flush) — triage and fix those individually first. + +Audit calibration (this codebase, 2026-06): a per-selector triage of the 10 highest-count components (9-19 reads each) ruled out ~90% of reads — feature-flag booleans, primitive accessors, and correctly `useMemo`'d factory selectors. The real findings were per-row parameterized selectors and deep-equal selectors over power-user-scaled data. The count was noise; the triage found what mattered. + +Consolidating related reads into **one memoized view selector** still earns its keep in two cases: a component repeated per row (per-row × per-flush multiplication of any expensive check), and derivation logic that would otherwise sit unmemoized in the component (where the React Compiler can't stabilize it — see [mm-selector-cascade.md](mm-selector-cascade.md)). One subscription, one equality check, one place where the shape is defined. + +The same consolidation applies to **duplicate derived-data implementations**: the extension audit found 4+ independent fiat-conversion code paths recomputing the same numbers in different components. One canonical selector ends both the wasted compute and the drift between implementations. + +## How to find + +```bash +# linear scans inside selectors/hooks +grep -rn "Object.values(.*)\.\(find\|filter\)\|\.find((" app/selectors app/components --include="*.ts*" | grep -v ".test." + +# reshaping selectors: nested loops/reduce building objects in a result function +grep -rn -B2 "??= {}\|reduce((acc" app/selectors --include="*.ts" + +# components with many subscriptions — triage the N selectors for cost/stability, don't flag the count itself +grep -rc "useSelector(" app/components --include="*.tsx" | awk -F: '$2>=5' | sort -t: -k2 -rn | head -20 +``` + +## Verify + +- Lookup fix: recomputation count on the index selector is ~1 per data change (not per render); list scroll/render time drops in the Profiler. +- Consolidation: the component's "why did this render" shows one subscription firing instead of N; render count per dispatch drops. +- Normalization: reducer tests confirm both shapes stay in sync on write. + +## Don't over-correct + +- Don't normalize a slice that's only ever iterated in full — indexes pay for themselves on *keyed lookups*, not on `.map()` over everything. +- Don't merge *unrelated* selectors into one mega view selector — that re-couples components to data they don't read and re-renders them for it. Consolidate related values consumed together. +- Don't flag a component for its `useSelector` **count** — with batched controller sync (250ms flush + `unstable_batchedUpdates`), N cheap subscriptions are noise. Flag the expensive or unstable selectors *among* them. +- `maxSize`/factory-selector machinery is for genuinely parameterized hot paths; for one or two call sites the lookup-map pattern is simpler and stays correct. + +## Related + +- [mm-selector-memoization.md](mm-selector-memoization.md) — memoization correctness for the selectors shaped here +- [mm-selector-cascade.md](mm-selector-cascade.md) — graph-level repair when a root selector poisons consumers +- [mm-redux-antipatterns.md](mm-redux-antipatterns.md) — inline selectors and `isEqual` band-aids diff --git a/domains/performance/skills/performance/references/mm-tools.md b/domains/performance/skills/performance/references/mm-tools.md index 9939808b..5de42764 100644 --- a/domains/performance/skills/performance/references/mm-tools.md +++ b/domains/performance/skills/performance/references/mm-tools.md @@ -84,6 +84,9 @@ Then read what's emitted — a load log (e.g. `source: 'cache' | 'fresh_fetch'`, "Components re-render too much" → React Native DevTools → "why did this render?" → mm-selector-memoization.md / mm-redux-antipatterns.md + → or WDYR (wired at wdyr.js, tracks useSelector diffs): ENABLE_WHY_DID_YOU_RENDER=true yarn start + — logs consumers re-rendering on same-values/new-reference; ideal for tracing a selector cascade + → mm-selector-cascade.md "Search/filter input lags while typing" → js-concurrent-react.md (useDeferredValue) — and memo() the expensive child @@ -166,6 +169,7 @@ endTrace({ name: TraceName.AssetDetails }); // end const x = trace({ name: TraceName.Tokens, op: TraceOperation.UIStartup }, () => build()); ``` - New flow → add a `TraceName` (+ `TraceOperation`) to `app/util/trace.ts`, then wrap it. +- **Quota guardrail:** never start a span per list item, per row, or per poll tick — span volume multiplies by data size × user count. A high-frequency span needs a deterministic sub-sample gate (and a kill-switch) before it ships. - **Component-level: use a per-feature measurement hook, not raw `trace()`.** The repo convention is a declarative `useXMeasurement` hook (e.g. `app/components/UI/Predict/hooks/usePredictMeasurement.ts`, `usePerpsMeasurement`, `useSectionPerformance`) that starts on mount and ends when conditions are true — which structurally enforces the "end on data-loaded, not mount" rule below: ```ts usePredictMeasurement({ traceName: TraceName.PredictMarketDetailsView, conditions: [dataLoaded, !isLoading] }); diff --git a/domains/performance/skills/performance/references/mm-useeffect-antipatterns.md b/domains/performance/skills/performance/references/mm-useeffect-antipatterns.md new file mode 100644 index 00000000..ad54d055 --- /dev/null +++ b/domains/performance/skills/performance/references/mm-useeffect-antipatterns.md @@ -0,0 +1,149 @@ +--- +title: useEffect Lifecycle Anti-Patterns (MetaMask) +impact: HIGH +tags: useEffect, setState, cleanup, AbortController, unmount, memory-leaks +--- + +# Skill: useEffect Lifecycle Anti-Patterns + +> **Scope.** The platform-agnostic taxonomy — unstable dependency identity, wrong +> dependencies, derived state via effect, cascading effect chains, missing timer cleanup, +> uncancelled async — is the single source in the **`effect-antipatterns`** knowledge file, +> installed alongside this skill under `knowledge/`. This file is the MetaMask Mobile +> instance of its lifecycle half: the verified instances, the repo's own idioms, and the +> fix recipes. + +[mm-hook-dependency-arrays.md](mm-hook-dependency-arrays.md) covers *when* effects re-run (the deps side). This file covers what goes wrong **inside and after** the effect: state derived in effects instead of render, effects chained off each other's setState, async work that outlives the component, and missing cleanup. These patterns cause extra render passes, memory leaks, and the classic "setState on unmounted component" warnings — and they're invisible to selector/re-render sweeps. + +## Pattern — derived state via useEffect + setState ("you might not need an effect") + +```tsx +// ❌ two render passes per change: render → effect → setState → render again +const [visibleTokens, setVisibleTokens] = useState([]); +useEffect(() => { + setVisibleTokens(tokens.filter((t) => !t.hidden)); +}, [tokens]); + +// ✅ derive during render — one pass, no state to drift out of sync +const visibleTokens = useMemo(() => tokens.filter((t) => !t.hidden), [tokens]); +``` + +If a value is computable from props/state/store, compute it in render (memoize only if it's expensive or feeds a memoized child). State + effect is for *synchronizing with something external*, not for derivation. + +## Pattern — cascading effect chains + +```tsx +// ❌ effect A sets state → triggers effect B → sets state → triggers effect C… +useEffect(() => { setAccount(deriveAccount(accounts, selected)); }, [accounts, selected]); +useEffect(() => { setBalances(deriveBalances(account)); }, [account]); +useEffect(() => { setFiat(deriveFiat(balances, rate)); }, [balances, rate]); +// 4 render passes for one upstream change, and the intermediate renders show stale combinations +``` + +**Fix:** collapse the chain into render-time derivation (one `useMemo` per step, or one for the lot). Each link in a setState-chain is a full extra render pass *and* a window where the UI shows an inconsistent intermediate state. + +## Pattern — async work that outlives the component + +```tsx +// ❌ fetch resolves after unmount (or after the input changed) → setState on dead component / stale data wins +useEffect(() => { + fetchTokenMetadata(address).then((meta) => setMetadata(meta)); +}, [address]); +``` + +Two equivalent fixes — pick one and use it consistently: + +```tsx +// ✅ cancelled flag — cheapest, works for any promise +useEffect(() => { + let cancelled = false; + fetchTokenMetadata(address).then((meta) => { + if (!cancelled) setMetadata(meta); + }); + return () => { cancelled = true; }; +}, [address]); + +// ✅ AbortController — also cancels the network request itself (RN fetch supports `signal`) +useEffect(() => { + const controller = new AbortController(); + fetch(url, { signal: controller.signal }) + .then((r) => r.json()) + .then(setData) + .catch((e) => { if (e.name !== 'AbortError') setError(e); }); + return () => controller.abort(); +}, [url]); +``` + +The cancelled flag prevents the *setState*; AbortController additionally stops the request from consuming bandwidth/battery. The race-condition variant (stale response overwriting fresh data when `address` changes quickly) is fixed by the same cleanup — the old effect's closure is cancelled before the new one runs. + +**Codify, don't copy-paste** (extension epic MetaMask-planning#6525): once a repo has three hand-rolled cancelled flags, extract shared hooks — `useIsMounted()`, `useAbortableEffect(fn, deps)` (effect receives a signal), `useEventListener(target, event, handler)` (auto-removes on unmount) — so cleanup is the default, not per-site diligence. + +## Pattern — missing cleanup for timers / subscriptions / listeners + +```tsx +// ❌ each mount adds another interval/listener; none are removed +useEffect(() => { + const id = setInterval(refreshGasEstimate, 15000); + emitter.on('update', onUpdate); +}, []); + +// ✅ every subscription returns its teardown +useEffect(() => { + const id = setInterval(refreshGasEstimate, 15000); + emitter.on('update', onUpdate); + return () => { clearInterval(id); emitter.off('update', onUpdate); }; +}, []); +``` + +Leaked intervals keep firing (and keep dispatching) forever; leaked listeners hold the closure — and everything it captured — out of garbage collection. See [js-memory-leaks.md](js-memory-leaks.md) for hunting these in a running app, and [mm-streaming-realtime.md](mm-streaming-realtime.md) for subscription lifecycles tied to visibility. + +## Pattern — regular variable where a ref is needed + +```tsx +// ❌ reset to false on every render — the guard never works +let hasLoggedImpression = false; +useEffect(() => { + if (!hasLoggedImpression) { logImpression(); hasLoggedImpression = true; } +}); + +// ✅ useRef persists across renders without triggering them +const hasLoggedImpression = useRef(false); +``` + +Any mutable flag/cache/previous-value that must survive re-renders but shouldn't cause them belongs in a ref, not a closure variable (and not state). + +## Pattern — large objects captured in effect closures + +An effect (or its cleanup) that closes over a large object — full token lists, raw API payloads — pins that object in memory for as long as the subscription lives. Extract the fields you need into locals *before* the closure, or read through a ref, so the big object can be collected. + +## How to find + +```bash +# setState-from-effect derivation candidates (review hits — some are legitimate syncs) +grep -rn -A3 "useEffect(" app --include="*.tsx" | grep -B1 "set[A-Z]" | grep -v ".test." + +# fetch/promises in effects with no signal/cancelled handling nearby +grep -rn -A6 "useEffect(" app --include="*.ts*" | grep -E "fetch\(|\.then\(" | grep -v "signal\|cancelled\|abort" | grep -v ".test." + +# intervals/timeouts/listeners inside effects — then eyeball for a `return () =>` teardown +grep -rn "setInterval\|setTimeout\|addEventListener\|\.on(" app --include="*.ts*" | grep -v ".test." | grep -v "clear\|remove\|off(" +``` + +## Verify + +- React DevTools highlight-updates: the derive-in-render fix removes the double render pass on the affected component. +- No "setState on unmounted component" / no stale-data flash when rapidly switching the input (account/network) that drives the effect. +- For cleanup fixes: navigate to the screen and back N times → timer/listener count stays flat (see [js-memory-leaks.md](js-memory-leaks.md)). + +## Don't over-correct + +- Effects that *synchronize with external systems* (subscriptions, navigation, imperative APIs) are the legitimate use — don't mechanically rewrite every effect as `useMemo`. +- An async effect whose component provably never unmounts mid-flight (e.g. root-level, app lifetime) doesn't need a cancelled flag — but say so in review rather than assuming. +- Don't wrap trivial derivations in `useMemo` while de-effecting — plain expressions are fine until profiling or a memoized child says otherwise. + +## Related + +- [mm-hook-dependency-arrays.md](mm-hook-dependency-arrays.md) — the deps side: JSON.stringify, inline literals, stale closures +- [js-memory-leaks.md](js-memory-leaks.md) — measuring leaks the missing cleanups cause +- [mm-streaming-realtime.md](mm-streaming-realtime.md) — subscription setup/teardown for real-time screens +- [mm-unstable-hook-return.md](mm-unstable-hook-return.md) — unstable hook returns that make effects re-run diff --git a/domains/performance/skills/performance/repos/metamask-mobile.md b/domains/performance/skills/performance/repos/metamask-mobile.md index f3d2d98a..c22f6162 100644 --- a/domains/performance/skills/performance/repos/metamask-mobile.md +++ b/domains/performance/skills/performance/repos/metamask-mobile.md @@ -54,6 +54,9 @@ Always pair measurement with the **power-user scenario on Android** — see [ref | `useSelector` returns new refs; `useSelector(x, isEqual)` band-aids | [mm-redux-antipatterns.md](references/mm-redux-antipatterns.md) | | Whole subtree re-renders under a Context provider | [mm-context-performance.md](references/mm-context-performance.md) | | `useEffect`/`useMemo` re-runs constantly; `JSON.stringify` in deps | [mm-hook-dependency-arrays.md](references/mm-hook-dependency-arrays.md) | +| Effect chains (`setState` in effect triggers next effect); setState after unmount; missing timer/listener cleanup | [mm-useeffect-antipatterns.md](references/mm-useeffect-antipatterns.md) | +| **One selector change re-renders half the app**; `isEqual`/`createDeepEqualSelector` band-aids accumulating downstream | [mm-selector-cascade.md](references/mm-selector-cascade.md) | +| O(n) `.find()` scans per render; parameterized selector recomputes for every list row; component with 5+ `useSelector` calls | [mm-state-normalization.md](references/mm-state-normalization.md) | | Animation janky; `useNativeDriver: false` on width/height | [mm-layout-animations.md](references/mm-layout-animations.md) → [js-animations-reanimated.md](references/js-animations-reanimated.md) | | List scroll jank / unbounded list | [js-lists-flatlist-flashlist.md](references/js-lists-flatlist-flashlist.md) | | Search/filter input blocks typing | [js-concurrent-react.md](references/js-concurrent-react.md) | @@ -69,6 +72,7 @@ Always pair measurement with the **power-user scenario on Android** — see [ref | Native module / sync method blocking JS | [native-sdks-over-polyfills.md](references/native-sdks-over-polyfills.md) | | Native lib crashes on 16KB-page Android | [native-android-16kb-alignment.md](references/native-android-16kb-alignment.md) | | Enable automatic memoization | [mm-react-compiler.md](references/mm-react-compiler.md) → [js-react-compiler.md](references/js-react-compiler.md) | +| Compiler is enabled but a component shows no `Memo ✨`; compiler errors in build output — which are real? | [mm-react-compiler-error-triage.md](references/mm-react-compiler-error-triage.md) | ## Verified anti-pattern catalogue (this codebase) @@ -88,6 +92,8 @@ Ordered by impact. Each links to the guide with the fix. **The `Where` column li | High | lodash main-package imports (98 files, no tree-shaking) | 98 files | [bundle-library-size.md](references/bundle-library-size.md) | | High | FlatList missing perf props on growing lists | 65 FlatList JSX | [js-lists-flatlist-flashlist.md](references/js-lists-flatlist-flashlist.md) | | High | AppState listener without cleanup | `app/core/SDKConnectV2/services/connection-registry.ts:487` | [js-memory-leaks.md](references/js-memory-leaks.md) | +| High | Parameterized selector (single-entry cache, busted per arg) doing an O(n) `Object.values().flat().find()` scan per call | `selectSingleTokenByAddressAndChainId` `app/selectors/tokensController.ts:174`; also `app/selectors/assets/assets-list.ts`, `app/selectors/moneyAccountController/index.ts` | [mm-state-normalization.md](references/mm-state-normalization.md) | +| Medium | Async effect without cancellation; setState-chain effects; derived state via useEffect+setState | feature-specific — run the guide's greps | [mm-useeffect-antipatterns.md](references/mm-useeffect-antipatterns.md) | | Medium | Inline `useSelector(state => state.x)` bypassing named selectors | 3 files | [mm-redux-antipatterns.md](references/mm-redux-antipatterns.md) | | Medium | Lottie where Rive fits (Rive already installed) | 5 files | [js-animations-reanimated.md](references/js-animations-reanimated.md) | | Low | dayjs + luxon both present (dedup) | 4 + 6 files | [bundle-library-size.md](references/bundle-library-size.md) | @@ -103,4 +109,4 @@ Ordered by impact. Each links to the guide with the fix. **The `Where` column li ## Attribution -Generic React Native references (`js-*`, `native-*`, `bundle-*`) adapted from "The Ultimate Guide to React Native Optimization" by Callstack. MetaMask-specific guidance (`mm-*`) from the internal Performance Guide for Engineers and verified codebase audits. +Generic React Native references (`js-*`, `native-*`, `bundle-*`) adapted from "The Ultimate Guide to React Native Optimization" by Callstack. MetaMask-specific guidance (`mm-*`) from the internal Performance Guide for Engineers and verified codebase audits. Cross-platform React/Redux guidance (`mm-selector-cascade`, `mm-useeffect-antipatterns`, `mm-state-normalization`, `mm-react-compiler-error-triage`) adapted from MetaMask contributor-docs [`frontend-performance.md`](https://github.com/MetaMask/contributor-docs/blob/main/docs/frontend-performance.md) and the extension performance audit (MetaMask-planning#6571; extension PRs metamask-extension#38007, metamask-extension#37147). diff --git a/domains/performance/skills/react-render-delta/skill.md b/domains/performance/skills/react-render-delta/skill.md new file mode 100644 index 00000000..7b50c00b --- /dev/null +++ b/domains/performance/skills/react-render-delta/skill.md @@ -0,0 +1,112 @@ +--- +name: react-render-delta +description: Prove a React rendering or memoization change actually reduced work, with a delivery gate and a reported band. Covers re-render counts (why-did-you-render), selector recomputes (reselect's real `.recomputations()` API), and A/B arms toggled at a FIXED commit rather than across a merge boundary. The falsifier is an arm whose treatment never reached the built bundle — a null from undelivered treatment is indistinguishable from a null from a small effect and reports as the second. Triggers on /mms-react-render-delta, or when asked to prove a component stopped over-rendering, measure selector recomputation, validate a memoization/React Compiler change, run a render-count A/B, or interpret a re-render benchmark. Callable by `evidence` as its React render & selector proof engine. +maturity: experimental +--- + +# /react-render-delta + +A render-count number is worthless until two things are true: the **treatment reached the +artifact the browser executes**, and the number is reported as a **band** rather than a point. +Most of this skill is those two checks. The measurement itself is easy; the failure mode is +reporting a difference between arms that never differed. + +> **Falsifier.** An arm whose manipulation cannot be observed in the built bundle. If you +> cannot point at output that differs *in kind* between arms — a symbol present in one and +> absent in the other, a flag line in a log — the A/B is not designed yet, and any delta it +> produces is noise with a story attached. + +## Method + +1. **Name the delivery check before building arms, and verify it emits.** State how the run + itself will show the arms differ, then confirm that output exists on one real build before + scaling to N repeats. This ordering is the whole skill. A measurement launched before the + instrument is proven emits a null that reads exactly like "no effect". + +2. **Derive the needle from output at the stage you will grep, not one stage upstream.** + Compiler and bundler output are not the same text. Worked failures, both real: + - React Compiler at `target: '17'` emits `react-compiler-runtime`; at `target: '19'` it + emits `react/compiler-runtime`. Grepping for the wrong one returns 0 in *both* arms and + fails the arm that actually got the treatment. + - `_c(` is the form **babel** emits. Metro transforms it further — in a real 119 MB React + Native bundle it scored 13 hits, every one a minified vendor identifier + (`function _c(e,t){return e|t}`), while the true compiler output went uncounted. The form + that survived metro was `memo_cache_sentinel` (4681 in the treated arm vs 166 in the + control). Same needle, two bundlers, two different answers. + + Compile one real file through the project's own config and read the output. Ten minutes + here saves a whole run. + +3. **A name is not a witness — count what only exists when the module is included.** A bare + module specifier appears in bundled `package.json` dependency lists whether or not the + module was ever pulled in; a clean control arm scored exactly 1 that way and was wrongly + failed. Gate on artifacts that cannot appear otherwise (a runtime sentinel, a compiled call + site). Keep the specifier count as a diagnostic — 3081-vs-1 is informative, it just isn't a + boolean. + +4. **Use the library's real counter before injecting your own.** `reselect` exposes + **`.recomputations()`** on memoized selectors — a genuine API, not a patch. Read it (sample + on an interval if the count should visibly climb). An injected `console.log` you added to a + selector body is an authored claim, not an observation; reach for it only when no real API + exists, and say so when you do. *(Note: the evidence catalog's render-and-selector entry long claimed there + was "no built-in selector-call counter". There is.)* + +5. **Toggle at a fixed commit, not across a merge boundary.** Same tree in both arms, one + thing different. A commit boundary drags in unrelated change you will then be unable to + exclude. When the real commit bundles two changes (a scope change *and* a version bump), + reproduce only the one under test — moving both reintroduces the confound the fixed-commit + design exists to remove, and can silently flip your delivery needle mid-experiment. + +6. **Repeat the capture, not the build; report the band.** Counts vary run to run — one + baseline measured 153/164/224 across three runs. The build dominates cost (~6 min vs ~90 s + per capture), so repeats are nearly free. Publish one artifact; report every repeat's count. + +7. **When the delta is under the spread, say "not resolvable at this n" and give the MDE.** + Not "no effect". State the smallest detectable effect and what n would resolve the observed + difference. A real worked result: 112–128 vs 115–133, delta 4.6%, t=1.33 — with delivery + proven (1244 compiled sites vs 0), so the null was about effect size, not plumbing. + +8. **A check that finds nothing needs a positive control.** Before believing a zero, confirm + the same check finds something it should. A search that returned "0 references" looked like + confirmation until searching for a string known to be present *also* returned 0 — the index + didn't reach that content and the zero meant nothing. + +## Gates, in order + +| gate | asserts | on failure | +|---|---|---| +| source manipulation | the intended edit applied, and *only* it | abort the arm | +| **delivery** | the change reached the built bundle | abort **before any capture** | +| metric | the instrument emitted a non-zero count on capture 1 | abort before spending repeats | + +Each catches what the previous cannot. Source changing is not delivery; delivery is not the +instrument working. Wire them as script-level aborts so a broken arm cannot report a number — +"refusing to emit a render count from an arm whose treatment is unproven" is the correct +output, and it is not a failure of the run. + +**Never relax a gate to make an arm pass.** When a gate fires, go read the artifact and find +the mechanism first. Loosening is the work-reducing direction, which is exactly where scrutiny +collapses. Demoting a needle from gate to diagnostic *after* proving it fires for an unrelated +reason is legitimate; doing it because the arm failed is not. + +## What the count does and does not mean + +WDYR counts **every** re-render in the measured window, including boot settling — not only the +cascade a given fix targeted. So an RCA predicting "→ 0 re-renders" for a specific cascade is +not refuted by a non-zero WDYR total. Say which quantity you measured, and don't let a global +counter stand in for a scoped claim. + +Global application does not imply a large effect: 1244 auto-memoized call sites moved one +interaction's re-render count under 5%. Reach for a flow the change plausibly dominates, and +treat a single flow as a lower bound on reach, not a summary of it. + +## Caveats to publish with the number + +Fixture parity (structurally matched vs byte-identical), arm ordering (randomized or not), +what window the counter covers, and how many flows were measured. State them; they are cheap +and their absence is what makes a number unfalsifiable. + +## Related + +- `evidence` — packages this skill's output as its [React render & selector proof category](https://github.com/MetaMask/skills/blob/main/domains/pr-workflow/skills/evidence/references/evidence-catalog.md). +- `memory-leak`, `supply-chain-audit` — sibling engines behind other categories. diff --git a/domains/performance/skills/selector-antipattern-scan/repos/metamask-extension.md b/domains/performance/skills/selector-antipattern-scan/repos/metamask-extension.md new file mode 100644 index 00000000..82801a08 --- /dev/null +++ b/domains/performance/skills/selector-antipattern-scan/repos/metamask-extension.md @@ -0,0 +1,43 @@ +--- +repo: metamask-extension +parent: selector-antipattern-scan +--- + +## Paths + +- Selector definitions: [`ui/selectors/`](https://github.com/MetaMask/metamask-extension/tree/main/ui/selectors) +- Selector creators: [`shared/lib/selectors/selector-creators.ts`](https://github.com/MetaMask/metamask-extension/blob/main/shared/lib/selectors/selector-creators.ts) — source of truth for `createSelector`, `createDeepEqualSelector`, `createResultEqualSelector`, `createShallowResultSelector` +- Controller state shape: [`app/scripts/metamask-controller.js`](https://github.com/MetaMask/metamask-extension/blob/main/app/scripts/metamask-controller.js) +- Component consumption sites: anywhere under [`ui/`](https://github.com/MetaMask/metamask-extension/tree/main/ui) that calls `useSelector` + +## Commands + +```bash +# Enable WDYR for post-merge diagnosis +ENABLE_WHY_DID_YOU_RENDER=true yarn start + +# Pre-merge grep checklist +grep -rE 'export function get' ui/selectors/ --include="*.ts" +grep -rn createDeepEqualSelector ui/ --include="*.ts" +grep -rnE 'useSelector\([^,]+,\s*(isEqual|shallowEqual)' ui/ --include="*.ts" --include="*.tsx" +grep -rnE '\.find\(' ui/selectors/ +``` + +## Selector Creators + +`shared/lib/selectors/selector-creators.ts` + +| Creator | Use Case | +|---------|----------| +| `createSelector` | Standard memoization (default) | +| `createDeepEqualSelector` | Genuinely unstable inputs (rare — see [narrow exception](../skill.md#overuse-of-createdeepequalselector)) | +| `createResultEqualSelector` | Unstable outputs requiring deep comparison | +| `createShallowResultSelector` | Unstable outputs, shallow comparison sufficient | + +## Example Fix Methodology + +[PR #37147](https://github.com/MetaMask/metamask-extension/pull/37147) fixed `getInternalAccounts` as the canonical example. Before: `createSelector(selectInternalAccounts, (accounts) => accounts)` (identity function, defeats memoization). After: `createSelector(getInternalAccountsObject, (accounts) => Object.values(accounts))`. Impact: 50+ component re-renders eliminated per state update. + +## Reference + +- [Frontend Performance Optimization Guidelines](https://github.com/MetaMask/contributor-docs/pull/159) (contributor-docs PR #159) diff --git a/domains/performance/skills/selector-antipattern-scan/repos/metamask-mobile.md b/domains/performance/skills/selector-antipattern-scan/repos/metamask-mobile.md new file mode 100644 index 00000000..1b324e3a --- /dev/null +++ b/domains/performance/skills/selector-antipattern-scan/repos/metamask-mobile.md @@ -0,0 +1,35 @@ +--- +repo: metamask-mobile +parent: selector-antipattern-scan +--- + +## Paths + +- Selector definitions: [`app/selectors/`](https://github.com/MetaMask/metamask-mobile/tree/main/app/selectors) +- Redux store: [`app/store/`](https://github.com/MetaMask/metamask-mobile/tree/main/app/store) +- Engine / state shape: [`app/core/Engine/Engine.ts`](https://github.com/MetaMask/metamask-mobile/blob/main/app/core/Engine/Engine.ts) +- WDYR setup: [`wdyr.js`](https://github.com/MetaMask/metamask-mobile/blob/main/wdyr.js) at repo root +- Component consumption sites: anywhere under [`app/`](https://github.com/MetaMask/metamask-mobile/tree/main/app) that calls `useSelector` + +## Commands + +```bash +# Enable WDYR (env var gate, same as extension) +ENABLE_WHY_DID_YOU_RENDER=true yarn start + +# Pre-merge grep checklist +grep -rE 'export function get' app/selectors/ --include="*.ts" +grep -rn createDeepEqualSelector app/ --include="*.ts" +grep -rnE 'useSelector\([^,]+,\s*(isEqual|shallowEqual)' app/ --include="*.ts" --include="*.tsx" +grep -rnE '\.find\(' app/selectors/ +``` + +## WDYR + +Mobile has `wdyr.js` at the repo root. It is gated on `__DEV__ && process.env.ENABLE_WHY_DID_YOU_RENDER === 'true'` and imported from the entry file. No manual setup required — flip the env var and restart Metro. + +Current configuration (at time of authoring): `trackAllPureComponents: true`, `onlyLogs: true` (Metro/Hermes console doesn't group well). + +## Differences from Extension + +- React Compiler adoption and `"use no memo"` opt-outs are extension-only at this time. diff --git a/domains/performance/skills/selector-antipattern-scan/skill.md b/domains/performance/skills/selector-antipattern-scan/skill.md new file mode 100644 index 00000000..ee32d7d7 --- /dev/null +++ b/domains/performance/skills/selector-antipattern-scan/skill.md @@ -0,0 +1,125 @@ +--- +maturity: experimental +name: selector-antipattern-scan +description: Review and diagnose Redux selector antipatterns that cause render cascades, pre-merge and post-merge +--- + +# Selector Anti-Pattern Review + +**Scope:** Redux selector antipatterns are the dominant cause of React render cascades in the MetaMask UI. This skill covers both review phases: pre-merge PR review (grep-driven checklist) and post-merge diagnosis (WDYR-driven workflow). Both modes resolve to the same root cause and the same fix set, catalogued in the **`selector-antipatterns`** and **`render-cascade`** knowledge files — the single source for their definitions (installed alongside this skill under `knowledge/`). + +Both `metamask-extension` and `metamask-mobile` share the same React + Redux architecture; this skill applies to both (see overlays for repo-specific paths). + +## When To Use + +- **Pre-merge.** Reviewing a PR that touches a `selectors/` directory, adds a `useSelector` call, or modifies a `createSelector` / `createDeepEqualSelector` definition +- **Post-merge.** Re-renders are disproportionate to state change size, performance degrades non-linearly with user data size, or components re-render during idle +- **Triage.** A WDYR counter jumps 5+ times per action, or a React render counter shows unexpected re-renders + +## Do Not Use When + +- Non-selector performance concerns (effects → use `effect-antipattern-scan`, context providers, virtualization) +- Network-bound slowness (use the Network panel, not WDYR) +- Startup or initial-mount perf (use startup profiling) +- Non-React trees (worker messaging, background script perf) + +## Mode A: Pre-Merge Review (grep-driven) + +1. **List changed selector/consumer files.** `git diff --name-only origin/main...HEAD | grep -E '(selectors|useSelector)'` +2. **Run the [grep checklist](#grep-checklist)** against the changed files. +3. **Match each hit to a pattern** in `selector-antipatterns` or to one of the [team-specific workarounds](#team-specific-workarounds) below. +4. **Block on Jest warning.** If the PR's test run surfaces `"result function returned its own inputs"`, the PR introduces an identity/passthrough result (`selector-antipatterns` §2). Do not merge. +5. **Require a fix, not a justification.** None of the five patterns have a valid use case. See [Pitfalls](#common-pitfalls) for the narrow `createDeepEqualSelector` exception. + +## Mode B: Post-Merge Diagnosis (WDYR-driven) + +1. **Confirm cascade.** Add a render counter to a high-level component. If count jumps 5+ per action, cascade is confirmed. + ```tsx + const [count, increment] = useReducer((n) => n + 1, 0) + useEffect(() => { increment() }) + console.log('Render:', count) + ``` +2. **Enable WDYR.** `ENABLE_WHY_DID_YOU_RENDER=true yarn start` (same env var on extension and mobile). +3. **Identify root component.** The first WDYR log is the cascade origin. Do not fix downstream symptoms first. +4. **Classify via the [WDYR message table](#wdyr-message-interpretation).** If the root cause is a selector, return to [Mode A](#mode-a-pre-merge-review-grep-driven) and apply the fix set. If it is a context value or prop identity issue, see the `render-cascade` knowledge file. +5. **Verify.** Repeat the action. Confirm the counter stabilizes (e.g. 0→2, not 0→25). Divide raw counts by 2 under React Strict Mode. + +## Grep Checklist + +| Pattern (`selector-antipatterns` §) | Detection | +|---|---| +| §1 Unmemoized selector | `grep -rE 'export function get' /` | +| §2 Identity / passthrough result | Jest warning `result function returned its own inputs` | +| §3 New collection in the result function | `grep -rnE 'new Set\|new Map\|Object\.(values\|keys\|entries)\|\?\? \{\}\|\?\? \[\]\|=> \(\{\|=> \[' /` | +| §4 Mutation in the result function | `grep -rnE '\.sort\(\|\.reverse\(\|\.push\(\|\.splice\(' /` | +| §5 Over-broad input | `grep -rn 'state) => state\b' /` | +| §6 Unnecessary deep equality | `grep -rn 'createDeepEqualSelector' /` then verify each input is genuinely unstable | +| §7 O(n) lookup | `grep -rnE '\.find\(.*=>.*address' /` | +| §8 Chained unmemoized transforms | `grep -rnE 'export function get.*\{' / -A5`, then look for several `.filter/.map/.sort` without memoization | + +The `=> ({` and `=> [` alternates catch a result function that *returns* a fresh literal rather than constructing a named collection. A trial run missed a real instance without them: `(metamask) => ({ userRegion: ..., ... })` builds a new object every recompute and matches none of the collection constructors. + +See the repo overlay for the concrete `` path. + +## Team-Specific Workarounds + +Two patterns show up beyond those in the knowledge file. Both are workarounds for broken selectors downstream. The fix is always to fix the selector, never to propagate the workaround. + +### `useSelector(selector, isEqual)` from `react-redux` + +```typescript +// Workaround that hides the real problem +const accounts = useSelector(getAccounts, isEqual) +``` + +- **Detection:** `grep -rnE 'useSelector\([^,]+,\s*(isEqual|shallowEqual)'` +- **Review action:** Find `getAccounts` (or whichever selector). Fix it to return a stable reference. Remove the `isEqual` argument in the same PR. +- **Why it's wrong:** Deep equality at the consumption site adds O(n) per render and leaves every other consumer of the same selector broken. + +### Overuse of `createDeepEqualSelector` + +```typescript +// Unnecessary when input is from Immer-managed Redux state +const getTokens = createDeepEqualSelector( + (state) => state.metamask.tokens, + (tokens) => transformTokens(tokens), +) +``` + +- **Detection:** `grep -rn createDeepEqualSelector /` +- **Review action:** For each instance, check if the inputs come from Redux state. If yes, swap to `createSelector`. Immer already gives stable references. +- **The narrow exception:** Inputs that are genuinely not from Immer/Redux state (e.g. derived from a non-Redux source, or passed in as props). These stay. + +## WDYR Message Interpretation + +For post-merge diagnosis, map the WDYR log message to the root cause: + +| Message | Root Cause | Fix | +|---------|------------|-----| +| `different objects that are equal by value` | Object recreated | `useMemo` (or fix selector that produced it) | +| `different functions with the same name` | Callback recreated | `useCallback` with stable deps | +| `different React elements` | JSX passed as prop | Extract to constant | +| `props object itself changed but values equal` | Parent cascade | Fix parent, not child | +| `[hook useContext result]` | Context value unstable | `useMemo` provider value | + +## Diagnostic Signals + +| Red | Green | +|-----|-------| +| Same component 5+ times in WDYR | Re-render count ≤ expected per action | +| Counter jumps 5+ per action | No WDYR logs during idle | +| Render count scales with data size | Render count stable regardless of data | +| Re-renders during idle | — | + +## Common Pitfalls + +| Mistake | Correct approach | +|---|---| +| Accept `useSelector(sel, isEqual)` because "it works" | The underlying selector is broken; fix it and remove the workaround | +| Approve `createDeepEqualSelector` without checking input source | Trace every input to verify it's not already Immer-stable | +| Treat the five patterns as preferences | They are measurably broken — each generates CI warnings | +| Ask the author to justify rather than fix | None of the patterns have a valid use case except the narrow exception above | +| Review only the selector definition, not consumption sites | Pattern 1 (plain function) hides at the call site | +| Fix downstream components first during post-merge diagnosis | Fix the root-cause selector; downstream fixes become wasted work | +| Add `React.memo` to symptom component | Requires stable parent. Fix the parent (usually a selector) first | +| Divide WDYR counts by 1 | React Strict Mode double-renders. Divide raw counts by 2 | diff --git a/domains/platform/knowledge/extension-architecture.md b/domains/platform/knowledge/extension-architecture.md new file mode 100644 index 00000000..9b87656e --- /dev/null +++ b/domains/platform/knowledge/extension-architecture.md @@ -0,0 +1,89 @@ +--- +name: extension-architecture +domain: platform +description: MetaMask extension — background/UI boundary, state sync, build types, key directories +--- + +# Extension Architecture + +## Background / UI Boundary + +The extension runs two separate JavaScript contexts that cannot share memory. + +| Context | Entry | Access | +|---------|-------|--------| +| Background (Service Worker / background page) | `app/scripts/` | DOM-less; controllers, wallet logic | +| UI (popup/tab) | `ui/` | React + Redux; rendering only | +| Shared | `shared/` | Constants, utilities, type definitions | + +Communication is message-based (Chrome runtime messaging). Code in `app/scripts/` cannot `import` from `ui/` and vice versa. + +## State Sync Flow + +``` +Controller state changes (app/scripts/) + ↓ +metamask-controller.js batches via debounce (200ms) + ↓ +UI receives batched state via sendUpdate + ↓ +Redux dispatches UPDATE_METAMASK_STATE + ↓ +Immer applies patches (structural sharing — unchanged paths keep stable references) + ↓ +useSelector evaluates; components re-render if output changed +``` + +Key file: `app/scripts/metamask-controller.js` — aggregates all controller state. + +## Build Types + +| Build | Command | Background | Security Policy | +|-------|---------|------------|-----------------| +| Development | `yarn start` | Webpack, hot reload | No LavaMoat | +| Production | `yarn dist` | Browserify | LavaMoat enforced | +| Test | `yarn build:test` | Browserify | Partial LavaMoat | + +LavaMoat restricts package capabilities at runtime. After adding/updating dependencies, run `yarn lavamoat:auto` to regenerate policies. + +## Manifest Versions + +| Version | Background | Lifecycle | +|---------|------------|-----------| +| MV3 (Chrome) | Service Worker | Can terminate and restart | +| MV2 (Firefox) | Background Page | Always running | + +Errors concentrated in MV3 (99%+) → root cause is service worker lifecycle, not application logic. + +## Key Directories + +``` +app/scripts/ +├── controllers/ # Feature controllers (one per domain) +├── lib/ # Background utilities +└── metamask-controller.js # Main aggregator; 200ms debounce + +ui/ +├── components/ # Reusable React components +├── pages/ # Page-level components +│ ├── routes/ # routes.component.tsx (high selector count) +│ └── home/ # home.container.js (legacy connect()) +├── ducks/ # Redux slices +├── selectors/ # All selectors +│ ├── selectors.js # Main file (~2500 lines) +│ └── .ts # Feature-specific selectors +└── contexts/ # React Context providers + +shared/ +├── constants/ +├── lib/ +└── modules/ + └── selectors/ + └── selector-creators.ts +``` + +## React Compiler Scope + +Enabled for `ui/components`, `ui/contexts`, `ui/hooks`, `ui/layouts`, `ui/pages`. + +Does NOT cross file boundaries — selector values from `useSelector` require manual `useMemo`. diff --git a/domains/platform/knowledge/mv3-service-worker.md b/domains/platform/knowledge/mv3-service-worker.md new file mode 100644 index 00000000..5acdc7a1 --- /dev/null +++ b/domains/platform/knowledge/mv3-service-worker.md @@ -0,0 +1,94 @@ +--- +name: mv3-service-worker +domain: platform +description: MV3 service worker lifecycle — Chrome background termination model, MetaMask's idle-termination mitigation, and cold-start failure modes +--- + +# MV3 Service Worker Lifecycle + +## MV2 vs MV3 + +| Manifest | Background | Default Lifecycle | Mitigated in MetaMask? | +|----------|------------|-------------------|------------------------| +| MV3 (Chrome) | Service Worker | Idle termination after 30s, hard cap ~5 min | Yes — see Idle Termination Mitigation | +| MV2 (Firefox) | Background Page | Always running | N/A | + +## Idle Termination Mitigation + +`app/scripts/background.js:750-758` runs a 2s `browser.storage.session` write loop. `saveTimestamp` (defined at `background.js:651-655`) writes an ISO timestamp into session storage: + + function saveTimestamp() { + const timestamp = new Date().toISOString(); + browser.storage.session.set({ timestamp }); + } + ... + const SAVE_TIMESTAMP_INTERVAL_MS = 2 * 1000; + saveTimestamp(); + setInterval(saveTimestamp, SAVE_TIMESTAMP_INTERVAL_MS); + +Each `chrome.*` / `browser.*` API call resets the 30s idle timer. At 2s cadence the worker stays alive indefinitely while the extension is active. `storage.session` (not `storage.local`) is deliberate — it is MV3-only, in-memory, and does not accumulate disk writes from a heartbeat. + +| Property | Value | +|---|---| +| API | `browser.storage.session.set` (MV3-only, in-memory) | +| Interval | 2000 ms (`SAVE_TIMESTAMP_INTERVAL_MS`) | +| Gate | `PreferencesController.enableMV3TimestampSave !== false` (default true) | +| Inline comment | `background.js:752` — "This keeps the service worker alive" | +| Pattern origin | De facto community consensus, not officially endorsed by Chrome DevRel | +| Re-verify if | Chromium policy change on idle-timer API interactions | + +Ongoing idle termination is **not** a live failure mode while the extension is running. Cold starts (browser launch, extension enable/reload, crash recovery) are the actual source of MV3-concentrated failures. + +## Verification Discipline + +Before attributing an MV3-concentrated error to "idle termination pressure": + +1. Verify `background.js:750-758` keepalive loop still exists and `saveTimestamp` still calls a `chrome.*` / `browser.*` API +2. Verify `enableMV3TimestampSave` is not disabled in affected Sentry events +3. Check whether error timing correlates with cold-start events, not idle periods + +If any check out, the working hypothesis is cold-start cascade race, not ongoing termination. + +## Error Concentration Signal + +| Distribution | Conclusion | +|---|---| +| ~50/50 MV3/MV2 | Application bug (affects both contexts equally) | +| 99%+ MV3 only | MV3 service worker lifecycle — check cold-start cascade before assuming idle termination | +| 99%+ MV2 only | Firefox-specific browser behavior | + +## Sentry Tag Dimensions + +Independent — do not conflate. + +| Tag | Meaning | +|-----|---------| +| `environment` | Build configuration (production, staging, development) | +| `installType` | How extension was loaded (normal, development, sideload, admin) | +| `dist` | Manifest version (mv3, mv2) | + +A production build can have `installType: development` if loaded unpacked. Filter carefully. + +## MV3-Specific Failure Modes + +| Failure | Cause | Mitigated? | +|---------|-------|------------| +| Cold-start cascade race (`APP_INIT_ALIVE` sent before UI listener bound) | `app-init.js` → dynamic-import `background.js` → listener registration races against an open port | No | +| `Background connection unresponsive` via ongoing idle termination | Worker idle-killed mid-session | Yes — 2s keepalive loop | +| `Background connection unresponsive` via cold-start latency | Cold start on browser launch + first-flush latency before `startUiSync` | No — keepalive does not apply before worker exists | +| Silent `postMessage` failure | Port disconnected during wake/termination, try/catch swallows error | No | +| In-memory state lost on cold start | New worker instance has empty in-memory state | No (fresh persistence read required) | + +## Sentry Diagnostic Instrumentation + +| Tag | Purpose | Status | +|-----|---------|--------| +| `uiStartup.receivedAppInitPing` | Distinguishes cold-start cascade race cases; `false` + `ALIVE` received ⇒ `APP_INIT_ALIVE` lost on cold start | Missing on `Background connection unresponsive` path as of 13.26.0 — instrumentation gap, being fixed | +| Phase-specific critical error types (`BACKGROUND_INITIALIZED`, `START_UI_SYNC`) | Distinguishes which startup phase hung | Added by 3-phase startup watchdog (PR #40306) | + +## When to Investigate MV3 Separately + +- Error volume is 10× higher in Chrome than Firefox +- Error involves background connectivity, keepalive, or startup handshake +- Error disappears when running with the worker kept alive manually +- Error correlates with browser-launch or extension-reload timestamps, not idle gaps diff --git a/domains/platform/skills/extension-errors-debugging/repos/metamask-extension.md b/domains/platform/skills/extension-errors-debugging/repos/metamask-extension.md new file mode 100644 index 00000000..5020917c --- /dev/null +++ b/domains/platform/skills/extension-errors-debugging/repos/metamask-extension.md @@ -0,0 +1,46 @@ +--- +repo: metamask-extension +parent: extension-errors-debugging +--- + +## Sentry Filters + +Filter by `dist` tag to isolate manifest version: +- `dist:mv3` — Chrome builds +- `dist:mv2` — Firefox builds + +Filter by `installType` to exclude developer-loaded builds: +- `installType:normal` — store-installed +- `installType:development` — sideloaded (unpacked); includes production builds loaded via developer mode + +## Build Commands + +```bash +# MV3 development (Chrome, service worker) +yarn start + +# MV2 development (Firefox, background page) +yarn start:mv2 + +# Production build (both manifests) +yarn dist + +# After dependency changes — regenerate LavaMoat policies +yarn lavamoat:auto +``` + +## Background Keepalive + +| Property | Value | +|---|---| +| Location | `app/scripts/background.js:750-758` | +| Function | `saveTimestamp` at `background.js:651-655` calls `browser.storage.session.set({ timestamp })` | +| Cadence | 2000 ms via `setInterval` | +| Effect | Each call resets Chrome's 30s SW idle timer — prevents idle eviction during active sessions | +| Gate | `PreferencesController.enableMV3TimestampSave !== false` | + +Active-session keepalive failures are rare and should be investigated as code bugs, not platform behavior. Cold-start cascade and first-flush latency are the actual MV3-concentrated failure modes — see `mv3-service-worker` knowledge for mechanism, failure modes table, and verification discipline. + +## Controller-Messenger Pattern + +Controllers communicate via `ControllerMessenger` (`@metamask/base-controller`). A controller's public API is its registered actions and events — not direct method calls. Cross-controller calls that bypass the messenger will not work across the background/UI boundary. diff --git a/domains/platform/skills/extension-errors-debugging/skill.md b/domains/platform/skills/extension-errors-debugging/skill.md new file mode 100644 index 00000000..885f59f8 --- /dev/null +++ b/domains/platform/skills/extension-errors-debugging/skill.md @@ -0,0 +1,59 @@ +--- +maturity: experimental +name: extension-errors-debugging +description: Diagnose browser extension errors — MV3 vs MV2, background/UI context, error tagging +--- + +# Extension Errors Debugging + +## When To Use + +- Errors appear in one manifest version but not the other +- Background connection or keepalive failures +- Errors that are hard to reproduce in development (only manifest in prod) +- Diagnosing Sentry errors before attributing root cause + +## Do Not Use When + +- Local development errors with full stack traces and reliable repro +- Build/compile errors (TypeScript, ESLint, bundler) +- Test failures unrelated to extension runtime behavior + +## Workflow + +1. **Check distribution** — Filter by `dist` tag. Is the error 99%+ MV3, MV2, or split? +2. **Classify root cause** — MV3-only → service worker lifecycle (specifically cold-start cascade; ongoing idle termination is mitigated — see `mv3-service-worker` knowledge). Split → application logic. MV2-only → Firefox behavior. +3. **Identify context** — Is the error from background (`app/scripts/`) or UI (`ui/`)? Stack trace file paths reveal this. +4. **Check error tags** — Verify `environment`, `installType`, and `dist` are what you expect (these are independent dimensions). +5. **Reproduce** — Use `dist` tag filter to reproduce in the right manifest version. + +## Context Identification from Stack Traces + +| Path prefix in trace | Context | +|---------------------|---------| +| `app/scripts/controllers/` | Background controller | +| `app/scripts/metamask-controller.js` | Background aggregator | +| `ui/components/` or `ui/pages/` | UI (React) | +| `shared/` | Either — shared module | + +## Background-Specific Error Types + +| Error | MV3 Root Cause | Mitigated? | +|-------|---------------|------------| +| Background connection unresponsive (cold-start cascade) | `app-init.js` → `background.js` listener race on worker cold start | No | +| Background connection unresponsive (first-flush latency) | Cold start + background state aggregation before `startUiSync` | No | +| Background connection unresponsive (idle termination) | Worker idle-killed mid-session | Yes — 2s `browser.storage.session` keepalive | +| Port disconnected (wake/termination race) | Port closed during worker lifecycle transition; silent via try/catch | No | +| Keepalive timer missed (active session) | Would imply `browser.storage.session.set` interval failed — rare; investigate as application bug, not platform behavior | N/A | +| In-memory state lost (cold start) | New worker instance re-reads persisted state | No | + +## Common Pitfalls + +| Mistake | Correct Approach | +|---------|-----------------| +| Attribute 99% MV3 error to application code | Check if error requires running background; MV3 SW lifecycle is the likely root cause | +| Default to "SW was terminated mid-session" for MV3 errors | Ongoing idle termination is mitigated by the 2s `browser.storage.session` keepalive. The likely mechanism is cold-start cascade or first-flush latency — see `mv3-service-worker` knowledge | +| "Keepalive timer missed" ⇒ SW slept | The 2s keepalive prevents idle sleep while active. A missed keepalive during active session is a code bug, not platform behavior | +| Use `environment` to filter for dev builds | Use `installType: development` — a prod build can be sideloaded | +| Conflate `dist` and `environment` | They are independent; filter both when needed | +| Reproduce MV2-only error in Chrome | Use Firefox; `installType` doesn't replicate MV3/MV2 lifecycle difference | diff --git a/domains/platform/skills/extension-lifecycle-decoupling/skill.md b/domains/platform/skills/extension-lifecycle-decoupling/skill.md new file mode 100644 index 00000000..cd7e5950 --- /dev/null +++ b/domains/platform/skills/extension-lifecycle-decoupling/skill.md @@ -0,0 +1,84 @@ +--- +maturity: experimental +name: extension-lifecycle-decoupling +description: Verify platform lifecycle events before assuming they cause application-level side effects +--- + +# Extension Lifecycle Decoupling + +## When To Use + +- Estimating event frequency based on service worker eviction +- Debugging behavior that "should" trigger on lock/unlock but doesn't +- Investigating keepalive, timer, or state persistence behavior + +## Do Not Use When + +- Working on UI-only code with no background process interaction +- The behavior reproduces reliably in development without service worker eviction + +## Core Distinction + +| Layer | Examples | Characteristics | +|-------|---------|----------------| +| Platform lifecycle | SW eviction, page unload | Infrastructure-level | +| Application lifecycle | Lock, unlock, init | User-level | + +These layers are often **decoupled**. The mapping between them is an implementation detail — verify it, don't assume it. + +## Verification Checklist + +Before claiming a platform lifecycle event causes application behavior: + +1. Is there an explicit handler (`onSuspend`, `beforeunload`) that triggers the claimed effect? +2. Is there a keepalive mechanism preventing the lifecycle event? +3. Does relevant state persist across restarts (`chrome.storage.session`, IndexedDB)? +4. Are timers alarm-based (persist across SW restart) or `setTimeout`-based (don't)? +5. Is the guard/flag reset by the lifecycle event or by a separate application event? + +## MV3 MetaMask Specifics + +| Assumption | Reality | +|------------|---------| +| SW eviction triggers lock | No `onSuspend` lock handler — SW eviction does NOT trigger lock | +| Timers lost on SW restart | Auto-lock uses Chrome Alarms API — persists across SW restarts | +| State lost on SW restart | Wallet state persists in `chrome.storage.session` and IndexedDB | +| SW evicts frequently during active use | A keepalive writes `browser.storage.session.set` on a short interval, and each `chrome.*`/`browser.*` call resets the 30s idle timer — so active-session eviction is effectively prevented. Cold starts (browser launch, extension reload) still happen. **Re-verify before relying on it — see below.** See `mv3-service-worker` knowledge for mechanism and verification discipline | + +### Re-verify the keepalive before reasoning from it + +This row is the only one that depends on a *current implementation detail* rather than on +absent handlers or persistent storage, and it is the one that inverts if the implementation +moves. If the interval grows past the idle timeout, or the keepalive is removed, the honest +answer flips from "eviction is prevented" to "eviction happens routinely" — and a skill that +still asserts the first would be worse than no skill. + +Confirm it in the target repo before drawing conclusions: + +```bash +# the keepalive writer and its cadence — symbol names, not line numbers +grep -rn "saveTimestamp\|SAVE_TIMESTAMP_INTERVAL_MS" app/scripts/background.js +``` + +Two things make the conclusion hold, and both must still be true: + +1. The interval is **well under the ~30s idle timeout** (last verified: `2 * 1000` ms). +2. The callback performs an **extension API call** — `browser.storage.session.set` — since it + is the API call that resets the timer, not the timer firing. + +If either has changed, treat active-session eviction as live and re-derive the rest of this +table's consequences. + +*Verified against `metamask-extension` at `d4dd55f300a` (2026-07-30): +`SAVE_TIMESTAMP_INTERVAL_MS = 2 * 1000`, `setInterval(saveTimestamp, …)`, +`saveTimestamp` calling `browser.storage.session.set`.* + +## Common Pitfalls + +| Mistake | Correct Approach | +|---------|-----------------| +| "SW evicts N times/day → event fires N times/day" | Check if application code has handler for eviction | +| Assume frequency from platform behavior | Grep for actual handler chains in `background.js`, `app-state-controller.ts` | +| Conflate platform restart with application reset | Check which state is persisted vs re-initialized | +| "Keepalive uses `chrome.alarms`" | It does not — keepalive works by making an extension API call (`browser.storage.session.set`) on a sub-idle-timeout interval. `chrome.alarms` is used separately, for auto-lock timers that must persist across SW restart | +| Citing this skill's keepalive claim without re-checking | It is the one row here that tracks a live implementation detail. Run the grep above; the conclusion inverts if the interval or the API call changes | diff --git a/domains/pr-workflow/skills/attest/references/dispatched-passes.md b/domains/pr-workflow/skills/attest/references/dispatched-passes.md new file mode 100644 index 00000000..0583920e --- /dev/null +++ b/domains/pr-workflow/skills/attest/references/dispatched-passes.md @@ -0,0 +1,41 @@ +# Phase 1 — the three dispatched briefs + +Send each to a **fresh instance** with the artifact and nothing else: not the transcript, not +your reasoning, not what you expect it to find. Context is what you are testing for. An instance +that knows what you meant will read what you meant. + +Run them concurrently — they are independent, and sequencing lets the first one's findings frame +the others. + +## outframe — contest the frame + +> You are reading a finished set of findings you did not produce. Do not check whether the +> findings are correct. Ask what claim was chosen and what a different framing makes visible: +> what question would a reader with different priorities have asked of the same material, what +> does the chosen frame make it impossible to notice, and which of the findings only look +> significant because of how the problem was cut. Return findings the framing hid, not a +> critique of the writing. + +## missing — contest the coverage + +> You are auditing a completed run for what it did not do. Enumerate: a modality that was not +> run, a claim asserted but not verified, a source cited but not read, a case the method +> structurally cannot reach. For each, say what running it would cost and what it could change. +> Do not restate what the run found. Absence is the deliverable. + +## press — read it as the stranger + +> You are the reviewer this lands in front of, with no context and a decision to make. Read only +> the artifact. Say what you would have to take on trust, which number you could not check if you +> wanted to, what reads as a measurement but is a sentence, and anything that assumes you were +> present for work you were not. Flag register slips: hedging that reads as concealment, +> confidence that outruns the evidence, and any place the author's process shows through. + +## Reading the returns + +A finding from any pass that invalidates the claim is `BLOCKED`. A finding that qualifies it is +`ATTESTED WITH` — and the caveat goes **into the published artifact**, not just into the verdict, +or the reader never sees it. + +Disagreement between passes is signal, not noise: `press` clearing something `outframe` flagged +usually means the artifact reads well and is framed wrong, which is the more dangerous state. diff --git a/domains/pr-workflow/skills/attest/references/phase-0-checks.md b/domains/pr-workflow/skills/attest/references/phase-0-checks.md new file mode 100644 index 00000000..994ad33c --- /dev/null +++ b/domains/pr-workflow/skills/attest/references/phase-0-checks.md @@ -0,0 +1,35 @@ +# Phase 0 — what each check catches + +Generated from the checks in `mms-evidence/scripts/attest-gate.sh`; that script is the +authority. Each entry exists because a run shipped without it. + +| # | check | run mode | diligence mode | +|---|---|---|---| +| 1 | marker pair | ✓ | ✓ | +| 2 | canonical header | ✓ | ✓ | +| 3 | verdict line | ✓ | ✓ | +| 4 | citations pinned | ✓ | ✓ | +| 5 | captured artifact | ✓ | ✓ | +| 6 | no prescriptions | ✓ | ✓ | +| 7 | no process narration | ✓ | ✓ | +| 8 | verdict is earned | ✓ | ✓ | +| 9 | verdict matches artifact | ✓ | ✓ | +| 10 | floats something for review | ✓ | ✓ | +| 11 | disclaimer present and early | ✓ | ✓ | +| 12 | destination is open | ✓ | ✓ | +| 13 | figures trace to an exhibit | ✓ | ✓ | + +Checks 1–4 differ by mode: in `--diligence` they test that contract's own marker pair, its +header, and that citations are pinned to a tag or SHA rather than a branch head, and the +verdict-line check reports SKIP because a diligence artifact renders none. Checks 8 and 9 SKIP +for the same reason. Everything from 5 down is shared, because those defects are shared. + +**Check 5 is the one that matters, and it asks for a medium.** Every earlier version tested a +property of the plaintext — does it carry a marker, does the command contain a placeholder — and +each caught one defect and missed the next, because every property of plaintext is forgeable by +whatever emits the plaintext. Four runs shipped that way. A `/blob/` permalink is a citation and +does not satisfy it: it witnesses a line in a file, never a run. + +**Check 12 tests the destination**, which no property of the text reveals. Across one register of +published runs, 22 of 27 comments went to pull requests that had already merged — median 22 days +after the merge, gate-clean every time. diff --git a/domains/pr-workflow/skills/attest/skill.md b/domains/pr-workflow/skills/attest/skill.md new file mode 100644 index 00000000..e5af6d89 --- /dev/null +++ b/domains/pr-workflow/skills/attest/skill.md @@ -0,0 +1,103 @@ +--- +name: attest +description: The gate an evidence artifact passes before it is published to a pull request, issue or shared tracker. Two halves that do not substitute for each other — a mechanical pass that greps for the properties a reader needs (marker pair, pinned environment, a captured artifact rather than typed prose, a destination that is still open) and a dispatched pass sent to fresh instances that contest the framing, the coverage, and how it reads to a stranger. The author is the wrong checker: they remember running the check, and the memory supplies the provenance the text lacks. Verdicts are attested, attested with named caveats, blocked, or not a run — the last being common and legitimate, because a run that could not execute has produced nothing to publish. Triggers on mms-attest, or before posting any evidence, validation or diligence output to a public surface. +maturity: experimental +--- + +# /mms-attest + +The gate an evidence artifact passes before it leaves your hands. Use before posting any +`/mms-evidence` or diligence output to a pull request, issue, or shared tracker. + +## The author is the wrong reader, and the wrong checker + +A validation run claims something was measured. Its characteristic failure is not a wrong number +— it is **prose that reads like a measurement**. An operator who ran the check cannot see this, +because they remember running it; the memory supplies the provenance the text lacks, before the +eye registers that it was missing. + +This is not hypothetical. A run in this workflow shipped a results section whose commands, exit +codes and "reached 100%" were typed by hand, while the real logs sat unpublished on disk. The +author had the skill installed that forbids exactly that. + +So the gate has two halves, and neither substitutes for the other. + +**The mechanical half is not advisory.** Marker presence, a pinned environment, whether any +fenced block is a tool's output rather than the author's transcription, whether the destination +is still open — all greppable. Anything checkable is checked before a model is asked for +judgement, because a model asked "is this good evidence?" answers from inside the frame that +produced it. + +**The dispatched half is positional.** Contesting the frame, the coverage, and the reading cannot +be self-run, for the same reason an author cannot proofread their own sentence for a word their +eye supplies. + +## Phase 0 — mechanical, no model + +``` +scripts/attest-gate.sh --target +scripts/attest-gate.sh --target --diligence +``` + +Thirteen checks; every one a hard fail. `--diligence` swaps the four Validation-Run envelope +checks for a no-verdict contract's own and shares everything downstream. See +[references/phase-0-checks.md](references/phase-0-checks.md) for what each check exists to catch +and the run that caused it to be written. + +**Run it as the same command that publishes, or it is a log line.** The gate and the write must +be one chain — `gate && publish`. Running both and reading the verdict afterwards is how a +blocked artifact reaches a public PR. The `hooks/pr-evidence-gate.py` PreToolUse hook enforces +this independently of your discipline, and fails closed; phase 0 is what you run to iterate +before it does. + +## Phase 1 — dispatched, three lenses + +| pass | reads for | returns | +|---|---|---| +| **outframe** | the frame — what claim was chosen, and what a different framing makes visible | findings the framing hid | +| **missing** | coverage — modality not run, claim unverified, source unread | the gap list | +| **press** | the text as it ships, as the stranger who has to act on it | leak and register findings | + +Dispatch to fresh instances is the mechanism, not an optimisation: a self-run frame check is +composed inside the frame it is meant to test. Briefs in +[references/dispatched-passes.md](references/dispatched-passes.md). + +Skipping a pass is allowed. Silently skipping it is not — name it as skipped in the verdict. + +## Phase 2 — shape + +Front-load the verdict, cut anything that does not change what the reader does, keep every +artifact and move only its placement. Shape only, after content is settled — a shape pass that +reaches content is how a capability table gets dissolved into paragraphs and the comment's +payload disappears. + +## Verdict + +``` +ATTESTED phase 0 clean, no blocking finding from phase 1 +ATTESTED WITH publishable, with named caveats carried INTO the artifact +BLOCKED phase 0 failure, or a phase 1 finding that invalidates the claim +NOT A RUN nothing was measured; there is no artifact to publish +``` + +`NOT A RUN` is legitimate and common. A run that could not execute its check produced no +evidence, and publishing the attempt with a disclaimer is worse than publishing nothing — the +disclaimer reads as hedging and the figure is kept anyway. + +## Anti-patterns + +| Bad | Good | +|---|---| +| Running phase 1 to decide phase 0 | Mechanical checks first; cheap and unarguable | +| Self-running the dispatched passes | Dispatch, or skip and say it was skipped | +| Attesting your own run | The gate is positional; an author attesting themselves attests nothing | +| Treating phase 0 items as advisory | Every one is a hard fail | +| `ATTESTED WITH` as a soft pass | The caveat goes *into the published artifact*, not just the verdict | +| Softening a check to fit the case in hand | If the new version could be satisfied by better prose alone, it is no longer the check | + +## Related + +- `mms-evidence` — produces the artifact this gates +- `mms-instrument-check` — prove the instrument fires before its output counts +- `mms-unmeasured-join` — audit the inference between the facts +- `mms-scope-of-search` — what a negative result is a fact about diff --git a/domains/pr-workflow/skills/coverage-partition/skill.md b/domains/pr-workflow/skills/coverage-partition/skill.md new file mode 100644 index 00000000..15e53fa3 --- /dev/null +++ b/domains/pr-workflow/skills/coverage-partition/skill.md @@ -0,0 +1,128 @@ +--- +name: coverage-partition +description: Measure which cases in a suite guard which mechanism, by defeating each mechanism in turn and recording the exact set of cases that go red. Reports the partition rather than the total, because "the suite has power" is a boolean while a suite's power is a distribution — and a suite credited with covering five behaviours routinely has one case standing between a mechanism and silence. Names why each survivor survived, since testing something else, being shielded by an upstream step, and being genuinely unaffected are three different facts that a count collapses into one. Use when a suite is offered as evidence for a specific claim, when one description credits one test set with covering several mechanisms, when deciding whether a green suite can be trusted to guard a security check, or when a mutation run reported a number and stopped there. Costs one full suite run per mechanism, two arms each. +--- + +# /coverage-partition + +A mutation run that reports "4 of 7 tests failed" has answered a question nobody asked. **Which** +four is the reviewable fact, and it costs the same probe to find out. + +"The test has power" is a boolean. A suite's power is a distribution, and the distribution is +almost never the one the author's sentence implies. + +## Why a total is the wrong number + +Totals compose badly. Seven cases that each defeat one mechanism and seven cases that all defeat +the same mechanism produce the same count and describe opposite suites. The count also hides its +own shape: a mechanism guarded by exactly one case looks identical, in the total, to a mechanism +guarded by four. + +And a total cannot be checked against a claim. "This suite covers signing and verification" is an +assertion about *which* mechanisms the cases reach — it is refuted or supported by the partition +and is untouched by the number. + +## The probe + +For each mechanism the suite is credited with guarding: + +1. **Defeat it minimally**, at its source, without breaking parsing — invert the condition, widen + the pattern, replace the verification call with a constant that succeeds. +2. **Run the whole suite** and record the exact set of cases that go red, by name. +3. **Name why each survivor survived.** This is the step that turns a count into a map, and it is + the step that gets skipped. + +Then report the sets, one row per case, one column per mechanism. + +## A worked partition + +A seven-case suite, described by its author as covering "real ECDSA signing and verification +across valid signed, unsigned, tampered, malformed, and invalid-signature cases". Three +mechanisms, three probes: + +| case | strip condition weakened | verification stubbed to succeed | value-format pattern widened | +|---|---|---|---| +| positive forward | — | — | — | +| legacy-signature | **fail** | — | — | +| missing-signature | **fail** | — | — | +| tampered | **fail** | **fail** | — | +| invalid-signature | **fail** | **fail** | — | +| unlisted-parameter | — | — | — | +| malformed-value | — | — | **fail** | +| | **4 of 7** | **2 of 7** | **1 of 7** | + +Five cases pass with signature verification entirely disabled. + +### Why each survivor survived + +The three cases that survive the strip mutation survive for three unrelated reasons, and the +distinction is the finding: + +- **The positive case is supposed to forward.** It tests the other side of the branch. Not a gap. +- **The unlisted-parameter case never reaches the strip** — an earlier canonicalization step + already dropped that parameter, so the mutated condition does not run on it. This one is + shielded, and it would keep passing no matter how badly the strip broke. +- **The malformed-value case is rejected by the format check first**, regardless of the strip. It + is genuinely unaffected, and it is load-bearing for a different mechanism. + +Three survivors, three facts. A count says "3 passed" and loses all of them. Shielded cases are +the ones worth naming out loud, because they read as coverage in a case list and provide none. + +### What the partition said that the total could not + +The suite's power is real and it is distributed — but most of it sits on the parameter strip, and +**exactly two cases would notice if signature verification stopped working entirely**. The +author's sentence reads as though all five case classes exercise verification. Five of them do +not. + +The format check has one guarding case, and that case asserts three keys at once. + +## Reading a partition + +**Name why each survivor survives.** A case that passes under mutation is testing something else, +or shielded by a step upstream, or genuinely unaffected. Those are different facts and only the +second one is a problem — but you cannot tell which you have without looking. + +**A mechanism with one guarding case is a finding, even when that case passes.** One case is one +refactor, one skip, one flaky quarantine away from zero, and nothing in a green run announces the +drop from one to none. Report it as a finding, not as coverage. + +**A case that asserts several things at once counts as thin.** When it goes red you cannot tell +which assertion fired, so it cannot serve as the guard for any one of them. Its column entry +should be read as "something in here broke", which is a weaker fact than it looks. + +**Overlap matters as much as coverage.** Cases that all fail under the same mutation are +redundant with each other under that mutation, however different their names and fixtures are. +Four cases failing on the strip is one guard with four expressions of it, and it will survive +deleting three of them. + +## When to reach for it + +When someone credits a suite as evidence for a claim — a PR description, a review reply, a +security sign-off. **"It has power" answers whether the suite is decorative. The partition answers +whether it has power over the mechanism named in the claim**, which is a different question and +usually an unasked one. + +Reach for it also when a mutation run has already produced a number, because the expensive part is +already paid for and the partition is what that run was capable of reporting all along. + +## Cost + +One full suite run per mechanism, two arms each — mutated and clean, since a case already red on +the clean arm is not evidence about anything. Three mechanisms is six suite runs. That is the +honest price, and it scales with mechanisms rather than with cases, so a large suite over three +mechanisms costs the same number of runs as a small one. + +Scope by mechanism, and pick them before running: the mechanisms the claim names, plus any +mechanism whose failure would be silent. + +## Related + +- [`falsifiers-first`](../falsifiers-first/skill.md) — supplies the defeats; this skill changes + what gets recorded when they run +- [`silent-failure`](../silent-failure/skill.md) — a mechanism with zero guarding cases is a + silent path by construction, and the partition is how the zero gets found +- [`evidence`](../evidence/skill.md) — the runners that execute the arms and capture the per-case + results the partition is built from +- [`unintended-breakage`](../unintended-breakage/skill.md) — reads the same per-case results in + the other direction, asking which cases went red that nobody meant to touch diff --git a/domains/pr-workflow/skills/debug/skill.md b/domains/pr-workflow/skills/debug/skill.md new file mode 100644 index 00000000..2a35b42c --- /dev/null +++ b/domains/pr-workflow/skills/debug/skill.md @@ -0,0 +1,101 @@ +--- +name: debug +description: Locate the cause of a symptom you cannot yet explain — a crash, a leak, a flake, a production error spike, a number that moved. The sibling of evidence: where evidence is handed a claim and looks for the observation that would falsify it, this is handed a symptom and must generate the hypothesis first, then kill it. Classifies the symptom into a defect class, routes to the engine skill that owns that class (memory-leak, race-condition-repro, react-render-proof, sentry-grafana-correlation, extension-errors-debugging, tsc-blindspots, supply-chain-audit), and holds the investigation to the same evidence bar evidence applies — an instrument that cannot fail is not evidence, a null needs its sensitivity stated, and a finding is scoped to what the change introduced versus what pre-existed. Stops when the cause is located or the class is excluded, not when a plausible story is available. Triggers on /mms-debug, or when asked to debug, diagnose, or investigate a symptom, find why something is slow, leaking, flaky, or erroring, chase a production alert to its cause, or reproduce a bug that cannot be reproduced by hand. +maturity: experimental +--- + +# /debug + +`evidence` is given a claim and looks for the observation that would prove it false. +This is given a **symptom** and has to produce the hypothesis before anything can falsify it. + +That difference is the whole skill. In review, the claim is someone else's and the social +pressure runs toward scepticism. In debugging, the hypothesis is *yours*, nobody else is +positioned to challenge it, and the expensive failure is building three hours of work on the +first theory that fit the first observation. + +## When To Use + +- A symptom with no established cause: a crash, a hang, a leak, an intermittent test, an + error-rate step change, a metric that moved without a deploy that explains it. +- A bug you cannot reproduce by hand and therefore cannot yet observe. +- A production signal that needs chasing back to code. + +## Do Not Use When + +- The PR states a claim and you need it settled — that is `/evidence`. +- The cause is known and you are validating the fix — that is `/evidence`, or the engine + skill directly. +- You want an after-the-fact writeup of a resolved failure — that is a postmortem, not this. + +## Workflow + +1. **State the symptom as an observation, not a theory.** "Popup memory grows ~105 MB per + open/close cycle" — not "the popup leaks because of the snow hook". The theory is the + output, never the input. +2. **Classify into a defect class** (table below). If two classes fit, run both; do not pick + the one you find more interesting. +3. **Delegate to the engine.** Each owns its own method and its own falsifier. This skill + routes and holds the bar; it does not re-implement the investigation. +4. **Kill the hypothesis before extending it.** Name the observation that would rule it out, + and go looking for that observation specifically. A hypothesis that has only ever been + confirmed has not been tested. +5. **Stop on a located cause or an excluded class.** A plausible story is not a stop condition. + +## Symptom → engine + +| Symptom | Class | Engine | +|---|---|---| +| Memory grows across a repeated flow; tab or worker dies over time | retention | `memory-leak` | +| Intermittent failure; passes on rerun; order-dependent | interleaving | `race-condition-repro` | +| UI janks, re-renders excessively, selector recomputes | wasted render work | `react-render-proof` | +| Production error spike, latency change, or a metric that moved | production signal | `sentry-grafana-correlation` | +| Extension-specific: MV3 vs MV2, background vs UI context, service-worker lifecycle | platform | `extension-errors-debugging` | +| Runtime value disagrees with its declared type; green typecheck, wrong behaviour | type/reality drift | `tsc-blindspots` | +| Started after a dependency change; new capability or transitive edge | supply chain | `supply-chain-audit` | +| None of the above, or several | — | bisect to a change first, then re-classify | + +## The evidence bar carries over + +The engines are shared with `evidence`, and so are its trust gates. They matter more here, +because in review a weak instrument produces a weak claim someone else will challenge — in +debugging it produces a wrong theory nobody checks. + +- **An instrument that cannot fail is not evidence.** Before trusting a measurement, establish + it can report the negative: a positive control that must move, a base arm that must fail. +- **A null needs its sensitivity stated.** "No difference" and "could not have detected one" + print identically. Calibrate, then report the zero against what the instrument demonstrably + resolves. +- **Scope to the change.** Classify each finding as introduced-here versus pre-existing. + Report pre-existing separately and uncharged, or you will attribute an old defect to a new + diff. +- **A negative result carries the scope of its search.** "No leak found", "nothing in the logs", + "the artifact does not exist" are claims about where you looked. Name the stores searched in the + finding itself — filesystem, artifact bucket, issue tracker, the other process's logs. If you + cannot name them, the search is not finished. +- **Measure on an isolated machine.** Timing- and GC-sensitive numbers taken on a contended host + are not noisy-but-usable, they are *stably wrong* — several runs will agree with each other and + disagree with reality. Replicate across hosts, not just across runs, before trusting a figure. +- **Collect the whole battery, not the discriminating member.** Instruments that stay flat are + data: the joint pattern localises the defect in a way no single reading does. + +## Output + +A short investigation record, not a narrative: + +``` +SYMPTOM observation, as measured +CLASS defect class, and why (with the classes considered and dropped) +HYPOTHESES each with the observation that would kill it, and whether that was found +CAUSE located mechanism, at file:line — or "class excluded", which is a real result +NOT CAUSE hypotheses killed, kept so the next person does not re-walk them +``` + +Killed hypotheses are part of the deliverable. Deleting them makes the surviving one look +inevitable and hands the next investigator the same dead ends. + +## Scope — what this is NOT + +- Not a fix. It locates and evidences the cause; the change is a separate act. +- Not a replacement for the engines. Each owns its method; this chooses among them. +- Not an incident-management process. No severity, comms, or timeline. diff --git a/domains/pr-workflow/skills/evidence/assets/evidence-run.yml b/domains/pr-workflow/skills/evidence/assets/evidence-run.yml new file mode 100644 index 00000000..1a65b24d --- /dev/null +++ b/domains/pr-workflow/skills/evidence/assets/evidence-run.yml @@ -0,0 +1,273 @@ +# Evidence runner — lives in ONE repo and measures any other. +# +# It does not need to be installed in the repo under review, and there is no reason to +# fork that repo either: `target_repo` is an input and the job checks it out read-only. +# Put this in whatever repo you want the runs and artifacts to belong to. +# +# Why this exists rather than running the measurement locally: a validation run is a +# claim that a command produced an output, and the reader has to be able to check that +# without going through the author. A local run's only witness is the author. Every +# failure class this suite has shipped was a local-environment failure — a helper script +# in /tmp, a probe deleted after the run, an absolute path, a toolchain that drifted, a +# contended host producing numbers that had to be retracted. +# +# In CI none of those is expressible. The workflow file is the recipe, the workspace is +# the repo, the run records its own ref, and the run URL is itself the capture — +# `actions/runs/` is what the publish gate accepts. +# +# Trigger from the CLI: +# gh workflow run evidence-run.yml \ +# -f runner=falsify-probe \ +# -f ref= \ +# -f args='--test path/to.test.ts --source path/to.ts --line 9 --replace " return x;"' +# +# Then cite the run URL in the comment. The artifacts are attached to the run; the +# orchestrator reads them to write the finding. +name: Evidence run + +'on': + workflow_dispatch: + inputs: + runner: + description: Runner to execute + required: true + type: choice + options: + - falsify-probe + - selector-recompute + - render-count + - tsc-substitution + - capture + target_repo: + description: >- + Repository to measure. No default on purpose: a default here is a standing + decision about what every unthinking dispatch touches, and this workflow's whole + argument is that it should be proved somewhere harmless first. + required: true + type: string + ref: + description: Commit SHA to measure. Pin it — a branch name makes the run unrepeatable. + required: true + type: string + args: + description: Arguments passed to the runner, verbatim + required: true + type: string + needs_install: + description: >- + Install the target's dependencies. Required for the jest and tsc runners; a waste + of ten minutes for `capture` wrapping git or a policy audit, which read files only. + required: false + default: true + type: boolean + baseline: + description: >- + Second SHA to measure identically. Twice now a run reported "no finding" when it + had no comparison, so the baseline arm is offered here rather than left to memory. + required: false + type: string + skills_repo: + description: >- + Where to source the runners. Defaults to upstream. Overridable because a runner + fix and the run that needs it cannot both wait on a review: point this at a fork + branch, and say in the artifact that you did. + required: false + default: MetaMask/skills + type: string + skills_ref: + description: >- + Ref within skills_repo. Overrides the EVIDENCE_SKILLS_REF variable. + required: false + type: string + probe_path: + description: >- + Path, within skills_repo, of a probe file to copy into the target tree before the + runner executes. `render-count` takes a hand-written probe, and a probe that lives + only on the author's disk is the exact defect the run URL exists to remove. + required: false + type: string + probe_dest: + description: Where in the target tree to place probe_path. Required with probe_path. + required: false + type: string + +permissions: + contents: read + +jobs: + measure: + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - name: Reject a moving ref + env: + REF: ${{ inputs.ref }} + BASE: ${{ inputs.baseline }} + run: | + # A branch name makes the artifact unrepeatable, which is the property this + # workflow exists to provide. Checked before anything is fetched. + for r in "$REF" ${BASE:+"$BASE"}; do + case "$r" in + *[!0-9a-f]* | "") echo "::error::'$r' is not a commit SHA"; exit 1 ;; + esac + [ ${#r} -eq 40 ] || { echo "::error::'$r' must be the full 40-char SHA"; exit 1; } + done + + - name: Checkout the target at the measured ref + uses: actions/checkout@v6 + with: + repository: ${{ inputs.target_repo }} + ref: ${{ inputs.ref }} + fetch-depth: 2 # the runners diff against the parent + + # Before setup-node, not after. A target that pins its package manager through + # `packageManager` in package.json makes setup-node's `cache: yarn` probe run + # `yarn cache dir` under the runner's global yarn 1.22, which refuses and fails + # the step — so every jest runner died at setup with nothing measured. The target + # repo's own workflows order it exactly this way. + - name: Enable corepack + if: inputs.needs_install + run: corepack enable + + - uses: actions/setup-node@v4 + if: inputs.needs_install + with: + node-version-file: .nvmrc + cache: yarn + + - name: Install + if: inputs.needs_install + run: yarn --immutable + + # Pinned to a commit, for the same reason the measured ref must be: a branch name + # makes the run unrepeatable, and "which version of the runner produced this" is + # exactly the question a reader asks. Override per-repo with the EVIDENCE_SKILLS_REF + # variable; bump the default when the runners land on the skills repo's main. + - name: Fetch the runners at a pinned version + uses: actions/checkout@v6 + with: + repository: ${{ inputs.skills_repo || 'MetaMask/skills' }} + ref: ${{ inputs.skills_ref || vars.EVIDENCE_SKILLS_REF || '56578cee0f679881e6f928177ef3cf6d45a5bfec' }} + path: .evidence-skills + # The security-domain analysis scripts (policy-audit.py and its siblings) are + # wrapped by the `capture` runner rather than being runners themselves, so they + # need no entry in the `runner` choice list — but they do need to be on disk. + # A sparse path absent from the chosen ref is silently empty, so listing it here + # costs nothing when it is not there. + sparse-checkout: | + domains/pr-workflow/skills/evidence/scripts + domains/pr-workflow/skills/evidence/probes + domains/security/skills + + - name: Verify the runners arrived + env: + RUNNER: ${{ inputs.runner }} + run: | + # A sparse checkout of a path that does not exist on the chosen ref succeeds and + # produces an empty directory, so the next step would fail with "No such file" + # and no indication that the REF was the problem. + F=".evidence-skills/domains/pr-workflow/skills/evidence/scripts/$RUNNER.sh" + [ -f "$F" ] || { + echo "::error::$RUNNER.sh not present at the pinned skills ref." + echo "::error::Set the EVIDENCE_SKILLS_REF repository variable to a commit that has it." + exit 1 + } + echo "runner $RUNNER.sh sourced from ${{ inputs.skills_repo || 'MetaMask/skills' }} @ ${{ inputs.skills_ref || vars.EVIDENCE_SKILLS_REF || 'pinned default' }}" \ + >> "$GITHUB_STEP_SUMMARY" + + - name: Place the probe + if: inputs.probe_path != '' + env: + SRC: .evidence-skills/${{ inputs.probe_path }} + DEST: ${{ inputs.probe_dest }} + run: | + # Copied from the runners checkout, so the probe has a permalink of its own and + # the reader can see the file that produced the count rather than taking the + # count on the author's word. + [ -n "$DEST" ] || { echo "::error::probe_dest is required with probe_path"; exit 1; } + [ -f "$SRC" ] || { echo "::error::probe not found at $SRC on the chosen skills ref"; exit 1; } + mkdir -p "$(dirname "$DEST")" + cp "$SRC" "$DEST" + echo "probe $SRC -> $DEST" >> "$GITHUB_STEP_SUMMARY" + + - name: Run + id: run + continue-on-error: true # the exit code IS the verdict; a finding is not a failure + env: + RUNNER: ${{ inputs.runner }} + ARGS: ${{ inputs.args }} + run: | + RUNNERS=.evidence-skills/domains/pr-workflow/skills/evidence/scripts + mkdir -p evidence-artifacts + set +e + eval "bash \"$RUNNERS/$RUNNER.sh\" --label \"$RUNNER-head\" --out evidence-artifacts $ARGS" + code=$? + set -e + echo "head_exit=$code" >> "$GITHUB_OUTPUT" + echo "runner exited $code — the exit code is the verdict, not a build failure" \ + >> "$GITHUB_STEP_SUMMARY" + + - name: Baseline arm + if: inputs.baseline != '' + continue-on-error: true + env: + RUNNER: ${{ inputs.runner }} + ARGS: ${{ inputs.args }} + BASE: ${{ inputs.baseline }} + run: | + git checkout --detach "$BASE" + if [ "${{ inputs.needs_install }}" = "true" ]; then yarn --immutable; fi + RUNNERS=.evidence-skills/domains/pr-workflow/skills/evidence/scripts + eval "bash \"$RUNNERS/$RUNNER.sh\" --label \"$RUNNER-base\" --out evidence-artifacts $ARGS" + + - name: Determinism check + # Contention produced numbers that were published and then retracted. Running the + # head arm twice and diffing costs one repeat and turns that into a pre-publish + # signal rather than a correction. + continue-on-error: true + env: + RUNNER: ${{ inputs.runner }} + ARGS: ${{ inputs.args }} + REF: ${{ inputs.ref }} + run: | + RUNNERS=.evidence-skills/domains/pr-workflow/skills/evidence/scripts + git checkout --detach "$REF" + eval "bash \"$RUNNERS/$RUNNER.sh\" --label \"$RUNNER-repeat\" --out evidence-artifacts $ARGS" || true + A="evidence-artifacts/$RUNNER-head.json" + B="evidence-artifacts/$RUNNER-repeat.json" + if [ -f "$A" ] && [ -f "$B" ]; then + # `label`, `log` and `logs` name the arm, and `env` carries timing — all differ + # between the two runs by construction. Comparing them makes the check fire on + # every run, which is the same as not having it. `logs` was missing from this + # list, so `render-count` — the only runner that writes the plural key — failed + # the check on every run while reporting identical counts. A warning that is + # always wrong for one runner teaches the operator to publish through it. + if diff <(jq -S 'del(.env, .label, .log, .logs)' "$A") \ + <(jq -S 'del(.env, .label, .log, .logs)' "$B") > determinism.diff; then + echo "deterministic across two runs" | tee -a "$GITHUB_STEP_SUMMARY" + else + echo "::warning::runner is NOT deterministic at this ref — do not publish these numbers" + cat determinism.diff >> "$GITHUB_STEP_SUMMARY" + fi + fi + + - name: Publish the run summary + if: always() + run: | + for f in evidence-artifacts/*.md; do + [ -f "$f" ] || continue + { echo; cat "$f"; } >> "$GITHUB_STEP_SUMMARY" + done + + - name: Upload artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: evidence-${{ inputs.runner }}-${{ inputs.ref }} + # the artifact name carries what was measured, so a downloaded zip is + # self-describing rather than needing the run page to interpret + path: | + evidence-artifacts/** + determinism.diff + retention-days: 90 + if-no-files-found: error diff --git a/domains/pr-workflow/skills/evidence/hooks/pr-evidence-gate.py b/domains/pr-workflow/skills/evidence/hooks/pr-evidence-gate.py new file mode 100755 index 00000000..03ac744e --- /dev/null +++ b/domains/pr-workflow/skills/evidence/hooks/pr-evidence-gate.py @@ -0,0 +1,481 @@ +#!/usr/bin/env python3 +""" +Emit-time evidence gate (PreToolUse:Bash). + +Blocks outward-facing `gh pr|issue edit|create|comment` — and `gh api` body +writes, which bypass the porcelain — whose body contains, in a validation-scoped +paragraph, a claim that the trustworthiness gate would reject. Rationale: an +unbacked "confirmed / verified / proven / observed / ingested / ✅" launders an +unverified assertion as fact under the author's name, and an untracked "remains +pending" decays to never. + +The trustworthiness gate is the checklist; THIS is the trigger that runs it. +Each class below implements a numbered item of `references/evidence-trustworthiness.md`. + +Contract: reads PreToolUse JSON on stdin. Exit 0 = allow. Exit 2 = block +(stderr shown to the model). Fails OPEN on anything it cannot parse, so it never +bricks unrelated Bash commands — but once it has identified a body it is going to +publish, it fails CLOSED: if attest-gate.sh cannot be found or run, the write is +refused rather than waved through. +""" +import json +import os +import re +import subprocess +import sys +import tempfile + + +def _out_allow(): + sys.exit(0) + + +def _block(msg): + sys.stderr.write(msg) + sys.exit(2) + + +# Outward-facing gh write surfaces. The porcelain set is wider than +# `gh pr edit|create` because the same unbacked verdict launders identically +# through a PR comment or an issue body. `gh api` is included because a PATCH +# to .../comments/ is the same publish with a different spelling — a gate +# that cannot see the write it is meant to police is not a gate. +GH_PORCELAIN = re.compile(r"\bgh\s+(?:pr|issue)\s+(?:edit|create|comment)\b") +GH_API = re.compile(r"\bgh\s+api\b") + + +def main(): + try: + payload = json.load(sys.stdin) + except Exception: + _out_allow() + + if payload.get("tool_name") != "Bash": + _out_allow() + + cmd = (payload.get("tool_input") or {}).get("command", "") + + is_porcelain = bool(GH_PORCELAIN.search(cmd)) + is_api = bool(GH_API.search(cmd)) and re.search(r"(?:-F|-f|--field|--raw-field)\s+body=|--input\b", cmd) + if not (is_porcelain or is_api): + _out_allow() + if is_porcelain and "--body" not in cmd: # covers --body and --body-file + _out_allow() + + body = _extract_body(cmd) + if not body: + _out_allow() # can't read it -> don't block; nothing to scan + + violations = _scan(body) + violations += _run_attest_gate(body, cmd) + if not violations: + _out_allow() + + lines = [ + "EVIDENCE GATE (PreToolUse) — blocked outward-facing GitHub write.", + "", + "Each finding names the trustworthiness-gate item it violates. Fix by", + "attaching the missing artifact in the SAME block, or by downgrading the", + "claim (⚠️ inconclusive / remove it). Do not rephrase around the check.", + "", + ] + for v in violations[:12]: + need = NEEDS.get(v.get("kind", "verdict"), "ARTIFACT") + lines.append(f' • [{v["kind"]}] "{v["token"]}"') + lines.append(f' needs: {need}') + lines.append(f' in: {v["snippet"]}') + if len(violations) > 12: + lines.append(f" … and {len(violations) - 12} more.") + lines += [ + "", + "If the evidence exists on disk, BIND it: every collected artifact the", + "claim rests on gets referenced or re-hosted before the write.", + ] + _block("\n".join(lines) + "\n") + + +NEEDS = { + "attest-gate": "the check named above to pass — run scripts/attest-gate.sh yourself to iterate", + "gate-missing": "attest-gate.sh on disk; refusing to publish a body nothing verified", + "gate-error": "attest-gate.sh to run successfully; refusing to publish unverified", + "verdict": "an inspectable ARTIFACT (https:// permalink, /blob//, or a *.test.ts ref)", + "observation": "an OBSERVATION artifact (screenshot/recording/log/JSON/permalink) — " + "a /blob/ code link witnesses code, not runtime behavior", + "deferral": "a co-located TRACKER (#issue, issues/pull URL, 'triage', 'tracked in')", + "ci-restatement": "removal — a validation surface carries zero CI references. " + "The Checks tab already shows them; cite CI only as the revert " + "lane's outcome, never as 'green at head'", + "inflated-verdict": "a downgraded verdict — 'live-proven' co-located with " + "'not exercised' is inflated; borrowed evidence never " + "upgrades an uncaptured lane", + "bare-identifier": "a resolving link for the id (permalink or absolute-windowed " + "query) OR the re-hosted capture showing it", + "truncated-identifier": "the FULL identifier, quoted verbatim — an ellipsized id " + "cannot be grepped against any artifact, and a co-located " + "resolver does not excuse it", + "mutable-ref": "a commit-pinned permalink (/blob//…#Lx-Ly) — a branch ref " + "can be rewritten after review", + "dump-resolver": "a reader-native exhibit — a live link or a visual. A raw " + "log/JSON/HAR dump is appendix-only, never the exhibit a claim rests on", + "link-only-exhibit": "an embedded visual of the linked view ALONGSIDE the permalink — " + "link-only defers validation behind click + auth + query rendering", + "data-only-exhibit": "an in-environment capture (the resolving UI with its query, " + "project/environment selectors and time window in-frame) — " + "quoted data alone carries no liveness provenance", + "step-waiver": "a per-step ⏳ + tracker whose blocker is that step's OWN unmet " + "precondition — an impossibility argument is not a discharge", +} + + +def _extract_body(cmd): + # 1) --body-file / --input + m = re.search(r"--(?:body-file|input)[=\s]+(?:'([^']+)'|\"([^\"]+)\"|(\S+))", cmd) + if m: + path = m.group(1) or m.group(2) or m.group(3) + try: + with open(os.path.expanduser(path), "r", encoding="utf-8") as fh: + raw = fh.read() + except Exception: + return "" + # `gh api --input` takes a JSON file; pull .body out of it. + try: + obj = json.loads(raw) + if isinstance(obj, dict) and isinstance(obj.get("body"), str): + return obj["body"] + except Exception: + pass + return raw + # 2) gh api -F body=@ / --field body=@ + m = re.search(r"(?:-F|--field|--raw-field)\s+body=@(?:'([^']+)'|\"([^\"]+)\"|(\S+))", cmd) + if m: + path = m.group(1) or m.group(2) or m.group(3) + try: + with open(os.path.expanduser(path), "r", encoding="utf-8") as fh: + return fh.read() + except Exception: + return "" + # 3) --body "$(cat <<'EOF' ... EOF)" heredoc + m = re.search(r"<<-?'?EOF'?\s*\n(.*?)\n\s*EOF", cmd, re.DOTALL) + if m: + return m.group(1) + # 4) --body '...' / --body "..." / gh api -f body='...' + m = re.search(r"(?:--body|(?:-f|--field|--raw-field)\s+body=)[=\s]*'((?:[^']|'\\'')*)'", cmd, re.DOTALL) + if m: + return m.group(1) + m = re.search(r'(?:--body|(?:-f|--field|--raw-field)\s+body=)[=\s]*"(.*?)"', cmd, re.DOTALL) + if m: + return m.group(1) + return "" + + +# ── item 1/5: verdict claims ──────────────────────────────────────────────── +VERDICT = re.compile( + r"(?i)(?:\bcapture[ds]?\s+confirm\w*|\bconfirm(?:s|ed)\b|\bverif(?:y|ies|ied)\b" + r"|\bproven\b|\bobserved\b|\bingested\b|\bdemonstrat(?:e|es|ed)\b" + r"|\blive-proven\b|\bsuccessful\b|\bvalidated\b" + r"|does not drop\b|✅)" +) +ARTIFACT = re.compile( + r"(?i)(?:https?://\S+|actions/runs/\d+|/blob/|\bjob/\d+" + r"|`?[\w./-]*\.(?:test|spec)\.[tj]sx?(?::\d+)?`?)" +) +# ── item 2: runtime observation claims ───────────────────────────────────── +OBSERVATION = re.compile( + r"(?i)(?:\brendered\b|byte-identical(?:ly)?|\bsnapshot\s+shows?\b" + r"|\bscreenshots?\s+show\w*|\breproduc(?:ed|es)\b" + r"|\bstill\s+(?:shown|shows|fails|failing|raises)\b" + r"|\bin\s+a\s+(?:real|live)\s+browser\b|\blive\s+test\s+build\b" + r"|\bin\s+two\s+independent\s+runs\b|\bworks\s+as\s+described\b)" +) +OBS_ARTIFACT = re.compile( + r"(?i)(?:!\[|/ instead of /blob// ───────── +MUTABLE_REF = re.compile( + r"(?i)https?://github\.com/[\w.-]+/[\w.-]+/blob/(?![0-9a-f]{7,40}[/#])[\w.-]+/" +) +# ── item 13: dump-as-resolver ────────────────────────────────────────────── +DUMP_LINK = re.compile(r"(?i)https?://\S+\.(?:log|json|har|txt)\b") +IMAGE_EMBED = re.compile( + r"(?i)(?:!\[|.*?", + "", body, flags=re.DOTALL) + violations = [] + section = "" + for block in re.split(r"(?m)^(?=\s*#{1,6}\s)", body): + hm = re.match(r"\s*#{1,6}\s*(.+)", block) + if hm: + section = hm.group(1) + section_in_scope = bool(SCOPE_HEADING.search(section)) + for para in re.split(r"\n\s*\n", block): + scan_lines = [] + for ln in para.splitlines(): + s = ln.strip() + if re.match(r"-\s*\[[ xX]\]", s): # checklist item + continue + if s.startswith(">"): # blockquote (bot NOTE) + continue + if s.startswith("_Status key"): # legend + continue + if s.startswith("#"): # heading line + continue + scan_lines.append(ln) + chunk = "\n".join(scan_lines) + if not chunk.strip(): + continue + if not (section_in_scope or SCOPE_PARA.search(chunk)): + continue + # A markdown table row is its own claim unit — scan each row so an + # artifact two rows down cannot excuse a bare row. + units = chunk.splitlines() if chunk.lstrip().startswith("|") else [chunk] + for unit in units: + _scan_unit(unit, violations) + return violations + + +def _add(violations, kind, token, unit): + violations.append({ + "kind": kind, + "token": token, + "snippet": re.sub(r"\s+", " ", unit.strip())[:120], + }) + + +def _positive_verdict(unit): + """A non-negated verdict token in this unit, or None.""" + for m in VERDICT.finditer(unit): + if not _negated(unit, m.start()): + return m.group(0) + return None + + +def _scan_unit(unit, violations): + # ── VERDICT: excused by a co-located inspectable artifact. + if not ARTIFACT.search(unit): + tok = _positive_verdict(unit) + if tok: + _add(violations, "verdict", tok, unit) + + # ── OBSERVATION: needs an observation-class artifact. A /blob/ code + # permalink does NOT excuse it. + if not OBS_ARTIFACT.search(unit): + for m in OBSERVATION.finditer(unit): + if _negated(unit, m.start()): + continue + _add(violations, "observation", m.group(0), unit) + break + + # ── DEFERRAL: excused by a co-located tracker, NOT by an artifact. + if not TRACKER.search(unit): + dm = DEFERRAL.search(unit) + if dm: + _add(violations, "deferral", dm.group(0), unit) + + # ── CI RESTATEMENT (item 11): unconditional in validation scope. No + # verdict co-location required, no "beyond-CI"/"as context" excuse — + # a carve-out here is an instruction to phrase every violation as the + # exception. + cm = CI_RESTATEMENT.search(unit) + if cm: + _add(violations, "ci-restatement", cm.group(0), unit) + + # ── INFLATED VERDICT (item 11): proof language co-located with an + # admission the surface was not exercised. + nm = NOT_EXERCISED.search(unit) + if nm and _positive_verdict(unit): + _add(violations, "inflated-verdict", nm.group(0), unit) + + # ── STEP WAIVER (item 14): an impossibility argument never discharges a + # lane derived from an executable Manual testing step. + sm = STEP_WAIVER.search(unit) + if sm: + _add(violations, "step-waiver", sm.group(0), unit) + + # ── TRUNCATED IDENTIFIER (item 16): a co-located resolver does NOT + # excuse — the resolver resolves the full id, not the fragment the + # reader holds. Hash-equality prose is exempt. + if not HASH_EQUALITY.search(unit): + tm = TRUNCATED_ID.search(unit) + if tm: + _add(violations, "truncated-identifier", tm.group(0), unit) + + # ── BARE IDENTIFIER (item 12): an id with no resolving link and no + # re-hosted capture is a digging assignment. + if not RESOLVER.search(unit) and not OBS_ARTIFACT.search(unit): + bm = BARE_ID.search(unit) + if bm: + _add(violations, "bare-identifier", bm.group(0), unit) + + # ── MUTABLE REF (item 16): pin evidence links to a SHA. + mm = MUTABLE_REF.search(unit) + if mm: + _add(violations, "mutable-ref", mm.group(0)[:60], unit) + + # ── DUMP RESOLVER (item 13): a positive verdict whose only resolver is a + # raw dump behind a link. The digging moved a hop away, it did not + # disappear. + if _positive_verdict(unit) and DUMP_LINK.search(unit) and not IMAGE_EMBED.search(unit) \ + and not LIVE_LINK.search(unit): + _add(violations, "dump-resolver", DUMP_LINK.search(unit).group(0)[:60], unit) + + # ── LINK-ONLY EXHIBIT (item 15): a live permalink defers validation + # behind click + auth + query rendering. Needs the visual too. + if _positive_verdict(unit) and LIVE_LINK.search(unit) and not IMAGE_EMBED.search(unit): + _add(violations, "link-only-exhibit", LIVE_LINK.search(unit).group(0)[:60], unit) + + # ── DATA-ONLY EXHIBIT (item 17): telemetry claim with neither a visual + # nor a live link carries no liveness provenance — extracted data is + # indistinguishable from data typed by hand. + if _positive_verdict(unit) and TELEMETRY_VOCAB.search(unit) \ + and not IMAGE_EMBED.search(unit) and not LIVE_LINK.search(unit): + _add(violations, "data-only-exhibit", TELEMETRY_VOCAB.search(unit).group(0), unit) + + +def _negated(text, pos): + """A verdict token preceded by a negator is a hedge, not a claim.""" + pre = text[max(0, pos - 16):pos].lower() + if re.search(r"\b(not|never|no|isn't|aren't|cannot|can't|without|un|yet)\s*$", pre): + return True + # 'unverified' / 'unproven' — negator fused onto the token + if pre.endswith("un"): + return True + return False + + +if __name__ == "__main__": + main() diff --git a/domains/pr-workflow/skills/evidence/probes/metametrics-context.test.tsx b/domains/pr-workflow/skills/evidence/probes/metametrics-context.test.tsx new file mode 100644 index 00000000..ae61df61 --- /dev/null +++ b/domains/pr-workflow/skills/evidence/probes/metametrics-context.test.tsx @@ -0,0 +1,68 @@ +// Probe — MetaMetrics context value identity. +// +// PLACEMENT: copy to `ui/contexts/__render_probe__.test.tsx` in a metamask-extension tree. +// The imports below are relative to `ui/contexts/`, so a different destination resolves +// nothing and the suite fails to run with "Cannot find module" — which is a failed probe, +// not a measurement. `probe_dest` in the evidence workflow must match this path. +// +// The claim under test is about breadth: "all N consumers avoid unnecessary re-renders". +// `useContext` re-renders a consumer when the value's IDENTITY changes, and that is not a +// per-consumer property — so one distinct value across N parent renders means every +// consumer is spared, and N distinct values means none is. Counting distinct values is +// therefore the measurement the claim actually rests on; counting one consumer's renders +// would only ever describe that consumer. +// +// Resolves against both `metametrics.js` and `metametrics.tsx`, so the same file measures a +// base commit and a head commit that renamed it — the comparison is the point. +import React, { useContext, useRef, useState } from 'react'; +import { act } from '@testing-library/react'; +import configureStore from '../store/store'; +import { renderWithProvider } from '../../test/lib/render-helpers-navigate'; +import mockState from '../../test/data/mock-state.json'; +import { MetaMetricsContext, MetaMetricsProvider } from './metametrics'; + +let consumerRenders = 0; +let distinctValues = 0; +let bump: (() => void) | undefined; + +function Consumer() { + const value = useContext(MetaMetricsContext); + const last = useRef(null); + if (last.current !== value) { + last.current = value; + distinctValues += 1; + } + consumerRenders += 1; + return null; +} + +function Parent() { + const [, setN] = useState(0); + bump = () => setN((n) => n + 1); + return ( + + + + ); +} + +describe('MetaMetrics context value identity', () => { + it('counts distinct context values across parent re-renders', () => { + const PARENT_RENDERS = 5; + consumerRenders = 0; + distinctValues = 0; + + renderWithProvider(, configureStore(mockState)); + for (let i = 0; i < PARENT_RENDERS; i++) { + act(() => { + bump?.(); + }); + } + + // eslint-disable-next-line no-console + console.log( + `RENDER_COUNT consumer=${distinctValues} parentRenders=${PARENT_RENDERS + 1} consumerRenders=${consumerRenders}`, + ); + expect(consumerRenders).toBeGreaterThan(0); + }); +}); diff --git a/domains/pr-workflow/skills/evidence/references/aep-local-run.md b/domains/pr-workflow/skills/evidence/references/aep-local-run.md new file mode 100644 index 00000000..cccb6b6e --- /dev/null +++ b/domains/pr-workflow/skills/evidence/references/aep-local-run.md @@ -0,0 +1,81 @@ +# Running AEP locally + +Everything the local Autonomous Engineering Platform run needs: bring-up, submit, poll, +fetch artifacts, tear down. The hosted instance (`aep.dev.web3factory.consensys.net`) has +not resolved since 2026-06, so local is the only path. + +The skill body links here rather than carrying this inline. An AEP run is the heaviest +lane in the catalog and most validations do not need it — a falsifying test, a single +screenshot, or an artifact CI already produced usually closes the same falsifier. Read +this when you have decided an AEP run is warranted. + +Every bullet below cost a failed run at least once. + +## Preflight — bring up what is down + +The hosted AEP doesn't resolve (`aep.dev.web3factory.consensys.net` is dead as of 2026-06). Everything runs locally. Health-check, then bring up only what's down. + +Fast checks: + +```bash +AEP=~/Code/metamask/metamask-autonomous-engineering-platform +curl -fsS localhost:3000/health >/dev/null && echo "control-plane up" || echo "control-plane DOWN" +curl -fsS localhost:8233 >/dev/null && echo "temporal UI up" || echo "temporal DOWN" +docker ps --format '{{.Names}}' | grep -E 'mm-aep-postgres-dev|mm-aep-temporal-dev' +``` + +Bring-up order (each in its own shell; details + env in the reference): +1. `yarn dev:postgres` (docker `postgres:16-alpine`, `mm-aep-postgres-dev`, port 5432) +2. `yarn dev:temporal` (temporal dev server; UI on 8233) +3. `yarn db:migrate` +4. **worker** — `yarn dev:worker` on **Node ≥ 24.13**, env `ANTHROPIC_API_KEY=host-subscription`, `CLAUDE_CODE_EXECUTABLE=~/.local/bin/claude`, `GITHUB_TOKEN="$(gh auth token)"`, `SANDBOX_PROVIDER=local` (needs JFrog `npm login` first; relies on uncommitted local patches) +5. `yarn dev:control-plane` (`localhost:3000`) + +If any of the local patches (`local-sandbox-adapter.ts` timeout, `claude-agent-runner.ts` auth, the `perf-validation/` graph) are missing from the working tree, the reference says how to restore them — `git status` in the AEP repo should show them modified/untracked. +## Run mechanics — submit, poll, fetch + +The control-plane is a thin REST shell. Submit a PR-validation task, poll the run, pull artifacts from the evidence bundle. + +```bash +CP=localhost:3000 +PR="https://github.com/MetaMask/metamask-extension/pull/" + +# Submit (publishEvidence:false ALWAYS for local runs — the platform otherwise +# writes to the public PR body even on failure, leaking local paths/usernames) +RUN_ID=$(curl -fsS -X POST "$CP/v1/tasks" -H 'content-type: application/json' -d '{ + "repo": "MetaMask/metamask-extension", + "title": "Visual validation — PR #", + "taskClass": "visual_validation", + "externalRef": "'"$PR"'", + "payload": { "prUrl": "'"$PR"'", "description": "", "publishEvidence": false } +}' | node -e 'process.stdin.on("data",d=>console.log(JSON.parse(d).runId||JSON.parse(d).id))') + +# Poll +curl -fsS "$CP/v1/runs/$RUN_ID" | node -e 'const r=JSON.parse(require("fs").readFileSync(0));console.log(r.status); (r.evidenceBundle?.artifactRefs||[]).forEach(a=>console.log(a.name,a.mediaType))' + +# Fetch an artifact +curl -fsS "$CP/v1/runs/$RUN_ID/artifacts/" -o /tmp/ +``` + +- `taskClass`: `visual_validation` or `perf_validation`. The worker auto-enriches the payload from `prUrl` (pulls headSha, base, diff, files, linked issues via the GitHub app) — you only supply `prUrl` + a `description` targeting hint. +- The **targeting hint** (`payload.description`) is how you steer the agent to the surface under test. Be specific: which screen, which control, what to toggle. For hard-to-reach surfaces, name the reachable fallback (e.g. the Shield entry modal stands in for the Perps tutorial modal, which is gated in the default fixture). +- Artifact regex allows **png/jpg/log/txt only** — no video. Screen recordings need the side-channel recipe (catalog + publishing reference). + +### Concurrent runs (multiple agents / parallel lanes) + +Five shared resources need per-run isolation on one machine — collisions cross-contaminate evidence *silently* (wrong session's logs attributed to a run), which is an integrity failure, not flakiness: **(1)** CDP debug ports — derive per run, never hardcode; **(2)** e2e harness service ports (anvil/proxy/fixture/mocha) — one e2e run at a time per worktree, one worktree per agent (`wt new`), and never rebuild `dist/` in a worktree with an active run; **(3)** artifact dirs — per-run namespaces; `test-artifacts/` is per-worktree shared state, harvest failure artifacts before the next run overwrites the same test-title dir; **(4)** evidence-repo uploads — run-scoped paths (`pr-//`), retry-with-fresh-sha on 409, never overwrite another run's published files; **(5)** commit-pinning — pin only after your own final upload lands, verifying your files exist at that sha. Safe to share: JFrog login, a read-only `dist/`, the AEP stack itself. + +### Trust the evidence (anti-reward-hacking) + +A green result is not proof. The vacuous-pass trap is the floor: if `promptCrafter` errors, the chain "passes" via skip with **zero artifacts** — a pass is only real if `evidenceBundle.artifactRefs` is non-empty with the expected media. Beyond that, every lane must clear a trustworthiness gate before you believe or publish it: **does the artifact show the *claimed* surface** (not a spinner/wrong screen), **does the test exercise the *changed* code** (fails on `main`), **does the signal exceed noise**, **could the assertion have failed**? The Claim Card's Falsifier is the anchor. Full gate + per-lane traps: **[references/evidence-trustworthiness.md](references/evidence-trustworthiness.md).** + +### perf_validation caveat + +The `perf-validation/` graph is **uncommitted local AEP work** (added 2026-06-11). It writes falsifiable network/static/smoke assertions and gives the tester deterministic `.aep/` helpers (CDP netlog, phase segmentation, source-map chunk membership). It requires a `yarn webpack --test` build first (the browserify `build:test` has no code splitting, so `import()` never hits the network there). Temporal caps activity results at ~2MB — artifact refs must be content-free; only `evidenceBundle` carries base64. If the graph isn't in the working tree, perf runs won't register — fall back to manual DevTools/CDP capture (catalog). +## Teardown — always, on every exit path + +The stack is the heaviest thing this skill starts — postgres + temporal + a Node worker + control-plane — and the worker holds a live Claude session while the autonomous run itself spends tokens. It is **on-demand, not resident**: bring it up for the validation window, **tear it down when the run(s) finish**. Left up, it's the single largest reclaimable footprint on a shared host and quietly keeps a Claude seat warm. + +- **On a host managed by `aep-stack` (systemd):** `aep-stack up` to preflight, **`aep-stack down` when done** — stops the services; the `--rm` postgres/temporal containers are removed, so state resets on the next `up` (fine — each run is fresh anyway). +- **Otherwise:** stop the `yarn dev:*` processes and `docker rm -f mm-aep-postgres-dev mm-aep-temporal-dev`. +- **Tear down on every exit path** — pass, refutation, *or* abort. A failed or abandoned run leaves the stack up exactly as much as a passing one; the usual leak is walking away after a refutation without stopping it. diff --git a/domains/pr-workflow/skills/evidence/references/claim-extraction.md b/domains/pr-workflow/skills/evidence/references/claim-extraction.md new file mode 100644 index 00000000..e5654608 --- /dev/null +++ b/domains/pr-workflow/skills/evidence/references/claim-extraction.md @@ -0,0 +1,63 @@ +# Claim extraction + +The linchpin of evidence: before choosing any lane, turn the PR into a **falsifiable, surface-specific claim**. Every lane is only as good as the claim it tests. A vague claim ("improves perf", "fixes the bug") can't be proven or refuted; a sharp claim names the precondition, action, observable outcome, and what would disprove it. + +## Read these, in order + +1. **PR body** — Description (what/why), `Fixes #N`, Manual testing steps, the Before/After intent. +2. **Linked issue(s)** — the bug report / acceptance criteria; "Steps to reproduce" and "Expected vs actual" are the claim in the reporter's words. +3. **The diff** (`gh pr diff`) — what actually changed: which surfaces, controllers, modules. The claim must be anchored to what the code can do, not only what the body promises. +4. **Labels / type** — bug vs feat vs perf vs refactor changes the claim shape (see [special cases](#special-cases)). + +## Extraction steps + +1. **Asserted change** — what does the PR say it does? (body + issue) +2. **Anchor to the diff** — which surface/module changed? Reconcile intent with the diff. If the body promises X but the diff can't deliver X, **flag the drift** — that's a finding, not a claim. +3. **Phrase as falsifiable** — `Given , when , then .` The outcome must be observable and checkable. Replace vague verbs (improve / fix / handle / support) with the concrete observable. +4. **Pin the surface + reachability** — exact screen / API / metric. Reachable in the default fixture, or does it need state seeding, a feature flag, or a fallback surface? + - **A surface need not be a screen.** A pipeline's job graph, a build artifact, a policy file, a telemetry shape, or a harness's determinism are all legitimate surfaces with their own falsifiers. Do not force a user-visible observable onto a claim that does not have one — routing a CI or build claim through a product effect is the *wrong* bar, not a stricter one. + - **When the changed code is the automation, the PR's own run may not exercise it.** A CI-config diff commonly skips the very path it edits (build reuse, `needs-*` resolution, event-type conditions). Execute the changed workflow where its trigger conditions hold — a test fork, a branch whose name satisfies the condition — with the failure state forced. Name that substitution explicitly; a claim about *this* repo's pipeline is not proven by a run on another. +5. **Classify the type** → routes to lanes via the matching guide: visible UI · non-visible perf · telemetry · persisted-state · build-output · behavior-no-UI. +6. **Decompose mixed claims** — a PR that changes UI *and* shifts a metric is two claims; validate each. + +## Claim Card (output) + +``` +Claim: Given , when , then . +Surface: (reachable? seed / flag / fallback: …) +Type: → lanes +Falsifier: +Baseline: +``` + +One card per claim. For a refactor, the claim is a **negation** (see below). + +## Claim quality bar + +A good claim is **falsifiable** (observable outcome + clear falsifier), **surface-specific** (names the exact screen/API/metric, not "the app"), **diff-anchored** (the changed code can plausibly produce it), **bounded** (one behavior, one precondition), and **measurable** where quantitative (a number + threshold, not "faster"). + +## Anti-patterns → refinements + +| Vague claim | Refined | +|---|---| +| "Improves performance" | "Opening the Activity tab: TBT drops below 200ms (was >600ms)" — name the interaction, metric, threshold | +| "Fixes the bug" | "With privacy mode on, the Perps tab balance is masked" — observable behavior + precondition + surface | +| "Refactor, no behavior change" | Negation claim: "behavior of `` is unchanged" → prove via a red-on-base test that stays green / snapshot / identical output, **not** a screenshot | +| "Adds a null check" (restates the diff) | "No crash when `` is null on ``" — the behavior, not the code | +| Body promises X, diff does Y | Not a claim — **flag the drift** to the author | + +## Special cases + +- **Refactor / no-op:** the claim is "nothing observable changed." Falsifier = any behavior/output diff. Lanes: regression test stays green, snapshot diff empty, bundle/output identical (D1/D2), benchmark within noise. A passing screenshot proves nothing here. +- **Bug fix:** the strongest claim form ships its own falsifier — a test that fails on `main` and passes on the branch (catalog **B3**). Extract the claim straight from the issue's "Expected vs actual." +- **Perf:** always quantify — metric + interaction + threshold + baseline. Without a number it isn't falsifiable. +- **Persisted-state / migration:** claim = "upgrading from `` preserves `` and applies ``." Falsifier = corrupted/lost state. Baseline = a profile from the prior version (catalog **F1**). +- **Flag-gated:** two claims, one per flag state (catalog **F5**). + +## Worked examples + +- **Visible (#42683):** body "privacy mode doesn't hide the Perps balance"; issue: expected masked, actual visible; diff touches the Perps balance component. → **Claim:** *Given privacy mode on, when I open the Perps tab, the balance is masked.* **Surface:** Perps tab (gated → fallback: Shield entry modal). **Type:** visible → A1/B1. **Falsifier:** balance digits visible under privacy mode. **Baseline:** same flow on base reproduces the bug. +- **Perf:** body "defer Rive wasm at startup"; diff: dynamic `import()` of the Rive runtime. → **Claim:** *On cold start of the home view, the Rive wasm chunk is not requested until the animation surface mounts.* **Surface:** startup network + chunk graph. **Type:** perf → A2/C6/D2. **Falsifier:** the chunk appears in the cold-start waterfall. **Baseline:** base requests it at startup. +- **Migration:** diff adds migration NNN. → **Claim:** *Loading a profile from `` applies migration NNN; `changedKeys = {}`; all other state intact.* **Type:** state → F1. **Falsifier:** an untouched controller mutated, or migrated state malformed. **Baseline:** a prior-version profile. + +A sharp claim is also a good recipe **proof target** (ADR-0058): precondition → action → observable maps to pre-conditions → assertions → screenshot points. Extraction pays off in both lanes. diff --git a/domains/pr-workflow/skills/evidence/references/evidence-catalog.md b/domains/pr-workflow/skills/evidence/references/evidence-catalog.md new file mode 100644 index 00000000..adca4077 --- /dev/null +++ b/domains/pr-workflow/skills/evidence/references/evidence-catalog.md @@ -0,0 +1,300 @@ +# Evidence catalog + +The menu of evidence kinds for validating a MetaMask **extension** PR, with **what each proves**, **how to capture it (verified against the live repo)**, and **when to reach for it**. AEP is the primary autonomous engine; the rest are complementary. The skill's job is to **match evidence to the claim** and to **proactively suggest kinds the author didn't think of**. + +Pick the evidence that would **falsify the claim if it were false**. Prefer a lane that yields an artifact a reviewer can independently re-check (a link, an image, a number, a replayable trace) over prose. Don't run the whole menu — match, then capture. Capture commands cite `~/Code/metamask/metamask-extension`; verify script names against its `package.json` (they drift). + +Legend: **first-class lanes** are `##`-headed; closely-related variants are sub-bullets. Capture marked *(manual)* has no repo helper — it's a DevTools/CDP action. + +--- + +## Lanes at a glance + +43 lanes in 7 families. Each lane below has a full spec in its own section — what it proves, how to capture it, and its trust gate. Family G is written as one-liners rather than full sections, because those lanes are links and counts rather than captures. + +| Family | Lanes | | +|---|---|---| +| **A. AEP harness (primary, autonomous)** | 3 | `A1` visual_validation · `A2` perf_validation · `A3` AEP bundle byproducts | +| **B. Behavior & flow proof** | 7 | `B1` Visual before/after via the mm CLI · `B2` E2E trace + video · `B3` Falsifying regression test · `B4` Component / Storybook visual · `B5` Accessibility · `B6` Flaky-stability rerun · `B7` Deterministic interleaving test | +| **C. Performance & render** | 9 | `C1` Startup / custom traces + phase segmentation · `C2` Web vitals · `C3` Long-task / TBT · `C4` React render & selector proof · `C5` Benchmark A/B · `C6` DevTools / CDP profiling · `C7` Memory stability over a flow · `C8` Same-window app + DevTools capture · `C9` Retention-path analysis | +| **D. Build** | 7 | `D1` Bundle-size diff · `D2` Chunk membership / source-map · `D3` LavaMoat policy / supply-chain capability diff · `D4` Manifest permissions diff · `D5` Build-variant matrix · `D6` Authored-vs-authoritative substitution A/B · `D7` Build & rebuild duration A/B | +| **E. Production telemetry** | 3 | `E1` Sentry query links · `E2` Tempo distributed traces · `E3` Sentry error-event / breadcrumb shape | +| **F. Extension integrity (high-stakes, extension-specific)** | 8 | `F1` State migration / upgrade · `F2` Vault / keyring round-trip · `F3` Transaction simulation / gas · `F4` Provider / dapp connectivity · `F5` Feature-flag matrix · `F6` Snaps / multichain execution · `F7` i18n usage · `F8` SES lockdown / runtime containment | +| **G. CI, review & process** | 6 | `G1` CI check links · `G2` Coverage delta · `G3` Automated-reviewer output · `G4` Manual reproduction steps · `G5` CI-workflow change, run on a test fork · `G6` CI job-duration delta | +--- + +# A. AEP harness (primary, autonomous) + +## A1. visual_validation — before/after screenshots +- **Proves:** a visible UI change on the real surface. Deterministic state seed + agent navigation; PNG artifacts in `evidenceBundle.artifactRefs`. +- **Capture:** `taskClass: visual_validation`, `payload.prUrl` + `description` hint. See [aep-local-run.md](aep-local-run.md). +- **Reach for it:** anything a human would screenshot for the PR's `### After`. + +## A2. perf_validation — falsifiable network/static/smoke assertions +- **Proves:** non-visible behavior (hover-preload, no double-fetch, chunk membership, smoke boot). CDP netlog / phase segmentation / source-map membership. +- **Capture:** `taskClass: perf_validation` (local/uncommitted graph; needs `yarn webpack --test`). Falls back to C6/D2 manually if the graph isn't present. + +## A3. AEP bundle byproducts (free with any run) +- Test results (`executionResult`/`checkResults`), diff stats, automated `reviewResult` findings, and the **LangSmith trace** of the run. Include the relevant subset; link the trace for auditability. + +--- + +# B. Behavior & flow proof + +## B1. Visual before/after via the `mm` CLI (`visual-testing`) +- **Proves:** UI behavior on a real headed build, with controlled state/network. Defers to the public `visual-testing` skill. +- **Capture:** `yarn build:test:webpack` → `dist/chrome`; `yarn mm launch` → `mm describe-screen` / `mm screenshot` / `mm click` / `mm type` / `mm navigate`. README: `test/e2e/playwright/llm-workflow/`. + - **Degraded-path:** `mm mock-network` to force error/slow responses (session-scoped; add after launch, before the action; can't intercept pre-launch startup). + - **a11y / DOM:** `mm accessibility-snapshot` and `mm cdp` (per the `visual-testing` skill; `a11yRef`s are ephemeral — re-describe after navigation). + +## B2. E2E trace + video (Playwright / Selenium) +- **Proves:** a full flow works, replayably. The strongest "it works end-to-end" artifact. +- **Capture (Playwright):** `yarn playwright test `; trace is `'on'` by default (`playwright.config.ts`), video is `'off'` (enable in config if needed). View: `yarn test:e2e:pw:report`. Artifacts under `public/playwright/`. +- **Capture (Selenium):** `yarn test:e2e:single --browser chrome|firefox|all [--retries n]`; screenshots auto-captured on failure to `test/test-results/e2e/`. + +## B3. Falsifying regression test ⭐ +- **Proves — strongest single proof a fix targets the bug:** a new test that **fails on `main` and passes on the branch**. Show both runs. + - **Engine: the `red-on-base` skill.** +- **Capture:** add the test, run it on the PR branch (pass) and on the PR's **merge-base** (fail) — pin the base, don't use whatever `main` points at today. Pair with the PR's `Fixes #N`. **Read the base failure's message, not its exit code:** it must fail on the assertion that encodes the bug. A `ModuleNotFoundError`, a missing fixture, or an unrelated pre-existing red produces an identical non-zero exit and falsifies nothing. +- **Reach for it:** every bug-fix PR. If you can't write a test that fails on main, question whether the fix addresses the reported bug. + +## B4. Component / Storybook visual +- **Proves:** a component renders across states/props in isolation. +- **Capture:** `.storybook/` present; `yarn storybook` (port 6006), `yarn storybook:build`, `yarn test-storybook` (visual + a11y via `@storybook/addon-a11y`). Jest snapshot diffs for serialized output. + +## B5. Accessibility (a11y) +- **Proves:** no a11y regression / an a11y improvement. +- **Capture:** `yarn test-storybook` (Storybook a11y addon) for components; `mm accessibility-snapshot` for live flows. (No axe-core in the e2e suite — don't claim it.) + +## B6. Flaky-stability rerun +- **Proves:** a flow/test is not flaky (or that a fix removed flakiness). +- **Capture:** Playwright retries `1` on CI / `0` local (`playwright.config.ts`); Selenium `--retries n`; benchmarks default `--retries 2`. Run N× and report the pass rate. See `e2e-flakiness-patterns`. +- Sub: jest snapshot diffs; a unit run for just the changed module (`yarn test:unit `); fuzz/property tests for parsers/encoders. + +--- + +## B7. Deterministic interleaving test (concurrency / temporal-ordering) ⭐ +- **Engine:** `race-condition-repro` — run it rather than hand-rolling the harness. +- **Proves:** an ordering guarantee under interleaving — retry, cancellation, supersession, debounce, locks, queues, async state machines — where the correctness *is* the ordering under races, not a value. +- **Capture:** force each race deterministically — `jest.useFakeTimers()` + `advanceTimersByTimeAsync(DELAY)` to fire the delayed action at a known point; `Promise.all([opA, opB])` to overlap operations; `advanceTimersByTimeAsync(0)` to step to a precise interleaving point; then assert the ordering/cancellation outcome for **each** guarantee, including asymmetric ones (one path canceled → its recovery event `.not.toHaveBeenCalled()`; another must complete → `.toHaveBeenCalledWith(...)`). Corroborate with transition telemetry; for the integration path, a live forced-race capture (C8/CDP, the #44610 technique). +- **Trust-gate:** the test must **actually interleave** — time advanced into the pending window, the superseding op injected *during* it. A sequential run exercises no race and is a vacuous green. Verify the interleaving, not just the assertion. + +# C. Performance & render + +## C1. Startup / custom traces + phase segmentation +- **Proves:** which startup phase moved (init → FirstRender → interactive), per named span. +- **Capture:** `shared/lib/trace.ts` `TraceName` enum (UIStartup, LoadScripts, FirstRender, …); read in test/debug via `window.stateHooks.getCustomTraces()`. LCP fallback mark: `performance.mark('mm-hero-painted')`. `driver.collectMetrics()` aggregates paint/navigation/long-task/custom traces in e2e. + +## C2. Web vitals — INP / FCP / LCP / CLS +- **Proves:** a user-centric metric moved. `ui/helpers/utils/web-vitals.ts` via `web-vitals/attribution` (attribution names the causing element). +- **Capture:** `window.stateHooks.getWebVitalsMetrics()` (test/debug) → `{inp, fcp, lcp, cls, *Rating}`. Thresholds: INP good<200/poor>500, FCP<1800/3000, LCP<2500/4000, CLS<0.1/0.25. +- **Caveat:** **INP fires on all pages; FCP/LCP/CLS do not fire on popup pages** (sidepanel/E2E only). For extensions, INP is the high-value runtime metric. + +## C3. Long-task / TBT +- **Proves:** main-thread blocking during an interaction dropped. This is where **TBT** lives (the web-vitals lib lane does *not* collect TBT). +- **Capture:** `ui/helpers/utils/performance-observers.ts`; `window.stateHooks.getLongTaskMetricsWithTBT()` → `{count, totalDuration, maxDuration, tbt, tbtRating}`. TBT good<200 / needs-improvement<600 / poor>600. Sampled 10% prod / 100% test. + +## C4. React render & selector proof + - **Engine: the `react-render-delta` skill.** Delegate the measurement to it; it runs the source/delivery/metric gates, derives the needle from real build output, repeats the capture, and returns a band (or "not resolvable at this n" with an MDE). evidence packages the result. +- **Proves:** a component/selector stopped over-rendering (cascade-amplification before/after). +- **Capture:** WDYR via `ENABLE_WHY_DID_YOU_RENDER` (`.metamaskrc` or env) — wired in `app/scripts/development/wdyr.ts` (`trackAllPureComponents`); console logs each unnecessary re-render. `yarn devtools:react` for the Profiler flame graph. Selectors use `reselect`'s `createSelector`, which **does expose a real `.recomputations()` counter** — read it (sample on an interval if the count should visibly climb) rather than injecting a log into the selector body; an injected log is an authored claim, a library API is an observation. *(This entry previously said there was no built-in counter. There is.)* +- **Bar:** the delivery check comes before the number. An arm whose manipulation cannot be observed in the built bundle produces a null indistinguishable from "small effect" — and reports as the second. + +## C5. Benchmark A/B +- **Proves:** a startup/journey/interaction timing moved, with a distribution not one sample. +- **Capture:** `yarn test:e2e:benchmark` (`test/e2e/benchmarks/run-benchmark.ts`); presets in `shared/constants/benchmarks.ts` (`startupStandardHome`, `sendTransactions`, `swap`, `dappPageLoad`, …). +- **Caveat:** the rolling baseline (`MetaMask/extension_benchmark_stats`) can **silently freeze** behind a green check (the `store-benchmark-stats` step is `continue-on-error`; happened 2026-04-02, PR #42947). Prefer a **paired A/B** (build both refs now, compare directly) over the stored baseline. +- **Treatment check first** — before trusting any delta, confirm the mechanism under test is actually active in each arm (split chunk present in head and absent in base; the span emitted; the flag evaluated). An arm without the treatment delivered is a no-op, not a control (2026-07-22, #42795). +- **A null needs its power stated** — "no change" and "underpowered" print the same result. When the run-to-run spread exceeds the effect under test, report **not resolvable at this n** and name the smallest detectable effect; never let it read as "no effect". Correcting a known bias (discarding a warm-up, alternating the starting arm) removes *that* bias and nothing more — it is not a trust gate, and the confounds you did not enumerate (thermal drift, background load, ordering within a round) stay live. + +### Capturing an authenticated view (the in-situ requirement) + +Headless Chrome's `--screenshot` cannot set cookies, so an authenticated dashboard +(Grafana/Tempo, Sentry Discover, an internal panel) screenshots as a login page. Drive +Chrome over CDP instead — inject the session cookie, navigate, capture: + +```bash +COOKIE_NAME=grafana_session COOKIE_VALUE="$sess" COOKIE_DOMAIN= \ + cdp-shot "" out.png 25000 1500 2400 +``` + +- **Deep-link to the exact view** so the capture and the reader's verification path are the + same URL (Grafana: `/explore?schemaVersion=1&panes=`). +- **Wait generously** — a trace waterfall or Discover table renders well after `load`. +- **Capture tall + `captureBeyondViewport`**, then crop; the interesting span is usually + below the fold, and cropping after the fact beats guessing a viewport. +- **Crop out the chrome that identifies the operator** (profile avatar, org switcher) + before the image leaves the machine. +- **Keep the trace/query id, timestamp, and result count in frame** — that is what makes + the exhibit reproducible rather than decorative. +- Never echo the cookie value, never commit it, never pass it to a subagent. + +## C6. DevTools / CDP profiling *(manual)* +- **Proves:** a flame-chart hot path shrank, a request was removed/deferred, frame rate held, or it holds on slow hardware. +- **Capture (manual via DevTools or `mm cdp`):** performance profile / flame chart; network waterfall (HAR) + request-count delta; **CPU throttling** (CDP `Emulation.setCPUThrottlingRate` — *no repo helper*, set it in DevTools); **animation/Rive FPS / dropped frames** (DevTools rendering FPS meter — *no repo helper*); JS coverage for dead-code. + +## C7. Memory stability over a flow *(manual)* +- **Proves:** a leak is fixed across repeated interactions (not one snapshot): retained heap stays flat, detached DOM nodes / listeners don't accumulate. +- **Capture:** DevTools heap snapshots before/after N cycles of the flow; compare retained size + detached nodes. +- Sub: redux dispatch/action count per interaction; network payload bytes; forced-reflow / layout-thrash count (DevTools Performance). + +## C8. Same-window app + DevTools capture *(manual)* +- **Proves:** the UI behavior **and** its internal evidence (console log, network row, storage state) in **one frame** — cause and effect temporally correlated in a single artifact. Two separate captures can't prove they came from the same run; one frame can. Canonical use: "the toast does NOT appear *while* the console shows the silent-handling path executed". +- **Capture (macOS, OS-level — Playwright `recordVideo` sees only the page viewport, never DevTools):** + 1. Tab-target DevTools: launch Chrome with `--auto-open-devtools-for-tabs` so DevTools opens **docked in the same window** (dock side persists per profile; set once via the DevTools ⋮ menu if a fresh profile defaults to undocked). + 2. MV3 **service-worker console has no dockable host** — open its dedicated inspector (`chrome://extensions` → *Inspect views: service worker*) and tile it flush beside the app window: `osascript -e 'tell application "Google Chrome" to set bounds of front window to {x, y, w, h}'` (the SW inspector is a Chrome window too and tiles the same way; CDP `Browser.setWindowBounds` also works per `windowId`).\ + 3. Record the union region, not a single window: stills `screencapture -x -R out.png`; video `screencapture -v -V -R out.mov`, then ffmpeg two-pass palette → GIF (recipe in [evidence-publishing](evidence-publishing.md)). First use prompts for macOS Screen Recording permission for the terminal. +- **Legibility rule:** console text dies in GIF downscale. Keep the GIF ≥720px wide, and pair it with (a) a full-res PNG of the same frame and (b) a text dump of the console via CDP (`Runtime.consoleAPICalled` on the SW target, `npx mm cdp` or a 20-line ws script) so the log lines are quotable/searchable. +- **Trust note:** arrange windows *before* triggering the behavior so the recording shows trigger → console line → UI (non-)reaction as one continuous take; a post-hoc composite of separate captures is exactly what this lane exists to avoid. + +--- + +## C9. Retention-path analysis — memory leak from code ⭐ *(static; lead for leak claims)* +- **Engine: the `memory-leak` skill.** For a memory-leak claim, delegate the analysis to `memory-leak` — it runs Phase-1 static pairing (and Phase-2 heap investigation if a primitive can't be paired) and returns the paired/unpaired sites + verdict. evidence keeps **memory leak** as the evidence category: it invokes the skill on the diff and packages the result (in-situ scan capture, plus the lifecycle test / retainer graph if Phase 2 ran) as the category's evidence. The lane spec below is the method that skill implements. +- **Proves:** "X is retained past its lifecycle boundary" / "collection Y grows unboundedly" — argued from code, no runtime needed. This is the lane that works at **review time** (does this PR *introduce* retention?) and leads fix-side validation (does the fix *break* the retention path?). C7 is the runtime corroborator, not the lead — leaks need many cycles to exceed noise. +- **Capture — the holder → held → boundary triple, per suspect:** (1) the **holder** (listener, closure, module singleton, accumulating collection, timer); (2) the **held set** — the *specific* objects pinned (list the closure's captures; note when a closure links two objects' GC); (3) the **outlived boundary** (`destroy()`, stream close, instance replacement, request completion). Method: **pair every acquire with its release site** (`on`↔`removeListener`, push↔drain, assign↔null) — the absence of the pair, cited at the acquire site, IS the finding. Four canonical shapes: unbounded accumulator (defeated guard, no drain) · stale-instance listeners on replacement · unremoved listener + capture set · retention past `destroy()`. +- **Scope to the diff, or you invent findings.** Classify every flagged primitive as *introduced by this PR* (in the added lines) vs *pre-existing* (already in the file). Charge only the introduced ones to the PR; report pre-existing un-paired primitives separately and uncharged. On extension#40684 the two new stream listeners each had a `removeListener` on `onStreamClosed` (the exact fix a reviewer suggested) and the new pending-request Map had its `.delete` — no leak introduced — while three pre-existing un-torn-down listeners were surfaced but left uncharged, matching how the human/bot reviewers treated them in-thread. This lane *is* the retention review automated; a heap snapshot (C7) is warranted only for an introduced primitive it cannot pair. +- **Corroborate:** a falsifying lifecycle test (force the boundary, assert release — listener count zero, singleton nulled, collection drained); C7 heap-over-flow with the **retainer graph naming the same path** the static argument named. +- **Trust-gate:** the triple must be specific ("this listener holds `patchStore` after `patchStore.destroy()`", not "might leak"); distinguish **bounded staleness vs unbounded growth** (severity differs); attribute **introduced vs pre-existing** honestly. + +# D. Build + +## D1. Bundle-size diff +- **Proves:** the build grew/shrank by a measured amount. Use the bundle-size CI output or a local build size comparison. + +## D2. Chunk membership / source-map +- **Proves:** a module moved to the intended (lazy) chunk and no longer ships on the critical path. Requires the webpack build. Mirrors AEP `perf-chunks`. + +## D3. LavaMoat policy / supply-chain capability diff + - **Engine: the `supply-chain-audit` skill** (umbrella — lockfile/manifest diff, advisories, Socket Security, install scripts), which delegates capability grants to **`lavamoat-policy`** (per-grant call-site justification). Delegate the dependency change to the umbrella; it returns a disposition per lane. evidence keeps **supply-chain capability diff** as the evidence category and packages the output. Note the lanes are independent: a clean policy diff does not mean a safe dependency, and a known CVE never appears as a new grant. +- **Proves:** a dependency change (bump/add/lockfile) grants **no *unjustified* new capability** — the supply-chain-risk lane. Note the bar: for a bump the policy *will* change, so "empty diff" is the WRONG test; the right test is **every new grant is justified by the dep's function**.. +- **Capture:** **Prefer the CI-generated policy whenever one is available.** `@metamaskbot update-policies` regenerates the policy files from a real run of the code and `validate-lavamoat-policies` fails the build on drift, so the committed policy on a bot-run PR *is* the authoritative artifact — diff that. Regenerating locally when a current CI policy exists only re-does a machine that is already trusted, and a local run's provenance is weaker (your node/OS/lockfile resolution, not CI's). **Local regen is the fallback**, for when the bot hasn't run yet, the branch is unpushed, or you need a variant CI didn't cover: `yarn webpack:lavamoat:policy:build` (`:mv2` / `:mv3` for variants) over `lavamoat/webpack/build/policy.json` (+ `policy-override.json`). Either way, `git diff` the policy across **all 8 variants** (mv{2,3}/{main,beta,flask,experimental}) — a grant can appear in one and not others. Then audit **grant-by-grant**: new **globals** (`fetch`, `importScripts`, `WebAssembly`) / **builtins** (`fs`, `child_process`) on a dep that shouldn't need them, new **packages** edges to powerful APIs, or an identifier substitution (`pkgC>name` replacing `pkgB>pkgA>name` = possible dep swap). Falsifier = a surprising grant ("I wonder what it's using this for"). Guide: lavamoat.github.io/guides/policy-diff/. `allowScripts` in `package.json` gates install scripts. + +## D4. Manifest permissions diff +- **Proves:** no permission/host-permission scope creep. +- **Capture:** `git diff app/manifest/v3/_base.json app/manifest/v2/_base.json` (+ `chrome.json`/`firefox.json`). Flag new sensitive perms (webRequest, broad host patterns). + +## D5. Build-variant matrix +- **Proves:** the change works across build types, not just main. +- **Capture:** `yarn build:test:flask` / `:beta` / `:mv2` (`ENABLE_MV3=false`, Firefox). Run the relevant lane per variant when behavior is build-type-gated. + +## D6. Authored-vs-authoritative substitution A/B ⭐ *(fixed head; lead for "the artifact restates a source" claims)* +- **Proves:** whether an artifact the PR *hand-wrote* agrees with the source it restates — a type vs the value's real type, a hand-maintained schema vs the generated one, a vendored constant vs the upstream export, a checked-in policy vs `update-policies` output. The finding is the **delta in a checker's output**, not a reading of the diff. +- **Shape:** both arms sit at the **same commit**; they differ by a *substitution*, not by a ref — so there is no build, no rebase, and no merge boundary to confound. + - **Arm A** — the PR as written, run through the checker. Must be **silent**. A non-empty Arm A means the instrument is broken and Arm B is unreadable (see trustworthiness gate item 19). + - **Arm B** — same tree, with the authored artifact replaced by the **derived** equivalent, exercised exactly as the real code exercises it. Every new diagnostic is a disagreement the authored version concealed. +- **Capture (TypeScript worked example — extension#44397, 2026-07-30):** + ```bash + # Arm A — baseline. Expect zero errors. + NODE_OPTIONS='--max-old-space-size=9216' npx tsc -p tsconfig.json --noEmit + # Arm B — probe files that substitute the derived type and call it as the caller does. + mkdir -p app/scripts/derive-probe && cp probe-*.ts app/scripts/derive-probe/ + NODE_OPTIONS='--max-old-space-size=9216' npx tsc -p tsconfig.json --noEmit # diagnostics = the findings + rm -rf app/scripts/derive-probe + ``` + One probe per claim, each naming the authoritative source in a header comment and calling the derived type the way the real call site does. Keep the probes as the artifact — they are the re-runnable falsifier. +- **Why it finds what review and CI miss:** the authored artifact compiles, so CI is green *by construction*. In a partially-migrated repo the asymmetry is structural — with `checkJs` off, a type written for a function whose callers are still `.js` is checked against nothing, and drifts silently forever. Those boundaries are where the lane pays. +- **Traps:** (a) **a substitution can fail for the wrong reason** — a diagnostic on an earlier property short-circuits the one under test, and counting exit codes reads that as confirmation; assert on the *specific* diagnostic, and re-probe with the earlier cause neutralised (`NonNullable<…>`, a targeted assertion) to isolate each claim. Same hazard as B3's "fails on base for the wrong reason." (b) **no authoritative source may exist** — an unshipped package's types, a lib not in tsconfig `lib`, a genuinely new boundary the repo owns. Hand-writing is then *correct*; report it as a cleared falsifier, not a finding. +- **Pairs with:** [lane-assertions.md](lane-assertions.md) for the recipe form; D3 when the substituted artifact is a LavaMoat policy. + +--- + +## D7. Build & rebuild duration A/B *(paired; lead for toolchain-change claims)* +- **Proves:** what a toolchain change costs or saves in the **dev loop** — a loader, transform, linter, or bundler swap. Distinct from `C5`, which times the shipped app at runtime; this times the build that produces it. The two move independently and in opposite directions often enough that measuring one and inferring the other is the failure this lane exists to prevent (`React Compiler` builds slower and runs faster; `thread-loader` builds faster and runs identically). +- **Shape:** paired A/B, both arms built now, on one machine, alternating order. **Cold and warm are separate questions and get separate numbers** — never one figure labelled "build time". +- **Capture:** N ≥ 5 per arm per mode, alternating. Cold: clear the cache explicitly between arms (`node_modules/.cache`, webpack `cache.cacheDirectory`) and state what was cleared. Warm: touch one source file, rebuild, discard the first result as pool warmup. Report median **and spread**; a median without spread hides a bimodal cache effect. +- **Falsifiers — each returns a favourable number when uncontrolled:** warm cache leaking into the "cold" arm (the largest confound, and the easiest to introduce by running arms in sequence); worker-pool startup counted once and amortised across rebuilds; core count, since parallel loaders scale with the runner and a laptop result does not transfer; watch-rebuild numbers presented as cold-build numbers. +- **Trust-gate:** state machine, core count, N, and cache handling per arm, or the number is unreproducible. A null result states the smallest effect the sample could have detected — "no difference" from N=3 is not a finding. Renders **no ship verdict**: a change that costs build time and buys runtime is a trade, and pricing it is not the same as taking it. +- **Corroborate:** `G6` for the CI half (different machine, different confounds), `C5` for the runtime half. A toolchain claim is not closed by one surface. + +# E. Production telemetry + +## E1. Sentry query links (before/after) +- **Proves:** error-rate / transaction count / latency moved in prod. A link a reviewer opens beats a chart screenshot. +- **Capture:** Sentry MCP (`search_events`/`search_issues`) → hand the discover/dashboard link with the before/after window, scoped to the release. Projects: `metamask` = prod, `metamask-performance` = CI. +- **Boundary:** PRs that *add/change span instrumentation* (volume/quota) → `/sentry-quota`, not this lane. +- **Perf-PR promotion (standard, not just complementary):** for a **performance-focused PR**, the main-branch Sentry **trend** for the affected metric across the PR's merge (before/after the merge commit's release) is **standalone lead evidence** — CI already sends every main/release `startupPowerUserHome` / journey benchmark to Sentry, so the trend is a real before/after on the actual metric, continuously tracked, with no local run. Prefer it over a local paired A/B when a clean merge-boundary window exists: it sidesteps the stale committed-baseline trap (`benchmark-baseline-staleness-paired-ab`). Still bound to the trust gate — a **windowed, release-scoped, one-click-resolvable** trend link with the merge boundary visible, never a prose "looks fine." A local interleaved paired A/B (C5) remains the precision complement when the merge window is noisy or the metric CI doesn't track (selector-eval count, re-render count, INP-on-typing — none of which CI captures). + +## E2. Tempo distributed traces +- **Proves:** a span/transaction now appears / is shaped correctly (e.g. background-RPC tracing). Link the trace + note the release. + +## E3. Sentry error-event / breadcrumb shape +- **Proves:** an instrumentation PR captures the intended error-event state / breadcrumbs (relevant after the Sentry-v10 error-event capture changes). Show the captured event payload. + +--- + +# F. Extension integrity (high-stakes, extension-specific) + +## F1. State migration / upgrade ⭐ +- **Proves:** a persisted-state change doesn't corrupt existing users. +- **Capture:** migrations in `app/scripts/migrations/NNN.ts`, runner `app/scripts/lib/migrator/`; scaffold with `./development/generate-migration.sh NNN`. The `NNN.test.js` asserts `meta.version` and that the `changedKeys` Set covers only mutated controllers — i.e. untouched state is preserved. Run it; show old-state-in / new-state-out. + +## F2. Vault / keyring round-trip +- **Proves:** no key/vault corruption; encrypt→decrypt is lossless. +- **Capture:** `app/scripts/lib/encryptor-factory.ts` (`@metamask/browser-passworder`, PBKDF2). E2E: `test/e2e/dist/vault-decryption-chrome.spec.ts`; `test/e2e/tests/vault-corruption/`. Storage-size via `getFileSize` on the encrypted blob. + +## F3. Transaction simulation / gas +- **Proves:** tx behavior/balance-changes/gas are correct before submit. +- **Capture:** `app/scripts/lib/transaction/containers/enforced-simulations.ts`; e2e `test/e2e/tests/simulation-details/`; mock `test/e2e/tests/confirmations/mocks/simulation.ts` (returns `gasUsed`, `callTrace`, `stateDiff`, token balance changes). TX_SENTINEL_URL in `shared/constants/transaction.ts`. + +## F4. Provider / dapp connectivity +- **Proves:** dapp integration works (injection, connect, requests). +- **Capture:** `yarn dapp` (serves `@metamask/test-dapp` on :8080); EIP-6963 `test/e2e/provider/eip-6963.spec.js`; multi-provider `test/e2e/multi-injected-provider/`; EIP-1193 reconnect tests under `test/e2e/tests/mm-connect/`. + +## F5. Feature-flag matrix (on/off) +- **Proves:** correct behavior in both remote-flag states (the Perps-gating class of bug). +- **Capture:** remote-feature-flag-controller (`app/scripts/lib/update-remote-feature-flags.ts`); flags come from `client-config.api.cx.metamask.io/v1/flags` — **not** `.metamaskrc`. In e2e, mock the response (see `test/e2e/tests/remote-feature-flag/`) to force each state; read via `uiState.metamask.remoteFeatureFlags`. + +## F6. Snaps / multichain execution +- **Proves:** snap behavior across multichain (e.g. `snap_startTrace`/`snap_endTrace`). +- **Capture:** `test/e2e/flask/snaps/preinstalled-example.spec.ts` (the snap-trace test), broader `test/e2e/snaps/`. Build flask (`yarn build:test:flask`). + +## F7. i18n usage +- **Proves:** no hardcoded strings; locales resolve. +- **Capture:** `yarn verify-locales` (`development/verify-locale-strings.js`); locales in `app/_locales/`. `yarn verify-locales:fix` to auto-fix. + +## F8. SES lockdown / runtime containment ⭐ +- **Proves:** the runtime defenses are **actually in force in the shipped artifact** — SES `lockdown()` and its taming levels, LavaMoat global scuttling, Snow's anti-escape hooks, Snaps compartments. Distinct from D3: D3 is the build-time *policy* (what a package may reach), this is whether containment *holds at runtime*. A correct policy ships alongside a lockdown that silently failed, and no policy diff would show it. +- **Capture:** `Runtime.evaluate` over CDP against the **built variant under discussion** — `Object.isFrozen(Object.prototype)`; a scuttled global throws while an exception-list global still resolves; `typeof SNOW === 'function'`; the `lockdown({…})` options as they appear *in the bundle*. Pair a positive with a negative — a check that only confirms the permitted case passes in a completely unlocked environment. +- **Bar — three divergences make this a lane, not a checkbox:** (1) the `lockdown()` call is wrapped in `try/catch` that logs to Sentry and **continues unlocked** (added for Firefox v56 contentscript injection), so it is a runtime assertion, never a guaranteed precondition; (2) **scuttling is off entirely in DEV builds** (`shouldScuttle = entryTask !== BUILD_TARGETS.DEV`); (3) **TEST builds widen the scuttling exception list** for chromedriver (`Proxy`, `ret_nodes`, `browser`, `chrome`, `indexedDB`). So **a green e2e run is evidence about a wider-open global than users get** — always state which build variant produced the evidence. +- **Reach for it:** any change touching the lockdown call site or its ordering (lockdown must precede untrusted code), the scuttling exception list, a taming level, compartment boundaries, or a `@lavamoat/snow` bump (Snow is patched in-repo — re-read the patch; see `supply-chain-audit`'s patch lane). + +--- + +# G. CI, review & process + +- **G1. CI check links** — `gh pr checks `; link the full suite (AEP's bundle is often `partial`). Always worth a one-line "all green" + link. +- **G2. Coverage delta** — `yarn test:unit:coverage` → `coverage/unit/` (and `yarn test:unit:webpack:coverage`); `codecov.yml`. Proves the new code is exercised. +- **G3. Automated-reviewer output** — independent bot (e.g. cursor[bot]) found nothing blocking. Complements, never replaces, behavior evidence. +- **G4. Manual reproduction steps** — human-followable steps that reproduce the fixed behavior; populates the PR template's Manual testing steps. +- **G5. CI-workflow change, run on a test fork** — a CI-YAML-only PR usually **cannot exercise the workflow it edits**: identical build output ⇒ builds reused from base ⇒ `needs-=false` ⇒ the workflow is *skipped* (`get-requirements.yml:654`). Escape: push to a branch literally named **`main`** (or `stable`) on your own test fork of the repo — `IS_RUN_EVERYTHING_BRANCH` (line 48) disables `find-reusable-builds` (line 310), so the workflow runs; `IS_CROSS_REPO_PR` is false inside the fork. Requires the workflow's secrets on the fork (`INFURA_PROJECT_ID`, `TEST_SRP_*` for benchmarks; `vars.`-gated Sentry/AWS steps skip cleanly) and a fork sync first. **State fork-scope in the published evidence** — it proves the workflow logic, not a run on the canonical repo.. +- **G6. CI job-duration delta** — compare job wall-clock across arms in the Actions UI or `gh run view`. **Falsifier: build reuse.** `get-requirements.yml` skips jobs when build output matches base, so a measured "speedup" is often a skipped job — confirm each arm actually ran the work before comparing. Runner class and queue time vary independently of the change; report job time, not wall-clock from push. Pairs with `D7`, which measures the same change on a machine you control. + +--- + +# Matching guide (claim → lanes) + +| The PR claims… | Lead with | Corroborate | +|---|---|---| +| a visible UI behavior | A1 / B1 visual | B2 recording for motion; B5 a11y | +| a fixed bug (any) | **B3 falsifying test** | A1/B1 if visible; E1 if it errored | +| preload / no-double-fetch / lazy-load | A2 perf | C6 netlog, D2 chunk | +| a render/over-render fix | C4 WDYR/profiler | C1 traces | +| interaction responsiveness | C2 INP, C3 TBT | C6 profile | +| startup/load timing | C5 benchmark (paired) | C1 phase traces, C2 FCP/LCP | +| smaller/cleaner bundle | D1 size | D2 chunk | +| a memory leak fixed / introduced | **C9 retention-path from code** (holder → held → boundary) | C7 heap-over-flow + retainer graph; falsifying lifecycle test | +| an error/crash fixed | E1 Sentry rate→0 | B3 test, A1 if visible | +| a dep change is safe | D3 LavaMoat + D4 manifest | D1 size; supply-chain-audit's patch/resolutions/ignore lanes | +| a mechanical migration / "rename-only" refactor | **D6 substitution A/B** (authored artifact vs its authoritative source) | B3 if behavior-visible; D1 for accidental output change | +| a hand-written type/schema/policy restates a source | **D6 substitution A/B** | G1 checks (as the *premise*: it compiles, which is why nobody noticed) | +| runtime containment / SES / scuttling | **F8 runtime containment** (on the shipped variant) | D3 policy; E1 for `Lockdown failed` events | +| persisted-state change | **F1 migration** | F2 vault | +| tx/confirmation behavior | F3 simulation | B2 e2e | +| dapp/provider behavior | F4 connectivity | B2 e2e | +| flag-gated behavior | F5 flag matrix | A1/B1 per state | +| snap behavior | F6 snaps | E2 trace | +| copy/localization | F7 i18n | A1 visual | +| CI workflow behavior | **G5 fork run** (branch named `main`) | G1 checks, G4 repro steps | + +Run the cheapest lane that yields an independently re-checkable artifact, confirm the claim holds, then escalate. Don't over-instrument a one-line copy fix; don't under-prove a startup-latency or migration claim with a single screenshot. diff --git a/domains/pr-workflow/skills/evidence/references/evidence-gate-setup.md b/domains/pr-workflow/skills/evidence/references/evidence-gate-setup.md new file mode 100644 index 00000000..21a2c998 --- /dev/null +++ b/domains/pr-workflow/skills/evidence/references/evidence-gate-setup.md @@ -0,0 +1,58 @@ +# Evidence gate — setup (optional, Claude Code only) + +`hooks/pr-evidence-gate.py` is an **optional** mechanical enforcement of the disciplines documented in [`evidence-trustworthiness.md`](./evidence-trustworthiness.md). It is a Claude Code `PreToolUse:Bash` hook: before an outward-facing write runs, it scans the body for a validation-scoped claim the trustworthiness gate would reject, and blocks the write if it finds one. + +**Surfaces policed:** the `gh pr|issue edit|create|comment` porcelain (`--body`, `--body-file`) *and* `gh api` body writes (`-f body=…`, `-F body=@file`, `--input file.json`) — a PATCH to a comment is the same publish with a different spelling, so a porcelain-only matcher is a hole rather than a gate. Read-only `gh api` calls pass through untouched. + +**Classes enforced:** `verdict`, `observation`, `deferral`, `ci-restatement`, `inflated-verdict`, `bare-identifier`, `truncated-identifier`, `mutable-ref`, `dump-resolver`, `link-only-exhibit`, `data-only-exhibit`, `step-waiver` — each implementing a numbered item of [`evidence-trustworthiness.md`](./evidence-trustworthiness.md). What the hook cannot see (whether a screenshot shows the resolving UI, whether a deferral's blocker matches its step, quotation fidelity) stays reader-applied. + +The hook is **Claude-Code-specific**. Other operators (Cursor, Codex, plain review) don't get the mechanical gate — for them the same disciplines apply as *documentation*, self-enforced by reading `evidence-trustworthiness.md`. The hook is not required to use the skill; it just moves the checklist from "remember to run it" to "runs automatically at emit time." + +It **fails open**: anything it cannot parse (non-`gh` command, unreadable body, malformed JSON) is allowed through, so it never bricks unrelated Bash commands. It uses the Python 3 standard library only (`json`, `re`, `sys`) — no dependencies to install. + +## Wire it up (Claude Code `settings.json`) + +Add a `PreToolUse` hook with matcher `Bash` that runs the script with `python3`. Put this in your user `~/.claude/settings.json` or a project `.claude/settings.json`: + +```json +{ + "hooks": { + "PreToolUse": [ + { + "matcher": "Bash", + "hooks": [ + { + "type": "command", + "command": "python3 /absolute/path/to/evidence/hooks/pr-evidence-gate.py" + } + ] + } + ] + } +} +``` + +Resolve the path to wherever `evidence` lives on disk. Note that `tools/install` copies only the `references`/`scripts`/`assets`/`adapters` bundles into `~/.claude/skills/mms-evidence/` — the `hooks/` directory is **not** part of the installed bundle. Point the `command` at your checked-out skills repo instead: + +``` +/domains/pr-workflow/skills/evidence/hooks/pr-evidence-gate.py +``` + +**When it blocks:** the hook exits `2` and prints the reason (which claim, what artifact/tracker it needs) to stderr. Claude Code surfaces that to the model, which self-corrects — attaches the missing artifact/tracker or downgrades the verdict — and re-posts. No manual intervention needed. + +## Two other setup requirements the skill needs + +These are independent of the hook; the skill needs them whether or not you install the gate. + +1. **`gh pr comment` must be permitted — pick a grant model.** evidence posts its evidence bundle as a PR review comment (`gh pr edit` if publishing into your own PR body). Four options, in descending order of standing safety: + + | Model | How | Tradeoff | + |---|---|---| + | **`ask` (recommended)** | `"Bash(gh pr comment:*)"` in `permissions.ask` | Per-post confirmation prompt. Combined with this hook (content gate) and a draft-confirm habit, that's three independent layers. | + | **`allow` + hook** | same pattern in `permissions.allow`, hook wired | Frictionless posting; safety rests entirely on the hook and your draft discipline. Only sensible where the hook is actually installed — not for operators without hook support. | + | **Allowlisted wrapper** | keep raw `gh pr comment` denied; allowlist a small script that takes `--repo`/`--pr`/`--body-file`, checks preconditions (canonical header present), and is the only sanctioned path | Tightest scoping — the raw verb stays blocked; costs a script to maintain. | + | **No grant — manual post** | the model prepares the body file; you run `gh pr comment --repo --body-file ` yourself | Zero standing grant; you are the bottleneck. The universal fallback, and the only option on operators with no permission system. | + + Avoid a bare **deny** on the comment verbs if you use this skill: it hard-blocks the publish step with no prompt, which reads as a mysterious failure mid-run. + +2. **Image re-hosting needs your own public evidence repo.** Screenshots and recordings captured locally must be re-hosted to a public URL before a reviewer can see them (see items 8–9 in `evidence-trustworthiness.md`). This repo is **yours to provide** — set it to a public repo you control, referenced here as ``. There is no shared/default host: parameterize it in your own configuration and push captures there, then reference the resulting raw URLs in the PR comment. Do not hardcode someone else's host. diff --git a/domains/pr-workflow/skills/evidence/references/evidence-publishing.md b/domains/pr-workflow/skills/evidence/references/evidence-publishing.md new file mode 100644 index 00000000..d86337b3 --- /dev/null +++ b/domains/pr-workflow/skills/evidence/references/evidence-publishing.md @@ -0,0 +1,335 @@ +# Publishing the evidence bundle to a PR body + +How to take run artifacts + complementary evidence and write a clean, idempotent, reviewer-familiar section into the PR body — **matching AEP's own format** so a re-run replaces in place instead of stacking duplicates. + +Canonical source for the format: `~/Code/metamask/metamask-autonomous-engineering-platform/packages/github/src/pr-body-builder.ts` (`upsertVisualValidationSection`). Mirror it. + +> **Publishing is public and outward-facing. Always render the section and get explicit confirmation before writing the PR body. Use `publishEvidence: false` on the run; this manual flow is the only publish path.** + +## Step 1 — Re-host images (artifacts are localhost) + +Control-plane artifact URLs (`localhost:3000/v1/runs/:id/artifacts/:name`) won't render on GitHub. Re-host each artifact and link the hosted URL. + +**Host: an S3 bucket you configure, prefix `public/`.** Set `EVIDENCE_BUCKET` and +`EVIDENCE_REGION` in your environment; this file does not name a bucket, because a bucket name +published here is an anonymously-readable endpoint advertised to everyone who reads it. + +``` +s3://$EVIDENCE_BUCKET/public/metamask/pr-// +https://$EVIDENCE_BUCKET.s3.$EVIDENCE_REGION.amazonaws.com/public/metamask/pr-// +``` + +The bucket must allow anonymous `GetObject` under `public/*` and must **not** allow listing, so +the prefix is not browsable — link individual files, and don't promise readers an index. If you +do not have one, that is the whole policy: + +```json +{ + "Version": "2012-10-17", + "Statement": [{ + "Effect": "Allow", + "Principal": "*", + "Action": "s3:GetObject", + "Resource": "arn:aws:s3:::YOUR-BUCKET/public/*" + }] +} +``` + +with `BlockPublicPolicy` and `RestrictPublicBuckets` disabled on that bucket and +`s3:ListBucket` granted to nobody. An org-owned bucket is preferable to a personal one: artifact +links outlive the person who published them. + +**Do not re-host to a personal repo.** A personal private repo returns 404 for every reader but +its owner, so every artifact link published from one is dead on arrival. That was the previous +target here, and this file simultaneously said such links were unreachable — guidance that +instructed you to publish dead links. Verified live in a published artifact. + +The test is **audience-reachability, not public-vs-private.** An org repo may be private and still +readable by colleagues, so an internal-audience link to one is fine. A personal repo is unreachable +by colleagues *and* by the public, so it fails for every audience. Re-host to an org-owned +destination, or to the configured bucket. + +- Path convention: `pr-//` keeps runs from colliding. +- **Verify unauthenticated before shipping**: `curl -s -o /dev/null -w "%{http_code}"` on each + published URL. A 200 from your own browser proves nothing — you are logged in. + +```bash +RUN_ID=; PR=; CP=localhost:3000 +BUCKET="$EVIDENCE_BUCKET" +BASE="https://$BUCKET.s3.$EVIDENCE_REGION.amazonaws.com" +for name in ; do + curl -fsS "$CP/v1/runs/$RUN_ID/artifacts/$name" -o "/tmp/$name" + key="public/metamask/pr-$PR/$RUN_ID/$name" + aws s3 cp "/tmp/$name" "s3://$BUCKET/$key" --only-show-errors + url="$BASE/$key" + # the link is not shippable until it resolves WITHOUT credentials + code=$(curl -s -o /dev/null -w '%{http_code}' --max-time 25 "$url") + [ "$code" = "200" ] || { echo "UNREACHABLE ($code): $url" >&2; exit 1; } + echo "$url" +done +``` + +No base64 round-trip and no 1 MB contents-API ceiling — the ceiling silently truncated a +1.7 MB gif to **0 bytes** on one run, and the loop reported success. Size-check anything you +transfer by another route. + +Files >1MB exceed `ARG_MAX` for an inline `-f content=` — use `gh api -F content=@` (write the base64 to a file first). For GIFs, re-host the same way. + +## Step 2 — Build the section (canonical header + mirror AEP) + +**Canonical header (2026-07-21):** every validation-run output — a PR comment *or* the PR-body section — leads with the exact literal `## 🧪 Validation Run`. Never reworded, never demoted to `###`: the constant string is the identifiability anchor, exactly like Copilot's fixed `## Pull request overview`. `hooks/pr-evidence-gate.py` blocks any `gh` write whose body has a validation/verification/evidence heading or AEP marker without this literal. + +Marker pairs, used so re-runs replace idempotently: + +- Whole section: `` … `` +- AEP status block (nested): `` … `` +- Screenshots block: `` … `` + +AEP prefers to inject screenshots into the PR template's `### **After**` section (replacing the `` placeholder), falling back to a `### Screenshots` block inside the status block when there's no After scaffold. Do the same. + +Section shape: + +```markdown + +## 🧪 Validation Run + +**Verdict:** ✅ proven — **Claim:** +head `` · · lanes: + + + + +### AEP Visual Validation + +**✅ Passed** + + + +
Validation details + +** — . " lines> + +
+ +Run `` · [LangSmith trace]() + + +``` + +Verdict icon: `✅` Passed, `❌` Failed, `ℹ️` otherwise. For perf, retitle the nested block `### AEP Perf Validation` and put `M/M assertions proven` in the headline. When AEP's *service* publishes its own `## AEP Visual Validation` block (publishEvidence:true, not the local flow), leave that block's heading alone — the demotion to `###` applies to hand-assembled bundles under the canonical header. + +Screenshots block (injected into `### After`, or appended under `### Screenshots`): + +```markdown + +
+ +<artifact-name> + +[Open full-size image]() + +
+ +``` + +`
` so reviewers see evidence without a click. One block per image; before/after read top-to-bottom. + +## Step 3 — Choose the surface by ownership, then publish + +**Publish surface depends on your relationship to the PR.** Determine it FIRST: + +```bash +PR=; REPO=MetaMask/metamask-extension +ME=$(gh api user --jq .login) +# Piped to jq rather than `gh --jq`: gh's built-in filter takes no --arg, and passing one +# fails with "accepts at most 1 arg(s)". +SURFACE=$(gh pr view "$PR" --repo "$REPO" --json author,commits | jq -r --arg me "$ME" ' + if .author.login==$me then "body" + elif ([.commits[] | select(.authors[].login==$me) + | select([.authors[].login] | map(select(.!=$me and .!="Copilot" and (test("claude|anthropic")|not))) | length == 0)] | length) > 0 + then "comment" else "skip" end') +``` + +- `body` — you authored the PR → upsert into the PR body (below). Validation is + part of your own claim. +- `comment` — not author but I have a solo commit (no HUMAN co-author) → post a + `gh pr comment` under the canonical `## 🧪 Validation Run` header. Never edit + someone else's PR body. +- `skip` — my only commits are co-authored with a human (review/pairing) OR I + have no commits → **do not publish**. Not my PR to validate outward. + +### Publish the script that produced a computed artifact, next to the artifact + +Any number you derived rather than read off a tool — a hash comparison, a count, a delta, a +statistic — is only as trustworthy as the reader's ability to re-run it. **A prose `method:` field +is not provenance.** Reviewers discount computed figures from an agent by default, and correctly: +on extension#45024 a reviewer dismissed a policy-identity check as *"we know LLMs are really bad at +this"*. It had in fact been a deterministic `sha256`, not the model counting — but the script lived +in a throwaway `python3 - <<'PY'` heredoc, so nothing could show that. The objection was +unanswerable because of how the evidence was packaged, not because of what it said. + +So: + +- **Write the script to a file, never an inline heredoc**, when its output will be published. The + heredoc survives only in the transcript, which the reader does not have. +- **Publish the script alongside its output**, and cross-reference: the artifact carries + `provenance: { script, script_sha256, command }`; the comment links the artifact. +- **Include the exact command** with its inputs (PR ref, head SHA), so the run is reproducible + rather than merely described. +- **Verify the round trip** — fetch the published script anonymously, hash it, and confirm it + matches `script_sha256`. A link that 200s is not proof the bytes are the ones you ran. +- Prefer a script that takes arguments and is re-runnable against a different PR. A one-off that + only works on your paths is weak provenance even when published. + +State plainly what the script does and does not do (`no model judgement; not a count of '+' +characters`) — that sentence is what actually retires the reviewer's prior. + +### Before any of the commands below: show the body in the response + +Every publish path here uses `--body-file`, so the **permission prompt displays a file path, not +the content**. The user is then asked to authorize publishing something under their name that they +cannot read, and the correct answer to that is no. + +**Paste the complete body inline in the response first, then run the command.** For an edit, also +say what changed relative to what is currently live. "I've drafted it, shall I post?" with a path +instead of the text is incomplete — pointing at `/tmp/validation-run.md` is the same failure as the +prompt itself. If the body is too long to show comfortably, that is a signal to trim it. +(Three consecutive denials on extension#45024, 2026-07-30, all from this.) + +### body surface (I own the PR) +```bash +gh pr view "$PR" --json body -q .body > /tmp/pr-body.md +# Replace the region between VALIDATION_RUN markers if present, else append. +# (Legacy bodies: replace the AEP_VISUAL_VALIDATION region and re-wrap it under +# the canonical "## 🧪 Validation Run" header + VALIDATION_RUN markers.) +# Replace the region between AEP_SCREENSHOTS markers if present, else inject after +# the "### **After**" heading (replacing the [screenshots/recordings] placeholder). +# ...edit /tmp/pr-body.md... +gh pr edit "$PR" --body-file /tmp/pr-body.md +``` + +### comment surface (I contributed but don't own) +```bash +# Same canonical "## 🧪 Validation Run" header + bundle; post as a comment. +gh pr comment "$PR" --repo "$REPO" --body-file /tmp/validation-run.md +``` + +Idempotency: because both regions are marker-delimited, re-running replaces them — never append a second copy. If the markers are absent (human-authored body), append the status block at the end and inject screenshots into `### After` when that heading exists. + +## Step 4 — Privacy scrub (before writing) + +Failure summaries and agent narratives leak the dev environment. Before publishing, strip: + +- Absolute local paths (`/Users//…`, `~/Code/…`) → describe the surface, not the path. +- The username anywhere it appears. +- `localhost` / `127.0.0.1` URLs → must be re-hosted public URLs only. +- Internal hostnames, JFrog/registry URLs, tokens. + +A failed run still must not publish raw — either omit the section or publish a scrubbed `❌ Failed` summary, with confirmation. + +## Recordings → GIF (for flows/motion a still can't prove) + +The platform can't collect video (artifact regex = png/jpg/log/txt). Capture out-of-band: + +1. In a **built** PR checkout (mm's fixture infra is required — a bare `dist/chrome` won't boot), write a preload `/tmp/patch-record.mjs` that monkey-patches `playwright-core`'s `chromium.launchPersistentContext` to inject `recordVideo: { dir }`. Resolve the module via `createRequire(/package.json)` so the patch hits the same module instance the `mm` daemon uses. +2. `NODE_OPTIONS="--import /tmp/patch-record.mjs" npx mm launch --state onboarding` → drive the flow (or let it sit) → `npx mm stop` flushes the `.webm`. States: `default | onboarding | custom`. +3. Convert with `ffmpeg` two-pass palette (better color than single-pass): + ```bash + ffmpeg -i in.webm -vf "fps=12,scale=480:-1:flags=lanczos,palettegen" -y /tmp/pal.png + ffmpeg -i in.webm -i /tmp/pal.png -lavfi "fps=12,scale=480:-1:flags=lanczos[x];[x][1:v]paletteuse" -y out.gif + ``` + webm/mp4 don't render inline in GitHub PR bodies; GIF does. +4. Re-host the GIF (Step 1) and embed like a screenshot. + +**Same-window app + DevTools (lane C8):** when the claim needs UI + console/network in one frame (e.g. "no toast *while* the log shows the silent path ran"), skip `recordVideo` entirely — it can't see DevTools. Use the OS-level region capture in [evidence-catalog C8](evidence-catalog.md): dock tab DevTools with `--auto-open-devtools-for-tabs`, tile the SW inspector window via `osascript`/CDP `Browser.setWindowBounds`, then `screencapture -v -V -R` → same ffmpeg GIF recipe. Publish the GIF + one full-res PNG + the CDP console text dump (GIF downscale makes log lines illegible on their own). + +## Re-validation runs: delta-first presentation, every verdict re-earned (2026-07-21) + +The common loop — a run refutes a claim, the author pushes a fix, `/evidence` re-runs at the new head — gets a **delta report**, not a second full bundle: + +- **Presentation is delta-only.** Full exhibits only for lanes whose outcome changed (flipped verdict / new lane / new residual). Unchanged lanes collapse to a `Prior run | This run` ledger, each row with a fresh run-log link from the new head plus one link to the prior run's comment for the full exhibits — and say so ("unchanged rows re-run at ``; full exhibits in the prior run"). +- **Evidence is never delta.** Evidence is head-pinned: re-run every automated lane at the new head and re-earn every verdict with a fresh artifact. "Unchanged" is a conclusion from the re-run, never a carried-over assumption (the stale-baseline trap at report level). Re-running is cheap — the falsifier harness already exists from the first run. +- Same canonical header + markers; the meta line names the fix commit and links the prior run. Comments: one per run, chronological, each linking its predecessor. PR-body section: replaced in place via markers. +- New head → **new hosted artifact directory keyed to the fix commit** (`pr-/fix-/`), commit-pinned raw URLs; never overwrite a prior run's published files. +- Residuals the fix intentionally leaves get their own row/section — don't round a fixed-with-residual claim up to fully proven. + + +## Lead with a lane-status ledger (no silent absence) + +The published section must **enumerate every lane the claim type calls for and give each an explicit status** — never render only the lanes you happen to have and let the rest be silently absent. An unmarked gap is indistinguishable from a lane that ran and came back empty; the reader (and you, on the next pass) can't tell "no evidence because none needed" from "no evidence because not done." This is the vacuous-pass trap at the publish layer — carry the run's `✅/❌/⚠️` verdict into the PR body, don't leave it in the internal report-back. + +Open the evidence section with a ledger: + +```markdown +| Lane | Status | Evidence | +|---|---|---| +| B3 falsifying test | ✅ proven | 32/32 head, 3/32 reverted | +| E1 Sentry before/after | ✅ proven | [discover](…) — distinct trace ids | +| A1 visual | ➖ N/A | background change, no UI surface | +| C6 CDP netlog | ⏳ not-captured | — | +``` + +Status vocabulary: `✅ proven` (link) · `⚠️ inconclusive` (name what's missing) · `➖ N/A` (reason) · `⏳ not-captured`. Mirror the `N/A — ` convention the `### Screenshots` block already uses for no-UI PRs. Never upgrade a `⏳`/`⚠️` to a pass by omission. + +**Sibling-PR parity:** when a set of PRs shares a claim shape (same program, same author, "root the X traces"), their ledgers must match lane-for-lane. A lane present on one and absent on another is either added or explicitly marked `➖ N/A — ` — a bar that silently drifts between siblings is a finding (postmortem 2026-07-17, #43929/#43930). + +## Non-visual & multi-lane evidence + +Screenshots are only one lane. Most claims (perf, telemetry, state, build) publish as **text/links/tables**, not images. Put them in the same verdict-first section so a reviewer sees one coherent bundle, not scattered comments. + +Per-lane rendering: + +- **Sentry / Tempo (E1/E2):** a markdown link to the discover/trace query with the before/after window baked in, plus the headline numbers inline (`errors: 1.2% → 0.0% over 24h post-release`). Link, not screenshot — reviewers re-run it. +- **Benchmark / web-vitals / TBT (C2/C3/C5):** a small before/after table (metric · base · head · Δ · threshold). State it's a **paired A/B** if the stored baseline was bypassed. +- **Migration (F1):** the `changedKeys` set + a before/after state-shape snippet, and a link to the migration-test run. +- **Bundle / chunk / LavaMoat / manifest (D1–D4):** the diff or size delta in a fenced block; for policy/manifest, the actual `git diff` (or "diff empty — no new capability"). +- **Trace artifacts (B2):** link the Playwright trace-viewer report / attach the `trace.zip`; don't paste raw. + +Multi-claim PRs get one sub-block per claim under the status section, each with its own ✅/❌/⚠️ verdict — mirror the Claim Cards. Keep the visual block (markers + `### After` injection) for the image lanes; render the rest as text beneath it. + +### One comment per evidence *kind*, not one comment per PR (2026-07-30) + +Sub-blocks are for several claims **of the same kind**. When a PR draws two different +kinds — say an executed Validation Run *and* a read-level capability triage — they get +**separate comments**, each with its own header, its own marker pair, and its own format. + +| | Validation Run | LavaMoat policy diligence | +|---|---|---| +| header | `## 🧪 Validation Run` | `## 🔒 LavaMoat Grants — → ` | +| markers | `VALIDATION_RUN_*` | `LAVAMOAT_DILIGENCE_*` | +| opens on | `**Verdict:** ✅/⚠️/❌` | the finding; **no verdict at all** | +| body | lane ledger, artifacts per lane | deny candidates, enumeration folded | +| audience | whoever owns the PR's claim | whoever owns the policy | + +Merging them forces one frame onto both. A read-level triage has no run to verdict, so it +would land as `⚠️ inconclusive` on a header promising a run; and a `⏳ not-captured` lane +needs a tracker it does not have. The marker pairs also collide — a re-run replacing the +`VALIDATION_RUN` region would silently eat the diligence output sharing it. + +**So: choose the format from the evidence kind, not from this document's default.** The +canonical `## 🧪 Validation Run` header applies when a run produced artifacts. An engine +skill that defines its own output contract (`lavamoat-policy`) publishes in that +contract. `hooks/pr-evidence-gate.py` enforces the canonical literal only on bodies that +*claim* validation/evidence framing — a diligence comment that renders no verdict does not +trip it, which is the tell that the two are different artifacts rather than one with a +different skin. + +**Per-scenario presentation (2026-07-21):** the same applies one level down — when the evidence spans multiple test scenarios (flag-on vs flag-off, control vs treatment in an A/B falsifier, numbered manual-testing steps), give each scenario its **own sub-section**: a heading naming the scenario in observation terms, one line on what it tests plus its verdict, and that scenario's artifacts co-located under it. Never bunch all scenarios' artifacts into one large evidence dump — the reviewer verifies "under condition X, artifact shows Y" one condition at a time, and a merged block destroys that mapping even when every artifact is real. For long artifact sets use a `
` block *per scenario*, not a merge. (Instance: #44610.) + +## Artifact contract (ADR-0058 alignment) + +To stay interoperable with the recipe-based verification system (MetaMask/decisions#173), shape the bundle like its reviewer-visible contract where practical: a `summary.json` (claim → verdict → evidence refs), a `trace.json` (the run/assertion log), and an artifact manifest (names + media types), with screenshots/video as the confidence layer. Publishing then becomes "render `summary.json` into the PR section." This keeps evidence's output and a recipe's output the same shape — see [lane-assertions.md](lane-assertions.md). Don't hand-roll a divergent format. + +## Checklist before you publish + +- [ ] Section opens with a **lane-status ledger** — every claim-required lane marked `✅`/`⚠️`/`➖ N/A`/`⏳`; no lane silently absent (and sibling PRs' ledgers match lane-for-lane) +- [ ] `evidenceBundle.artifactRefs` non-empty with expected media (not a vacuous pass) +- [ ] Each lane passed the [trustworthiness gate](evidence-trustworthiness.md) (shows the claimed surface, signal > noise, could-have-failed) +- [ ] Multi-scenario evidence rendered **per scenario** (own heading + verdict + co-located artifacts), not bunched into one block +- [ ] **Automated-process voice, no first person** — published validation output never says "I ran/captured/verified"; attribute to the process ("Automated validation ran…", "the harness captured…") so readers know the evidence is machine-generated, not a manual account under the author's name +- [ ] Every image/GIF re-hosted to the configured bucket under `public/…`; no localhost/local-path URLs in the body +- [ ] **Every published link curl'd unauthenticated and returning 200** — never a personal private repo +- [ ] Work cited by **PR link** rather than tracking-ticket id, unless the ticket's own content (an RCA, a spec) is the referent +- [ ] Narrative scrubbed of username/paths/internal hosts +- [ ] Marker pairs present so the upsert is idempotent +- [ ] Section rendered and **confirmed by the user** diff --git a/domains/pr-workflow/skills/evidence/references/evidence-trustworthiness.md b/domains/pr-workflow/skills/evidence/references/evidence-trustworthiness.md new file mode 100644 index 00000000..f398b2ef --- /dev/null +++ b/domains/pr-workflow/skills/evidence/references/evidence-trustworthiness.md @@ -0,0 +1,42 @@ +# Evidence trustworthiness (anti-reward-hacking) + +A green result is not proof. An agent — or an eager run — can produce evidence that *looks* like it validates the claim but doesn't. Before believing or publishing any lane, run it through this gate. It extends the vacuous-pass trap to all lanes; the Claim Card's **Falsifier** is the anchor: trustworthy evidence is evidence that *could* have shown the falsifier and didn't. + +## The gate (per lane, before publish) + +1. **Non-empty & expected media** — the bundle has artifacts of the expected kind. Zero artifacts = not a pass (the vacuous-pass guard). +2. **Shows the claimed surface** — the screenshot/recording is the Claim Card surface in the asserted state — not a loading spinner, an error toast, the wrong screen, or a pre-action frame. Eyeball it. +3. **Exercises the changed code** — the test/flow actually hits the diff. For a test: it **fails on `main`** (catalog B3). For a flow: the changed component/route is on the path. A green test that never imports the changed module proves nothing. +4. **Signal exceeds noise — and a null states its power** — a perf delta must be beyond run-to-run variance (paired A/B, multiple iterations); a 3% move on a noisy metric is not evidence. The same bar applies in reverse: when the spread is wider than the effect being looked for, the finding is **"not resolvable at this sample size"**, never "no change" — an underpowered run and a true null print the same word, and reporting the word alone lets the reader infer the stronger claim. State the smallest effect the design could have detected. + - **Removing a bias is not establishing validity.** Correcting a flaw you found (discarding a warm-up, alternating the starting arm, pinning CPU governor) removes *that* bias and licenses no more than that. It is not a trust gate, because a trust gate names how the evidence could **still** be vacuous — residual risk, not completed work. List what remains uncontrolled (thermal drift, background load, ordering within a round); an unenumerated confound reads as a nonexistent one. + - **When correcting an overclaim, cut the certainty, not the evidence.** A falsifier that actually caught something is the strongest thing on the page — downgrade the conclusion around it, don't delete it with the overclaim. +5. **Could have failed** — the assertion has a reachable failure mode. Always-true assertions (`expect(true)`, a screenshot with no assertion, a Sentry query with no time bound) can't falsify anything. +6. **Right baseline** — "before" is the actual base ref / prior version / pre-window, not a stale or mismatched comparison. +7. **Artifacts are independent & honestly labeled** — checksum every capture set (`md5 *`). Byte-identical files across supposedly independent runs/cases cannot stand as separate observations: either explain the identity in the artifact bundle (deterministic fixture rendering) with per-run provenance that *does* differ (the harness state dump, timestamps, a manifest), or re-capture at distinct moments. Labels must describe the observation, not the interpretation — a file named for the state it *should* show under the claim (`steady-state`, `no-toast`) misleads when the capture shows the refutation. +8. **The finding ships with its artifacts** — a findings comment (including a refutation shared privately) carries functional links to the re-hosted observation artifacts at *draft* time, not descriptions of artifacts that exist only on the capturing machine. "Would need re-hosting" is not a reason to omit: re-hosting is the procedure ([evidence-publishing](evidence-publishing.md) Step 1). Code permalinks + a runnable repro are corroboration, not a substitute for the observation itself. +9. **Signal is surfaced — least-effort validation** — evidence is judged at the reader's eyes, not the author's disk: signal the reader must excavate from a mountain of attached data is, for evidence purposes, no evidence. Every published exhibit leads with a one-line pointer — *what to open, where to look, what it should show*. Deltas are presented **as** deltas (annotated side-by-side, diff, before→after crop of the differing region), never two full captures for the reader to compare by eye; if the claim is "no visual change," publish one image plus the hash-equality line, never N identical-looking copies as separate exhibits. Bulk artifacts (MB-scale JSON, full logs) are excerpted inline to the discriminating lines, with the full file linked as appendix. Emit-time test, per exhibit: can a reader who did not run the session confirm the claim in ~30 seconds from what is directly visible? If not, restructure the presentation — attaching more data cannot fix it. Coverage is the converse constraint (2026-07-21): this item governs *form*, never column-set minimalism — a valid, relevant dimension is never omitted because it duplicates another's signal (redundant corroboration costs a skippable glance; an omitted column is unfalsifiable and reads as cherry-picking). Exclusion requires invalidity (metric void on this surface, e.g. TTFB on `chrome-extension://` pages) or irrelevance (different claim/different data → sibling exhibit, not a column), each stated in a one-line disposition. +10. **Parallel exhibits are format-uniform** — sibling exhibits (table rows, per-scenario blocks, the legs of an A/B pair) carry the same evidence format and quality. If one row links its artifact inline, every row does; if one scenario gets an annotated timeline, action-log provenance, and co-located full-res/raw links, every scenario does. The bar is the **best sibling**: when the presentation standard improves mid-session, re-normalize the whole document up to it before publish — never apply the improvement only to the exhibit being produced (append-only drafting). Any asymmetry carries an explicit stated reason co-located with the weaker exhibit ("close-event variant unit-uncoverable", "manual-only trigger"); an unexplained format gap reads as an evidence gap — the reader cannot tell an unlinked artifact from a missing one, and inconsistency spends credibility on *every* exhibit, including the strong ones. Emit-time test: enumerate the sibling sets, diff each against the best-formatted member, and for every deviation either normalize it or state the reason. +11. **Lanes derive from the Manual testing steps — a CI-green row is not a lane** — the Validation Run's rows are generated top-down from the claim and the PR's own **Manual testing steps**, never bottom-up from whatever links already exist. For each step the claim depends on, the lane's payload is the **captured output of executing that step** (step "in Discover, group by `trace`" → a Discover permalink / **linked** trace-id table showing N rounds → N distinct `trace_id`s, per item 12), or an honest ⏳ naming the missing capture with a tracker. A row restating CI ("tests green at head `` in [CI run]") duplicates the Checks tab and is deleted — and a validation surface carries **zero** CI references, full stop: no `actions/runs` links, no "green at head" clauses, no "as context (only)" retention. The earlier carve-out here ("a CI link is admissible as context on a beyond-CI row") was itself the next costume: within a day all four sibling bodies (extension#43928–#43931) shipped restatements phrased as the exception — rows *leading* with "green at head … in [Unit tests CI]", the same link repeated 3× per body, the remediated row keeping it re-labeled "as context only" — while the gate's excuse regex matched the mere word "revert", so vocabulary, not evidence, discharged the class. The revert lane cites the revert **outcome** (which blocks failed, at which commit); its green-at-head half is the Checks tab's information and is omitted. A carve-out in an emit-time gate is an instruction to generation to phrase every violation as the exception — deliberate exceptions route through the human, never through an excuse predicate. Borrowed evidence — a sibling PR's capture, a unit falsifier standing in for the named live surface — never upgrades an uncaptured lane to ✅: "mechanism live-proven" co-located with "was not exercised" is an inflated verdict; downgrade it. Emit-time trigger: `pr-evidence-gate.py` classes `ci-restatement` (unconditional since 2026-07-21: any CI link / CI-green phrase in validation scope fires — no verdict co-location required, no beyond-CI excuse) and `inflated-verdict`, with the shipped extension#43928 rows and the carve-out-blessed "as context" shape as regression cases (2026-07-21). +12. **Identifiers resolve in one click — a bare id is a digging assignment** — trace ids, event ids, run ids, SHAs are *pointers into a system*, not evidence. Publishing a bunch of raw trace ids hands the reviewer the job of reconstructing project/environment/time window and querying Sentry themselves — it fails item 9's ~30-second test by construction (item 9 makes the signal *findable*; this item makes it *checkable*). Every identifier published as evidence is either hyperlinked to its resolving surface (the Sentry trace/event permalink, or an absolute-windowed Discover query pre-filtered to exactly those ids) or accompanied by the re-hosted captured output (query-result rows / envelope excerpt showing the discriminating fields) — ideally both. Special case that produced the rule: ids captured **locally** (mockttp forwarder, envelope intercept) never reached Sentry, so no permalink can exist — the re-hosted capture is the *only* admissible form, and pasting the id fragments plus a re-run recipe is the "spec necessary / output sufficient" violation wearing ids as decoration (extension#43931 Validation row, 2026-07-21). Rule of construction: when any item in this gate blesses an evidence class by name ("trace-id table", "envelope log"), it means the class's *resolvable instance*, never its bare tokens — a blessed class name is otherwise the next costume. Emit-time trigger: `pr-evidence-gate.py` class `bare-identifier`; converse-of-gate note: the prior gate *whitelisted* `trace_ids?` as beyond-CI payload and its own fix-message recommended "trace-id table" unqualified — second occurrence of "audit the gate for whitelists of the violating shape." +13. **Terminal exhibits are reader-native — a live link or a visual; a dump behind a link is still an opaque reference** — item 12 makes every pointer resolve in one click; this item constrains what it may resolve *to*. A positive verdict's terminal artifact is one of the two media a reviewer natively consumes: a **live link into the resolving system** (Sentry trace/event permalink, absolute-windowed Discover query pre-filtered to the claim) or a **visual capture** (screenshot/recording, annotated or cropped to the discriminating region). Raw files (`.log`/`.json`/`.har`, MB-scale dumps) are **appendix-only** — linked once for auditability, never the exhibit a claim rests on: a link whose target is a raw dump passes item 12 and fails item 9 one click later; the digging moved a hop away, it did not disappear (extension#43931 *second* remediation, 2026-07-21: the `bare-identifier` fix shipped a ✅ row whose sole resolver was a re-hosted ~70KB run log). Two corollaries: (a) **the gate items are conjunctive** — a fix for the newest item must re-pass all prior items; satisfying resolvability with an artifact that fails legibility is the generator's next costume; (b) **ascertain the terminal medium at step zero and pick the capture lane that can produce it** — a local intercept (mockttp envelope forwarder) can never yield a live Sentry permalink, so for Sentry-observable claims it is the supplementary falsifier lane and live ingest (dev build → `SENTRY_DSN_DEV`/test-metamask) is primary, precisely because it terminates in permalinks + screenshots; choosing a lane that cannot produce the terminal medium silently displaces it. Emit-time trigger: `pr-evidence-gate.py` class `dump-resolver`, with the remediated extension#43931 row as the regression case (2026-07-21). +14. **Manual testing steps are the validation contract — steps present ⇒ live evidence definitionally required; an impossibility waiver contradicting an executable step is invalid** — the PR's own **Manual testing steps** are the author's assertion that the claimed behavior *is* live-observable, and how: each numbered step is an executability proof (a step a human reviewer can run, `/evidence` can run) and its text is the capture spec. Item 11 derives the lanes from the steps top-down; this item closes the other side of the hatch — a lane derived from a step may not then be *waived by argument*. Emit-time procedure: build the per-step coverage map (step → executed-output artifact); for any step without one, the only admissible state is a **per-step** ⏳ + tracker whose blocker is that step's *own* unmet precondition, checked against the step text. Three waiver-inflation patterns from the producing instance (extension#43228/#42869/#44538, 2026-07-21 — all three shipped articulate impossibility rationales in the same body whose Manual testing steps asserted the opposite): (a) **borrowed impossibility** — the excuse imported from a different mechanism or sibling PR (#43228 waived its live lane citing the async remote-flag read race, which is #44538's mechanism; #43228's overrides are build-time env vars, and its steps 1–4 are directly executable in a dev build); (b) **lane-limitation universalized** — one harness's gap stated as global impossibility (#42869: "the e2e harness emits no error `event` envelope, so … not capturable pre-merge" — the step says *dev build with Sentry enabled, trigger an error*, which ingests error events into test-metamask without the e2e harness); (c) **blocked-scope inflation** — a genuinely blocked precondition of one half of the claim expanded to waive the whole lane (#44538: LaunchDarkly provisioning [#7482](https://github.com/MetaMask/MetaMask-planning/issues/7482) blocks only the *prod-flag* half; the step's own text names the dev-injectable alternative — "or inject it into persisted `RemoteFeatureFlagController` state"). If a step is *truly* non-executable, the waiver is still inadmissible alone: the Manual testing steps are then wrong and are corrected in the same edit — a document may not simultaneously instruct a reviewer to observe X and declare X unobservable. The honest-⏳ lane blessed throughout this gate is for *not yet done*, never for *argued away*: an eloquent impossibility rationale is the cheapest token sequence that satisfies every prior item (no fake capture, no CI link, no bare id, no dump) — the generator's costume for the coverage axis. Emit-time trigger: `pr-evidence-gate.py` class `step-waiver` ("not demonstrable / capturable / observable", "not separately captured", "not attached", "rests entirely/solely on the falsifiers/unit/revert" — unconditional in validation scope; no tracker or artifact excuses it), with the three shipped waiver paragraphs as regression cases and the gate verified-blocking on all three live bodies (2026-07-21). Detection gap: the per-step consistency check (does a deferral's blocker match the step's own precondition?) stays procedural — the gate sees vocabulary, not step semantics. +15. **The exhibit lives in the body — link AND visual; a live link alone is the verification path, not the exhibit** — item 13 blessed the terminal media as a *disjunction* (live link OR visual), and generation took the cheaper disjunct: a Discover permalink is producible from the API token alone, a screenshot needs a browser session — so extension#44540's live-ingestion exhibit shipped as a permalink + prose counts, with nothing in the PR body a reader could look at (2026-07-21: "only sentry link and not screenshot that makes it immediately obvious how evidence validates pr"). A live link defers validation behind **click + auth + query rendering + column interpretation** — the dump-resolver displacement one hop further, with the mountain now behind a login: it fails item 9's ~30-second test at the moment of the click, and for any reader *without* Sentry org access (most PR reviewers) a link-only exhibit degrades to a bare identifier (item 12) behind an auth wall. The repaired rule is a **conjunction**: a positive verdict's headline exhibit is an **embedded visual** — screenshot/recording of the linked resolving view (Discover result rows, trace waterfall), cropped/annotated to the discriminating region, captioned with what it should show — **and** the co-located live permalink (absolute-windowed) as the independent-verification path. Neither substitutes for the other: link-only hides the exhibit; visual-only is independently unverifiable. The 2026-07-16 clause "screenshots ride along when a browser session is available; the API token alone yields links + JSON, which is the automatable minimum" was the self-authored escape hatch of this axis (family: the "as context" carve-out, the honest-⏳ waiver): the *automatable minimum* got promoted to the shipped standard because it was the cheapest compliant artifact. A capture lane that cannot screenshot its resolving view is a lane gap to fix before publish (drive a browser session to the Discover URL), never a licensed downgrade — deliberate exceptions route through the human. Emit-time trigger: `pr-evidence-gate.py` class `link-only-exhibit` (non-negated verdict + `sentry.io` link + no image/recording embed in the unit), with the shipped #44540 paragraph as the regression case and the prior suite's permalink-only ALLOW cases flipped/augmented — third occurrence of "an ALLOW case containing the violating tokens is a specification of the next costume." Detection gaps: verdict co-location is required, so a no-verdict link-only paragraph evades mechanically; the visual-without-link converse stays procedural under item 12. +16. **The audit chain is mechanical — quote, don't transcribe; pin, don't point** — an exhibit's inline data must be a **verbatim, greppable excerpt** of the artifact (full-length identifiers, raw capture lines quoted exactly), and every repo-hosted artifact link must be a **commit-pinned, line-anchored permalink** (`/blob//…#Lx-Ly`) to the discriminating lines. Producing real artifacts and then hand-transcribing digests severs the claim→artifact bridge at every link: an ellipsized id (`24b1e2da…`) cannot be grepped against any artifact even when the log is linked in the same block; a reformatted data block cannot be distinguished from confabulation without redoing the dig, so it reads as *claims in the form of data*; a branch-ref `/blob//` link is a mutable pointer whose target can be rewritten after review (not tamper-evident); a file-level link without line anchors lands the reader at the top of a 1,000-line dump. The producing instance (extension#43929 validation comment, 2026-07-21) had every number substantiated by four re-hosted run logs — real, included, resolvable — and still drew "still no immediately auditable evidence just claims in the form of data": each prior item individually near-passed while the exhibit↔artifact binding stayed **editorial** (transcription + file link) instead of **mechanical** (quotation + pinned line anchor). Emit-time procedure: for every inline datum, quote the raw capture line it comes from (fenced, verbatim, full ids) and anchor it (`#L`); pin every evidence link to the SHA (press `y` on the GitHub file view). Emit-time trigger: `pr-evidence-gate.py` classes `truncated-identifier` (a co-located resolver does NOT excuse it — the resolver resolves the full id, not the fragment the reader holds; hash-equality prose exempt) and `mutable-ref`, plus the **surface hole** — the shipped comment was published via `gh api` PATCH, which the porcelain-only matcher (`gh pr|issue edit|create|comment`) never saw; fifth occurrence of the converse-of-gate rule, one level down: audit the gate for *spellings of the write it cannot see*, not just tokens it excuses. **CORRECTION 2026-07-30 — this hole was recorded as closed and is not.** Verified against the deployed `hooks/pr-evidence-gate.py` (259 lines): line 47 is the only command matcher, `\bgh\s+(?:pr|issue)\s+(?:edit|create|comment)\b`, so `gh api` body writes are still invisible; and the file implements essentially one check (verdict-needs-artifact), **not** the ~9 classes named across items 11–18 (`ci-restatement`, `bare-identifier`, `dump-resolver`, `link-only-exhibit`, `data-only-exhibit`, `step-waiver`, `truncated-identifier`, `mutable-ref`, `inflated-verdict`). Treat every "Emit-time trigger: `pr-evidence-gate.py` class …" line in this document as **specified, not implemented**, until re-verified in the code — a doc asserting a class the code lacks retires the vigilance it claims to replace, which is the failure this very item warns about. Consequence observed the same day: 14 unlinked `path:line` references shipped across 12 review comments via `gh api`, with the gate both classless for that shape and unwired in `settings.json`. Detection gaps: the line-anchor half of pinning and the paraphrase-vs-quotation judgment stay procedural — the gate sees ellipses and branch refs, not editorial fidelity. +17. **Evidence is captured in its environment — data alone is insufficient even when correct** — item 16 makes the data trustworthy as *transcription* (verbatim, greppable, pinned); this item polices what transcription can never carry: **liveness provenance**. A quoted `EVIDENCE trace_id=…` line, a re-hosted gist, a hand-assembled id table can all be correct and still show nothing about *where they came from* — extracted data is indistinguishable from data typed by hand, so it cannot make it immediately apparent that the evidence was captured **live** from a **functioning** system. The exhibit for a system-of-record-observable claim therefore includes an **in-environment capture**: a screenshot/recording of the resolving system's own UI (the Sentry Discover/trace view with the query, project/environment selectors, absolute time window, and result rows all in-frame) — the environmental chrome is not decoration, it *is* the provenance: it shows the query really ran, in the real dashboard, over the real window, and returned these rows. Correctness was never the failing dimension (2026-07-21: "just the data is insufficient even if correct — it needs to be immediately apparent that evidence was captured live and is functional"). Relation to prior items: item 15's link+visual conjunction fired only when a `sentry.io` link was present, and item 13's `NATIVE_MEDIUM` blessed an inline fenced excerpt as a terminal medium — so a no-link, quoted-data exhibit (the fidelity-remediated shape: full ids, verbatim excerpts, pinned line anchors, zero environment captures) passed the whole regime while carrying zero liveness provenance. The joint rule after this item: a telemetry-observable positive verdict always carries the in-environment visual (plus the live permalink per item 15); quoted excerpts, gists, and data files are appendix beside it, never the exhibit. Emit-time trigger: `pr-evidence-gate.py` class `data-only-exhibit` (non-negated verdict + telemetry-observation vocabulary + no image/recording embed + no sentry link — with a sentry link, `link-only-exhibit` already fires), with the re-hosted-gist ALLOW case flipped (fifth occurrence of "the ALLOW case was the next costume's spec") and the #43929 quoted-excerpt shape as a regression case. Detection gaps: vocabulary-scoped (telemetry-observation terms, not bare code tokens like `trace.test.ts`), so a claim phrased entirely without them evades mechanically; and the gate cannot see whether an embedded image actually shows the environment's chrome — screenshot content stays procedural (item 2's "eyeball it" applies: the capture must show the *resolving UI*, not a cropped data region indistinguishable from a spreadsheet). +18. **"Successful" is an evidence predicate, not a run status — and the default Sentry exhibit is fixed in advance** — a validation run may be scored/reported "successful"/"validated" only when its published surface already carries, for every Sentry-observable lane, the default exhibit pair: an **in-environment Sentry-UI screenshot** (item 17) **plus the co-located live permalink** (item 15). Completed runs, green falsifiers, staged drafts, and honest ⏳ lanes do not confer success — a run without the pair is at most "run-complete, evidence-owed." The default recipe needs no per-PR ascertainment: for Sentry, **generally capture actual screenshots from the Sentry UI and attach the link** — that pair is step zero's pre-computed answer for any Sentry-observable claim, never the terminus of axis-by-axis escalation. Capture-first ordering: the capture executes before any rule/gate/postmortem authoring may close a validation session — writing a new rule or gate class discharges nothing (2026-07-22: ten postmortems and 17 gate items shipped while zero Sentry-UI screenshots did; every "successful" run was claims-only, because success was assigned by run-completion and meta-work substituted for capture work). Emit-time trigger: `pr-evidence-gate.py` `VERDICT` vocabulary now includes the status spellings `successful`/`validated`/`live-proven`, so a claims-only unit scoring itself successful blocks like any bare "confirmed." Detection gap: the gate fires only on re-emit — already-shipped "successful" surfaces are audited by backward re-score, enumerated from live state, never from the ledger (the discharge-granularity rule applies to success statuses verbatim). + +19. **A substitution A/B is readable only if the unmodified arm is silent — and only if each diagnostic fires for the reason claimed** — the substitution lane (catalog D6) derives its finding from the *delta* in a checker's output between the PR as written (Arm A) and the PR with one authored artifact replaced by its authoritative equivalent (Arm B). Two ways that delta lies, both of which look like a confirmed finding. (a) **A noisy Arm A destroys attribution.** If the unmodified tree already emits diagnostics, nothing in Arm B is attributable to the substitution — the reader cannot tell a concealed disagreement from ambient breakage, and "N errors in Arm B" is then a count, not a finding. Publish Arm A's result explicitly (`0 errors`, verbatim) as the delivery check; if it is non-empty, the instrument is broken and the lane is **inconclusive**, not a pass — fix the baseline (pin the toolchain, raise the heap, exclude the unrelated project) or drop the lane. This is the substitution analogue of item 1's vacuous-pass guard: item 1 asks whether the artifact exists, this asks whether the *comparison* means anything. (b) **A diagnostic can fire for the wrong reason.** A checker reports the first failure it reaches, so an earlier cause short-circuits the claim under test and an exit-code read scores it as confirmation — the same hazard item 3 polices for tests that fail on base for an import error rather than the bug. Producing instance (extension#44397, 2026-07-30): a probe asserting a hand-written provider return type was unsound errored on *nullability* one property earlier, and the return-type claim — re-probed with the nullability neutralised via `NonNullable<…>` — turned out to be **sound**, i.e. a finding that would have shipped as real. Emit-time procedure: for every substitution claim, assert on the *specific* expected diagnostic (code + message + line), not on non-zero exit; where an earlier cause intervenes, isolate it and re-run; and report the claims the re-probe **cleared** alongside the ones it confirmed — a substitution sweep that only ever confirms is indistinguishable from one that never isolated anything. Corollary on the negative case: when no authoritative source exists (a package that ships no types, a lib absent from tsconfig `lib`, a boundary the repo genuinely owns), hand-writing is *correct* — record it as a cleared falsifier with the reason, never as an unreported non-finding. Detection gap: procedural — a gate can see whether Arm A's result is published, not whether the diagnostic it cites is the one the claim needs. + +## Lane-specific traps + +- **Visual:** spinner/skeleton mistaken for the loaded state; the toggle (privacy/redaction) not actually flipped; a cached screenshot from a prior run; the fallback surface shown without saying so. +- **Perf / benchmark:** stale frozen baseline (catalog C5 caveat); single sample; warm-vs-cold mismatch; measuring a different interaction than the claim. +- **Test:** snapshot regenerated to match the bug (`--updateSnapshot` masking a regression); the test mocks out the changed path; it passes on `main` too (so it's not a regression test). +- **Substitution A/B (D6):** a non-silent Arm A (attribution destroyed); a diagnostic that fires one property earlier than the claim (isolate and re-probe); a substitution the checker never reaches because the caller is untyped JS or `any`; treating "no authoritative source exists" as a null result rather than a cleared falsifier. +- **Telemetry:** query window excludes the release; the error regrouped under a different fingerprint; sample-rate makes "0 events" meaningless. +- **Migration:** only the happy path asserted; `changedKeys` not checked against actual mutations; no real prior-version fixture. +- **Coverage:** a line covered ≠ a behavior asserted (executed but never checked). + +## When evidence fails the gate + +Don't publish it. Either re-capture correctly, **downgrade the verdict to ⚠️ inconclusive** and name what's missing, or — if the evidence shows the claim is false — switch to the [refutation path](SKILL.md). Never round a weak pass up to "proven." diff --git a/domains/pr-workflow/skills/evidence/references/lane-assertions.md b/domains/pr-workflow/skills/evidence/references/lane-assertions.md new file mode 100644 index 00000000..7ba6dca7 --- /dev/null +++ b/domains/pr-workflow/skills/evidence/references/lane-assertions.md @@ -0,0 +1,26 @@ +# Lane → declarative assertion mapping (ADR-0058 bridge) + +Maps each evidence-catalog lane to a declarative assertion form, so a Claim Card can be expressed as an ADR-0058 recipe (pre-conditions → proof targets → assertions → screenshot points) where possible — and so we know which lanes are **CDP-expressible** vs **out-of-band**. State/log assertions give determinism; screenshots/video give reviewer confidence. See [[../ITERATION]] items 9–11 and MetaMask/decisions#173. + +| Lane | Assertion form | Expressible as a CDP recipe action? | +|---|---|---| +| A1 / B1 visual | screenshot at a proof point + (optional) DOM/a11y assertion | **yes** — Chrome CDP | +| B2 e2e | the spec's own assertions; trace.zip as artifact | yes — it *is* a driver | +| B3 falsifying test | test exit code: fail@`main`, pass@branch | out-of-band (test runner) | +| C1 startup traces | `stateHooks.getCustomTraces()[name] < threshold` | **yes** — `Runtime.evaluate` | +| C2 web-vitals | `stateHooks.getWebVitalsMetrics().inp < 200` | **yes** | +| C3 long-task / TBT | `stateHooks.getLongTaskMetricsWithTBT().tbt < 200` | **yes** | +| C4 render (WDYR) | console-log assertion: 0 unnecessary re-renders | partial — needs console capture | +| C5 benchmark | metric delta vs paired baseline > threshold | out-of-band (benchmark runner) | +| C6 DevTools/CDP | netlog: request absent/present; profile metric | **yes** | +| D1 / D2 bundle/chunk | static: chunk-manifest membership / size delta | out-of-band (build artifact) | +| D3 LavaMoat | static: `policy.json` diff empty / justified | out-of-band (git diff) | +| D4 manifest | static: permissions diff empty | out-of-band | +| E1 / E2 Sentry/Tempo | external query link (before/after window) | out-of-band (dashboard) | +| F1 migration | `changedKeys == expected` + state shape valid | out-of-band (migration test) | +| F3 simulation | `simulationData.{gasUsed,stateDiff}` matches | **yes** — `Runtime.evaluate` on state | +| F5 flag matrix | the same assertion repeated per `remoteFeatureFlags` state | **yes** | +| F7 i18n | static: `verify-locales` exit 0 | out-of-band | +| F8 runtime containment | `Object.isFrozen(Object.prototype)`; scuttled global throws + exception resolves; `typeof SNOW` | **yes** — `Runtime.evaluate`, but only against the SHIPPED build variant (dev is unscuttled, test's exception list is wider) | + +**Takeaway.** UI-state and runtime-metric lanes (A/B-visual, C1–C3/C6, F3/F5) map cleanly to CDP recipe assertions — ADR-0058's sweet spot. Static (D, F7), test-runner (B3, C5, F1), and dashboard (E) lanes are **out-of-band**: the recipe should *reference* them as proof targets without executing them. That out-of-band reference is precisely the **non-UI scaling gap** flagged in review of decisions#173 — a recipe schema that admits out-of-band assertion references (not only CDP actions) closes it. This table is the proposed taxonomy to contribute back (ITERATION item 10). diff --git a/domains/pr-workflow/skills/evidence/references/output-templates.md b/domains/pr-workflow/skills/evidence/references/output-templates.md new file mode 100644 index 00000000..752ac449 --- /dev/null +++ b/domains/pr-workflow/skills/evidence/references/output-templates.md @@ -0,0 +1,81 @@ +# Output templates + +The shape a validation run ships in. **This file is the generator.** A correction to how a +run reads is a defect here, not in the comment it was noticed on — fix it here and regenerate, +or the same correction arrives again on the next run. + +Drafting a template in a scratch directory is how that goes wrong: the comment gets better and +nothing else does. + +## The template + +```markdown + +## 🧪 Validation Run + +**Verdict:** — **Claim:** head `` · · + +> [!NOTE] +> Trial run of the [MetaMask evidence skills]() — feedback welcome, on the finding or +> on whether this format is useful to a reviewer. Not a review verdict; nothing here blocks +> the PR. + + + + + + + + + +**Follows from the above** + +- +- + +**Open for review:** + + +``` + +## What each slot is for + +**Verdict line.** The conclusion, not the topic. *"one of the two conjuncts is tested"* and +*"six renders where one would do"* are conclusions; *"tested the hash predicate"* is a topic. +Icons: `✅` proven · `⚠️` partial or scoped · `📋` measured, no verdict asserted · `❌` failed. +Never `❌` for a gap in *evidence* — that reads as a verdict on the author's work. + +**Check name, in words.** *red-on-base check*, *render-count check*, +*dependency-containment check*. Never the lane id: `B3` is an address into +[evidence-catalog.md](evidence-catalog.md), which the reviewer cannot open. + +**The exhibits.** Whatever the runner wrote, pasted whole and unfolded. They should outweigh +everything else in the comment; 70% is a reasonable floor. Do not summarise them above +themselves — a table of your own restating theirs turns a measurement into your word for it. + +**Follows from the above.** Bullets, each traceable to a number in an exhibit. If a bullet +needs three sentences, the exhibit is not carrying its weight. + +**The disclaimer sits directly under the verdict, and stays a callout.** It is the frame a +reviewer needs *before* they read a verdict on their own PR from a source they have not seen +before — where feedback goes, and that nothing here blocks them. Edited by the same rules as +prose it drifts to the foot of the page in ``, where it arrives after the reaction it +exists to shape. Check 11 tests its position, not just its presence. + +**Open for review.** One question, about this change. The runners' generic limits go to stderr +and the `.json` precisely so they do not end up here three times over; read them, and write +the thing a human should actually look at. + +## Assembly + +Templates carry `@@TOKEN@@` placeholders, one per exhibit, substituted with the runner's `.md` +verbatim. Substitution — never retyping — is what keeps the provenance line attached to the +numbers it vouches for. + +Before posting, `scripts/attest-gate.sh ` must exit 0. + +## Worked instantiations + +Three runs assembled from this template, with the reasoning behind each choice, are in +[worked-examples.md](worked-examples.md). diff --git a/domains/pr-workflow/skills/evidence/references/worked-examples.md b/domains/pr-workflow/skills/evidence/references/worked-examples.md new file mode 100644 index 00000000..e98a5f1f --- /dev/null +++ b/domains/pr-workflow/skills/evidence/references/worked-examples.md @@ -0,0 +1,30 @@ +# Worked examples (end-to-end) + +Full runs: claim → lanes → capture → trust-gate → publish. The visual case is in SKILL.md; these cover the non-visual claim shapes. + +## Perf — "defer Rive wasm at startup" +- **Claim:** on cold start of the home view, the Rive wasm chunk isn't requested until the animation surface mounts. **Surface:** startup network + chunk graph. **Falsifier:** the chunk appears in the cold-start waterfall. **Baseline:** base requests it at startup. +- **Lanes:** A2 `perf_validation` (primary) → D2 chunk membership + C6 CDP netlog (corroborate). +- **Capture:** paired build of base vs head (`yarn webpack --test`); CDP netlog over cold start for each; source-map chunk membership of the Rive runtime. +- **Trust gate:** cold-vs-cold (not warm); the chunk truly absent (not deferred by a few ms); the netlog covers the whole startup window. +- **Publish:** before/after request list + a chunk-membership table in the PR body. No screenshot needed. + +## Migration — "add migration NNN" +- **Claim:** loading a profile from `` applies NNN; `changedKeys = {X, Y}`; all other state intact. **Falsifier:** an untouched controller mutated / malformed state. **Baseline:** a prior-version profile. +- **Lanes:** F1 migration test (primary) → F2 vault round-trip (if the vault is touched). +- **Capture:** run `NNN.test.js` (old-state-in → new-state-out); assert `changedKeys`; load a real prior-version profile and confirm boot. +- **Trust gate:** the test asserts more than the happy path; `changedKeys` matches the actual mutations; the fixture is a real prior profile, not synthetic. +- **Publish:** the `changedKeys` assertion + before/after state shape; link the test run. + +## Flag-gated — "Perps banner behind a remote feature flag" +- **Claim (×2):** flag on → banner shows; flag off → banner absent. **Surface:** home/Perps. **Falsifier:** banner state ≠ flag state. **Baseline:** each flag state is its own baseline. +- **Lanes:** F5 flag matrix → A1/B1 visual per state. +- **Capture:** mock the client-config response for each flag state; screenshot each. +- **Trust gate:** the flag is actually toggled (read `remoteFeatureFlags`); two distinct states are shown, not the same frame twice. +- **Publish:** a two-up before/after (flag off / flag on) in the PR body. + +## Refactor / no-op — "extract a hook, no behavior change" +- **Claim (negation):** behavior of `` is unchanged. **Falsifier:** any output/behavior diff. **Baseline:** base behavior. +- **Lanes:** B3 regression suite stays green + B4 snapshot diff empty + D1 bundle within noise. +- **Trust gate:** snapshots were *not* regenerated to hide a diff; the tests actually cover the surface; bundle delta is within noise, not "small but real". +- **Publish:** "no behavior change — regression suite green, snapshots unchanged, bundle ±0"; link CI. A passing screenshot is not evidence here. diff --git a/domains/pr-workflow/skills/evidence/scripts/attest-gate.sh b/domains/pr-workflow/skills/evidence/scripts/attest-gate.sh new file mode 100755 index 00000000..4168e740 --- /dev/null +++ b/domains/pr-workflow/skills/evidence/scripts/attest-gate.sh @@ -0,0 +1,300 @@ +#!/usr/bin/env bash +# +# attest-gate — phase 0 of /attest. Mechanical, no model, hard fails only. +# +# Everything checkable is checked before anything is asked of a model, because a +# model asked "is this good evidence?" answers from inside the frame that produced +# the text. These are greppable, so they are not a matter of judgement. +# +# Usage: attest-gate.sh [--reference ] +# +# 0 all checks pass → proceed to the dispatched passes +# 1 one or more failed → BLOCKED, do not publish +# +# --target owner/repo#N is how check 12 learns where this is going. Without it the gate +# cannot tell a live review from a merged one, and the difference is the whole point. +# 2 usage error +set -uo pipefail + +FILE="${1:-}"; REF=""; TARGET=""; MODE="run" +shift || true +while [ $# -gt 0 ]; do + case "$1" in + --reference) REF="${2:-}"; shift 2 ;; + --target) TARGET="${2:-}"; shift 2 ;; + --diligence) MODE="diligence"; shift ;; + *) shift ;; + esac +done +[ -n "$FILE" ] || { echo "usage: attest-gate.sh [--reference ] [--target ] [--diligence]" >&2; exit 2; } +[ -f "$FILE" ] || { echo "attest-gate: not found: $FILE" >&2; exit 2; } + +FAILED=0 +pass() { printf ' PASS %s\n' "$1"; } +fail() { printf ' FAIL %s\n %s\n' "$1" "$2"; FAILED=$((FAILED+1)); } +has() { grep -qF "$1" "$FILE"; } +hasre(){ grep -qE "$1" "$FILE"; } +hasi() { grep -qiE "$1" "$FILE"; } # case-insensitive; a separate function because + # `hasre -i ''` silently greps for "-i". + +echo "attest-gate: $FILE" +echo + +# A diligence comment (lavamoat-policy and its siblings) renders no verdict and deliberately +# does not use the Validation Run envelope — see "One comment per evidence kind" in +# references/evidence-publishing.md. That exemption used to mean it was checked by nothing at +# all: attest-gate only knew the Validation Run shape, and pr-evidence-gate.py by design does +# not trip on a body claiming no verdict. So every rule the diligence skills state about their +# own output had no execution path, and a comment shipped with an unwitnessed local `npm pack` +# result and untraceable integers. --diligence swaps the envelope checks for that contract's +# own; everything downstream of the envelope is shared, because those defects are shared. +if [ "$MODE" = diligence ]; then + has 'LAVAMOAT_DILIGENCE_START' && has 'LAVAMOAT_DILIGENCE_END' \ + && pass "1 marker pair" \ + || fail "1 marker pair" "no LAVAMOAT_DILIGENCE_START/_END — a re-run appends a duplicate instead of replacing, and the pair must not be VALIDATION_RUN_* or an evidence re-run would eat this region" + + hasre '^\*\*LavaMoat grants|^LavaMoat grants' \ + && pass "2 canonical header" \ + || fail "2 canonical header" "missing the 'LavaMoat grants — -> ' opener" + + printf ' SKIP %s\n' "3 verdict line — a diligence comment renders none, by contract" +else +has 'VALIDATION_RUN_START' && has 'VALIDATION_RUN_END' \ + && pass "1 marker pair" \ + || fail "1 marker pair" "no VALIDATION_RUN_START/_END — a re-run appends a duplicate instead of replacing" + +has '## 🧪 Validation Run' \ + && pass "2 canonical header" \ + || fail "2 canonical header" "missing '## 🧪 Validation Run'" + +hasre '^\*\*Verdict:\*\*.*\*\*Claim:\*\*' \ + && pass "3 verdict line" \ + || fail "3 verdict line" "no '**Verdict:** … — **Claim:** …' — valence is not legible at a glance" +fi + +# A run outside the repo's toolchain pins a different thing. A browser-memory lane +# names "Firefox 153.0"; a repo lane names a head SHA and a lockfile hash. Both are +# pins, and a check that only knows the second one fails every run of the first — +# telling an author their pinned environment is unpinned. +# A diligence comment pins a read, not a run: its citations are permalinks, and the thing +# that can rot is a branch-head link drifting out from under the line it names. +if [ "$MODE" = diligence ]; then + if grep -qE 'https://github\.com/[^ )]+/blob/(main|master|develop|HEAD)/' "$FILE"; then + fail "4 citations pinned" "a permalink points at a branch head; it will drift off the line it cites. Pin a tag or a 40-char SHA" + elif grep -qE 'https://github\.com/[^ )]+/blob/[^/]+/' "$FILE"; then + pass "4 citations pinned" + else + fail "4 citations pinned" "no source permalink at all — the permalink IS the evidence here; a retyped 'it needs X' proves nothing about provenance" + fi +else +hasre 'head `[0-9a-f]{7,}|sha256|node `v|yarn\.lock `|[Ff]irefox [0-9]+\.[0-9]|[Cc]hrom(e|ium) [0-9]+\.|[Ss]afari [0-9]+\.|[Nn]ode v?[0-9]+\.[0-9]' \ + && pass "4 environment pinned" \ + || fail "4 environment pinned" "no head SHA, lockfile hash, or pinned toolchain/browser version" +fi + +# 5 — the one that matters, and it asks for a MEDIUM, not for better text. +# +# Every earlier version of this check tested a property of the plaintext: does it carry a +# provenance marker, does the command contain a placeholder, is the path local. Each caught +# one defect and missed the next, because every property of plaintext is forgeable by +# whatever emits the plaintext. Four runs shipped that way. +# +# So the block below is necessary but is no longer the evidence. The evidence is an image +# of the tool's own surface, a link that re-executes, or a hosted artifact the reader +# fetches without going through the author. If the artifact is small, nothing was attached. +# +# `Produced by` attests who WROTE the block, not that the block is the tool's own output. +# A script that composes a summary table and stamps itself passes on the marker alone — +# which is how a run shipped with a table the script had written, one grepped line, and a +# command reading `yarn jest `. A `$` line carrying a placeholder is the +# tell: it looks reproducible and cannot be run. +# An image, a re-executing link, or a hosted artifact — verification that does not route +# through the author. `Produced by` and `evidence-artifacts/` are provenance, not this. +# In diligence mode the medium is the permalink, already required by check 4 — a reader +# clicks it and lands on the line. What a permalink cannot witness is what the AUTHOR RAN, +# and that is the defect this variant catches: an `npm pack` unpacked locally, a grep over a +# tarball, a byte-comparison across policy files. Those read as properties of the package +# and are actually properties of an unwitnessed local run. State them as the search +# ("searched N files, no match") or publish the output; do not assert them as fact. +if [ "$MODE" = diligence ]; then + # A permalink is the medium for a CITATION — a reader clicks it and lands on the line. It is + # not the medium for anything you RAN. The first version of this branch tested for phrases + # ("npm pack", "complete specifier set") and passed an artifact whose entire results section + # was hand-typed to look like terminal output, because none of those words appeared in it. + # That is the regression the block below already documents as having shipped four times: + # every property of plaintext is forgeable by whatever emits the plaintext. So the test is + # the same one, on the same terms — if the artifact shows a command or a run result, it owes + # the reader something fetchable. + # `/blob/` is a CITATION, never a capture — it witnesses a line in a file, not a run. + # Excluding it matters: a permalink to `policy-override.json` ends in `.json` and satisfied + # a naive extension test, so an artifact whose entire results section was hand-typed passed + # on the strength of a source link. + if hasre '^\$ |^ *\$ |\bexit [0-9]|\bexit=[0-9]' \ + && ! grep -qE 'actions/runs/[0-9]|/gist\.|!\[[^]]*\]\(https?://' "$FILE" \ + && ! grep -E 'https?://[^ )]+\.(txt|log|json)\b' "$FILE" | grep -qv '/blob/'; then + fail "5 captured artifact" "shows a command or a run result with nothing a reader can fetch — a fenced block is your transcription, whatever produced it. Publish the log/gist/run and link it" + else + pass "5 captured artifact" + fi +elif ! hasre '!\[[^]]*\]\(https?://|]*src="https?://|actions/runs/[0-9]|/gist\.|https?://[^ )]+\.(png|jpg|jpeg|gif|svg|txt|log|json)\b'; then + fail "5 captured artifact" "no reader-verifiable capture — an image of the tool surface, a run link, or a hosted artifact. A fenced block is the author\'s transcription, whatever produced it" + # No separate attribution test: a hosted artifact the reader fetches is its own + # attribution, and requiring `Produced by` on top of it only fails runs whose + # evidence is stronger than a stamped fenced block. +elif grep -qE '^\$ .*<[a-z][a-z ._-]*>' "$FILE"; then + fail "5 captured artifact" "a console command contains a placeholder — $(grep -m1 -oE '^\$ .*' "$FILE") is not a command a reader can run" +elif grep -qE '^\$ .*(/tmp/|/home/|/Users/)' "$FILE"; then + # A helper script in /tmp, or any absolute local path, is unreproducible by + # construction. `capture.sh` records the command honestly — but honestly + # recording `bash /tmp/dup.sh` still publishes a recipe nobody else can follow. + # Inline the commands, or ship the helper where the reader can reach it. + fail "5 captured artifact" "a console command references a local-only path — $(grep -m1 -oE '^\$ .*(/tmp/|/home/|/Users/)[^ ]*' "$FILE") cannot be run by a reader" +elif [ "$(grep -cE '^\$ ' "$FILE")" -gt 1 ] && \ + [ "$(grep -E '^\$ ' "$FILE" | sed 's/ *#.*$//' | sort -u | wc -l)" -lt "$(grep -cE '^\$ ' "$FILE")" ]; then + # Two identical commands shown as producing different outputs. The difference + # came from an edit made between runs, so the block misstates its own cause: + # running it twice reproduces the first number twice. + fail "5 captured artifact" "two console commands are identical but shown with different output — the block does not say what actually differed between them" +else + pass "5 captured artifact" +fi + +if hasi 'what would close it|what would prove it|closing it requires'; then + fail "6 no prescriptions" "contains a 'what would close it' section — that is an unfinished run, formatted to look finished" +elif hasre '^\s*(Run|Switch|Assert|Scroll|Compare) '; then + fail "6 no prescriptions" "imperative-mood instructions to the reader — the artifact does not exist" +else + pass "6 no prescriptions" +fi + +# Drafting history is the author's, not the reader's: a reader who never saw the earlier +# version learns nothing from being told it existed, and the byline may not be yours. +# The list grew after a comment shipped a '### Correction:' section retracting its own +# previous revision in place — right instinct, wrong surface. Retract by restating the +# finding correctly; the account of how it changed belongs in a postmortem. +if hasi "I originally|correction to my earlier|filed by me|hard to calibrate|I withdraw|my earlier comment|earlier revision|previous revision|an earlier version of this|is withdrawn|that claim was wrong|^#{1,4} *Correction[: ]|corrected below|see the correction"; then + fail "7 no process narration" "contains first-person process commentary — the reader did not see the earlier draft, and the byline may not be yours" +else + pass "7 no process narration" +fi + +if [ "$MODE" = diligence ]; then + printf ' SKIP %s\n' "8 verdict is earned — no verdict rendered" + printf ' SKIP %s\n' "9 verdict matches artifact — no verdict rendered" +else +if hasi '\*\*Verdict:\*\*.*proven' && ! hasre 'Produced by |actions/runs|evidence-artifacts/'; then + fail "8 verdict is earned" "claims 'proven' with no execution artifact — reading yields 'unverified'" +else + pass "8 verdict is earned" +fi + +# 9 — the wrapper's verdict must not contradict the artifact it embeds. A comment is +# assembled by hand around machine output, and the hand-written header is exactly where +# a "vacuous" result acquires a "proven" label. +HDR="$(grep -m1 '^\*\*Verdict:\*\*' "$FILE" | tr 'A-Z' 'a-z')" +BODY="$(grep -ioE 'vacuous|value unstable|no delta|nothing falsified|broke the module|substitution silent|probe-failed' "$FILE" | head -1 | tr 'A-Z' 'a-z')" +if printf '%s' "$HDR" | grep -q 'proven' && [ -n "$BODY" ]; then + fail "9 verdict matches artifact" "header claims 'proven' while the embedded artifact reports '$BODY'" +else + pass "9 verdict matches artifact" +fi +fi + +# 10 — the positive counterpart to check 6. A run succeeds by putting concerns in front +# of a reviewer, so an artifact that floats nothing has reported only what it happened to +# measure and called that the whole picture. This is NOT satisfied by a "what would close +# it" section, which check 6 rejects: that hands the reader the run's own unfinished work, +# whereas this names a limit or a question the run is right to leave open. +# +# The vocabulary is a fixed list because this phase asks no model anything. That makes +# it blind to a limit phrased outside the list — a real run stated its limit as "what it +# does not establish" and the check called it absent. Add phrases when that happens; +# judging whether the stated limit is substantive is the dispatched passes' job. +if hasi 'open for review|raise with a human|falsifier|worth a look|left unmeasured|not covered by this run|no verdict offered|does not establish|what it does not|cannot attribute'; then + pass "10 floats something for review" +else + fail "10 floats something for review" "no limit, open question, or falsifier named — an artifact that floats nothing implies its measurement was the whole surface" +fi + +# 11 — the trial-run disclaimer, and its POSITION. This is not content, it is the frame +# the reader needs before they read a verdict on their own PR from an unfamiliar source. +# Compressed and moved to the foot of the page — which is what happens when it is edited +# by the same rules as prose — it arrives after the reaction it exists to shape. +DISC="$(grep -n -i 'trial run' "$FILE" | head -1 | cut -d: -f1)" +FIRST_EXHIBIT="$(grep -n '^```' "$FILE" | head -1 | cut -d: -f1)" +if [ -z "$DISC" ]; then + fail "11 disclaimer present and early" "no trial-run disclaimer — a reviewer cannot tell what this is or where to send feedback" +elif ! grep -qi 'trial run' "$FILE" || ! grep -q 'skills/pull/\|MetaMask/skills' "$FILE"; then + fail "11 disclaimer present and early" "disclaimer does not link the skills PR, so feedback has nowhere to go" +elif [ -n "$FIRST_EXHIBIT" ] && [ "$DISC" -gt "$FIRST_EXHIBIT" ]; then + fail "11 disclaimer present and early" "disclaimer is at line $DISC, after the first exhibit at line $FIRST_EXHIBIT — it frames nothing from there" +else + pass "11 disclaimer present and early" +fi + +if [ -n "$REF" ] && [ -f "$REF" ]; then + r=$(grep -coE '!\[|/dev/null 2>&1; then + # Blocks rather than warns. An earlier version printed UNVERIFIED and exited 0, so on a + # machine without `gh` — which is to say, running locally — this check announced that it + # had not run and passed anyway. That is the shape it exists to catch, one level up. + fail "12 destination is open" "gh not on PATH, so the destination was not checked. Unverified is not passing: install gh, or confirm the target is open and re-run." +else + t_repo="${TARGET%%#*}"; t_num="${TARGET##*#}" + t_state="$(gh api "repos/$t_repo/pulls/$t_num" --jq 'if .merged_at then "merged" else .state end' 2>/dev/null || echo unknown)" + case "$t_state" in + open) pass "12 destination is open" ;; + unknown) fail "12 destination is open" "could not read $TARGET — do not publish to a destination you could not check" ;; + *) fail "12 destination is open" "$TARGET is $t_state. A run published to a closed pull request reaches no reviewer and changes no decision." ;; + esac +fi + +# 13 — a number in the prose that appears in no exhibit. Check 9 compares verdict WORDS; +# nothing compared the figures. Measured on a demonstration artifact built to test this +# gate: the prose said "0 errors over 48 skills" directly above an exhibit reading +# "47 skill(s) checked", and named a warning class with zero instances in the output it +# was describing. Both survived every other check. Prose drifts from the exhibit it sits +# beside, and it is the most common way one of these goes wrong. +# +# Deliberately narrow, because a noisy check is an ignored check: integers of two or more +# digits only, and only those absent from every fenced block. Excluded as references +# rather than measurements — whole URLs, issue refs, version strings, dates, SHAs, +# file:line citations, hyphenated identifiers like P-256, and regex quantifiers. Every +# one of those was added after a control run flagged something that was not a figure. +echo +NUM_ORPHANS="$( + awk '/^```/{f=!f; next} f{print}' "$FILE" > "$FILE.exh" 2>/dev/null + awk '/^```/{f=!f; next} !f{print}' "$FILE" \ + | sed -E 's#https?://[^ )]*##g' \ + | sed -E 's/#[0-9]+//g; s/\bv?[0-9]+\.[0-9]+(\.[0-9]+)?\b//g; s/\b[0-9]{4}-[0-9]{2}-[0-9]{2}\b//g; s/\b[0-9a-f]{7,}\b//g; s/:[0-9]+\b//g; s/[A-Za-z]+-[0-9]+//g; s/\{[0-9,]+\}//g' \ + | grep -oE '\b[0-9]{2,}\b' | sort -u \ + | while read -r n; do grep -qF "$n" "$FILE.exh" || printf '%s ' "$n"; done + rm -f "$FILE.exh" +)" +if [ -n "$NUM_ORPHANS" ]; then + fail "13 figures trace to an exhibit" "these appear in the prose and in no exhibit: $NUM_ORPHANS — either they came from somewhere the reader cannot see, or they disagree with what is shown" +else + pass "13 figures trace to an exhibit" +fi + +echo +if [ "$FAILED" -eq 0 ]; then + echo "attest-gate: phase 0 clean — proceed to /outframe ‖ /missing ‖ /press" + exit 0 +fi +echo "attest-gate: BLOCKED — $FAILED check(s) failed. Do not publish." +exit 1 diff --git a/domains/pr-workflow/skills/evidence/scripts/capture.sh b/domains/pr-workflow/skills/evidence/scripts/capture.sh new file mode 100755 index 00000000..298cd619 --- /dev/null +++ b/domains/pr-workflow/skills/evidence/scripts/capture.sh @@ -0,0 +1,154 @@ +#!/usr/bin/env bash +# +# capture — turn any analysis command into a contract-compliant evidence artifact. +# +# The analysis scripts in this repo (retention-scan.py, policy-audit.py, a jest +# probe, a selector recomputation counter) all print to stdout. Printing to stdout +# means the operator is the capture device: they read it, retype some of it into a +# comment, and the result carries their provenance rather than the measurement's. +# +# This wraps any command so the ARTIFACT is written by the tool. Nothing is retyped. +# +# capture.sh --label --lane --claim "" [--verdict ] +# [--open ""] +# [--max-log-lines N | --head-lines N --tail-lines N] -- +# +# --verdict is stated by the caller, never inferred from the exit code: a wrapped +# tool's exit convention is its own, and guessing prints "pass" over real findings. +# +# --open is the same discipline pointed the other way. A run succeeds by putting +# concerns in front of a reviewer, not by closing them, so what the wrapped tool +# could not reach is publishable content. Omitting it is recorded, not hidden. +# +# Emits, under --out (default evidence-artifacts/): +# " +} > "$STAMP.md" + +printf 'capture: %s (exit %s)\n %s\n %s\n %s\n' "$VERDICT" "$CODE" "$STAMP.log" "$STAMP.json" "$STAMP.md" >&2 +# Stated limits reach the orchestrator, not the pasted exhibit: one open question per +# comment, about this diff, beats the same sentence repeated under every block. +if [ -n "$OPEN" ]; then + printf 'limits: %s\n' "$OPEN" >&2 +else + printf 'limits: none stated. This tool answered one question; what it does not cover was +not recorded, which is not the same as it covering everything.\n' >&2 +fi +exit "$CODE" diff --git a/domains/pr-workflow/skills/evidence/scripts/falsify-probe.sh b/domains/pr-workflow/skills/evidence/scripts/falsify-probe.sh new file mode 100755 index 00000000..46fed7f0 --- /dev/null +++ b/domains/pr-workflow/skills/evidence/scripts/falsify-probe.sh @@ -0,0 +1,214 @@ +#!/usr/bin/env bash +# +# falsify-probe — prove a test is falsifying, by mutation rather than by reading. +# +# A test is evidence only if it FAILS when the mechanism it guards is removed. +# Reading the test establishes its shape; only this establishes its power. +# +# Runs two arms against the same tree: +# Arm A baseline — the suite as committed +# Arm B mutant — one line replaced, suite re-run, source restored +# +# Emits a captured artifact (JSON + markdown) written by this script, not +# transcribed by an operator. Exit code IS the verdict, so CI can gate on it. +# +# 0 falsifying arm A passed, arm B failed ON ASSERTIONS → the test has power +# 1 vacuous arm A passed, arm B ALSO passed → the test proves nothing +# 2 broken arm A failed, or arm B did not run → nothing to conclude +# 3 usage/env error +# +# Arm B failing is NOT sufficient. A mutation that breaks syntax fails every test in +# the file, which looks identical to a falsification and is worth nothing: the suite +# never executed. So arm B must run the SAME number of tests as arm A and fail some of +# them. A dropped test count means the mutation broke the module, not the mechanism. +# +# Usage: +# falsify-probe.sh --test --source --line --replace +# [--expect-fail ]... +# [--label ] [--out ] [--runner ""] +# +# Example: +# falsify-probe.sh \ +# --test ui/hooks/perps/coalesceBackgroundRequest.test.ts \ +# --source ui/hooks/perps/coalesceBackgroundRequest.ts \ +# --line 54 --replace ' const existing = undefined as Promise | undefined;' \ +# --label coalesce-inflight +set -uo pipefail + +# A run's artifact has to say whether a reader can verify it. In CI the run URL is that +# verification; locally there is none, and the artifact says so rather than leaving the +# omission for a gate to catch later. +capture_provenance() { + if [ -n "${GITHUB_RUN_ID:-}" ]; then + printf 'Run: %s/%s/actions/runs/%s — logs and artifacts attached there.' \ + "${GITHUB_SERVER_URL:-https://github.com}" "${GITHUB_REPOSITORY:-}" "$GITHUB_RUN_ID" + else + printf 'Produced on a local machine: no reader-verifiable capture. Re-run through the evidence workflow before publishing.' + fi +} + +RUNNER="yarn jest" +OUT_DIR="evidence-artifacts" +LABEL="" +TEST="" SOURCE="" LINE="" REPLACE="" +# Which test names the caller predicts will fail. Caller-stated, like every other judgement +# word here, and checked rather than trusted: the guards ask whether arm B failed and whether +# it ran the same tests, never whether the RIGHT ones failed. A mutation silently corrupted +# before it reached the file failed a different case than it aimed at, ran the full suite, and +# was reported `falsifying` — a green verdict for a mechanism the run never touched. +EXPECT="" + +die() { printf 'falsify-probe: %s\n' "$1" >&2; exit 3; } + +while [ $# -gt 0 ]; do + case "$1" in + --test) TEST="${2:-}"; shift 2 ;; + --source) SOURCE="${2:-}"; shift 2 ;; + --line) LINE="${2:-}"; shift 2 ;; + --replace) REPLACE="${2:-}"; shift 2 ;; + --label) LABEL="${2:-}"; shift 2 ;; + --out) OUT_DIR="${2:-}"; shift 2 ;; + --runner) RUNNER="${2:-}"; shift 2 ;; + --expect-fail) EXPECT="$EXPECT${EXPECT:+\n}${2:-}"; shift 2 ;; + -h|--help) sed -n '2,32p' "$0"; exit 0 ;; + *) die "unknown argument: $1" ;; + esac +done + +[ -n "$TEST" ] || die "--test is required" +[ -n "$SOURCE" ] || die "--source is required" +[ -n "$LINE" ] || die "--line is required" +[ -n "$REPLACE" ] || die "--replace is required (use '' only if deleting the line)" +[ -f "$TEST" ] || die "test not found: $TEST" +[ -f "$SOURCE" ] || die "source not found: $SOURCE" +case "$LINE" in ''|*[!0-9]*) die "--line must be numeric: $LINE" ;; esac +[ "$LINE" -le "$(wc -l < "$SOURCE")" ] || die "--line $LINE is past the end of $SOURCE" + +LABEL="${LABEL:-$(basename "$SOURCE" | sed 's/\.[^.]*$//')-L$LINE}" +mkdir -p "$OUT_DIR" || die "cannot create $OUT_DIR" +STAMP="$OUT_DIR/falsify-$LABEL" + +# --- environment pin: two operators on different machines must be comparable --- +HEAD_SHA="$(git rev-parse HEAD 2>/dev/null || echo unknown)" +DIRTY="$(git status --porcelain 2>/dev/null | grep -v '^??' | wc -l | tr -d ' ')" +NODE_V="$(node -v 2>/dev/null || echo unknown)" +LOCK_SHA="$( { sha256sum yarn.lock 2>/dev/null || shasum -a 256 yarn.lock 2>/dev/null; } | cut -c1-16)" +ORIGINAL_LINE="$(sed -n "${LINE}p" "$SOURCE")" + +BACKUP="$(mktemp)" || die "mktemp failed" +cp "$SOURCE" "$BACKUP" +restore() { cp "$BACKUP" "$SOURCE"; rm -f "$BACKUP"; } +trap restore EXIT INT TERM + +run_arm() { # $1=logfile ; prints "passed|failed" + if $RUNNER "$TEST" > "$1" 2>&1; then echo passed; else echo failed; fi +} + +total_tests() { sed -n 's/.*Tests:.*[^0-9]\([0-9][0-9]*\) total.*/\1/p' "$1" | head -1; } +load_failed() { grep -qiE "SyntaxError|Cannot find module|Unexpected token|Transform failed" "$1"; } + +ARM_A="$(run_arm "$STAMP-armA.log")" + +if [ "$ARM_A" != "passed" ]; then + VERDICT="baseline-already-failing"; CODE=2; ARM_B="not-run" + : > "$STAMP-armB.log" +else + # Mutate exactly one line. The replacement travels through the environment, not + # through `awk -v`: awk runs escape processing on a `-v` assignment, so `[\s\S]` + # arrived as `[sS]` and the mutation written to the file was not the mutation asked + # for — it narrowed the regex it was meant to widen, failed a different test, and the + # runner reported `falsifying` for a mechanism it never touched. `ENVIRON` does no + # such processing. + MUTANT_LINE="$REPLACE" awk -v n="$LINE" 'NR==n{print ENVIRON["MUTANT_LINE"]; next}{print}' \ + "$SOURCE" > "$SOURCE.tmp" && mv "$SOURCE.tmp" "$SOURCE" || die "mutation failed" + # What the artifact reports as the mutation is read back off disk, never taken from the + # argument. The two differed once and nothing in the output said so. + APPLIED_LINE="$(sed -n "${LINE}p" "$SOURCE")" + if [ "$APPLIED_LINE" != "$REPLACE" ]; then + printf 'falsify-probe: the line written differs from --replace\n asked: %s\n written: %s\n' \ + "$REPLACE" "$APPLIED_LINE" >&2 + fi + ARM_B="$(run_arm "$STAMP-armB.log")" + restore; trap - EXIT INT TERM + A_TOTAL="$(total_tests "$STAMP-armA.log")"; A_TOTAL="${A_TOTAL:-0}" + B_TOTAL="$(total_tests "$STAMP-armB.log")"; B_TOTAL="${B_TOTAL:-0}" + if [ "$ARM_B" != "failed" ]; then + VERDICT="vacuous"; CODE=1 + elif load_failed "$STAMP-armB.log" || [ "$B_TOTAL" -lt "$A_TOTAL" ]; then + # The suite did not execute under mutation, so nothing was falsified. Reported as + # broken rather than falsifying: a module that will not load fails every test, which + # is indistinguishable from a real failure by exit code alone. + VERDICT="mutation broke the module — suite ran $B_TOTAL of $A_TOTAL tests, nothing falsified" + CODE=2 + else + VERDICT="falsifying"; CODE=0 + fi +fi + +# Runs last, on the verdict the guards already reached: a mutation can only fail the wrong +# case if it failed something, so this narrows `falsifying` and never widens it. +MISSED="" +if [ "$CODE" -eq 0 ] && [ -n "$EXPECT" ]; then + FAILED_SO_FAR="$(grep -E "^[[:space:]]+.[^\u203a]*\u203a" "$STAMP-armB.log" 2>/dev/null)" + printf '%b\n' "$EXPECT" | while IFS= read -r want; do + [ -n "$want" ] || continue + printf '%s' "$FAILED_SO_FAR" | grep -qF "$want" || printf '%s\n' "$want" + done > "$STAMP.missed" + MISSED="$(tr '\n' '|' < "$STAMP.missed" | sed 's/|$//;s/|/, /g')" + rm -f "$STAMP.missed" + if [ -n "$MISSED" ]; then + VERDICT="falsified a different case — predicted failure absent: $MISSED" + CODE=2 + fi +fi + +summarise() { grep -E '^(Tests|Test Suites):' "$1" 2>/dev/null | tr '\n' ' ' | sed 's/ */ /g'; } +A_SUM="$(summarise "$STAMP-armA.log")" +B_SUM="$(summarise "$STAMP-armB.log")" +FAILED_NAMES="$(grep -E '^\s+●[^›]*›' "$STAMP-armB.log" 2>/dev/null | sed 's/^ *//' | head -10)" + +cat > "$STAMP.json" <Produced by \`falsify-probe.sh\` at \`$HEAD_SHA\` · node \`$NODE_V\` · yarn.lock \`$LOCK_SHA\` · $DIRTY tracked changes. $(capture_provenance)" +} > "$STAMP.md" + +printf 'falsify-probe: %s (exit %s)\n %s\n %s\n' "$VERDICT" "$CODE" "$STAMP.json" "$STAMP.md" >&2 +# The limits below are identical on every run: they describe the instrument, not the +# change under review. Pasted into a PR comment they read as boilerplate to a reviewer +# who has no stake in this tooling, so they go to stderr and to the .json instead. The +# orchestrator reads them and writes ONE open question about THIS diff. +printf 'limits: one line of one file was mutated. Says nothing about other paths into the +same mechanism, whether it is reachable in production, or whether the guarded behaviour is +correct. A falsifying test proves the test has power, not that the fix is right.%s\n' \ + "$([ "$VERDICT" = vacuous ] && printf '\n vacuous: the mechanism is unguarded by this suite — what else depends on it?')" >&2 +exit "$CODE" diff --git a/domains/pr-workflow/skills/evidence/scripts/render-count.sh b/domains/pr-workflow/skills/evidence/scripts/render-count.sh new file mode 100755 index 00000000..ab3dcca2 --- /dev/null +++ b/domains/pr-workflow/skills/evidence/scripts/render-count.sh @@ -0,0 +1,157 @@ +#!/usr/bin/env bash +# +# render-count — lane C4, the component half. +# +# `selector-recompute` answers "how often does this selector recompute". This +# answers the other C4 question: "how many times does a consumer actually +# render". A memoization claim about context or props is a claim about that +# count, and a count of call sites is not it — 149 consumers can mean 149 +# avoided renders or none. +# +# Generates a probe that mounts a provider with a counting consumer, forces the +# parent to re-render N times with the memoised value unchanged, and reports the +# consumer's render count. Arm B re-runs with one line changed, so the delta is +# attributable rather than assumed. +# +# Usage: +# render-count.sh --probe [--defeat --defeat-line --defeat-with ] +# [--arm-b