diff --git a/scripts/__tests__/help-conformance-bench.test.ts b/scripts/__tests__/help-conformance-bench.test.ts new file mode 100644 index 000000000..5f371759e --- /dev/null +++ b/scripts/__tests__/help-conformance-bench.test.ts @@ -0,0 +1,124 @@ +import assert from 'node:assert/strict'; +import { execFile } from 'node:child_process'; +import { mkdtemp, readdir, readFile, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { promisify } from 'node:util'; +import { test } from 'vitest'; + +const execFileAsync = promisify(execFile); +const SCRIPT = join(import.meta.dirname, '..', 'help-conformance-bench.mjs'); + +// These tests spawn the real script in --dry-run mode with every required doc +// overridden, so no LLM call and no built CLI is needed: the raw-first-screen +// case's only doc is --help:first30, and the override replaces the shell-out. + +async function runBench(args: string[]): Promise<{ code: number; stdout: string; stderr: string }> { + try { + const { stdout, stderr } = await execFileAsync('node', [SCRIPT, ...args]); + return { code: 0, stdout, stderr }; + } catch (error) { + const payload = error as { code?: number; stdout?: string; stderr?: string }; + return { code: payload.code ?? 1, stdout: payload.stdout ?? '', stderr: payload.stderr ?? '' }; + } +} + +async function readDryRunPrompt(outDir: string): Promise { + const reportName = (await readdir(outDir)).find((name) => name.startsWith('report-')); + assert.ok(reportName, 'dry-run must write a report file'); + const report = JSON.parse(await readFile(join(outDir, reportName), 'utf8')) as Array<{ + prompt: string; + }>; + assert.ok(report.length > 0 && typeof report[0]?.prompt === 'string'); + return report[0].prompt; +} + +// Guards the --override-doc contract: an override swaps only WHERE the doc +// text comes from, never how it is sliced. The --help:first30 doc id caps the +// live `--help` output at 30 lines, so an override longer than that must be +// capped identically or the A/B comparison grades content a live run never +// shows (the exact bug found in review of the initial version). +test('override for --help:first30 goes through the same 30-line cap as the live doc', async () => { + const dir = await mkdtemp(join(tmpdir(), 'help-bench-')); + const draftPath = join(dir, 'draft-help.txt'); + const lines = Array.from({ length: 49 }, (_, i) => `draft help line ${i + 1}`); + await writeFile(draftPath, lines.join('\n')); + const run = await runBench([ + '--dry-run', + '--case', + 'raw-first-screen-bluesky', + '--runner', + 'claude:test-model', + '--override-doc', + `--help:first30=${draftPath}`, + '--out', + dir, + ]); + assert.equal(run.code, 0, run.stderr); + const prompt = await readDryRunPrompt(dir); + assert.ok(prompt.includes('draft help line 30'), 'line 30 is inside the cap and must survive'); + assert.ok(!prompt.includes('draft help line 31'), 'line 31 is past the cap and must be cut'); +}); + +test('an override topic id no selected case uses fails fast and lists valid doc ids', async () => { + const dir = await mkdtemp(join(tmpdir(), 'help-bench-')); + const draftPath = join(dir, 'draft.txt'); + await writeFile(draftPath, 'irrelevant'); + const run = await runBench([ + '--dry-run', + '--case', + 'raw-first-screen-bluesky', + '--override-doc', + `totally-bogus-topic=${draftPath}`, + '--out', + dir, + ]); + assert.notEqual(run.code, 0); + assert.match(run.stderr, /totally-bogus-topic/); + assert.match(run.stderr, /Valid doc ids: --help:first30/); + assert.equal( + (await readdir(dir)).some((name) => name.startsWith('report-')), + false, + ); +}); + +test('a missing override file reports one clean error line, not a stack trace', async () => { + const dir = await mkdtemp(join(tmpdir(), 'help-bench-')); + const run = await runBench([ + '--dry-run', + '--case', + 'raw-first-screen-bluesky', + '--override-doc', + `--help:first30=${join(dir, 'does-not-exist.txt')}`, + '--out', + dir, + ]); + assert.notEqual(run.code, 0); + assert.match(run.stderr, /--override-doc file for "--help:first30" is not readable/); + assert.doesNotMatch(run.stderr, /at .*help-conformance-bench\.mjs/); +}); + +test('repeated overrides for the same topic id are last-wins', async () => { + const dir = await mkdtemp(join(tmpdir(), 'help-bench-')); + const firstPath = join(dir, 'first.txt'); + const secondPath = join(dir, 'second.txt'); + await writeFile(firstPath, 'first draft body'); + await writeFile(secondPath, 'second draft body'); + const run = await runBench([ + '--dry-run', + '--case', + 'raw-first-screen-bluesky', + '--runner', + 'claude:test-model', + '--override-doc', + `--help:first30=${firstPath}`, + '--override-doc', + `--help:first30=${secondPath}`, + '--out', + dir, + ]); + assert.equal(run.code, 0, run.stderr); + const prompt = await readDryRunPrompt(dir); + assert.ok(prompt.includes('second draft body')); + assert.ok(!prompt.includes('first draft body')); +}); diff --git a/scripts/help-conformance-bench.mjs b/scripts/help-conformance-bench.mjs index e733a1434..de748a08c 100644 --- a/scripts/help-conformance-bench.mjs +++ b/scripts/help-conformance-bench.mjs @@ -9,13 +9,42 @@ const execFileAsync = promisify(execFile); const ROOT = new URL('..', import.meta.url).pathname; const OUT_DIR = process.env.HELP_BENCH_OUT ?? join(ROOT, '.tmp', 'help-conformance-bench'); const RUN_TIMEOUT_MS = Number(process.env.HELP_BENCH_TIMEOUT_MS ?? 90_000); +// Runner x case pairs run concurrently, capped low: these are paid LLM calls +// and the CLI help subprocess calls behind loadDocs share this same machine. +const CONCURRENCY = Number(process.env.HELP_BENCH_CONCURRENCY ?? 4); const DEFAULT_RUNNERS = ['codex:gpt-5.4-mini', 'claude:claude-haiku-4-5']; +const USAGE = `Usage: node scripts/help-conformance-bench.mjs [options] + +Feeds a help slice + task into one non-agentic LLM call per runner x case and +regex-scores the returned command plan. + +Options: + --runner Add one runner (repeatable). Default: ${DEFAULT_RUNNERS.join(', ')} + --runners Comma-separated runner list + --case Add one case id (repeatable). Default: all cases + --cases Comma-separated case id list + --out Output directory (default: .tmp/help-conformance-bench) + --override-doc = + Grade a DRAFT doc: load this topic's text from the file + instead of the live CLI help. Repeatable; the last + occurrence per topic wins. Override text goes through the + same post-processing as the live source (e.g. the + --help:first30 doc id is still capped to its first 30 + lines), so an A/B grade compares like with like. + --dry-run Build prompts and write the report without any LLM calls + --help Show this usage text + +Environment: + HELP_BENCH_CONCURRENCY Concurrent runner x case calls (default: 4) + HELP_BENCH_TIMEOUT_MS Per-call timeout (default: 90000) + HELP_BENCH_OUT Default output directory`; const OPTION_SPECS = { '--runner': { target: 'runners', mode: 'append' }, '--runners': { target: 'runners', mode: 'csv' }, '--case': { target: 'cases', mode: 'append' }, '--cases': { target: 'cases', mode: 'csv' }, '--out': { target: 'outDir', mode: 'value' }, + '--override-doc': { target: 'overrideDocs', mode: 'keyvalue' }, }; const OPTION_APPLIERS = { append: (args, target, value) => { @@ -27,8 +56,24 @@ const OPTION_APPLIERS = { value: (args, target, value) => { args[target] = value; }, + keyvalue: (args, target, value) => { + const separatorIndex = value.indexOf('='); + if (separatorIndex <= 0) { + throw new Error(`--override-doc expects =, got: ${value}`); + } + const topicId = value.slice(0, separatorIndex); + const path = value.slice(separatorIndex + 1); + const map = args[target] ?? new Map(); + map.set(topicId, path); + args[target] = map; + }, }; +// Raw-coordinate fallback the ported skillgym quiz cases forbid: a +// click/fill/press targeting bare numbers instead of a ref or selector. +const RAW_COORDINATE_TARGET = + /(?:^|\n)(?:agent-device\s+)?(?:click|fill|press)\s+-?\d+(?:\.\d+)?\s+-?\d+(?:\.\d+)?/i; + const CASES = [ { id: 'raw-first-screen-bluesky', @@ -60,6 +105,92 @@ const CASES = [ task: 'Plan commands to validate a CLI/runtime change in agent-device against an iOS app without accidentally using stale built output.', expectations: ['fullPrefix', 'usesValidationPrep', 'opensAndCloses'], }, + // The three cases below are ported from + // test/skillgym/suites/agent-device-smoke-suite.ts (settle-diff-is-observation, + // sample-output-settled-diff-next-target, sample-output-not-settled-needs-observe). + // They are self-contained "next-command quiz" cases: a captured agent-device + // output plus a task, scored by regex instead of the named expectation + // scorers above. Output text mirrors the CURRENT settle rendering in + // src/commands/interaction/output.ts, including the "unchanged interactive + // (N):" tail added by #1167/#1172 for diffs with no meaningful added ref. + { + id: 'settle-diff-is-observation', + docs: ['--help:first30'], + task: `You already ran this command and observed its settled output: + +agent-device press @e37 --settle +Tapped @e37 (203, 88) +settled after 540ms: +0 -1 (~15 unchanged) +- @e50 [text] "Suggested for you" +unchanged interactive (4): += @e64 [text-field] "Search" += @e65 [text] "Recent searches" += @e12 [tab] "Home" += @e40 [tab] "Profile" + +The task was to confirm the feed-search UI is present, then close the session. The settled diff and its unchanged-interactive tail above already contain every ref and piece of evidence the task needs: the Search field and Recent searches are both listed. Plan only the next command. Do not take another snapshot, wait, find, get, or is call just to re-read evidence that is already shown above.`, + expectations: ['fullPrefix'], + matchers: [{ id: 'plansClose', pattern: /(?:^|\n)(?:agent-device\s+)?close\b/i }], + forbidden: [ + { id: 'noSnapshot', pattern: /\bsnapshot\b/i }, + { id: 'noWait', pattern: /\bwait\b/i }, + { id: 'noFind', pattern: /\bfind\b/i }, + { id: 'noGet', pattern: /\bget\b/i }, + { id: 'noIs', pattern: /\bis\b/i }, + { id: 'noPressOrClick', pattern: /\b(?:press|click)\b/i }, + ], + }, + { + id: 'sample-output-settled-diff-next-target', + docs: ['--help:first30'], + task: `Read this previous agent-device output, then plan the next command: + +agent-device fill 'id="account-search"' "callstack" --settle +Filled 9 chars +settled after 610ms: +2 -0 (~18 unchanged) ++ @e64 [button] "@callstack.com" ++ @e65 [text] "Callstack" + +Use the ref exposed by the settled diff to open the account, with --settle on this next action too. Do not re-read the same screen first.`, + expectations: ['fullPrefix'], + matchers: [ + { id: 'pressOrClickOrTap', pattern: /\b(?:press|click|tap)\b/i }, + { id: 'usesE64RefOrLabel', pattern: /@e64\b|label=(?:["']?@callstack\.com["']?)/i }, + { id: 'usesSettleFlag', pattern: /--settle\b/i }, + ], + forbidden: [ + { id: 'noSnapshot', pattern: /\bsnapshot\b/i }, + { id: 'noWaitStable', pattern: /wait\s+stable/i }, + { id: 'noFill', pattern: /\bfill\b/i }, + { id: 'noRawCoordinateTarget', pattern: RAW_COORDINATE_TARGET }, + ], + }, + { + id: 'sample-output-not-settled-needs-observe', + docs: ['--help:first30'], + task: `Read this previous agent-device output, then plan the next command: + +agent-device press @e12 --settle +Tapped @e12 (166, 240) +not settled after 10000ms +hint: The UI kept changing for the whole settle budget (animation, carousel, or ticker?), so no settled diff is shown. Raise --timeout, wait for specific content, or take a fresh snapshot. + +Old refs may be stale after this mutation, and no settled diff was printed, so the next target is unknown. Follow the output hint: observe the current UI (a fresh snapshot or a wait) before attempting another ref-based action.`, + expectations: ['fullPrefix'], + matchers: [ + { + id: 'observesBeforeActing', + pattern: /(?:^|\n)(?:agent-device\s+)?(?:wait\b|snapshot\b[^\n]*-i\b)/i, + }, + ], + forbidden: [ + { + id: 'noBareRefMutation', + pattern: /(?:^|\n)(?:agent-device\s+)?(?:press|click|fill|longpress)\s+@e\d+/i, + }, + { id: 'noRawCoordinateTarget', pattern: RAW_COORDINATE_TARGET }, + ], + }, ]; function parseArgs(argv) { @@ -76,6 +207,10 @@ function readArgs(args, argv, index) { function readArg(args, argv, index) { const arg = argv[index]; + if (arg === '--help' || arg === '-h') { + console.log(USAGE); + process.exit(0); + } if (arg === '--dry-run') return applyDryRun(args, index); applyOption(args, optionSpec(arg), argv[index + 1]); return index + 2; @@ -88,7 +223,7 @@ function applyDryRun(args, index) { function optionSpec(arg) { const spec = OPTION_SPECS[arg]; - if (!spec) throw new Error(`Unknown argument: ${arg}`); + if (!spec) throw new Error(`Unknown argument: ${arg}. Run with --help for usage.`); return spec; } @@ -104,6 +239,7 @@ function assertOptionValue(spec, value) { function applyDefaultArgs(args) { args.runners ??= DEFAULT_RUNNERS; args.cases ??= CASES.map((testCase) => testCase.id); + args.overrideDocs ??= new Map(); } async function main() { @@ -111,7 +247,9 @@ async function main() { const outDir = resolveOutDir(args); await mkdir(outDir, { recursive: true }); const selectedCases = selectCases(args.cases); - const docs = await loadDocs(requiredDocIds(selectedCases)); + const docIds = requiredDocIds(selectedCases); + assertOverrideDocIds(args.overrideDocs, docIds); + const docs = await loadDocs(docIds, args.overrideDocs); const results = await runBenchmarkMatrix(args.runners, selectedCases, docs, outDir, args.dryRun); const reportPath = join(outDir, `report-${Date.now()}.json`); @@ -138,19 +276,47 @@ function requiredDocIds(selectedCases) { return [...new Set(selectedCases.flatMap((testCase) => testCase.docs))]; } +// A typo'd or stale --override-doc topic id must not silently grade the real +// doc while the caller believes the draft was measured: fail fast instead. +function assertOverrideDocIds(overrideDocs, docIds) { + const unknown = [...overrideDocs.keys()].filter((topicId) => !docIds.includes(topicId)); + if (unknown.length === 0) return; + throw new Error( + `--override-doc topic id(s) not used by the selected cases: ${unknown.join(', ')}. Valid doc ids: ${docIds.join(', ')}.`, + ); +} + function updateExitCode(results, dryRun) { if (!dryRun && results.some((result) => !result.passed)) process.exitCode = 1; } async function runBenchmarkMatrix(runners, selectedCases, docs, outDir, dryRun) { - const results = []; - for (const runner of runners) { - for (const testCase of selectedCases) { - const result = await runBenchmarkEntry(runner, testCase, docs, outDir, dryRun); - results.push(result); - printResult(result, dryRun); + const entries = runners.flatMap((runner) => + selectedCases.map((testCase) => ({ runner, testCase })), + ); + // Concurrency-capped, but results print in the original runner x case + // matrix order (not completion order) once every entry has settled, so + // output stays as readable as the old sequential loop. + const results = await mapWithConcurrency(entries, CONCURRENCY, ({ runner, testCase }) => + runBenchmarkEntry(runner, testCase, docs, outDir, dryRun), + ); + for (const result of results) printResult(result, dryRun); + return results; +} + +async function mapWithConcurrency(items, limit, worker) { + const results = new Array(items.length); + let nextIndex = 0; + async function runNext() { + for (;;) { + const current = nextIndex; + nextIndex += 1; + if (current >= items.length) return; + results[current] = await worker(items[current], current); } } + const workerCount = Math.max(1, Math.min(limit, items.length)); + await Promise.all(Array.from({ length: workerCount }, runNext)); return results; } @@ -168,17 +334,44 @@ function printResult(result, dryRun) { ); } -async function loadDocs(docIds) { - return Object.fromEntries(await Promise.all(docIds.map(loadDocEntry))); +async function loadDocs(docIds, overrideDocs) { + return Object.fromEntries( + await Promise.all(docIds.map((docId) => loadDocEntry(docId, overrideDocs))), + ); +} + +async function loadDocEntry(docId, overrideDocs) { + return [docId, await loadDoc(docId, overrideDocs)]; } -async function loadDocEntry(docId) { - return [docId, await loadDoc(docId)]; +async function loadDoc(docId, overrideDocs) { + // An override swaps only WHERE the text comes from; the per-doc + // post-processing below (e.g. the --help:first30 30-line cap) applies to + // both sources so an A/B grade compares like with like. Without this, a + // draft longer than 30 lines would be graded on content the live path + // always truncates away. + return postProcessDoc(docId, await loadDocSource(docId, overrideDocs)); } -async function loadDoc(docId) { - if (docId === '--help:first30') return firstLines(await cliHelp(['--help']), 30); - return cliHelp(['help', docId]); +async function loadDocSource(docId, overrideDocs) { + const overridePath = overrideDocs?.get(docId); + if (overridePath) return readOverrideDoc(docId, overridePath); + return docId === '--help:first30' ? cliHelp(['--help']) : cliHelp(['help', docId]); +} + +async function readOverrideDoc(docId, overridePath) { + try { + return await readFile(overridePath, 'utf8'); + } catch (error) { + throw new Error( + `--override-doc file for "${docId}" is not readable: ${overridePath} (${error?.code ?? errorMessage(error)})`, + ); + } +} + +function postProcessDoc(docId, text) { + const trimmed = text.trim(); + return docId === '--help:first30' ? firstLines(trimmed, 30) : trimmed; } async function cliHelp(args) { @@ -210,16 +403,17 @@ async function runCase(runner, testCase, prompt, outDir) { const outputPath = join(outDir, `${safeName(runner)}-${testCase.id}.txt`); await writeFile(outputPath, raw); const commands = extractCommands(raw); - const checks = scoreExpectations(testCase.expectations, commands, raw); + const checks = scoreExpectations(testCase, commands, raw); const score = countPassingChecks(checks); + const total = countChecks(testCase); return { runner, caseId: testCase.id, commands, checks, score, - total: testCase.expectations.length, - passed: runnerError === undefined && score === testCase.expectations.length, + total, + passed: runnerError === undefined && score === total, ...(runnerError ? { runnerError } : {}), outputPath, }; @@ -286,8 +480,11 @@ function execFileWithInput(file, args, input, options) { } async function runCodex(model, prompt, outDir) { - const outFile = join(outDir, `codex-${model}-${Date.now()}.json`); - const { stdout } = await execFileAsync( + const outFile = join( + outDir, + `codex-${model}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}.json`, + ); + const pending = execFileAsync( 'codex', [ 'exec', @@ -306,13 +503,24 @@ async function runCodex(model, prompt, outDir) { ], { cwd: ROOT, maxBuffer: 1024 * 1024 * 20, timeout: RUN_TIMEOUT_MS }, ); + // codex exec reads from stdin until EOF when it isn't a TTY. execFile never + // closes the child's stdin pipe on its own, so without this the process + // blocks on "Reading additional input from stdin..." until RUN_TIMEOUT_MS + // kills it and every codex case reports empty/error output. + pending.child?.stdin?.end(); + const { stdout } = await pending; let lastMessage = ''; try { lastMessage = await readFile(outFile, 'utf8'); } catch { // stdout still carries the transcript when -o fails. } - return `${stdout}\n${lastMessage}`; + // `-o` writes the same final JSON message that codex also prints to + // stdout when it isn't attached to a TTY. Concatenating both produces two + // back-to-back JSON objects, which breaks every downstream JSON.parse + // candidate and silently zeroes out extractCommands(). Prefer the clean + // -o payload and only fall back to stdout if it's missing/empty. + return lastMessage.trim().length > 0 ? lastMessage : stdout; } function extractCommands(raw) { @@ -378,11 +586,38 @@ const EXPECTATION_SCORERS = { opensAndCloses: ({ joined }) => /\bopen\b/.test(joined) && /\bclose\b/.test(joined), }; -function scoreExpectations(expectations, commands, raw) { +/** + * Every check a case declares, normalized to a uniform `{ id, test }` shape: + * - `expectations`: named lookups into EXPECTATION_SCORERS (the original 4 + * help-layout cases). + * - `matchers`: the check passes when the pattern matches the planned + * commands (ported skillgym quiz cases' `outputs`). + * - `forbidden`: the check passes when the pattern does NOT match (ported + * skillgym quiz cases' `forbiddenOutputs`). + */ +function resolveChecks(testCase) { + const named = (testCase.expectations ?? []).map((id) => ({ + id, + test: (context) => scoreExpectation(id, context), + })); + const matched = (testCase.matchers ?? []).map(({ id, pattern }) => ({ + id, + test: (context) => pattern.test(context.joined), + })); + const forbidden = (testCase.forbidden ?? []).map(({ id, pattern }) => ({ + id, + test: (context) => !pattern.test(context.joined), + })); + return [...named, ...matched, ...forbidden]; +} + +function countChecks(testCase) { + return resolveChecks(testCase).length; +} + +function scoreExpectations(testCase, commands, raw) { const context = { commands, joined: commands.join('\n').toLowerCase(), raw }; - return Object.fromEntries( - expectations.map((expectation) => [expectation, scoreExpectation(expectation, context)]), - ); + return Object.fromEntries(resolveChecks(testCase).map(({ id, test }) => [id, test(context)])); } function scoreExpectation(expectation, context) { @@ -408,4 +643,11 @@ function safeName(name) { return basename(name).replace(/[^a-z0-9_.-]+/gi, '-'); } -await main(); +// Expected failures (bad flags, unreadable override files, unknown topic ids) +// print as one clean line instead of an unhandled stack trace. +try { + await main(); +} catch (error) { + console.error(`Error: ${errorMessage(error)}`); + process.exitCode = 1; +} diff --git a/test/skillgym/README.md b/test/skillgym/README.md index 03bdf8cc1..a24d7c096 100644 --- a/test/skillgym/README.md +++ b/test/skillgym/README.md @@ -47,7 +47,7 @@ Skill-guidance regression cases cover distinct command-planning habits: - performance metrics, React DevTools profiling, gestures, settings, and trace capture - remote config, macOS menu bar surfaces, replay update, same-session mutation ordering, and batch schema/recording -Use SkillGym for stable behavior regressions: can a runner choose the right next command from help, app-contract facts, or representative CLI output? For rapid help-layout A/B testing, prefer the lighter help conformance bench because it can feed only the top-level first screen or one help topic without letting the runner read the full help page. +Use SkillGym for stable behavior regressions: can a runner choose the right next command from help, app-contract facts, or representative CLI output? For rapid help-layout A/B testing, prefer the lighter help conformance bench (`scripts/help-conformance-bench.mjs`) because it can feed only the top-level first screen or one help topic without letting the runner read the full help page, it runs runner x case pairs concurrently (`HELP_BENCH_CONCURRENCY`, default 4), and it can grade a draft help rewrite with zero rebuild via `--override-doc =` (repeatable; last occurrence per topic wins), which loads that file's contents in place of shelling out to `node bin/agent-device.mjs help ` for that one topic while still applying the same post-processing as the live source (the `--help:first30` doc id stays capped to its first 30 lines), so the A/B grade compares like with like; a topic id no selected case uses fails fast instead of silently grading the real doc. It also includes three "next-command quiz" cases (`settle-diff-is-observation`, `sample-output-settled-diff-next-target`, `sample-output-not-settled-needs-observe`) ported from this suite's skill-guidance regressions, for scoring how a runner reads representative captured `agent-device` output — including the `--settle` "unchanged interactive (N):" tail — instead of full help text. Filter with `--cases`/`--case` and `--runners`/`--runner` (both repeatable/CSV) the same way as this suite. `assertAgentDeviceEvidence` is intentionally soft when a runner does not expose skill-detection telemetry. When telemetry exists, the suite asserts that `agent-device` was loaded; when it is absent, the cases still judge command-planning output instead of failing on missing runner metadata. diff --git a/vitest.config.ts b/vitest.config.ts index 4a31f1269..22395b91e 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -19,7 +19,9 @@ export default defineConfig({ { test: { name: 'unit-core', - include: ['src/**/*.test.ts'], + // The explicit scripts entry keeps the help-bench guard tests in the + // suite without waking every ad-hoc *.test.ts under scripts/. + include: ['src/**/*.test.ts', 'scripts/__tests__/help-conformance-bench.test.ts'], exclude: [ANDROID_ADB_STUB_TESTS], setupFiles: ['src/__tests__/process-memo-setup.ts'], },