diff --git a/.gitignore b/.gitignore index 8c2df2fb1..cbe293ff4 100644 --- a/.gitignore +++ b/.gitignore @@ -20,6 +20,9 @@ coverage-isolated *.junit.xml test/skill-eval/results.json +# local bench baselines — machine-specific, not for version control +.bench/ + # logs logs _.log diff --git a/bun.lock b/bun.lock index 406cc30ea..41f9d1d01 100644 --- a/bun.lock +++ b/bun.lock @@ -17,6 +17,7 @@ "@types/bun": "latest", "@types/http-cache-semantics": "^4.2.0", "@types/node": "^22", + "@types/picomatch": "^4.0.3", "@types/qrcode-terminal": "^0.12.2", "@types/semver": "^7.7.1", "binpunch": "^1.0.0", @@ -226,6 +227,8 @@ "@types/node-fetch": ["@types/node-fetch@2.6.13", "", { "dependencies": { "@types/node": "*", "form-data": "^4.0.4" } }, "sha512-QGpRVpzSaUs30JBSGPjOg4Uveu384erbHBoT1zeONvyCfwQxIkUshLAOqN/k9EjGviPRmWTTe6aH2qySWKTVSw=="], + "@types/picomatch": ["@types/picomatch@4.0.3", "", {}, "sha512-iG0T6+nYJ9FAPmx9SsUlnwcq1ZVRuCXcVEvWnntoPlrOpwtSTKNDC9uVAxTsC3PUvJ+99n4RpAcNgBbHX3JSnQ=="], + "@types/qrcode-terminal": ["@types/qrcode-terminal@0.12.2", "", {}, "sha512-v+RcIEJ+Uhd6ygSQ0u5YYY7ZM+la7GgPbs0V/7l/kFs2uO4S8BcIUEMoP7za4DNIqNnUD5npf0A/7kBhrCKG5Q=="], "@types/qs": ["@types/qs@6.14.0", "", {}, "sha512-eOunJqu0K1923aExK6y8p6fsihYEn/BYuQ4g0CxAAgFc4b/ZLN4CrsRZ55srTdqoiLzU2B2evC+apEIxprEzkQ=="], diff --git a/package.json b/package.json index de121753b..c5fad8711 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,7 @@ "@types/bun": "latest", "@types/http-cache-semantics": "^4.2.0", "@types/node": "^22", + "@types/picomatch": "^4.0.3", "@types/qrcode-terminal": "^0.12.2", "@types/semver": "^7.7.1", "binpunch": "^1.0.0", @@ -91,6 +92,10 @@ "generate:schema": "bun run script/generate-api-schema.ts", "generate:command-docs": "bun run script/generate-command-docs.ts", "eval:skill": "bun run script/eval-skill.ts", + "bench": "bun run script/bench.ts", + "bench:save": "bun run script/bench.ts --save-baseline", + "bench:compare": "bun run script/bench.ts --compare", + "bench:sweep": "bun run script/bench-sweep.ts", "check:fragments": "bun run script/check-fragments.ts", "check:deps": "bun run script/check-no-deps.ts", "check:errors": "bun run script/check-error-patterns.ts", diff --git a/script/bench-sweep.ts b/script/bench-sweep.ts new file mode 100644 index 000000000..c168b3438 --- /dev/null +++ b/script/bench-sweep.ts @@ -0,0 +1,375 @@ +#!/usr/bin/env bun +/** + * Concurrency sweep for `src/lib/scan/` hot paths. + * + * Goal: measure how the walker + grep scale with `concurrency` on + * the synthetic bench fixtures, so we can pick a data-driven default + * for `CONCURRENCY_LIMIT`. + * + * The main bench harness (`script/bench.ts`) uses a fixed concurrency + * inherited from the DSN scanner. This script is a one-shot + * diagnostic run by contributors when tuning perf — it's not wired + * into CI. + * + * Usage: + * bun run bench:sweep # full sweep on medium+large + * bun run bench:sweep --size small # one preset + * bun run bench:sweep --values 1,2,4,8,16,32 # custom concurrency grid + * bun run bench:sweep --runs 10 --warmup 3 # override run counts + * bun run bench:sweep --json > sweep.json # machine-readable + * + * Output: a per-(fixture, op) table of p50 times across the + * concurrency grid, plus a "knee" annotation flagging the value + * past which additional parallelism yields < 3% improvement. + */ + +import { existsSync, mkdirSync } from "node:fs"; +import { arch, availableParallelism, cpus, platform, tmpdir } from "node:os"; +import { join } from "node:path"; +import { + type FixtureSpec, + generateFixture, + hashSpec, +} from "../test/fixtures/bench/generate.js"; +import { + measure, + summarize, + withBenchDb, +} from "../test/fixtures/bench/helpers.js"; +import { + PRESET_NAMES, + PRESETS, + type PresetName, +} from "../test/fixtures/bench/presets.js"; + +/** Default concurrency values we sweep across. */ +const DEFAULT_VALUES = [1, 2, 4, 8, 16, 32, 50, 100, 200] as const; + +/** Default fixture sizes we sweep on. `small` rarely shows signal. */ +const DEFAULT_SIZES: readonly PresetName[] = ["medium", "large"]; + +/** + * DSN scanner hot regex — reused by the `scan.grepFiles` op. Kept at + * module scope to satisfy Biome's `useTopLevelRegex` rule. + */ +const DSN_PATTERN = + /https?:\/\/[a-z0-9]+(?::[a-z0-9]+)?@[a-z0-9.-]+(?:\.[a-z]+|:[0-9]+)\/\d+/i; + +type SweepArgs = { + sizes: readonly PresetName[]; + values: readonly number[]; + runs: number; + warmup: number; + json: boolean; + kneeThresholdPct: number; +}; + +// biome-ignore lint/complexity/noExcessiveCognitiveComplexity: CLI flag switch is inherently branchy +function parseArgs(argv: readonly string[]): SweepArgs { + const sizes: PresetName[] = [...DEFAULT_SIZES]; + let values: number[] = [...DEFAULT_VALUES]; + let runs = 5; + let warmup = 2; + let json = false; + const kneeThresholdPct = 0.03; // 3% + + for (let i = 0; i < argv.length; i += 1) { + const arg = argv[i] as string; + const next = argv[i + 1]; + switch (arg) { + case "--size": { + if (!next) { + throw new Error("--size requires a value"); + } + if (next === "all") { + sizes.length = 0; + sizes.push(...PRESET_NAMES); + } else if ((PRESET_NAMES as readonly string[]).includes(next)) { + sizes.length = 0; + sizes.push(next as PresetName); + } else { + throw new Error( + `Unknown size '${next}'. Valid: ${PRESET_NAMES.join(", ")}, all` + ); + } + i += 1; + break; + } + case "--values": { + if (!next) { + throw new Error("--values requires a comma-separated list"); + } + values = next + .split(",") + .map((s) => Number(s.trim())) + .filter((n) => Number.isFinite(n) && n > 0); + if (values.length === 0) { + throw new Error("--values must contain at least one positive number"); + } + i += 1; + break; + } + case "--runs": { + if (!next) { + throw new Error("--runs requires a number"); + } + runs = Number(next); + i += 1; + break; + } + case "--warmup": { + if (!next) { + throw new Error("--warmup requires a number"); + } + warmup = Number(next); + i += 1; + break; + } + case "--json": { + json = true; + break; + } + case "-h": + case "--help": { + printHelp(); + process.exit(0); + break; + } + default: + throw new Error(`Unknown flag: ${arg}`); + } + } + + return { sizes, values, runs, warmup, json, kneeThresholdPct }; +} + +function printHelp(): void { + console.log("Usage: bun run bench:sweep [--size small|medium|large|all]"); + console.log(" [--values 1,2,4,8,...]"); + console.log(" [--runs N] [--warmup N]"); + console.log(" [--json]"); +} + +/** Resolve (or create) a synthetic fixture, same as bench.ts. */ +function resolveFixture(name: PresetName): { label: string; rootDir: string } { + const preset = PRESETS[name]; + // biome-ignore lint/suspicious/noBitwiseOperators: deterministic seed mix + const seed = (0xde_ad_be_ef ^ hashStr(name)) >>> 0; + const spec: FixtureSpec = { + ...preset, + seed, + rootDir: "", + }; + const { rootDir: _unused, ...specNoRoot } = spec; + const hash = hashSpec(specNoRoot); + const rootDir = join(tmpdir(), "sentry-cli-bench", `fx-${name}-${hash}`); + mkdirSync(rootDir, { recursive: true }); + generateFixture({ ...spec, rootDir }); + return { label: `synthetic/${name}`, rootDir }; +} + +/** Cheap 32-bit FNV-1a over a short string — same as bench.ts. */ +// biome-ignore-start lint/suspicious/noBitwiseOperators: FNV-1a is a bitwise hash +function hashStr(s: string): number { + let h = 0x81_1c_9d_c5; + for (let i = 0; i < s.length; i += 1) { + h = Math.imul(h ^ s.charCodeAt(i), 0x01_00_01_93); + } + return h >>> 0; +} +// biome-ignore-end lint/suspicious/noBitwiseOperators: FNV-1a is a bitwise hash + +/** + * Single op × concurrency → p50 in ms. Returns NaN when the op + * throws or the fixture doesn't exist. + */ +type SweepResult = { + fixture: string; + operation: string; + concurrency: number; + p50: number; + p95: number; + runs: number; +}; + +/** + * Sweepable ops. Only ops that accept a `concurrency` override are + * included — the walker itself is sequential, so sweeping + * `scan.walk` would produce identical numbers across the grid. + * + * `scan.grepFiles` is the closest shape to `scanCodeForDsns` (walker + * + per-file read + regex). The knee we find here should transfer to + * the DSN scanner once we update `CONCURRENCY_LIMIT`. + */ +async function buildOps(): Promise< + Array<{ + label: string; + run: (cwd: string, concurrency: number) => Promise; + setup?: (cwd: string) => Promise; + }> +> { + const { collectGrep } = await import("../src/lib/scan/index.js"); + const { dsnScanOptions } = await import("../src/lib/dsn/scan-options.js"); + + return [ + { + label: "scan.grepFiles", + run: async (cwd, concurrency) => { + await collectGrep({ + cwd, + pattern: DSN_PATTERN, + ...dsnScanOptions(), + concurrency, + }); + }, + }, + ]; +} + +async function runSweep(args: SweepArgs): Promise { + const fixtures = args.sizes.map(resolveFixture); + const ops = await buildOps(); + const results: SweepResult[] = []; + + // Silence CLI telemetry — we don't want Sentry events from bench runs. + process.env.SENTRY_CLI_NO_TELEMETRY = "1"; + + for (const fx of fixtures) { + if (!existsSync(fx.rootDir)) { + if (!args.json) { + console.error(`✗ fixture missing: ${fx.rootDir}`); + } + continue; + } + if (!args.json) { + console.log(`\n${fx.label} (${fx.rootDir})`); + } + await withBenchDb(async () => { + await sweepFixture(fx, ops, args, results); + }); + } + + return results; +} + +/** Inner loop body extracted to keep `runSweep`'s arity + complexity low. */ +async function sweepFixture( + fx: { label: string; rootDir: string }, + ops: Awaited>, + args: SweepArgs, + results: SweepResult[] +): Promise { + for (const op of ops) { + for (const concurrency of args.values) { + const samples = await measure(() => op.run(fx.rootDir, concurrency), { + runs: args.runs, + warmup: args.warmup, + beforeEach: op.setup ? () => op.setup?.(fx.rootDir) : undefined, + }); + const stats = summarize(samples); + results.push({ + fixture: fx.label, + operation: op.label, + concurrency, + p50: stats.p50, + p95: stats.p95, + runs: stats.runs, + }); + if (!args.json) { + console.log( + ` ${op.label.padEnd(24)} conc=${String(concurrency).padStart(3)} p50 ${stats.p50.toFixed(1).padStart(6)}ms p95 ${stats.p95.toFixed(1).padStart(6)}ms` + ); + } + } + } +} + +/** + * Given sorted-by-concurrency results for one (fixture, op), return + * the smallest concurrency value past which increasing concurrency + * yields < `thresholdPct` improvement in p50. + */ +function findKnee( + entries: readonly SweepResult[], + thresholdPct: number +): number | null { + const sorted = [...entries].sort((a, b) => a.concurrency - b.concurrency); + let bestP50 = Number.POSITIVE_INFINITY; + let kneeAt: number | null = null; + for (const e of sorted) { + const improvementRatio = (bestP50 - e.p50) / bestP50; + if ( + !Number.isFinite(improvementRatio) || + improvementRatio >= thresholdPct + ) { + bestP50 = Math.min(bestP50, e.p50); + kneeAt = e.concurrency; + } else { + // Stop improving. Previous kneeAt is our answer. + break; + } + } + return kneeAt; +} + +/** Render the per-(fixture, op) knee table. */ +function printKnees( + results: readonly SweepResult[], + thresholdPct: number +): void { + const byKey = new Map(); + for (const r of results) { + const key = `${r.fixture}||${r.operation}`; + const list = byKey.get(key) ?? []; + list.push(r); + byKey.set(key, list); + } + console.log(""); + console.log( + `Knee analysis (smallest concurrency past which each additional step gains < ${(thresholdPct * 100).toFixed(0)}%)` + ); + console.log("─".repeat(72)); + for (const [key, entries] of byKey) { + const [fixture, operation] = key.split("||"); + const knee = findKnee(entries, thresholdPct); + const minP50 = Math.min(...entries.map((e) => e.p50)); + console.log( + ` ${String(fixture).padEnd(20)} ${String(operation).padEnd(24)} knee = ${knee ?? "?"} (best p50 ${minP50.toFixed(1)}ms)` + ); + } + console.log(""); +} + +async function main(): Promise { + const args = parseArgs(process.argv.slice(2)); + const results = await runSweep(args); + + if (args.json) { + const report = { + generatedAt: new Date().toISOString(), + runtime: { + platform: platform(), + arch: arch(), + cpus: cpus().length, + availableParallelism: availableParallelism(), + }, + kneeThresholdPct: args.kneeThresholdPct, + results, + }; + process.stdout.write(`${JSON.stringify(report, null, 2)}\n`); + } else { + console.log( + `\nsystem: ${platform()}/${arch()}, availableParallelism=${availableParallelism()}` + ); + printKnees(results, args.kneeThresholdPct); + } + + return 0; +} + +main() + .then((code) => process.exit(code)) + .catch((error) => { + console.error(error); + process.exit(1); + }); diff --git a/script/bench.ts b/script/bench.ts new file mode 100644 index 000000000..63fa2a4d6 --- /dev/null +++ b/script/bench.ts @@ -0,0 +1,650 @@ +#!/usr/bin/env bun +/** + * Local benchmark harness for DSN detection, project-root finding, and + * (after the `src/lib/scan/` refactor lands) the generic scanner module. + * + * Goals: + * 1. Capture objective baselines *before* the scanner refactor so we can + * verify that the new pure-TS implementation lands within ~1.2x. + * 2. Feed data-driven decisions for the worker-pool + caching follow-ups. + * 3. Use operation labels that match the Sentry spans already emitted in + * production (`findProjectRoot`, `scanCodeForDsns`, etc.) so local + * numbers correlate with prod telemetry. + * + * The harness is deliberately zero-dependency: it builds synthetic repos + * from `test/fixtures/bench/` (parameterized + deterministic) or, if you + * pass `--repo /path` or set `BENCH_REPO=`, benches against a real repo. + * Baselines go to `.bench/baseline.json` (gitignored) — they're machine- + * specific and intentionally not version-controlled. + * + * Usage: + * bun run bench # all ops, all preset sizes + * bun run bench --size small # only the 'small' preset + * bun run bench --op detectDsn.cold # filter by operation (substring) + * bun run bench --repo /path/to/repo # bench a real repo (disables --save-baseline) + * bun run bench --warmup 3 --runs 10 # override default run counts + * bun run bench --json > report.json # machine-readable stdout + * bun run bench --save-baseline # write .bench/baseline.json + * bun run bench --compare # diff current vs .bench/baseline.json + * # (exit 1 if any p50 regresses >20%) + * bun run bench --regen-fixtures # force fixture regeneration + * + * Environment variables: + * BENCH_REPO Path to a real repo (equivalent to --repo) + * BENCH_RUNS Default measured run count (default: 10) + * BENCH_WARMUP Default warmup run count (default: 3) + * BENCH_THRESHOLD Default regression threshold for --compare (default: 0.2) + * + * Exit codes: + * 0 - Bench completed; no regression on --compare + * 1 - Invalid args, or --compare detected a p50 regression over threshold + */ + +import { existsSync, mkdirSync, readFileSync, rmSync } from "node:fs"; +import { readdir } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { + type FixtureSpec, + generateFixture, + hashSpec, +} from "../test/fixtures/bench/generate.js"; +import { + type BenchEntry, + type BenchReport, + clearDsnDetectionCache, + compareReports, + measure, + printComparison, + printReport, + runtimeInfo, + summarize, + withBenchDb, + writeJsonReport, +} from "../test/fixtures/bench/helpers.js"; +import { + PRESET_NAMES, + PRESETS, + type PresetName, +} from "../test/fixtures/bench/presets.js"; + +/** + * The DSN scanner's hot regex. Pinned at module scope per Biome's + * `useTopLevelRegex` rule; reused by the `scan.grepFiles` op. + */ +const DSN_PATTERN = + /https?:\/\/[a-z0-9]+(?::[a-z0-9]+)?@[a-z0-9.-]+(?:\.[a-z]+|:[0-9]+)\/\d+/i; + +// -------- Arg parsing -------- + +type CliArgs = { + sizes: readonly PresetName[]; + opFilter: string | undefined; + repo: string | undefined; + runs: number; + warmup: number; + json: boolean; + saveBaseline: boolean; + compare: boolean; + regenFixtures: boolean; + thresholdPct: number; +}; + +type ParseState = { + sizes: PresetName[]; + opFilter: string | undefined; + repo: string | undefined; + runs: number; + warmup: number; + json: boolean; + saveBaseline: boolean; + compare: boolean; + regenFixtures: boolean; +}; + +/** Apply a single flag (and its optional value) to the mutable parse state. */ +function applyFlag( + state: ParseState, + arg: string, + next: string | undefined +): boolean { + switch (arg) { + case "--size": { + if (!next) { + throw new Error("--size requires a value"); + } + if (next === "all") { + state.sizes = [...PRESET_NAMES]; + } else if ((PRESET_NAMES as readonly string[]).includes(next)) { + state.sizes = [next as PresetName]; + } else { + throw new Error( + `Unknown size '${next}'. Valid: ${PRESET_NAMES.join(", ")}, all` + ); + } + return true; + } + case "--op": { + if (!next) { + throw new Error("--op requires a value"); + } + state.opFilter = next; + return true; + } + case "--repo": { + if (!next) { + throw new Error("--repo requires a path"); + } + state.repo = next; + return true; + } + case "--runs": { + if (!next) { + throw new Error("--runs requires a number"); + } + state.runs = Number(next); + return true; + } + case "--warmup": { + if (!next) { + throw new Error("--warmup requires a number"); + } + state.warmup = Number(next); + return true; + } + case "--json": + state.json = true; + return false; + case "--save-baseline": + state.saveBaseline = true; + return false; + case "--compare": + state.compare = true; + return false; + case "--regen-fixtures": + state.regenFixtures = true; + return false; + case "-h": { + printHelp(); + process.exit(0); + break; + } + case "--help": { + printHelp(); + process.exit(0); + break; + } + default: + throw new Error(`Unknown flag: ${arg}`); + } +} + +function parseArgs(argv: readonly string[]): CliArgs { + const state: ParseState = { + sizes: [...PRESET_NAMES], + opFilter: undefined, + repo: process.env.BENCH_REPO, + runs: Number(process.env.BENCH_RUNS ?? 10), + warmup: Number(process.env.BENCH_WARMUP ?? 3), + json: false, + saveBaseline: false, + compare: false, + regenFixtures: false, + }; + const thresholdPct = Number(process.env.BENCH_THRESHOLD ?? 0.2); + + for (let i = 0; i < argv.length; i += 1) { + const arg = argv[i] as string; + const next = argv[i + 1]; + if (applyFlag(state, arg, next)) { + i += 1; + } + } + + if (!Number.isFinite(state.runs) || state.runs < 1) { + throw new Error("--runs must be >= 1"); + } + if (!Number.isFinite(state.warmup) || state.warmup < 0) { + throw new Error("--warmup must be >= 0"); + } + if (state.saveBaseline && state.repo) { + throw new Error( + "--save-baseline is only valid with synthetic fixtures; remove --repo/BENCH_REPO" + ); + } + + return { ...state, thresholdPct }; +} + +function printHelp(): void { + console.log( + "Usage: bun run bench [--size small|medium|large|all] [--op NAME] [--repo PATH]" + ); + console.log(" [--runs N] [--warmup N]"); + console.log( + " [--json] [--save-baseline] [--compare] [--regen-fixtures]" + ); +} + +// -------- Fixture resolution -------- + +type FixtureHandle = { + label: string; + rootDir: string; + /** True when the fixture is an ephemeral synthetic tree that we may discard. */ + synthetic: boolean; + /** Reported file count for the header / JSON. */ + fileCount: number; +}; + +// biome-ignore-start lint/suspicious/noBitwiseOperators: FNV-1a is a bitwise hash +/** 32-bit FNV-1a hash of a string — supplies a stable per-preset seed. */ +function hashToSeed(s: string): number { + let h = 0x81_1c_9d_c5; + for (let i = 0; i < s.length; i += 1) { + h = Math.imul(h ^ s.charCodeAt(i), 0x01_00_01_93); + } + // Keep the result in the 32-bit signed range for determinism across engines. + return h >>> 0; +} +// biome-ignore-end lint/suspicious/noBitwiseOperators: FNV-1a is a bitwise hash + +/** Build (or reuse) a synthetic fixture for the given preset. */ +function resolveSyntheticFixture( + name: PresetName, + forceRegen: boolean +): FixtureHandle { + const preset = PRESETS[name]; + // Deterministic seed per preset so every contributor lands on the same tree. + // We XOR an anchor constant with the per-preset name hash so seeds are + // spread across the 32-bit space even when preset names are similar. + // biome-ignore lint/suspicious/noBitwiseOperators: deterministic 32-bit seed mix + const seed = (0xde_ad_be_ef ^ hashToSeed(name)) >>> 0; + const specNoRoot = { ...preset, seed }; + const hash = hashSpec(specNoRoot); + const rootDir = join(tmpdir(), "sentry-cli-bench", `fx-${name}-${hash}`); + mkdirSync(rootDir, { recursive: true }); + if (forceRegen) { + rmSync(rootDir, { recursive: true, force: true }); + mkdirSync(rootDir, { recursive: true }); + } + const spec: FixtureSpec = { ...preset, seed, rootDir }; + const meta = generateFixture(spec, { force: forceRegen }); + return { + label: `synthetic/${name}`, + rootDir, + synthetic: true, + fileCount: meta.fileCount, + }; +} + +/** Count files under a path with a lightweight walk (used for real-repo headers). */ +async function roughFileCount(root: string): Promise { + const skip = new Set([ + ".git", + "node_modules", + "dist", + "build", + ".venv", + "venv", + ]); + let count = 0; + async function walk(dir: string): Promise { + let entries: Awaited>; + try { + entries = await readdir(dir, { withFileTypes: true }); + } catch { + return; + } + for (const entry of entries) { + if (entry.isDirectory()) { + if (!skip.has(entry.name)) { + await walk(join(dir, entry.name)); + } + } else if (entry.isFile()) { + count += 1; + } + } + } + await walk(root); + return count; +} + +// -------- Operation registry -------- + +type OpRunner = (cwd: string) => Promise; +type OpEntry = { + label: string; + warm: boolean; + /** Called before every measured iteration — use for cold-cache resets. */ + setup?: (cwd: string) => Promise; + run: OpRunner; +}; + +async function buildOps(): Promise { + // Lazy-import production code so unit tests can import the helpers/fixtures + // without loading all of @sentry/node-core. + const { detectDsn, detectAllDsns } = await import( + "../src/lib/dsn/detector.js" + ); + const { findProjectRoot } = await import("../src/lib/dsn/project-root.js"); + const { scanCodeForDsns, scanCodeForFirstDsn } = await import( + "../src/lib/dsn/code-scanner.js" + ); + // Scan module — not yet wired into DSN detection (PR 3 will do that). + // These ops give us standalone baselines so PR 2/PR 3 can compare. + const { walkFiles, IgnoreStack, TEXT_EXTENSIONS, collectGrep } = await import( + "../src/lib/scan/index.js" + ); + // DSN-parity preset — used by the `scan.walk.dsnParity` op below. + const { dsnScanOptions } = await import("../src/lib/dsn/scan-options.js"); + + const coldSetup = (cwd: string) => clearDsnDetectionCache(cwd); + + return [ + { + label: "findProjectRoot", + warm: false, + setup: coldSetup, + run: async (cwd) => { + await findProjectRoot(cwd); + }, + }, + { + label: "detectDsn.cold", + warm: false, + setup: coldSetup, + run: async (cwd) => { + await detectDsn(cwd); + }, + }, + { + label: "detectDsn.warm", + warm: true, + // No setup hook — leverage whatever the previous run cached. + run: async (cwd) => { + await detectDsn(cwd); + }, + }, + { + label: "detectAllDsns.cold", + warm: false, + setup: coldSetup, + run: async (cwd) => { + await detectAllDsns(cwd); + }, + }, + { + label: "detectAllDsns.warm", + warm: true, + run: async (cwd) => { + await detectAllDsns(cwd); + }, + }, + { + label: "scanCodeForDsns", + warm: false, + // scanCodeForDsns bypasses the cache entirely — no setup needed, but we + // still clear so any sibling-test leftover doesn't skew timings. + setup: coldSetup, + run: async (cwd) => { + await scanCodeForDsns(cwd); + }, + }, + { + label: "scanCodeForFirstDsn", + warm: false, + setup: coldSetup, + run: async (cwd) => { + await scanCodeForFirstDsn(cwd); + }, + }, + { + // scan.walk — iterate the walker with the DSN scanner's extension + // allowlist. This is the closest standalone comparison point with + // `scanCodeForDsns` (same file set, just no regex work). + label: "scan.walk", + warm: false, + run: async (cwd) => { + // We intentionally discard the entries — the bench measures + // the cost of iterating the generator, not anything downstream. + for await (const _ of walkFiles({ + cwd, + extensions: TEXT_EXTENSIONS, + })) { + // body intentionally empty + } + }, + }, + { + // scan.walk.noExt — no extension filter, so every unknown-extension + // file is opened and sniffed for NUL. Tells us how expensive lazy + // binary detection is on mixed trees. + label: "scan.walk.noExt", + warm: false, + run: async (cwd) => { + for await (const _ of walkFiles({ cwd })) { + // body intentionally empty + } + }, + }, + { + // scan.walk.dsnParity — walker configured with the DSN scanner's + // exact options (TEXT_EXTENSIONS + full skip list + depth 3 with + // monorepo reset). This is the apples-to-apples comparison with + // `scanCodeForDsns`; the success bar for PR 1.5 is p50 ≤ 1.2x. + label: "scan.walk.dsnParity", + warm: false, + run: async (cwd) => { + for await (const _ of walkFiles({ cwd, ...dsnScanOptions() })) { + // body intentionally empty + } + }, + }, + { + // scan.grepFiles — walker + regex pass using the same DSN + // preset. Adds the per-file `readFile` + line-by-line + // `regex.test` cost on top of `scan.walk.dsnParity` so PR 3 + // has a direct apples-to-apples comparison with + // `scanCodeForDsns` (which does the same work). + label: "scan.grepFiles", + warm: false, + run: async (cwd) => { + await collectGrep({ + cwd, + pattern: DSN_PATTERN, + ...dsnScanOptions(), + }); + }, + }, + { + // scan.ignore — micro-benchmark for IgnoreStack.isIgnored(). We + // build a stack once then hit it 10k times with synthetic paths + // so the reported timing is dominated by the query itself, not + // tree walking. + label: "scan.ignore", + warm: false, + run: async (cwd) => { + const stack = await IgnoreStack.create({ + cwd, + alwaysSkipDirs: ["node_modules", ".git", "dist", "build"], + respectGitignore: true, + includeGitInfoExclude: true, + }); + const queries = [ + "src/index.ts", + "node_modules/foo/bar.js", + "packages/pkg/src/deep/file.tsx", + "dist/bundle.js", + "build/out.css", + "test/fixtures/secret.env", + "README.md", + ".git/HEAD", + ]; + for (let i = 0; i < 10_000; i += 1) { + const q = queries[i % queries.length] as string; + stack.isIgnored(q, false); + } + }, + }, + ]; +} + +// -------- Main -------- + +/** Resolve the fixture list for this invocation (synthetic or real repo). */ +async function resolveFixtures(args: CliArgs): Promise { + if (args.repo) { + const abs = resolve(args.repo); + if (!existsSync(abs)) { + throw new Error(`--repo ${abs} does not exist`); + } + return [ + { + label: `real:${abs}`, + rootDir: abs, + synthetic: false, + fileCount: await roughFileCount(abs), + }, + ]; + } + return args.sizes.map((size) => + resolveSyntheticFixture(size, args.regenFixtures) + ); +} + +/** Filter the available ops by substring. Throws when nothing matches. */ +function filterOps(ops: OpEntry[], opFilter: string | undefined): OpEntry[] { + if (!opFilter) { + return ops; + } + const filtered = ops.filter((op) => op.label.includes(opFilter)); + if (filtered.length === 0) { + throw new Error( + `--op ${opFilter} matched no operations.\n Available: ${ops.map((o) => o.label).join(", ")}` + ); + } + return filtered; +} + +/** Run every op on every fixture and return the flattened entry list. */ +async function runAll( + fixtures: readonly FixtureHandle[], + ops: readonly OpEntry[], + args: CliArgs +): Promise { + const entries: BenchEntry[] = []; + for (const fx of fixtures) { + if (!args.json) { + console.log(`${fx.label} (${fx.fileCount} files @ ${fx.rootDir})`); + } + await withBenchDb(async () => { + for (const op of ops) { + const samples = await measure(() => op.run(fx.rootDir), { + runs: args.runs, + warmup: args.warmup, + beforeEach: op.setup ? () => op.setup?.(fx.rootDir) : undefined, + }); + const stats = summarize(samples); + entries.push({ + fixture: fx.label, + operation: op.label, + warm: op.warm, + stats, + }); + if (!args.json) { + console.log( + ` ${op.label.padEnd(24)} p50 ${stats.p50.toFixed(2)}ms p95 ${stats.p95.toFixed(2)}ms (${stats.runs} runs)` + ); + } + } + }); + } + return entries; +} + +/** Perform the --compare step. Returns false on regression. */ +function compareAgainstBaseline( + report: BenchReport, + thresholdPct: number +): boolean { + const baselinePath = ".bench/baseline.json"; + if (!existsSync(baselinePath)) { + console.error( + `✗ No baseline found at ${baselinePath}. Run with --save-baseline first.` + ); + return false; + } + const baseline = JSON.parse( + readFileSync(baselinePath, "utf8") + ) as BenchReport; + const rows = compareReports(baseline, report, thresholdPct); + const ok = printComparison(rows, thresholdPct); + if (!ok) { + console.error("✗ One or more operations regressed beyond threshold"); + return false; + } + console.log("✓ No regressions beyond threshold"); + return true; +} + +async function main(): Promise { + let args: CliArgs; + try { + args = parseArgs(process.argv.slice(2)); + } catch (error) { + console.error(`✗ ${(error as Error).message}`); + printHelp(); + return 1; + } + + // Silence CLI telemetry — we don't want bench runs filing Sentry events. + process.env.SENTRY_CLI_NO_TELEMETRY = "1"; + + let fixtures: FixtureHandle[]; + let ops: OpEntry[]; + try { + fixtures = await resolveFixtures(args); + const allOps = await buildOps(); + ops = filterOps(allOps, args.opFilter); + } catch (error) { + console.error(`✗ ${(error as Error).message}`); + return 1; + } + + const entries = await runAll(fixtures, ops, args); + + const report: BenchReport = { + version: 1, + generatedAt: new Date().toISOString(), + runtime: runtimeInfo(), + entries, + }; + + if (args.json) { + process.stdout.write(`${JSON.stringify(report, null, 2)}\n`); + } else { + printReport(report); + } + + if (args.saveBaseline) { + mkdirSync(".bench", { recursive: true }); + await writeJsonReport(report, ".bench/baseline.json"); + if (!args.json) { + console.log("✓ Baseline written to .bench/baseline.json"); + } + } + + if (args.compare && !compareAgainstBaseline(report, args.thresholdPct)) { + return 1; + } + + return 0; +} + +main() + .then((code) => process.exit(code)) + .catch((error) => { + console.error(error); + process.exit(1); + }); diff --git a/src/lib/dsn/code-scanner.ts b/src/lib/dsn/code-scanner.ts index 446765163..24b6f8516 100644 --- a/src/lib/dsn/code-scanner.ts +++ b/src/lib/dsn/code-scanner.ts @@ -1,210 +1,81 @@ /** - * Language-Agnostic Code Scanner + * Language-Agnostic DSN Code Scanner (policy layer). * - * Scans source code for Sentry DSNs using a simple grep-based approach. - * This replaces the language-specific detectors with a unified scanner that: + * This module owns the DSN-specific policy (URL regex, comment-line + * filtering, host validation, package-path inference, stop-on-first + * semantics). All file walking, `.gitignore` handling, extension + * filtering, and bounded concurrency are delegated to the shared + * `src/lib/scan/` module. * - * 1. Greps for DSN URL pattern directly: https://KEY@HOST/PROJECT_ID - * 2. Filters out DSNs appearing in commented lines - * 3. Respects .gitignore using the `ignore` package - * 4. Validates DSN hosts (SaaS when no SENTRY_URL, or self-hosted host when set) - * 5. Scans concurrently with p-limit for performance - * 6. Skips large files and known non-source directories + * Flow: + * 1. `scanDirectory(cwd, stopOnFirst)` calls `walkFiles` with the + * DSN preset (`dsnScanOptions()`), passing `recordMtimes` and an + * `onDirectoryVisit` hook so the cache-invalidation map is + * populated in one traversal. + * 2. Each yielded file is read + passed through `extractDsnsFromContent` + * via `mapFilesConcurrent`. Per-file `ConfigError` re-throws up + * to the caller; all other errors are logged at debug level and + * the file is skipped. + * 3. `onResult` in `mapFilesConcurrent` dedups into a shared Map + * and raises the early-exit flag on first unique DSN when + * `stopOnFirst: true`. + * + * Behavior change landed in PR 3: the walker's `nestedGitignore: true` + * default (via `dsnScanOptions()`) means nested `.gitignore` files are + * now honored. Pre-PR-3 code only read the project-root `.gitignore`. + * This is a correctness improvement matching git's cumulative semantics; + * DSNs in files covered by a subdir `.gitignore` are no longer detected. */ -import type { Dirent } from "node:fs"; -import { readdir, stat } from "node:fs/promises"; import path from "node:path"; -import ignore, { type Ignore } from "ignore"; -import pLimit from "p-limit"; import { DEFAULT_SENTRY_HOST, getConfiguredSentryUrl } from "../constants.js"; import { ConfigError } from "../errors.js"; import { logger } from "../logger.js"; +import { + mapFilesConcurrent, + normalizePath, + type WalkEntry, + walkFiles, +} from "../scan/index.js"; import { withTracingSpan } from "../telemetry.js"; import { createDetectedDsn, inferPackagePath, parseDsn } from "./parser.js"; +import { DSN_MAX_DEPTH, dsnScanOptions } from "./scan-options.js"; import type { DetectedDsn } from "./types.js"; -import { MONOREPO_ROOTS } from "./types.js"; /** Scoped logger for DSN code scanning */ const log = logger.withTag("dsn-scan"); /** * Result of scanning code for DSNs, including mtimes for caching. + * + * Shape is stable — `src/lib/db/dsn-cache.ts` stores this via + * `setCachedDetection` and verifies `sourceMtimes` / `dirMtimes` + * against the filesystem. Do NOT change keys/values without also + * bumping the cache schema. */ export type CodeScanResult = { /** All detected DSNs */ dsns: DetectedDsn[]; - /** Map of source file paths to their mtimes (only files containing DSNs) */ + /** + * Map of source file paths (POSIX, relative to cwd) to their mtimes. + * Only files that contained at least one DSN are present — the cache + * verifier uses this to detect "source file touched since last scan". + */ sourceMtimes: Record; - /** Mtimes of scanned directories (for detecting new files added to subdirs) */ + /** + * Map of scanned directories (POSIX, relative to cwd; `.` for the + * root) to their floored `stat.mtimeMs`. The verifier uses this to + * detect "files added to a scanned dir since last scan". + */ dirMtimes: Record; }; -/** - * Maximum file size to scan (256KB). - * Files larger than this are skipped as they're unlikely to be source files - * with DSN configuration. - * - * Note: This check happens during file processing rather than collection to - * avoid extra stat() calls. Bun.file().size is a cheap operation once we - * have the file handle. - */ -const MAX_FILE_SIZE = 256 * 1024; - -/** - * Concurrency limit for file reads. - * Balances performance with file descriptor limits. - */ -const CONCURRENCY_LIMIT = 50; - -/** - * Maximum depth to scan from project root. - * Depth 0 = files in root directory - * Depth 3 = files in third-level subdirectories (e.g., src/lib/config/sentry.ts) - * - * In monorepos, depth resets to 0 when entering a package directory - * (e.g., packages/spotlight/), giving each package its own depth budget. - */ -const MAX_SCAN_DEPTH = 3; - -/** - * Directories that are always skipped regardless of .gitignore. - * These are common dependency/build/cache directories that should never contain DSNs. - * Added to the gitignore instance as built-in patterns. - */ -const ALWAYS_SKIP_DIRS = [ - // Version control - ".git", - ".hg", - ".svn", - // IDE/Editor - ".idea", - ".vscode", - ".cursor", - // Node.js - "node_modules", - // Test directories (contain fixture DSNs, not real configuration) - "test", - "tests", - "__mocks__", - "fixtures", - "__fixtures__", - // Python - "__pycache__", - ".pytest_cache", - ".mypy_cache", - ".ruff_cache", - "venv", - ".venv", - // Java/Kotlin/Gradle - "build", - "target", - ".gradle", - // Go - "vendor", - // Ruby - ".bundle", - // General build outputs - "dist", - "out", - ".next", - ".nuxt", - ".output", - "coverage", -]; - -/** - * File extensions to scan for DSNs. - * Covers source code, config files, and data formats that might contain DSNs. - */ -const TEXT_EXTENSIONS = new Set([ - // JavaScript/TypeScript ecosystem - ".ts", - ".tsx", - ".js", - ".jsx", - ".mjs", - ".cjs", - ".astro", - ".vue", - ".svelte", - // Python - ".py", - // Go - ".go", - // Ruby - ".rb", - ".erb", - // PHP - ".php", - // JVM languages - ".java", - ".kt", - ".kts", - ".scala", - ".groovy", - // .NET languages - ".cs", - ".fs", - ".vb", - // Rust - ".rs", - // Swift/Objective-C - ".swift", - ".m", - ".mm", - // Dart/Flutter - ".dart", - // Elixir/Erlang - ".ex", - ".exs", - ".erl", - // Lua - ".lua", - // Config/data formats - ".json", - ".yaml", - ".yml", - ".toml", - ".xml", - ".properties", - ".config", -]); - /** * Common comment prefixes to detect commented-out DSNs. * Lines starting with these (after trimming whitespace) are ignored. */ const COMMENT_PREFIXES = ["//", "#", "--", "