diff --git a/README.md b/README.md index 14a747b8..c5913e34 100644 --- a/README.md +++ b/README.md @@ -459,6 +459,7 @@ For the full capability matrix, limitations, and fixture coverage, see [docs/lan - [docs/agent-workflows.md](./docs/agent-workflows.md): explore, orientation packets, search anchors, MCP, sessions, streaming, tool wrappers, review bundles, and agent-oriented review recipes - [docs/mcp.md](./docs/mcp.md): MCP server setup, tool list, safety model, and client configuration examples - [docs/how-it-works.md](./docs/how-it-works.md): performance, caching, native runtime behavior, architecture, and testing guidance +- [docs/benchmarks/README.md](./docs/benchmarks/README.md): reproducible local benchmark methodology, checked evidence, metric semantics, and limitations - [docs/language-parity.md](./docs/language-parity.md): per-language capability matrix - [docs/scenario-catalog.md](./docs/scenario-catalog.md): scenario and fixture coverage - [docs/coverage/README.md](./docs/coverage/README.md): committed Markdown coverage summaries diff --git a/docs/benchmarks/README.md b/docs/benchmarks/README.md new file mode 100644 index 00000000..c8221099 --- /dev/null +++ b/docs/benchmarks/README.md @@ -0,0 +1,71 @@ +# Documentation benchmarks + +## What the checked results show + +On these tiny local fixtures, with Codegraph caching disabled and a fresh process for each sample: + +- Direct reads are far faster: their median wall times are tens of milliseconds, while the Codegraph runs take several seconds. +- Codegraph reduces each declared workflow from three calls to one `explore` call. +- Every expected evidence anchor is present in every checked run. + +These findings describe only the declared workflows and checked fixtures below. Completeness means that expected path anchors appear in captured evidence. It does not measure answer quality, reasoning, relevance, correctness, or whether an agent could produce a good final answer. The benchmark also does not show that either workflow is generally faster, cheaper, or better at repository discovery. + +## Reproduce locally + +From the repository root, run: + +```bash +npm run bench:docs +``` + +This command rebuilds stale `dist` output when needed, runs every scenario and both variants serially, requires complete anchor evidence, writes `.tmp/public-docs-benchmark-results.json`, and prints the median table. The fixtures are local and the run makes no network requests. + +## Workflows and metrics + +Each checked scenario in [`scenarios.json`](./scenarios.json) defines a local fixture, a task, ordered expected anchors, and the exact steps for both variants. The scenarios cover TypeScript request paths, Python imports, SQL migration and application coupling, and Markdown-to-TypeScript request paths. + +- **Baseline workflow:** three declared direct UTF-8 file reads. The files are selected in advance, so this is not an unaided discovery task. +- **Codegraph workflow:** one local `codegraph explore --root --cache off --json` call. It starts a fresh CLI process and builds a cold in-process index for every sample. +- **Tool calls:** declared workflow steps. A direct read and an `explore` call both count as one, despite doing unequal work. +- **File reads:** baseline read steps, or unique source paths Codegraph returns in `packets` or `fileView`. This is a context-delivery count, not total parser, indexer, operating-system, or disk I/O. +- **Wall time:** elapsed time around declared steps. It includes Codegraph process startup and cold indexing but excludes harness setup. +- **Completeness:** the fraction of expected path anchors found as text in captured evidence. It is evidence-anchor presence, not an answer-quality score. + +Medians are calculated independently for each scenario and variant. Tool calls and returned files can approximate workflow coordination and delivered context, but they are not units of time, tokens, bytes, or effort. + +## Checked results + +[`results.example.json`](./results.example.json) contains the checked runs behind this table. A SHA-256 digest binds it to the exact ordered scenario definitions. Validation requires every selected scenario, both variants, and every expected run exactly once; it also checks declared step counts. The summarizer generates and orders every table cell, and `npm run bench:docs:check` detects drift. + + + +| Scenario | Variant | Samples | Median tool calls | Median file reads | Median wall time (ms) | Complete runs | Minimum completeness | +| -------------------------------- | --------- | ------: | ----------------: | ----------------: | --------------------: | ------------: | -------------------: | +| repo-orientation-small-ts | baseline | 3 | 3 | 3 | 60.458 | 3 | 100% | +| repo-orientation-small-ts | codegraph | 3 | 1 | 3 | 3762.148 | 3 | 100% | +| python-import-reference | baseline | 3 | 3 | 3 | 27.293 | 3 | 100% | +| python-import-reference | codegraph | 3 | 1 | 2 | 3620.385 | 3 | 100% | +| sql-migration-application-review | baseline | 3 | 3 | 3 | 39.783 | 3 | 100% | +| sql-migration-application-review | codegraph | 3 | 1 | 3 | 3569.998 | 3 | 100% | +| mixed-docs-source-graph | baseline | 3 | 3 | 3 | 39.589 | 3 | 100% | +| mixed-docs-source-graph | codegraph | 3 | 1 | 3 | 3437.486 | 3 | 100% | + + + +The checked result document records Node, platform, architecture, CPU, logical CPU count, and memory so reruns can be compared in context. + +## Limitations and variability + +- These fixtures are tiny, local, synthetic, and network-free. They do not represent large repositories, remote tools, warm indexes, long sessions, concurrent agents, or ambiguous discovery tasks. +- The baseline reads preselected files, while Codegraph starts a process, parses, indexes, and returns structured evidence. The workflows do not perform equivalent internal work. +- Codegraph runs with `--cache off`; the harness does not clear operating-system file caches or reboot the host. Node version, hardware, storage, memory pressure, antivirus, scheduling, build freshness, and system load can change wall times. +- Three samples per variant are a modest evidence set. Treat small timing differences cautiously and compare reruns using the recorded environment. +- The benchmark does not measure answer quality, tokens, output size, or human effort. Complete anchors can still accompany misleading evidence or a wrong answer. +- Fixture trees, scenario files, and output parents are trusted local maintainer inputs. Traversal and symlink checks prevent common mistakes, but the harness is not an adversarial sandbox; do not rename or retarget these paths during a run. + +Use the table only for claims directly supported by its checked fixtures and rows, not for broad claims about speed, quality, scale, or universal performance. + +## Related documentation + +- [How it works](../how-it-works.md) explains runtime performance and cache behavior. +- [Agent workflows](../agent-workflows.md) describes the agent-facing `explore` workflow measured here. diff --git a/docs/benchmarks/results.example.json b/docs/benchmarks/results.example.json new file mode 100644 index 00000000..28d613ba --- /dev/null +++ b/docs/benchmarks/results.example.json @@ -0,0 +1,418 @@ +{ + "schemaVersion": 1, + "generatedAt": "2026-07-10T17:51:31.958Z", + "command": [ + "node", + "scripts/benchmarks/run-scenario.mjs", + "--scenario-file", + "docs/benchmarks/scenarios.json", + "--runs", + "3", + "--output", + ".tmp/public-docs-benchmark-results.json", + "--require-complete" + ], + "environment": { + "nodeVersion": "v24.15.0", + "platform": "linux", + "arch": "x64", + "cpuModel": "Intel(R) Core(TM) i9-14900K", + "logicalCpus": 32, + "totalMemoryBytes": 67280605184 + }, + "scenarioFile": "docs/benchmarks/scenarios.json", + "scenarioDigest": "sha256:7d1d6583adf2f3cbaf904cae40837c8e25ca1d540cd1b5a28e444d3fc4db67fa", + "scenarioIds": [ + "repo-orientation-small-ts", + "python-import-reference", + "sql-migration-application-review", + "mixed-docs-source-graph" + ], + "runsPerVariant": 3, + "runs": [ + { + "scenarioId": "repo-orientation-small-ts", + "variant": "baseline", + "run": 1, + "metrics": { + "toolCalls": 3, + "fileReads": 3, + "wallTimeMs": 88.527 + }, + "checks": { + "anchorsExpected": 3, + "anchorsFound": 3, + "missingAnchors": [], + "completeness": 1 + } + }, + { + "scenarioId": "repo-orientation-small-ts", + "variant": "baseline", + "run": 2, + "metrics": { + "toolCalls": 3, + "fileReads": 3, + "wallTimeMs": 60.458 + }, + "checks": { + "anchorsExpected": 3, + "anchorsFound": 3, + "missingAnchors": [], + "completeness": 1 + } + }, + { + "scenarioId": "repo-orientation-small-ts", + "variant": "baseline", + "run": 3, + "metrics": { + "toolCalls": 3, + "fileReads": 3, + "wallTimeMs": 44.596 + }, + "checks": { + "anchorsExpected": 3, + "anchorsFound": 3, + "missingAnchors": [], + "completeness": 1 + } + }, + { + "scenarioId": "repo-orientation-small-ts", + "variant": "codegraph", + "run": 1, + "metrics": { + "toolCalls": 1, + "fileReads": 3, + "wallTimeMs": 3725.869 + }, + "checks": { + "anchorsExpected": 3, + "anchorsFound": 3, + "missingAnchors": [], + "completeness": 1 + } + }, + { + "scenarioId": "repo-orientation-small-ts", + "variant": "codegraph", + "run": 2, + "metrics": { + "toolCalls": 1, + "fileReads": 3, + "wallTimeMs": 3762.148 + }, + "checks": { + "anchorsExpected": 3, + "anchorsFound": 3, + "missingAnchors": [], + "completeness": 1 + } + }, + { + "scenarioId": "repo-orientation-small-ts", + "variant": "codegraph", + "run": 3, + "metrics": { + "toolCalls": 1, + "fileReads": 3, + "wallTimeMs": 3768.807 + }, + "checks": { + "anchorsExpected": 3, + "anchorsFound": 3, + "missingAnchors": [], + "completeness": 1 + } + }, + { + "scenarioId": "python-import-reference", + "variant": "baseline", + "run": 1, + "metrics": { + "toolCalls": 3, + "fileReads": 3, + "wallTimeMs": 28.868 + }, + "checks": { + "anchorsExpected": 3, + "anchorsFound": 3, + "missingAnchors": [], + "completeness": 1 + } + }, + { + "scenarioId": "python-import-reference", + "variant": "baseline", + "run": 2, + "metrics": { + "toolCalls": 3, + "fileReads": 3, + "wallTimeMs": 27.293 + }, + "checks": { + "anchorsExpected": 3, + "anchorsFound": 3, + "missingAnchors": [], + "completeness": 1 + } + }, + { + "scenarioId": "python-import-reference", + "variant": "baseline", + "run": 3, + "metrics": { + "toolCalls": 3, + "fileReads": 3, + "wallTimeMs": 21.963 + }, + "checks": { + "anchorsExpected": 3, + "anchorsFound": 3, + "missingAnchors": [], + "completeness": 1 + } + }, + { + "scenarioId": "python-import-reference", + "variant": "codegraph", + "run": 1, + "metrics": { + "toolCalls": 1, + "fileReads": 2, + "wallTimeMs": 3474.61 + }, + "checks": { + "anchorsExpected": 3, + "anchorsFound": 3, + "missingAnchors": [], + "completeness": 1 + } + }, + { + "scenarioId": "python-import-reference", + "variant": "codegraph", + "run": 2, + "metrics": { + "toolCalls": 1, + "fileReads": 2, + "wallTimeMs": 3620.385 + }, + "checks": { + "anchorsExpected": 3, + "anchorsFound": 3, + "missingAnchors": [], + "completeness": 1 + } + }, + { + "scenarioId": "python-import-reference", + "variant": "codegraph", + "run": 3, + "metrics": { + "toolCalls": 1, + "fileReads": 2, + "wallTimeMs": 3934.762 + }, + "checks": { + "anchorsExpected": 3, + "anchorsFound": 3, + "missingAnchors": [], + "completeness": 1 + } + }, + { + "scenarioId": "sql-migration-application-review", + "variant": "baseline", + "run": 1, + "metrics": { + "toolCalls": 3, + "fileReads": 3, + "wallTimeMs": 37.275 + }, + "checks": { + "anchorsExpected": 3, + "anchorsFound": 3, + "missingAnchors": [], + "completeness": 1 + } + }, + { + "scenarioId": "sql-migration-application-review", + "variant": "baseline", + "run": 2, + "metrics": { + "toolCalls": 3, + "fileReads": 3, + "wallTimeMs": 40.696 + }, + "checks": { + "anchorsExpected": 3, + "anchorsFound": 3, + "missingAnchors": [], + "completeness": 1 + } + }, + { + "scenarioId": "sql-migration-application-review", + "variant": "baseline", + "run": 3, + "metrics": { + "toolCalls": 3, + "fileReads": 3, + "wallTimeMs": 39.783 + }, + "checks": { + "anchorsExpected": 3, + "anchorsFound": 3, + "missingAnchors": [], + "completeness": 1 + } + }, + { + "scenarioId": "sql-migration-application-review", + "variant": "codegraph", + "run": 1, + "metrics": { + "toolCalls": 1, + "fileReads": 3, + "wallTimeMs": 3569.998 + }, + "checks": { + "anchorsExpected": 3, + "anchorsFound": 3, + "missingAnchors": [], + "completeness": 1 + } + }, + { + "scenarioId": "sql-migration-application-review", + "variant": "codegraph", + "run": 2, + "metrics": { + "toolCalls": 1, + "fileReads": 3, + "wallTimeMs": 3544.761 + }, + "checks": { + "anchorsExpected": 3, + "anchorsFound": 3, + "missingAnchors": [], + "completeness": 1 + } + }, + { + "scenarioId": "sql-migration-application-review", + "variant": "codegraph", + "run": 3, + "metrics": { + "toolCalls": 1, + "fileReads": 3, + "wallTimeMs": 3721.044 + }, + "checks": { + "anchorsExpected": 3, + "anchorsFound": 3, + "missingAnchors": [], + "completeness": 1 + } + }, + { + "scenarioId": "mixed-docs-source-graph", + "variant": "baseline", + "run": 1, + "metrics": { + "toolCalls": 3, + "fileReads": 3, + "wallTimeMs": 39.589 + }, + "checks": { + "anchorsExpected": 3, + "anchorsFound": 3, + "missingAnchors": [], + "completeness": 1 + } + }, + { + "scenarioId": "mixed-docs-source-graph", + "variant": "baseline", + "run": 2, + "metrics": { + "toolCalls": 3, + "fileReads": 3, + "wallTimeMs": 40.26 + }, + "checks": { + "anchorsExpected": 3, + "anchorsFound": 3, + "missingAnchors": [], + "completeness": 1 + } + }, + { + "scenarioId": "mixed-docs-source-graph", + "variant": "baseline", + "run": 3, + "metrics": { + "toolCalls": 3, + "fileReads": 3, + "wallTimeMs": 39.422 + }, + "checks": { + "anchorsExpected": 3, + "anchorsFound": 3, + "missingAnchors": [], + "completeness": 1 + } + }, + { + "scenarioId": "mixed-docs-source-graph", + "variant": "codegraph", + "run": 1, + "metrics": { + "toolCalls": 1, + "fileReads": 3, + "wallTimeMs": 3946.935 + }, + "checks": { + "anchorsExpected": 3, + "anchorsFound": 3, + "missingAnchors": [], + "completeness": 1 + } + }, + { + "scenarioId": "mixed-docs-source-graph", + "variant": "codegraph", + "run": 2, + "metrics": { + "toolCalls": 1, + "fileReads": 3, + "wallTimeMs": 3437.486 + }, + "checks": { + "anchorsExpected": 3, + "anchorsFound": 3, + "missingAnchors": [], + "completeness": 1 + } + }, + { + "scenarioId": "mixed-docs-source-graph", + "variant": "codegraph", + "run": 3, + "metrics": { + "toolCalls": 1, + "fileReads": 3, + "wallTimeMs": 3410.832 + }, + "checks": { + "anchorsExpected": 3, + "anchorsFound": 3, + "missingAnchors": [], + "completeness": 1 + } + } + ] +} diff --git a/docs/benchmarks/scenarios.json b/docs/benchmarks/scenarios.json new file mode 100644 index 00000000..e1f862bb --- /dev/null +++ b/docs/benchmarks/scenarios.json @@ -0,0 +1,125 @@ +{ + "schemaVersion": 1, + "scenarios": [ + { + "id": "repo-orientation-small-ts", + "repo": "tests/samples/benchmarks/typescript-service", + "task": "Trace how serveRequest sends an incoming request through dispatchRequest to the getHealth handler, and identify each file in the request path.", + "expectedAnchors": ["src/server.ts", "src/routes.ts", "src/handlers/health.ts"], + "metrics": ["toolCalls", "fileReads", "wallTimeMs"], + "variants": { + "baseline": [ + { + "type": "read", + "path": "src/server.ts" + }, + { + "type": "read", + "path": "src/routes.ts" + }, + { + "type": "read", + "path": "src/handlers/health.ts" + } + ], + "codegraph": [ + { + "type": "codegraph", + "command": "explore", + "query": "Trace serveRequest from src/server.ts through dispatchRequest in src/routes.ts to getHealth in src/handlers/health.ts." + } + ] + } + }, + { + "id": "python-import-reference", + "repo": "tests/samples/python", + "task": "Trace the local imports used by main.py, including helper_function, UtilityClass, and another_helper, to the modules that define them.", + "expectedAnchors": ["main.py", "utils.py", "helpers.py"], + "metrics": ["toolCalls", "fileReads", "wallTimeMs"], + "variants": { + "baseline": [ + { + "type": "read", + "path": "main.py" + }, + { + "type": "read", + "path": "utils.py" + }, + { + "type": "read", + "path": "helpers.py" + } + ], + "codegraph": [ + { + "type": "codegraph", + "command": "explore", + "query": "Trace the imports in main.py for helper_function, UtilityClass, and another_helper to their definitions in utils.py and helpers.py." + } + ] + } + }, + { + "id": "sql-migration-application-review", + "repo": "tests/samples/benchmarks/sql-application-review", + "task": "Locate the coupling between the accounts schema, the migration that introduces status, and the TypeScript query that reads active accounts; do not assume a persistent source-to-SQL graph edge.", + "expectedAnchors": ["schema.sql", "migrations/001_add_account_status.sql", "src/list-active-accounts.ts"], + "metrics": ["toolCalls", "fileReads", "wallTimeMs"], + "variants": { + "baseline": [ + { + "type": "read", + "path": "schema.sql" + }, + { + "type": "read", + "path": "migrations/001_add_account_status.sql" + }, + { + "type": "read", + "path": "src/list-active-accounts.ts" + } + ], + "codegraph": [ + { + "type": "codegraph", + "command": "explore", + "query": "Locate how the accounts definition in schema.sql, the status addition in migrations/001_add_account_status.sql, and the active-account query in src/list-active-accounts.ts are coupled by table and column usage." + } + ] + } + }, + { + "id": "mixed-docs-source-graph", + "repo": "tests/samples/benchmarks/mixed-docs-source", + "task": "Follow the local request-flow guide into sendRequest, then trace the source import to formatRequest.", + "expectedAnchors": ["docs/request-flow.md", "src/client.ts", "src/transport.ts"], + "metrics": ["toolCalls", "fileReads", "wallTimeMs"], + "variants": { + "baseline": [ + { + "type": "read", + "path": "docs/request-flow.md" + }, + { + "type": "read", + "path": "src/client.ts" + }, + { + "type": "read", + "path": "src/transport.ts" + } + ], + "codegraph": [ + { + "type": "codegraph", + "command": "explore", + "query": "Follow docs/request-flow.md to sendRequest in src/client.ts, then trace its import and call to formatRequest in src/transport.ts." + } + ] + } + } + ] +} diff --git a/docs/cli.md b/docs/cli.md index 0f0e9b9f..d319c22d 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -67,10 +67,10 @@ codegraph graph ./ # Default graph output to stdout codegraph graph ./ --stdout -# Fast graph-only overview +# Opt-in text specifier shortcut for plain .js and .ts files codegraph graph ./src --fast-graph -# Full AST-based graph +# Default extraction (Tree-sitter for supported source languages) codegraph graph ./src # Build a dependency graph from multiple roots @@ -667,7 +667,7 @@ Important review-bundle details: - `standard`: symbol snippets plus up to 2 callsites, `maxCandidates=25` - `deep`: symbol snippets plus up to 10 callsites, `maxCandidates=50` - Explicit flags like `--include-symbol-details`, `--max-callsites`, `--max-tests`, or `--fast-graph` override preset defaults. -- For review accuracy, keep full parsing enabled unless you intentionally want a faster, less complete pass. +- For review accuracy, keep the default Tree-sitter import extraction unless you intentionally accept less complete JavaScript or TypeScript edges. - `--incremental-strict` disables fast graph extraction for changed files while still using incremental file selection. - `--cache-verify` validates the manifest before reuse and falls back to a full rebuild if mismatches are detected. @@ -712,7 +712,7 @@ Format notes: - Use `--mermaid` for a Mermaid flowchart. - Use `--dot` for Graphviz DOT. - In DOT output, type-only edges are dotted and external nodes are dashed ellipses. -- Use `--fast-graph` for faster JS and TS specifier extraction. +- `--fast-graph` bypasses native import queries only for plain `.js` and `.ts` files, using lightweight text extraction that may miss multiline or complex patterns. TSX and other languages keep their normal extraction path. When using `--symbols`: diff --git a/docs/how-it-works.md b/docs/how-it-works.md index 70d3b370..58593be7 100644 --- a/docs/how-it-works.md +++ b/docs/how-it-works.md @@ -1,172 +1,103 @@ # How It Works -Runtime behavior, performance characteristics, architecture, extension points, and testing notes. +## Short version -## Performance +Codegraph turns source files into a resolved dependency graph and a semantic index: -### Quick start for large repos +1. Discover files under the selected roots. +2. Parse supported source languages with Tree-sitter and run language queries for imports, exports, definitions, and bindings. +3. Resolve each module specifier to a project file or an external dependency, then assemble forward and reverse graph indexes. +4. Use the same indexed symbols, scopes, and edges for navigation, impact analysis, review, and agent queries. +5. Reuse compatible parsed files and graph edges when caching or a long-lived session is enabled. -- Graph only: `codegraph graph --fast-graph --threads 8 --mermaid > graph.mmd` -- Full index: `codegraph index --workers --threads 8 --cache disk` -- Detailed symbols, pruned: `codegraph graph --root . ./src --symbols-detailed --symbols-detailed-scope imported --symbols-detailed-members-only --symbols-detailed-max-edges 5000 --mermaid > graph.symbols.pruned.mmd` +The normal supported-language path is Tree-sitter. The native addon accelerates the shared parse and query path; it is not a separate analysis model. `--fast-graph` is a narrower, opt-in shortcut for plain `.js` and `.ts` import specifiers, not a switch that disables all AST work. -### Fast graph +## Discovery -- Regex-based specifier extraction is available for JS and TS only. -- It covers common `import`, `export ... from`, `require()`, and `import()` patterns and ignores commented imports. -- If the output looks off, rerun without `--fast-graph`. -- Programmatic callers can set `graph.fastRegexDisabledLanguages` to opt specific languages out of regex fast paths. +Commands start from the project root and any selected scan roots. Codegraph walks those roots, applies supported-extension filtering, honors `.gitignore` by default, and then applies configured and command-line include or ignore globs. Excluding generated, vendored, and fixture trees here avoids reading or parsing them later. -### Scan scope +Each discovered file is mapped to a language definition. First-class source languages feed the shared Tree-sitter pipeline. Graph-first formats such as documentation and templates use their dedicated link or specifier extractors where the [language parity matrix](./language-parity.md) says graph support is available. -- Keep large fixtures, generated trees, and vendored code out of default scans with `codegraph.config.json`. -- `discovery.ignoreGlobs` reduces the file set before indexing, search text reads, unresolved-import checks, graphing, impact, and review work. -- CLI `--include-glob` and `--ignore-glob` are additive one-off scan-root-relative filters. `--no-gitignore` disables gitignore filtering for that command. +See [CLI scan and root behavior](./cli.md#project-config) for the distinction between project configuration and one-off command flags. -### Caching +## Extraction -- Modes: `off` (default), `memory` (per-process), `disk` (persist across runs under `.codegraph-cache/index-v1`) -- Content-hash caching is the default: parsed-module cache keys use content SHA1 for reliability. -- `cacheStrict` defaults to true. Set `cacheStrict: false` only when mtime and size checks are an acceptable speed tradeoff. -- Per-file parsed caches are versioned; version mismatches trigger a rebuild of that file's cached outputs. -- Persistent SQLite caches record table schema versions. Older compatible tables are migrated in place, while corrupt or unsupported schema metadata causes that cache table to be rebuilt. -- Bloom filters are built automatically during indexing for faster reference scanning. Disable them with `useBloomFilters: false` if needed. -- `.codegraph-cache/index-v1/manifest.json` stores the last indexed commit, graph options, and per-file signatures plus resolved edges. -- Incremental runs treat the manifest as a cached base graph: unchanged files keep their edges, while changed files are reparsed and their edges replaced. -- `codegraph hotspots` and `codegraph inspect` reuse the disk index cache when the manifest is present and log the manifest path, timestamp, and last commit hash to stderr. -- Agent tool wrappers and agent sessions default to incremental warm-cache reuse so repeated local and MCP queries pay the cold build cost once, then reuse compatible manifests and parsed state. -- Remove the manifest, clear `.codegraph-cache/index-v1`, or rerun with different graph flags to force a full graph rebuild. +### Default: Tree-sitter queries -### Read paths +For supported source languages, language definitions provide a Tree-sitter grammar plus queries and classification rules. The indexer parses each file and normalizes query captures into common records for module specifiers, exports, local definitions, import bindings, and scopes. Later stages therefore consume the same shapes across languages. -- `ProjectIndex` builds derived forward and reverse file-adjacency maps for common dependency reads. -- `deps`, `rdeps`, and `path` use adjacency traversal instead of scanning every edge at each BFS step. -- CLI graph queries use the indexed graph path when no direct graph collector is injected, so manifest-backed builds can serve repeated reads without changing command output. -- High-level SQLite dependency-chain and affected-function questions walk `file_edges` through indexed neighbor lookups instead of loading all file edges into memory. +In the normal `auto` runtime mode, the native addon performs this Tree-sitter parse and query work when available. TypeScript code owns discovery, normalization, resolution, graph assembly, semantic operations, and output formatting, so native acceleration does not change the public result contract. -### Threads +### Opt-in: `--fast-graph` -- Use `--threads` to increase concurrency. -- A typical sweet spot is CPU cores or cores times two. -- Very high values may become I/O-bound; 8-32 is a good SSD-era range. +`--fast-graph` bypasses native import queries only for plain `.js` and `.ts` files and extracts their module specifiers with a lightweight text matcher instead. TSX and other parser-backed languages keep their normal extraction path, and other analysis work can still require parsed syntax. -### Native Tree-sitter acceleration +The shortcut recognizes common `import`, `export ... from`, `require()`, and `import()` forms. It can miss multiline or complex patterns, so use it for a quick dependency overview when that accuracy tradeoff is acceptable. Rerun without it when edges from `.js` or `.ts` files look incomplete or when review accuracy matters. -- `npm run build` requires the local native workspace build to succeed when Cargo is available. -- Use `npm run build:native` when you want native-only rebuilds or a hard failure if Rust is missing. -- When the addon is present, Codegraph runs supported Tree-sitter parse and query work in Rust. -- If native mode is `auto` or `off` and native is unavailable, Codegraph drops to reduced graph-only and regex recovery mode. -- If native mode is `on`, a missing native addon is a hard error. -- `--workers` uses a Piscina worker pool to offload per-file Rust extraction across CPU cores. -- Vue, Svelte, and Astro files stay on the main thread because they need source preprocessing before extraction. -- Falls back silently to single-threaded extraction if Piscina is unavailable or pool creation fails. -- Use `--report` to inspect `workerPool` statistics. +### Recovery when parsing is unavailable -### Monorepo resolution +Recovery is separate from `--fast-graph`. If the native addon is unavailable in `auto` mode, Codegraph continues in a reduced graph-only mode and uses the available regex or graph-first recovery extractors. It does not load a second JavaScript grammar backend. Semantic features that need definitions, scopes, or precise syntax may be unavailable or less complete in this mode. -- Workspace detection precedence: `package.json` workspaces, then `pnpm-workspace.yaml`, then `lerna.json` -- `pnpm-workspace.yaml` supports `packages:` include globs and `!` exclude globs. -- Bare-specifier resolution precedence: - - nearest TypeScript `paths` and `baseUrl` - - workspace packages - - `node_modules` only when `--resolve-node-modules` is enabled -- Package subpaths are resolved via `exports` and `main` heuristics. +A failed or empty import query can also trigger a language-specific recovery extractor for that file. This is resilience behavior, not the normal architecture. Use `--report` or `codegraph doctor` when you need to confirm which backend ran. `--native on` makes a missing native addon an error; `--native off` explicitly selects reduced behavior. -### Troubleshooting +## Resolution and graph construction -- Missing JS or TS edges: disable `--fast-graph`. -- Dynamic JS or TS specifiers or bare imports from custom roots: use `--dynamic-import-heuristics` or `--resolution-hint ` only when needed because they can introduce false positives. -- Stale results after non-strict cache runs: rerun with default strict caching or clear `.codegraph-cache`. -- Windows path separators are normalized to `/` where relevant. +Extraction produces raw module specifiers. Resolution then turns them into graph targets: -## Architecture +- Relative and absolute-like paths are matched to discovered files with supported extension and index-file rules. +- TypeScript path aliases and `baseUrl`, workspace packages, and package metadata participate where configured. +- `node_modules` resolution is opt-in with `--resolve-node-modules`. +- Resolution hints and dynamic-import heuristics are opt-in because guesses can create false positives. +- A specifier that cannot be mapped to a project file remains an `external` node instead of disappearing. -### 1. Language adapters +The resulting file nodes and typed edges are stored with forward and reverse adjacency indexes. That graph powers `graph`, `deps`, `rdeps`, `path`, `cycles`, unresolved-import checks, and the dependency portion of higher-level analysis. Mermaid, DOT, JSON, and SQLite are output views of the same graph model. -Language adapters expose: +## Semantic navigation and impact -- file extensions -- the Tree-sitter grammar -- a few node-type helpers -- four small query groups for imports, exports, locals, and import bindings -- definition classification and scope behavior +The semantic index connects definitions, scopes, exports, and import bindings to graph edges. `goto` resolves local definitions and imported bindings, including supported namespace-member cases. `refs` follows local and imported references through the same model. -### 2. Indexing +Impact analysis maps diff hunks to changed symbols, follows resolved references and reverse dependencies, and ranks the affected files. Review and agent-facing commands build on that evidence, adding bounded snippets or related findings as requested. Call compatibility is deliberately narrower than type checking: it reports high-confidence arity mismatches for resolved callable changes, not overload, dispatch, macro, or data-flow conclusions. -- TypeScript owns the shared indexing pipeline, resolution logic, and output shapes. -- `src/indexer.ts` stays as the stable public facade while the implementation is split across focused modules under `src/indexer/` for parsing, imports, symbols, navigation, and build orchestration. -- The parser and query hot path stays on Tree-sitter for every supported language. -- When available, the addon inside `@lzehrung/codegraph-native` runs those Tree-sitter parses and queries natively, then returns plain capture data to TypeScript. +Precise semantic navigation depends on successful parsing and queries. Reduced recovery can preserve useful file edges without claiming equivalent symbol or scope analysis. Current per-language capabilities are listed in the [language parity matrix](./language-parity.md). -### 3. Graph building +## Cache and session behavior -- For each file, Codegraph collects module specifiers and resolves them. -- Path-like specifiers resolve to best-effort file targets. -- Unresolved targets become `external` nodes instead of being discarded. +Caching avoids repeating work; it does not change extraction or resolution semantics. -### 4. Navigation +- `off` is useful for a deliberately cold run. +- `memory` reuses parsed work inside one process. +- `disk` persists compatible parsed-file entries and an incremental graph manifest under `.codegraph-cache/index-v1` for later commands. +- Strict content hashes favor reliable reuse. Non-strict metadata checks are an explicit speed tradeoff. -- `goToDefinition` checks local scope first, then imported bindings, and understands namespace-member access. -- `findReferences` builds per-file scope, seeds imports as bindings, records occurrences, and resolves through imports and namespace members. +An incremental graph starts from a compatible manifest, keeps edges for unchanged files, and replaces edges for changed files. Changes to file signatures, discovery configuration, graph options, cache schema, or relevant build options invalidate the affected reuse boundary. Corrupt or unsupported cache data is rebuilt rather than treated as current analysis. -### 5. Impact and Call Compatibility +Long-lived agent and review sessions retain compatible index state so repeated questions avoid the cold build. They check relevant file, configuration, and project freshness signals before reuse and refresh when drift is detected. Clear the disk cache or run cache-off when you specifically need a cold rebuild. -- Impact maps diff hunks to changed symbols, then scans resolved references and dependency edges to rank affected files. -- Duplicate detection compares indexed symbols and chunks with exact hashes, normalized token hashes, token fingerprints, and AST shape hashes when parser context is available. -- Pretty impact and review summaries add a small duplicate-lead pass over scoped files only; git copy or rename similarity metadata can boost those leads when both files exist in the indexed snapshot. JSON callers use `findDuplicates()` for the full grouped contract. -- Signature-change detection uses Tree-sitter byte ranges so body-only edits do not look like parameter-list edits. -- Call compatibility runs only for changed callable signatures with provider-backed signature extraction and high-confidence callsite argument counts. -- Hints compare arity only. They do not perform type checking, overload resolution, data-flow analysis, macro expansion, or dynamic dispatch. -- Existing impact filters apply before hints are emitted, so ignored files and tests excluded by default stay out of call compatibility results. -- Long-lived `CodeReviewSession` instances keep cheap freshness baselines for config files and project-directory mtimes. Navigation also checks the requested file signature, while impact calls add an interval-throttled tracked-file scan before reuse. When those signals show drift, the session refreshes before serving results, and `getStats()` exposes stale/refresh metadata for callers that want to surface it. +## Performance choices -### 6. AST grep +Choose the least lossy option that solves the actual bottleneck: -AST grep runs any Tree-sitter query across matched files and prints hits as `file:line:col: @capture: snippet`. +1. **Narrow discovery first.** Scan only the roots you need and ignore generated or vendored trees. Work avoided here benefits every later stage. +2. **Use disk cache for repeated CLI or agent work.** Use memory cache for repeated operations in one process and cache-off for controlled cold runs. +3. **Use `--threads N` for file-level indexing and I/O concurrency.** Start near the available CPU count, then measure; excessive concurrency can become I/O-bound. +4. **Use `--workers` for CPU-parallel native extraction on larger index-building workloads.** It requires the native addon; direct file-only graph output does not use this pool. Single-threaded extraction remains the fallback if a worker pool cannot start, and preprocessed single-file component formats stay on the main thread. +5. **Use `--fast-graph` only for a quick plain-JavaScript or plain-TypeScript dependency pass.** It trades import-specifier completeness for less native query work on `.js` and `.ts` files; TSX and other languages keep their normal extraction path. -### Design tradeoffs +Use `--report` to inspect cache, backend, and worker-pool behavior instead of assuming a tuning flag helped. For controlled evidence about agent-oriented discovery, see the [benchmark methodology and checked results](./benchmarks/README.md); those fixtures do not establish universal speedups. -- Native parse and query work, TypeScript indexing and reporting: Rust handles the syntax-tree hot path when the native addon is available, while TypeScript keeps graph assembly, resolution, review logic, CLI behavior, and SQLite export in one shared implementation. That keeps the performance-critical layer small and lets native and non-native modes share output contracts and tests. -- Read-only SQLite inspection: `codegraph sql` and `queryGraphSqliteRaw()` treat the SQLite export as a query artifact, not a writable application database. Keeping that surface read-only makes CI and agent workflows safer, avoids accidental artifact corruption, and preserves reproducible graph output. -- Stable symbol handles for automation: cursor-based navigation is convenient for editor-style entrypoints, but serialized handles are a better contract for review bundles, repeated agent calls, and cross-step automation. They let downstream tools refer back to definitions and import aliases without depending on the exact cursor that found them originally. +## Extension and testing -## Extending to other languages +New source-language support should extend the shared definition, native grammar, extraction, resolution, semantic, and fixture conventions rather than introduce a parallel parser architecture. Graph-first languages should state their narrower contract explicitly. -Codegraph uses one unified language-definition system that powers both dependency graph extraction and semantic chunking. +- [Adding language support](./adding-language-support.md) gives the implementation and review checklist. +- [Language parity](./language-parity.md) records the public capability matrix. +- [Scenario catalog](./scenario-catalog.md) links claimed behavior to fixtures and regression coverage. -For the full checklist, see [docs/adding-language-support.md](./adding-language-support.md). - -The short version: - -1. Add a definition file in `src/languages/definitions/.ts`. -2. Register it in `src/languages.ts` and `src/bootstrap/treeSitterLanguages.ts`. -3. Add the nearest language sample and regression coverage in `tests/languages`. - -Do not stop at "80/20" support unless the parity docs and scenario catalog explicitly mark the limitation and the tests prove it. - -## Testing - -The core stays intentionally pure and test-friendly. Good seams to target directly include: - -- `collectLocalsAndExportsFromSource(file, source, support, lang)` -- `collectModuleSpecifiersFromSource(support, lang, source)` -- `buildScopeIndexFromSource(file, source, support, lang, imports)` -- `resolveExport(index, file, exportedName)` -- `goToDefinition(index, req)` -- `findReferences(index, req)` - -Recommended strategy: - -- Feed inline strings as source and assert on JSON-serializable structures. -- For end-to-end tests, create a small temp directory with a few files and run the CLI with `tsx`. -- Use `npm run test:native:required` for JS suites that must fail when the native addon is unavailable. -- Use `npm run test:native:fallback` for reduced-mode fallback coverage on hosts without native support. -- `npm run test:native` runs Rust native tests, native-required JS suites, and reduced-mode fallback suites together. +Test the smallest layer that owns a change, then add a small end-to-end fixture when discovery, resolution, or CLI output is part of the contract. Native-required suites verify the normal Tree-sitter path; native-fallback suites verify reduced recovery without presenting it as parity. ## Related docs -- [docs/cli.md](./cli.md) -- [docs/library-api.md](./library-api.md) -- [docs/agent-workflows.md](./agent-workflows.md) -- [docs/language-parity.md](./language-parity.md) -- [docs/scenario-catalog.md](./scenario-catalog.md) +- [CLI reference](./cli.md) +- [Library API](./library-api.md) +- [Agent workflows](./agent-workflows.md) +- [Benchmark methodology and results](./benchmarks/README.md) diff --git a/package.json b/package.json index a1a545b5..56edaedc 100644 --- a/package.json +++ b/package.json @@ -70,6 +70,8 @@ "bench:native": "node ./scripts/bench-native.mjs", "bench:native:smoke": "node ./scripts/bench-native.mjs --runs 1 --fixtures typescript,python,go,rust,mixed --workloads full --temperatures cold --max-slowdown 4", "bench:native:full": "node ./scripts/bench-native.mjs --runs 1 --fixtures typescript,python,go,rust,mixed --workloads full,graph --temperatures cold", + "bench:docs": "node ./scripts/ensure-dist-for-tests.mjs && node ./scripts/benchmarks/run-scenario.mjs --output .tmp/public-docs-benchmark-results.json --require-complete && node ./scripts/benchmarks/summarize-results.mjs --input .tmp/public-docs-benchmark-results.json", + "bench:docs:check": "node ./scripts/benchmarks/summarize-results.mjs --input docs/benchmarks/results.example.json --scenario-file docs/benchmarks/scenarios.json --readme docs/benchmarks/README.md --check", "test:ui": "node ./scripts/ensure-dist-for-tests.mjs && vitest --ui", "test:run": "node ./scripts/ensure-dist-for-tests.mjs && vitest run", "test:bench": "node ./scripts/ensure-dist-for-tests.mjs && vitest tests/bench-harness.test.ts tests/native-query-scope.test.ts", diff --git a/scripts/benchmarks/benchmark-contract-lib.mjs b/scripts/benchmarks/benchmark-contract-lib.mjs new file mode 100644 index 00000000..246c0d21 --- /dev/null +++ b/scripts/benchmarks/benchmark-contract-lib.mjs @@ -0,0 +1,22 @@ +import { createHash } from "node:crypto"; + +function canonicalizeJsonValue(value) { + if (Array.isArray(value)) return value.map(canonicalizeJsonValue); + if (value !== null && typeof value === "object") { + return Object.fromEntries( + Object.keys(value) + .sort() + .map((key) => [key, canonicalizeJsonValue(value[key])]), + ); + } + return value; +} + +export function canonicalizeSelectedScenarios(schemaVersion, scenarios) { + return JSON.stringify(canonicalizeJsonValue({ schemaVersion, scenarios })); +} + +export function calculateScenarioDigest(schemaVersion, scenarios) { + const canonical = canonicalizeSelectedScenarios(schemaVersion, scenarios); + return `sha256:${createHash("sha256").update(canonical, "utf8").digest("hex")}`; +} diff --git a/scripts/benchmarks/run-scenario-lib.mjs b/scripts/benchmarks/run-scenario-lib.mjs new file mode 100644 index 00000000..8208c5f9 --- /dev/null +++ b/scripts/benchmarks/run-scenario-lib.mjs @@ -0,0 +1,708 @@ +import { spawn as spawnChild } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { performance } from "node:perf_hooks"; +import { fileURLToPath } from "node:url"; +import { calculateScenarioDigest } from "./benchmark-contract-lib.mjs"; + +export const DEFAULT_SCENARIO_FILE = "docs/benchmarks/scenarios.json"; +export const DEFAULT_RUNS = 3; +export const METRICS = Object.freeze(["toolCalls", "fileReads", "wallTimeMs"]); +export const VARIANTS = Object.freeze(["baseline", "codegraph"]); +export const repositoryRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..", ".."); + +const SCHEME_OR_DRIVE_PATH = /^[a-zA-Z][a-zA-Z\d+.-]*:/; + +function fail(message) { + throw new Error(message); +} + +function isPlainObject(value) { + return value !== null && typeof value === "object" && !Array.isArray(value); +} + +function assertExactKeys(value, expectedKeys, label) { + if (!isPlainObject(value)) fail(`${label} must be an object.`); + const actual = Object.keys(value).sort(); + const expected = [...expectedKeys].sort(); + const missing = expected.filter((key) => !actual.includes(key)); + const extra = actual.filter((key) => !expected.includes(key)); + if (missing.length || extra.length) { + const details = [ + missing.length ? `missing ${missing.join(", ")}` : "", + extra.length ? `unexpected ${extra.join(", ")}` : "", + ] + .filter(Boolean) + .join("; "); + fail(`${label} must have exactly these keys: ${expectedKeys.join(", ")} (${details}).`); + } +} + +function assertNonEmptyString(value, label) { + if (typeof value !== "string" || value.trim() === "") fail(`${label} must be a non-empty string.`); + if (value.includes("\0")) fail(`${label} must not contain a NUL byte.`); +} + +function normalizePortableRelativePath(value, label) { + assertNonEmptyString(value, label); + if (value.startsWith("~")) fail(`${label} must be relative to the repository, not a home-directory path.`); + if (value.includes("\\")) fail(`${label} must use forward slashes.`); + if (path.posix.isAbsolute(value) || path.win32.isAbsolute(value)) { + fail(`${label} must be a relative path.`); + } + if (SCHEME_OR_DRIVE_PATH.test(value)) fail(`${label} must be a local path, not a URL or drive-relative path.`); + const normalized = path.posix.normalize(value); + if (normalized === "." || normalized === ".." || normalized.startsWith("../")) { + fail(`${label} must stay within the repository.`); + } + return normalized.replace(/^\.\//, ""); +} + +function assertCanonicalScenarioPath(value, label) { + const normalized = normalizePortableRelativePath(value, label); + if (normalized !== value) fail(`${label} must be normalized as "${normalized}".`); + return normalized; +} + +function isPathInside(parent, candidate) { + const relative = path.relative(parent, candidate); + return relative === "" || (!path.isAbsolute(relative) && relative !== ".." && !relative.startsWith(`..${path.sep}`)); +} + +function resolveConfinedPath(rootDir, relativePath, label) { + const absolute = path.resolve(rootDir, ...relativePath.split("/")); + if (!isPathInside(rootDir, absolute)) fail(`${label} escapes the repository.`); + return absolute; +} + +function realpathExisting(entryPath, label, fsImpl = fs) { + try { + return fsImpl.realpathSync(entryPath); + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + fail(`${label} does not exist or cannot be resolved: ${detail}`); + } +} + +function assertNoSymlinkEscape(parentRealpath, entryPath, label, fsImpl = fs) { + const entryRealpath = realpathExisting(entryPath, label, fsImpl); + if (!isPathInside(parentRealpath, entryRealpath)) fail(`${label} escapes its allowed root through a symlink.`); + return entryRealpath; +} + +function validateMetrics(metrics, label) { + if (!Array.isArray(metrics) || metrics.length !== METRICS.length) { + fail(`${label} must be exactly [${METRICS.join(", ")}].`); + } + for (let index = 0; index < METRICS.length; index += 1) { + if (metrics[index] !== METRICS[index]) fail(`${label} must be exactly [${METRICS.join(", ")}].`); + } +} + +function validateAnchors(anchors, label) { + if (!Array.isArray(anchors) || anchors.length === 0) fail(`${label} must be a non-empty array.`); + const seen = new Set(); + for (let index = 0; index < anchors.length; index += 1) { + const anchorLabel = `${label}[${index}]`; + assertCanonicalScenarioPath(anchors[index], anchorLabel); + if (seen.has(anchors[index])) fail(`${label} contains duplicate anchor "${anchors[index]}".`); + seen.add(anchors[index]); + } +} + +function validateStep(step, variant, label) { + if (variant === "baseline") { + assertExactKeys(step, ["type", "path"], label); + if (step.type !== "read") fail(`${label}.type must be "read".`); + assertCanonicalScenarioPath(step.path, `${label}.path`); + return; + } + + assertExactKeys(step, ["type", "command", "query"], label); + if (step.type !== "codegraph") fail(`${label}.type must be "codegraph".`); + if (step.command !== "explore") fail(`${label}.command must be "explore".`); + assertNonEmptyString(step.query, `${label}.query`); +} + +function validateVariants(variants, label) { + assertExactKeys(variants, VARIANTS, label); + for (const variant of VARIANTS) { + const steps = variants[variant]; + if (!Array.isArray(steps) || steps.length === 0) fail(`${label}.${variant} must be a non-empty array.`); + for (let index = 0; index < steps.length; index += 1) { + validateStep(steps[index], variant, `${label}.${variant}[${index}]`); + } + } +} + +function validateRepoFile(repoAbsolute, repoRealpath, relativePath, label, fsImpl) { + const fileAbsolute = resolveConfinedPath(repoAbsolute, relativePath, label); + const fileRealpath = assertNoSymlinkEscape(repoRealpath, fileAbsolute, label, fsImpl); + let fileStat; + try { + fileStat = fsImpl.statSync(fileRealpath); + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + fail(`${label} cannot be inspected: ${detail}`); + } + if (!fileStat.isFile()) fail(`${label} must name an existing file.`); +} + +function validateFilesystemEntries(document, rootDir, fsImpl = fs) { + const rootRealpath = realpathExisting(rootDir, "Repository root", fsImpl); + for (let scenarioIndex = 0; scenarioIndex < document.scenarios.length; scenarioIndex += 1) { + const scenario = document.scenarios[scenarioIndex]; + const label = `scenarios[${scenarioIndex}]`; + const repoAbsolute = resolveConfinedPath(rootDir, scenario.repo, `${label}.repo`); + const repoRealpath = assertNoSymlinkEscape(rootRealpath, repoAbsolute, `${label}.repo`, fsImpl); + let repoStat; + try { + repoStat = fsImpl.statSync(repoRealpath); + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + fail(`${label}.repo cannot be inspected: ${detail}`); + } + if (!repoStat.isDirectory()) fail(`${label}.repo must name an existing directory.`); + + for (let anchorIndex = 0; anchorIndex < scenario.expectedAnchors.length; anchorIndex += 1) { + validateRepoFile( + repoAbsolute, + repoRealpath, + scenario.expectedAnchors[anchorIndex], + `${label}.expectedAnchors[${anchorIndex}]`, + fsImpl, + ); + } + + for (let stepIndex = 0; stepIndex < scenario.variants.baseline.length; stepIndex += 1) { + const step = scenario.variants.baseline[stepIndex]; + validateRepoFile(repoAbsolute, repoRealpath, step.path, `${label}.variants.baseline[${stepIndex}].path`, fsImpl); + } + } +} + +export function validateScenarioDocument(value, options = {}) { + const { rootDir = repositoryRoot, fs: fsImpl = fs, checkFilesystem = true } = options; + assertExactKeys(value, ["schemaVersion", "scenarios"], "Scenario document"); + if (value.schemaVersion !== 1) fail("Scenario document schemaVersion must be 1."); + if (!Array.isArray(value.scenarios) || value.scenarios.length === 0) { + fail("Scenario document scenarios must be a non-empty array."); + } + + const ids = new Set(); + for (let index = 0; index < value.scenarios.length; index += 1) { + const scenario = value.scenarios[index]; + const label = `scenarios[${index}]`; + assertExactKeys(scenario, ["id", "repo", "task", "expectedAnchors", "metrics", "variants"], label); + assertNonEmptyString(scenario.id, `${label}.id`); + if (!/^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(scenario.id)) { + fail(`${label}.id may contain only letters, digits, dots, underscores, and hyphens.`); + } + if (ids.has(scenario.id)) fail(`Scenario id "${scenario.id}" is duplicated.`); + ids.add(scenario.id); + assertCanonicalScenarioPath(scenario.repo, `${label}.repo`); + assertNonEmptyString(scenario.task, `${label}.task`); + validateAnchors(scenario.expectedAnchors, `${label}.expectedAnchors`); + validateMetrics(scenario.metrics, `${label}.metrics`); + validateVariants(scenario.variants, `${label}.variants`); + } + + if (checkFilesystem) validateFilesystemEntries(value, path.resolve(rootDir), fsImpl); + return value; +} + +export function loadScenarioFile(scenarioFile = DEFAULT_SCENARIO_FILE, options = {}) { + const { rootDir = repositoryRoot, fs: fsImpl = fs } = options; + const normalizedFile = normalizePortableRelativePath(scenarioFile, "Scenario file"); + const absoluteRoot = path.resolve(rootDir); + const scenarioAbsolute = resolveConfinedPath(absoluteRoot, normalizedFile, "Scenario file"); + const rootRealpath = realpathExisting(absoluteRoot, "Repository root", fsImpl); + assertNoSymlinkEscape(rootRealpath, scenarioAbsolute, "Scenario file", fsImpl); + let source; + try { + source = fsImpl.readFileSync(scenarioAbsolute, "utf8"); + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + fail(`Unable to read scenario file "${normalizedFile}": ${detail}`); + } + let parsed; + try { + parsed = JSON.parse(source); + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + fail(`Scenario file "${normalizedFile}" is not valid JSON: ${detail}`); + } + return validateScenarioDocument(parsed, { rootDir: absoluteRoot, fs: fsImpl, checkFilesystem: true }); +} + +function readOptionValue(argv, index, option) { + const argument = argv[index]; + const prefix = `${option}=`; + if (argument.startsWith(prefix)) { + const value = argument.slice(prefix.length); + if (!value) fail(`${option} requires a value.`); + return { value, nextIndex: index }; + } + if (argument === option) { + const value = argv[index + 1]; + if (value === undefined || value.startsWith("--")) fail(`${option} requires a value.`); + return { value, nextIndex: index + 1 }; + } + return undefined; +} + +function parsePositiveInteger(value, option) { + if (!/^[1-9]\d*$/.test(value)) fail(`${option} must be a positive integer.`); + const parsed = Number(value); + if (!Number.isSafeInteger(parsed)) fail(`${option} must be a safe positive integer.`); + return parsed; +} + +export function parseArguments(argv) { + if (!Array.isArray(argv)) fail("Arguments must be an array."); + const options = { + scenarioIds: [], + runs: DEFAULT_RUNS, + scenarioFile: DEFAULT_SCENARIO_FILE, + output: undefined, + requireComplete: false, + json: false, + }; + const seenOptions = new Set(); + + for (let index = 0; index < argv.length; index += 1) { + const argument = argv[index]; + if (typeof argument !== "string") fail(`Argument at index ${index} must be a string.`); + + const scenario = readOptionValue(argv, index, "--scenario"); + if (scenario) { + const ids = scenario.value.split(",").map((id) => id.trim()); + if (ids.some((id) => id === "")) fail("--scenario must contain non-empty comma-separated ids."); + for (const id of ids) { + if (options.scenarioIds.includes(id)) fail(`--scenario selects "${id}" more than once.`); + options.scenarioIds.push(id); + } + index = scenario.nextIndex; + continue; + } + + const runs = readOptionValue(argv, index, "--runs"); + if (runs) { + if (seenOptions.has("--runs")) fail("--runs may be specified only once."); + seenOptions.add("--runs"); + options.runs = parsePositiveInteger(runs.value, "--runs"); + index = runs.nextIndex; + continue; + } + + const output = readOptionValue(argv, index, "--output"); + if (output) { + if (seenOptions.has("--output")) fail("--output may be specified only once."); + seenOptions.add("--output"); + options.output = normalizePortableRelativePath(output.value, "--output"); + index = output.nextIndex; + continue; + } + + const scenarioFile = readOptionValue(argv, index, "--scenario-file"); + if (scenarioFile) { + if (seenOptions.has("--scenario-file")) fail("--scenario-file may be specified only once."); + seenOptions.add("--scenario-file"); + options.scenarioFile = normalizePortableRelativePath(scenarioFile.value, "--scenario-file"); + index = scenarioFile.nextIndex; + continue; + } + + if (argument === "--require-complete") { + if (seenOptions.has("--require-complete")) fail("--require-complete may be specified only once."); + seenOptions.add("--require-complete"); + options.requireComplete = true; + continue; + } + if (argument === "--json") { + if (seenOptions.has("--json")) fail("--json may be specified only once."); + seenOptions.add("--json"); + options.json = true; + continue; + } + if (argument.startsWith("--require-complete=") || argument.startsWith("--json=")) { + fail(`${argument.split("=", 1)[0]} does not take a value.`); + } + fail(`Unknown argument "${argument}".`); + } + + return options; +} + +export const parseArgs = parseArguments; + +function stringifyCapturedOutput(output) { + if (typeof output === "string") return output; + if (Buffer.isBuffer(output)) return output.toString("utf8"); + try { + return JSON.stringify(output); + } catch { + return String(output); + } +} + +export function calculateCompleteness(expectedAnchors, capturedStepOutputs) { + if (!Array.isArray(expectedAnchors) || !Array.isArray(capturedStepOutputs)) { + fail("Completeness requires expected anchor and captured output arrays."); + } + const captured = capturedStepOutputs.map(stringifyCapturedOutput).join("\n"); + const found = expectedAnchors.filter((anchor) => captured.includes(anchor)); + const missingAnchors = expectedAnchors.filter((anchor) => !captured.includes(anchor)); + return { + anchorsExpected: expectedAnchors.length, + anchorsFound: found.length, + missingAnchors, + completeness: expectedAnchors.length === 0 ? 1 : found.length / expectedAnchors.length, + }; +} + +function normalizeReturnedFile(value) { + if (typeof value !== "string" || value === "" || value.includes("\0") || value.includes("\\")) return undefined; + if (path.posix.isAbsolute(value) || path.win32.isAbsolute(value) || SCHEME_OR_DRIVE_PATH.test(value)) + return undefined; + const normalized = path.posix.normalize(value).replace(/^\.\//, ""); + if (normalized === "." || normalized === ".." || normalized.startsWith("../")) return undefined; + return normalized; +} + +export function collectSourceFiles(exploreOutputs) { + const outputs = Array.isArray(exploreOutputs) ? exploreOutputs : [exploreOutputs]; + const files = new Set(); + for (const output of outputs) { + if (!isPlainObject(output)) continue; + if (isPlainObject(output.fileView)) { + const file = normalizeReturnedFile(output.fileView.file); + if (file) files.add(file); + } + if (!Array.isArray(output.packets)) continue; + for (const packet of output.packets) { + if (!isPlainObject(packet)) continue; + const resolvedFile = + isPlainObject(packet.packet) && isPlainObject(packet.packet.target) + ? normalizeReturnedFile(packet.packet.target.file) + : undefined; + const directFile = normalizeReturnedFile(packet.target); + const file = resolvedFile ?? directFile; + if (file) files.add(file); + } + } + return [...files].sort((left, right) => left.localeCompare(right)); +} + +export function countSourceFiles(exploreOutputs) { + return collectSourceFiles(exploreOutputs).length; +} + +export function spawnCaptured(executable, argv, options = {}) { + const { cwd, spawn = spawnChild } = options; + return new Promise((resolve, reject) => { + const child = spawn(executable, argv, { + cwd, + shell: false, + windowsHide: true, + stdio: ["ignore", "pipe", "pipe"], + }); + let stdout = ""; + let stderr = ""; + child.stdout.setEncoding("utf8"); + child.stderr.setEncoding("utf8"); + child.stdout.on("data", (chunk) => { + stdout += chunk; + }); + child.stderr.on("data", (chunk) => { + stderr += chunk; + }); + child.once("error", reject); + child.once("close", (code, signal) => resolve({ code, signal, stdout, stderr })); + }); +} + +export async function executeCodegraphExplore(options) { + const { + rootDir = repositoryRoot, + repo, + query, + distCliPath = path.join(rootDir, "dist", "cli.js"), + spawn = spawnChild, + } = options; + if (!fs.existsSync(distCliPath)) { + fail("Built CLI not found at dist/cli.js. Run node scripts/ensure-dist-for-tests.mjs first."); + } + const absoluteRoot = path.resolve(rootDir); + const rootRealpath = realpathExisting(absoluteRoot, "Repository root"); + const repoAbsolute = resolveConfinedPath(absoluteRoot, repo, "Scenario repo"); + const repoRealpath = assertNoSymlinkEscape(rootRealpath, repoAbsolute, "Scenario repo"); + if (!fs.statSync(repoRealpath).isDirectory()) fail("Scenario repo must name an existing directory."); + const argv = [distCliPath, "explore", query, "--root", repoRealpath, "--cache", "off", "--json"]; + let result; + try { + result = await spawnCaptured(process.execPath, argv, { cwd: rootDir, spawn }); + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + fail(`Unable to start Codegraph explore: ${detail}`); + } + if (result.code !== 0) { + const status = result.signal ? `signal ${result.signal}` : `exit ${result.code}`; + const detail = result.stderr.trim() || result.stdout.trim() || "no diagnostic output"; + fail(`Codegraph explore failed (${status}): ${detail}`); + } + let data; + try { + data = JSON.parse(result.stdout); + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + const stderr = result.stderr.trim(); + fail(`Codegraph explore returned invalid JSON: ${detail}${stderr ? `; stderr: ${stderr}` : ""}`); + } + if (!isPlainObject(data)) fail("Codegraph explore returned JSON that is not an object."); + return { data, stdout: result.stdout, stderr: result.stderr }; +} + +function normalizeExploreExecution(result) { + if (isPlainObject(result) && Object.hasOwn(result, "data")) { + return { + data: result.data, + stdout: typeof result.stdout === "string" ? result.stdout : JSON.stringify(result.data), + stderr: typeof result.stderr === "string" ? result.stderr : "", + }; + } + return { data: result, stdout: JSON.stringify(result), stderr: "" }; +} +export function captureCodegraphEvidence(response) { + if (!isPlainObject(response)) fail("Codegraph explore returned JSON that is not an object."); + return JSON.stringify({ + anchors: response.anchors, + packets: response.packets, + fileView: response.fileView, + paths: response.paths, + blastRadius: response.blastRadius, + candidateTests: response.candidateTests, + }); +} + +async function runVariant(scenario, variant, runNumber, options) { + const { rootDir, readFile, executeCodegraph, now } = options; + const capturedOutputs = []; + const exploreOutputs = []; + const startedAt = now(); + const steps = scenario.variants[variant]; + + for (let stepIndex = 0; stepIndex < steps.length; stepIndex += 1) { + const step = steps[stepIndex]; + try { + if (variant === "baseline") { + const rootRealpath = realpathExisting(rootDir, "Repository root"); + const repoAbsolute = resolveConfinedPath(rootDir, scenario.repo, "Scenario repo"); + const repoRealpath = assertNoSymlinkEscape(rootRealpath, repoAbsolute, "Scenario repo"); + const fileAbsolute = resolveConfinedPath(repoAbsolute, step.path, "Baseline read path"); + const fileRealpath = assertNoSymlinkEscape(repoRealpath, fileAbsolute, "Baseline read path"); + if (!fs.statSync(fileRealpath).isFile()) fail("Baseline read path must name an existing file."); + const content = await readFile(fileRealpath, "utf8"); + capturedOutputs.push(JSON.stringify({ path: step.path, content: String(content) })); + } else { + const execution = normalizeExploreExecution( + await executeCodegraph({ rootDir, repo: scenario.repo, query: step.query, command: step.command }), + ); + if (!isPlainObject(execution.data)) fail("Codegraph explore returned JSON that is not an object."); + exploreOutputs.push(execution.data); + capturedOutputs.push(captureCodegraphEvidence(execution.data)); + } + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + fail(`Scenario "${scenario.id}" ${variant} run ${runNumber}, step ${stepIndex + 1} failed: ${detail}`); + } + } + + const elapsed = Math.max(0, now() - startedAt); + return { + scenarioId: scenario.id, + variant, + run: runNumber, + metrics: { + toolCalls: steps.length, + fileReads: variant === "baseline" ? steps.length : countSourceFiles(exploreOutputs), + wallTimeMs: Number(elapsed.toFixed(3)), + }, + checks: calculateCompleteness(scenario.expectedAnchors, capturedOutputs), + }; +} + +export async function runScenario(scenario, options = {}) { + const rootDir = path.resolve(options.rootDir ?? repositoryRoot); + const runs = options.runs ?? DEFAULT_RUNS; + if (!Number.isSafeInteger(runs) || runs < 1) fail("runs must be a positive safe integer."); + const dependencies = { + rootDir, + runs, + readFile: options.readFile ?? fs.promises.readFile, + executeCodegraph: options.executeCodegraph ?? executeCodegraphExplore, + now: options.now ?? performance.now.bind(performance), + }; + const results = []; + for (const variant of VARIANTS) { + for (let runNumber = 1; runNumber <= runs; runNumber += 1) { + results.push(await runVariant(scenario, variant, runNumber, dependencies)); + } + } + return results; +} + +export const runOneScenario = runScenario; + +export async function runScenarios(scenarios, options = {}) { + if (!Array.isArray(scenarios)) fail("scenarios must be an array."); + const runs = []; + for (const scenario of scenarios) runs.push(...(await runScenario(scenario, options))); + return runs; +} + +export function createEnvironmentMetadata(options = {}) { + const osModule = options.os ?? os; + const processObject = options.process ?? process; + const cpus = osModule.cpus(); + const cpuModel = cpus[0]?.model?.replace(/\s+/g, " ").trim() || "unknown"; + return { + nodeVersion: processObject.version, + platform: processObject.platform, + arch: processObject.arch, + cpuModel, + logicalCpus: cpus.length, + totalMemoryBytes: osModule.totalmem(), + }; +} + +export function buildCommand(options) { + const command = [ + "node", + "scripts/benchmarks/run-scenario.mjs", + "--scenario-file", + options.scenarioFile, + "--runs", + String(options.runs), + ]; + for (const id of options.scenarioIds) command.push("--scenario", id); + if (options.output) command.push("--output", options.output); + if (options.requireComplete) command.push("--require-complete"); + if (options.json) command.push("--json"); + return command; +} + +function selectScenarios(scenarios, ids) { + if (!ids.length) return scenarios; + const byId = new Map(scenarios.map((scenario) => [scenario.id, scenario])); + const selectedIds = new Set(ids); + const missing = [...selectedIds].filter((id) => !byId.has(id)); + if (missing.length) { + fail( + `Unknown scenario id${missing.length === 1 ? "" : "s"}: ${missing.join(", ")}. Available: ${[...byId.keys()].join(", ")}.`, + ); + } + return scenarios.filter((scenario) => selectedIds.has(scenario.id)); +} + +export async function runBenchmark(options = {}, dependencies = {}) { + const rootDir = path.resolve(dependencies.rootDir ?? repositoryRoot); + const normalizedOptions = { + scenarioIds: options.scenarioIds ?? [], + runs: options.runs ?? DEFAULT_RUNS, + scenarioFile: normalizePortableRelativePath(options.scenarioFile ?? DEFAULT_SCENARIO_FILE, "Scenario file"), + output: options.output === undefined ? undefined : normalizePortableRelativePath(options.output, "--output"), + requireComplete: options.requireComplete ?? false, + json: options.json ?? false, + }; + const document = dependencies.scenarioDocument + ? validateScenarioDocument(dependencies.scenarioDocument, { + rootDir, + fs: dependencies.fs ?? fs, + checkFilesystem: dependencies.checkFilesystem ?? true, + }) + : loadScenarioFile(normalizedOptions.scenarioFile, { rootDir, fs: dependencies.fs ?? fs }); + const selected = selectScenarios(document.scenarios, normalizedOptions.scenarioIds); + const scenarioDigest = calculateScenarioDigest(document.schemaVersion, selected); + const scenarioIds = selected.map((scenario) => scenario.id); + const benchmarkRuns = await runScenarios(selected, { + rootDir, + runs: normalizedOptions.runs, + readFile: dependencies.readFile, + executeCodegraph: dependencies.executeCodegraph, + now: dependencies.now, + }); + const date = dependencies.date ?? (() => new Date()); + return { + schemaVersion: 1, + generatedAt: date().toISOString(), + command: buildCommand(normalizedOptions), + environment: dependencies.environment ?? createEnvironmentMetadata(), + scenarioFile: normalizedOptions.scenarioFile, + scenarioDigest, + scenarioIds, + runsPerVariant: normalizedOptions.runs, + runs: benchmarkRuns, + }; +} + +export function assertComplete(result) { + const incomplete = result.runs.filter((run) => run.checks.completeness < 1); + if (!incomplete.length) return; + const details = incomplete + .map((run) => `${run.scenarioId}/${run.variant}/run-${run.run}: ${run.checks.missingAnchors.join(", ")}`) + .join("; "); + fail(`Benchmark completeness requirement failed: ${details}.`); +} + +function nearestExistingAncestor(entryPath, fsImpl = fs) { + let current = entryPath; + while (!fsImpl.existsSync(current)) { + const parent = path.dirname(current); + if (parent === current) fail(`No existing ancestor for output path "${entryPath}".`); + current = parent; + } + return current; +} + +export function writeBenchmarkResult(relativeOutput, result, options = {}) { + const { rootDir = repositoryRoot, fs: fsImpl = fs } = options; + const normalizedOutput = normalizePortableRelativePath(relativeOutput, "--output"); + const absoluteRoot = path.resolve(rootDir); + const outputAbsolute = resolveConfinedPath(absoluteRoot, normalizedOutput, "--output"); + const rootRealpath = realpathExisting(absoluteRoot, "Repository root", fsImpl); + const ancestor = nearestExistingAncestor(path.dirname(outputAbsolute), fsImpl); + assertNoSymlinkEscape(rootRealpath, ancestor, "--output parent", fsImpl); + fsImpl.mkdirSync(path.dirname(outputAbsolute), { recursive: true }); + assertNoSymlinkEscape(rootRealpath, path.dirname(outputAbsolute), "--output parent", fsImpl); + let outputStat; + try { + outputStat = fsImpl.lstatSync(outputAbsolute); + } catch (error) { + if (!isPlainObject(error) || error.code !== "ENOENT") { + const detail = error instanceof Error ? error.message : String(error); + fail(`--output cannot be inspected: ${detail}`); + } + } + if (outputStat?.isSymbolicLink()) fail("--output must not be a symbolic link."); + if (outputStat) assertNoSymlinkEscape(rootRealpath, outputAbsolute, "--output", fsImpl); + const serialized = `${JSON.stringify(result, null, 2)}\n`; + const noFollow = fs.constants.O_NOFOLLOW ?? 0; + let descriptor; + try { + descriptor = fsImpl.openSync( + outputAbsolute, + fs.constants.O_WRONLY | fs.constants.O_CREAT | fs.constants.O_TRUNC | noFollow, + 0o666, + ); + fsImpl.writeFileSync(descriptor, serialized, "utf8"); + } finally { + if (descriptor !== undefined) fsImpl.closeSync(descriptor); + } + return serialized; +} + +export function serializeBenchmarkResult(result) { + return `${JSON.stringify(result, null, 2)}\n`; +} diff --git a/scripts/benchmarks/run-scenario.mjs b/scripts/benchmarks/run-scenario.mjs new file mode 100644 index 00000000..b97ac022 --- /dev/null +++ b/scripts/benchmarks/run-scenario.mjs @@ -0,0 +1,38 @@ +#!/usr/bin/env node + +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { + assertComplete, + parseArguments, + repositoryRoot, + runBenchmark, + serializeBenchmarkResult, + writeBenchmarkResult, +} from "./run-scenario-lib.mjs"; + +export async function main(argv = process.argv.slice(2), runtime = {}) { + const stdout = runtime.stdout ?? process.stdout; + const stderr = runtime.stderr ?? process.stderr; + const rootDir = runtime.rootDir ?? repositoryRoot; + try { + const options = parseArguments(argv); + const result = await runBenchmark(options, { rootDir }); + let serialized; + if (options.output) { + serialized = writeBenchmarkResult(options.output, result, { rootDir }); + } else { + serialized = serializeBenchmarkResult(result); + } + if (options.json || !options.output) stdout.write(serialized); + if (options.requireComplete) assertComplete(result); + return 0; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + stderr.write(`Benchmark error: ${message}\n`); + return 1; + } +} + +const isMain = process.argv[1] !== undefined && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url); +if (isMain) process.exitCode = await main(); diff --git a/scripts/benchmarks/summarize-results-lib.mjs b/scripts/benchmarks/summarize-results-lib.mjs new file mode 100644 index 00000000..4ea4b2d2 --- /dev/null +++ b/scripts/benchmarks/summarize-results-lib.mjs @@ -0,0 +1,757 @@ +import fs from "node:fs"; +import path from "node:path"; +import { calculateScenarioDigest } from "./benchmark-contract-lib.mjs"; + +export const README_START_MARKER = ""; +export const README_END_MARKER = ""; + +const RESULT_KEYS = [ + "schemaVersion", + "generatedAt", + "command", + "environment", + "scenarioFile", + "scenarioDigest", + "scenarioIds", + "runsPerVariant", + "runs", +]; +const ENVIRONMENT_KEYS = ["nodeVersion", "platform", "arch", "cpuModel", "logicalCpus", "totalMemoryBytes"]; +const RUN_KEYS = ["scenarioId", "variant", "run", "metrics", "checks"]; +const METRIC_KEYS = ["toolCalls", "fileReads", "wallTimeMs"]; +const CHECK_KEYS = ["anchorsExpected", "anchorsFound", "missingAnchors", "completeness"]; +const SCENARIO_DOCUMENT_KEYS = ["schemaVersion", "scenarios"]; +const SCENARIO_KEYS = ["id", "repo", "task", "expectedAnchors", "metrics", "variants"]; +const VARIANT_KEYS = ["baseline", "codegraph"]; +const VARIANT_ORDER = new Map([ + ["baseline", 0], + ["codegraph", 1], +]); + +function fail(location, message) { + throw new Error(`${location}: ${message}`); +} + +function isPlainObject(value) { + if (value === null || typeof value !== "object" || Array.isArray(value)) { + return false; + } + const prototype = Object.getPrototypeOf(value); + return prototype === Object.prototype || prototype === null; +} + +function requireObject(value, location) { + if (!isPlainObject(value)) { + fail(location, "expected an object"); + } + return value; +} + +function requireExactKeys(value, expectedKeys, location) { + requireObject(value, location); + const expected = new Set(expectedKeys); + const actualKeys = Object.keys(value); + const missing = expectedKeys.filter((key) => !Object.hasOwn(value, key)); + const unknown = actualKeys.filter((key) => !expected.has(key)); + if (missing.length || unknown.length) { + const details = []; + if (missing.length) details.push(`missing ${missing.join(", ")}`); + if (unknown.length) details.push(`unknown ${unknown.join(", ")}`); + fail(location, details.join("; ")); + } +} + +function requireNonEmptyString(value, location) { + if (typeof value !== "string" || value.trim().length === 0) { + fail(location, "expected a non-empty string"); + } + if (value.includes("\0")) { + fail(location, "must not contain a NUL byte"); + } + return value; +} + +function requireArray(value, location, { nonEmpty = false } = {}) { + if (!Array.isArray(value)) { + fail(location, "expected an array"); + } + if (nonEmpty && value.length === 0) { + fail(location, "must not be empty"); + } + return value; +} + +function requireNonNegativeNumber(value, location, { integer = false } = {}) { + if (typeof value !== "number" || !Number.isFinite(value)) { + fail(location, "expected a finite number"); + } + if (value < 0) { + fail(location, "must not be negative"); + } + if (integer && !Number.isSafeInteger(value)) { + fail(location, "expected a safe integer count"); + } + return value; +} + +function requirePositiveInteger(value, location) { + requireNonNegativeNumber(value, location, { integer: true }); + if (value < 1) { + fail(location, "expected an integer greater than zero"); + } + return value; +} + +function looksLikeNetworkOrAbsolutePath(value) { + return ( + path.posix.isAbsolute(value) || + path.win32.isAbsolute(value) || + value.startsWith("~") || + /^[A-Za-z][A-Za-z\d+.-]*:/u.test(value) + ); +} + +function requireRepoRelativePath(value, location) { + requireNonEmptyString(value, location); + if (value.includes("\\")) { + fail(location, "must use forward slashes"); + } + if (looksLikeNetworkOrAbsolutePath(value)) { + fail(location, "must be a local repo-relative path"); + } + const segments = value.split("/"); + if (segments.some((segment) => segment === "" || segment === "." || segment === "..")) { + fail(location, "must be normalized and confined to the repository"); + } + return value; +} + +function requireSafeCommandArgument(value, location) { + requireNonEmptyString(value, location); + const optionValue = value.startsWith("--") && value.includes("=") ? value.slice(value.indexOf("=") + 1) : value; + if (looksLikeNetworkOrAbsolutePath(optionValue)) { + fail(location, "must not contain an absolute path or network URL"); + } + return value; +} + +function requireUniqueStrings(value, location, { paths = false, nonEmpty = false } = {}) { + requireArray(value, location, { nonEmpty }); + const seen = new Set(); + value.forEach((item, index) => { + const itemLocation = `${location}[${index}]`; + if (paths) requireRepoRelativePath(item, itemLocation); + else requireNonEmptyString(item, itemLocation); + if (seen.has(item)) { + fail(itemLocation, `duplicate value ${JSON.stringify(item)}`); + } + seen.add(item); + }); + return value; +} + +function validateBaselineStep(step, location) { + requireExactKeys(step, ["type", "path"], location); + if (step.type !== "read") { + fail(`${location}.type`, 'expected "read"'); + } + requireRepoRelativePath(step.path, `${location}.path`); +} + +function validateCodegraphStep(step, location) { + requireExactKeys(step, ["type", "command", "query"], location); + if (step.type !== "codegraph") { + fail(`${location}.type`, 'expected "codegraph"'); + } + if (step.command !== "explore") { + fail(`${location}.command`, 'expected "explore"'); + } + requireNonEmptyString(step.query, `${location}.query`); +} + +export function validateScenarioFile(value) { + requireExactKeys(value, SCENARIO_DOCUMENT_KEYS, "scenario file"); + if (value.schemaVersion !== 1) { + fail("scenario file.schemaVersion", "expected 1"); + } + requireArray(value.scenarios, "scenario file.scenarios", { nonEmpty: true }); + + const ids = new Set(); + value.scenarios.forEach((scenario, index) => { + const location = `scenario file.scenarios[${index}]`; + requireExactKeys(scenario, SCENARIO_KEYS, location); + requireNonEmptyString(scenario.id, `${location}.id`); + if (ids.has(scenario.id)) { + fail(`${location}.id`, `duplicate scenario id ${JSON.stringify(scenario.id)}`); + } + ids.add(scenario.id); + requireRepoRelativePath(scenario.repo, `${location}.repo`); + requireNonEmptyString(scenario.task, `${location}.task`); + requireUniqueStrings(scenario.expectedAnchors, `${location}.expectedAnchors`, { + paths: true, + nonEmpty: true, + }); + requireArray(scenario.metrics, `${location}.metrics`); + if ( + scenario.metrics.length !== METRIC_KEYS.length || + scenario.metrics.some((metric, metricIndex) => metric !== METRIC_KEYS[metricIndex]) + ) { + fail(`${location}.metrics`, `expected exactly ${METRIC_KEYS.join(", ")} in that order`); + } + + requireExactKeys(scenario.variants, VARIANT_KEYS, `${location}.variants`); + requireArray(scenario.variants.baseline, `${location}.variants.baseline`, { nonEmpty: true }); + scenario.variants.baseline.forEach((step, stepIndex) => + validateBaselineStep(step, `${location}.variants.baseline[${stepIndex}]`), + ); + requireArray(scenario.variants.codegraph, `${location}.variants.codegraph`, { nonEmpty: true }); + scenario.variants.codegraph.forEach((step, stepIndex) => + validateCodegraphStep(step, `${location}.variants.codegraph[${stepIndex}]`), + ); + }); + + return value; +} + +function scenarioLookupFromOptions(options) { + const scenarioFile = options?.scenarioFile ?? options?.scenarios ?? null; + if (scenarioFile === null || scenarioFile === undefined) { + return null; + } + validateScenarioFile(scenarioFile); + return new Map( + scenarioFile.scenarios.map((scenario, index) => [ + scenario.id, + { + expectedAnchorCount: scenario.expectedAnchors.length, + expectedAnchors: new Set(scenario.expectedAnchors), + variantStepCounts: { + baseline: scenario.variants.baseline.length, + codegraph: scenario.variants.codegraph.length, + }, + baselineReadCount: scenario.variants.baseline.filter((step) => step.type === "read").length, + scenario, + index, + }, + ]), + ); +} + +function validateRun(run, index) { + const location = `results.runs[${index}]`; + requireExactKeys(run, RUN_KEYS, location); + requireNonEmptyString(run.scenarioId, `${location}.scenarioId`); + if (!VARIANT_ORDER.has(run.variant)) { + fail(`${location}.variant`, 'expected "baseline" or "codegraph"'); + } + requirePositiveInteger(run.run, `${location}.run`); + + requireExactKeys(run.metrics, METRIC_KEYS, `${location}.metrics`); + requireNonNegativeNumber(run.metrics.toolCalls, `${location}.metrics.toolCalls`, { integer: true }); + requireNonNegativeNumber(run.metrics.fileReads, `${location}.metrics.fileReads`, { integer: true }); + requireNonNegativeNumber(run.metrics.wallTimeMs, `${location}.metrics.wallTimeMs`); + + requireExactKeys(run.checks, CHECK_KEYS, `${location}.checks`); + requirePositiveInteger(run.checks.anchorsExpected, `${location}.checks.anchorsExpected`); + requireNonNegativeNumber(run.checks.anchorsFound, `${location}.checks.anchorsFound`, { + integer: true, + }); + if (run.checks.anchorsFound > run.checks.anchorsExpected) { + fail(`${location}.checks.anchorsFound`, "must not exceed anchorsExpected"); + } + requireUniqueStrings(run.checks.missingAnchors, `${location}.checks.missingAnchors`, { + paths: true, + }); + const expectedMissing = run.checks.anchorsExpected - run.checks.anchorsFound; + if (run.checks.missingAnchors.length !== expectedMissing) { + fail( + `${location}.checks`, + `anchorsFound (${run.checks.anchorsFound}) plus missingAnchors (${run.checks.missingAnchors.length}) must equal anchorsExpected (${run.checks.anchorsExpected})`, + ); + } + requireNonNegativeNumber(run.checks.completeness, `${location}.checks.completeness`); + if (run.checks.completeness > 1) { + fail(`${location}.checks.completeness`, "must be between 0 and 1"); + } + const expectedCompleteness = run.checks.anchorsFound / run.checks.anchorsExpected; + if (run.checks.completeness !== expectedCompleteness) { + fail( + `${location}.checks.completeness`, + `expected ${expectedCompleteness} from anchorsFound / anchorsExpected, received ${run.checks.completeness}`, + ); + } +} + +export function validateResults(value, options = {}) { + requireExactKeys(value, RESULT_KEYS, "results"); + if (value.schemaVersion !== 1) { + fail("results.schemaVersion", "expected 1"); + } + requireNonEmptyString(value.generatedAt, "results.generatedAt"); + const generatedDate = new Date(value.generatedAt); + if (Number.isNaN(generatedDate.getTime()) || generatedDate.toISOString() !== value.generatedAt) { + fail("results.generatedAt", "expected an ISO 8601 UTC timestamp"); + } + + requireArray(value.command, "results.command", { nonEmpty: true }); + value.command.forEach((argument, index) => requireSafeCommandArgument(argument, `results.command[${index}]`)); + requireExactKeys(value.environment, ENVIRONMENT_KEYS, "results.environment"); + for (const key of ["nodeVersion", "platform", "arch", "cpuModel"]) { + requireNonEmptyString(value.environment[key], `results.environment.${key}`); + } + requirePositiveInteger(value.environment.logicalCpus, "results.environment.logicalCpus"); + requirePositiveInteger(value.environment.totalMemoryBytes, "results.environment.totalMemoryBytes"); + requireRepoRelativePath(value.scenarioFile, "results.scenarioFile"); + requireNonEmptyString(value.scenarioDigest, "results.scenarioDigest"); + if (!/^sha256:[0-9a-f]{64}$/u.test(value.scenarioDigest)) { + fail("results.scenarioDigest", 'expected "sha256:" followed by 64 lowercase hexadecimal characters'); + } + requireUniqueStrings(value.scenarioIds, "results.scenarioIds", { nonEmpty: true }); + requirePositiveInteger(value.runsPerVariant, "results.runsPerVariant"); + requireArray(value.runs, "results.runs", { nonEmpty: true }); + + const scenarioLookup = scenarioLookupFromOptions(options); + if (scenarioLookup) { + value.scenarioIds.forEach((scenarioId, index) => { + if (!scenarioLookup.has(scenarioId)) { + fail(`results.scenarioIds[${index}]`, `unknown scenario id ${JSON.stringify(scenarioId)}`); + } + }); + const selectedIds = new Set(value.scenarioIds); + const selectedScenarios = [...scenarioLookup.values()] + .sort((left, right) => left.index - right.index) + .filter(({ scenario }) => selectedIds.has(scenario.id)) + .map(({ scenario }) => scenario); + const declaredOrder = selectedScenarios.map((scenario) => scenario.id); + const outOfOrderIndex = value.scenarioIds.findIndex((scenarioId, index) => scenarioId !== declaredOrder[index]); + if (outOfOrderIndex !== -1) { + fail( + `results.scenarioIds[${outOfOrderIndex}]`, + `must follow scenario-file order; expected ${JSON.stringify(declaredOrder[outOfOrderIndex])}, received ${JSON.stringify(value.scenarioIds[outOfOrderIndex])}`, + ); + } + const scenarioFile = options?.scenarioFile ?? options?.scenarios; + const expectedDigest = calculateScenarioDigest(scenarioFile.schemaVersion, selectedScenarios); + if (value.scenarioDigest !== expectedDigest) { + fail( + "results.scenarioDigest", + `does not match the selected scenario definitions; expected ${expectedDigest}, received ${value.scenarioDigest}`, + ); + } + } + + const selectedIds = new Set(value.scenarioIds); + const runKeys = new Set(); + const runNumbersByScenarioVariant = new Map(); + const expectedByScenario = new Map(); + + value.runs.forEach((run, index) => { + validateRun(run, index); + const location = `results.runs[${index}]`; + if (!selectedIds.has(run.scenarioId)) { + fail(`${location}.scenarioId`, `is not listed in results.scenarioIds`); + } + if (run.run > value.runsPerVariant) { + fail( + `${location}.run`, + `must be between 1 and results.runsPerVariant (${value.runsPerVariant}); received ${run.run}`, + ); + } + const runKey = `${run.scenarioId}\0${run.variant}\0${run.run}`; + if (runKeys.has(runKey)) { + fail(location, `duplicate scenario+variant+run key ${run.scenarioId}/${run.variant}/${run.run}`); + } + runKeys.add(runKey); + const scenarioVariantKey = `${run.scenarioId}\0${run.variant}`; + const runNumbers = runNumbersByScenarioVariant.get(scenarioVariantKey) ?? []; + runNumbers.push(run.run); + runNumbersByScenarioVariant.set(scenarioVariantKey, runNumbers); + + const priorExpected = expectedByScenario.get(run.scenarioId); + if (priorExpected !== undefined && priorExpected !== run.checks.anchorsExpected) { + fail( + `${location}.checks.anchorsExpected`, + `inconsistent total for scenario ${JSON.stringify(run.scenarioId)}; expected ${priorExpected}, received ${run.checks.anchorsExpected}`, + ); + } + expectedByScenario.set(run.scenarioId, run.checks.anchorsExpected); + + if (scenarioLookup) { + const scenario = scenarioLookup.get(run.scenarioId); + if (scenario.expectedAnchorCount !== run.checks.anchorsExpected) { + fail( + `${location}.checks.anchorsExpected`, + `expected ${scenario.expectedAnchorCount} from the scenario file, received ${run.checks.anchorsExpected}`, + ); + } + const expectedToolCalls = scenario.variantStepCounts[run.variant]; + if (run.metrics.toolCalls !== expectedToolCalls) { + fail( + `${location}.metrics.toolCalls`, + `expected ${expectedToolCalls} from scenario ${JSON.stringify(run.scenarioId)} variant ${JSON.stringify(run.variant)}, received ${run.metrics.toolCalls}`, + ); + } + if (run.variant === "baseline" && run.metrics.fileReads !== scenario.baselineReadCount) { + fail( + `${location}.metrics.fileReads`, + `expected ${scenario.baselineReadCount} from the declared baseline read steps, received ${run.metrics.fileReads}`, + ); + } + run.checks.missingAnchors.forEach((anchor, anchorIndex) => { + if (!scenario.expectedAnchors.has(anchor)) { + fail( + `${location}.checks.missingAnchors[${anchorIndex}]`, + `anchor ${JSON.stringify(anchor)} is not an expected anchor for scenario ${JSON.stringify(run.scenarioId)}`, + ); + } + }); + } + }); + + for (const scenarioId of value.scenarioIds) { + for (const variant of VARIANT_KEYS) { + const scenarioVariantKey = `${scenarioId}\0${variant}`; + const runNumbers = (runNumbersByScenarioVariant.get(scenarioVariantKey) ?? []).sort( + (left, right) => left - right, + ); + if (runNumbers.length === value.runsPerVariant) continue; + let missingRunNumber = 1; + for (const runNumber of runNumbers) { + if (runNumber !== missingRunNumber) break; + missingRunNumber += 1; + } + fail( + "results.runs", + `missing required run tuple ${scenarioId}/${variant}/${missingRunNumber} for results.runsPerVariant=${value.runsPerVariant}`, + ); + } + } + + return value; +} + +export function median(values) { + requireArray(values, "median values", { nonEmpty: true }); + const sorted = values + .map((value, index) => { + if (typeof value !== "number" || !Number.isFinite(value)) { + fail(`median values[${index}]`, "expected a finite number"); + } + if (value < 0) { + fail(`median values[${index}]`, "must not be negative"); + } + return value; + }) + .sort((left, right) => left - right); + const middle = Math.floor(sorted.length / 2); + if (sorted.length % 2 === 1) { + return sorted[middle]; + } + return sorted[middle - 1] / 2 + sorted[middle] / 2; +} + +function requireScenarioOrder(value, selectedScenarioIds) { + if (value === undefined || value === null) return selectedScenarioIds; + const order = requireUniqueStrings(value, "scenarioOrder", { nonEmpty: true }); + const selected = new Set(selectedScenarioIds); + order.forEach((scenarioId, index) => { + if (!selected.has(scenarioId)) { + fail(`scenarioOrder[${index}]`, `is not listed in results.scenarioIds`); + } + }); + const ordered = new Set(order); + const missing = selectedScenarioIds.filter((scenarioId) => !ordered.has(scenarioId)); + if (missing.length) { + fail("scenarioOrder", `missing selected scenario id${missing.length === 1 ? "" : "s"} ${missing.join(", ")}`); + } + return order; +} + +function summaryOrder(results, options, hasScenarioFile) { + if (hasScenarioFile) return results.scenarioIds; + return requireScenarioOrder(options.scenarioOrder, results.scenarioIds); +} + +export function summarizeResults(results, options = {}) { + validateResults(results, options); + const hasScenarioFile = (options?.scenarioFile ?? options?.scenarios ?? null) !== null; + const requestedOrder = summaryOrder(results, options, hasScenarioFile); + const orderIndex = new Map(requestedOrder.map((scenarioId, index) => [scenarioId, index])); + const groups = new Map(); + + for (const run of results.runs) { + const key = `${run.scenarioId}\0${run.variant}`; + const group = groups.get(key) ?? { + scenarioId: run.scenarioId, + variant: run.variant, + runs: [], + }; + group.runs.push(run); + groups.set(key, group); + } + + return [...groups.values()] + .sort((left, right) => { + const leftIndex = orderIndex.get(left.scenarioId); + const rightIndex = orderIndex.get(right.scenarioId); + if (leftIndex !== undefined || rightIndex !== undefined) { + if (leftIndex === undefined) return 1; + if (rightIndex === undefined) return -1; + if (leftIndex !== rightIndex) return leftIndex - rightIndex; + } + if (left.scenarioId < right.scenarioId) return -1; + if (left.scenarioId > right.scenarioId) return 1; + return VARIANT_ORDER.get(left.variant) - VARIANT_ORDER.get(right.variant); + }) + .map((group) => ({ + scenarioId: group.scenarioId, + variant: group.variant, + sampleCount: group.runs.length, + medians: { + toolCalls: median(group.runs.map((run) => run.metrics.toolCalls)), + fileReads: median(group.runs.map((run) => run.metrics.fileReads)), + wallTimeMs: median(group.runs.map((run) => run.metrics.wallTimeMs)), + }, + completeRunCount: group.runs.filter((run) => run.checks.completeness === 1).length, + minimumCompleteness: group.runs.reduce((minimum, run) => Math.min(minimum, run.checks.completeness), 1), + })); +} + +export function escapeMarkdown(value) { + return String(value) + .replaceAll("\\", "\\\\") + .replaceAll("|", "\\|") + .replace(/([`*_[\]{}<>])/gu, "\\$1") + .replace(/\r\n|\r|\n/gu, "
"); +} + +function formatNumber(value) { + if (Object.is(value, -0)) return "0"; + return String(value); +} + +export function renderMarkdownTable(summaries) { + requireArray(summaries, "summaries", { nonEmpty: true }); + const headers = [ + "Scenario", + "Variant", + "Samples", + "Median tool calls", + "Median file reads", + "Median wall time (ms)", + "Complete runs", + "Minimum completeness", + ]; + const rows = summaries.map((summary, index) => { + const location = `summaries[${index}]`; + requireExactKeys( + summary, + ["scenarioId", "variant", "sampleCount", "medians", "completeRunCount", "minimumCompleteness"], + location, + ); + requireNonEmptyString(summary.scenarioId, `${location}.scenarioId`); + if (!VARIANT_ORDER.has(summary.variant)) { + fail(`${location}.variant`, 'expected "baseline" or "codegraph"'); + } + requirePositiveInteger(summary.sampleCount, `${location}.sampleCount`); + requireExactKeys(summary.medians, METRIC_KEYS, `${location}.medians`); + for (const key of METRIC_KEYS) { + requireNonNegativeNumber(summary.medians[key], `${location}.medians.${key}`); + } + requireNonNegativeNumber(summary.completeRunCount, `${location}.completeRunCount`, { integer: true }); + if (summary.completeRunCount > summary.sampleCount) { + fail(`${location}.completeRunCount`, "must not exceed sampleCount"); + } + requireNonNegativeNumber(summary.minimumCompleteness, `${location}.minimumCompleteness`); + if (summary.minimumCompleteness > 1) { + fail(`${location}.minimumCompleteness`, "must be between 0 and 1"); + } + + return [ + escapeMarkdown(summary.scenarioId), + escapeMarkdown(summary.variant), + String(summary.sampleCount), + formatNumber(summary.medians.toolCalls), + formatNumber(summary.medians.fileReads), + formatNumber(summary.medians.wallTimeMs), + String(summary.completeRunCount), + `${formatNumber(summary.minimumCompleteness * 100)}%`, + ]; + }); + const widths = headers.map((header, column) => + rows.reduce((width, row) => Math.max(width, row[column].length), header.length), + ); + const numericColumns = new Set([2, 3, 4, 5, 6, 7]); + const formatRow = (cells) => + `| ${cells + .map((cell, column) => (numericColumns.has(column) ? cell.padStart(widths[column]) : cell.padEnd(widths[column]))) + .join(" | ")} |`; + const separator = widths.map((width, column) => { + if (numericColumns.has(column)) return `${"-".repeat(width - 1)}:`; + return "-".repeat(width); + }); + return `${[formatRow(headers), formatRow(separator), ...rows.map(formatRow)].join("\n")}\n`; +} + +function generatedBlockBounds(markdown) { + if (typeof markdown !== "string") { + fail("README", "expected Markdown text"); + } + const start = markdown.indexOf(README_START_MARKER); + const end = markdown.indexOf(README_END_MARKER); + if (start === -1 || end === -1) { + fail("README", `expected exactly one ${README_START_MARKER} and one ${README_END_MARKER}`); + } + if (start !== markdown.lastIndexOf(README_START_MARKER) || end !== markdown.lastIndexOf(README_END_MARKER)) { + fail("README", "generated benchmark markers must each appear exactly once"); + } + const contentStart = start + README_START_MARKER.length; + if (end < contentStart) { + fail("README", "benchmark-results:end marker appears before benchmark-results:start"); + } + return { contentStart, end }; +} + +export function replaceGeneratedBlock(markdown, generatedContent) { + const { contentStart, end } = generatedBlockBounds(markdown); + if (typeof generatedContent !== "string" || generatedContent.trim().length === 0) { + fail("generated content", "expected non-empty text"); + } + if (generatedContent.includes(README_START_MARKER) || generatedContent.includes(README_END_MARKER)) { + fail("generated content", "must not contain benchmark result markers"); + } + const newline = markdown.includes("\r\n") ? "\r\n" : "\n"; + const normalizedContent = generatedContent.trim().replace(/\r\n|\r|\n/gu, newline); + return `${markdown.slice(0, contentStart)}${newline}${newline}${normalizedContent}${newline}${newline}${markdown.slice(end)}`; +} + +export function checkGeneratedBlock(markdown, generatedContent) { + return markdown === replaceGeneratedBlock(markdown, generatedContent); +} + +function consumeValue(argv, index, flag) { + const value = argv[index + 1]; + if (value === undefined || value.startsWith("--")) { + fail(flag, "expected a value"); + } + return value; +} + +export function parseArgs(argv) { + if (!Array.isArray(argv)) { + fail("CLI arguments", "expected an array"); + } + const options = { + input: null, + scenarioFile: null, + json: false, + readme: null, + write: false, + check: false, + }; + const seen = new Set(); + const valueFlags = new Map([ + ["--input", "input"], + ["--scenario-file", "scenarioFile"], + ["--readme", "readme"], + ]); + const booleanFlags = new Map([ + ["--json", "json"], + ["--write", "write"], + ["--check", "check"], + ]); + + for (let index = 0; index < argv.length; index += 1) { + const argument = argv[index]; + requireNonEmptyString(argument, `CLI arguments[${index}]`); + const equalsIndex = argument.indexOf("="); + const flag = equalsIndex === -1 ? argument : argument.slice(0, equalsIndex); + if (valueFlags.has(flag)) { + if (seen.has(flag)) fail(flag, "specified more than once"); + seen.add(flag); + const value = equalsIndex === -1 ? consumeValue(argv, index, flag) : argument.slice(equalsIndex + 1); + requireNonEmptyString(value, flag); + options[valueFlags.get(flag)] = value; + if (equalsIndex === -1) index += 1; + continue; + } + if (booleanFlags.has(flag)) { + if (equalsIndex !== -1) fail(flag, "does not take a value"); + if (seen.has(flag)) fail(flag, "specified more than once"); + seen.add(flag); + options[booleanFlags.get(flag)] = true; + continue; + } + fail("CLI arguments", `unknown argument ${JSON.stringify(argument)}`); + } + + if (options.input === null) fail("--input", "is required"); + if (options.write && options.check) fail("CLI arguments", "--write and --check are mutually exclusive"); + if ((options.write || options.check) && options.readme === null) { + fail("CLI arguments", "--write and --check require --readme"); + } + if (options.readme !== null && !options.write && !options.check) { + fail("CLI arguments", "--readme requires either --write or --check"); + } + + return options; +} + +function parseJsonFile(filePath, label) { + let text; + try { + text = fs.readFileSync(filePath, "utf8"); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + fail(label, `cannot read ${JSON.stringify(filePath)}: ${message}`); + } + try { + return JSON.parse(text); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + fail(label, `invalid JSON in ${JSON.stringify(filePath)}: ${message}`); + } +} + +export function runCli(argv, io = {}) { + const options = parseArgs(argv); + const scenarioFile = options.scenarioFile + ? validateScenarioFile(parseJsonFile(options.scenarioFile, "--scenario-file")) + : null; + const results = parseJsonFile(options.input, "--input"); + const summaries = summarizeResults(results, scenarioFile ? { scenarioFile } : {}); + const table = renderMarkdownTable(summaries); + + if (options.readme) { + let readme; + try { + readme = fs.readFileSync(options.readme, "utf8"); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + fail("--readme", `cannot read ${JSON.stringify(options.readme)}: ${message}`); + } + if (options.check) { + if (!checkGeneratedBlock(readme, table)) { + fail( + "--check", + `generated benchmark block in ${JSON.stringify(options.readme)} is out of date; rerun with --write`, + ); + } + } else { + const updated = replaceGeneratedBlock(readme, table); + try { + fs.writeFileSync(options.readme, updated, "utf8"); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + fail("--write", `cannot write ${JSON.stringify(options.readme)}: ${message}`); + } + } + } + + const output = options.json ? `${JSON.stringify(summaries, null, 2)}\n` : table; + const stdout = io.stdout ?? process.stdout; + if (typeof stdout === "function") stdout(output); + else if (stdout && typeof stdout.write === "function") stdout.write(output); + else fail("CLI output", "stdout must be a function or writable stream"); + return { summaries, table }; +} diff --git a/scripts/benchmarks/summarize-results.mjs b/scripts/benchmarks/summarize-results.mjs new file mode 100644 index 00000000..dfd8181d --- /dev/null +++ b/scripts/benchmarks/summarize-results.mjs @@ -0,0 +1,19 @@ +#!/usr/bin/env node +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { runCli } from "./summarize-results-lib.mjs"; + +export function main(argv = process.argv.slice(2), runtime = {}) { + const stderr = runtime.stderr ?? process.stderr; + try { + runCli(argv, { stdout: runtime.stdout ?? process.stdout }); + return 0; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + stderr.write(`${message}\n`); + return 1; + } +} + +const isMain = process.argv[1] !== undefined && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url); +if (isMain) process.exitCode = main(); diff --git a/src/cli/help.ts b/src/cli/help.ts index f8f83bca..4d395d19 100644 --- a/src/cli/help.ts +++ b/src/cli/help.ts @@ -42,10 +42,9 @@ Commands: version Print the installed codegraph version Graph Options: - --fast-graph Skip AST parsing, use regex for imports. - 5-10x faster but may miss dynamic imports, - re-exports, and complex patterns. Best for - quick overviews of large codebases. + --fast-graph Use text import extraction for plain .js and .ts + files. May miss multiline or complex patterns; + TSX and other languages keep normal extraction. --resolve-node-modules Include node_modules in resolution --dynamic-import-heuristics Attempt to resolve dynamic imports --resolution-hint Custom resolution hint (e.g., tsconfig:path) diff --git a/tests/public-docs-benchmarks.test.ts b/tests/public-docs-benchmarks.test.ts new file mode 100644 index 00000000..8d3cc457 --- /dev/null +++ b/tests/public-docs-benchmarks.test.ts @@ -0,0 +1,1243 @@ +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { + collectSourceFiles, + loadScenarioFile, + parseArguments, + runBenchmark, + runScenario, + serializeBenchmarkResult, + validateScenarioDocument, +} from "../scripts/benchmarks/run-scenario-lib.mjs"; +import { calculateScenarioDigest } from "../scripts/benchmarks/benchmark-contract-lib.mjs"; +import { + checkGeneratedBlock, + median, + renderMarkdownTable, + runCli as runSummarizerCli, + summarizeResults, + validateResults, +} from "../scripts/benchmarks/summarize-results-lib.mjs"; + +const rootDir = path.resolve(import.meta.dirname, ".."); +const scenarioFilePath = path.join(rootDir, "docs", "benchmarks", "scenarios.json"); +const resultsFilePath = path.join(rootDir, "docs", "benchmarks", "results.example.json"); +const readmePath = path.join(rootDir, "docs", "benchmarks", "README.md"); +const runnerPath = path.join(rootDir, "scripts", "benchmarks", "run-scenario.mjs"); +const metrics = ["toolCalls", "fileReads", "wallTimeMs"]; +const scenarioIds = [ + "repo-orientation-small-ts", + "python-import-reference", + "sql-migration-application-review", + "mixed-docs-source-graph", +]; + +interface BaselineStep { + type: string; + path: string; +} + +interface CodegraphStep { + type: string; + command: string; + query: string; +} + +interface Scenario { + id: string; + repo: string; + task: string; + expectedAnchors: string[]; + metrics: string[]; + variants: { + baseline: BaselineStep[]; + codegraph: CodegraphStep[]; + }; +} + +interface ScenarioDocument { + schemaVersion: number; + scenarios: Scenario[]; +} + +type Variant = "baseline" | "codegraph"; + +interface BenchmarkRun { + scenarioId: string; + variant: Variant; + run: number; + metrics: { + toolCalls: number; + fileReads: number; + wallTimeMs: number; + }; + checks: { + anchorsExpected: number; + anchorsFound: number; + missingAnchors: string[]; + completeness: number; + }; +} + +interface BenchmarkResults { + schemaVersion: number; + generatedAt: string; + command: string[]; + environment: { + nodeVersion: string; + platform: string; + arch: string; + cpuModel: string; + logicalCpus: number; + totalMemoryBytes: number; + }; + scenarioFile: string; + scenarioDigest: string; + scenarioIds: string[]; + runsPerVariant: number; + runs: BenchmarkRun[]; +} + +interface RunInput { + scenarioId?: string; + variant?: Variant; + run?: number; + toolCalls?: number; + fileReads?: number; + wallTimeMs?: number; + anchorsExpected?: number; + anchorsFound?: number; + missingAnchors?: string[]; + completeness?: number; +} + +interface ResultsInput { + scenarioDigest?: string; + scenarioIds?: string[]; + runsPerVariant?: number; +} + +const validScenarioDigest = `sha256:${"0".repeat(64)}`; + +const tempRoots: string[] = []; + +afterEach(() => { + for (const tempRoot of tempRoots.splice(0)) { + fs.rmSync(tempRoot, { recursive: true, force: true }); + } +}); + +function createTempRoot(): string { + const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), "codegraph-public-docs-benchmarks-")); + tempRoots.push(tempRoot); + return tempRoot; +} + +function clone(value: T): T { + return structuredClone(value); +} + +function createScenarioFixture(tempRoot: string): ScenarioDocument { + const fixtureRoot = path.join(tempRoot, "fixture", "src"); + fs.mkdirSync(fixtureRoot, { recursive: true }); + fs.writeFileSync(path.join(fixtureRoot, "actual.ts"), "export const actual = true;\n", "utf8"); + fs.writeFileSync(path.join(fixtureRoot, "anchor.ts"), "export const anchor = true;\n", "utf8"); + return { + schemaVersion: 1, + scenarios: [ + { + id: "fixture-scenario", + repo: "fixture", + task: "Trace the fixture files.", + expectedAnchors: ["src/actual.ts", "src/anchor.ts"], + metrics: [...metrics], + variants: { + baseline: [ + { type: "read", path: "src/actual.ts" }, + { type: "read", path: "src/anchor.ts" }, + ], + codegraph: [{ type: "codegraph", command: "explore", query: "Trace actual to anchor." }], + }, + }, + ], + }; +} + +function addSecondaryScenario(document: ScenarioDocument): ScenarioDocument { + document.scenarios.push({ + id: "fixture-secondary", + repo: "fixture", + task: "Trace the secondary fixture scenario.", + expectedAnchors: ["src/actual.ts", "src/anchor.ts"], + metrics: [...metrics], + variants: { + baseline: [{ type: "read", path: "src/actual.ts" }], + codegraph: [ + { type: "codegraph", command: "explore", query: "Find the actual fixture file." }, + { type: "codegraph", command: "explore", query: "Find the anchor fixture file." }, + ], + }, + }); + return document; +} + +function makeRun(input: RunInput = {}): BenchmarkRun { + const anchorsExpected = input.anchorsExpected ?? 2; + const anchorsFound = input.anchorsFound ?? anchorsExpected; + return { + scenarioId: input.scenarioId ?? "alpha", + variant: input.variant ?? "baseline", + run: input.run ?? 1, + metrics: { + toolCalls: input.toolCalls ?? 1, + fileReads: input.fileReads ?? 1, + wallTimeMs: input.wallTimeMs ?? 1, + }, + checks: { + anchorsExpected, + anchorsFound, + missingAnchors: input.missingAnchors ?? [], + completeness: input.completeness ?? anchorsFound / anchorsExpected, + }, + }; +} + +function makeResults(runs: BenchmarkRun[], input: ResultsInput = {}): BenchmarkResults { + const inferredScenarioIds = [...new Set(runs.map((run) => run.scenarioId))]; + const inferredRunsPerVariant = Math.max(...runs.map((run) => run.run)); + return { + schemaVersion: 1, + generatedAt: "2026-07-10T12:00:00.000Z", + command: ["node", "scripts/benchmarks/run-scenario.mjs"], + environment: { + nodeVersion: "v24.0.0", + platform: "linux", + arch: "x64", + cpuModel: "test cpu", + logicalCpus: 4, + totalMemoryBytes: 1024, + }, + scenarioFile: "docs/benchmarks/scenarios.json", + scenarioDigest: input.scenarioDigest ?? validScenarioDigest, + scenarioIds: input.scenarioIds ?? inferredScenarioIds, + runsPerVariant: input.runsPerVariant ?? inferredRunsPerVariant, + runs, + }; +} + +function makeScenarioResults( + document: ScenarioDocument, + selectedScenarioIds = document.scenarios.map((scenario) => scenario.id), + runsPerVariant = 2, +): BenchmarkResults { + const selectedIds = new Set(selectedScenarioIds); + const selectedScenarios = document.scenarios.filter((scenario) => selectedIds.has(scenario.id)); + const runs: BenchmarkRun[] = []; + for (const scenario of selectedScenarios) { + for (const variant of ["baseline", "codegraph"] as const) { + for (let run = 1; run <= runsPerVariant; run += 1) { + runs.push( + makeRun({ + scenarioId: scenario.id, + variant, + run, + toolCalls: scenario.variants[variant].length, + fileReads: variant === "baseline" ? scenario.variants.baseline.length : scenario.expectedAnchors.length, + anchorsExpected: scenario.expectedAnchors.length, + }), + ); + } + } + } + return makeResults(runs, { + scenarioDigest: calculateScenarioDigest(document.schemaVersion, selectedScenarios), + scenarioIds: selectedScenarios.map((scenario) => scenario.id), + runsPerVariant, + }); +} + +function completeVariantPair(scenarioId = "alpha"): BenchmarkResults { + return makeResults([makeRun({ scenarioId, variant: "baseline" }), makeRun({ scenarioId, variant: "codegraph" })]); +} + +function findRun(results: BenchmarkResults, scenarioId: string, variant: Variant, run: number): BenchmarkRun { + const match = results.runs.find( + (candidate) => candidate.scenarioId === scenarioId && candidate.variant === variant && candidate.run === run, + ); + if (!match) throw new Error(`Missing test run ${scenarioId}/${variant}/${run}.`); + return match; +} + +describe("public documentation benchmark scenarios", () => { + it("loads exactly the four checked local scenarios with canonical metrics, variants, steps, and existing fixtures", () => { + const document = loadScenarioFile("docs/benchmarks/scenarios.json", { rootDir }) as ScenarioDocument; + + expect(document.schemaVersion).toBe(1); + expect(document.scenarios.map((scenario) => scenario.id)).toEqual(scenarioIds); + for (const scenario of document.scenarios) { + expect(scenario.metrics).toEqual(metrics); + expect(Object.keys(scenario.variants)).toEqual(["baseline", "codegraph"]); + expect(scenario.variants.baseline.length).toBeGreaterThan(0); + expect(scenario.variants.codegraph.length).toBeGreaterThan(0); + expect(scenario.variants.baseline.every((step) => step.type === "read")).toBe(true); + expect(scenario.variants.codegraph.every((step) => step.type === "codegraph" && step.command === "explore")).toBe( + true, + ); + expect(scenario.repo).not.toMatch(/^(?:[a-z][a-z\d+.-]*:|[/\\~])/iu); + const fixtureRoot = path.join(rootDir, ...scenario.repo.split("/")); + expect(fs.statSync(fixtureRoot).isDirectory()).toBe(true); + for (const relativeFile of [ + ...scenario.expectedAnchors, + ...scenario.variants.baseline.map((step) => step.path), + ]) { + expect(fs.statSync(path.join(fixtureRoot, ...relativeFile.split("/"))).isFile()).toBe(true); + } + } + }); + + it("rejects malformed JSON, unsupported schema versions, and unknown fields at every schema layer", () => { + const tempRoot = createTempRoot(); + fs.writeFileSync(path.join(tempRoot, "broken.json"), "{ not json", "utf8"); + expect(() => loadScenarioFile("broken.json", { rootDir: tempRoot })).toThrow(/not valid JSON/); + + const mutations: Array<{ name: string; mutate: (document: ScenarioDocument) => void }> = [ + { + name: "unsupported schema version", + mutate: (document) => { + document.schemaVersion = 2; + }, + }, + { + name: "unknown document field", + mutate: (document) => { + Object.assign(document, { unexpected: true }); + }, + }, + { + name: "unknown scenario field", + mutate: (document) => { + Object.assign(document.scenarios[0], { unexpected: true }); + }, + }, + { + name: "unknown baseline step field", + mutate: (document) => { + Object.assign(document.scenarios[0].variants.baseline[0], { unexpected: true }); + }, + }, + { + name: "unknown codegraph step field", + mutate: (document) => { + Object.assign(document.scenarios[0].variants.codegraph[0], { unexpected: true }); + }, + }, + ]; + + for (const testCase of mutations) { + const document = createScenarioFixture(tempRoot); + testCase.mutate(document); + expect(() => validateScenarioDocument(document, { rootDir: tempRoot }), testCase.name).toThrow(); + } + }); + + it("rejects duplicate scenario IDs and duplicate expected anchors", () => { + const tempRoot = createTempRoot(); + const duplicateIdDocument = createScenarioFixture(tempRoot); + duplicateIdDocument.scenarios.push(clone(duplicateIdDocument.scenarios[0])); + expect(() => validateScenarioDocument(duplicateIdDocument, { rootDir: tempRoot })).toThrow(/duplicated/i); + + const duplicateAnchorDocument = createScenarioFixture(tempRoot); + duplicateAnchorDocument.scenarios[0].expectedAnchors.push("src/actual.ts"); + expect(() => validateScenarioDocument(duplicateAnchorDocument, { rootDir: tempRoot })).toThrow(/duplicate anchor/i); + }); + + it("rejects URL, absolute, and traversal paths for repos, reads, and anchors", () => { + const tempRoot = createTempRoot(); + const cases: Array<{ + name: string; + mutate: (scenario: Scenario) => void; + }> = [ + { + name: "URL repo", + mutate: (scenario) => { + scenario.repo = "https://example.test/repo"; + }, + }, + { + name: "absolute repo", + mutate: (scenario) => { + scenario.repo = "/outside/repo"; + }, + }, + { + name: "traversing repo", + mutate: (scenario) => { + scenario.repo = "../outside"; + }, + }, + { + name: "URL read", + mutate: (scenario) => { + scenario.variants.baseline[0].path = "https://example.test/file.ts"; + }, + }, + { + name: "absolute read", + mutate: (scenario) => { + scenario.variants.baseline[0].path = "/outside/file.ts"; + }, + }, + { + name: "traversing read", + mutate: (scenario) => { + scenario.variants.baseline[0].path = "../outside.ts"; + }, + }, + { + name: "URL anchor", + mutate: (scenario) => { + scenario.expectedAnchors[0] = "https://example.test/file.ts"; + }, + }, + { + name: "absolute anchor", + mutate: (scenario) => { + scenario.expectedAnchors[0] = "/outside/file.ts"; + }, + }, + { + name: "traversing anchor", + mutate: (scenario) => { + scenario.expectedAnchors[0] = "../outside.ts"; + }, + }, + ]; + + for (const testCase of cases) { + const document = createScenarioFixture(tempRoot); + testCase.mutate(document.scenarios[0]); + expect(() => validateScenarioDocument(document, { rootDir: tempRoot }), testCase.name).toThrow(); + } + }); + + it("rejects missing scenario repos, baseline reads, and expected-anchor files", () => { + const tempRoot = createTempRoot(); + const cases: Array<{ + name: string; + mutate: (scenario: Scenario) => void; + expected: RegExp; + }> = [ + { + name: "missing repo", + mutate: (scenario) => { + scenario.repo = "missing-repo"; + }, + expected: /repo.*does not exist|repo.*cannot be resolved/i, + }, + { + name: "missing read", + mutate: (scenario) => { + scenario.variants.baseline[0].path = "src/missing-read.ts"; + }, + expected: /baseline.*missing-read|missing-read.*does not exist|missing-read.*cannot be resolved/i, + }, + { + name: "missing anchor", + mutate: (scenario) => { + scenario.expectedAnchors[0] = "src/missing-anchor.ts"; + }, + expected: /anchor|missing-anchor/i, + }, + ]; + + for (const testCase of cases) { + const document = createScenarioFixture(tempRoot); + testCase.mutate(document.scenarios[0]); + expect(() => validateScenarioDocument(document, { rootDir: tempRoot }), testCase.name).toThrow(testCase.expected); + } + }); +}); + +describe("public documentation benchmark runner contracts", () => { + it("reports declared tool calls, unique delivered source files, elapsed time, completeness, and a portable result shape", async () => { + const tempRoot = createTempRoot(); + const scenarioDocument = createScenarioFixture(tempRoot); + const clock = [100, 112.3456, 200, 225.556]; + const environment = { + nodeVersion: "v24.0.0", + platform: "test-platform", + arch: "test-arch", + cpuModel: "test cpu", + logicalCpus: 2, + totalMemoryBytes: 4096, + }; + const result = await runBenchmark( + { + scenarioIds: ["fixture-scenario"], + runs: 1, + scenarioFile: "scenario.json", + requireComplete: true, + json: true, + }, + { + rootDir: tempRoot, + scenarioDocument, + environment, + date: () => new Date("2026-07-10T12:00:00.000Z"), + now: () => { + const value = clock.shift(); + if (value === undefined) throw new Error("test clock exhausted"); + return value; + }, + executeCodegraph: async () => ({ + anchors: ["src/actual.ts", "src/anchor.ts"], + fileView: { file: "src/actual.ts" }, + packets: [{ target: "src/actual.ts" }, { packet: { target: { file: "src/anchor.ts" } } }], + }), + }, + ); + + expect(result).toEqual({ + schemaVersion: 1, + generatedAt: "2026-07-10T12:00:00.000Z", + command: [ + "node", + "scripts/benchmarks/run-scenario.mjs", + "--scenario-file", + "scenario.json", + "--runs", + "1", + "--scenario", + "fixture-scenario", + "--require-complete", + "--json", + ], + environment, + scenarioFile: "scenario.json", + scenarioDigest: calculateScenarioDigest(scenarioDocument.schemaVersion, scenarioDocument.scenarios), + scenarioIds: ["fixture-scenario"], + runsPerVariant: 1, + runs: [ + { + scenarioId: "fixture-scenario", + variant: "baseline", + run: 1, + metrics: { toolCalls: 2, fileReads: 2, wallTimeMs: 12.346 }, + checks: { + anchorsExpected: 2, + anchorsFound: 2, + missingAnchors: [], + completeness: 1, + }, + }, + { + scenarioId: "fixture-scenario", + variant: "codegraph", + run: 1, + metrics: { toolCalls: 1, fileReads: 2, wallTimeMs: 25.556 }, + checks: { + anchorsExpected: 2, + anchorsFound: 2, + missingAnchors: [], + completeness: 1, + }, + }, + ], + }); + const serialized = serializeBenchmarkResult(result); + expect(JSON.parse(serialized)).toEqual(result); + expect(serialized.endsWith("\n")).toBe(true); + expect(serialized).not.toContain(tempRoot); + expect(serialized).not.toContain(JSON.stringify(tempRoot).slice(1, -1)); + }); + + it("binds the scenario digest to repo, task, query, and ordered baseline steps", () => { + const tempRoot = createTempRoot(); + const document = createScenarioFixture(tempRoot); + const originalDigest = calculateScenarioDigest(document.schemaVersion, document.scenarios); + const cases: Array<{ name: string; mutate: (scenario: Scenario) => void }> = [ + { + name: "repo", + mutate: (scenario) => { + scenario.repo = "a-different-fixture"; + }, + }, + { + name: "task", + mutate: (scenario) => { + scenario.task = "Trace the fixture in a different way."; + }, + }, + { + name: "codegraph query", + mutate: (scenario) => { + scenario.variants.codegraph[0].query = "Trace anchor to actual instead."; + }, + }, + { + name: "baseline step order", + mutate: (scenario) => { + scenario.variants.baseline.reverse(); + }, + }, + ]; + + for (const testCase of cases) { + const changed = clone(document); + testCase.mutate(changed.scenarios[0]); + expect(calculateScenarioDigest(changed.schemaVersion, changed.scenarios), testCase.name).not.toBe(originalDigest); + } + }); + + it("digests and validates only the selected scenario definitions in scenario-file order", async () => { + const tempRoot = createTempRoot(); + const document = addSecondaryScenario(createScenarioFixture(tempRoot)); + const dependencies = { + rootDir: tempRoot, + scenarioDocument: document, + date: () => new Date("2026-07-10T12:00:00.000Z"), + now: () => 0, + executeCodegraph: async () => ({ + anchors: ["src/actual.ts", "src/anchor.ts"], + packets: [{ target: "src/actual.ts" }, { target: "src/anchor.ts" }], + }), + }; + const result = await runBenchmark( + { scenarioIds: ["fixture-secondary"], runs: 1, scenarioFile: "scenario.json" }, + dependencies, + ); + const changedUnselected = clone(document); + changedUnselected.scenarios[0].task = "This unselected task changed."; + const resultAfterUnselectedChange = await runBenchmark( + { scenarioIds: ["fixture-secondary"], runs: 1, scenarioFile: "scenario.json" }, + { ...dependencies, scenarioDocument: changedUnselected }, + ); + + expect(result.scenarioIds).toEqual(["fixture-secondary"]); + expect(result.runsPerVariant).toBe(1); + expect(result.scenarioDigest).toBe(calculateScenarioDigest(1, [document.scenarios[1]])); + expect(resultAfterUnselectedChange.scenarioDigest).toBe(result.scenarioDigest); + expect( + summarizeResults(result, { scenarioFile: document }).map((summary: { scenarioId: string; variant: Variant }) => [ + summary.scenarioId, + summary.variant, + ]), + ).toEqual([ + ["fixture-secondary", "baseline"], + ["fixture-secondary", "codegraph"], + ]); + }); + + it("calculates completeness from captured step evidence rather than task or query metadata", async () => { + const tempRoot = createTempRoot(); + const fixtureRoot = path.join(tempRoot, "fixture", "src"); + fs.mkdirSync(fixtureRoot, { recursive: true }); + fs.writeFileSync(path.join(fixtureRoot, "actual.ts"), "export const actual = true;\n", "utf8"); + fs.writeFileSync(path.join(fixtureRoot, "metadata-only.ts"), "export const hidden = true;\n", "utf8"); + const metadataAnchor = "src/metadata-only.ts"; + const scenario: Scenario = { + id: "metadata-is-not-evidence", + repo: "fixture", + task: `Find ${metadataAnchor}`, + expectedAnchors: [metadataAnchor], + metrics: [...metrics], + variants: { + baseline: [{ type: "read", path: "src/actual.ts" }], + codegraph: [{ type: "codegraph", command: "explore", query: `Find ${metadataAnchor}` }], + }, + }; + + const runs = await runScenario(scenario, { + rootDir: tempRoot, + runs: 1, + now: () => 0, + executeCodegraph: async () => ({ anchors: [], packets: [] }), + }); + + expect(runs.map((run: BenchmarkRun) => run.checks)).toEqual([ + { + anchorsExpected: 1, + anchorsFound: 0, + missingAnchors: [metadataAnchor], + completeness: 0, + }, + { + anchorsExpected: 1, + anchorsFound: 0, + missingAnchors: [metadataAnchor], + completeness: 0, + }, + ]); + }); + + it("deduplicates normalized packet and file-view paths when counting delivered source files", () => { + expect( + collectSourceFiles([ + { + fileView: { file: "src/shared.ts" }, + packets: [ + { target: "src/shared.ts" }, + { target: "./src/direct.ts" }, + { packet: { target: { file: "src/nested.ts" } } }, + ], + }, + { + fileView: { file: "src/nested.ts" }, + packets: [ + { target: "src/direct.ts" }, + { target: "../outside.ts" }, + { packet: { target: { file: "/absolute.ts" } } }, + ], + }, + ]), + ).toEqual(["src/direct.ts", "src/nested.ts", "src/shared.ts"]); + }); + + it("accepts explicit runner selections and rejects ambiguous or unsafe arguments", () => { + const parsed = parseArguments([ + "--scenario", + "one,two", + "--runs=2", + "--scenario-file", + "fixtures/scenarios.json", + "--output=tmp/results.json", + "--require-complete", + "--json", + ]); + expect(parsed.scenarioIds).toEqual(["one", "two"]); + expect(parsed.runs).toBe(2); + expect(parsed.scenarioFile).toBe("fixtures/scenarios.json"); + expect(parsed.output).toBe("tmp/results.json"); + expect(parsed.requireComplete).toBe(true); + expect(parsed.json).toBe(true); + + const invalidCases: Array<{ args: string[]; expected: RegExp }> = [ + { args: ["--runs", "0"], expected: /positive integer/ }, + { args: ["--runs", "1.5"], expected: /positive integer/ }, + { args: ["--runs", "9007199254740992"], expected: /safe positive integer/ }, + { args: ["--scenario", "one,,two"], expected: /non-empty comma-separated/ }, + { args: ["--scenario", "one", "--scenario=one"], expected: /more than once/ }, + { args: ["--scenario-file", "https://example.test/scenarios.json"], expected: /local path/ }, + { args: ["--output", "/tmp/results.json"], expected: /relative path/ }, + { args: ["--output", "../results.json"], expected: /within the repository/ }, + { args: ["--json", "--json"], expected: /only once/ }, + { args: ["--require-complete=yes"], expected: /does not take a value/ }, + { args: ["--unknown"], expected: /Unknown argument/ }, + ]; + for (const testCase of invalidCases) { + expect(() => parseArguments(testCase.args), testCase.args.join(" ")).toThrow(testCase.expected); + } + }); +}); + +describe("public documentation benchmark summarizer contracts", () => { + it("computes odd, even, single, and zero medians without mutating input", () => { + const values = [9, 1, 5, 3]; + const original = [...values]; + + expect(median([9, 1, 5])).toBe(5); + expect(median(values)).toBe(4); + expect(median([7])).toBe(7); + expect(median([0])).toBe(0); + expect(values).toEqual(original); + }); + + it("rejects empty, nonfinite, and negative median inputs", () => { + const cases: Array<{ values: number[]; expected: RegExp }> = [ + { values: [], expected: /must not be empty/ }, + { values: [Number.NaN], expected: /finite number/ }, + { values: [Number.POSITIVE_INFINITY], expected: /finite number/ }, + { values: [-0.001], expected: /must not be negative/ }, + ]; + for (const testCase of cases) { + expect(() => median(testCase.values)).toThrow(testCase.expected); + } + }); + + it("keeps fractional even-count medians isolated by scenario and variant", () => { + const runs = [ + makeRun({ scenarioId: "alpha", variant: "codegraph", run: 2, toolCalls: 102, fileReads: 202, wallTimeMs: 302 }), + makeRun({ scenarioId: "beta", variant: "baseline", run: 1, toolCalls: 10, fileReads: 20, wallTimeMs: 30 }), + makeRun({ scenarioId: "alpha", variant: "baseline", run: 2, toolCalls: 2, fileReads: 12, wallTimeMs: 22 }), + makeRun({ scenarioId: "beta", variant: "codegraph", run: 1, toolCalls: 1000, fileReads: 2000, wallTimeMs: 3000 }), + makeRun({ scenarioId: "alpha", variant: "baseline", run: 1, toolCalls: 1, fileReads: 11, wallTimeMs: 21 }), + makeRun({ scenarioId: "beta", variant: "codegraph", run: 2, toolCalls: 1001, fileReads: 2001, wallTimeMs: 3001 }), + makeRun({ scenarioId: "beta", variant: "baseline", run: 2, toolCalls: 11, fileReads: 21, wallTimeMs: 31 }), + makeRun({ scenarioId: "alpha", variant: "codegraph", run: 1, toolCalls: 101, fileReads: 201, wallTimeMs: 301 }), + ]; + + expect(summarizeResults(makeResults(runs), { scenarioOrder: ["beta", "alpha"] })).toEqual([ + { + scenarioId: "beta", + variant: "baseline", + sampleCount: 2, + medians: { toolCalls: 10.5, fileReads: 20.5, wallTimeMs: 30.5 }, + completeRunCount: 2, + minimumCompleteness: 1, + }, + { + scenarioId: "beta", + variant: "codegraph", + sampleCount: 2, + medians: { toolCalls: 1000.5, fileReads: 2000.5, wallTimeMs: 3000.5 }, + completeRunCount: 2, + minimumCompleteness: 1, + }, + { + scenarioId: "alpha", + variant: "baseline", + sampleCount: 2, + medians: { toolCalls: 1.5, fileReads: 11.5, wallTimeMs: 21.5 }, + completeRunCount: 2, + minimumCompleteness: 1, + }, + { + scenarioId: "alpha", + variant: "codegraph", + sampleCount: 2, + medians: { toolCalls: 101.5, fileReads: 201.5, wallTimeMs: 301.5 }, + completeRunCount: 2, + minimumCompleteness: 1, + }, + ]); + }); + + it("surfaces incomplete run counts and the minimum completeness in each group", () => { + const results = makeResults([ + makeRun({ scenarioId: "alpha", variant: "baseline", run: 1 }), + makeRun({ + scenarioId: "alpha", + variant: "baseline", + run: 2, + anchorsExpected: 2, + anchorsFound: 1, + missingAnchors: ["src/missing.ts"], + completeness: 0.5, + }), + makeRun({ scenarioId: "alpha", variant: "codegraph", run: 1 }), + makeRun({ scenarioId: "alpha", variant: "codegraph", run: 2 }), + ]); + + expect(summarizeResults(results)).toEqual([ + { + scenarioId: "alpha", + variant: "baseline", + sampleCount: 2, + medians: { toolCalls: 1, fileReads: 1, wallTimeMs: 1 }, + completeRunCount: 1, + minimumCompleteness: 0.5, + }, + { + scenarioId: "alpha", + variant: "codegraph", + sampleCount: 2, + medians: { toolCalls: 1, fileReads: 1, wallTimeMs: 1 }, + completeRunCount: 2, + minimumCompleteness: 1, + }, + ]); + }); + + it("rejects malformed and stale scenario digests", () => { + const tempRoot = createTempRoot(); + const document = createScenarioFixture(tempRoot); + const malformedDigests = ["", "sha256:abc", `sha256:${"A".repeat(64)}`, `sha512:${"0".repeat(64)}`]; + + for (const digest of malformedDigests) { + const results = makeScenarioResults(document, ["fixture-scenario"], 1); + results.scenarioDigest = digest; + expect(() => validateResults(results, { scenarioFile: document }), digest).toThrow(/results\.scenarioDigest/); + } + + const stale = makeScenarioResults(document, ["fixture-scenario"], 1); + stale.scenarioDigest = validScenarioDigest; + expect(() => validateResults(stale, { scenarioFile: document })).toThrow( + /scenarioDigest.*does not match the selected scenario definitions/, + ); + }); + + it("requires selected scenario IDs to be unique, known, ordered, and exactly represented", () => { + const tempRoot = createTempRoot(); + const document = addSecondaryScenario(createScenarioFixture(tempRoot)); + const cases: Array<{ name: string; create: () => BenchmarkResults; expected: RegExp }> = [ + { + name: "duplicate selected id", + create: () => { + const results = makeScenarioResults(document); + results.scenarioIds.push("fixture-scenario"); + return results; + }, + expected: /scenarioIds\[2\].*duplicate value/, + }, + { + name: "unknown selected id", + create: () => { + const results = makeScenarioResults(document); + results.scenarioIds[1] = "unknown-scenario"; + return results; + }, + expected: /scenarioIds\[1\].*unknown scenario id/, + }, + { + name: "selected ids out of scenario-file order", + create: () => { + const results = makeScenarioResults(document); + results.scenarioIds.reverse(); + return results; + }, + expected: /must follow scenario-file order/, + }, + { + name: "selected scenario has no runs", + create: () => { + const results = makeScenarioResults(document); + results.runs = results.runs.filter((run) => run.scenarioId !== "fixture-secondary"); + return results; + }, + expected: /missing required run tuple fixture-secondary\/baseline\/1/, + }, + { + name: "run tuple belongs to an unselected scenario", + create: () => { + const results = makeScenarioResults(document, ["fixture-scenario"], 1); + results.runs.push( + makeRun({ + scenarioId: "fixture-secondary", + variant: "baseline", + toolCalls: 1, + fileReads: 1, + }), + ); + return results; + }, + expected: /scenarioId.*not listed in results\.scenarioIds/, + }, + ]; + + for (const testCase of cases) { + expect(() => validateResults(testCase.create(), { scenarioFile: document }), testCase.name).toThrow( + testCase.expected, + ); + } + }); + + it("rejects incomplete, noncontiguous, duplicate, extra, and input-amplified run matrices", () => { + const tempRoot = createTempRoot(); + const document = addSecondaryScenario(createScenarioFixture(tempRoot)); + const cases: Array<{ name: string; create: () => BenchmarkResults; expected: RegExp }> = [ + { + name: "one missing tuple", + create: () => { + const results = makeScenarioResults(document); + results.runs = results.runs.filter( + (run) => !(run.scenarioId === "fixture-scenario" && run.variant === "codegraph" && run.run === 2), + ); + return results; + }, + expected: /missing required run tuple fixture-scenario\/codegraph\/2/, + }, + { + name: "noncontiguous set mismatches the other scenario-variant sets", + create: () => { + const results = makeScenarioResults( + document, + document.scenarios.map((scenario) => scenario.id), + 3, + ); + results.runs = results.runs.filter( + (run) => !(run.scenarioId === "fixture-scenario" && run.variant === "baseline" && run.run === 2), + ); + return results; + }, + expected: /missing required run tuple fixture-scenario\/baseline\/2/, + }, + { + name: "extra out-of-range tuple", + create: () => { + const results = makeScenarioResults(document); + const extra = clone(findRun(results, "fixture-scenario", "baseline", 1)); + extra.run = 3; + results.runs.push(extra); + return results; + }, + expected: /run.*must be between 1 and results\.runsPerVariant \(2\).*received 3/, + }, + { + name: "deleted incomplete run", + create: () => { + const results = makeScenarioResults(document); + const incomplete = findRun(results, "fixture-scenario", "baseline", 2); + incomplete.checks.anchorsFound = 1; + incomplete.checks.missingAnchors = ["src/anchor.ts"]; + incomplete.checks.completeness = 0.5; + results.runs = results.runs.filter((run) => run !== incomplete); + return results; + }, + expected: /missing required run tuple fixture-scenario\/baseline\/2/, + }, + { + name: "duplicate tuple", + create: () => { + const results = makeScenarioResults(document); + results.runs.push(clone(findRun(results, "fixture-secondary", "codegraph", 2))); + return results; + }, + expected: /duplicate scenario\+variant\+run key fixture-secondary\/codegraph\/2/, + }, + { + name: "absurd runsPerVariant remains input-bounded", + create: () => { + const results = makeScenarioResults(document); + results.runsPerVariant = Number.MAX_SAFE_INTEGER; + return results; + }, + expected: /missing required run tuple fixture-scenario\/baseline\/3/, + }, + ]; + + for (const testCase of cases) { + expect(() => validateResults(testCase.create(), { scenarioFile: document }), testCase.name).toThrow( + testCase.expected, + ); + } + }); + + it("binds declared step counts while preserving output-derived Codegraph file reads", () => { + const tempRoot = createTempRoot(); + const document = addSecondaryScenario(createScenarioFixture(tempRoot)); + const mismatches: Array<{ + name: string; + mutate: (results: BenchmarkResults) => void; + expected: RegExp; + }> = [ + { + name: "baseline tool calls", + mutate: (results) => { + findRun(results, "fixture-scenario", "baseline", 1).metrics.toolCalls = 1; + }, + expected: /toolCalls.*expected 2.*variant "baseline"/, + }, + { + name: "Codegraph tool calls", + mutate: (results) => { + findRun(results, "fixture-scenario", "codegraph", 1).metrics.toolCalls = 2; + }, + expected: /toolCalls.*expected 1.*variant "codegraph"/, + }, + { + name: "baseline file reads", + mutate: (results) => { + findRun(results, "fixture-scenario", "baseline", 1).metrics.fileReads = 1; + }, + expected: /fileReads.*expected 2 from the declared baseline read steps/, + }, + ]; + + for (const testCase of mismatches) { + const results = makeScenarioResults(document, ["fixture-scenario"], 1); + testCase.mutate(results); + expect(() => validateResults(results, { scenarioFile: document }), testCase.name).toThrow(testCase.expected); + } + + const outputDerived = makeScenarioResults(document, ["fixture-secondary"], 1); + findRun(outputDerived, "fixture-secondary", "codegraph", 1).metrics.fileReads = 7; + expect( + summarizeResults(outputDerived, { scenarioFile: document }).map( + (summary: { variant: Variant; medians: { fileReads: number } }) => ({ + variant: summary.variant, + fileReads: summary.medians.fileReads, + }), + ), + ).toEqual([ + { variant: "baseline", fileReads: 1 }, + { variant: "codegraph", fileReads: 7 }, + ]); + }); + + it("rejects malformed, duplicate, nonfinite, negative, inconsistent, and unsafe result data", () => { + const cases: Array<{ + name: string; + mutate: (results: BenchmarkResults) => void; + expected: RegExp; + }> = [ + { + name: "unknown result field", + mutate: (results) => { + Object.assign(results, { unexpected: true }); + }, + expected: /unknown unexpected/, + }, + { + name: "missing metric", + mutate: (results) => { + Reflect.deleteProperty(results.runs[0].metrics, "fileReads"); + }, + expected: /missing fileReads/, + }, + { + name: "nonfinite metric", + mutate: (results) => { + results.runs[0].metrics.wallTimeMs = Number.NaN; + }, + expected: /finite number/, + }, + { + name: "negative metric", + mutate: (results) => { + results.runs[0].metrics.fileReads = -1; + }, + expected: /must not be negative/, + }, + { + name: "unsafe count", + mutate: (results) => { + results.runs[0].metrics.toolCalls = Number.MAX_SAFE_INTEGER + 1; + }, + expected: /safe integer count/, + }, + { + name: "absolute command path", + mutate: (results) => { + results.command[0] = "/absolute/node"; + }, + expected: /absolute path or network URL/, + }, + { + name: "traversing scenario file", + mutate: (results) => { + results.scenarioFile = "../outside.json"; + }, + expected: /confined to the repository/, + }, + { + name: "missing-anchor arithmetic", + mutate: (results) => { + results.runs[0].checks.anchorsFound = 1; + results.runs[0].checks.completeness = 0.5; + }, + expected: /plus missingAnchors.*must equal anchorsExpected/, + }, + { + name: "completeness arithmetic", + mutate: (results) => { + results.runs[0].checks.completeness = 0.5; + }, + expected: /expected 1 from anchorsFound \/ anchorsExpected/, + }, + { + name: "inconsistent expected-anchor total", + mutate: (results) => { + results.runs[1].checks.anchorsExpected = 3; + results.runs[1].checks.anchorsFound = 3; + results.runs[1].checks.completeness = 1; + }, + expected: /inconsistent total for scenario/, + }, + ]; + + for (const testCase of cases) { + const results = completeVariantPair(); + testCase.mutate(results); + expect(() => validateResults(results), testCase.name).toThrow(testCase.expected); + } + }); + + it("renders deterministic scenario and variant order with Markdown cell escaping", () => { + const escapedId = "z|[x]\nnext"; + const plainId = "alpha"; + const runs = [ + makeRun({ scenarioId: plainId, variant: "codegraph", toolCalls: 4, fileReads: 5, wallTimeMs: 6 }), + makeRun({ scenarioId: escapedId, variant: "codegraph", toolCalls: 7, fileReads: 8, wallTimeMs: 9 }), + makeRun({ scenarioId: plainId, variant: "baseline", toolCalls: 1, fileReads: 2, wallTimeMs: 3 }), + makeRun({ scenarioId: escapedId, variant: "baseline", toolCalls: 10, fileReads: 11, wallTimeMs: 12 }), + ]; + const order = [escapedId, plainId]; + const render = (orderedRuns: BenchmarkRun[]) => + renderMarkdownTable(summarizeResults(makeResults(orderedRuns), { scenarioOrder: order })); + const expected = [ + "| Scenario | Variant | Samples | Median tool calls | Median file reads | Median wall time (ms) | Complete runs | Minimum completeness |", + "| ---------------- | --------- | ------: | ----------------: | ----------------: | --------------------: | ------------: | -------------------: |", + "| z\\|\\[x\\]
next | baseline | 1 | 10 | 11 | 12 | 1 | 100% |", + "| z\\|\\[x\\]
next | codegraph | 1 | 7 | 8 | 9 | 1 | 100% |", + "| alpha | baseline | 1 | 1 | 2 | 3 | 1 | 100% |", + "| alpha | codegraph | 1 | 4 | 5 | 6 | 1 | 100% |", + "", + ].join("\n"); + + expect(render(runs)).toBe(expected); + expect(render([...runs].reverse())).toBe(expected); + const checkedScenarios: unknown = JSON.parse(fs.readFileSync(scenarioFilePath, "utf8")); + const checkedResults: unknown = JSON.parse(fs.readFileSync(resultsFilePath, "utf8")); + const checkedTable = renderMarkdownTable(summarizeResults(checkedResults, { scenarioFile: checkedScenarios })); + const formatted = spawnSync( + process.execPath, + [path.join(rootDir, "node_modules", "prettier", "bin", "prettier.cjs"), "--parser", "markdown"], + { + cwd: rootDir, + input: checkedTable, + encoding: "utf8", + }, + ); + expect(formatted.status, formatted.stderr).toBe(0); + expect(formatted.stdout).toBe(checkedTable); + }); + + it("matches the README generated block to results.example.json and passes the summarizer --check flow", () => { + const scenarioDocument: unknown = JSON.parse(fs.readFileSync(scenarioFilePath, "utf8")); + const results: unknown = JSON.parse(fs.readFileSync(resultsFilePath, "utf8")); + const readmeBefore = fs.readFileSync(readmePath, "utf8"); + const table = renderMarkdownTable(summarizeResults(results, { scenarioFile: scenarioDocument })); + + expect(checkGeneratedBlock(readmeBefore, table)).toBe(true); + let stdout = ""; + runSummarizerCli( + ["--input", resultsFilePath, "--scenario-file", scenarioFilePath, "--readme", readmePath, "--check"], + { + stdout: (chunk: string) => { + stdout += chunk; + }, + }, + ); + expect(stdout).toBe(table); + expect(fs.readFileSync(readmePath, "utf8")).toBe(readmeBefore); + }); +}); + +describe("public documentation benchmark subprocess", () => { + it("runs one complete local scenario through both variants without native or network requirements", () => { + const result = spawnSync( + process.execPath, + [runnerPath, "--runs", "1", "--scenario", "repo-orientation-small-ts", "--require-complete", "--json"], + { + cwd: rootDir, + encoding: "utf8", + timeout: 60_000, + env: { ...process.env, CODEGRAPH_DISABLE_NATIVE: "1" }, + }, + ); + + expect(result.error).toBeUndefined(); + expect(result.status).toBe(0); + expect(result.stderr).toBe(""); + const output: BenchmarkResults = JSON.parse(result.stdout); + const scenarioDocument: ScenarioDocument = JSON.parse(fs.readFileSync(scenarioFilePath, "utf8")); + expect(() => validateResults(output, { scenarioFile: scenarioDocument })).not.toThrow(); + expect(output.runs.map((run) => run.variant)).toEqual(["baseline", "codegraph"]); + for (const run of output.runs) { + expect(run.scenarioId).toBe("repo-orientation-small-ts"); + expect(Object.keys(run.metrics)).toEqual(metrics); + expect(run.checks).toEqual({ + anchorsExpected: 3, + anchorsFound: 3, + missingAnchors: [], + completeness: 1, + }); + } + const serializedRoot = JSON.stringify(rootDir).slice(1, -1); + expect(result.stdout).not.toContain(rootDir); + expect(result.stdout).not.toContain(serializedRoot); + }, 60_000); +}); diff --git a/tests/samples/benchmarks/mixed-docs-source/docs/request-flow.md b/tests/samples/benchmarks/mixed-docs-source/docs/request-flow.md new file mode 100644 index 00000000..7c23d4be --- /dev/null +++ b/tests/samples/benchmarks/mixed-docs-source/docs/request-flow.md @@ -0,0 +1,4 @@ +# Request flow + +The local request pipeline starts at `sendRequest` in the [client source](../src/client.ts). +The client delegates request formatting to `formatRequest` in the [transport source](../src/transport.ts). diff --git a/tests/samples/benchmarks/mixed-docs-source/src/client.ts b/tests/samples/benchmarks/mixed-docs-source/src/client.ts new file mode 100644 index 00000000..6e5008ae --- /dev/null +++ b/tests/samples/benchmarks/mixed-docs-source/src/client.ts @@ -0,0 +1,5 @@ +import { formatRequest } from "./transport.js"; + +export function sendRequest(resource: string): string { + return formatRequest(resource); +} diff --git a/tests/samples/benchmarks/mixed-docs-source/src/transport.ts b/tests/samples/benchmarks/mixed-docs-source/src/transport.ts new file mode 100644 index 00000000..8cd60ef2 --- /dev/null +++ b/tests/samples/benchmarks/mixed-docs-source/src/transport.ts @@ -0,0 +1,3 @@ +export function formatRequest(resource: string): string { + return `GET ${resource}`; +} diff --git a/tests/samples/benchmarks/sql-application-review/migrations/001_add_account_status.sql b/tests/samples/benchmarks/sql-application-review/migrations/001_add_account_status.sql new file mode 100644 index 00000000..8e8a6aa0 --- /dev/null +++ b/tests/samples/benchmarks/sql-application-review/migrations/001_add_account_status.sql @@ -0,0 +1,2 @@ +ALTER TABLE accounts +ADD COLUMN status TEXT NOT NULL DEFAULT 'active'; diff --git a/tests/samples/benchmarks/sql-application-review/schema.sql b/tests/samples/benchmarks/sql-application-review/schema.sql new file mode 100644 index 00000000..c812fc3b --- /dev/null +++ b/tests/samples/benchmarks/sql-application-review/schema.sql @@ -0,0 +1,4 @@ +CREATE TABLE accounts ( + id INTEGER PRIMARY KEY, + email TEXT NOT NULL UNIQUE +); diff --git a/tests/samples/benchmarks/sql-application-review/src/list-active-accounts.ts b/tests/samples/benchmarks/sql-application-review/src/list-active-accounts.ts new file mode 100644 index 00000000..6f906bed --- /dev/null +++ b/tests/samples/benchmarks/sql-application-review/src/list-active-accounts.ts @@ -0,0 +1,11 @@ +export interface AccountRow { + id: number; + email: string; + status: string; +} + +export const listActiveAccountsQuery = ` + SELECT id, email, status + FROM accounts + WHERE status = 'active' +`; diff --git a/tests/samples/benchmarks/typescript-service/src/handlers/health.ts b/tests/samples/benchmarks/typescript-service/src/handlers/health.ts new file mode 100644 index 00000000..9e725c84 --- /dev/null +++ b/tests/samples/benchmarks/typescript-service/src/handlers/health.ts @@ -0,0 +1,3 @@ +export function getHealth(): string { + return "healthy"; +} diff --git a/tests/samples/benchmarks/typescript-service/src/routes.ts b/tests/samples/benchmarks/typescript-service/src/routes.ts new file mode 100644 index 00000000..33fadcb4 --- /dev/null +++ b/tests/samples/benchmarks/typescript-service/src/routes.ts @@ -0,0 +1,13 @@ +import { getHealth } from "./handlers/health.js"; + +export interface RoutedRequest { + path: string; +} + +export function dispatchRequest(request: RoutedRequest): string { + if (request.path === "/health") { + return getHealth(); + } + + return "not found"; +} diff --git a/tests/samples/benchmarks/typescript-service/src/server.ts b/tests/samples/benchmarks/typescript-service/src/server.ts new file mode 100644 index 00000000..66b72a8b --- /dev/null +++ b/tests/samples/benchmarks/typescript-service/src/server.ts @@ -0,0 +1,9 @@ +import { dispatchRequest } from "./routes.js"; + +export interface Request { + path: string; +} + +export function serveRequest(request: Request): string { + return dispatchRequest(request); +}