diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index b25788e1e..043ea1ace 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -388,7 +388,9 @@ Interactive terminal controls are allowed only when stdout is not redirected or Query commands that accept path filters (`search`, `definition`, `references`, `callers`, `callees`, `symbols`, `files`, `find`, `map`, `inspect`, `deps`, `impact`, `unused`, `hotspots`, and `validate`) expand `--project` into the matching project directory glob before hitting `DbReader`, so all existing SQL path predicates keep working. When the indexed project root cannot be resolved and project expansion falls back to the process current directory, CLI query context and MCP structured payloads include `project_filter_root` and `project_filter_root_fallback_reason`. `index --project` expands to the files under the selected project directory and reuses the existing `--files` update path, but rejects expansions above 65,536 files for one project or 131,072 unique files across all requested projects with an explicit-files recovery hint. -`cdidx batch` is a CLI-side query loop for editor integrations and scripts that need several query commands against the same DB without spawning `cdidx` repeatedly. It opens one `DbContext` / `DbReader`, reads at most 1,024 newline-delimited JSON string arrays from stdin, caps each decoded string argument at 8,192 characters, and dispatches only commands in the side-effect-free allowlist owned by `CliCommandCatalog`. That schema includes the query and read-only discovery surfaces such as `goto` and `audit`; adding a top-level command or a dispatcher arm alone cannot cross the batch safety boundary. Immediate EOF with no commands remains exit 0 with no output by default; `--json-summary` appends a final JSON object with `commands_processed`, `line_errors`, `command_failures`, and `exit_code` for non-interactive callers that need an explicit empty-input signal. +`cdidx batch` is a CLI-side query loop for editor integrations and scripts that need several query commands against the same DB without spawning `cdidx` repeatedly. Each newline-delimited stdin record may use the established JSON string-array form or the validated `{"command": "...", "args": [...]}` object form. Object input rejects duplicate/unknown properties, missing or blank commands, non-array `args`, non-string values, and the same argument count/length violations as array input. Serial mode opens one `DbContext` / `DbReader`; `--parallel ` requires `--json-summary`, is capped at 16 workers, and opens one isolated query-only context per active worker. Every form dispatches only commands in the side-effect-free allowlist owned by `CliCommandCatalog`. That schema includes query and read-only discovery surfaces such as `goto` and `audit`; adding a top-level command or a dispatcher arm alone cannot cross the batch safety boundary. + +The default input budget remains 1,024 lines and is configurable through `--max-input-lines ` up to 65,536. Each decoded string argument remains capped at 8,192 characters. The JSON-summary output budget defaults to 10,485,760 characters and `--max-output-chars ` accepts 4,096 through 67,108,864. Immediate EOF with no commands remains exit 0 with no output by default; `--json-summary` appends a final JSON object with `commands_processed`, `line_errors`, `command_failures`, and `exit_code` for non-interactive callers that need an explicit empty-input signal. By default, child query commands stream their normal stdout/stderr directly. In `--json-summary` mode, every non-blank stdin line must instead emit one machine-readable batch envelope before the final summary: parsed commands use @@ -401,12 +403,20 @@ commands remain raw `stdout` text so diagnostics are not lost. Malformed or over-limit input lines use `record: "batch_error"` and an `error` object. Child output must not be written directly beside batch metadata in this mode. The entire serialized stream—including envelopes, arguments, escaping expansion, terminal -errors, and the final summary—is capped at 10,485,760 characters. An item that -exhausts it retains its exit/error metadata with `error.scope: "batch"`. The +errors, and the final summary—uses the configured `--max-output-chars` budget +(default 10,485,760; maximum 67,108,864). An item that exhausts it retains its +exit/error metadata with `error.scope: "batch"`. The final `record: "batch_summary"` retains `commands_processed`, `line_errors`, `command_failures`, and `exit_code`, and publishes `output_chars`, -`output_char_limit`, `input_line_limit`, and input/output limit state for -empty-input, failure, and budget accounting. +`output_char_limit`, `input_line_limit`, `parallelism`, and input/output limit +state for empty-input, failure, and budget accounting. Parallel workers route +stdout/stderr through per-command bounded writers, keep a separate read-only +SQLite connection and thread-local batch reader, and buffer only the active +worker window. `ScopedConsoleOutput` keeps nested JSON-envelope capture on the +current worker's routed stdout instead of replacing another worker's process-wide +writer. Completed records are committed to the shared output writer in input +order; an ordinary item failure remains isolated, while caller cancellation +stops scheduling and propagates. Editor integrations can request standard location shapes directly. `definition`, `references`, `search`, `find`, and `validate` accept `--format `; `lsp` emits LSP `Location` arrays, `qf` emits Vim quickfix lines, and `sarif` emits SARIF 2.1.0. `goto ` returns the single unambiguous definition as one LSP `Location`, while `goto --all ` returns all matching locations. @@ -3319,7 +3329,9 @@ override が文書化されていない限り ANSI/progress control を抑止す path filter を受け付ける query コマンド（`search`, `definition`, `references`, `callers`, `callees`, `symbols`, `files`, `find`, `map`, `inspect`, `deps`, `impact`, `unused`, `hotspots`, `validate`）は、`--project` を対応する project directory glob に展開してから `DbReader` に渡す。これにより既存の SQL path predicate をそのまま利用できる。indexed project root を解決できず process current directory に fallback して project expansion する場合、CLI query context と MCP structured payload は `project_filter_root` と `project_filter_root_fallback_reason` を含める。`index --project` は選択された project directory 配下のファイルに展開し、既存の `--files` 更新経路を再利用する。ただし 1 project で 65,536 files、requested projects 全体で 131,072 unique files を超える展開は拒否し、明示的な `--files` を使う recovery hint を返す。 -`cdidx batch` は、同じ DB に複数の query command を投げる editor integration や script 向けの CLI 側 query loop である。1 つの `DbContext` / `DbReader` を開き、stdin から最大 1,024 行の newline-delimited JSON 文字列配列を読み、デコード後の各文字列引数を 8,192 文字に制限し、`CliCommandCatalog` が正本となる副作用なし allowlist の command だけを dispatch する。この schema には `goto` や `audit` などの query / read-only discovery surface が含まれ、top-level command や dispatcher arm を追加しただけでは batch の安全境界を越えられない。command がない即時 EOF は既定で exit 0 かつ無出力のまま維持される。非対話の呼び出し元が空入力を明示的に判定したい場合は、`--json-summary` が `commands_processed`、`line_errors`、`command_failures`、`exit_code` を含む最終 JSON オブジェクトを追加する。 +`cdidx batch` は、同じ DB に複数の query command を投げる editor integration や script 向けの CLI 側 query loop である。newline-delimited な stdin record は従来の JSON 文字列配列 form、または検証済みの `{"command": "...", "args": [...]}` object form を使用できる。object input は重複/未知 property、欠落または空白 command、array でない `args`、文字列でない値、array input と同じ引数数/長さ違反を拒否する。serial mode は 1 つの `DbContext` / `DbReader` を開く。`--parallel ` は `--json-summary` を必須とし、最大 16 workers に制限し、active worker ごとに分離した query-only context を開く。すべての form は `CliCommandCatalog` が正本となる副作用なし allowlist の command だけを dispatch する。この schema には `goto` や `audit` などの query / read-only discovery surface が含まれ、top-level command や dispatcher arm を追加しただけでは batch の安全境界を越えられない。 + +既定の入力 budget は 1,024 行のままで、`--max-input-lines ` により最大 65,536 まで設定できる。デコード後の各文字列引数は引き続き 8,192 文字に制限する。JSON-summary 出力 budget は既定で 10,485,760 文字であり、`--max-output-chars ` は 4,096 から 67,108,864 までを受け付ける。command がない即時 EOF は既定で exit 0 かつ無出力のまま維持される。非対話の呼び出し元が空入力を明示的に判定したい場合は、`--json-summary` が `commands_processed`、`line_errors`、`command_failures`、`exit_code` を含む最終 JSON オブジェクトを追加する。 既定では child query command の通常の stdout / stderr を直接 stream する。`--json-summary` mode では、空白でない stdin 行ごとに final summary より前へ 1 つの machine-readable batch envelope を出力しなければならない。parse 済み command は `record: "batch_result"` として @@ -3329,12 +3341,18 @@ envelope を出力しなければならない。parse 済み command は `record text と失敗 command の出力は診断を失わないよう raw `stdout` text のまま 保持する。malformed line や入力上限超過 line は `record: "batch_error"` と `error` object を使う。 この mode では child output を batch metadata と並べて直接出力してはならない。envelope、 -arguments、escape 展開、terminal error、final summary を含む serialized stream 全体を -10,485,760 文字に制限し、使い切った item も +arguments、escape 展開、terminal error、final summary を含む serialized stream 全体には +設定された `--max-output-chars` budget（既定 10,485,760、最大 67,108,864）を適用し、 +使い切った item も `error.scope: "batch"` と exit / error metadata を保持する。final `record: "batch_summary"` は empty input、failure、budget accounting のために `commands_processed`、`line_errors`、 -`command_failures`、`exit_code`、`output_chars`、`output_char_limit`、`input_line_limit` と -input / output limit state を保持する。 +`command_failures`、`exit_code`、`output_chars`、`output_char_limit`、`input_line_limit`、 +`parallelism` と input / output limit state を保持する。parallel worker は stdout / stderr を +command ごとの bounded writer へ route し、分離した read-only SQLite connection と thread-local +batch reader を使い、active worker window だけを buffer する。`ScopedConsoleOutput` は nested +JSON-envelope capture を現在の worker の routed stdout に保ち、他 worker の process-wide writer を +置き換えない。完了 record は入力順で共有 output writer へ commit する。通常の item failure は +他 item から隔離し、caller cancellation は scheduling を停止して伝播する。 editor integration は標準的な location 形状を直接要求できる。`definition`、`references`、`search`、`find`、`validate` は `--format ` を受け付け、`lsp` は LSP `Location` 配列、`qf` は Vim quickfix 行、`sarif` は SARIF 2.1.0 を出力する。`goto ` は曖昧でない単一定義を 1 つの LSP `Location` として返し、`goto --all ` は一致する全 location を返す。 diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index 67cc83499..347373275 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -572,6 +572,8 @@ Use `docs/test-doc-maintenance-plan.md` before moving oversized suites or adding Database corruption recovery and graceful degradation behavior. Filesystem setup failures for `cdidx index` (read-only DB files and unwritable DB parent directories) are covered in `IndexCommandRunnerTests.cs` so they exercise the same CLI JSON/stderr boundary users see. - `JsonOutputSnapshotTests.cs`, `JsonOutputSnapshotHelper.cs` Golden-file regression fixtures for the CLI `--json` output contracts (issue #1548). Each test runs one command (`status`, `search`, `references`, `impact`, `excerpt`) against a deterministic in-memory fixture, normalizes volatile fields (timestamps, absolute paths, commit SHAs, FTS5 scores, SQLite page counts), and diffs against the matching file under `tests/CodeIndex.Tests/golden/`. Renames, removals, reordered arrays, or new keys fail the snapshot so the contract change is forced to land alongside an intentional golden update. See "JSON `--json` output snapshots" below for the update procedure. +- `QueryCommandRunnerBatchIssue4723Tests.cs` + CLI batch coverage for structured command objects, configurable input/output budgets, bounded parallel read overlap, input-order result emission, per-item failure isolation, and cancellation/console restoration. The deterministic overlap test blocks the first worker until the second finishes through batch-only test seams; keep those seams reset in `finally` and do not replace the signal with timing assertions. - `PropertyBasedParserTests.cs` FsCheck-driven property tests for parser-heavy paths called out in issue #1572: `ArgHelper.WantsHelp` and `ProgramRunner.IsProjectPathArg` never throw on arbitrary inputs; `FileIndexer.NormalizePathSeparators` is idempotent under double application; the literal-safe FTS5 sanitizer (`DbReader.SanitizeFtsQuery`) always emits a query that a real in-memory FTS5 virtual table can parse. They complement, not replace, the example-based tests in `ArgHelperTests.cs` / `QueryCommandRunnerTests.cs`. - `TestProjectHelper.cs`, `TestDeterminism.cs`, `RepositoryTestPaths.cs`, `TestConsoleLock.cs` @@ -1389,6 +1391,8 @@ dotnet test --filter "FullyQualifiedName~GitHelperTests" DB破損からの復旧とグレースフル劣化のテスト。`cdidx index` の filesystem setup failure（read-only DB file や書き込み不可の DB 親ディレクトリ）は、ユーザーが見る CLI JSON/stderr 境界を通すため `IndexCommandRunnerTests.cs` で扱います。 - `JsonOutputSnapshotTests.cs`、`JsonOutputSnapshotHelper.cs` CLI の `--json` 出力契約に対するゴールデンファイル回帰フィクスチャ (issue #1548)。各テストは `status` / `search` / `references` / `impact` / `excerpt` を決定的なインメモリ fixture に対して実行し、揺らぐフィールド（timestamp、絶対パス、commit SHA、FTS5 score、SQLite page count など）を正規化したうえで `tests/CodeIndex.Tests/golden/` 配下のファイルと差分比較します。フィールドの rename / 削除 / 並び替え / 新規追加が起きると snapshot が失敗するため、契約変更は意図的な golden 更新と同じ PR で揃えざるを得ません。更新手順は下記「JSON `--json` 出力 snapshot」を参照してください。 +- `QueryCommandRunnerBatchIssue4723Tests.cs` + structured command object、設定可能な input / output budget、上限付き parallel read の重複実行、入力順の result 出力、item ごとの failure isolation、cancellation / console 復元を対象とする CLI batch test です。決定的な overlap test は batch 専用 test seam を通じて第 1 worker を第 2 worker の完了まで block します。seam は `finally` で必ず reset し、signal を timing assertion に置き換えないでください。 - `PropertyBasedParserTests.cs` issue #1572 で挙げられたパーサー系経路に対する FsCheck 駆動の property テスト: `ArgHelper.WantsHelp` と `ProgramRunner.IsProjectPathArg` が任意入力で例外を投げないこと、`FileIndexer.NormalizePathSeparators` が二重適用で idempotent であること、literal-safe な FTS5 サニタイザ (`DbReader.SanitizeFtsQuery`) が常にインメモリ FTS5 仮想テーブルで parse 可能なクエリを出力すること。`ArgHelperTests.cs` / `QueryCommandRunnerTests.cs` の例ベーステストを置き換えるものではなく補完します。 - `TestProjectHelper.cs`、`TestDeterminism.cs`、`RepositoryTestPaths.cs`、`TestConsoleLock.cs` diff --git a/USER_GUIDE.md b/USER_GUIDE.md index 9e4985621..6ed21466c 100644 --- a/USER_GUIDE.md +++ b/USER_GUIDE.md @@ -519,6 +519,8 @@ be audited whenever the matching help text changes. | JSON envelope capture | `10,485,760` characters | `JsonEnvelopeWrapper` | | CLI batch line | `1,048,576` characters | `QueryCommandRunner` | | CLI batch arguments | `256` arguments after command name | `QueryCommandRunner` | +| CLI batch input lines | `1,024` by default; configurable from `1` through `65,536` | `QueryCommandRunner` | +| CLI batch JSON-summary output | `10,485,760` characters by default; configurable from `4,096` through `67,108,864` | `QueryCommandRunner` | When a default changes, update the help text, this table, affected examples, and the changelog fragment in the same PR so users are not asked to reconcile @@ -1363,31 +1365,45 @@ cdidx search "authenticate" --json --verbose ``` For scripts or editor integrations that need several queries against the same -index, `cdidx batch --db ` keeps one SQLite connection open and reads one -JSON string array per stdin line. Each array starts with a command from the +index, `cdidx batch --db ` reads one JSON command per stdin line. A line +may be the established string-array form +`["search","Needle","--json"]` or the structured form +`{"command":"search","args":["Needle","--json"]}`. Both forms dispatch only the schema-owned side-effect-free allowlist, which includes read-only navigation and -audit commands such as `goto` and `audit`, followed by that command's normal -arguments. Each stdin line is capped at 1,048,576 characters, each decoded -string argument is capped at 8,192 characters, each command can carry at most -256 arguments after the command name, and one invocation reads at most 1,024 -input lines. By default, child commands stream their normal stdout/stderr -directly, so callers can keep the standalone command output shape. With no -input, `batch` exits 0 and prints nothing by default. Pass `--json-summary` when -a non-interactive caller needs a machine-readable batch stream: each non-blank -stdin line emits one JSON envelope before the final summary. Parsed commands use +audit commands such as `goto` and `audit`. Each stdin line is capped at +1,048,576 characters, each decoded string argument is capped at 8,192 +characters, and each command can carry at most 256 arguments after the command +name. The default 1,024-line input budget can be changed with +`--max-input-lines ` up to the safe maximum of 65,536. + +Serial execution is the default and keeps one SQLite connection open. Child +commands stream their normal stdout/stderr directly, so callers can keep the +standalone command output shape. With no input, `batch` exits 0 and prints +nothing by default. Pass `--json-summary` when a non-interactive caller needs a +machine-readable batch stream: each non-blank stdin line emits one JSON envelope +before the final summary. Parsed commands use `record: "batch_result"` with `line`, `command`, `arguments`, `exit_code`, and captured child `stderr`. Successful single-document JSON is embedded as typed `result`, successful NDJSON is embedded as a stable `results` array even when it contains one row, and text or failed output remains raw `stdout`. Malformed or over-limit lines use `record: "batch_error"` with an `error` object. The complete serialized stream, including envelopes, arguments, JSON escaping, terminal -errors, and the final summary, is capped at 10,485,760 characters. The final -`record: "batch_summary"` reports `commands_processed`, `line_errors`, -`command_failures`, `exit_code`, `output_chars`, `output_char_limit`, and input / -output limit state, including `commands_processed: 0` for immediate EOF. For -clean automation, feed stdin from a pipe or file; an interactive TTY may echo -typed JSONL before `cdidx` reads it, but that echo is terminal behavior rather -than process stdout/stderr. +errors, and the final summary, defaults to a 10,485,760-character budget. +`--max-output-chars ` can change that budget from 4,096 through the safe +maximum of 67,108,864 characters. The final `record: "batch_summary"` reports +`commands_processed`, `line_errors`, `command_failures`, `exit_code`, +`output_chars`, `output_char_limit`, `input_line_limit`, `parallelism`, and +input/output limit state, including `commands_processed: 0` for immediate EOF. + +Use `--parallel ` with `--json-summary` to run up to 16 independent read-only +items concurrently. Each worker uses an isolated query-only SQLite connection +and isolated stdout/stderr capture; records are still emitted in input order, +one command failure does not cancel sibling items, and caller cancellation stops +new work. A bounded producer/consumer pipeline starts work as each line arrives +and emits the earliest eligible ordered record without waiting for a full worker +window or stdin EOF. For clean automation, feed stdin from a pipe or file; an interactive +TTY may echo typed JSONL before `cdidx` reads it, but that echo is terminal +behavior rather than process stdout/stderr. ```bash printf '%s\n' \ @@ -1401,6 +1417,14 @@ printf '' | cdidx batch --db .cdidx/codeindex.db --json-summary # Emits one batch_summary object; output_chars equals the complete serialized stream length. ``` +```bash +printf '%s\n' \ + '{"command":"search","args":["Authenticate","--json","--exact"]}' \ + '{"command":"symbols","args":["AuthFixture","--json","--exact-name"]}' \ + | cdidx batch --db .cdidx/codeindex.db --json-summary --parallel 2 \ + --max-input-lines 4096 --max-output-chars 16777216 +``` + Output: ``` @@ -3673,6 +3697,8 @@ render できます。 | JSON envelope capture | `10,485,760` 文字 | `JsonEnvelopeWrapper` | | CLI batch line | `1,048,576` 文字 | `QueryCommandRunner` | | CLI batch arguments | command 名の後ろに `256` 引数 | `QueryCommandRunner` | +| CLI batch input lines | 既定 `1,024`、`1` から `65,536` まで設定可能 | `QueryCommandRunner` | +| CLI batch JSON-summary output | 既定 `10,485,760` 文字、`4,096` から `67,108,864` まで設定可能 | `QueryCommandRunner` | 既定値を変更するときは、help text、この表、影響する examples、changelog fragment を 同じ PR で更新してください。 @@ -4511,26 +4537,39 @@ cdidx search "authenticate" --json --verbose ``` 同じインデックスに対して複数の query を投げる script や editor integration では、 -`cdidx batch --db ` を使うと 1 つの SQLite connection を開いたまま処理できます。 -stdin の各行は JSON 文字列配列で、先頭に `goto` や `audit` を含む schema 管理の副作用なし -allowlist の command 名、その後ろに通常の引数を並べます。各 stdin 行は 1,048,576 文字まで、 -デコード後の各文字列引数は 8,192 文字まで、各 command は command 名の後ろに最大 256 引数までで、 -1 回の呼び出しが読む入力は最大 1,024 行です。既定では child command の通常の stdout / stderr を -直接 stream するため、単発 command と同じ出力形状を維持できます。入力がない場合、`batch` は -既定で exit 0 かつ無出力です。非対話の呼び出し元が machine-readable な batch stream を -必要とする場合は `--json-summary` を渡します。この場合、空白でない stdin 行ごとに 1 つの JSON -envelope を出力してから final summary を出します。parse 済み command は +`cdidx batch --db ` が stdin の各行から 1 つの JSON command を読みます。各行は従来の +文字列配列 `["search","Needle","--json"]`、または structured form +`{"command":"search","args":["Needle","--json"]}` を使用できます。どちらの form も +`goto` や `audit` を含む schema 管理の副作用なし allowlist だけを dispatch します。各 stdin 行は +1,048,576 文字まで、デコード後の各文字列引数は 8,192 文字まで、各 command は command 名の後ろに +最大 256 引数までです。既定の入力 budget は 1,024 行で、`--max-input-lines ` により安全な最大値 +65,536 まで変更できます。 + +既定の serial execution は 1 つの SQLite connection を開いたまま処理します。child command の +通常の stdout / stderr を直接 stream するため、単発 command と同じ出力形状を維持できます。入力が +ない場合、`batch` は既定で exit 0 かつ無出力です。非対話の呼び出し元が machine-readable な batch +stream を必要とする場合は `--json-summary` を渡します。この場合、空白でない stdin 行ごとに +1 つの JSON envelope を出力してから final summary を出します。parse 済み command は `record: "batch_result"` として `line`、`command`、`arguments`、`exit_code`、捕捉した child `stderr` を持ちます。成功した単一 document JSON は型付き `result`、成功した NDJSON は 1 row の場合も安定して `results` array に埋め込み、text または失敗時の出力は raw `stdout` のまま 保持します。malformed line や上限超過 line は `record: "batch_error"` と `error` object を 持ちます。envelope、arguments、JSON escape、terminal error、final summary を含む serialized -stream 全体は 10,485,760 文字までです。最後の `record: "batch_summary"` object は +stream 全体の既定 budget は 10,485,760 文字です。`--max-output-chars ` で 4,096 から安全な +最大値 67,108,864 文字まで変更できます。最後の `record: "batch_summary"` object は `commands_processed`、`line_errors`、`command_failures`、`exit_code`、`output_chars`、 -`output_char_limit` と input / output limit state を報告し、即時 EOF では -`commands_processed: 0` を含みます。automation で clean な入出力が必要な場合は stdin を pipe -または file から渡してください。interactive TTY では入力した JSONL が `cdidx` の読み取り前に -echo される場合がありますが、これは process の stdout / stderr ではなく terminal の挙動です。 +`output_char_limit`、`input_line_limit`、`parallelism` と input / output limit state を報告し、 +即時 EOF では `commands_processed: 0` を含みます。 + +`--json-summary` とともに `--parallel ` を使うと、最大 16 個の独立した read-only item を +並列実行できます。各 worker は分離された query-only SQLite connection と stdout / stderr capture +を使い、record は完了順にかかわらず入力順で出力されます。1 command の失敗は sibling item を +cancel せず、caller cancellation は新しい work を停止します。bounded producer / consumer +pipeline は各行の到着時に work を開始し、worker window が満杯になることや stdin EOF を待たずに、 +入力順を守って出力可能になった最初の record を出力します。automation で clean な入出力が +必要な場合は stdin を pipe または file から渡してください。interactive TTY では入力した JSONL が +`cdidx` の読み取り前に echo される場合がありますが、これは process の stdout / stderr ではなく +terminal の挙動です。 ```bash printf '%s\n' \ @@ -4544,6 +4583,14 @@ printf '' | cdidx batch --db .cdidx/codeindex.db --json-summary # batch_summary object を 1 件出力し、output_chars は serialized stream 全体の実際の長さと一致します。 ``` +```bash +printf '%s\n' \ + '{"command":"search","args":["Authenticate","--json","--exact"]}' \ + '{"command":"symbols","args":["AuthFixture","--json","--exact-name"]}' \ + | cdidx batch --db .cdidx/codeindex.db --json-summary --parallel 2 \ + --max-input-lines 4096 --max-output-chars 16777216 +``` + 出力: ``` diff --git a/changelog.d/unreleased/4723.added.md b/changelog.d/unreleased/4723.added.md new file mode 100644 index 000000000..067810682 --- /dev/null +++ b/changelog.d/unreleased/4723.added.md @@ -0,0 +1,24 @@ +--- +category: added +issues: + - 4723 +affected: + - src/CodeIndex/Cli/CliFlagSchema.cs + - src/CodeIndex/Cli/ConsoleUi.cs + - src/CodeIndex/Cli/JsonEnvelopeWrapper.cs + - src/CodeIndex/Cli/JsonEnvelopeWrapper.Bounded.cs + - src/CodeIndex/Cli/QueryCommandRunner.Batch.cs + - src/CodeIndex/Cli/QueryCommandRunner.BatchLimits.cs + - src/CodeIndex/Cli/ScopedConsoleOutput.cs + - DEVELOPER_GUIDE.md + - TESTING_GUIDE.md + - USER_GUIDE.md +--- + +## English + +- **CLI batch now supports structured input, configurable budgets, and bounded parallel reads (#4723)** — `cdidx batch` accepts validated `{"command":"...","args":[...]}` records alongside string arrays, lets generated pipelines tune input-line and JSON-summary output budgets within safe maxima, and can run up to 16 isolated read-only workers with stable input-order records, per-item error isolation, and incremental output before stdin EOF. + +## 日本語 + +- **CLI batch が structured input、設定可能な budget、上限付き parallel read に対応しました (#4723)** — `cdidx batch` は従来の文字列配列に加えて検証済みの `{"command":"...","args":[...]}` record を受け付け、generated pipeline が安全な最大値の範囲で input line / JSON-summary output budget を調整できるようになりました。また、最大 16 個の分離された read-only worker を実行し、入力順の record、item ごとの error isolation、stdin EOF 前の逐次出力を維持します。 diff --git a/src/CodeIndex/Cli/CliFlagSchema.cs b/src/CodeIndex/Cli/CliFlagSchema.cs index 5b4038516..83485bc16 100644 --- a/src/CodeIndex/Cli/CliFlagSchema.cs +++ b/src/CodeIndex/Cli/CliFlagSchema.cs @@ -244,6 +244,10 @@ private static IReadOnlyList BuildAll() new() { Name = "--workspace-db", ValuePlaceholder = "", Description = "Additional workspace member database path for dependency aggregation; repeat up to 7 distinct additional DBs", PrimaryCommands = Set(WorkspaceDbCommands) }, new() { Name = "--data-dir", ValuePlaceholder = "", Description = "Directory containing codeindex.db; overrides CDIDX_DATA_DIR/XDG/workspace defaults", PrimaryCommands = Set(DataDirCommands) }, new() { Name = "--json", Description = "JSON output; search/symbols/files/validate also accept --json=array for a single JSON array", PrimaryCommands = Set(JsonCommands) }, + new() { Name = "--json-summary", Description = "Batch: emit one typed result/error record per input plus a final summary", PrimaryCommands = Set("batch") }, + new() { Name = "--max-input-lines", ValuePlaceholder = "", Description = $"Batch: input-line budget (default {QueryCommandRunner.BatchDefaultInputLines}, max {QueryCommandRunner.BatchMaxInputLines})", PrimaryCommands = Set("batch") }, + new() { Name = "--max-output-chars", ValuePlaceholder = "", Description = $"Batch JSON-summary output budget (default {QueryCommandRunner.BatchDefaultTotalOutputChars}, max {QueryCommandRunner.BatchMaxTotalOutputChars})", PrimaryCommands = Set("batch") }, + new() { Name = "--parallel", ValuePlaceholder = "", Description = $"Batch JSON-summary worker count (default 1, max {QueryCommandRunner.BatchMaxParallelism}); results retain input order", PrimaryCommands = Set("batch") }, new() { Name = "--pretty", Description = CliOutputFormatCapabilities.PrettyDescription, PrimaryCommands = Set(JsonCommands), TopLevel = true }, new() { Name = "--compact", Description = "AI-oriented compact JSON with capped list sections and truncation metadata", PrimaryCommands = Set(CompactJsonCommands) }, new() { Name = "--format", ValuePlaceholder = CliOutputFormatCapabilities.FormatValuePlaceholder, Description = CliOutputFormatCapabilities.FormatDescription, PrimaryCommands = Set(FormatCommands) }, diff --git a/src/CodeIndex/Cli/ConsoleUi.cs b/src/CodeIndex/Cli/ConsoleUi.cs index 3ba80894b..6e1b71b09 100644 --- a/src/CodeIndex/Cli/ConsoleUi.cs +++ b/src/CodeIndex/Cli/ConsoleUi.cs @@ -130,7 +130,7 @@ private static readonly (string Command, string Usage)[] CommandUsageLines = ("export", "cdidx export ctags [--output ] [--db ] [--json] [--lang ] [--path ] [--exclude-path ] [--exclude-tests] [--include-generated]"), ("import", "cdidx import [--db ] [--prune-paths] [--dry-run|--check] [--limit ] [--offset ] [--json]"), ("languages", "cdidx languages [--db ] [--json] [--format ] [--summary-only] [--indexed-only] [--language |--extension |--alias ] [--capability ]"), - ("batch", "cdidx batch [--db ] [--json-summary] # stdin is JSON Lines; --json-summary embeds typed child JSON plus a final summary"), + ("batch", "cdidx batch [--db ] [--json-summary] [--max-input-lines ] [--max-output-chars ] [--parallel ] # stdin is JSON Lines; --json-summary embeds typed child JSON plus a final summary"), ("hooks-install", "cdidx hooks install [--project ] [--force] [--dry-run] [--json]"), ("hooks-uninstall", "cdidx hooks uninstall [--project ] [--force] [--json]"), ("hooks-status", "cdidx hooks status [--project ] [--json]"), @@ -169,9 +169,11 @@ private static readonly (string Command, string Note)[] CommandUsageNotes = ("inspect", "In query mode --path is a glob filter; in line mode --path --line selects a source location."), ("status", "`stats` is a compatibility alias for `status`; prefer `status` in scripts and documentation."), ("backfill-fold", "`fold` is a compatibility alias for `backfill-fold`; prefer `backfill-fold` in scripts and documentation."), - ("batch", "Each stdin line must be a JSON string array such as [\"search\",\"Needle\",\"--json\"]; blank lines are skipped."), + ("batch", "Each stdin line may be a JSON string array such as [\"search\",\"Needle\",\"--json\"] or an object such as {\"command\":\"search\",\"args\":[\"Needle\",\"--json\"]}; blank lines are skipped."), ("batch", "By default child commands stream their normal stdout/stderr; with --json-summary each non-blank line writes a batch_result or batch_error envelope."), - ("batch", $"Successful child JSON is embedded as result, NDJSON as stable results, and text or failed output remains raw stdout; the full serialized stream is capped at {QueryCommandRunner.BatchMaxTotalOutputChars} characters and {QueryCommandRunner.BatchMaxInputLines} input lines."), + ("batch", $"--max-input-lines and --max-output-chars tune the default {QueryCommandRunner.BatchDefaultInputLines}-line / {QueryCommandRunner.BatchDefaultTotalOutputChars}-character budgets up to safe maxima of {QueryCommandRunner.BatchMaxInputLines} lines / {QueryCommandRunner.BatchMaxTotalOutputChars} characters."), + ("batch", $"--parallel (max {QueryCommandRunner.BatchMaxParallelism}) requires --json-summary, uses isolated read-only DB connections, and emits stable results in input order."), + ("batch", "Successful child JSON is embedded as result, NDJSON as stable results, and text or failed output remains raw stdout; configured serialized-output budgets include envelopes and escaping."), ("batch", "Malformed lines and failed commands set a non-zero exit status after draining stdin; --json-summary still appends a batch_summary record."), ("hooks", "install writes `.git/hooks/pre-commit`; use `cdidx hooks install --dry-run` to preview the managed hook and planned action first."), ("hooks-install", "Writes `.git/hooks/pre-commit`; --dry-run reports the planned create, managed replacement, custom-hook chain, or no-op action and prints the managed hook without changing files; use --force only when replacing an existing chained-hook backup is intended."), diff --git a/src/CodeIndex/Cli/JsonEnvelopeWrapper.Bounded.cs b/src/CodeIndex/Cli/JsonEnvelopeWrapper.Bounded.cs index 68762870f..d052315c3 100644 --- a/src/CodeIndex/Cli/JsonEnvelopeWrapper.Bounded.cs +++ b/src/CodeIndex/Cli/JsonEnvelopeWrapper.Bounded.cs @@ -137,14 +137,13 @@ private static int RunBoundedResponse( } var innerArgs = PrepareBoundedInnerArgs(command, args, controls); - var originalOut = Console.Out; using var captured = new BoundedStringWriter(MaxCapturedOutputChars); var stopwatch = System.Diagnostics.Stopwatch.StartNew(); int exitCode; JsonEnvelopeCaptureLimitExceededException? captureLimitExceeded = null; try { - Console.SetOut(captured); + using var outputScope = ScopedConsoleOutput.Redirect(captured); using var executionScope = EnterBoundedExecution(command, controls.Offset, controls.PageLimit, controls.Fields, controls.Compact); exitCode = runInner(innerArgs); } @@ -156,7 +155,6 @@ private static int RunBoundedResponse( finally { stopwatch.Stop(); - Console.SetOut(originalOut); } if (captureLimitExceeded is not null) @@ -616,23 +614,17 @@ private static ResponseCount ResolveTotalCount( return new ResponseCount(offset + availableCount, false); var countArgs = PrepareCountArgs(command, args); - var originalOut = Console.Out; using var captured = new BoundedStringWriter(MaxRawJsonItemChars); int countExitCode; try { - Console.SetOut(captured); + using var outputScope = ScopedConsoleOutput.Redirect(captured); countExitCode = runInner(countArgs); } catch { return new ResponseCount(offset + availableCount, false); } - finally - { - Console.SetOut(originalOut); - } - try { var countItems = ParseRawJsonItems(command, captured.ToString(), out _, out _); diff --git a/src/CodeIndex/Cli/JsonEnvelopeWrapper.cs b/src/CodeIndex/Cli/JsonEnvelopeWrapper.cs index e327a9cc2..7ac3af9ad 100644 --- a/src/CodeIndex/Cli/JsonEnvelopeWrapper.cs +++ b/src/CodeIndex/Cli/JsonEnvelopeWrapper.cs @@ -106,14 +106,13 @@ internal static int RunWrapped( ? Path.Combine(".cdidx", "codeindex.db") : explicitDbPath!; - var originalOut = Console.Out; using var captured = new BoundedStringWriter(MaxCapturedOutputChars); var stopwatch = Stopwatch.StartNew(); int exitCode; JsonEnvelopeCaptureLimitExceededException? captureLimitExceeded = null; try { - Console.SetOut(captured); + using var outputScope = ScopedConsoleOutput.Redirect(captured); exitCode = runInner(innerArgs); } catch (JsonEnvelopeCaptureLimitExceededException ex) @@ -124,7 +123,6 @@ internal static int RunWrapped( finally { stopwatch.Stop(); - Console.SetOut(originalOut); } if (captureLimitExceeded is not null) diff --git a/src/CodeIndex/Cli/ProgramRunner.Dispatch.cs b/src/CodeIndex/Cli/ProgramRunner.Dispatch.cs index 5bd835531..f39e72f77 100644 --- a/src/CodeIndex/Cli/ProgramRunner.Dispatch.cs +++ b/src/CodeIndex/Cli/ProgramRunner.Dispatch.cs @@ -83,7 +83,7 @@ private static int RunDispatchedCommand( "deps" => a => QueryCommandRunner.RunDeps(a, context.JsonOptions, context.CancellationToken), "unused" => a => QueryCommandRunner.RunUnused(a, context.JsonOptions), "hotspots" => a => QueryCommandRunner.RunHotspots(a, context.JsonOptions), - "batch" => a => QueryCommandRunner.RunBatch(a, context.JsonOptions, context.AppVersion), + "batch" => a => QueryCommandRunner.RunBatch(a, context.JsonOptions, context.AppVersion, context.CancellationToken), "suggestions" => a => SuggestionsCommandRunner.Run(a, context.JsonOptions, context.CancellationToken), _ => null, }; diff --git a/src/CodeIndex/Cli/QueryCommandRunner.Batch.cs b/src/CodeIndex/Cli/QueryCommandRunner.Batch.cs index 62d950570..dd6859a01 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.Batch.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.Batch.cs @@ -1,6 +1,8 @@ +using System.Globalization; using System.Text; using System.Text.Json; using System.Text.Json.Nodes; +using System.Threading.Channels; using CodeIndex.Database; using CodeIndex.Diagnostics; @@ -9,12 +11,23 @@ namespace CodeIndex.Cli; public static partial class QueryCommandRunner { private const int BatchMaxCapturedOutputChars = JsonEnvelopeWrapper.MaxCapturedOutputChars; + internal static Action? BatchParallelCommandStartedForTesting { get; set; } + internal static Action? BatchParallelCommandCompletedForTesting { get; set; } - public static int RunBatch(string[] cmdArgs, JsonSerializerOptions jsonOptions, string appVersion = "") + public static int RunBatch( + string[] cmdArgs, + JsonSerializerOptions jsonOptions, + string appVersion = "", + CancellationToken cancellationToken = default) { var dbPath = Path.Combine(".cdidx", "codeindex.db"); var dbPathExplicit = false; var jsonSummary = false; + var maxInputLines = BatchDefaultInputLines; + var maxOutputChars = BatchDefaultTotalOutputChars; + var maxOutputCharsSpecified = false; + var parallelism = 1; + var parallelismSpecified = false; for (var i = 0; i < cmdArgs.Length; i++) { var arg = cmdArgs[i]; @@ -48,11 +61,74 @@ public static int RunBatch(string[] cmdArgs, JsonSerializerOptions jsonOptions, continue; } + if (arg == "--max-input-lines" || arg.StartsWith("--max-input-lines=", StringComparison.Ordinal)) + { + if (!TryReadBatchBoundedOption( + cmdArgs, + ref i, + arg, + "--max-input-lines", + 1, + BatchMaxInputLines, + out maxInputLines)) + { + return CommandExitCodes.UsageError; + } + continue; + } + + if (arg == "--max-output-chars" || arg.StartsWith("--max-output-chars=", StringComparison.Ordinal)) + { + if (!TryReadBatchBoundedOption( + cmdArgs, + ref i, + arg, + "--max-output-chars", + BatchMinTotalOutputChars, + BatchMaxTotalOutputChars, + out maxOutputChars)) + { + return CommandExitCodes.UsageError; + } + maxOutputCharsSpecified = true; + continue; + } + + if (arg == "--parallel" || arg.StartsWith("--parallel=", StringComparison.Ordinal)) + { + if (!TryReadBatchBoundedOption( + cmdArgs, + ref i, + arg, + "--parallel", + 1, + BatchMaxParallelism, + out parallelism)) + { + return CommandExitCodes.UsageError; + } + parallelismSpecified = true; + continue; + } + CommandErrorWriter.WriteStderr($"Error: {ConsoleUi.FormatBoundedValue(arg)} is not supported for batch."); CommandErrorWriter.WriteStderr($"Usage: {ConsoleUi.GetUsageLine("batch")}"); return CommandExitCodes.UsageError; } + if (parallelismSpecified && !jsonSummary) + { + CommandErrorWriter.WriteStderr("Error: --parallel requires --json-summary so concurrent child output can be isolated and emitted in input order."); + CommandErrorWriter.WriteStderr($"Usage: {ConsoleUi.GetUsageLine("batch")}"); + return CommandExitCodes.UsageError; + } + if (maxOutputCharsSpecified && !jsonSummary) + { + CommandErrorWriter.WriteStderr("Error: --max-output-chars requires --json-summary because ordinary batch output streams directly."); + CommandErrorWriter.WriteStderr($"Usage: {ConsoleUi.GetUsageLine("batch")}"); + return CommandExitCodes.UsageError; + } + var isUri = dbPath.StartsWith("file:", StringComparison.OrdinalIgnoreCase); if (!isUri && !File.Exists(dbPath)) { @@ -61,9 +137,28 @@ public static int RunBatch(string[] cmdArgs, JsonSerializerOptions jsonOptions, return CommandExitCodes.DatabaseError; } + if (parallelism > 1) + { + using (var validationDb = new DbContext(DbOpenIntent.QueryOnly, dbPath, cancellationToken)) + { + if (!validationDb.TryValidateIsCodeIndexDb(out var validationReason)) + return WriteInvalidCodeIndexDbError(dbPath, validationReason, json: false, jsonOptions); + } + + return RunBatchParallel( + dbPath, + dbPathExplicit, + maxInputLines, + maxOutputChars, + parallelism, + jsonOptions, + appVersion, + cancellationToken); + } + try { - using var db = new DbContext(DbOpenIntent.QueryOnly, dbPath); + using var db = new DbContext(DbOpenIntent.QueryOnly, dbPath, cancellationToken); if (!db.TryValidateIsCodeIndexDb(out var validationReason)) return WriteInvalidCodeIndexDbError(dbPath, validationReason, json: false, jsonOptions); @@ -73,7 +168,7 @@ public static int RunBatch(string[] cmdArgs, JsonSerializerOptions jsonOptions, var jsonOutput = jsonSummary ? new BatchJsonOutputWriter( Console.Out, - BatchMaxTotalOutputChars, + maxOutputChars, BatchTerminalOutputReserveChars, jsonOptions) : null; @@ -86,11 +181,12 @@ public static int RunBatch(string[] cmdArgs, JsonSerializerOptions jsonOptions, var inputLimitReached = false; while (TryReadBatchLine(Console.In, out var line, out var lineExceededLimit)) { + cancellationToken.ThrowIfCancellationRequested(); lineNumber++; - if (lineNumber > BatchMaxInputLines) + if (lineNumber > maxInputLines) { var lineError = new BatchLineError( - $"batch input exceeds the {BatchMaxInputLines} line limit.", + $"batch input exceeds the {maxInputLines} line limit.", CommandExitCodes.UsageError, Hint: "Split the request into smaller batch invocations.", ErrorCode: CommandErrorCodes.UsageError, @@ -116,7 +212,12 @@ public static int RunBatch(string[] cmdArgs, JsonSerializerOptions jsonOptions, { if (!WriteBatchLineErrorJson(lineNumber, lineError, jsonOutput!)) { - WriteBatchOutputLimitErrorJson(lineNumber, commandName: null, CommandExitCodes.UsageError, jsonOutput!); + WriteBatchOutputLimitErrorJson( + lineNumber, + commandName: null, + CommandExitCodes.UsageError, + maxOutputChars, + jsonOutput!); outputLimitReached = true; } } @@ -142,7 +243,12 @@ public static int RunBatch(string[] cmdArgs, JsonSerializerOptions jsonOptions, parseError ?? BuildGenericBatchLineError(lineNumber), jsonOutput!)) { - WriteBatchOutputLimitErrorJson(lineNumber, commandName: null, parseExitCode, jsonOutput!); + WriteBatchOutputLimitErrorJson( + lineNumber, + commandName: null, + parseExitCode, + maxOutputChars, + jsonOutput!); outputLimitReached = true; } } @@ -156,9 +262,17 @@ public static int RunBatch(string[] cmdArgs, JsonSerializerOptions jsonOptions, commandsProcessed++; var batchResult = jsonSummary - ? RunBatchQueryCommandWithJsonRecord(lineNumber, commandName, subArgs, jsonOutput!, jsonOptions, appVersion) + ? RunBatchQueryCommandWithJsonRecord( + lineNumber, + commandName, + subArgs, + maxOutputChars, + jsonOutput!, + jsonOptions, + appVersion, + cancellationToken) : new BatchCommandRunResult( - RunBatchQueryCommand(commandName, subArgs, jsonOptions, appVersion), + RunBatchQueryCommand(commandName, subArgs, jsonOptions, appVersion, cancellationToken), OutputLimitReached: false); var exitCode = batchResult.ExitCode; if (exitCode != CommandExitCodes.Success) @@ -183,6 +297,9 @@ public static int RunBatch(string[] cmdArgs, JsonSerializerOptions jsonOptions, firstFailure, outputLimitReached, inputLimitReached, + maxInputLines, + maxOutputChars, + parallelism, jsonOutput!); return firstFailure; @@ -195,6 +312,461 @@ public static int RunBatch(string[] cmdArgs, JsonSerializerOptions jsonOptions, } } + private static bool TryReadBatchBoundedOption( + string[] args, + ref int index, + string currentArg, + string optionName, + int minimum, + int maximum, + out int value) + { + value = 0; + string rawValue; + if (currentArg == optionName) + { + if (index + 1 >= args.Length || string.IsNullOrWhiteSpace(args[index + 1])) + { + CommandErrorWriter.WriteStderr(BuildMissingOptionValueError(optionName)); + return false; + } + rawValue = args[++index]; + } + else + { + rawValue = currentArg[(optionName.Length + 1)..]; + if (string.IsNullOrWhiteSpace(rawValue)) + { + CommandErrorWriter.WriteStderr(BuildMissingOptionValueError(optionName)); + return false; + } + } + + if (!int.TryParse(rawValue, NumberStyles.None, CultureInfo.InvariantCulture, out value) + || value < minimum + || value > maximum) + { + CommandErrorWriter.WriteStderr($"Error: {optionName} must be an integer from {minimum} to {maximum}."); + CommandErrorWriter.WriteStderr($"Usage: {ConsoleUi.GetUsageLine("batch")}"); + return false; + } + + return true; + } + + private static int RunBatchParallel( + string dbPath, + bool dbPathExplicit, + int maxInputLines, + int maxOutputChars, + int parallelism, + JsonSerializerOptions jsonOptions, + string appVersion, + CancellationToken cancellationToken) + { + var originalOut = Console.Out; + var originalError = Console.Error; + var stdoutRouter = new BatchConsoleRouter(originalOut); + var stderrRouter = new BatchConsoleRouter(originalError); + var jsonOutput = new BatchJsonOutputWriter( + originalOut, + maxOutputChars, + BatchTerminalOutputReserveChars, + jsonOptions); + var firstFailure = CommandExitCodes.Success; + var lineNumber = 0; + var commandsProcessed = 0; + var lineErrors = 0; + var commandFailures = 0; + var outputLimitReached = false; + var inputLimitReached = false; + using var stopProducing = new CancellationTokenSource(); + using var producerCancellation = CancellationTokenSource.CreateLinkedTokenSource( + cancellationToken, + stopProducing.Token); + var input = Channel.CreateBounded( + new BoundedChannelOptions(1) + { + FullMode = BoundedChannelFullMode.Wait, + SingleReader = true, + SingleWriter = true, + }); + + Console.SetOut(stdoutRouter); + Console.SetError(stderrRouter); + try + { + var producer = Task.Run(() => + { + try + { + while (!stopProducing.IsCancellationRequested) + { + cancellationToken.ThrowIfCancellationRequested(); + if (!TryReadBatchLine(Console.In, out var line, out var lineExceededLimit)) + break; + + cancellationToken.ThrowIfCancellationRequested(); + lineNumber++; + if (lineNumber > maxInputLines) + { + var lineError = new BatchLineError( + $"batch input exceeds the {maxInputLines} line limit.", + CommandExitCodes.UsageError, + Hint: "Split the request into smaller batch invocations.", + ErrorCode: CommandErrorCodes.UsageError, + Category: "batch_input_line_limit"); + input.Writer.WriteAsync( + new BatchPendingItem(lineNumber, null, [], lineError, Terminal: true), + producerCancellation.Token) + .AsTask() + .GetAwaiter() + .GetResult(); + lineErrors++; + inputLimitReached = true; + break; + } + + if (lineExceededLimit) + { + var lineError = new BatchLineError( + $"batch line {lineNumber} exceeds the {BatchMaxLineChars} character limit.", + CommandExitCodes.UsageError, + ErrorCode: CommandErrorCodes.UsageError); + input.Writer.WriteAsync( + new BatchPendingItem(lineNumber, null, [], lineError, Terminal: false), + producerCancellation.Token) + .AsTask() + .GetAwaiter() + .GetResult(); + lineErrors++; + continue; + } + + if (string.IsNullOrWhiteSpace(line)) + continue; + + BatchPendingItem item; + if (!TryParseBatchLine( + line, + lineNumber, + jsonOptions, + writeDiagnostics: false, + out var commandName, + out var subArgs, + out _, + out var parseError)) + { + item = new BatchPendingItem( + lineNumber, + null, + [], + parseError ?? BuildGenericBatchLineError(lineNumber), + Terminal: false); + lineErrors++; + } + else + { + item = new BatchPendingItem( + lineNumber, + commandName, + subArgs, + null, + Terminal: false); + commandsProcessed++; + } + + input.Writer.WriteAsync(item, producerCancellation.Token) + .AsTask() + .GetAwaiter() + .GetResult(); + } + + input.Writer.TryComplete(); + } + catch (Exception ex) + { + input.Writer.TryComplete(ex); + throw; + } + }); + var active = new Queue<( + BatchPendingItem Item, + Task Result)>(); + + try + { + while (!outputLimitReached) + { + while (active.Count < parallelism && input.Reader.TryRead(out var item)) + { + var result = item.Error is not null + ? Task.FromResult(null) + : Task.Run( + () => RunBatchParallelCommand( + item.LineNumber, + item.CommandName!, + item.Arguments, + dbPath, + dbPathExplicit, + stdoutRouter, + stderrRouter, + jsonOptions, + appVersion, + cancellationToken), + cancellationToken); + active.Enqueue((item, result)); + } + + if (active.Count > 0 + && (active.Peek().Result.IsCompleted + || active.Count == parallelism + || input.Reader.Completion.IsCompleted)) + { + var (item, resultTask) = active.Dequeue(); + var result = resultTask.GetAwaiter().GetResult(); + if (item.Error is not null) + { + if (firstFailure == CommandExitCodes.Success) + firstFailure = item.Error.ExitCode; + + if (item.Terminal) + { + jsonOutput.WriteTerminal(BuildBatchLineErrorJson(item.LineNumber, item.Error)); + continue; + } + + if (!WriteBatchLineErrorJson(item.LineNumber, item.Error, jsonOutput)) + { + WriteBatchOutputLimitErrorJson( + item.LineNumber, + commandName: null, + item.Error.ExitCode, + maxOutputChars, + jsonOutput); + outputLimitReached = true; + break; + } + continue; + } + + if (result is null) + { + throw new InvalidOperationException( + "A parallel batch command completed without a result."); + } + if (WriteBatchCommandRecordJson( + item.LineNumber, + item.CommandName!, + item.Arguments, + result.ExitCode, + result.Stdout, + result.Stderr, + result.Error, + ClassifyBatchOutput(item.CommandName!, item.Arguments), + jsonOutput)) + { + if (result.ExitCode != CommandExitCodes.Success) + { + commandFailures++; + if (firstFailure == CommandExitCodes.Success) + firstFailure = result.ExitCode; + } + continue; + } + + WriteBatchOutputLimitErrorJson( + item.LineNumber, + item.CommandName, + result.ExitCode, + maxOutputChars, + jsonOutput); + outputLimitReached = true; + commandFailures++; + if (firstFailure == CommandExitCodes.Success) + firstFailure = CommandExitCodes.InvalidArgument; + break; + } + + if (outputLimitReached) + break; + if (active.Count == 0 && input.Reader.Completion.IsCompleted) + break; + if (active.Count > 0 && active.Peek().Result.IsCompleted) + continue; + + using var waitCancellation = CancellationTokenSource.CreateLinkedTokenSource( + cancellationToken); + var waitForInput = input.Reader.WaitToReadAsync(waitCancellation.Token).AsTask(); + if (active.Count == 0) + { + if (!waitForInput.GetAwaiter().GetResult()) + break; + continue; + } + + var completed = Task.WhenAny(active.Peek().Result, waitForInput) + .GetAwaiter() + .GetResult(); + if (ReferenceEquals(completed, active.Peek().Result)) + { + waitCancellation.Cancel(); + try + { + waitForInput.GetAwaiter().GetResult(); + } + catch (OperationCanceledException) when (waitCancellation.IsCancellationRequested) + { + } + } + } + } + catch + { + stopProducing.Cancel(); + while (active.Count > 0) + { + try + { + active.Dequeue().Result.GetAwaiter().GetResult(); + } + catch + { + // Preserve the first failure while ensuring sibling workers have exited. + } + } + + if (producer.IsCompleted) + { + try + { + producer.GetAwaiter().GetResult(); + } + catch + { + // Preserve the first failure from the consumer or ordered worker. + } + } + else + { + _ = producer.ContinueWith( + static completedProducer => _ = completedProducer.Exception, + CancellationToken.None, + TaskContinuationOptions.ExecuteSynchronously | TaskContinuationOptions.OnlyOnFaulted, + TaskScheduler.Default); + } + throw; + } + + if (outputLimitReached) + { + stopProducing.Cancel(); + while (active.Count > 0) + active.Dequeue().Result.GetAwaiter().GetResult(); + } + + try + { + producer.GetAwaiter().GetResult(); + } + catch (OperationCanceledException) when ( + outputLimitReached + && !cancellationToken.IsCancellationRequested) + { + } + + WriteBatchSummaryJson( + lineNumber, + commandsProcessed, + lineErrors, + commandFailures, + firstFailure, + outputLimitReached, + inputLimitReached, + maxInputLines, + maxOutputChars, + parallelism, + jsonOutput); + return firstFailure; + } + finally + { + Console.SetOut(originalOut); + Console.SetError(originalError); + } + } + + private static BatchParallelCommandResult RunBatchParallelCommand( + int lineNumber, + string commandName, + string[] subArgs, + string dbPath, + bool dbPathExplicit, + BatchConsoleRouter stdoutRouter, + BatchConsoleRouter stderrRouter, + JsonSerializerOptions jsonOptions, + string appVersion, + CancellationToken cancellationToken) + { + using var stdout = new BatchBoundedStringWriter(BatchMaxCapturedOutputChars, "stdout"); + using var stderr = new BatchBoundedStringWriter(BatchMaxCapturedOutputChars, "stderr"); + using var stdoutRouterRegistration = ScopedConsoleOutput.Register(stdoutRouter); + using var stdoutScope = stdoutRouter.Push(stdout); + using var stderrScope = stderrRouter.Push(stderr); + var exitCode = CommandExitCodes.DatabaseError; + JsonObject? error = null; + try + { + cancellationToken.ThrowIfCancellationRequested(); + using var db = new DbContext(DbOpenIntent.QueryOnly, dbPath, cancellationToken); + if (!db.TryValidateIsCodeIndexDb(out var validationReason)) + { + exitCode = WriteInvalidCodeIndexDbError(dbPath, validationReason, json: false, jsonOptions); + } + else + { + s_batchReader = new DbReader(db); + s_batchDbPath = dbPath; + s_batchDbPathExplicit = dbPathExplicit; + BatchParallelCommandStartedForTesting?.Invoke(lineNumber); + cancellationToken.ThrowIfCancellationRequested(); + exitCode = RunBatchQueryCommand(commandName, subArgs, jsonOptions, appVersion, cancellationToken); + BatchParallelCommandCompletedForTesting?.Invoke(lineNumber); + } + } + catch (BatchOutputCaptureLimitExceededException ex) + { + exitCode = CommandExitCodes.InvalidArgument; + error = BuildBatchCaptureLimitError(ex); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception ex) + { + exitCode = CommandExitCodes.DatabaseError; + error = new JsonObject + { + ["message"] = "batch command failed without affecting other batch items.", + ["error_code"] = CommandErrorCodes.DbError, + ["category"] = SafeDiagnosticFormatter.FormatCategoryType( + "batch_command_failure", + ex.GetType().Name), + ["scope"] = "command", + }; + } + finally + { + s_batchReader = null; + s_batchDbPath = null; + s_batchDbPathExplicit = false; + s_activeQueryProjectRoot = null; + } + + return new BatchParallelCommandResult(exitCode, stdout.ToString(), stderr.ToString(), error); + } + private static void WriteBatchSummaryJson( int inputLinesRead, int commandsProcessed, @@ -203,6 +775,9 @@ private static void WriteBatchSummaryJson( int exitCode, bool outputLimitReached, bool inputLimitReached, + int inputLineLimit, + int outputCharLimit, + int parallelism, BatchJsonOutputWriter output) { var payload = new JsonObject @@ -216,10 +791,11 @@ private static void WriteBatchSummaryJson( ["command_failures"] = commandFailures, ["exit_code"] = exitCode, ["output_chars"] = 0, - ["output_char_limit"] = BatchMaxTotalOutputChars, + ["output_char_limit"] = outputCharLimit, ["output_limit_reached"] = outputLimitReached, - ["input_line_limit"] = BatchMaxInputLines, + ["input_line_limit"] = inputLineLimit, ["input_limit_reached"] = inputLimitReached, + ["parallelism"] = parallelism, }; output.WriteSummary(payload); @@ -229,9 +805,11 @@ private static BatchCommandRunResult RunBatchQueryCommandWithJsonRecord( int lineNumber, string commandName, string[] subArgs, + int outputCharLimit, BatchJsonOutputWriter output, JsonSerializerOptions jsonOptions, - string appVersion) + string appVersion, + CancellationToken cancellationToken) { using var capture = new BatchCommandOutputCapture(); int exitCode; @@ -239,7 +817,7 @@ private static BatchCommandRunResult RunBatchQueryCommandWithJsonRecord( try { capture.Start(); - exitCode = RunBatchQueryCommand(commandName, subArgs, jsonOptions, appVersion); + exitCode = RunBatchQueryCommand(commandName, subArgs, jsonOptions, appVersion, cancellationToken); } catch (BatchOutputCaptureLimitExceededException ex) { @@ -253,18 +831,7 @@ private static BatchCommandRunResult RunBatchQueryCommandWithJsonRecord( JsonObject? error = null; if (captureLimitExceeded is not null) - { - var message = $"batch command {captureLimitExceeded.StreamName} exceeded {captureLimitExceeded.MaxChars} captured characters."; - error = new JsonObject - { - ["message"] = message, - ["hint"] = "Reduce the result set or run cdidx batch without --json-summary for streaming output.", - ["error_code"] = CommandErrorCodes.UsageError, - ["max_chars"] = captureLimitExceeded.MaxChars, - ["stream"] = captureLimitExceeded.StreamName, - ["scope"] = "command", - }; - } + error = BuildBatchCaptureLimitError(captureLimitExceeded); var recordWritten = WriteBatchCommandRecordJson( lineNumber, @@ -279,10 +846,23 @@ private static BatchCommandRunResult RunBatchQueryCommandWithJsonRecord( if (recordWritten) return new BatchCommandRunResult(exitCode, OutputLimitReached: false); - WriteBatchOutputLimitErrorJson(lineNumber, commandName, exitCode, output); + WriteBatchOutputLimitErrorJson(lineNumber, commandName, exitCode, outputCharLimit, output); return new BatchCommandRunResult(CommandExitCodes.InvalidArgument, OutputLimitReached: true); } + private static JsonObject BuildBatchCaptureLimitError(BatchOutputCaptureLimitExceededException exception) + { + return new JsonObject + { + ["message"] = $"batch command {exception.StreamName} exceeded {exception.MaxChars} captured characters.", + ["hint"] = "Reduce the result set or run cdidx batch without --json-summary for streaming output.", + ["error_code"] = CommandErrorCodes.UsageError, + ["max_chars"] = exception.MaxChars, + ["stream"] = exception.StreamName, + ["scope"] = "command", + }; + } + private static bool WriteBatchCommandRecordJson( int lineNumber, string commandName, @@ -487,16 +1067,17 @@ private static void WriteBatchOutputLimitErrorJson( int lineNumber, string? commandName, int attemptedExitCode, + int outputCharLimit, BatchJsonOutputWriter output) { var error = new JsonObject { - ["message"] = $"batch serialized output reached the {BatchMaxTotalOutputChars} character limit.", + ["message"] = $"batch serialized output reached the {outputCharLimit} character limit.", ["hint"] = "Split the request into smaller batches or reduce child output with --limit/--top.", ["error_code"] = CommandErrorCodes.UsageError, ["category"] = "batch_output_limit", ["scope"] = "batch", - ["max_chars"] = BatchMaxTotalOutputChars, + ["max_chars"] = outputCharLimit, ["attempted_exit_code"] = attemptedExitCode, }; var payload = new JsonObject @@ -597,10 +1178,23 @@ private static bool TryParseBatchLine( try { using var document = BoundedJson.ParseDocument(line, BatchMaxLineUtf8Bytes, BatchMaxJsonDepth); + if (document.RootElement.ValueKind == JsonValueKind.Object) + { + return TryParseBatchCommandObject( + document.RootElement, + lineNumber, + jsonOptions, + writeDiagnostics, + out commandName, + out subArgs, + out exitCode, + out error); + } + if (document.RootElement.ValueKind != JsonValueKind.Array || document.RootElement.GetArrayLength() == 0) { error = new BatchLineError( - $"batch line {lineNumber} must be a non-empty JSON string array.", + $"batch line {lineNumber} must be a non-empty JSON string array or a command object.", CommandExitCodes.UsageError, ErrorCode: CommandErrorCodes.UsageError); if (writeDiagnostics) @@ -654,7 +1248,7 @@ private static bool TryParseBatchLine( error = new BatchLineError( $"batch line {lineNumber} {SafeDiagnosticFormatter.FormatCategoryType("invalid_batch_json", nameof(JsonException))}.", CommandExitCodes.UsageError, - Hint: "ensure each batch input line is a JSON string array.", + Hint: "ensure each batch input line is a JSON string array or a {\"command\",\"args\"} object.", ErrorCode: CommandErrorCodes.UsageError, Category: "invalid_batch_json", WriteAsJson: true); @@ -664,6 +1258,128 @@ private static bool TryParseBatchLine( } } + private static bool TryParseBatchCommandObject( + JsonElement root, + int lineNumber, + JsonSerializerOptions jsonOptions, + bool writeDiagnostics, + out string commandName, + out string[] subArgs, + out int exitCode, + out BatchLineError? error) + { + commandName = string.Empty; + subArgs = []; + exitCode = CommandExitCodes.UsageError; + error = null; + JsonElement commandElement = default; + JsonElement argumentsElement = default; + var commandSeen = false; + var argumentsSeen = false; + + foreach (var property in root.EnumerateObject()) + { + if (property.NameEquals("command")) + { + if (commandSeen) + { + error = BuildBatchObjectError(lineNumber, "must not repeat the command property."); + return WriteBatchObjectErrorIfNeeded(error, jsonOptions, writeDiagnostics); + } + commandSeen = true; + commandElement = property.Value; + continue; + } + + if (property.NameEquals("args")) + { + if (argumentsSeen) + { + error = BuildBatchObjectError(lineNumber, "must not repeat the args property."); + return WriteBatchObjectErrorIfNeeded(error, jsonOptions, writeDiagnostics); + } + argumentsSeen = true; + argumentsElement = property.Value; + continue; + } + + error = BuildBatchObjectError(lineNumber, "contains an unsupported property."); + return WriteBatchObjectErrorIfNeeded(error, jsonOptions, writeDiagnostics); + } + + if (!commandSeen + || commandElement.ValueKind != JsonValueKind.String + || string.IsNullOrWhiteSpace(commandElement.GetString())) + { + error = BuildBatchObjectError(lineNumber, "requires a non-empty string command property."); + return WriteBatchObjectErrorIfNeeded(error, jsonOptions, writeDiagnostics); + } + + commandName = commandElement.GetString()!; + if (commandName.Length > BatchMaxArgumentChars) + { + error = BuildBatchObjectError( + lineNumber, + $"command exceeds the {BatchMaxArgumentChars} character limit."); + return WriteBatchObjectErrorIfNeeded(error, jsonOptions, writeDiagnostics); + } + + if (!argumentsSeen) + return true; + if (argumentsElement.ValueKind != JsonValueKind.Array) + { + error = BuildBatchObjectError(lineNumber, "requires args to be a JSON string array."); + return WriteBatchObjectErrorIfNeeded(error, jsonOptions, writeDiagnostics); + } + if (argumentsElement.GetArrayLength() > BatchMaxArgumentCount) + { + error = BuildBatchObjectError( + lineNumber, + $"must contain at most {BatchMaxArgumentCount} command arguments."); + return WriteBatchObjectErrorIfNeeded(error, jsonOptions, writeDiagnostics); + } + + var values = new List(argumentsElement.GetArrayLength()); + foreach (var element in argumentsElement.EnumerateArray()) + { + if (element.ValueKind != JsonValueKind.String) + { + error = BuildBatchObjectError(lineNumber, "args must contain only strings."); + return WriteBatchObjectErrorIfNeeded(error, jsonOptions, writeDiagnostics); + } + + var value = element.GetString() ?? string.Empty; + if (value.Length > BatchMaxArgumentChars) + { + error = BuildBatchObjectError( + lineNumber, + $"argument {values.Count + 1} exceeds the {BatchMaxArgumentChars} character limit."); + return WriteBatchObjectErrorIfNeeded(error, jsonOptions, writeDiagnostics); + } + values.Add(value); + } + + subArgs = values.ToArray(); + return true; + } + + private static BatchLineError BuildBatchObjectError(int lineNumber, string detail) + => new( + $"batch line {lineNumber} command object {detail}", + CommandExitCodes.UsageError, + ErrorCode: CommandErrorCodes.UsageError, + Category: "invalid_batch_command_object"); + + private static bool WriteBatchObjectErrorIfNeeded( + BatchLineError error, + JsonSerializerOptions jsonOptions, + bool writeDiagnostics) + { + if (writeDiagnostics) + WriteBatchLineErrorDiagnostic(error, jsonOptions); + return false; + } + private static void WriteBatchLineErrorDiagnostic(BatchLineError error, JsonSerializerOptions jsonOptions) { if (error.WriteAsJson) @@ -690,16 +1406,21 @@ private static BatchLineError BuildGenericBatchLineError(int lineNumber) CommandExitCodes.UsageError, ErrorCode: CommandErrorCodes.UsageError); - private static int RunBatchQueryCommand(string commandName, string[] subArgs, JsonSerializerOptions jsonOptions, string appVersion) + private static int RunBatchQueryCommand( + string commandName, + string[] subArgs, + JsonSerializerOptions jsonOptions, + string appVersion, + CancellationToken cancellationToken) { if (!CliCommandCatalog.IsBatchReadOnlyCommand(commandName)) return WriteBatchUnsupportedCommand(commandName); Func runner = commandName switch { - "search" => args => RunSearch(args, jsonOptions), - "recipes" => args => RunRecipes(args, jsonOptions), - "audit" => args => RunAudit(args, jsonOptions), + "search" => args => RunSearch(args, jsonOptions, cancellationToken), + "recipes" => args => RunRecipes(args, jsonOptions, cancellationToken), + "audit" => args => RunAudit(args, jsonOptions, cancellationToken), "definition" => args => RunDefinition(args, jsonOptions), "goto" => args => RunGoto(args, jsonOptions), "references" => args => RunReferences(args, jsonOptions), @@ -712,11 +1433,11 @@ private static int RunBatchQueryCommand(string commandName, string[] subArgs, Js "map" => args => RunMap(args, jsonOptions), "inspect" => args => RunInspect(args, jsonOptions), "outline" => args => RunOutline(args, jsonOptions), - "status" => args => RunStatus(args, jsonOptions), + "status" => args => RunStatus(args, jsonOptions, appVersion, cancellationToken), "validate" => args => RunValidate(args, jsonOptions), "languages" => args => RunLanguages(args, jsonOptions), "impact" => args => RunImpact(args, jsonOptions), - "deps" => args => RunDeps(args, jsonOptions), + "deps" => args => RunDeps(args, jsonOptions, cancellationToken), "unused" => args => RunUnused(args, jsonOptions), "hotspots" => args => RunHotspots(args, jsonOptions), _ => throw new InvalidOperationException($"Batch schema command '{commandName}' has no dispatcher."), @@ -744,6 +1465,19 @@ private sealed record BatchLineError( private sealed record BatchCommandRunResult(int ExitCode, bool OutputLimitReached); + private sealed record BatchPendingItem( + int LineNumber, + string? CommandName, + string[] Arguments, + BatchLineError? Error, + bool Terminal); + + private sealed record BatchParallelCommandResult( + int ExitCode, + string Stdout, + string Stderr, + JsonObject? Error); + private enum BatchOutputKind { Text, @@ -751,6 +1485,56 @@ private enum BatchOutputKind Ndjson, } + private sealed class BatchConsoleRouter(TextWriter fallback) : TextWriter, IScopedConsoleOutputRouter + { + private readonly AsyncLocal _target = new(); + + public override Encoding Encoding => fallback.Encoding; + + public IDisposable Push(TextWriter target) + { + var previous = _target.Value; + _target.Value = target; + return new BatchConsoleRouteScope(this, previous); + } + + public override void Write(char value) + => Current.Write(value); + + public override void Write(string? value) + => Current.Write(value); + + public override void Write(char[]? buffer, int index, int count) + { + if (buffer is null) + return; + Current.Write(buffer, index, count); + } + + public override void Write(ReadOnlySpan buffer) + => Current.Write(buffer); + + public override void Flush() + => Current.Flush(); + + private TextWriter Current => _target.Value ?? fallback; + + private sealed class BatchConsoleRouteScope( + BatchConsoleRouter owner, + TextWriter? previous) : IDisposable + { + private bool _disposed; + + public void Dispose() + { + if (_disposed) + return; + _disposed = true; + owner._target.Value = previous; + } + } + } + private sealed class BatchJsonOutputWriter( TextWriter output, int maxChars, diff --git a/src/CodeIndex/Cli/QueryCommandRunner.BatchLimits.cs b/src/CodeIndex/Cli/QueryCommandRunner.BatchLimits.cs index 0788361f6..ca43181f8 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.BatchLimits.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.BatchLimits.cs @@ -7,7 +7,11 @@ public static partial class QueryCommandRunner internal const int BatchMaxArgumentCount = 256; internal const int BatchMaxArgumentChars = 8192; internal const int BatchMaxJsonDepth = 32; - internal const int BatchMaxInputLines = 1024; - internal const int BatchMaxTotalOutputChars = JsonEnvelopeWrapper.MaxCapturedOutputChars; + internal const int BatchDefaultInputLines = 1024; + internal const int BatchMaxInputLines = 64 * 1024; + internal const int BatchDefaultTotalOutputChars = JsonEnvelopeWrapper.MaxCapturedOutputChars; + internal const int BatchMinTotalOutputChars = 4096; + internal const int BatchMaxTotalOutputChars = 64 * 1024 * 1024; + internal const int BatchMaxParallelism = 16; internal const int BatchTerminalOutputReserveChars = 4096; } diff --git a/src/CodeIndex/Cli/ScopedConsoleOutput.cs b/src/CodeIndex/Cli/ScopedConsoleOutput.cs new file mode 100644 index 000000000..4e19b4abf --- /dev/null +++ b/src/CodeIndex/Cli/ScopedConsoleOutput.cs @@ -0,0 +1,41 @@ +namespace CodeIndex.Cli; + +internal interface IScopedConsoleOutputRouter +{ + IDisposable Push(TextWriter target); +} + +internal static class ScopedConsoleOutput +{ + private static readonly AsyncLocal s_activeRouter = new(); + + internal static IDisposable Register(IScopedConsoleOutputRouter router) + { + var previous = s_activeRouter.Value; + s_activeRouter.Value = router; + return new DelegateScope(() => s_activeRouter.Value = previous); + } + + internal static IDisposable Redirect(TextWriter target) + { + if (s_activeRouter.Value is { } router) + return router.Push(target); + + var original = Console.Out; + Console.SetOut(target); + return new DelegateScope(() => Console.SetOut(original)); + } + + private sealed class DelegateScope(Action dispose) : IDisposable + { + private bool _disposed; + + public void Dispose() + { + if (_disposed) + return; + _disposed = true; + dispose(); + } + } +} diff --git a/tests/CodeIndex.Tests/ConsoleUiTests.cs b/tests/CodeIndex.Tests/ConsoleUiTests.cs index f7a5cae0f..54b646798 100644 --- a/tests/CodeIndex.Tests/ConsoleUiTests.cs +++ b/tests/CodeIndex.Tests/ConsoleUiTests.cs @@ -337,11 +337,15 @@ public void PrintCommandUsage_DocumentsJsonLineAndBatchContracts_Issue3916() Assert.Contains("JSON Lines", referencesStdout); Assert.Contains("one JSON object per result", referencesStdout); Assert.Contains("stdin is JSON Lines", batchStdout); - Assert.Contains("--json-summary embeds typed child JSON plus a final summary", batchStdout); + Assert.Contains("--max-input-lines ", batchStdout); + Assert.Contains("--max-output-chars ", batchStdout); + Assert.Contains("--parallel ", batchStdout); + Assert.Contains("\"command\":\"search\"", batchStdout); Assert.Contains("batch_result or batch_error envelope", batchStdout); Assert.Contains("JSON is embedded as result, NDJSON as stable results", batchStdout); - Assert.Contains("full serialized stream is capped", batchStdout); - Assert.Contains("input lines", batchStdout); + Assert.Contains("configured serialized-output budgets", batchStdout); + Assert.Contains("safe maxima", batchStdout); + Assert.Contains("stable results in input order", batchStdout); Assert.Contains("batch_summary record", batchStdout); } diff --git a/tests/CodeIndex.Tests/QueryCommandRunnerBatchIssue4723Tests.cs b/tests/CodeIndex.Tests/QueryCommandRunnerBatchIssue4723Tests.cs new file mode 100644 index 000000000..cb4bac948 --- /dev/null +++ b/tests/CodeIndex.Tests/QueryCommandRunnerBatchIssue4723Tests.cs @@ -0,0 +1,435 @@ +using CodeIndex.Cli; + +namespace CodeIndex.Tests; + +public partial class QueryCommandRunnerTests +{ + [Fact] + public void RunBatch_AcceptsStructuredCommandsAndValidatesTheirShape_Issue4723() + { + using var project = TestProjectHelper.CreateTempProjectScope("cdidx_batch_structured_4723"); + var dbPath = TestProjectHelper.CreateProjectDb(project.Root); + var input = """ + {"command":"status","args":["--json"]} + {"command":"languages","args":["--format","count"]} + {"command":"status","args":"--json"} + {"command":"status","extra":true} + + """; + + var (exitCode, stdout, stderr) = CaptureConsoleWithInput( + input, + () => QueryCommandRunner.RunBatch(["--db", dbPath, "--json-summary"], _jsonOptions)); + var lines = ParseJsonLines(stdout); + try + { + Assert.Equal(CommandExitCodes.UsageError, exitCode); + Assert.Equal(string.Empty, stderr); + Assert.Equal(5, lines.Count); + + Assert.Equal("status", lines[0].RootElement.GetProperty("command").GetString()); + Assert.True( + lines[0].RootElement.TryGetProperty("result", out var statusResult), + lines[0].RootElement.GetRawText()); + Assert.True(statusResult.TryGetProperty("files", out _)); + Assert.Equal("languages", lines[1].RootElement.GetProperty("command").GetString()); + Assert.Equal("count", lines[1].RootElement.GetProperty("result").GetProperty("format").GetString()); + + foreach (var errorDocument in lines.Skip(2).Take(2)) + { + var errorRecord = errorDocument.RootElement; + Assert.Equal("batch_error", errorRecord.GetProperty("record").GetString()); + Assert.Equal( + "invalid_batch_command_object", + errorRecord.GetProperty("error").GetProperty("category").GetString()); + } + + var summary = lines[^1].RootElement; + Assert.Equal(2, summary.GetProperty("commands_processed").GetInt32()); + Assert.Equal(2, summary.GetProperty("line_errors").GetInt32()); + } + finally + { + foreach (var document in lines) + document.Dispose(); + } + } + + [Fact] + public void RunBatch_ConfigurableBudgetsUseEffectiveValuesAndRejectUnsafeValues_Issue4723() + { + using var project = TestProjectHelper.CreateTempProjectScope("cdidx_batch_budgets_4723"); + var dbPath = TestProjectHelper.CreateProjectDb(project.Root); + var input = """ + [] + [] + [] + + """; + + var (exitCode, stdout, stderr) = CaptureConsoleWithInput( + input, + () => QueryCommandRunner.RunBatch( + [ + "--db", dbPath, + "--json-summary", + "--max-input-lines", "2", + "--max-output-chars=8192", + ], + _jsonOptions)); + var lines = ParseJsonLines(stdout); + try + { + Assert.Equal(CommandExitCodes.UsageError, exitCode); + Assert.Equal(string.Empty, stderr); + var summary = lines[^1].RootElement; + Assert.Equal(2, summary.GetProperty("input_line_limit").GetInt32()); + Assert.Equal(8192, summary.GetProperty("output_char_limit").GetInt32()); + Assert.True(summary.GetProperty("input_limit_reached").GetBoolean()); + } + finally + { + foreach (var document in lines) + document.Dispose(); + } + + var (invalidExitCode, _, invalidStderr) = CaptureConsole(() => QueryCommandRunner.RunBatch( + ["--db", dbPath, "--json-summary", "--parallel", (QueryCommandRunner.BatchMaxParallelism + 1).ToString()], + _jsonOptions)); + Assert.Equal(CommandExitCodes.UsageError, invalidExitCode); + Assert.Contains($"from 1 to {QueryCommandRunner.BatchMaxParallelism}", invalidStderr); + + var (parallelWithoutSummaryExitCode, _, parallelWithoutSummaryStderr) = CaptureConsole( + () => QueryCommandRunner.RunBatch(["--db", dbPath, "--parallel", "2"], _jsonOptions)); + Assert.Equal(CommandExitCodes.UsageError, parallelWithoutSummaryExitCode); + Assert.Contains("--parallel requires --json-summary", parallelWithoutSummaryStderr); + + var (defaultParallelWithoutSummaryExitCode, _, defaultParallelWithoutSummaryStderr) = CaptureConsole( + () => QueryCommandRunner.RunBatch(["--db", dbPath, "--parallel", "1"], _jsonOptions)); + Assert.Equal(CommandExitCodes.UsageError, defaultParallelWithoutSummaryExitCode); + Assert.Contains("--parallel requires --json-summary", defaultParallelWithoutSummaryStderr); + + var (outputWithoutSummaryExitCode, _, outputWithoutSummaryStderr) = CaptureConsole( + () => QueryCommandRunner.RunBatch(["--db", dbPath, "--max-output-chars", "8192"], _jsonOptions)); + Assert.Equal(CommandExitCodes.UsageError, outputWithoutSummaryExitCode); + Assert.Contains("--max-output-chars requires --json-summary", outputWithoutSummaryStderr); + + var (defaultOutputWithoutSummaryExitCode, _, defaultOutputWithoutSummaryStderr) = CaptureConsole( + () => QueryCommandRunner.RunBatch( + [ + "--db", dbPath, + "--max-output-chars", QueryCommandRunner.BatchDefaultTotalOutputChars.ToString(), + ], + _jsonOptions)); + Assert.Equal(CommandExitCodes.UsageError, defaultOutputWithoutSummaryExitCode); + Assert.Contains("--max-output-chars requires --json-summary", defaultOutputWithoutSummaryStderr); + + var (inputAboveMaximumExitCode, _, inputAboveMaximumStderr) = CaptureConsole( + () => QueryCommandRunner.RunBatch( + [ + "--db", dbPath, + "--json-summary", + "--max-input-lines", (QueryCommandRunner.BatchMaxInputLines + 1).ToString(), + ], + _jsonOptions)); + Assert.Equal(CommandExitCodes.UsageError, inputAboveMaximumExitCode); + Assert.Contains($"from 1 to {QueryCommandRunner.BatchMaxInputLines}", inputAboveMaximumStderr); + + var (outputAboveMaximumExitCode, _, outputAboveMaximumStderr) = CaptureConsole( + () => QueryCommandRunner.RunBatch( + [ + "--db", dbPath, + "--json-summary", + "--max-output-chars", (QueryCommandRunner.BatchMaxTotalOutputChars + 1).ToString(), + ], + _jsonOptions)); + Assert.Equal(CommandExitCodes.UsageError, outputAboveMaximumExitCode); + Assert.Contains( + $"from {QueryCommandRunner.BatchMinTotalOutputChars} to {QueryCommandRunner.BatchMaxTotalOutputChars}", + outputAboveMaximumStderr); + } + + [Fact] + public void RunBatch_ParallelReadsOverlapButEmitInInputOrderAndIsolateFailures_Issue4723() + { + using var project = TestProjectHelper.CreateTempProjectScope("cdidx_batch_parallel_4723"); + var dbPath = TestProjectHelper.CreateProjectDb(project.Root); + using var secondCompleted = new ManualResetEventSlim(); + QueryCommandRunner.BatchParallelCommandStartedForTesting = lineNumber => + { + if (lineNumber == 1 && !secondCompleted.Wait(TimeSpan.FromSeconds(5))) + throw new TimeoutException("The second parallel batch command did not complete."); + }; + QueryCommandRunner.BatchParallelCommandCompletedForTesting = lineNumber => + { + if (lineNumber == 2) + secondCompleted.Set(); + }; + + try + { + var input = """ + {"command":"status","args":["--json-envelope"]} + {"command":"unknown"} + {"command":"languages","args":["--format","count"]} + + """; + var (exitCode, stdout, stderr) = CaptureConsoleWithInput( + input, + () => QueryCommandRunner.RunBatch( + ["--db", dbPath, "--json-summary", "--parallel", "3"], + _jsonOptions)); + var lines = ParseJsonLines(stdout); + try + { + Assert.Equal(CommandExitCodes.UsageError, exitCode); + Assert.Equal(string.Empty, stderr); + Assert.Equal(4, lines.Count); + Assert.Equal(1, lines[0].RootElement.GetProperty("line").GetInt32()); + Assert.Equal("status", lines[0].RootElement.GetProperty("command").GetString()); + Assert.Equal("ok", lines[0].RootElement.GetProperty("status").GetString()); + var envelopeStdout = lines[0].RootElement.GetProperty("stdout").GetString(); + Assert.Contains("\"metadata\":", envelopeStdout); + Assert.Contains("\"command\":\"status\"", envelopeStdout); + Assert.Equal(2, lines[1].RootElement.GetProperty("line").GetInt32()); + Assert.Equal("unknown", lines[1].RootElement.GetProperty("command").GetString()); + Assert.Equal("error", lines[1].RootElement.GetProperty("status").GetString()); + Assert.Equal(3, lines[2].RootElement.GetProperty("line").GetInt32()); + Assert.Equal("languages", lines[2].RootElement.GetProperty("command").GetString()); + Assert.Equal("ok", lines[2].RootElement.GetProperty("status").GetString()); + Assert.Equal(3, lines[^1].RootElement.GetProperty("parallelism").GetInt32()); + Assert.Equal(1, lines[^1].RootElement.GetProperty("command_failures").GetInt32()); + } + finally + { + foreach (var document in lines) + document.Dispose(); + } + } + finally + { + QueryCommandRunner.BatchParallelCommandStartedForTesting = null; + QueryCommandRunner.BatchParallelCommandCompletedForTesting = null; + } + } + + [Fact] + public async Task RunBatch_ParallelStreamsFirstResultBeforeMoreInputOrEof_Issue4723() + { + using var project = TestProjectHelper.CreateTempProjectScope("cdidx_batch_parallel_streaming_4723"); + var dbPath = TestProjectHelper.CreateProjectDb(project.Root); + using var input = new InteractiveBatchTextReader(); + using var firstRecordEmitted = new ManualResetEventSlim(); + using var stdout = new NotifyingStringWriter(firstRecordEmitted); + using var stderr = new StringWriter(); + var originalIn = Console.In; + var originalOut = Console.Out; + var originalError = Console.Error; + Task? runTask = null; + var emittedBeforeEof = false; + var exitCode = CommandExitCodes.UnhandledException; + + Console.SetIn(input); + Console.SetOut(stdout); + Console.SetError(stderr); + try + { + runTask = Task.Run(() => QueryCommandRunner.RunBatch( + ["--db", dbPath, "--json-summary", "--parallel", "2"], + _jsonOptions)); + input.WriteLine("""{"command":"languages","args":["--format","count"]}"""); + + emittedBeforeEof = firstRecordEmitted.Wait(TimeSpan.FromSeconds(15)); + Assert.False(runTask.IsCompleted); + + input.Complete(); + exitCode = await runTask.WaitAsync(TimeSpan.FromSeconds(15)); + } + finally + { + input.Complete(); + if (runTask is not null && !runTask.IsCompleted) + { + try + { + await runTask.WaitAsync(TimeSpan.FromSeconds(15)); + } + catch + { + // Preserve the primary assertion while still making cleanup bounded. + } + } + Console.SetIn(originalIn); + Console.SetOut(originalOut); + Console.SetError(originalError); + } + + Assert.True(emittedBeforeEof); + Assert.Equal(CommandExitCodes.Success, exitCode); + Assert.Equal(string.Empty, stderr.ToString()); + var lines = ParseJsonLines(stdout.ToString()); + try + { + Assert.Equal(2, lines.Count); + Assert.Equal("batch_result", lines[0].RootElement.GetProperty("record").GetString()); + Assert.Equal("batch_summary", lines[1].RootElement.GetProperty("record").GetString()); + } + finally + { + foreach (var document in lines) + document.Dispose(); + } + } + + [Fact] + public void RunBatch_ParallelFailureExitCodeFollowsInputOrder_Issue4723() + { + using var project = TestProjectHelper.CreateTempProjectScope("cdidx_batch_parallel_exit_order_4723"); + var dbPath = TestProjectHelper.CreateProjectDb(project.Root); + var input = """ + {"command":"search","args":["missing-symbol-4723","--json","--strict-not-found"]} + [] + + """; + + var (exitCode, stdout, stderr) = CaptureConsoleWithInput( + input, + () => QueryCommandRunner.RunBatch( + ["--db", dbPath, "--json-summary", "--parallel", "2"], + _jsonOptions)); + var lines = ParseJsonLines(stdout); + try + { + Assert.Equal(CommandExitCodes.NotFound, exitCode); + Assert.Equal(string.Empty, stderr); + Assert.Equal(CommandExitCodes.NotFound, lines[0].RootElement.GetProperty("exit_code").GetInt32()); + Assert.Equal(CommandExitCodes.UsageError, lines[1].RootElement.GetProperty("exit_code").GetInt32()); + Assert.Equal(CommandExitCodes.NotFound, lines[^1].RootElement.GetProperty("exit_code").GetInt32()); + } + finally + { + foreach (var document in lines) + document.Dispose(); + } + } + + [Fact] + public void RunBatch_ParallelReadsPropagateCancellationAndRestoreConsole_Issue4723() + { + using var project = TestProjectHelper.CreateTempProjectScope("cdidx_batch_cancel_4723"); + var dbPath = TestProjectHelper.CreateProjectDb(project.Root); + using var cancellation = new CancellationTokenSource(); + QueryCommandRunner.BatchParallelCommandStartedForTesting = lineNumber => + { + if (lineNumber == 1) + cancellation.Cancel(); + }; + + try + { + var exception = Record.Exception(() => CaptureConsoleWithInput( + """ + {"command":"recipes","args":["--json"]} + {"command":"languages","args":["--format","count"]} + + """, + () => QueryCommandRunner.RunBatch( + ["--db", dbPath, "--json-summary", "--parallel", "2"], + _jsonOptions, + cancellationToken: cancellation.Token))); + Assert.True(cancellation.IsCancellationRequested); + Assert.IsAssignableFrom(exception); + } + finally + { + QueryCommandRunner.BatchParallelCommandStartedForTesting = null; + } + + var (_, stdout, _) = CaptureConsole(() => + { + Console.Write("restored"); + return 0; + }); + Assert.Equal("restored", stdout); + } + + private sealed class InteractiveBatchTextReader : TextReader + { + private readonly System.Collections.Concurrent.BlockingCollection _characters = new(); + + public void WriteLine(string line) + { + foreach (var character in line) + _characters.Add(character); + _characters.Add('\n'); + } + + public void Complete() + { + if (!_characters.IsAddingCompleted) + _characters.CompleteAdding(); + } + + public override int Read() + { + try + { + return _characters.Take(); + } + catch (InvalidOperationException) when (_characters.IsCompleted) + { + return -1; + } + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + Complete(); + _characters.Dispose(); + } + base.Dispose(disposing); + } + } + + private sealed class NotifyingStringWriter(ManualResetEventSlim lineWritten) : StringWriter + { + private readonly object _sync = new(); + + public override void Write(char value) + { + lock (_sync) + { + base.Write(value); + if (value == '\n') + lineWritten.Set(); + } + } + + public override void Write(string? value) + { + lock (_sync) + { + base.Write(value); + if (value?.Contains('\n') == true) + lineWritten.Set(); + } + } + + public override void WriteLine(string? value) + { + lock (_sync) + { + base.WriteLine(value); + lineWritten.Set(); + } + } + + public override string ToString() + { + lock (_sync) + return base.ToString(); + } + } +} diff --git a/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs b/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs index 13ee0ab92..34263738b 100644 --- a/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs +++ b/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs @@ -945,10 +945,10 @@ public void RunBatch_JsonSummaryReportsEmptyInput_Issue3906() Assert.Equal(CommandExitCodes.Success, summary.GetProperty("exit_code").GetInt32()); Assert.Equal(stdout.Length, summary.GetProperty("output_chars").GetInt32()); Assert.Equal( - QueryCommandRunner.BatchMaxTotalOutputChars, + QueryCommandRunner.BatchDefaultTotalOutputChars, summary.GetProperty("output_char_limit").GetInt32()); Assert.False(summary.GetProperty("output_limit_reached").GetBoolean()); - Assert.Equal(QueryCommandRunner.BatchMaxInputLines, summary.GetProperty("input_line_limit").GetInt32()); + Assert.Equal(QueryCommandRunner.BatchDefaultInputLines, summary.GetProperty("input_line_limit").GetInt32()); Assert.False(summary.GetProperty("input_limit_reached").GetBoolean()); } @@ -994,7 +994,7 @@ public void RunBatch_JsonSummaryReportsProcessedCommandsAndFailures_Issue3906_Is Assert.Equal(CommandExitCodes.UsageError, lineError.GetProperty("exit_code").GetInt32()); Assert.Equal(string.Empty, lineError.GetProperty("stdout").GetString()); Assert.Equal(string.Empty, lineError.GetProperty("stderr").GetString()); - Assert.Contains("batch line 2 must be a non-empty JSON string array", lineError.GetProperty("error").GetProperty("message").GetString()); + Assert.Contains("batch line 2 must be a non-empty JSON string array or a command object", lineError.GetProperty("error").GetProperty("message").GetString()); var unsupportedRecord = unsupportedRecordDocument.RootElement; Assert.Equal("batch_result", unsupportedRecord.GetProperty("record").GetString()); @@ -1222,7 +1222,7 @@ public void RunBatch_JsonSummaryBoundsSerializedOutputIncludingEscaping_Issue458 { using var project = TestProjectHelper.CreateTempProjectScope("cdidx_batch_summary_total_output_limit"); var dbPath = TestProjectHelper.CreateProjectDb(project.Root); - var largeLine = new string('"', QueryCommandRunner.BatchMaxTotalOutputChars * 3 / 5); + var largeLine = new string('"', QueryCommandRunner.BatchDefaultTotalOutputChars * 3 / 5); TestProjectHelper.InsertIndexedFile(dbPath, "docs/large.txt", "text", largeLine); var input = """ ["excerpt","docs/large.txt","--start","1","--max-line-width","0"] @@ -1247,7 +1247,7 @@ public void RunBatch_JsonSummaryBoundsSerializedOutputIncludingEscaping_Issue458 Assert.Equal("batch", error.GetProperty("scope").GetString()); Assert.Equal("batch_output_limit", error.GetProperty("category").GetString()); Assert.Equal( - QueryCommandRunner.BatchMaxTotalOutputChars, + QueryCommandRunner.BatchDefaultTotalOutputChars, error.GetProperty("max_chars").GetInt32()); var summary = summaryDocument.RootElement; @@ -1255,9 +1255,9 @@ public void RunBatch_JsonSummaryBoundsSerializedOutputIncludingEscaping_Issue458 Assert.Equal(1, summary.GetProperty("command_failures").GetInt32()); Assert.True(summary.GetProperty("output_limit_reached").GetBoolean()); Assert.Equal(stdout.Length, summary.GetProperty("output_chars").GetInt32()); - Assert.InRange(stdout.Length, 1, QueryCommandRunner.BatchMaxTotalOutputChars); + Assert.InRange(stdout.Length, 1, QueryCommandRunner.BatchDefaultTotalOutputChars); Assert.Equal( - QueryCommandRunner.BatchMaxTotalOutputChars, + QueryCommandRunner.BatchDefaultTotalOutputChars, summary.GetProperty("output_char_limit").GetInt32()); } @@ -1266,7 +1266,7 @@ public void RunBatch_JsonSummaryBoundsInputLinesAndMalformedRecords_Issue4582() { using var project = TestProjectHelper.CreateTempProjectScope("cdidx_batch_summary_input_line_limit"); var dbPath = TestProjectHelper.CreateProjectDb(project.Root); - var input = string.Concat(Enumerable.Repeat("[]\n", QueryCommandRunner.BatchMaxInputLines + 1)); + var input = string.Concat(Enumerable.Repeat("[]\n", QueryCommandRunner.BatchDefaultInputLines + 1)); var (exitCode, stdout, stderr) = CaptureConsoleWithInput( input, @@ -1276,7 +1276,7 @@ public void RunBatch_JsonSummaryBoundsInputLinesAndMalformedRecords_Issue4582() { Assert.Equal(CommandExitCodes.UsageError, exitCode); Assert.Equal(string.Empty, stderr); - Assert.Equal(QueryCommandRunner.BatchMaxInputLines + 2, lines.Count); + Assert.Equal(QueryCommandRunner.BatchDefaultInputLines + 2, lines.Count); var limitRecord = lines[^2].RootElement; Assert.Equal("batch_error", limitRecord.GetProperty("record").GetString()); Assert.Equal( @@ -1284,12 +1284,12 @@ public void RunBatch_JsonSummaryBoundsInputLinesAndMalformedRecords_Issue4582() limitRecord.GetProperty("error").GetProperty("category").GetString()); var summary = lines[^1].RootElement; - Assert.Equal(QueryCommandRunner.BatchMaxInputLines + 1, summary.GetProperty("input_lines_read").GetInt32()); - Assert.Equal(QueryCommandRunner.BatchMaxInputLines + 1, summary.GetProperty("line_errors").GetInt32()); + Assert.Equal(QueryCommandRunner.BatchDefaultInputLines + 1, summary.GetProperty("input_lines_read").GetInt32()); + Assert.Equal(QueryCommandRunner.BatchDefaultInputLines + 1, summary.GetProperty("line_errors").GetInt32()); Assert.True(summary.GetProperty("input_limit_reached").GetBoolean()); Assert.False(summary.GetProperty("output_limit_reached").GetBoolean()); Assert.Equal(stdout.Length, summary.GetProperty("output_chars").GetInt32()); - Assert.InRange(stdout.Length, 1, QueryCommandRunner.BatchMaxTotalOutputChars); + Assert.InRange(stdout.Length, 1, QueryCommandRunner.BatchDefaultTotalOutputChars); } finally {