diff --git a/AGENT_GUIDE.md b/AGENT_GUIDE.md index 879629af89..d8375944ff 100644 --- a/AGENT_GUIDE.md +++ b/AGENT_GUIDE.md @@ -131,7 +131,7 @@ CI watching must be bounded. Do not loop indefinitely. ## Status Contract -- `status --json` and related JSON/MCP payloads currently expose the trust fields documented in `README.md` and `DEVELOPER_GUIDE.md`, including `fold_ready`, `fold_ready_reason`, `graph_table_available`, `issues_table_available`, `file_issues_data_current`, `migration_in_progress`, `sql_graph_contract_ready`, `sql_graph_contract_degraded_reason`, `hotspot_family_ready`, `hotspot_family_degraded_reason`, `csharp_symbol_name_ready`, `csharp_metadata_target_ready`, `csharp_metadata_target_degraded_reason`, `indexed_head_commit`, `worktree_head_changed`, `index_writer_version`, `index_newer_than_reader`, `index_newer_than_reader_reason`, `unknown_extension_file_count`, `path_case_sensitive`, `data_dir`, `data_dir_source`, `data_dir_mode`, `mac_profile`, `db_pragma_settings` (`journal_mode`, `synchronous`, `wal_autocheckpoint`, `page_count`, `freelist_count`, `page_size`), `hooks`, MCP-only `mcp_session`, and the `status --check`-only `stale_after_seconds` / `index_age_seconds` threshold audit fields. +- `status --json` and related JSON/MCP payloads currently expose the trust fields documented in `README.md` and `DEVELOPER_GUIDE.md`, including `fold_ready`, `fold_ready_reason`, `graph_table_available`, `issues_table_available`, `file_issues_data_current`, `migration_in_progress`, `sql_graph_contract_ready`, `sql_graph_contract_degraded_reason`, `hotspot_family_ready`, `hotspot_family_degraded_reason`, `csharp_symbol_name_ready`, `csharp_metadata_target_ready`, `csharp_metadata_target_degraded_reason`, `indexed_head_commit`, `worktree_head_changed`, `index_writer_version`, `index_newer_than_reader`, `index_newer_than_reader_reason`, `unknown_extension_file_count`, `path_case_sensitive`, `data_dir`, `data_dir_source`, `data_dir_mode`, `mac_profile`, `db_size_bytes`, `wal_size_bytes`, `db_pragma_settings` (`journal_mode`, `synchronous`, `wal_autocheckpoint`, `page_count`, `freelist_count`, `page_size`), `symbols_by_language`, `process`, `last_index_run`, `hooks`, MCP-only `mcp_session`, and the `status --check`-only `stale_after_seconds` / `index_age_seconds` threshold audit fields. - When any readiness field is degraded, the CLI adds `degraded_root_cause`, `degraded_reason`, `recommended_action`, `alternative_action`, and `readiness_degradations[]`. `degraded_root_cause` is the primary stable machine code; `readiness_degradations[]` lists every degraded field with `root_cause`, human reason, and remediation strings. - `issues_table_available` reports physical `file_issues` table presence only. `file_issues_data_current` reports whether the table is also stamped current for the active index generation. - `index_writer_version` records the `cdidx` version that last wrote to the DB (stamped into `codeindex_meta` as `cdidx_writer_version` on every full scan, update, and MCP index). `index_newer_than_reader` flips to `true` whenever any persisted numeric contract stamp in `codeindex_meta` (or unknown `PRAGMA user_version` readiness bits) exceeds the current binary's compiled maximum, so an older CLI re-opening a DB written by a newer CLI degrades loudly with an audit trail instead of silently dropping back to text-search fallbacks. `index_newer_than_reader_reason` enumerates the specific newer-than-reader stamps. @@ -140,6 +140,7 @@ CI watching must be bounded. Do not loop indefinitely. - `status` also surfaces `.cdidx` data-directory permissions via `data_dir_mode` on POSIX filesystems. New `.cdidx` data directories are forced to `0700`; the field is omitted on Windows, URI DBs, or when the directory mode cannot be inspected. - `status` also surfaces filesystem case-sensitivity via `path_case_sensitive`, stamped on every successful `cdidx index` run (full scan AND partial update, plus MCP-driven indexes) from `core.ignorecase` + a live filesystem probe. `true` means the volume is case-sensitive (`Foo.cs` and `foo.cs` are distinct); `false` means case-insensitive. Omitted on legacy DBs that predate the stamp. Use it to audit path-equality decisions on case-sensitive APFS, WSL NTFS / dev-drive, and ReFS mounts where the prior OS-keyed heuristic could mis-classify the workspace (#1546). - `status` also surfaces Linux mandatory-access-control context via `mac_profile` when `/proc/self/attr/current` or `/proc/self/attr/exec` indicates an AppArmor or SELinux profile. It is omitted on non-Linux hosts, unconstrained processes, or unreadable proc attributes (#1768). +- `status` also surfaces DB/WAL size, per-language symbol-kind histograms, current process heap/GC/working-set metrics, and the last successful index run metadata. `process` is captured at status-call time; `last_index_run` is persisted at the end of successful CLI and MCP index runs and can include a peak-memory summary when CLI `--memory-trace` was used. - MCP `status` also surfaces session diagnostics via `mcp_session`. It is not persisted DB state; it includes the current `log_level`, captured `roots`, optional `client_info`, and optional `client_capabilities`. - Keep `README.md`, `DEVELOPER_GUIDE.md`, and this file synchronized if this contract changes. diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 6cf13e5aea..4d5cf59b88 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -317,6 +317,8 @@ Index write batches also stamp `codeindex_meta.batch_in_progress=true` before st Read-only fallback uses an immutable SQLite URI when the normal writable open cannot create or lock journal/WAL side files, so query commands can still read a DB from read-only or sandboxed storage. That fallback intentionally skips writable pragmas, migrations, and WAL recovery writes; if a WAL is present and must be observed, copy the `.db`, `.db-wal`, and `.db-shm` files together to a writable location or use a SQLite backup from an environment that can open the full WAL set. `status --json` exposes the resolved connection values under `db_pragma_settings` (`journal_mode`, `synchronous`, `wal_autocheckpoint`, `page_count`, `freelist_count`, `page_size`) for automation and support diagnostics. `cdidx vacuum` runs `PRAGMA incremental_vacuum` against writable incremental-auto-vacuum DBs, and performs a one-time `PRAGMA auto_vacuum=INCREMENTAL` plus full `VACUUM` conversion for legacy no-autovacuum DBs. +`status --json` also reports `db_size_bytes`, `wal_size_bytes`, `symbols_by_language`, current `process` heap/GC/working-set metrics, and `last_index_run` metadata from successful CLI and MCP index runs. `index --json --memory-trace` adds a `memory_timeline` block to the CLI index result and persists peak working-set MB into `last_index_run`; `CDIDX_MEM_WARN_MB=` prints a warning when the sampled working set crosses that threshold. + Writable opens also reject databases whose `PRAGMA user_version` contains readiness bits outside the current binary's `CurrentSchemaVersion` mask. Read-only status/query paths may still surface `index_newer_than_reader=true` as a degraded audit signal, but write-capable paths must fail with `E003_SCHEMA_TOO_NEW` so an older cdidx cannot silently rewrite a DB stamped by a newer one. ### Data directory resolution @@ -1983,6 +1985,8 @@ WAL では `NORMAL` により 500 行単位の indexing batch ごとの fsync 通常の writable open が journal/WAL side file を作成または lock できない場合、read-only fallback は immutable SQLite URI を使うため、query command は read-only / sandboxed storage 上の DB でも読み取りを継続できる。この fallback は意図的に writable pragma、migration、WAL recovery write を skip する。WAL が存在し、その内容を観測する必要がある場合は、`.db` / `.db-wal` / `.db-shm` をまとめて writable location に copy するか、full WAL set を open できる環境で SQLite backup を使う。`status --json` は automation / support diagnostics 用に、解決済みの接続値を `db_pragma_settings` (`journal_mode`, `synchronous`, `wal_autocheckpoint`, `page_count`, `freelist_count`, `page_size`) で公開する。`cdidx vacuum` は incremental-auto-vacuum DB では `PRAGMA incremental_vacuum` を実行し、legacy no-autovacuum DB では初回のみ `PRAGMA auto_vacuum=INCREMENTAL` と full `VACUUM` で変換する。 +`status --json` は `db_size_bytes`、`wal_size_bytes`、`symbols_by_language`、現在の `process` heap / GC / working-set metrics、成功した CLI / MCP index 実行由来の `last_index_run` metadata も公開する。`index --json --memory-trace` は CLI index 結果に `memory_timeline` block を追加し、peak working-set MB を `last_index_run` に保存する。`CDIDX_MEM_WARN_MB=` は sampled working set がしきい値を超えたときに warning を出す。 + ## データベーススキーマ ### テーブル diff --git a/README.md b/README.md index 130c297a9f..14989aa0c8 100644 --- a/README.md +++ b/README.md @@ -165,9 +165,10 @@ The documented `status --json` trust contract covers these fields: worktree_head_changedindexed_head_shaindexed_head_branchindexed_head_timestamp commits_ahead_of_indexed_headindex_writer_versionindex_newer_than_readerindex_newer_than_reader_reason unknown_extension_file_countpath_case_sensitivedata_dirdata_dir_source -data_dir_modemac_profiledb_pragma_settingshooks -stale_after_secondsindex_age_secondsdegraded_reasonrecommended_action -alternative_actionmcp_session +data_dir_modemac_profiledb_size_byteswal_size_bytes +db_pragma_settingssymbols_by_languageprocesslast_index_run +hooksstale_after_secondsindex_age_secondsdegraded_reason +recommended_actionalternative_actionmcp_session @@ -175,6 +176,8 @@ When any readiness field is degraded, `degraded_root_cause` identifies the prima For MCP `status`, `mcp_session` is session-scoped diagnostic data rather than persisted index state. It includes `log_level`, `roots`, optional `client_info`, and optional `client_capabilities`. +`process` is captured at status-call time and includes heap, GC collection, and working-set counters. `last_index_run` is persisted by successful CLI and MCP index runs with the run mode, duration, file counts, byte count, row-change counts, and optional peak-memory summary from CLI `--memory-trace`. + `hotspot_family_degraded_reason` uses these values: | Value | Meaning | @@ -378,9 +381,10 @@ upgrade / downgrade 後はインストール済み補完 script を再生成し worktree_head_changedindexed_head_shaindexed_head_branchindexed_head_timestamp commits_ahead_of_indexed_headindex_writer_versionindex_newer_than_readerindex_newer_than_reader_reason unknown_extension_file_countpath_case_sensitivedata_dirdata_dir_source -data_dir_modemac_profiledb_pragma_settingshooks -stale_after_secondsindex_age_secondsdegraded_reasonrecommended_action -alternative_actionmcp_session +data_dir_modemac_profiledb_size_byteswal_size_bytes +db_pragma_settingssymbols_by_languageprocesslast_index_run +hooksstale_after_secondsindex_age_secondsdegraded_reason +recommended_actionalternative_actionmcp_session @@ -388,6 +392,8 @@ readiness field のいずれかが degraded の場合、`degraded_root_cause` MCP `status` の `mcp_session` は永続化された index 状態ではなく、セッション単位の診断情報です。`log_level`、`roots`、任意の `client_info`、任意の `client_capabilities` を含みます。 +`process` は status 呼び出し時点の heap、GC collection、working-set counters です。`last_index_run` は成功した CLI / MCP index 実行が永続化し、run mode、duration、file counts、byte count、row-change counts、CLI `--memory-trace` 由来の任意の peak-memory summary を含みます。 + `hotspot_family_degraded_reason` は次の値を使います。 | 値 | 意味 | diff --git a/changelog.d/unreleased/1637.added.md b/changelog.d/unreleased/1637.added.md new file mode 100644 index 0000000000..5ea302fd47 --- /dev/null +++ b/changelog.d/unreleased/1637.added.md @@ -0,0 +1,24 @@ +--- +category: added +issues: + - 1637 + - 1680 + - 1792 + - 1817 +affected: + - src/CodeIndex/Models/QueryResults.cs + - src/CodeIndex/Database/DbReader.FilesStatus.cs + - src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs + - src/CodeIndex/Cli/IndexCommandRunner.Update.cs + - README.md + - DEVELOPER_GUIDE.md + - AGENT_GUIDE.md +--- + +## English + +- **Expanded status and index diagnostics (#1637, #1680, #1792, #1817)** — `status --json` now reports DB/WAL size, per-language symbol-kind counts, process heap/GC/working-set metrics, and last-index-run metadata, while `index --json --memory-trace` emits memory samples and persists the peak working set. + +## 日本語 + +- **status と index diagnostics を拡張しました (#1637, #1680, #1792, #1817)** — `status --json` は DB/WAL サイズ、言語別 symbol-kind 件数、process heap / GC / working-set metrics、直近 index 実行 metadata を返し、`index --json --memory-trace` は memory samples を出力して peak working set を保存するようになりました。 diff --git a/src/CodeIndex/Cli/CliFlagSchema.cs b/src/CodeIndex/Cli/CliFlagSchema.cs index 8501977961..5ed4c3a1fa 100644 --- a/src/CodeIndex/Cli/CliFlagSchema.cs +++ b/src/CodeIndex/Cli/CliFlagSchema.cs @@ -276,6 +276,7 @@ private static IReadOnlyList BuildAll() new() { Name = "--duration-format", ValuePlaceholder = "", Description = "Index elapsed time display format", Commands = Set("index") }, new() { Name = "--max-file-bytes", ValuePlaceholder = "", Description = "Override the per-file indexing size limit", Commands = Set("index") }, new() { Name = "--parallelism", ValuePlaceholder = "", Description = "Full-scan extraction worker count (default: CPU count capped at 16; also honors CDIDX_INDEX_PARALLELISM)", Commands = Set("index") }, + new() { Name = "--memory-trace", Description = "Include phase memory samples in index JSON output", Commands = Set("index") }, new() { Name = "--commits", ValuePlaceholder = "", Description = "Update files changed in given git commits", Commands = Set("index") }, new() { Name = "--changed-between", ValuePlaceholder = " ", Description = "Update files changed between two git refs", Commands = Set("index") }, new() { Name = "--files", ValuePlaceholder = "", Description = "Update only the specified files", Commands = Set("index") }, diff --git a/src/CodeIndex/Cli/ConsoleUi.cs b/src/CodeIndex/Cli/ConsoleUi.cs index 4cf0ba5369..54ea0dd8d6 100644 --- a/src/CodeIndex/Cli/ConsoleUi.cs +++ b/src/CodeIndex/Cli/ConsoleUi.cs @@ -60,14 +60,14 @@ public static class ConsoleUi private static readonly (string Command, string Usage)[] CommandUsageLines = [ - ("index", "cdidx index [--db ] [--rebuild] [--optimize] [--verbose] [--dry-run] [--force] [--quiet] [--json] [--duration-format ] [--max-file-bytes ] [--follow-symlinks ] [--include-symbol-kind [,]] [--exclude-symbol-kind [,]] [--watch [--debounce ]]"), + ("index", "cdidx index [--db ] [--rebuild] [--optimize] [--verbose] [--dry-run] [--force] [--quiet] [--json] [--memory-trace] [--duration-format ] [--max-file-bytes ] [--follow-symlinks ] [--include-symbol-kind [,]] [--exclude-symbol-kind [,]] [--watch [--debounce ]]"), ("hooks", "cdidx hooks [--project ] [--force] [--json]"), ("backfill-fold", "cdidx backfill-fold [--db ] [--json]"), ("optimize", "cdidx optimize [--db ] [--json]"), ("vacuum", "cdidx vacuum [--db ] [--json]"), - ("index-commits", "cdidx index --commits [id ...] [--db ] [--verbose] [--dry-run] [--json] [--duration-format ] [--max-file-bytes ] [--include-symbol-kind [,]] [--exclude-symbol-kind [,]]"), - ("index-changed-between", "cdidx index --changed-between [--db ] [--verbose] [--dry-run] [--json] [--duration-format ] [--max-file-bytes ] [--include-symbol-kind [,]] [--exclude-symbol-kind [,]]"), - ("index-files", "cdidx index --files [path ...] [--db ] [--verbose] [--dry-run] [--json] [--duration-format ] [--max-file-bytes ] [--include-symbol-kind [,]] [--exclude-symbol-kind [,]]"), + ("index-commits", "cdidx index --commits [id ...] [--db ] [--verbose] [--dry-run] [--json] [--memory-trace] [--duration-format ] [--max-file-bytes ] [--include-symbol-kind [,]] [--exclude-symbol-kind [,]]"), + ("index-changed-between", "cdidx index --changed-between [--db ] [--verbose] [--dry-run] [--json] [--memory-trace] [--duration-format ] [--max-file-bytes ] [--include-symbol-kind [,]] [--exclude-symbol-kind [,]]"), + ("index-files", "cdidx index --files [path ...] [--db ] [--verbose] [--dry-run] [--json] [--memory-trace] [--duration-format ] [--max-file-bytes ] [--include-symbol-kind [,]] [--exclude-symbol-kind [,]]"), ("search", "cdidx search |--query |-- [--db ] [--json[=ndjson|array]] [--verbose] [--limit |--top ] [--lang ] [--path ] [--exclude-path ] [--exclude-tests] [--snippet-lines ] [--snippet-focus ] [--max-line-width ] [--fts] [--exact|--exact-substring] [--prefix] [--count] [--since ] [--no-dedup] [--no-visibility-rank]"), ("definition", "cdidx definition |--query |-- [--db ] [--json] [--verbose] [--limit |--top ] [--lang ] [--kind ] [--visibility ] [--exclude-visibility ] [--path ] [--exclude-path ] [--exclude-tests] [--body] [--exact|--exact-name] [--count] [--since ]"), ("goto", "cdidx goto |--query |-- [--db ] [--json] [--limit |--top ] [--lang ] [--kind ] [--path ] [--exclude-path ] [--exclude-tests] [--exact|--exact-name] [--all]"), @@ -772,6 +772,7 @@ private static void PrintFlagReference(Action WriteHelpLine) Console.WriteLine(" --dry-run Scan files without writing to the database"); Console.WriteLine(" --force Bypass the per-database index lock; only use when no other cdidx index is active"); Console.WriteLine(" --json Output results as JSON (for AI/machine use)"); + Console.WriteLine(" --memory-trace Include phase memory samples in index JSON output"); Console.WriteLine(" --quiet, -q, --silent Suppress informational stderr output; errors still print (also honors CDIDX_QUIET=1)"); Console.WriteLine(" --duration-format Index elapsed time format: `auto` (default), `seconds`, or `hms`; JSON keeps raw elapsed_ms"); WriteHelpLine(" --max-file-bytes Index only files up to this size (default: 4MiB; also honors CDIDX_MAX_FILE_BYTES; accepts K/M/G suffixes)"); diff --git a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs index 64ce35e144..19f19f69fc 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs @@ -356,6 +356,7 @@ private static int RunFullScan( string resolvedDbPath, IndexCommandOptions options, Stopwatch stopwatch, + DateTime runStartedAtUtc, string[] spinnerFrames, JsonSerializerOptions jsonOptions, string? priorFoldVersion, @@ -375,6 +376,7 @@ private static int RunFullScan( CancellationToken cancellationToken) { var jsonContext = CliJsonSerializerContextFactory.Create(jsonOptions); + var memorySamples = options.MemoryTrace ? new List { CaptureMemorySample("start", stopwatch) } : []; _ = priorMetadataTargetCsharp; // full-scan resolver runs unconditionally on success / 成功時に常に再解決するため不要 var unresolvedMergeExitCode = RejectUnresolvedMergeState(projectRoot, options.Json, jsonOptions); if (unresolvedMergeExitCode != null) @@ -517,6 +519,8 @@ void ThrowIfFullScanCancelled(int filesProcessed, int? filesTotal) StopJsonPhaseHeartbeat(scanHeartbeat); } var files = scanResult.Files; + if (options.MemoryTrace) + memorySamples.Add(CaptureMemorySample("scan", stopwatch)); ConsoleUi.StopSpinner(spinnerCts); WriteJsonLiveness($"found {ConsoleUi.Counted(files.Count, "file", format: "N0")}; preparing database..."); var fatalScanErrors = scanResult.Errors @@ -606,6 +610,8 @@ void ThrowIfFullScanCancelled(int filesProcessed, int? filesTotal) } if (purged > 0) WriteProjectRootOnce(); + if (options.MemoryTrace) + memorySamples.Add(CaptureMemorySample("purge", stopwatch)); ConsoleUi.StopSpinner(purgeCts); WriteJsonLiveness(purged > 0 ? $"purged {purged:N0} stale file(s); preparing index writes..." @@ -1273,10 +1279,27 @@ void StopJsonHeartbeat() // #1509: あらゆる成功 index の終端で更新する HEAD トリプル (SHA + branch + 時刻) も // ここで stamp する。full scan / partial update を問わず最新の HEAD を保存する。 StampIndexedHeadMetadata(writer, projectRoot); + if (options.MemoryTrace) + memorySamples.Add(CaptureMemorySample("finalize", stopwatch)); + var memoryTimelineForStamp = BuildMemoryTimeline(memorySamples); + StampLastIndexRunMetadata( + writer, + options.Rebuild ? "rebuild" : "incremental", + runStartedAtUtc, + stopwatch.ElapsedMilliseconds, + files.Count, + skipped, + errors, + SumReadableFileBytes(files), + processed, + purged, + memoryTimelineForStamp); } writer.ClearBatchInProgress(); fullScanTxn.Commit(); stopwatch.Stop(); + var memoryTimeline = BuildMemoryTimeline(memorySamples); + WarnIfMemoryThresholdExceeded(memoryTimeline); // Detect cwd drift between option-parsing and finalize. See RunUpdateMode for the // rationale; the warning is informational because we already absolutized paths. // Issue #1577. @@ -1357,6 +1380,7 @@ void StopJsonHeartbeat() CwdDriftNotice = cwdDriftNotice, Errors = errorList.Count > 0 ? errorList : null, Warnings = warningList.Count > 0 ? warningList : null, + MemoryTimeline = memoryTimeline, ElapsedMs = stopwatch.ElapsedMilliseconds, }, jsonContext.IndexFullScanJsonResult)); } diff --git a/src/CodeIndex/Cli/IndexCommandRunner.Parse.cs b/src/CodeIndex/Cli/IndexCommandRunner.Parse.cs index b19a2f245d..097365fcc4 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.Parse.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.Parse.cs @@ -14,7 +14,7 @@ public static partial class IndexCommandRunner [ "--db", "--data-dir", "--rebuild", "--verbose", "--json", "--dry-run", "--force", "--yes", "--watch", "--debounce", "--duration-format", "--max-file-bytes", - "--parallelism", "--follow-symlinks", + "--parallelism", "--memory-trace", "--follow-symlinks", "--commits", "--changed-between", "--files", "--solution", "--project", "--include-symbol-kind", "--exclude-symbol-kind", "--optimize", "--help", "--read-only", "--immutable", @@ -37,6 +37,7 @@ public static IndexCommandOptions ParseArgs(string[] args) bool yes = false; bool watch = false; bool optimizeOnly = false; + bool memoryTrace = false; int? watchDebounceMs = null; var durationFormat = DurationOutputFormat.Auto; long? maxFileSizeBytes = ReadMaxFileSizeBytesFromEnvironment(); @@ -115,6 +116,9 @@ public static IndexCommandOptions ParseArgs(string[] args) case "--optimize": optimizeOnly = true; break; + case "--memory-trace": + memoryTrace = true; + break; case "--debounce" when i + 1 < args.Length: if (int.TryParse(args[i + 1], System.Globalization.NumberStyles.Integer, System.Globalization.CultureInfo.InvariantCulture, out var parsedDebounce) && parsedDebounce >= 0) { @@ -292,6 +296,7 @@ public static IndexCommandOptions ParseArgs(string[] args) Yes = yes, Watch = watch, OptimizeOnly = optimizeOnly, + MemoryTrace = memoryTrace, WatchDebounceMs = watchDebounceMs, DurationFormat = durationFormat, MaxFileSizeBytes = maxFileSizeBytes, diff --git a/src/CodeIndex/Cli/IndexCommandRunner.Update.cs b/src/CodeIndex/Cli/IndexCommandRunner.Update.cs index e107fa8a40..2af6cd689a 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.Update.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.Update.cs @@ -16,6 +16,7 @@ private static int RunUpdateMode( string resolvedDbPath, IndexCommandOptions options, Stopwatch stopwatch, + DateTime runStartedAtUtc, string[] spinnerFrames, JsonSerializerOptions jsonOptions, int priorReadiness, @@ -36,6 +37,7 @@ private static int RunUpdateMode( CancellationToken cancellationToken) { var jsonContext = CliJsonSerializerContextFactory.Create(jsonOptions); + var memorySamples = options.MemoryTrace ? new List { CaptureMemorySample("start", stopwatch) } : []; var currentSqlGraphContractVersion = DbContext.SqlGraphContractVersion.ToString(System.Globalization.CultureInfo.InvariantCulture); var sqlGraphContractMatchesCurrent = priorSqlGraphContractVersion == currentSqlGraphContractVersion; var unresolvedMergeExitCode = RejectUnresolvedMergeState(projectRoot, options.Json, jsonOptions); @@ -162,6 +164,7 @@ private static int RunUpdateMode( resolvedDbPath, options, stopwatch, + runStartedAtUtc, spinnerFrames, jsonOptions, priorFoldVersion, @@ -1069,8 +1072,25 @@ void ThrowIfUpdateCancelled() { StampIndexedHeadMetadata(writer, projectRoot); StampCommitScopedFreshHeadMetadata(writer, options, currentHeadCommit); + if (options.MemoryTrace) + memorySamples.Add(CaptureMemorySample("finalize", stopwatch)); + var memoryTimelineForStamp = BuildMemoryTimeline(memorySamples); + StampLastIndexRunMetadata( + writer, + "update", + runStartedAtUtc, + stopwatch.ElapsedMilliseconds, + updated + removed + skipped, + skipped, + errors, + SumReadableFileBytes(targetPaths.Select(path => Path.Combine(projectRoot, path.Replace('/', Path.DirectorySeparatorChar)))), + updated, + removed, + memoryTimelineForStamp); } stopwatch.Stop(); + var memoryTimeline = BuildMemoryTimeline(memorySamples); + WarnIfMemoryThresholdExceeded(memoryTimeline); // Detect cwd drift between option-parsing and finalize. Paths used in this run are // already absolute, but a drifted cwd is a strong signal that an embedded host or // signal handler mutated process state -- surface it so the operator can correct @@ -1148,6 +1168,7 @@ void ThrowIfUpdateCancelled() CwdDriftNotice = cwdDriftNotice, Errors = errorList.Count > 0 ? errorList : null, Warnings = warningList.Count > 0 ? warningList : null, + MemoryTimeline = memoryTimeline, ElapsedMs = stopwatch.ElapsedMilliseconds, }, jsonContext.IndexUpdateJsonResult)); } diff --git a/src/CodeIndex/Cli/IndexCommandRunner.cs b/src/CodeIndex/Cli/IndexCommandRunner.cs index 0519e64206..0b26b16c7e 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.cs @@ -87,6 +87,7 @@ internal static int Run(string[] indexArgs, JsonSerializerOptions jsonOptions, C var dbResolution = DbPathResolver.ResolveForIndex(options.ProjectPath, options.DbPath, options.DataDir); var dbPath = dbResolution.DbPath; var stopwatch = Stopwatch.StartNew(); + var runStartedAtUtc = DateTime.UtcNow; var isUpdateMode = options.Commits.Count > 0 || options.ChangedBetweenSpecified || options.UpdateFiles.Count > 0; var mode = options.Rebuild ? "rebuild" : isUpdateMode ? "update" : "incremental"; @@ -244,8 +245,8 @@ internal static int Run(string[] indexArgs, JsonSerializerOptions jsonOptions, C var projectRoot = Path.GetFullPath(options.ProjectPath!); initialExitCode = isUpdateMode - ? RunUpdateMode(writer, indexer, projectRoot, resolvedDbPath, options, stopwatch, spinnerFrames, jsonOptions, priorReadiness, priorFoldVersion, priorFoldFingerprint, priorSymbolExtractorVersionsMatchCurrent, priorCSharpSymbolNameContractVersion, priorMetadataTargetCsharp, priorSqlGraphContractVersion, priorHotspotFamilyVersions, priorHotspotFamilyMarkerFingerprints, currentHotspotFamilyMarkerFingerprints, priorIndexedProjectRoot, priorIndexedHeadCommit, currentHeadCommit, priorSymbolKindFilterSignature, initialCwd, indexCancellation.Token) - : RunFullScan(writer, indexer, projectRoot, resolvedDbPath, options, stopwatch, spinnerFrames, jsonOptions, priorFoldVersion, priorFoldFingerprint, priorSymbolExtractorVersionsMatchCurrent, priorCSharpSymbolNameContractVersion, priorMetadataTargetCsharp, priorSqlGraphContractVersion, priorHotspotFamilyVersions, priorHotspotFamilyMarkerFingerprints, currentHotspotFamilyMarkerFingerprints, priorIndexedProjectRoot, priorIndexedHeadCommit, currentHeadCommit, priorSymbolKindFilterSignature, initialCwd, indexCancellation.Token); + ? RunUpdateMode(writer, indexer, projectRoot, resolvedDbPath, options, stopwatch, runStartedAtUtc, spinnerFrames, jsonOptions, priorReadiness, priorFoldVersion, priorFoldFingerprint, priorSymbolExtractorVersionsMatchCurrent, priorCSharpSymbolNameContractVersion, priorMetadataTargetCsharp, priorSqlGraphContractVersion, priorHotspotFamilyVersions, priorHotspotFamilyMarkerFingerprints, currentHotspotFamilyMarkerFingerprints, priorIndexedProjectRoot, priorIndexedHeadCommit, currentHeadCommit, priorSymbolKindFilterSignature, initialCwd, indexCancellation.Token) + : RunFullScan(writer, indexer, projectRoot, resolvedDbPath, options, stopwatch, runStartedAtUtc, spinnerFrames, jsonOptions, priorFoldVersion, priorFoldFingerprint, priorSymbolExtractorVersionsMatchCurrent, priorCSharpSymbolNameContractVersion, priorMetadataTargetCsharp, priorSqlGraphContractVersion, priorHotspotFamilyVersions, priorHotspotFamilyMarkerFingerprints, currentHotspotFamilyMarkerFingerprints, priorIndexedProjectRoot, priorIndexedHeadCommit, currentHeadCommit, priorSymbolKindFilterSignature, initialCwd, indexCancellation.Token); if (initialExitCode == CommandExitCodes.Success) db.RunPlannerStatisticsMaintenance(forceAnalyze: !databaseExistedBeforeIndex); } @@ -289,6 +290,94 @@ private static string DescribeLockHolder(IndexLockInfo? holder) return values; } + private static IndexMemorySampleJsonResult CaptureMemorySample(string phase, Stopwatch stopwatch) + { + var gcInfo = GC.GetGCMemoryInfo(); + return new IndexMemorySampleJsonResult + { + Phase = phase, + ElapsedMs = stopwatch.ElapsedMilliseconds, + HeapBytes = GC.GetTotalMemory(forceFullCollection: false), + TotalAllocatedBytes = GC.GetTotalAllocatedBytes(), + GcHeapSizeBytes = gcInfo.HeapSizeBytes, + FragmentedBytes = gcInfo.FragmentedBytes, + WorkingSetBytes = Process.GetCurrentProcess().WorkingSet64, + Gen0Collections = GC.CollectionCount(0), + Gen1Collections = GC.CollectionCount(1), + Gen2Collections = GC.CollectionCount(2), + }; + } + + private static IndexMemoryTimelineJsonResult? BuildMemoryTimeline(List samples) + { + if (samples.Count == 0) + return null; + + return new IndexMemoryTimelineJsonResult + { + Samples = samples, + PeakWorkingSetBytes = samples.Max(static sample => sample.WorkingSetBytes), + PeakHeapBytes = samples.Max(static sample => sample.HeapBytes), + }; + } + + private static void WarnIfMemoryThresholdExceeded(IndexMemoryTimelineJsonResult? timeline) + { + var rawThreshold = Environment.GetEnvironmentVariable("CDIDX_MEM_WARN_MB"); + if (timeline == null || !long.TryParse(rawThreshold, System.Globalization.NumberStyles.Integer, System.Globalization.CultureInfo.InvariantCulture, out var thresholdMb) || thresholdMb <= 0) + return; + + var peakMb = timeline.PeakWorkingSetBytes / (1024 * 1024); + if (peakMb >= thresholdMb) + Console.Error.WriteLine($"Warning: cdidx working set reached {peakMb:N0} MB (CDIDX_MEM_WARN_MB={thresholdMb:N0})."); + } + + private static void StampLastIndexRunMetadata( + DbWriter writer, + string mode, + DateTime startedAtUtc, + long durationMs, + long filesScanned, + long filesSkipped, + long parseErrors, + long bytesRead, + long rowsUpserted, + long rowsDeleted, + IndexMemoryTimelineJsonResult? memoryTimeline) + { + writer.SetMeta(DbContext.LastIndexRunModeMetaKey, mode); + writer.SetMeta(DbContext.LastIndexRunStartedAtMetaKey, startedAtUtc.ToString("o", System.Globalization.CultureInfo.InvariantCulture)); + writer.SetMeta(DbContext.LastIndexRunDurationMsMetaKey, durationMs.ToString(System.Globalization.CultureInfo.InvariantCulture)); + writer.SetMeta(DbContext.LastIndexRunFilesScannedMetaKey, filesScanned.ToString(System.Globalization.CultureInfo.InvariantCulture)); + writer.SetMeta(DbContext.LastIndexRunFilesSkippedMetaKey, filesSkipped.ToString(System.Globalization.CultureInfo.InvariantCulture)); + writer.SetMeta(DbContext.LastIndexRunParseErrorsMetaKey, parseErrors.ToString(System.Globalization.CultureInfo.InvariantCulture)); + writer.SetMeta(DbContext.LastIndexRunBytesReadMetaKey, bytesRead.ToString(System.Globalization.CultureInfo.InvariantCulture)); + writer.SetMeta(DbContext.LastIndexRunRowsUpsertedMetaKey, rowsUpserted.ToString(System.Globalization.CultureInfo.InvariantCulture)); + writer.SetMeta(DbContext.LastIndexRunRowsDeletedMetaKey, rowsDeleted.ToString(System.Globalization.CultureInfo.InvariantCulture)); + writer.SetMeta(DbContext.LastIndexRunPeakMemoryMbMetaKey, memoryTimeline == null + ? null + : (memoryTimeline.PeakWorkingSetBytes / (1024 * 1024)).ToString(System.Globalization.CultureInfo.InvariantCulture)); + } + + private static long SumReadableFileBytes(IEnumerable paths) + { + long total = 0; + foreach (var path in paths) + { + try + { + var info = new FileInfo(path); + if (info.Exists) + total += info.Length; + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or NotSupportedException or ArgumentException) + { + } + } + + return total; + } + private static Dictionary GetHotspotFamilyMarkerFingerprints(FileIndexer indexer) { var values = new Dictionary(StringComparer.Ordinal); @@ -1180,6 +1269,7 @@ public sealed class IndexCommandOptions public DurationOutputFormat DurationFormat { get; init; } = DurationOutputFormat.Auto; public long? MaxFileSizeBytes { get; init; } public int Parallelism { get; init; } = IndexCommandRunner.DefaultIndexParallelism(); + public bool MemoryTrace { get; init; } public FileIndexer.SymlinkPolicy SymlinkPolicy { get; init; } = FileIndexer.SymlinkPolicy.None; public SymbolKindFilter SymbolKindFilter { get; init; } = SymbolKindFilter.Empty; } diff --git a/src/CodeIndex/Cli/JsonOutputContracts.cs b/src/CodeIndex/Cli/JsonOutputContracts.cs index d1f56bb050..4363f6bebd 100644 --- a/src/CodeIndex/Cli/JsonOutputContracts.cs +++ b/src/CodeIndex/Cli/JsonOutputContracts.cs @@ -166,6 +166,27 @@ internal sealed class IndexFullScanSummaryJsonResult public int SymbolsDroppedByKindFilter { get; init; } } +internal sealed class IndexMemorySampleJsonResult +{ + public string Phase { get; init; } = string.Empty; + public long ElapsedMs { get; init; } + public long HeapBytes { get; init; } + public long TotalAllocatedBytes { get; init; } + public long GcHeapSizeBytes { get; init; } + public long FragmentedBytes { get; init; } + public long WorkingSetBytes { get; init; } + public int Gen0Collections { get; init; } + public int Gen1Collections { get; init; } + public int Gen2Collections { get; init; } +} + +internal sealed class IndexMemoryTimelineJsonResult +{ + public List Samples { get; init; } = []; + public long PeakWorkingSetBytes { get; init; } + public long PeakHeapBytes { get; init; } +} + public sealed class IndexSymbolKindFilterJsonResult { public IReadOnlyList Include { get; init; } = []; @@ -199,6 +220,7 @@ internal sealed class IndexUpdateJsonResult public string? CwdDriftNotice { get; init; } public List? Errors { get; init; } public List? Warnings { get; init; } + public IndexMemoryTimelineJsonResult? MemoryTimeline { get; init; } public long ElapsedMs { get; init; } } @@ -233,6 +255,7 @@ internal sealed class IndexFullScanJsonResult public string? CwdDriftNotice { get; init; } public List? Errors { get; init; } public List? Warnings { get; init; } + public IndexMemoryTimelineJsonResult? MemoryTimeline { get; init; } public long ElapsedMs { get; init; } } @@ -300,6 +323,8 @@ internal sealed record VersionInfoJsonResult( [JsonSerializable(typeof(IndexFreshnessCheckResult))] [JsonSerializable(typeof(IndexFullScanJsonResult))] [JsonSerializable(typeof(IndexFullScanSummaryJsonResult))] +[JsonSerializable(typeof(IndexMemorySampleJsonResult))] +[JsonSerializable(typeof(IndexMemoryTimelineJsonResult))] [JsonSerializable(typeof(IndexUpdateJsonResult))] [JsonSerializable(typeof(IndexUpdateSummaryJsonResult))] [JsonSerializable(typeof(IndexWatchEventJsonResult))] @@ -349,6 +374,8 @@ internal sealed record VersionInfoJsonResult( [JsonSerializable(typeof(StatusResult))] [JsonSerializable(typeof(StatusReadinessDegradation))] [JsonSerializable(typeof(StatusDbPragmaSettings))] +[JsonSerializable(typeof(StatusLastIndexRun))] +[JsonSerializable(typeof(StatusProcessMetrics))] [JsonSerializable(typeof(SuggestionDetailJsonResult))] [JsonSerializable(typeof(SuggestionExportJsonResult))] [JsonSerializable(typeof(SuggestionListItemJsonResult))] diff --git a/src/CodeIndex/Database/DbContext.cs b/src/CodeIndex/Database/DbContext.cs index 8d9b18e2cb..4ee75a4570 100644 --- a/src/CodeIndex/Database/DbContext.cs +++ b/src/CodeIndex/Database/DbContext.cs @@ -1231,6 +1231,16 @@ private static void RegisterConnectionFunctionsWithRetry( public const string IndexedHeadTimestampMetaKey = "indexed_head_timestamp"; public const string CommitScopedFreshHeadShaMetaKey = "commit_scoped_fresh_head_sha"; public const string LastFullScanElapsedMsMetaKey = "last_full_scan_elapsed_ms"; + public const string LastIndexRunModeMetaKey = "last_index_run_mode"; + public const string LastIndexRunStartedAtMetaKey = "last_index_run_started_at"; + public const string LastIndexRunDurationMsMetaKey = "last_index_run_duration_ms"; + public const string LastIndexRunFilesScannedMetaKey = "last_index_run_files_scanned"; + public const string LastIndexRunFilesSkippedMetaKey = "last_index_run_files_skipped"; + public const string LastIndexRunParseErrorsMetaKey = "last_index_run_parse_errors"; + public const string LastIndexRunBytesReadMetaKey = "last_index_run_bytes_read"; + public const string LastIndexRunRowsUpsertedMetaKey = "last_index_run_rows_upserted"; + public const string LastIndexRunRowsDeletedMetaKey = "last_index_run_rows_deleted"; + public const string LastIndexRunPeakMemoryMbMetaKey = "last_index_run_peak_memory_mb"; // Issue #1585: count of files seen by the most recent successful full-repository scan // whose non-empty extension did not map to a known language. This is a scan coverage // signal, not an indexed-file count, and is omitted by readers until a current index pass diff --git a/src/CodeIndex/Database/DbReader.FilesStatus.cs b/src/CodeIndex/Database/DbReader.FilesStatus.cs index be104e8bad..d60c6ca48b 100644 --- a/src/CodeIndex/Database/DbReader.FilesStatus.cs +++ b/src/CodeIndex/Database/DbReader.FilesStatus.cs @@ -437,6 +437,30 @@ public StatusResult GetStatus() langs[reader.GetString(0)] = reader.GetInt64(1); } + var symbolsByLanguage = new Dictionary>(StringComparer.Ordinal); + { + using var cmd = _conn.CreateCommand(); + cmd.CommandText = @" + SELECT COALESCE(f.lang, 'unknown'), s.kind, COUNT(*) + FROM symbols s + JOIN files f ON f.id = s.file_id + GROUP BY COALESCE(f.lang, 'unknown'), s.kind + ORDER BY COALESCE(f.lang, 'unknown'), COUNT(*) DESC, s.kind"; + using var reader = cmd.ExecuteTrackedReader(); + while (reader.TrackedRead()) + { + var lang = reader.GetString(0); + var kind = reader.GetString(1); + if (!symbolsByLanguage.TryGetValue(lang, out var kinds)) + { + kinds = new Dictionary(StringComparer.Ordinal); + symbolsByLanguage[lang] = kinds; + } + + kinds[kind] = reader.GetInt64(2); + } + } + // #1509: pull persisted HEAD metadata while the SHARED snapshot is open so the // recorded SHA / branch / timestamp can't drift relative to the counts and freshness // reported by the same status call. @@ -450,6 +474,9 @@ public StatusResult GetStatus() // #1546: case-sensitivity stamp も同 snapshot で読む。stamp 無し旧 DB は null。 var pathCaseSensitive = ParseMetaBool(TryGetMetaStringInternal(DbContext.WorkspacePathCaseSensitiveMetaKey)); var dbPragmaSettings = GetDbPragmaSettings(); + var dbSizeBytes = TryGetDatabaseFileSize(); + var walSizeBytes = TryGetWalFileSize(); + var lastIndexRun = GetLastIndexRun(); var batchInProgress = string.Equals( TryGetMetaStringInternal(DbContext.BatchInProgressMetaKey), "true", @@ -468,6 +495,7 @@ public StatusResult GetStatus() IndexedHeadBranch = indexedHeadBranch, IndexedHeadTimestamp = indexedHeadTimestamp, Languages = langs, + SymbolsByLanguage = symbolsByLanguage.Count > 0 ? symbolsByLanguage : null, GraphTableAvailable = _hasReferencesTable, IssuesTableAvailable = _hasIssuesPhysicalTable, FileIssuesDataCurrent = _hasIssuesTable, @@ -486,6 +514,10 @@ public StatusResult GetStatus() IndexNewerThanReaderReason = _indexNewerThanReaderReason, PathCaseSensitive = pathCaseSensitive, DbPragmaSettings = dbPragmaSettings, + DbSizeBytes = dbSizeBytes, + WalSizeBytes = walSizeBytes, + Process = StatusProcessMetrics.Capture(), + LastIndexRun = lastIndexRun, ReadOnlyFallback = _readOnlyFallback, WalCheckpointAttempted = _walCheckpointAttempted, WalCheckpointSucceeded = _walCheckpointSucceeded, @@ -523,6 +555,73 @@ private long ExecuteScalar(string sql) PageSize = ExecuteNullableLong("PRAGMA page_size"), }; + private long? TryGetDatabaseFileSize() + { + var path = _conn.DataSource; + if (string.IsNullOrWhiteSpace(path) || path.StartsWith("file:", StringComparison.OrdinalIgnoreCase)) + return null; + + try + { + var info = new FileInfo(path); + return info.Exists ? info.Length : null; + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or NotSupportedException or ArgumentException) + { + return null; + } + } + + private long? TryGetWalFileSize() + { + var path = _conn.DataSource; + if (string.IsNullOrWhiteSpace(path) || path.StartsWith("file:", StringComparison.OrdinalIgnoreCase)) + return null; + + try + { + var info = new FileInfo(path + "-wal"); + return info.Exists ? info.Length : 0; + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or NotSupportedException or ArgumentException) + { + return null; + } + } + + private StatusLastIndexRun? GetLastIndexRun() + { + var mode = TryGetMetaStringInternal(DbContext.LastIndexRunModeMetaKey); + var startedAt = ParseMetaDateTime(TryGetMetaStringInternal(DbContext.LastIndexRunStartedAtMetaKey)); + var durationMs = ParseMetaLong(TryGetMetaStringInternal(DbContext.LastIndexRunDurationMsMetaKey)); + var filesScanned = ParseMetaLong(TryGetMetaStringInternal(DbContext.LastIndexRunFilesScannedMetaKey)); + var filesSkipped = ParseMetaLong(TryGetMetaStringInternal(DbContext.LastIndexRunFilesSkippedMetaKey)); + var parseErrors = ParseMetaLong(TryGetMetaStringInternal(DbContext.LastIndexRunParseErrorsMetaKey)); + var bytesRead = ParseMetaLong(TryGetMetaStringInternal(DbContext.LastIndexRunBytesReadMetaKey)); + var rowsUpserted = ParseMetaLong(TryGetMetaStringInternal(DbContext.LastIndexRunRowsUpsertedMetaKey)); + var rowsDeleted = ParseMetaLong(TryGetMetaStringInternal(DbContext.LastIndexRunRowsDeletedMetaKey)); + var peakMemoryMb = ParseMetaLong(TryGetMetaStringInternal(DbContext.LastIndexRunPeakMemoryMbMetaKey)); + if (mode == null && startedAt == null && durationMs == null && filesScanned == null && filesSkipped == null + && parseErrors == null && bytesRead == null && rowsUpserted == null && rowsDeleted == null && peakMemoryMb == null) + { + return null; + } + + return new StatusLastIndexRun + { + Mode = mode, + StartedAt = startedAt, + DurationMs = durationMs, + FilesScanned = filesScanned, + FilesSkipped = filesSkipped, + ParseErrors = parseErrors, + BytesRead = bytesRead, + RowsUpserted = rowsUpserted, + RowsDeleted = rowsDeleted, + PeakMemoryMb = peakMemoryMb, + }; + } + private string? ExecuteScalarString(string sql) { using var cmd = _conn.CreateCommand(); diff --git a/src/CodeIndex/Mcp/McpToolHandlers.cs b/src/CodeIndex/Mcp/McpToolHandlers.cs index a7f42aa0c0..502d2b33d7 100644 --- a/src/CodeIndex/Mcp/McpToolHandlers.cs +++ b/src/CodeIndex/Mcp/McpToolHandlers.cs @@ -2786,6 +2786,8 @@ private JsonNode ExecuteIndex(JsonNode? id, JsonNode? args, JsonNode? progressTo if (maxFileBytes is <= 0 or > int.MaxValue) return CreateToolErrorResponse(id, "maxFileBytes must be a positive integer less than or equal to 2147483647"); var projectPath = Path.GetFullPath(path); + var runStartedAtUtc = DateTime.UtcNow; + var runStopwatch = Stopwatch.StartNew(); // Prevent path traversal — only allow indexing within current working directory // パストラバーサル防止 — カレントディレクトリ配下のみインデックスを許可 @@ -2872,6 +2874,25 @@ void WriteProjectRootOnce() } } + static long SumReadableFileBytes(IEnumerable paths) + { + long total = 0; + foreach (var filePath in paths) + { + try + { + var info = new FileInfo(filePath); + if (info.Exists) + total += info.Length; + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or NotSupportedException or ArgumentException) + { + } + } + + return total; + } + // First mutation point — demote readiness just before any write. // 実書き込み直前で readiness をクリア。 writer.ClearReadyFlags(); @@ -3098,6 +3119,15 @@ void WriteProjectRootOnce() writer.SetMeta( DbContext.UnknownExtensionFileCountMetaKey, scanResult.UnknownExtensionFiles.Count.ToString(System.Globalization.CultureInfo.InvariantCulture)); + writer.SetMeta(DbContext.LastIndexRunModeMetaKey, rebuild ? "rebuild" : "mcp"); + writer.SetMeta(DbContext.LastIndexRunStartedAtMetaKey, runStartedAtUtc.ToString("o", System.Globalization.CultureInfo.InvariantCulture)); + writer.SetMeta(DbContext.LastIndexRunDurationMsMetaKey, runStopwatch.ElapsedMilliseconds.ToString(System.Globalization.CultureInfo.InvariantCulture)); + writer.SetMeta(DbContext.LastIndexRunFilesScannedMetaKey, files.Count.ToString(System.Globalization.CultureInfo.InvariantCulture)); + writer.SetMeta(DbContext.LastIndexRunFilesSkippedMetaKey, skipped.ToString(System.Globalization.CultureInfo.InvariantCulture)); + writer.SetMeta(DbContext.LastIndexRunParseErrorsMetaKey, errors.ToString(System.Globalization.CultureInfo.InvariantCulture)); + writer.SetMeta(DbContext.LastIndexRunBytesReadMetaKey, SumReadableFileBytes(files).ToString(System.Globalization.CultureInfo.InvariantCulture)); + writer.SetMeta(DbContext.LastIndexRunRowsUpsertedMetaKey, processed.ToString(System.Globalization.CultureInfo.InvariantCulture)); + writer.SetMeta(DbContext.LastIndexRunRowsDeletedMetaKey, purged.ToString(System.Globalization.CultureInfo.InvariantCulture)); // Persist the current HEAD only after the run is fully successful (errors == 0). // Mirrors the CLI full-scan contract (Issue #1508) so MCP-driven re-indexes also // refresh `worktree_head_changed`; partial / failed runs leave the prior HEAD diff --git a/src/CodeIndex/Models/QueryResults.cs b/src/CodeIndex/Models/QueryResults.cs index 852d66b97d..de7b56144a 100644 --- a/src/CodeIndex/Models/QueryResults.cs +++ b/src/CodeIndex/Models/QueryResults.cs @@ -556,6 +556,9 @@ public class StatusResult public int? CommitsAheadOfIndexedHead { get; set; } public Dictionary Languages { get; set; } = new(); public Dictionary? SymbolKinds { get; set; } + [JsonPropertyName("symbols_by_language")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public Dictionary>? SymbolsByLanguage { get; set; } public List? GraphSupportedLanguages { get; set; } [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public List? Hooks { get; set; } @@ -727,6 +730,70 @@ public class StatusResult /// [JsonPropertyName("db_pragma_settings")] public StatusDbPragmaSettings DbPragmaSettings { get; set; } = new(); + [JsonPropertyName("db_size_bytes")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public long? DbSizeBytes { get; set; } + [JsonPropertyName("wal_size_bytes")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public long? WalSizeBytes { get; set; } + [JsonPropertyName("process")] + public StatusProcessMetrics Process { get; set; } = StatusProcessMetrics.Capture(); + [JsonPropertyName("last_index_run")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public StatusLastIndexRun? LastIndexRun { get; set; } +} + +public sealed class StatusProcessMetrics +{ + [JsonPropertyName("heap_bytes")] + public long HeapBytes { get; set; } + [JsonPropertyName("gc_heap_size_bytes")] + public long GcHeapSizeBytes { get; set; } + [JsonPropertyName("gc_gen0_count")] + public int GcGen0Count { get; set; } + [JsonPropertyName("gc_gen1_count")] + public int GcGen1Count { get; set; } + [JsonPropertyName("gc_gen2_count")] + public int GcGen2Count { get; set; } + [JsonPropertyName("working_set_bytes")] + public long WorkingSetBytes { get; set; } + + public static StatusProcessMetrics Capture() + { + var gcInfo = GC.GetGCMemoryInfo(); + return new StatusProcessMetrics + { + HeapBytes = GC.GetTotalMemory(forceFullCollection: false), + GcHeapSizeBytes = gcInfo.HeapSizeBytes, + GcGen0Count = GC.CollectionCount(0), + GcGen1Count = GC.CollectionCount(1), + GcGen2Count = GC.CollectionCount(2), + WorkingSetBytes = System.Diagnostics.Process.GetCurrentProcess().WorkingSet64, + }; + } +} + +public sealed class StatusLastIndexRun +{ + public string? Mode { get; set; } + [JsonPropertyName("started_at")] + public DateTime? StartedAt { get; set; } + [JsonPropertyName("duration_ms")] + public long? DurationMs { get; set; } + [JsonPropertyName("files_scanned")] + public long? FilesScanned { get; set; } + [JsonPropertyName("files_skipped")] + public long? FilesSkipped { get; set; } + [JsonPropertyName("parse_errors")] + public long? ParseErrors { get; set; } + [JsonPropertyName("bytes_read")] + public long? BytesRead { get; set; } + [JsonPropertyName("rows_upserted")] + public long? RowsUpserted { get; set; } + [JsonPropertyName("rows_deleted")] + public long? RowsDeleted { get; set; } + [JsonPropertyName("peak_memory_mb")] + public long? PeakMemoryMb { get; set; } } public class StatusReadinessDegradation diff --git a/tests/CodeIndex.Tests/ConsoleUiTests.cs b/tests/CodeIndex.Tests/ConsoleUiTests.cs index 047c04ba8c..b45d9e9e9c 100644 --- a/tests/CodeIndex.Tests/ConsoleUiTests.cs +++ b/tests/CodeIndex.Tests/ConsoleUiTests.cs @@ -57,10 +57,10 @@ public void PrintUsage_WithoutBanner_HidesAsciiArtAndEasterEggFlags() Assert.DoesNotContain("██████╗", output); Assert.Contains("Usage:", output); - Assert.Contains("cdidx index [--db ] [--rebuild] [--optimize] [--verbose] [--dry-run] [--force] [--quiet] [--json] [--duration-format ]", output); + Assert.Contains("cdidx index [--db ] [--rebuild] [--optimize] [--verbose] [--dry-run] [--force] [--quiet] [--json] [--memory-trace] [--duration-format ] [--max-file-bytes ] [--follow-symlinks ]", output); Assert.Contains("cdidx hooks [--project ] [--force] [--json]", output); - Assert.Contains("cdidx index --commits [id ...] [--db ] [--verbose] [--dry-run] [--json] [--duration-format ]", output); - Assert.Contains("cdidx index --files [path ...] [--db ] [--verbose] [--dry-run] [--json] [--duration-format ]", output); + Assert.Contains("cdidx index --commits [id ...] [--db ] [--verbose] [--dry-run] [--json] [--memory-trace] [--duration-format ]", output); + Assert.Contains("cdidx index --files [path ...] [--db ] [--verbose] [--dry-run] [--json] [--memory-trace] [--duration-format ]", output); Assert.Contains("cdidx backfill-fold [--db ] [--json]", output); Assert.Contains("cdidx optimize [--db ] [--json]", output); Assert.Contains("cdidx license", output); diff --git a/tests/CodeIndex.Tests/DbReaderTests.cs b/tests/CodeIndex.Tests/DbReaderTests.cs index 590c40b71f..31e3a9d96f 100644 --- a/tests/CodeIndex.Tests/DbReaderTests.cs +++ b/tests/CodeIndex.Tests/DbReaderTests.cs @@ -71,6 +71,35 @@ public void DegradationReasonCodes_AllCodesHaveActionableMetadata() } } + [Fact] + public void GetStatus_ExposesOperationalMetrics() + { + _writer.SetMeta(DbContext.LastIndexRunModeMetaKey, "incremental"); + _writer.SetMeta(DbContext.LastIndexRunStartedAtMetaKey, "2026-05-31T00:00:00.0000000Z"); + _writer.SetMeta(DbContext.LastIndexRunDurationMsMetaKey, "1234"); + _writer.SetMeta(DbContext.LastIndexRunFilesScannedMetaKey, "3"); + _writer.SetMeta(DbContext.LastIndexRunFilesSkippedMetaKey, "1"); + _writer.SetMeta(DbContext.LastIndexRunParseErrorsMetaKey, "0"); + _writer.SetMeta(DbContext.LastIndexRunBytesReadMetaKey, "4096"); + _writer.SetMeta(DbContext.LastIndexRunRowsUpsertedMetaKey, "2"); + _writer.SetMeta(DbContext.LastIndexRunRowsDeletedMetaKey, "1"); + _writer.SetMeta(DbContext.LastIndexRunPeakMemoryMbMetaKey, "64"); + + var status = _reader.GetStatus(); + + Assert.True(status.DbSizeBytes > 0); + Assert.True(status.WalSizeBytes >= 0); + Assert.NotNull(status.SymbolsByLanguage); + Assert.True(status.SymbolsByLanguage.Values.Sum(kinds => kinds.Values.Sum()) > 0); + Assert.True(status.Process.HeapBytes > 0); + Assert.True(status.Process.WorkingSetBytes > 0); + Assert.NotNull(status.LastIndexRun); + Assert.Equal("incremental", status.LastIndexRun.Mode); + Assert.Equal(1234, status.LastIndexRun.DurationMs); + Assert.Equal(3, status.LastIndexRun.FilesScanned); + Assert.Equal(64, status.LastIndexRun.PeakMemoryMb); + } + [Theory] [InlineData(DegradationReasonCodes.MissingFoldBackfill, "--exact falls back")] [InlineData(DegradationReasonCodes.StaleFoldKeyVersion, "older fold-key version")] diff --git a/tests/CodeIndex.Tests/JsonOutputSnapshotHelper.cs b/tests/CodeIndex.Tests/JsonOutputSnapshotHelper.cs index 32bff6abd1..e2ed629ad8 100644 --- a/tests/CodeIndex.Tests/JsonOutputSnapshotHelper.cs +++ b/tests/CodeIndex.Tests/JsonOutputSnapshotHelper.cs @@ -63,6 +63,14 @@ internal static class JsonOutputSnapshotHelper private static readonly HashSet VolatileCountKeys = new(StringComparer.Ordinal) { "page_count", + "db_size_bytes", + "wal_size_bytes", + "heap_bytes", + "gc_heap_size_bytes", + "gc_gen0_count", + "gc_gen1_count", + "gc_gen2_count", + "working_set_bytes", }; public static void AssertMatches( diff --git a/tests/CodeIndex.Tests/golden/status.json b/tests/CodeIndex.Tests/golden/status.json index 6fe004c779..aa31679b19 100644 --- a/tests/CodeIndex.Tests/golden/status.json +++ b/tests/CodeIndex.Tests/golden/status.json @@ -20,6 +20,13 @@ "function": 1, "namespace": 1 }, + "symbols_by_language": { + "csharp": { + "class": 1, + "function": 1, + "namespace": 1 + } + }, "graph_supported_languages": [ "assembly", "batch", @@ -138,5 +145,15 @@ "page_count": "\u003CCOUNT\u003E", "freelist_count": 0, "page_size": 4096 + }, + "db_size_bytes": "\u003CCOUNT\u003E", + "wal_size_bytes": "\u003CCOUNT\u003E", + "process": { + "heap_bytes": "\u003CCOUNT\u003E", + "gc_heap_size_bytes": "\u003CCOUNT\u003E", + "gc_gen0_count": "\u003CCOUNT\u003E", + "gc_gen1_count": "\u003CCOUNT\u003E", + "gc_gen2_count": "\u003CCOUNT\u003E", + "working_set_bytes": "\u003CCOUNT\u003E" } }