From 62e8b636f0caf3374f07020d350d654973c2a53e Mon Sep 17 00:00:00 2001 From: Widthdom Date: Wed, 29 Jul 2026 12:41:06 +0900 Subject: [PATCH 1/2] Fix YAML sequence outline hierarchy (#4873) --- TESTING_GUIDE.md | 4 + USER_GUIDE.md | 4 +- changelog.d/unreleased/4873.fixed.md | 19 ++++ .../Symbols/SymbolExtractor.StructuredData.cs | 78 +++++++++---- .../QueryCommandRunnerInspectTests.cs | 104 ++++++++++++++++++ tests/CodeIndex.Tests/SymbolExtractorTests.cs | 58 ++++++++++ 6 files changed, 242 insertions(+), 25 deletions(-) create mode 100644 changelog.d/unreleased/4873.fixed.md diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index 1f35aaea5..798db890c 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -78,6 +78,7 @@ Use `docs/test-doc-maintenance-plan.md` before moving oversized suites or adding Console writer synchronization coverage yields between character writes instead of sleeping per character; use enough whole-line iterations to expose interleaving without adding wall-clock delay. - `SymbolExtractor*Tests.cs` and `ReferenceExtractor*Tests.cs` Extractor coverage is split by language or feature area with partial test classes, while shared helpers remain on the root `SymbolExtractorTests` / `ReferenceExtractorTests` parts. + YAML sequence-hierarchy fixtures must keep sequence items path-only, count empty/scalar/alias items in stable indexes, preserve anchor and block-scalar boundaries, and assert logical qualified paths separately from nearest emitted symbol parents. C# declaration-boundary regressions should pair a direct extractor fixture with a real-index `symbols --exact-name` query. Keep invocation and parameter continuations beside valid multi-line methods, constructors, delegates, and local functions so both false-positive rejection and declaration ranges remain observable. C# callable-containment fixtures should cover block-bodied test methods, local and nested local functions, named lambdas, expression-bodied members, and nested types together, asserting both symbol parents and call-reference containers. Repository-metadata coverage lives in `SymbolExtractorRepositoryMetadataTests.cs` and `ReferenceExtractorRepositoryMetadataTests.cs`; keep TOML, JSON Lines, ignore/attributes, EditorConfig, `.rules`, and application-manifest capability assertions coordinated with conservative local-path and malformed-record controls. @@ -180,6 +181,7 @@ Use `docs/test-doc-maintenance-plan.md` before moving oversized suites or adding - `LegacySchemaMigrationTests.cs` End-to-end upgrade path: seeds a pre-column legacy DB, opens it through `TryMigrateForRead`, and exercises the read paths that touch nullable symbol ordinals (outline, symbol search, nearby, unused, analyze bundle) to lock in the real-world failure mode behind #58 / #49. Migration ownership coverage must also distinguish caller-owned transactions from owned transactions, propagate unrelated `BEGIN` errors, preserve populated foreign-key rows across rebuilds, and verify rollback, foreign-key restoration, and successful retry after injected failures. - `IndexCommandRunner*Tests.cs`, `QueryCommandRunner*Tests.cs`, `ProgramCliTests.cs`, `InstallScriptTests.cs` + YAML `outline` integration coverage must assert nested sequence paths, mapping-parent depth, unchanged source ranges and symbol counts, deterministic sorting, and cursor replay across a page boundary. CLI parsing, command execution, and installer behavior. Index command coverage is split by run mode or feature area, and query command coverage is split by command family with partial test classes so shared console and fixture helpers stay centralized. Keep repeated query-result fixtures, such as overlapping chunk content used by multiple search deduplication tests, in narrow class-level helpers instead of duplicating local builders. `ProgramCliTests.cs` covers top-level entrypoint behavior that must be exercised through a subprocess, while `InstallScriptTests.cs` runs focused bash snippets against `install.sh` in library mode to lock in release-installer regressions without performing real network installs. Installer bundle-generation tests must also verify that `install.sh` is marked generated while every canonical `install_modules/` source remains unmarked. C# `unused` partial-family coverage must reuse one multi-file fixture across regular JSON, compact, `--by-bucket`, and `--actionable`; include top-level and nested partial types, genuinely unused private members, an unrelated same-named family, a containing-type generic-arity collision, and a family-external occurrence in a matched peer file so semantic membership cannot regress into name-only, flattened-qualified-name, or whole-file evidence sharing. `CliFlagSchemaTests.cs` and `ConsoleUiTests.cs` keep command help, runtime value validation, and each generated shell's option/value catalog aligned with `CliFlagSchema`, and check every Fish command scope against the shared per-command completion projection. When an accepted value or alias changes, assert its registry normalization plus command usage, runtime validation, and Bash / Zsh / Fish / PowerShell completion visibility; add the value to the schema instead of weakening this parity contract. @@ -996,6 +998,7 @@ dotnet test --filter "FullyQualifiedName~GitHelperTests" console writer synchronization coverageは文字writeごとのsleepではなくyieldを使い、wall-clock delayを追加せずinterleavingを露出できる十分なwhole-line iterationを維持してください。 - `SymbolExtractor*Tests.cs` と `ReferenceExtractor*Tests.cs` extractor のカバレッジは言語または機能領域ごとの partial test class に分割し、共有 helper は root 側の `SymbolExtractorTests` / `ReferenceExtractorTests` に残します。 + YAML sequence hierarchy の fixture では、sequence item を path 専用のまま保ち、empty / scalar / alias item を安定した index に数え、anchor と block scalar の境界を維持し、論理 qualified path と最寄りの出力済み symbol parent を分けて検証してください。 C# の declaration boundary に関する regression では、extractor を直接呼ぶ fixture と、実 index に対する `symbols --exact-name` query を組み合わせてください。呼び出し・parameter の continuation と、正当な複数行 method、constructor、delegate、local function を同居させ、false positive の拒否と宣言 range の両方を観測可能にします。 C# の callable containment fixture では、block body の test method、local / nested local function、named lambda、expression-bodied member、nested type を同居させ、symbol の親と call reference の container の両方を検証してください。 repository metadata の coverage は `SymbolExtractorRepositoryMetadataTests.cs` と `ReferenceExtractorRepositoryMetadataTests.cs` に置きます。TOML、JSON Lines、ignore / attributes、EditorConfig、`.rules`、application manifest の capability assertion を、保守的な local-path 抽出と malformed-record control に同期させてください。 @@ -1101,6 +1104,7 @@ dotnet test --filter "FullyQualifiedName~GitHelperTests" - `LegacySchemaMigrationTests.cs` エンドツーエンドのアップグレード経路: カラム追加前のレガシー DB を用意し、`TryMigrateForRead` 経由で開いてから NULL になりうるシンボル列を触る read path（outline、シンボル検索、近傍、unused、analyze バンドル）を一通り叩き、#58 / #49 の実機失敗モードを固定する。migration ownership の coverage では、caller-owned transaction と cdidx-owned transaction を区別し、無関係な `BEGIN` error を伝播させ、foreign key が設定された既存 row を rebuild 後も保持し、注入 failure 後の rollback、foreign key 復元、retry 成功まで検証する。 - `IndexCommandRunner*Tests.cs`、`QueryCommandRunner*Tests.cs`、`ProgramCliTests.cs`、`InstallScriptTests.cs` + YAML `outline` の integration coverage では、入れ子 sequence path、mapping parent の depth、不変な source range と symbol count、決定的な sort、page 境界を越える cursor replay を検証してください。 CLI の引数解析、コマンド実行、installer 挙動のテスト。Index command coverage は run mode または機能領域ごとの partial suite に分割し、Query command coverage は command family ごとの partial test class に分割して、共有 console / fixture helper は一箇所に保ちます。`ProgramCliTests.cs` はグローバル引数の解釈や完全な CLI 起動フローのように subprocess 経由で確認すべき Program エントリポイント挙動を扱い、`InstallScriptTests.cs` は `install.sh` を library mode で source した bash snippet を実行して、実ネットワーク install を行わずに release installer の回帰を固定する。installer bundle 生成テストでは、`install.sh` が generated と判定される一方、canonical な `install_modules/` source はすべて unmarked のままであることも検証してください。 C# `unused` の partial-family coverage では、通常の JSON、compact、`--by-bucket`、`--actionable` で1つの multi-file fixture を共有してください。top-level / nested partial type、本当に未使用の private member、無関係な同名 family、containing type の generic-arity collision、matched peer file 内の family 外 occurrence を含め、semantic membership が name-only、平坦化された qualified name、または file 全体の evidence 共有へ戻らないことを固定します。 `CliFlagSchemaTests.cs` と `ConsoleUiTests.cs` は command help、runtime value validation、各生成 shell の option / value 全カタログを `CliFlagSchema` と同期させ、Fish の全 command scope を共有の command 別 completion 射影と照合します。受理値または alias を変更するときは、registry normalization に加えて command usage、runtime validation、Bash / Zsh / Fish / PowerShell completion への露出を検証し、この parity 契約を弱めず schema に値を追加してください。 diff --git a/USER_GUIDE.md b/USER_GUIDE.md index 326243213..83c98e8d4 100644 --- a/USER_GUIDE.md +++ b/USER_GUIDE.md @@ -2566,7 +2566,7 @@ All indexed languages are searchable through FTS5. Rows with **Symbols = yes** a - Dynamic/declarative graph languages: Crystal, Groovy, Tcl, and Prolog expose conservative declarations, imports, and call relationships. Crystal, Groovy, and Prolog parenthesized calls use the shared extractor; command-style calls are limited to callables declared in the same file. Tcl recognizes command substitutions and common control-command script arguments without treating ordinary `name()` words as calls, while Tcl proc / Prolog predicate bodies preserve caller containers. An index created before this graph contract reports `reference_graph_complete=false` and `graph_data_current=false` with `dynamic_reference_graph_contract_stale`; rerun `cdidx index ` to refresh affected rows before treating absent edges as authoritative. - Scientific and native-extension graphs: Julia, MATLAB, Nim, D, Cython, and Ada emit bounded language-aware import/module, base/type, and call references. Julia macro invocations and Ada procedure-style calls without parentheses are also represented. -- Markdown, JSON/YAML, and CSS: Markdown headings and explicit HTML anchors are indexed as definitions, while local and cross-document fragment links are indexed as path-scoped references. Heading slugs use rendered inline text; explicit HTML IDs preserve exact case and punctuation after HTML entity decoding. JSON/YAML configuration keys are indexed as structural key paths; CSS variables, placeholders, and `@extend` references are indexed. +- Markdown, JSON/YAML, and CSS: Markdown headings and explicit HTML anchors are indexed as definitions, while local and cross-document fragment links are indexed as path-scoped references. Heading slugs use rendered inline text; explicit HTML IDs preserve exact case and punctuation after HTML entity decoding. JSON/YAML configuration keys are indexed as structural key paths. YAML sequence elements remain path-only (`steps[14]`) rather than adding synthetic symbols, while their mapping descendants retain the nearest indexed mapping parent so `outline` reports stable paths and meaningful depth. CSS variables, placeholders, and `@extend` references are indexed. - Dockerfile, Assembly, Common Lisp, and Racket: `ARG` build args, labels/PROC/MACRO blocks, package/module forms, definitions, classes/structs, requires, and provides are surfaced as symbols where applicable. - Shell, PowerShell, and Batch: command-style function calls, functions/filters, classes/enums, imports, labels, `goto` / `call` targets, and inline control-flow forms are indexed where the language supports them. - C# and Java: modern C# partial members remain visible to `symbols`, `definition`, and `outline`; Java sealed `permits` lists are recorded as `type_reference` graph edges. @@ -5766,7 +5766,7 @@ indexing はファイル単位の SQLite transaction を commit します。長 - 動的・宣言型言語の graph 対応: Crystal、Groovy、Tcl、Prolog は保守的な宣言、import、call relationship を公開します。Crystal、Groovy、Prolog の括弧付き call は共通 extractor を使い、command-style call は同一ファイルで宣言済みの callable に限定します。Tcl は通常の `name()` word を call とみなさず、command substitution と主要な制御 command の script 引数を認識し、Tcl proc / Prolog predicate の本体では caller container を保持します。 この graph contract より前に作成された index は `dynamic_reference_graph_contract_stale` とともに `reference_graph_complete=false`、`graph_data_current=false` を報告します。欠落 edge を authoritative とみなす前に `cdidx index ` を再実行して対象 row を更新してください。 - 科学技術・ネイティブ拡張言語のグラフ: Julia、MATLAB、Nim、D、Cython、Ada は、言語構文に応じた import/module、基底型/type、call 参照を上限付きで出力します。Julia の macro invocation と、括弧を伴わない Ada の procedure call も記録します。 -- Markdown、JSON/YAML、CSS: Markdown の heading と明示的な HTML anchor は定義として、同一文書・文書間の fragment link は対象 path に限定した参照として索引します。heading slug は表示される inline text から作り、明示的な HTML ID は HTML entity の decode 後も大文字小文字と句読点を正確に保持します。JSON/YAML の configuration key path、CSS の variable、placeholder、`@extend` もシンボルとして扱います。 +- Markdown、JSON/YAML、CSS: Markdown の heading と明示的な HTML anchor は定義として、同一文書・文書間の fragment link は対象 path に限定した参照として索引します。heading slug は表示される inline text から作り、明示的な HTML ID は HTML entity の decode 後も大文字小文字と句読点を正確に保持します。JSON/YAML の configuration key path を索引します。YAML の sequence element は synthetic symbol を追加せず path 専用（`steps[14]`）のまま扱い、その mapping descendant には最寄りの索引済み mapping parent を保持するため、`outline` は安定した path と意味のある depth を返します。CSS の variable、placeholder、`@extend` もシンボルとして扱います。 - Dockerfile、Assembly、Common Lisp、Racket: `ARG` build arg、label、PROC/MACRO、package/module form、definition、class/struct、require/provide を必要に応じて表面化します。 - Shell、PowerShell、Batch: command-style function call、function/filter、class/enum、import、label、`goto` / `call` target、inline control-flow を言語仕様に合わせて索引します。 - C# と Java: C# の近年の partial member は `symbols`、`definition`、`outline` から見えます。Java の sealed `permits` list は `type_reference` graph edge として記録します。 diff --git a/changelog.d/unreleased/4873.fixed.md b/changelog.d/unreleased/4873.fixed.md new file mode 100644 index 000000000..1cee2f140 --- /dev/null +++ b/changelog.d/unreleased/4873.fixed.md @@ -0,0 +1,19 @@ +--- +category: fixed +issues: + - 4873 +affected: + - src/CodeIndex/Indexer/Symbols/SymbolExtractor.StructuredData.cs + - tests/CodeIndex.Tests/SymbolExtractorTests.cs + - tests/CodeIndex.Tests/QueryCommandRunnerInspectTests.cs + - USER_GUIDE.md + - TESTING_GUIDE.md +--- + +## English + +- **YAML sequence descendants now retain their outline hierarchy (#4873)** — indexed mapping keys keep stable `[n]` paths while attaching to the nearest emitted mapping symbol, so nested workflow members report correct parents and non-zero depth without synthetic sequence-item symbols or changed source ranges. + +## 日本語 + +- **YAML sequence の descendant が outline hierarchy を保持するようになりました（#4873）** — 索引した mapping key は安定した `[n]` path を維持しながら最寄りの出力済み mapping symbol に接続されるため、synthetic な sequence-item symbol や source range の変更なしで、入れ子 workflow member が正しい parent と非ゼロ depth を報告します。 diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.StructuredData.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.StructuredData.cs index 92db92bf2..8179732e7 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.StructuredData.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.StructuredData.cs @@ -33,7 +33,7 @@ public static partial class SymbolExtractor @"^(?[ ]*)(?:-\s*)?(?:""(?(?:[^""]|"""")+)""|'(?(?:[^']|'')+)'|(?[A-Za-z0-9_.-][A-Za-z0-9_. -]*))\s*:\s*(?.*)$", RegexOptions.Compiled | RegexOptions.CultureInvariant); - private readonly record struct YamlPathFrame(int Indent, string Path); + private readonly record struct YamlPathFrame(int Indent, string Path, string? SymbolPath); private static List ExtractJsonSymbols(long fileId, string content, string[] lines) { @@ -417,17 +417,30 @@ private static List ExtractYamlSymbols(long fileId, string[] lines continue; } - if (trimmed.SequenceEqual("-")) + var isSequenceItem = trimmed[0] == '-' + && (trimmed.Length == 1 || char.IsWhiteSpace(trimmed[1])); + if (isSequenceItem) { while (stack != null && stack.Count > 0 && indent <= stack[^1].Indent) stack.RemoveAt(stack.Count - 1); var sequenceParent = stack == null || stack.Count == 0 ? null : stack[^1].Path; + var sequenceSymbolParent = stack == null || stack.Count == 0 ? null : stack[^1].SymbolPath; var sequenceKey = $"{indent}:{sequenceParent}"; sequenceIndexes ??= new Dictionary(StringComparer.Ordinal); sequenceIndexes.TryGetValue(sequenceKey, out var sequenceIndex); sequenceIndexes[sequenceKey] = sequenceIndex + 1; - (stack ??= []).Add(new YamlPathFrame(indent, $"{sequenceParent}[{sequenceIndex}]")); - continue; + (stack ??= []).Add(new YamlPathFrame( + indent, + $"{sequenceParent}[{sequenceIndex}]", + sequenceSymbolParent)); + + var sequenceValue = trimmed[1..].TrimStart(); + if (sequenceValue.IsEmpty + || sequenceValue[0] == '#' + || line.IndexOf(':') < 0) + { + continue; + } } if (line.IndexOf(':') < 0) @@ -441,20 +454,14 @@ private static List ExtractYamlSymbols(long fileId, string[] lines if (key.Length == 0) continue; - while (stack != null && stack.Count > 0 && indent <= stack[^1].Indent) - stack.RemoveAt(stack.Count - 1); - - var parentPath = stack == null || stack.Count == 0 ? null : stack[^1].Path; - if (trimmed.StartsWith("- ", StringComparison.Ordinal)) + if (!isSequenceItem) { - var sequenceKey = $"{indent}:{parentPath}"; - sequenceIndexes ??= new Dictionary(StringComparer.Ordinal); - sequenceIndexes.TryGetValue(sequenceKey, out var sequenceIndex); - sequenceIndexes[sequenceKey] = sequenceIndex + 1; - var itemPath = $"{parentPath}[{sequenceIndex}]"; - (stack ??= []).Add(new YamlPathFrame(indent, itemPath)); - parentPath = itemPath; + while (stack != null && stack.Count > 0 && indent <= stack[^1].Indent) + stack.RemoveAt(stack.Count - 1); } + + var parentPath = stack == null || stack.Count == 0 ? null : stack[^1].Path; + var symbolParentPath = stack == null || stack.Count == 0 ? null : stack[^1].SymbolPath; var pathLength = string.IsNullOrEmpty(parentPath) ? key.Length : parentPath.Length + 1 + key.Length; @@ -474,10 +481,11 @@ private static List ExtractYamlSymbols(long fileId, string[] lines path, i + 1, lines, - parentPath, + symbolParentPath, symbols ??= CreateSymbolListForLines(lines.Length), "structured_data_symbol_budget_exceeded", - ref truncated)) + ref truncated, + parentPath)) break; if (isBlockScalar) @@ -486,7 +494,7 @@ private static List ExtractYamlSymbols(long fileId, string[] lines } else if (isContainer) { - (stack ??= []).Add(new YamlPathFrame(indent, path)); + (stack ??= []).Add(new YamlPathFrame(indent, path, path)); } } @@ -502,11 +510,19 @@ private static bool TryAddStructuredDataSymbol( string? parentPath, List symbols, string category, - ref bool truncated) + ref bool truncated, + string? qualifiedParentPath = null) { if (symbols.Count < StructuredDataMaxSymbols) { - symbols.Add(CreateStructuredDataSymbol(fileId, kind, name, line, lines, parentPath)); + symbols.Add(CreateStructuredDataSymbol( + fileId, + kind, + name, + line, + lines, + parentPath, + qualifiedParentPath)); return true; } @@ -587,7 +603,8 @@ private static SymbolRecord CreateStructuredDataSymbol( string name, int line, string[] lines, - string? parentPath) + string? parentPath, + string? qualifiedParentPath) { var signatureIndex = Math.Clamp(line - 1, 0, Math.Max(0, lines.Length - 1)); return new SymbolRecord @@ -601,7 +618,7 @@ private static SymbolRecord CreateStructuredDataSymbol( Signature = lines.Length == 0 ? null : LimitStructuredDataLineSignature(lines[signatureIndex]), ContainerKind = parentPath == null ? null : "namespace", ContainerName = parentPath, - ContainerQualifiedName = parentPath, + ContainerQualifiedName = qualifiedParentPath ?? parentPath, }; } @@ -766,6 +783,7 @@ private static string UnescapeYamlQuotedKey(ReadOnlySpan value, string esc private static bool IsYamlContainerValue(ReadOnlySpan value) => value.IsEmpty + || IsYamlAnchorDeclaration(value) || value.SequenceEqual("|") || value.SequenceEqual(">") || value.SequenceEqual("|-") @@ -773,6 +791,20 @@ private static bool IsYamlContainerValue(ReadOnlySpan value) || value.SequenceEqual("|+") || value.SequenceEqual(">+"); + private static bool IsYamlAnchorDeclaration(ReadOnlySpan value) + { + if (value.Length < 2 || value[0] != '&') + return false; + + for (var i = 1; i < value.Length; i++) + { + if (char.IsWhiteSpace(value[i])) + return false; + } + + return true; + } + private static ReadOnlySpan StripYamlInlineComment(ReadOnlySpan value) { var inSingle = false; diff --git a/tests/CodeIndex.Tests/QueryCommandRunnerInspectTests.cs b/tests/CodeIndex.Tests/QueryCommandRunnerInspectTests.cs index 0ca51e5ee..0eaf79a4f 100644 --- a/tests/CodeIndex.Tests/QueryCommandRunnerInspectTests.cs +++ b/tests/CodeIndex.Tests/QueryCommandRunnerInspectTests.cs @@ -1620,6 +1620,110 @@ public void RunOutline_Json_YamlNestedPathsDoNotDuplicateParentKeys_Issue4151() } } + [Fact] + public void RunOutline_Json_YamlSequenceHierarchyRetainsDepthAcrossPages_Issue4873() + { + var projectRoot = TestProjectHelper.CreateTempProject("cdidx_outline_yaml_sequence_hierarchy_4873"); + try + { + var dbPath = TestProjectHelper.CreateProjectDb(projectRoot); + var lines = new List + { + "jobs:", + " build:", + " steps:", + }; + for (var index = 0; index < 14; index++) + lines.Add($" - run: echo {index}"); + lines.AddRange( + [ + " - name: Upload", + " with:", + " path: artifacts", + ]); + TestProjectHelper.InsertIndexedFile( + dbPath, + ".github/workflows/release.yml", + "yaml", + string.Join('\n', lines)); + + var outlineFields = "name,path,container_name,depth,line,start_line,end_line"; + var (firstExitCode, firstStdout, firstStderr) = CaptureConsole(() => QueryCommandRunner.RunOutline( + [ + ".github/workflows/release.yml", + "--db", dbPath, + "--json", + "--sort", "source", + "--limit", "19", + "--outline-fields", outlineFields + ], + _jsonOptions)); + + Assert.Equal(CommandExitCodes.Success, firstExitCode); + Assert.Equal(string.Empty, firstStderr); + using var firstDocument = ParseJsonOutput(firstStdout); + var firstJson = firstDocument.RootElement; + var firstSymbols = firstJson.GetProperty("symbols").EnumerateArray().ToList(); + + Assert.Equal(20, firstJson.GetProperty("total_symbol_count").GetInt32()); + Assert.Equal(19, firstJson.GetProperty("returned_symbol_count").GetInt32()); + Assert.True(firstJson.GetProperty("has_more").GetBoolean()); + Assert.DoesNotContain(firstSymbols, symbol => + symbol.GetProperty("name").GetString() is "jobs.build.steps[14]" or "jobs.build.steps[14].path"); + Assert.Equal( + new[] { "jobs", "jobs.build", "jobs.build.steps" }, + firstSymbols.Take(3).Select(symbol => symbol.GetProperty("name").GetString()).ToArray()); + + var steps = Assert.Single(firstSymbols, symbol => + symbol.GetProperty("name").GetString() == "jobs.build.steps"); + Assert.Equal(2, steps.GetProperty("depth").GetInt32()); + Assert.Equal("jobs.build", steps.GetProperty("container_name").GetString()); + + var with = Assert.Single(firstSymbols, symbol => + symbol.GetProperty("name").GetString() == "jobs.build.steps[14].with"); + Assert.Equal("jobs.build.steps[14].with", with.GetProperty("path").GetString()); + Assert.Equal("jobs.build.steps", with.GetProperty("container_name").GetString()); + Assert.Equal(3, with.GetProperty("depth").GetInt32()); + Assert.Equal(19, with.GetProperty("line").GetInt32()); + + var nextCursor = firstJson.GetProperty("next_cursor").GetString(); + Assert.StartsWith("page:v1:", nextCursor, StringComparison.Ordinal); + var (secondExitCode, secondStdout, secondStderr) = CaptureConsole(() => QueryCommandRunner.RunOutline( + [ + ".github/workflows/release.yml", + "--db", dbPath, + "--json", + "--sort", "source", + "--limit", "19", + "--cursor", nextCursor!, + "--outline-fields", outlineFields + ], + _jsonOptions)); + + Assert.Equal(CommandExitCodes.Success, secondExitCode); + Assert.Equal(string.Empty, secondStderr); + using var secondDocument = ParseJsonOutput(secondStdout); + var secondJson = secondDocument.RootElement; + var path = Assert.Single(secondJson.GetProperty("symbols").EnumerateArray()); + + Assert.Equal(20, secondJson.GetProperty("total_symbol_count").GetInt32()); + Assert.Equal(1, secondJson.GetProperty("returned_symbol_count").GetInt32()); + Assert.Equal(19, secondJson.GetProperty("cursor_offset").GetInt32()); + Assert.False(secondJson.GetProperty("has_more").GetBoolean()); + Assert.Equal("jobs.build.steps[14].with.path", path.GetProperty("name").GetString()); + Assert.Equal("jobs.build.steps[14].with.path", path.GetProperty("path").GetString()); + Assert.Equal("jobs.build.steps[14].with", path.GetProperty("container_name").GetString()); + Assert.Equal(4, path.GetProperty("depth").GetInt32()); + Assert.Equal(20, path.GetProperty("line").GetInt32()); + Assert.Equal(20, path.GetProperty("start_line").GetInt32()); + Assert.Equal(20, path.GetProperty("end_line").GetInt32()); + } + finally + { + TestProjectHelper.DeleteDirectory(projectRoot); + } + } + [Fact] public void RunOutline_CompactJson_CapsSymbolsAndReportsTruncation_Issue3009() { diff --git a/tests/CodeIndex.Tests/SymbolExtractorTests.cs b/tests/CodeIndex.Tests/SymbolExtractorTests.cs index 7dd72699b..2caba3126 100644 --- a/tests/CodeIndex.Tests/SymbolExtractorTests.cs +++ b/tests/CodeIndex.Tests/SymbolExtractorTests.cs @@ -1055,6 +1055,64 @@ public void Extract_Yaml_PreservesBareDashSequenceIdentity_Issue4411() Assert.DoesNotContain(symbols, symbol => symbol.Name == "steps.name"); } + [Fact] + public void Extract_Yaml_SequenceItemsUsePathOnlyIdentityAndNearestSymbolParent_Issue4873() + { + const string content = """ + - &first + name: Build + with: + path: | + ignored: value + - *first + - + - null + - name: Final + nested: + value: yes + defaults: &defaults + retries: 3 + copy: *defaults + """; + + var symbols = SymbolExtractor.Extract(1, "yaml", content); + + Assert.Equal(9, symbols.Count); + Assert.DoesNotContain(symbols, symbol => symbol.Name is "[0]" or "[1]" or "[2]" or "[3]" or "[4]"); + + var firstName = Assert.Single(symbols, symbol => symbol.Name == "[0].name"); + Assert.Null(firstName.ContainerName); + Assert.Equal("[0]", firstName.ContainerQualifiedName); + + var firstWith = Assert.Single(symbols, symbol => symbol.Name == "[0].with"); + Assert.Equal("namespace", firstWith.Kind); + Assert.Null(firstWith.ContainerName); + Assert.Equal("[0]", firstWith.ContainerQualifiedName); + + var firstPath = Assert.Single(symbols, symbol => symbol.Name == "[0].with.path"); + Assert.Equal("[0].with", firstPath.ContainerName); + Assert.Equal("[0].with", firstPath.ContainerQualifiedName); + Assert.Equal(4, firstPath.Line); + Assert.Equal(4, firstPath.StartLine); + Assert.Equal(4, firstPath.EndLine); + + var finalName = Assert.Single(symbols, symbol => symbol.Name == "[4].name"); + Assert.Null(finalName.ContainerName); + Assert.Equal("[4]", finalName.ContainerQualifiedName); + Assert.Contains(symbols, symbol => + symbol.Name == "[4].nested.value" + && symbol.ContainerName == "[4].nested" + && symbol.ContainerQualifiedName == "[4].nested"); + + Assert.Contains(symbols, symbol => symbol.Kind == "namespace" && symbol.Name == "defaults"); + Assert.Contains(symbols, symbol => + symbol.Name == "defaults.retries" + && symbol.ContainerName == "defaults" + && symbol.ContainerQualifiedName == "defaults"); + Assert.Contains(symbols, symbol => symbol.Kind == "property" && symbol.Name == "copy"); + Assert.DoesNotContain(symbols, symbol => symbol.Name.Contains("ignored", StringComparison.Ordinal)); + } + [Fact] public void Extract_Yaml_CapsBroadMappings_Issue3808() { From 3e3df14749231fa75b5c9f0cf86dd6e852aabdc5 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Wed, 29 Jul 2026 13:31:48 +0900 Subject: [PATCH 2/2] Refresh stale YAML hierarchy indexes (#4873) --- TESTING_GUIDE.md | 4 +- changelog.d/unreleased/4873.fixed.md | 2 + .../Symbols/SymbolExtractor.Contracts.cs | 4 +- .../Symbols/SymbolExtractor.StructuredData.cs | 10 ++- .../IndexCommandRunnerFullScanTests.cs | 67 +++++++++++++++++++ tests/CodeIndex.Tests/SymbolExtractorTests.cs | 32 ++++++--- 6 files changed, 104 insertions(+), 15 deletions(-) diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index e59553fbc..b53ed8b9a 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -79,7 +79,7 @@ Use `docs/test-doc-maintenance-plan.md` before moving oversized suites or adding Console writer synchronization coverage yields between character writes instead of sleeping per character; use enough whole-line iterations to expose interleaving without adding wall-clock delay. - `SymbolExtractor*Tests.cs` and `ReferenceExtractor*Tests.cs` Extractor coverage is split by language or feature area with partial test classes, while shared helpers remain on the root `SymbolExtractorTests` / `ReferenceExtractorTests` parts. - YAML sequence-hierarchy fixtures must keep sequence items path-only, count empty/scalar/alias items in stable indexes, preserve anchor and block-scalar boundaries, and assert logical qualified paths separately from nearest emitted symbol parents. + YAML sequence-hierarchy fixtures must keep sequence items path-only, count empty/scalar/alias items in stable indexes, preserve anchor and block-scalar boundaries, assert logical qualified paths separately from nearest emitted symbol parents, and prove a normal full scan refreshes unchanged YAML rows stamped with the prior extractor contract. C# declaration-boundary regressions should pair a direct extractor fixture with a real-index `symbols --exact-name` query. Keep invocation and parameter continuations beside valid multi-line methods, constructors, delegates, and local functions so both false-positive rejection and declaration ranges remain observable. C# callable-containment fixtures should cover block-bodied test methods, local and nested local functions, named lambdas, expression-bodied members, and nested types together, asserting both symbol parents and call-reference containers. Repository-metadata coverage lives in `SymbolExtractorRepositoryMetadataTests.cs` and `ReferenceExtractorRepositoryMetadataTests.cs`; keep TOML, JSON Lines, ignore/attributes, EditorConfig, `.rules`, and application-manifest capability assertions coordinated with conservative local-path and malformed-record controls. @@ -1001,7 +1001,7 @@ dotnet test --filter "FullyQualifiedName~GitHelperTests" console writer synchronization coverageは文字writeごとのsleepではなくyieldを使い、wall-clock delayを追加せずinterleavingを露出できる十分なwhole-line iterationを維持してください。 - `SymbolExtractor*Tests.cs` と `ReferenceExtractor*Tests.cs` extractor のカバレッジは言語または機能領域ごとの partial test class に分割し、共有 helper は root 側の `SymbolExtractorTests` / `ReferenceExtractorTests` に残します。 - YAML sequence hierarchy の fixture では、sequence item を path 専用のまま保ち、empty / scalar / alias item を安定した index に数え、anchor と block scalar の境界を維持し、論理 qualified path と最寄りの出力済み symbol parent を分けて検証してください。 + YAML sequence hierarchy の fixture では、sequence item を path 専用のまま保ち、empty / scalar / alias item を安定した index に数え、anchor と block scalar の境界を維持し、論理 qualified path と最寄りの出力済み symbol parent を分けて検証し、旧 extractor contract が stamp された未変更 YAML row を通常の full scan が更新することも確認してください。 C# の declaration boundary に関する regression では、extractor を直接呼ぶ fixture と、実 index に対する `symbols --exact-name` query を組み合わせてください。呼び出し・parameter の continuation と、正当な複数行 method、constructor、delegate、local function を同居させ、false positive の拒否と宣言 range の両方を観測可能にします。 C# の callable containment fixture では、block body の test method、local / nested local function、named lambda、expression-bodied member、nested type を同居させ、symbol の親と call reference の container の両方を検証してください。 repository metadata の coverage は `SymbolExtractorRepositoryMetadataTests.cs` と `ReferenceExtractorRepositoryMetadataTests.cs` に置きます。TOML、JSON Lines、ignore / attributes、EditorConfig、`.rules`、application manifest の capability assertion を、保守的な local-path 抽出と malformed-record control に同期させてください。 diff --git a/changelog.d/unreleased/4873.fixed.md b/changelog.d/unreleased/4873.fixed.md index 1cee2f140..825a6d2e0 100644 --- a/changelog.d/unreleased/4873.fixed.md +++ b/changelog.d/unreleased/4873.fixed.md @@ -4,8 +4,10 @@ issues: - 4873 affected: - src/CodeIndex/Indexer/Symbols/SymbolExtractor.StructuredData.cs + - src/CodeIndex/Indexer/Symbols/SymbolExtractor.Contracts.cs - tests/CodeIndex.Tests/SymbolExtractorTests.cs - tests/CodeIndex.Tests/QueryCommandRunnerInspectTests.cs + - tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs - USER_GUIDE.md - TESTING_GUIDE.md --- diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.Contracts.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.Contracts.cs index 43c4cada5..45199f83a 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.Contracts.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.Contracts.cs @@ -4,6 +4,7 @@ public static partial class SymbolExtractor { public const int DefaultContractVersion = 1; public const int ExpandedLanguageContractVersion = 2; + public const int YamlContractVersion = 3; public const int PythonContractVersion = 2; public const int CSharpContractVersion = 8; public const int DockerfileContractVersion = 2; @@ -72,7 +73,8 @@ public static int GetContractVersion(string? lang) "app_manifest" => ApplicationManifestContractVersion, "markdown" => MarkdownContractVersion, "dependency_lock" => DependencyLockContractVersion, - "cmake" or "dependency_manifest" or "graphql" or "html" or "json" or "justfile" or "msbuild" or "solution" or "yaml" => ExpandedLanguageContractVersion, + "yaml" => YamlContractVersion, + "cmake" or "dependency_manifest" or "graphql" or "html" or "json" or "justfile" or "msbuild" or "solution" => ExpandedLanguageContractVersion, _ => DefaultContractVersion, }; } diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.StructuredData.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.StructuredData.cs index 8179732e7..0caaac340 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.StructuredData.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.StructuredData.cs @@ -419,6 +419,7 @@ private static List ExtractYamlSymbols(long fileId, string[] lines var isSequenceItem = trimmed[0] == '-' && (trimmed.Length == 1 || char.IsWhiteSpace(trimmed[1])); + var mappingIndent = indent; if (isSequenceItem) { while (stack != null && stack.Count > 0 && indent <= stack[^1].Indent) @@ -434,7 +435,12 @@ private static List ExtractYamlSymbols(long fileId, string[] lines $"{sequenceParent}[{sequenceIndex}]", sequenceSymbolParent)); - var sequenceValue = trimmed[1..].TrimStart(); + var rawSequenceValue = trimmed[1..]; + var sequenceValueStart = rawSequenceValue.TrimStart(); + mappingIndent = indent + 1 + rawSequenceValue.Length - sequenceValueStart.Length; + var sequenceValue = StripYamlInlineComment(sequenceValueStart).Trim(); + if (!sequenceValue.IsEmpty && (sequenceValue[0] == '|' || sequenceValue[0] == '>')) + blockScalarIndent = indent; if (sequenceValue.IsEmpty || sequenceValue[0] == '#' || line.IndexOf(':') < 0) @@ -494,7 +500,7 @@ private static List ExtractYamlSymbols(long fileId, string[] lines } else if (isContainer) { - (stack ??= []).Add(new YamlPathFrame(indent, path, path)); + (stack ??= []).Add(new YamlPathFrame(mappingIndent, path, path)); } } diff --git a/tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs b/tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs index 7173e93c8..4031b87de 100644 --- a/tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs +++ b/tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs @@ -4218,6 +4218,73 @@ def target(): } } + [Fact] + public void Run_FullScan_ReindexesYamlHierarchyWhenExtractorContractChanges_Issue4873() + { + var projectRoot = CreateTempProject(); + try + { + File.WriteAllText(Path.Combine(projectRoot, "app.cs"), "public class App { public void Run() { } }"); + File.WriteAllText( + Path.Combine(projectRoot, "workflow.yml"), + """ + jobs: + build: + steps: + - name: Upload + with: + path: artifacts + """); + + var initialExitCode = IndexCommandRunner.Run([projectRoot, "--json"], _jsonOptions); + Assert.Equal(CommandExitCodes.Success, initialExitCode); + + var dbPath = Path.Combine(projectRoot, ".cdidx", "codeindex.db"); + using (var conn = OpenNonPoolingConnection(dbPath)) + { + conn.Open(); + using var cmd = conn.CreateCommand(); + cmd.CommandText = """ + UPDATE symbols + SET container_name = 'jobs.build.steps[0]', + container_qualified_name = 'jobs.build.steps[0]' + WHERE name = 'jobs.build.steps[0].with'; + UPDATE codeindex_meta + SET value = '2' + WHERE key = 'symbol_extractor_version_yaml'; + """; + cmd.ExecuteNonQuery(); + } + + var (exitCode, json) = RunAndCaptureJson([projectRoot, "--json"]); + + Assert.Equal(CommandExitCodes.Success, exitCode); + Assert.Equal("success", json.GetProperty("status").GetString()); + Assert.True(json.GetProperty("summary").GetProperty("files_skipped").GetInt32() > 0); + + using var verify = OpenNonPoolingConnection(dbPath); + verify.Open(); + + using var parentCmd = verify.CreateCommand(); + parentCmd.CommandText = """ + SELECT container_name || '|' || container_qualified_name + FROM symbols + WHERE name = 'jobs.build.steps[0].with' + """; + Assert.Equal("jobs.build.steps|jobs.build.steps[0]", parentCmd.ExecuteScalar() as string); + + using var versionCmd = verify.CreateCommand(); + versionCmd.CommandText = "SELECT value FROM codeindex_meta WHERE key = 'symbol_extractor_version_yaml'"; + Assert.Equal( + SymbolExtractor.YamlContractVersion.ToString(CultureInfo.InvariantCulture), + versionCmd.ExecuteScalar() as string); + } + finally + { + DeleteDirectory(projectRoot); + } + } + [Fact] public void Run_FullScan_ReindexesNormalizedCSharpFieldsWhenExtractorContractChanges_Issue4865() { diff --git a/tests/CodeIndex.Tests/SymbolExtractorTests.cs b/tests/CodeIndex.Tests/SymbolExtractorTests.cs index aa6acd094..73a61a3e8 100644 --- a/tests/CodeIndex.Tests/SymbolExtractorTests.cs +++ b/tests/CodeIndex.Tests/SymbolExtractorTests.cs @@ -1060,6 +1060,8 @@ public void Extract_Yaml_SequenceItemsUsePathOnlyIdentityAndNearestSymbolParent_ { const string content = """ - &first + env: + FOO: bar name: Build with: path: | @@ -1067,6 +1069,8 @@ public void Extract_Yaml_SequenceItemsUsePathOnlyIdentityAndNearestSymbolParent_ - *first - - null + - | + phantom: text - name: Final nested: value: yes @@ -1077,13 +1081,18 @@ public void Extract_Yaml_SequenceItemsUsePathOnlyIdentityAndNearestSymbolParent_ var symbols = SymbolExtractor.Extract(1, "yaml", content); - Assert.Equal(9, symbols.Count); - Assert.DoesNotContain(symbols, symbol => symbol.Name is "[0]" or "[1]" or "[2]" or "[3]" or "[4]"); + Assert.Equal(11, symbols.Count); + Assert.DoesNotContain(symbols, symbol => symbol.Name is "[0]" or "[1]" or "[2]" or "[3]" or "[4]" or "[5]"); var firstName = Assert.Single(symbols, symbol => symbol.Name == "[0].name"); Assert.Null(firstName.ContainerName); Assert.Equal("[0]", firstName.ContainerQualifiedName); + Assert.Contains(symbols, symbol => + symbol.Name == "[0].env.FOO" + && symbol.ContainerName == "[0].env" + && symbol.ContainerQualifiedName == "[0].env"); + var firstWith = Assert.Single(symbols, symbol => symbol.Name == "[0].with"); Assert.Equal("namespace", firstWith.Kind); Assert.Null(firstWith.ContainerName); @@ -1092,17 +1101,17 @@ public void Extract_Yaml_SequenceItemsUsePathOnlyIdentityAndNearestSymbolParent_ var firstPath = Assert.Single(symbols, symbol => symbol.Name == "[0].with.path"); Assert.Equal("[0].with", firstPath.ContainerName); Assert.Equal("[0].with", firstPath.ContainerQualifiedName); - Assert.Equal(4, firstPath.Line); - Assert.Equal(4, firstPath.StartLine); - Assert.Equal(4, firstPath.EndLine); + Assert.Equal(6, firstPath.Line); + Assert.Equal(6, firstPath.StartLine); + Assert.Equal(6, firstPath.EndLine); - var finalName = Assert.Single(symbols, symbol => symbol.Name == "[4].name"); + var finalName = Assert.Single(symbols, symbol => symbol.Name == "[5].name"); Assert.Null(finalName.ContainerName); - Assert.Equal("[4]", finalName.ContainerQualifiedName); + Assert.Equal("[5]", finalName.ContainerQualifiedName); Assert.Contains(symbols, symbol => - symbol.Name == "[4].nested.value" - && symbol.ContainerName == "[4].nested" - && symbol.ContainerQualifiedName == "[4].nested"); + symbol.Name == "[5].nested.value" + && symbol.ContainerName == "[5].nested" + && symbol.ContainerQualifiedName == "[5].nested"); Assert.Contains(symbols, symbol => symbol.Kind == "namespace" && symbol.Name == "defaults"); Assert.Contains(symbols, symbol => @@ -1111,6 +1120,7 @@ public void Extract_Yaml_SequenceItemsUsePathOnlyIdentityAndNearestSymbolParent_ && symbol.ContainerQualifiedName == "defaults"); Assert.Contains(symbols, symbol => symbol.Kind == "property" && symbol.Name == "copy"); Assert.DoesNotContain(symbols, symbol => symbol.Name.Contains("ignored", StringComparison.Ordinal)); + Assert.DoesNotContain(symbols, symbol => symbol.Name.Contains("phantom", StringComparison.Ordinal)); } [Fact] @@ -12845,6 +12855,8 @@ public void GetContractVersion_LanguageSpecificKinds_UseDedicatedVersions() Assert.True(SymbolExtractor.MakefileContractVersion > SymbolExtractor.DefaultContractVersion); Assert.Equal(SymbolExtractor.DependencyLockContractVersion, SymbolExtractor.GetContractVersion("dependency_lock")); Assert.True(SymbolExtractor.DependencyLockContractVersion > SymbolExtractor.ExpandedLanguageContractVersion); + Assert.Equal(SymbolExtractor.YamlContractVersion, SymbolExtractor.GetContractVersion("yaml")); + Assert.True(SymbolExtractor.YamlContractVersion > SymbolExtractor.ExpandedLanguageContractVersion); Assert.Equal(SymbolExtractor.StyleAndXamlContractVersion, SymbolExtractor.GetContractVersion("sass")); Assert.Equal(SymbolExtractor.StyleAndXamlContractVersion, SymbolExtractor.GetContractVersion("stylus")); Assert.True(SymbolExtractor.StyleAndXamlContractVersion > SymbolExtractor.DefaultContractVersion);