From 7f19cea626558122a8de30faa836c41966b45397 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Tue, 2 Jun 2026 15:36:43 +0900 Subject: [PATCH 1/2] Expose literal exact-substring highlights (#2820) --- DEVELOPER_GUIDE.md | 4 +-- USER_GUIDE.md | 8 +++++ changelog.d/unreleased/2820.fixed.md | 22 ++++++++++++ src/CodeIndex/Cli/QueryCommandRunner.cs | 2 +- src/CodeIndex/Cli/SearchSnippetFormatter.cs | 24 +++++++++---- src/CodeIndex/Mcp/McpToolHandlers.cs | 2 +- tests/CodeIndex.Tests/McpServerTests.cs | 17 +++++++++ .../QueryCommandRunnerTests.cs | 35 +++++++++++++++++++ .../SearchSnippetFormatterTests.cs | 23 ++++++++++++ 9 files changed, 127 insertions(+), 10 deletions(-) create mode 100644 changelog.d/unreleased/2820.fixed.md diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index a20e189ff8..124df684d2 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -1007,7 +1007,7 @@ Exact-match flag compatibility is documented in [USER_GUIDE.md](USER_GUIDE.md#fl `search`, `definition`, `references`, `callers`, `callees`, `symbols`, and `files` also share path-aware narrowing via `--path`, repeatable `--exclude-path`, and `--exclude-tests`. The read layer ranks source files ahead of tests and docs, and `search` further boosts exact symbol-name and path matches so AI clients are more likely to land on implementation files first. -`search --json` and MCP `search` project full chunks into compact match-centered snippets with `chunk_start_line`, `chunk_end_line`, `snippet_start_line`, `snippet_end_line`, `snippet`, `match_lines`, `highlights`, `context_before`, `context_after`, `truncated_line_count`, `dropped_match_line_count`, and `truncation_context`. `--snippet-lines` caps the snippet length up front (default: 8, max: 20), and `--max-line-width` (CLI) / `maxLineWidth` (MCP) clamps each individual snippet line around the first match token via the shared `LineWidthFormatter.ClampLine` contract used by `find` / `references` / `excerpt` / `inspect` (default: 512, max: 4096) so a single match inside a minified / transpiled / generated single-line file no longer returns hundreds of KB per hit. Clamped lines surface `...(+N)...` markers inside the snippet and expose `truncation_context.char_counts`, `truncation_context.total_chars`, `highlights[].truncated`, `highlights[].original_line_length`, and `highlights[].truncated_char_counts` so AI clients can detect clamping and quantify omitted characters. `highlights[].terms` remains a distinct term list for compatibility; `highlights[].term_occurrences` records every matched occurrence with `term`, 1-based `line`, 1-based `column`, and `length`. `dropped_match_line_count` reports match lines omitted because they fell outside the selected snippet window. +`search --json` and MCP `search` project full chunks into compact match-centered snippets with `chunk_start_line`, `chunk_end_line`, `snippet_start_line`, `snippet_end_line`, `snippet`, `match_lines`, `highlights`, `context_before`, `context_after`, `truncated_line_count`, `dropped_match_line_count`, and `truncation_context`. `--snippet-lines` caps the snippet length up front (default: 8, max: 20), and `--max-line-width` (CLI) / `maxLineWidth` (MCP) clamps each individual snippet line around the first match token via the shared `LineWidthFormatter.ClampLine` contract used by `find` / `references` / `excerpt` / `inspect` (default: 512, max: 4096) so a single match inside a minified / transpiled / generated single-line file no longer returns hundreds of KB per hit. Clamped lines surface `...(+N)...` markers inside the snippet and expose `truncation_context.char_counts`, `truncation_context.total_chars`, `highlights[].truncated`, `highlights[].original_line_length`, and `highlights[].truncated_char_counts` so AI clients can detect clamping and quantify omitted characters. `highlights[].terms` remains a distinct term list for compatibility; `highlights[].term_occurrences` records every matched occurrence with `term`, 1-based `line`, 1-based `column`, and `length`. Exact substring search also adds `highlights[].literal_terms` and `highlights[].literal_term_occurrences` (camelCase in MCP) so clients can render only the requested literal phrase while preserving the broader diagnostic token list. `dropped_match_line_count` reports match lines omitted because they fell outside the selected snippet window. `excerpt --json` includes `semantic_tokens`, a lightweight range list with 1-based start/end positions, token `type`, and `modifiers`, so IDE and LLM clients can render or post-process excerpt spans without reparsing the raw `content` string. @@ -2619,7 +2619,7 @@ exact-match flag の互換性は [USER_GUIDE.md](USER_GUIDE.md#フラグ互換 `search`、`definition`、`references`、`callers`、`callees`、`symbols`、`files` は `--path`、繰り返し指定できる `--exclude-path`、`--exclude-tests` による絞り込みを共有します。読み取り層は tests や docs より source を優先し、`search` はシンボル名やパスがクエリと正確に一致する候補をさらに上位に出して、AIクライアントが実装ファイルへ早く到達できるようにします。 -`search --json` と MCP の `search` は、フルチャンクを `chunk_start_line`、`chunk_end_line`、`snippet_start_line`、`snippet_end_line`、`snippet`、`match_lines`、`highlights`、`context_before`、`context_after`、`truncated_line_count`、`dropped_match_line_count`、`truncation_context` を持つ軽量スニペットへ投影します。`--snippet-lines` で抜粋長を先に制限でき（デフォルト: 8、最大: 20）、`--max-line-width`（CLI）/ `maxLineWidth`（MCP）は `find` / `references` / `excerpt` / `inspect` と同じ共有 `LineWidthFormatter.ClampLine` 契約（デフォルト: 512、最大: 4096、`0` で切り詰め解除）で各スニペット行を最初のマッチトークン周辺にクランプするため、minified / transpiled / 生成された 1 行ファイル内の 1 ヒットで数百 KB を返さなくなります。クランプされた行はスニペットに `...(+N)...` マーカーが入り、`truncation_context.char_counts`、`truncation_context.total_chars`、`highlights[].truncated`、`highlights[].original_line_length`、`highlights[].truncated_char_counts` で AI クライアントがクランプの有無と省略文字数を検出できます。`highlights[].terms` は互換性のため distinct な term list のまま残し、`highlights[].term_occurrences` は一致ごとの `term`、1-based の `line` / `column`、`length` を記録します。`dropped_match_line_count` は選択された snippet window 外に落ちた一致行数を示します。 +`search --json` と MCP の `search` は、フルチャンクを `chunk_start_line`、`chunk_end_line`、`snippet_start_line`、`snippet_end_line`、`snippet`、`match_lines`、`highlights`、`context_before`、`context_after`、`truncated_line_count`、`dropped_match_line_count`、`truncation_context` を持つ軽量スニペットへ投影します。`--snippet-lines` で抜粋長を先に制限でき（デフォルト: 8、最大: 20）、`--max-line-width`（CLI）/ `maxLineWidth`（MCP）は `find` / `references` / `excerpt` / `inspect` と同じ共有 `LineWidthFormatter.ClampLine` 契約（デフォルト: 512、最大: 4096、`0` で切り詰め解除）で各スニペット行を最初のマッチトークン周辺にクランプするため、minified / transpiled / 生成された 1 行ファイル内の 1 ヒットで数百 KB を返さなくなります。クランプされた行はスニペットに `...(+N)...` マーカーが入り、`truncation_context.char_counts`、`truncation_context.total_chars`、`highlights[].truncated`、`highlights[].original_line_length`、`highlights[].truncated_char_counts` で AI クライアントがクランプの有無と省略文字数を検出できます。`highlights[].terms` は互換性のため distinct な term list のまま残し、`highlights[].term_occurrences` は一致ごとの `term`、1-based の `line` / `column`、`length` を記録します。exact substring search では `highlights[].literal_terms` と `highlights[].literal_term_occurrences`（MCP では camelCase）も追加され、広めの診断 token list を残したまま、要求された literal phrase だけを render できます。`dropped_match_line_count` は選択された snippet window 外に落ちた一致行数を示します。 `excerpt --json` は 1-based の開始/終了位置、token `type`、`modifiers` を持つ軽量 range list の `semantic_tokens` を返すため、IDE や LLM クライアントは生の `content` 文字列を再パースせずに抜粋範囲を描画・後処理できます。 diff --git a/USER_GUIDE.md b/USER_GUIDE.md index 4c8730e4a6..92d7b03df0 100644 --- a/USER_GUIDE.md +++ b/USER_GUIDE.md @@ -358,6 +358,10 @@ not scheduled for removal before the next major release. If removal is planned, the release notes will announce the timeline before the alias stops working. MCP mirrors the same split: use `exactSubstring` on `search`, `exactName` on name-based tools, and keep `exact` only for backward-compatible clients. +In `search --json` and MCP `search` responses, exact substring highlights add +`literal_terms` / `literal_term_occurrences` (camelCase in MCP) so clients can +render only the requested literal phrase while keeping the broader diagnostic +`terms` / `term_occurrences` fields. For single-value CLI flags, repeated occurrences keep the long-standing rightmost-wins behavior. `cdidx` prints a warning that names the winning value: @@ -2452,6 +2456,10 @@ name に対する NFKC + Unicode CaseFold の等価比較です。 削除する予定はありません。削除する場合は、alias が使えなくなる前に release notes で timeline を告知します。MCP も同じ分割を反映します。`search` では `exactSubstring`、 name-based tools では `exactName` を使い、`exact` は後方互換 client 向けに残します。 +`search --json` と MCP `search` の exact substring highlight には +`literal_terms` / `literal_term_occurrences`（MCP では camelCase）も追加されるため、 +広めの診断用 `terms` / `term_occurrences` を残したまま、要求した literal phrase だけを +render できます。 単一値の CLI フラグを複数回指定した場合は、従来どおり右端の指定が採用されます。 `cdidx` は採用される値を含む警告を出し、最後の CLI 指定がそれ以前の CLI 指定や diff --git a/changelog.d/unreleased/2820.fixed.md b/changelog.d/unreleased/2820.fixed.md new file mode 100644 index 0000000000..4a0c7871b8 --- /dev/null +++ b/changelog.d/unreleased/2820.fixed.md @@ -0,0 +1,22 @@ +--- +category: fixed +issues: + - 2820 +affected: + - src/CodeIndex/Cli/SearchSnippetFormatter.cs + - src/CodeIndex/Cli/QueryCommandRunner.cs + - src/CodeIndex/Mcp/McpToolHandlers.cs + - tests/CodeIndex.Tests/SearchSnippetFormatterTests.cs + - tests/CodeIndex.Tests/QueryCommandRunnerTests.cs + - tests/CodeIndex.Tests/McpServerTests.cs + - DEVELOPER_GUIDE.md + - USER_GUIDE.md +--- + +## English + +- **Exact substring highlights expose literal-only terms (#2820)** — `search --json` and MCP `search` now add `literal_terms` / `literal_term_occurrences` for exact substring results so clients can render the requested phrase without also highlighting tokenized punctuation fragments. + +## 日本語 + +- **exact substring highlight が literal-only terms を公開するようになりました (#2820)** — `search --json` と MCP `search` は exact substring 結果に `literal_terms` / `literal_term_occurrences` を追加し、tokenize された記号片まで highlight せずに要求された phrase だけを render できるようになりました。 diff --git a/src/CodeIndex/Cli/QueryCommandRunner.cs b/src/CodeIndex/Cli/QueryCommandRunner.cs index 87e3e5219c..deca452de9 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.cs @@ -488,7 +488,7 @@ public static int RunSearch(string[] cmdArgs, JsonSerializerOptions jsonOptions) return CommandExitCodes.Success; } var compactResults = results - .Select(r => SearchSnippetFormatter.ToCompactResult(r, options.Query, options.SnippetLines, exact, options.MaxLineWidth, r.Lang, options.SnippetFocus)) + .Select(r => SearchSnippetFormatter.ToCompactResult(r, options.Query, options.SnippetLines, exact, options.MaxLineWidth, r.Lang, options.SnippetFocus, exposeLiteralHighlights: exact)) .ToArray(); if (options.JsonOutputFormat == JsonOutputFormatArray) { diff --git a/src/CodeIndex/Cli/SearchSnippetFormatter.cs b/src/CodeIndex/Cli/SearchSnippetFormatter.cs index b1b597bc67..4af3e8dfe2 100644 --- a/src/CodeIndex/Cli/SearchSnippetFormatter.cs +++ b/src/CodeIndex/Cli/SearchSnippetFormatter.cs @@ -30,9 +30,9 @@ public static IReadOnlyList Format(string content, string query, int max return snippet; } - public static CompactSearchResult ToCompactResult(SearchResult result, string query, int maxLines = DefaultSnippetLines, bool caseSensitive = false, int maxLineWidth = LineWidthFormatter.DefaultMaxLineWidth, string? lang = null, SearchSnippetFocusMode focusMode = SearchSnippetFocusMode.Quality) + public static CompactSearchResult ToCompactResult(SearchResult result, string query, int maxLines = DefaultSnippetLines, bool caseSensitive = false, int maxLineWidth = LineWidthFormatter.DefaultMaxLineWidth, string? lang = null, SearchSnippetFocusMode focusMode = SearchSnippetFocusMode.Quality, bool exposeLiteralHighlights = false) { - var excerpt = BuildExcerpt(result.Content, query, result.StartLine, maxLines, caseSensitive, maxLineWidth, lang ?? result.Lang, focusMode); + var excerpt = BuildExcerpt(result.Content, query, result.StartLine, maxLines, caseSensitive, maxLineWidth, lang ?? result.Lang, focusMode, exposeLiteralHighlights); return new CompactSearchResult { Query = query, @@ -55,13 +55,13 @@ public static CompactSearchResult ToCompactResult(SearchResult result, string qu }; } - public static IEnumerable ToCompactResults(IEnumerable results, string query, int maxLines = DefaultSnippetLines, bool caseSensitive = false, int maxLineWidth = LineWidthFormatter.DefaultMaxLineWidth, string? lang = null, SearchSnippetFocusMode focusMode = SearchSnippetFocusMode.Quality) + public static IEnumerable ToCompactResults(IEnumerable results, string query, int maxLines = DefaultSnippetLines, bool caseSensitive = false, int maxLineWidth = LineWidthFormatter.DefaultMaxLineWidth, string? lang = null, SearchSnippetFocusMode focusMode = SearchSnippetFocusMode.Quality, bool exposeLiteralHighlights = false) { foreach (var result in results) - yield return ToCompactResult(result, query, maxLines, caseSensitive, maxLineWidth, lang ?? result.Lang, focusMode); + yield return ToCompactResult(result, query, maxLines, caseSensitive, maxLineWidth, lang ?? result.Lang, focusMode, exposeLiteralHighlights); } - public static SearchSnippetExcerpt BuildExcerpt(string content, string query, int absoluteStartLine, int maxLines = DefaultSnippetLines, bool caseSensitive = false, int maxLineWidth = LineWidthFormatter.DefaultMaxLineWidth, string? lang = null, SearchSnippetFocusMode focusMode = SearchSnippetFocusMode.Quality) + public static SearchSnippetExcerpt BuildExcerpt(string content, string query, int absoluteStartLine, int maxLines = DefaultSnippetLines, bool caseSensitive = false, int maxLineWidth = LineWidthFormatter.DefaultMaxLineWidth, string? lang = null, SearchSnippetFocusMode focusMode = SearchSnippetFocusMode.Quality, bool exposeLiteralHighlights = false) { maxLines = ClampSnippetLines(maxLines); maxLineWidth = LineWidthFormatter.ClampMaxLineWidth(maxLineWidth); @@ -170,9 +170,15 @@ public static SearchSnippetExcerpt BuildExcerpt(string content, string query, in var absoluteLine = absoluteStartLine + i; matchLines.Add(absoluteLine); + var matchLineForTerms = normalizeCSharpVerbatimNames && normalizedLines != null ? normalizedLines[i] : originalLine; var termOccurrences = normalizeCSharpVerbatimNames && normalizedLines != null && rawIndexMaps != null ? GetMatchedTermOccurrences(normalizedLines[i], absoluteLine, normalizedQuery, tokens, caseSensitive, originalLine, rawIndexMaps[i]) : GetMatchedTermOccurrences(originalLine, absoluteLine, normalizedQuery, tokens, caseSensitive); + var literalTermOccurrences = exposeLiteralHighlights + ? normalizeCSharpVerbatimNames && normalizedLines != null && rawIndexMaps != null + ? GetMatchedTermOccurrences(normalizedLines[i], absoluteLine, normalizedQuery, [], caseSensitive, originalLine, rawIndexMaps[i]) + : GetMatchedTermOccurrences(originalLine, absoluteLine, normalizedQuery, [], caseSensitive) + : null; highlights.Add(new SearchHighlight { Line = absoluteLine, @@ -180,8 +186,10 @@ public static SearchSnippetExcerpt BuildExcerpt(string content, string query, in OriginalLineLength = originalLine.Length, Truncated = clamped.Truncated, TruncatedCharCounts = clamped.Truncated ? [clamped.TruncatedCharCount] : [], - Terms = GetMatchedTerms(normalizeCSharpVerbatimNames && normalizedLines != null ? normalizedLines[i] : originalLine, normalizedQuery, tokens, caseSensitive), + Terms = GetMatchedTerms(matchLineForTerms, normalizedQuery, tokens, caseSensitive), TermOccurrences = termOccurrences, + LiteralTerms = exposeLiteralHighlights ? GetMatchedTerms(matchLineForTerms, normalizedQuery, [], caseSensitive) : null, + LiteralTermOccurrences = literalTermOccurrences, }); } @@ -528,6 +536,10 @@ public sealed class SearchHighlight public List TruncatedCharCounts { get; set; } = []; public List Terms { get; set; } = []; public List TermOccurrences { get; set; } = []; + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public List? LiteralTerms { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public List? LiteralTermOccurrences { get; set; } } public sealed class SearchTermOccurrence diff --git a/src/CodeIndex/Mcp/McpToolHandlers.cs b/src/CodeIndex/Mcp/McpToolHandlers.cs index dbf4f4cabb..b7f5133604 100644 --- a/src/CodeIndex/Mcp/McpToolHandlers.cs +++ b/src/CodeIndex/Mcp/McpToolHandlers.cs @@ -1039,7 +1039,7 @@ private JsonNode ExecuteSearch(JsonNode? id, JsonNode? args) ["maxLineWidth"] = maxLineWidth, ["path"] = PathEcho(pathPatterns), ["excludeTests"] = excludeTests, - ["results"] = ToJsonArray(SearchSnippetFormatter.ToCompactResults(results, query, snippetLines, exact, maxLineWidth)) + ["results"] = ToJsonArray(SearchSnippetFormatter.ToCompactResults(results, query, snippetLines, exact, maxLineWidth, exposeLiteralHighlights: exact)) }; AddSearchStabilityMetadata(structured, reader, cursor, results); AddResultEnvelope(structured, results.Count, truncated ? null : results.Count, truncated); diff --git a/tests/CodeIndex.Tests/McpServerTests.cs b/tests/CodeIndex.Tests/McpServerTests.cs index af8243bf6d..50f10da7b0 100644 --- a/tests/CodeIndex.Tests/McpServerTests.cs +++ b/tests/CodeIndex.Tests/McpServerTests.cs @@ -3080,6 +3080,23 @@ public void ToolsCall_Search_ReturnsResults() Assert.Null(structured["results"]![0]!["content"]); } + [Fact] + public void ToolsCall_Search_ExactSubstringReturnsLiteralHighlightMetadata() + { + InsertIndexedFile("src/sql.cs", "csharp", "var CommandText = $\"SELECT 1\";\nvar CommandText = other;\n"); + + var request = JsonNode.Parse("""{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"search","arguments":{"query":"CommandText = $","exactSubstring":true}}}""")!; + var response = _server.HandleMessage(request)!; + + var highlight = response["result"]!["structuredContent"]!["results"]![0]!["highlights"]![0]!; + var literalOccurrence = highlight["literalTermOccurrences"]![0]!; + Assert.Equal("CommandText = $", highlight["literalTerms"]![0]!.GetValue()); + Assert.Equal("CommandText = $", literalOccurrence["term"]!.GetValue()); + Assert.Equal(1, literalOccurrence["line"]!.GetValue()); + Assert.Equal(5, literalOccurrence["column"]!.GetValue()); + Assert.Equal("CommandText = $".Length, literalOccurrence["length"]!.GetValue()); + } + [Fact] public void ToolsCall_Search_ExcludesGeneratedFilesByDefault() { diff --git a/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs b/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs index 3e4afb92d6..1d72a926ee 100644 --- a/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs +++ b/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs @@ -10925,6 +10925,41 @@ public void RunSearch_WithJsonOutputsCompactSnippetMetadata() } } + [Fact] + public void RunSearch_ExactSubstringJsonOutputsLiteralHighlightMetadata() + { + var projectRoot = TestProjectHelper.CreateTempProject("cdidx_query_runner_search_exact_literal_highlight"); + try + { + var dbPath = TestProjectHelper.CreateProjectDb(projectRoot); + TestProjectHelper.InsertIndexedFile( + dbPath, + "src/sql.cs", + "csharp", + "var CommandText = $\"SELECT 1\";\nvar CommandText = other;"); + + var (exitCode, stdout, stderr) = CaptureConsole(() => QueryCommandRunner.RunSearch( + ["CommandText = $", "--db", dbPath, "--json", "--exact-substring", "--snippet-lines", "2"], + _jsonOptions)); + + using var document = ParseJsonOutput(stdout); + var highlight = document.RootElement.GetProperty("highlights")[0]; + var literalOccurrence = highlight.GetProperty("literal_term_occurrences")[0]; + + Assert.Equal(CommandExitCodes.Success, exitCode); + Assert.Equal(string.Empty, stderr); + Assert.Equal("CommandText = $", highlight.GetProperty("literal_terms")[0].GetString()); + Assert.Equal("CommandText = $", literalOccurrence.GetProperty("term").GetString()); + Assert.Equal(1, literalOccurrence.GetProperty("line").GetInt32()); + Assert.Equal(5, literalOccurrence.GetProperty("column").GetInt32()); + Assert.Equal("CommandText = $".Length, literalOccurrence.GetProperty("length").GetInt32()); + } + finally + { + TestProjectHelper.DeleteDirectory(projectRoot); + } + } + [Fact] public void RunSearch_ExactSubstringAliasMatchesBackwardCompatibleExact() { diff --git a/tests/CodeIndex.Tests/SearchSnippetFormatterTests.cs b/tests/CodeIndex.Tests/SearchSnippetFormatterTests.cs index 121b23b861..6e8c7f80fa 100644 --- a/tests/CodeIndex.Tests/SearchSnippetFormatterTests.cs +++ b/tests/CodeIndex.Tests/SearchSnippetFormatterTests.cs @@ -63,6 +63,29 @@ public void BuildExcerpt_ReportsEveryTermOccurrenceWithPositions() }); } + [Fact] + public void BuildExcerpt_ExactSubstringExposesLiteralOnlyHighlights() + { + const string content = "CommandText = $\"SELECT\";\nCommandText only"; + + var excerpt = SearchSnippetFormatter.BuildExcerpt( + content, + "CommandText = $", + absoluteStartLine: 1, + maxLines: 1, + caseSensitive: true, + exposeLiteralHighlights: true); + + var highlight = Assert.Single(excerpt.Highlights); + Assert.Equal(["CommandText = $", "CommandText", "=", "$"], highlight.Terms); + Assert.Equal(["CommandText = $"], highlight.LiteralTerms); + var literalOccurrence = Assert.Single(highlight.LiteralTermOccurrences!); + Assert.Equal("CommandText = $", literalOccurrence.Term); + Assert.Equal(1, literalOccurrence.Line); + Assert.Equal(1, literalOccurrence.Column); + Assert.Equal("CommandText = $".Length, literalOccurrence.Length); + } + [Fact] public void ToCompactResult_UsesSnippetInsteadOfFullChunkContent() { From f8db3462edde77653b49da13b1838052508f6713 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Tue, 2 Jun 2026 16:16:21 +0900 Subject: [PATCH 2/2] Suggest exact substring for code phrase search (#2813) --- DEVELOPER_GUIDE.md | 4 +- USER_GUIDE.md | 7 ++ changelog.d/unreleased/2813.fixed.md | 23 ++++++ src/CodeIndex/Cli/JsonOutputContracts.cs | 1 + src/CodeIndex/Cli/QueryCommandRunner.cs | 53 +++++++++++-- src/CodeIndex/Cli/SearchQueryAdvisor.cs | 75 +++++++++++++++++++ src/CodeIndex/Cli/SearchSnippetFormatter.cs | 2 + src/CodeIndex/Mcp/McpToolHandlers.cs | 37 +++++++-- tests/CodeIndex.Tests/McpServerTests.cs | 15 ++++ .../QueryCommandRunnerTests.cs | 59 +++++++++++++++ 10 files changed, 261 insertions(+), 15 deletions(-) create mode 100644 changelog.d/unreleased/2813.fixed.md create mode 100644 src/CodeIndex/Cli/SearchQueryAdvisor.cs diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 124df684d2..c359d8133b 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -1007,7 +1007,7 @@ Exact-match flag compatibility is documented in [USER_GUIDE.md](USER_GUIDE.md#fl `search`, `definition`, `references`, `callers`, `callees`, `symbols`, and `files` also share path-aware narrowing via `--path`, repeatable `--exclude-path`, and `--exclude-tests`. The read layer ranks source files ahead of tests and docs, and `search` further boosts exact symbol-name and path matches so AI clients are more likely to land on implementation files first. -`search --json` and MCP `search` project full chunks into compact match-centered snippets with `chunk_start_line`, `chunk_end_line`, `snippet_start_line`, `snippet_end_line`, `snippet`, `match_lines`, `highlights`, `context_before`, `context_after`, `truncated_line_count`, `dropped_match_line_count`, and `truncation_context`. `--snippet-lines` caps the snippet length up front (default: 8, max: 20), and `--max-line-width` (CLI) / `maxLineWidth` (MCP) clamps each individual snippet line around the first match token via the shared `LineWidthFormatter.ClampLine` contract used by `find` / `references` / `excerpt` / `inspect` (default: 512, max: 4096) so a single match inside a minified / transpiled / generated single-line file no longer returns hundreds of KB per hit. Clamped lines surface `...(+N)...` markers inside the snippet and expose `truncation_context.char_counts`, `truncation_context.total_chars`, `highlights[].truncated`, `highlights[].original_line_length`, and `highlights[].truncated_char_counts` so AI clients can detect clamping and quantify omitted characters. `highlights[].terms` remains a distinct term list for compatibility; `highlights[].term_occurrences` records every matched occurrence with `term`, 1-based `line`, 1-based `column`, and `length`. Exact substring search also adds `highlights[].literal_terms` and `highlights[].literal_term_occurrences` (camelCase in MCP) so clients can render only the requested literal phrase while preserving the broader diagnostic token list. `dropped_match_line_count` reports match lines omitted because they fell outside the selected snippet window. +`search --json` and MCP `search` project full chunks into compact match-centered snippets with `chunk_start_line`, `chunk_end_line`, `snippet_start_line`, `snippet_end_line`, `snippet`, `match_lines`, `highlights`, `context_before`, `context_after`, `truncated_line_count`, `dropped_match_line_count`, and `truncation_context`. `--snippet-lines` caps the snippet length up front (default: 8, max: 20), and `--max-line-width` (CLI) / `maxLineWidth` (MCP) clamps each individual snippet line around the first match token via the shared `LineWidthFormatter.ClampLine` contract used by `find` / `references` / `excerpt` / `inspect` (default: 512, max: 4096) so a single match inside a minified / transpiled / generated single-line file no longer returns hundreds of KB per hit. Clamped lines surface `...(+N)...` markers inside the snippet and expose `truncation_context.char_counts`, `truncation_context.total_chars`, `highlights[].truncated`, `highlights[].original_line_length`, and `highlights[].truncated_char_counts` so AI clients can detect clamping and quantify omitted characters. `highlights[].terms` remains a distinct term list for compatibility; `highlights[].term_occurrences` records every matched occurrence with `term`, 1-based `line`, 1-based `column`, and `length`. Exact substring search also adds `highlights[].literal_terms` and `highlights[].literal_term_occurrences` (camelCase in MCP) so clients can render only the requested literal phrase while preserving the broader diagnostic token list. Non-exact punctuation-heavy code-phrase searches add `exact_substring_hint` to CLI JSON compact results and `recovery_hint` to MCP `search` responses so clients can retry with exact substring semantics when FTS tokenization is likely to hide punctuation. `dropped_match_line_count` reports match lines omitted because they fell outside the selected snippet window. `excerpt --json` includes `semantic_tokens`, a lightweight range list with 1-based start/end positions, token `type`, and `modifiers`, so IDE and LLM clients can render or post-process excerpt spans without reparsing the raw `content` string. @@ -2619,7 +2619,7 @@ exact-match flag の互換性は [USER_GUIDE.md](USER_GUIDE.md#フラグ互換 `search`、`definition`、`references`、`callers`、`callees`、`symbols`、`files` は `--path`、繰り返し指定できる `--exclude-path`、`--exclude-tests` による絞り込みを共有します。読み取り層は tests や docs より source を優先し、`search` はシンボル名やパスがクエリと正確に一致する候補をさらに上位に出して、AIクライアントが実装ファイルへ早く到達できるようにします。 -`search --json` と MCP の `search` は、フルチャンクを `chunk_start_line`、`chunk_end_line`、`snippet_start_line`、`snippet_end_line`、`snippet`、`match_lines`、`highlights`、`context_before`、`context_after`、`truncated_line_count`、`dropped_match_line_count`、`truncation_context` を持つ軽量スニペットへ投影します。`--snippet-lines` で抜粋長を先に制限でき（デフォルト: 8、最大: 20）、`--max-line-width`（CLI）/ `maxLineWidth`（MCP）は `find` / `references` / `excerpt` / `inspect` と同じ共有 `LineWidthFormatter.ClampLine` 契約（デフォルト: 512、最大: 4096、`0` で切り詰め解除）で各スニペット行を最初のマッチトークン周辺にクランプするため、minified / transpiled / 生成された 1 行ファイル内の 1 ヒットで数百 KB を返さなくなります。クランプされた行はスニペットに `...(+N)...` マーカーが入り、`truncation_context.char_counts`、`truncation_context.total_chars`、`highlights[].truncated`、`highlights[].original_line_length`、`highlights[].truncated_char_counts` で AI クライアントがクランプの有無と省略文字数を検出できます。`highlights[].terms` は互換性のため distinct な term list のまま残し、`highlights[].term_occurrences` は一致ごとの `term`、1-based の `line` / `column`、`length` を記録します。exact substring search では `highlights[].literal_terms` と `highlights[].literal_term_occurrences`（MCP では camelCase）も追加され、広めの診断 token list を残したまま、要求された literal phrase だけを render できます。`dropped_match_line_count` は選択された snippet window 外に落ちた一致行数を示します。 +`search --json` と MCP の `search` は、フルチャンクを `chunk_start_line`、`chunk_end_line`、`snippet_start_line`、`snippet_end_line`、`snippet`、`match_lines`、`highlights`、`context_before`、`context_after`、`truncated_line_count`、`dropped_match_line_count`、`truncation_context` を持つ軽量スニペットへ投影します。`--snippet-lines` で抜粋長を先に制限でき（デフォルト: 8、最大: 20）、`--max-line-width`（CLI）/ `maxLineWidth`（MCP）は `find` / `references` / `excerpt` / `inspect` と同じ共有 `LineWidthFormatter.ClampLine` 契約（デフォルト: 512、最大: 4096、`0` で切り詰め解除）で各スニペット行を最初のマッチトークン周辺にクランプするため、minified / transpiled / 生成された 1 行ファイル内の 1 ヒットで数百 KB を返さなくなります。クランプされた行はスニペットに `...(+N)...` マーカーが入り、`truncation_context.char_counts`、`truncation_context.total_chars`、`highlights[].truncated`、`highlights[].original_line_length`、`highlights[].truncated_char_counts` で AI クライアントがクランプの有無と省略文字数を検出できます。`highlights[].terms` は互換性のため distinct な term list のまま残し、`highlights[].term_occurrences` は一致ごとの `term`、1-based の `line` / `column`、`length` を記録します。exact substring search では `highlights[].literal_terms` と `highlights[].literal_term_occurrences`（MCP では camelCase）も追加され、広めの診断 token list を残したまま、要求された literal phrase だけを render できます。exact ではない記号の多い code phrase 検索では、FTS tokenization が記号を失いやすい場合に exact substring semantics で再検索できるよう、CLI JSON compact result に `exact_substring_hint`、MCP `search` に `recovery_hint` を追加します。`dropped_match_line_count` は選択された snippet window 外に落ちた一致行数を示します。 `excerpt --json` は 1-based の開始/終了位置、token `type`、`modifiers` を持つ軽量 range list の `semantic_tokens` を返すため、IDE や LLM クライアントは生の `content` 文字列を再パースせずに抜粋範囲を描画・後処理できます。 diff --git a/USER_GUIDE.md b/USER_GUIDE.md index 92d7b03df0..64e11fba9f 100644 --- a/USER_GUIDE.md +++ b/USER_GUIDE.md @@ -362,6 +362,10 @@ In `search --json` and MCP `search` responses, exact substring highlights add `literal_terms` / `literal_term_occurrences` (camelCase in MCP) so clients can render only the requested literal phrase while keeping the broader diagnostic `terms` / `term_occurrences` fields. +When a normal `search` query looks like a punctuation-heavy code phrase, +text output suggests `--exact-substring`, JSON results include +`exact_substring_hint`, and MCP `search` includes a `recovery_hint` with +`exactSubstring: true` arguments. For single-value CLI flags, repeated occurrences keep the long-standing rightmost-wins behavior. `cdidx` prints a warning that names the winning value: @@ -2460,6 +2464,9 @@ name-based tools では `exactName` を使い、`exact` は後方互換 client `literal_terms` / `literal_term_occurrences`（MCP では camelCase）も追加されるため、 広めの診断用 `terms` / `term_occurrences` を残したまま、要求した literal phrase だけを render できます。 +通常の `search` query が記号の多い code phrase に見える場合、text output は +`--exact-substring` を提案し、JSON 結果は `exact_substring_hint`、MCP `search` は +`exactSubstring: true` arguments 付きの `recovery_hint` を返します。 単一値の CLI フラグを複数回指定した場合は、従来どおり右端の指定が採用されます。 `cdidx` は採用される値を含む警告を出し、最後の CLI 指定がそれ以前の CLI 指定や diff --git a/changelog.d/unreleased/2813.fixed.md b/changelog.d/unreleased/2813.fixed.md new file mode 100644 index 0000000000..46fa29b215 --- /dev/null +++ b/changelog.d/unreleased/2813.fixed.md @@ -0,0 +1,23 @@ +--- +category: fixed +issues: + - 2813 +affected: + - src/CodeIndex/Cli/SearchQueryAdvisor.cs + - src/CodeIndex/Cli/SearchSnippetFormatter.cs + - src/CodeIndex/Cli/QueryCommandRunner.cs + - src/CodeIndex/Cli/JsonOutputContracts.cs + - src/CodeIndex/Mcp/McpToolHandlers.cs + - tests/CodeIndex.Tests/QueryCommandRunnerTests.cs + - tests/CodeIndex.Tests/McpServerTests.cs + - DEVELOPER_GUIDE.md + - USER_GUIDE.md +--- + +## English + +- **Punctuation-heavy searches now suggest exact substring matching (#2813)** — `search` now hints when a normal FTS query looks like a literal code phrase, including text stderr, CLI JSON `exact_substring_hint`, and MCP `recovery_hint` retry metadata. + +## 日本語 + +- **記号の多い検索で exact substring matching を提案するようになりました (#2813)** — 通常の FTS `search` query が literal code phrase に見える場合、text stderr、CLI JSON の `exact_substring_hint`、MCP の `recovery_hint` retry metadata で案内するようになりました。 diff --git a/src/CodeIndex/Cli/JsonOutputContracts.cs b/src/CodeIndex/Cli/JsonOutputContracts.cs index 9745644871..927e3d2e39 100644 --- a/src/CodeIndex/Cli/JsonOutputContracts.cs +++ b/src/CodeIndex/Cli/JsonOutputContracts.cs @@ -431,6 +431,7 @@ internal sealed record VersionInfoJsonResult( [JsonSerializable(typeof(RepoModuleResult))] [JsonSerializable(typeof(ReportBundleSummary))] [JsonSerializable(typeof(SearchHighlight))] +[JsonSerializable(typeof(SearchQueryHint))] [JsonSerializable(typeof(SearchResult))] [JsonSerializable(typeof(SearchTermOccurrence))] [JsonSerializable(typeof(SearchTruncationContext))] diff --git a/src/CodeIndex/Cli/QueryCommandRunner.cs b/src/CodeIndex/Cli/QueryCommandRunner.cs index deca452de9..b5f68fd818 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.cs @@ -413,6 +413,7 @@ public static int RunSearch(string[] cmdArgs, JsonSerializerOptions jsonOptions) if (TryWriteUnexpectedExtraPositionals("search", options)) return CommandExitCodes.UsageError; + var exactSubstringHint = SearchQueryAdvisor.BuildExactSubstringHint(options.Query, options.RawFts, exact, options.Prefix); int? jsonDoneCount = null; return WithDb(options, jsonOptions, reader => { @@ -422,15 +423,27 @@ public static int RunSearch(string[] cmdArgs, JsonSerializerOptions jsonOptions) var queryDiagnostics = DbReader.AnalyzeFtsQuery(options.Query, options.RawFts, options.Prefix, options.Lang); if (counts.Count == 0) { - Console.WriteLine(options.Json - ? BuildJsonZeroResultPayload(reader, jsonOptions, includeFiles: true, query: options.Query, ftsQueryDiagnostics: queryDiagnostics, queryOptions: options).ToJsonString(jsonOptions) - : "0"); + if (options.Json) + { + Console.WriteLine(BuildJsonZeroResultPayload(reader, jsonOptions, includeFiles: true, query: options.Query, ftsQueryDiagnostics: queryDiagnostics, queryOptions: options, exactSubstringHint: exactSubstringHint).ToJsonString(jsonOptions)); + } + else + { + Console.WriteLine("0"); + WriteExactSubstringHintIfNeeded(exactSubstringHint); + } return CommandExitCodes.Success; } - Console.WriteLine(options.Json - ? JsonSerializer.Serialize(new QueryCountFilesJsonResult(counts.Count, counts.FileCount, options.Query), CliJsonSerializerContextFactory.Create(jsonOptions).QueryCountFilesJsonResult) - : $"{counts.Count}"); + if (options.Json) + { + Console.WriteLine(JsonSerializer.Serialize(new QueryCountFilesJsonResult(counts.Count, counts.FileCount, options.Query), CliJsonSerializerContextFactory.Create(jsonOptions).QueryCountFilesJsonResult)); + } + else + { + Console.WriteLine($"{counts.Count}"); + WriteExactSubstringHintIfNeeded(exactSubstringHint); + } return CommandExitCodes.Success; } @@ -452,7 +465,7 @@ public static int RunSearch(string[] cmdArgs, JsonSerializerOptions jsonOptions) } else { - Console.WriteLine(BuildJsonZeroResultPayload(reader, jsonOptions, resultsKey: "results", query: options.Query, ftsQueryDiagnostics: ftsQueryDiagnostics, queryOptions: options).ToJsonString(jsonOptions)); + Console.WriteLine(BuildJsonZeroResultPayload(reader, jsonOptions, resultsKey: "results", query: options.Query, ftsQueryDiagnostics: ftsQueryDiagnostics, queryOptions: options, exactSubstringHint: exactSubstringHint).ToJsonString(jsonOptions)); jsonDoneCount = 0; } } @@ -460,6 +473,7 @@ public static int RunSearch(string[] cmdArgs, JsonSerializerOptions jsonOptions) { Console.Error.WriteLine(BuildZeroResultLine("No results found", options)); WriteLangHint(options.Lang, reader); + WriteExactSubstringHintIfNeeded(exactSubstringHint); WriteZeroResultHints(options, reader); } return ZeroResultExitCode(options); @@ -490,6 +504,11 @@ public static int RunSearch(string[] cmdArgs, JsonSerializerOptions jsonOptions) var compactResults = results .Select(r => SearchSnippetFormatter.ToCompactResult(r, options.Query, options.SnippetLines, exact, options.MaxLineWidth, r.Lang, options.SnippetFocus, exposeLiteralHighlights: exact)) .ToArray(); + if (exactSubstringHint != null) + { + foreach (var result in compactResults) + result.ExactSubstringHint = exactSubstringHint; + } if (options.JsonOutputFormat == JsonOutputFormatArray) { Console.WriteLine(JsonSerializer.Serialize( @@ -517,6 +536,7 @@ public static int RunSearch(string[] cmdArgs, JsonSerializerOptions jsonOptions) } var fileCount = results.Select(r => r.Path).Distinct().Count(); Console.Error.WriteLine($"({results.Count} results in {fileCount} files)"); + WriteExactSubstringHintIfNeeded(exactSubstringHint); } return CommandExitCodes.Success; }, exitCode => @@ -6726,6 +6746,14 @@ private static void WriteZeroResultHints(QueryCommandOptions options, DbReader r } } + private static void WriteExactSubstringHintIfNeeded(SearchQueryHint? hint) + { + if (hint == null) + return; + + Console.Error.WriteLine($"Hint: {hint.SuggestedAction}"); + } + private static string BuildZeroResultLine(string message, QueryCommandOptions options) { var context = BuildQueryContextParts(options, includeDefaultLimit: true).ToList(); @@ -6877,6 +6905,7 @@ private static JsonObject BuildJsonZeroResultPayload( bool? degraded = null, ExactQuerySignal? exactSignal = null, QueryCommandOptions? queryOptions = null, + SearchQueryHint? exactSubstringHint = null, Action? extraFields = null) { var payload = new JsonObject @@ -6907,6 +6936,8 @@ private static JsonObject BuildJsonZeroResultPayload( payload["query_degraded_reason"] = ftsQueryDiagnostics.QueryDegradedReason; payload["tokens_dropped"] = JsonSerializer.SerializeToNode(ftsQueryDiagnostics.TokensDropped.ToList(), CliJsonSerializerContextFactory.Create(jsonOptions).ListString); } + if (exactSubstringHint != null) + payload["exact_substring_hint"] = BuildSearchQueryHintJson(exactSubstringHint); if (queryOptions != null) payload["query_context"] = BuildQueryContextJson(queryOptions, jsonOptions); extraFields?.Invoke(payload); @@ -6915,6 +6946,14 @@ private static JsonObject BuildJsonZeroResultPayload( return payload; } + private static JsonObject BuildSearchQueryHintJson(SearchQueryHint hint) => new() + { + ["reason"] = hint.Reason, + ["suggested_action"] = hint.SuggestedAction, + ["flag"] = hint.Flag, + ["mcp_argument"] = hint.McpArgument, + }; + private static JsonObject BuildGroupedHotspotsZeroJsonPayload(DbReader reader, JsonSerializerOptions jsonOptions, bool countOnly, bool graphAvailable, QueryCommandOptions? queryOptions = null) { var payload = BuildJsonZeroResultPayload( diff --git a/src/CodeIndex/Cli/SearchQueryAdvisor.cs b/src/CodeIndex/Cli/SearchQueryAdvisor.cs new file mode 100644 index 0000000000..f4d4ad445f --- /dev/null +++ b/src/CodeIndex/Cli/SearchQueryAdvisor.cs @@ -0,0 +1,75 @@ +namespace CodeIndex.Cli; + +internal static class SearchQueryAdvisor +{ + internal const string ExactSubstringHintReason = "punctuation_heavy_query"; + internal const string CliExactSubstringSuggestedAction = "This looks like a literal code phrase; try --exact-substring for punctuation-sensitive matching."; + internal const string McpExactSubstringSuggestedAction = "This looks like a literal code phrase; try exactSubstring for punctuation-sensitive matching."; + + internal static SearchQueryHint? BuildExactSubstringHint(string? query, bool rawQuery, bool exact, bool prefix) + => ShouldSuggestExactSubstring(query, rawQuery, exact, prefix) + ? SearchQueryHint.ExactSubstring() + : null; + + internal static bool ShouldSuggestExactSubstring(string? query, bool rawQuery, bool exact, bool prefix) + { + if (rawQuery || exact || prefix || string.IsNullOrWhiteSpace(query)) + return false; + + var trimmed = query.Trim(); + if (!trimmed.Any(char.IsLetterOrDigit)) + return false; + + var tokens = trimmed.Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries); + if (tokens.Length == 1 && IsOptionLookingLiteral(tokens[0])) + return false; + + var punctuationCount = trimmed.Count(IsCodePunctuation); + if (punctuationCount >= 2) + return true; + + return tokens.Any(IsStandaloneOperatorToken); + } + + private static bool IsStandaloneOperatorToken(string token) + => token.Length > 0 + && token.All(ch => !char.IsLetterOrDigit(ch) && !char.IsWhiteSpace(ch) && ch != '_') + && token.Any(IsCodePunctuation); + + private static bool IsOptionLookingLiteral(string token) + => token.StartsWith("-", StringComparison.Ordinal) + && token.SkipWhile(ch => ch == '-').Any(char.IsLetterOrDigit); + + private static bool IsCodePunctuation(char ch) + { + if (char.IsLetterOrDigit(ch) || char.IsWhiteSpace(ch) || ch == '_') + return false; + + return ch is '.' + or ':' or ';' or ',' + or '=' or '$' or '@' or '#' + or '%' or '^' or '&' or '|' + or '!' or '?' or '+' or '-' + or '*' or '/' or '\\' + or '<' or '>' + or '(' or ')' or '[' or ']' + or '{' or '}' + or '"' or '\'' or '`' or '~'; + } +} + +public sealed class SearchQueryHint +{ + public string Reason { get; set; } = string.Empty; + public string SuggestedAction { get; set; } = string.Empty; + public string Flag { get; set; } = string.Empty; + public string McpArgument { get; set; } = string.Empty; + + internal static SearchQueryHint ExactSubstring() => new() + { + Reason = SearchQueryAdvisor.ExactSubstringHintReason, + SuggestedAction = SearchQueryAdvisor.CliExactSubstringSuggestedAction, + Flag = "--exact-substring", + McpArgument = "exactSubstring", + }; +} diff --git a/src/CodeIndex/Cli/SearchSnippetFormatter.cs b/src/CodeIndex/Cli/SearchSnippetFormatter.cs index 4af3e8dfe2..e1b74d8351 100644 --- a/src/CodeIndex/Cli/SearchSnippetFormatter.cs +++ b/src/CodeIndex/Cli/SearchSnippetFormatter.cs @@ -517,6 +517,8 @@ public sealed class CompactSearchResult public int TruncatedLineCount { get; set; } public int DroppedMatchLineCount { get; set; } public SearchTruncationContext TruncationContext { get; set; } = new(); + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public SearchQueryHint? ExactSubstringHint { get; set; } public double Score { get; set; } } diff --git a/src/CodeIndex/Mcp/McpToolHandlers.cs b/src/CodeIndex/Mcp/McpToolHandlers.cs index b7f5133604..c9010132e2 100644 --- a/src/CodeIndex/Mcp/McpToolHandlers.cs +++ b/src/CodeIndex/Mcp/McpToolHandlers.cs @@ -231,6 +231,19 @@ private static void AddRecoveryHint(JsonObject payload, string reason, string su payload["recovery_hint"] = hint; } + private static void AddExactSubstringRecoveryHint(JsonObject payload, string query) + => AddRecoveryHint( + payload, + SearchQueryAdvisor.ExactSubstringHintReason, + SearchQueryAdvisor.McpExactSubstringSuggestedAction, + "search", + new JsonObject + { + ["query"] = query, + ["exactSubstring"] = true, + ["limit"] = 5, + }); + private static void AddSymbolRecoveryHint(JsonObject payload, string query, string toolName, string? lang, string? kind, JsonNode? path) { var args = new JsonObject @@ -984,6 +997,7 @@ private JsonNode ExecuteSearch(JsonNode? id, JsonNode? args) var prefix = args?["prefix"]?.GetValue() ?? false; if (prefix && exact) return CreateToolErrorResponse(id, "'prefix' cannot be combined with 'exact' / 'exactSubstring' (exact uses instr(), not FTS5 prefix phrases)."); + var suggestExactSubstring = SearchQueryAdvisor.ShouldSuggestExactSubstring(query, rawQuery, exact, prefix); return WithDbReader(id, args, reader => { @@ -997,6 +1011,8 @@ private JsonNode ExecuteSearch(JsonNode? id, JsonNode? args) payload["path"] = PathEcho(pathPatterns); payload["excludeTests"] = excludeTests; AddSearchStabilityMetadata(payload, reader, cursor, []); + if (suggestExactSubstring) + AddExactSubstringRecoveryHint(payload, query); if (countResults.Count == 0) AddFtsQueryDiagnostics(payload, DbReader.AnalyzeFtsQuery(query, rawQuery, prefix, lang)); return CreateToolResult(id, $"Counted {countResults.Count} search result(s).", payload); @@ -1020,12 +1036,19 @@ private JsonNode ExecuteSearch(JsonNode? id, JsonNode? args) AddSearchStabilityMetadata(payload, reader, cursor, results); AddFtsQueryDiagnostics(payload, ftsDiagnostics); AddResultEnvelope(payload, 0, 0, truncated: false); - AddRecoveryHint( - payload, - "no_results", - "search returned no rows; try removing lang/path filters, using prefix for token-prefix matches, or using exactSubstring for literal punctuation or emoji.", - "search", - new JsonObject { ["query"] = query, ["limit"] = 5 }); + if (suggestExactSubstring) + { + AddExactSubstringRecoveryHint(payload, query); + } + else + { + AddRecoveryHint( + payload, + "no_results", + "search returned no rows; try removing lang/path filters, using prefix for token-prefix matches, or using exactSubstring for literal punctuation or emoji.", + "search", + new JsonObject { ["query"] = query, ["limit"] = 5 }); + } AddFreshnessHint(payload, reader); return CreateToolResult(id, "No results found.", payload); } @@ -1050,6 +1073,8 @@ private JsonNode ExecuteSearch(JsonNode? id, JsonNode? args) structured, "excerpt", BuildExcerptArgs(topResult.Path, topResult.StartLine, topResult.EndLine)); + if (suggestExactSubstring) + AddExactSubstringRecoveryHint(structured, query); // Include top file paths in summary for quick AI orientation // AIが素早く位置把握できるよう、サマリにトップファイルパスを含める var topPaths = results.Select(r => r.Path).Distinct().Take(3); diff --git a/tests/CodeIndex.Tests/McpServerTests.cs b/tests/CodeIndex.Tests/McpServerTests.cs index 50f10da7b0..0e808ac5bc 100644 --- a/tests/CodeIndex.Tests/McpServerTests.cs +++ b/tests/CodeIndex.Tests/McpServerTests.cs @@ -3097,6 +3097,21 @@ public void ToolsCall_Search_ExactSubstringReturnsLiteralHighlightMetadata() Assert.Equal("CommandText = $".Length, literalOccurrence["length"]!.GetValue()); } + [Fact] + public void ToolsCall_Search_PunctuationHeavyQueryAddsExactSubstringRecoveryHint() + { + InsertIndexedFile("src/sql.cs", "csharp", "var CommandText = $\"SELECT 1\";\nvar CommandText = other;\n"); + + var request = JsonNode.Parse("""{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"search","arguments":{"query":"CommandText = $","limit":1}}}""")!; + var response = _server.HandleMessage(request)!; + + var recoveryHint = response["result"]!["structuredContent"]!["recovery_hint"]!; + Assert.Equal("punctuation_heavy_query", recoveryHint["reason"]!.GetValue()); + Assert.Equal("search", recoveryHint["tool"]!.GetValue()); + Assert.Equal("CommandText = $", recoveryHint["args"]!["query"]!.GetValue()); + Assert.True(recoveryHint["args"]!["exactSubstring"]!.GetValue()); + } + [Fact] public void ToolsCall_Search_ExcludesGeneratedFilesByDefault() { diff --git a/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs b/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs index 1d72a926ee..6bb46e74c5 100644 --- a/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs +++ b/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs @@ -10960,6 +10960,65 @@ public void RunSearch_ExactSubstringJsonOutputsLiteralHighlightMetadata() } } + [Fact] + public void RunSearch_PunctuationHeavyTextSuggestsExactSubstring() + { + var projectRoot = TestProjectHelper.CreateTempProject("cdidx_query_runner_search_exact_substring_hint_text"); + try + { + var dbPath = TestProjectHelper.CreateProjectDb(projectRoot); + TestProjectHelper.InsertIndexedFile( + dbPath, + "src/sql.cs", + "csharp", + "var CommandText = $\"SELECT 1\";\nvar CommandText = other;"); + + var (exitCode, stdout, stderr) = CaptureConsole(() => QueryCommandRunner.RunSearch( + ["CommandText = $", "--db", dbPath, "--limit", "1"], + _jsonOptions)); + + Assert.Equal(CommandExitCodes.Success, exitCode); + Assert.Contains("src/sql.cs", stdout); + Assert.Contains("Hint: This looks like a literal code phrase; try --exact-substring for punctuation-sensitive matching.", stderr); + } + finally + { + TestProjectHelper.DeleteDirectory(projectRoot); + } + } + + [Fact] + public void RunSearch_PunctuationHeavyJsonAddsExactSubstringHint() + { + var projectRoot = TestProjectHelper.CreateTempProject("cdidx_query_runner_search_exact_substring_hint_json"); + try + { + var dbPath = TestProjectHelper.CreateProjectDb(projectRoot); + TestProjectHelper.InsertIndexedFile( + dbPath, + "src/sql.cs", + "csharp", + "var CommandText = $\"SELECT 1\";\nvar CommandText = other;"); + + var (exitCode, stdout, stderr) = CaptureConsole(() => QueryCommandRunner.RunSearch( + ["CommandText = $", "--db", dbPath, "--json", "--limit", "1"], + _jsonOptions)); + + using var document = ParseJsonOutput(stdout); + var hint = document.RootElement.GetProperty("exact_substring_hint"); + + Assert.Equal(CommandExitCodes.Success, exitCode); + Assert.Equal(string.Empty, stderr); + Assert.Equal("punctuation_heavy_query", hint.GetProperty("reason").GetString()); + Assert.Equal("--exact-substring", hint.GetProperty("flag").GetString()); + Assert.Equal("exactSubstring", hint.GetProperty("mcp_argument").GetString()); + } + finally + { + TestProjectHelper.DeleteDirectory(projectRoot); + } + } + [Fact] public void RunSearch_ExactSubstringAliasMatchesBackwardCompatibleExact() {