Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 18 additions & 0 deletions changelog.d/unreleased/2102.fixed.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
---
category: fixed
issues:
- 2102
affected:
- src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs
- src/CodeIndex/Indexer/References/Languages/SqlReferenceExtractor.cs
- tests/CodeIndex.Tests/SymbolExtractorTests.cs
- tests/CodeIndex.Tests/ReferenceExtractorTests.cs
---

## English

- **SQL generated/computed columns are now indexed (#2102)** — `CREATE TABLE` and `ALTER TABLE ... ADD` generated/computed columns emit column symbols and dependency references for generation and `DEFAULT NEXT VALUE FOR` expressions.

## 日本語

- **SQL の generated/computed column を index するようにしました (#2102)** — `CREATE TABLE` と `ALTER TABLE ... ADD` の生成/計算列で列シンボルを出し、生成式と `DEFAULT NEXT VALUE FOR` 式の依存参照も抽出します。
152 changes: 152 additions & 0 deletions src/CodeIndex/Indexer/References/Languages/SqlReferenceExtractor.cs
Original file line number Diff line number Diff line change
Expand Up @@ -285,6 +285,18 @@ internal sealed class State
private static readonly Regex CreateTempRoutineRegex = new(
$@"(?<![\w$])CREATE(?:\s+OR\s+(?:REPLACE|ALTER))?(?:\s+(?:TEMP|TEMPORARY))?\s+(?:PROC(?:EDURE)?|FUNCTION)\b(?:\s+IF\s+NOT\s+EXISTS)?\s+(?<name>{TempIdentifierPattern})",
RegexOptions.Compiled | RegexOptions.IgnoreCase);
private static readonly Regex GeneratedColumnMarkerRegex = new(
@"\b(?:GENERATED\s+(?:ALWAYS\s+)?AS|NEXT\s+VALUE\s+FOR)\b|(?<![\w$])AS\s*\(",
RegexOptions.Compiled | RegexOptions.IgnoreCase);
private static readonly Regex GeneratedColumnExpressionStartRegex = new(
@"\b(?:GENERATED\s+(?:ALWAYS\s+)?AS|AS)\s*\(",
RegexOptions.Compiled | RegexOptions.IgnoreCase);
private static readonly Regex DefaultNextValueForExpressionRegex = new(
$@"\bDEFAULT\s+NEXT\s+VALUE\s+FOR\s+(?<name>{QualifiedIdentifierNoCapturePattern})",
RegexOptions.Compiled | RegexOptions.IgnoreCase);
private static readonly Regex SqlExpressionIdentifierRegex = new(
$@"(?<name>{QualifiedIdentifierNoCapturePattern})",
RegexOptions.Compiled | RegexOptions.IgnoreCase);
private static readonly Regex TrailingTempIdentifierRegex = new(
$@"^(?:(?:ONLY)\b\s+)?(?<item>(?:{TempIdentifierPattern}|{QualifiedIdentifierNoCapturePattern}))(?:\s+(?:AS\s+)?(?:{QuotedIdentifierPattern}|{BareIdentifierPattern}))?\s*$",
RegexOptions.Compiled | RegexOptions.IgnoreCase);
Expand Down Expand Up @@ -509,6 +521,19 @@ private static void EmitStatementReferences(
fileId,
resolveContainerForCall);

EmitGeneratedColumnDependencyReferences(
statement,
statementStart,
statementLineOffset,
lineOffset,
context,
lineNumber,
references,
seen,
fileId,
resolveContainerForCall,
shouldIgnoreName);

EmitSourceCaptureReferences(
FromSourceListRegex.Matches(statement),
statement,
Expand Down Expand Up @@ -1547,6 +1572,133 @@ private static void EmitMergeUsingReferences(
}
}

private static void EmitGeneratedColumnDependencyReferences(
string statement,
int statementStart,
int statementLineOffset,
int lineOffset,
string context,
int lineNumber,
List<ReferenceRecord> references,
HashSet<string> seen,
long fileId,
Func<int, SymbolRecord?> resolveContainerForCall,
Func<string, bool> shouldIgnoreName)
{
if (!GeneratedColumnMarkerRegex.IsMatch(statement))
return;

foreach (Match match in GeneratedColumnExpressionStartRegex.Matches(statement))
{
if (match.Index < statementLineOffset || IsInsideDoubleQuotedRegion(statement, match.Index))
continue;
if (match.Value.TrimStart().StartsWith("AS", StringComparison.OrdinalIgnoreCase)
&& !IsLikelyComputedColumnAsExpression(statement, match.Index))
{
continue;
}

var openParenIndex = statement.IndexOf('(', match.Index + match.Length - 1);
if (openParenIndex < 0)
continue;

var closeParenIndex = FindMatchingParen(statement, openParenIndex);
if (closeParenIndex <= openParenIndex)
continue;

EmitSqlExpressionIdentifierDependencies(
statement,
openParenIndex + 1,
closeParenIndex,
statementStart,
statementLineOffset,
lineOffset,
context,
lineNumber,
references,
seen,
fileId,
resolveContainerForCall,
shouldIgnoreName);
}

foreach (Match match in DefaultNextValueForExpressionRegex.Matches(statement))
{
if (match.Index < statementLineOffset || IsInsideDoubleQuotedRegion(statement, match.Index))
continue;

var sequence = match.Groups["name"];
EmitSqlExpressionIdentifierDependencies(
statement,
sequence.Index,
sequence.Index + sequence.Length,
statementStart,
statementLineOffset,
lineOffset,
context,
lineNumber,
references,
seen,
fileId,
resolveContainerForCall,
shouldIgnoreName);
}
}

private static void EmitSqlExpressionIdentifierDependencies(
string statement,
int startIndex,
int endIndexExclusive,
int statementStart,
int statementLineOffset,
int lineOffset,
string context,
int lineNumber,
List<ReferenceRecord> references,
HashSet<string> seen,
long fileId,
Func<int, SymbolRecord?> resolveContainerForCall,
Func<string, bool> shouldIgnoreName)
{
var expression = statement[startIndex..endIndexExclusive];
foreach (Match match in SqlExpressionIdentifierRegex.Matches(expression))
{
var rawIndex = startIndex + match.Index;
if (rawIndex < statementLineOffset || IsInsideDoubleQuotedRegion(statement, rawIndex))
continue;

var rawName = match.Value;
NormalizeIdentifier(rawName, rawIndex, out var resolvedName, out var nameIndex, out var wasQuoted);
if (!wasQuoted && (shouldIgnoreName(resolvedName) || IsGeneratedColumnDependencyKeyword(resolvedName)))
continue;

var nameColumn = nameIndex + statementStart - lineOffset;
var container = resolveContainerForCall(rawIndex);
ReferenceExtractor.AddReference(references, seen, fileId, resolvedName, nameColumn, "generated_column_dependency", context, lineNumber, container);
}
}

private static bool IsGeneratedColumnDependencyKeyword(string name)
=> name.Equals("GENERATED", StringComparison.OrdinalIgnoreCase)
|| name.Equals("ALWAYS", StringComparison.OrdinalIgnoreCase)
|| name.Equals("AS", StringComparison.OrdinalIgnoreCase)
|| name.Equals("DEFAULT", StringComparison.OrdinalIgnoreCase)
|| name.Equals("NEXT", StringComparison.OrdinalIgnoreCase)
|| name.Equals("VALUE", StringComparison.OrdinalIgnoreCase)
|| name.Equals("FOR", StringComparison.OrdinalIgnoreCase)
|| name.Equals("STORED", StringComparison.OrdinalIgnoreCase)
|| name.Equals("VIRTUAL", StringComparison.OrdinalIgnoreCase)
|| name.Equals("PERSISTED", StringComparison.OrdinalIgnoreCase)
|| name.Equals("NULL", StringComparison.OrdinalIgnoreCase)
|| name.Equals("NOT", StringComparison.OrdinalIgnoreCase);

private static bool IsLikelyComputedColumnAsExpression(string statement, int asIndex)
{
var prefix = statement[..asIndex];
return Regex.IsMatch(prefix, @"(?<![\w$])ALTER\s+TABLE\b[\s\S]*\bADD\b", RegexOptions.IgnoreCase)
|| Regex.IsMatch(prefix, @"(?<![\w$])CREATE\s+(?:OR\s+(?:REPLACE|ALTER)\s+)?(?:(?:(?:GLOBAL|LOCAL)\s+)?(?:TEMP|TEMPORARY)\s+|UNLOGGED\s+)?TABLE\b", RegexOptions.IgnoreCase);
}

private static void EmitSourceReference(
string rawName,
int rawIndex,
Expand Down
152 changes: 152 additions & 0 deletions src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs
Original file line number Diff line number Diff line change
Expand Up @@ -361,6 +361,18 @@ public static int GetContractVersion(string? lang)
private static readonly Regex SqlCteDefinitionRegex = new(
$@"(?<![\w$])(?:WITH\s+(?:RECURSIVE\s+)?|,\s*)(?<name>{SqlQualifiedIdentifierSegmentPattern})(?:\s*\([^)]*\))?\s+AS\s*\(",
RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant);
private static readonly Regex SqlAlterTableAddGeneratedColumnRegex = new(
$@"(?<![\w$])ALTER\s+TABLE\s+(?<table>{SqlQualifiedIdentifierPattern})\s+ADD(?:\s+COLUMN)?\s+(?!CONSTRAINT\b)(?<name>{SqlQualifiedIdentifierSegmentPattern})\b(?=[^;]*?\b(?:GENERATED\s+(?:ALWAYS\s+)?AS|AS\s*\(|DEFAULT\s+NEXT\s+VALUE\s+FOR)\b)",
RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant);
private static readonly Regex SqlCreateTableBodyRegex = new(
$@"(?<![\w$])CREATE\s+(?:OR\s+(?:REPLACE|ALTER)\s+)?(?:(?:(?:GLOBAL|LOCAL)\s+)?(?:TEMP|TEMPORARY)\s+|UNLOGGED\s+)?TABLE\s+(?:IF\s+NOT\s+EXISTS\s+)?(?<table>{SqlQualifiedIdentifierPattern})\s*\((?<body>[\s\S]*?)\)\s*;",
RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant);
private static readonly Regex SqlGeneratedColumnDefinitionMarkerRegex = new(
@"\b(?:GENERATED\s+(?:ALWAYS\s+)?AS|AS\s*\(|DEFAULT\s+NEXT\s+VALUE\s+FOR)\b",
RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant);
private static readonly Regex SqlColumnDefinitionNameRegex = new(
$@"^\s*(?<name>{SqlQualifiedIdentifierSegmentPattern})\b",
RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant);
private static readonly Regex SqlReturnsTableRegex = new(
@"\bRETURNS\s+TABLE\s*\((?<columns>(?:[^()]|\([^()]*\))*)\)",
RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant | RegexOptions.Singleline);
Expand Down Expand Up @@ -1391,6 +1403,8 @@ private enum JavaScriptTypeScriptFunctionHeaderConsumeResult
// file module declarations and inline modules / ファイルモジュール宣言とインラインモジュール
new("file_module", new Regex(@"^\s*(?:(?<visibility>pub(?:\([^)]*\))?)\s+)?mod\s+(?<name>(?:r#)?\w+)\s*;", RegexOptions.Compiled), BodyStyle.None, "visibility"),
new("namespace", new Regex(@"^\s*(?:(?<visibility>pub(?:\([^)]*\))?)\s+)?mod\s+(?<name>(?:r#)?\w+)", RegexOptions.Compiled), BodyStyle.Brace, "visibility"),
// Trait associated type defaults / trait 関連型のデフォルト
new("property", new Regex(@"^\s*(?:(?<visibility>pub(?:\([^)]*\))?)\s+)?type\s+(?<name>(?:r#)?\w+)(?:\s*<[^=>]+>)?(?:\s*:\s*[^=;]+)?\s*=\s*(?<returnType>[^;]+)", RegexOptions.Compiled), BodyStyle.None, "visibility", "returnType"),
// type alias / 型エイリアス
new("import", new Regex(@"^\s*(?:(?<visibility>pub(?:\([^)]*\))?)\s+)?type\s+(?<name>(?:r#)?\w+)(?:\s*<[^=]+>)?", RegexOptions.Compiled), BodyStyle.None, "visibility"),
new("import", new Regex(@"^\s*(?:(?<visibility>pub(?:\([^)]*\))?)\s+)?use\s+(?<name>.+);", RegexOptions.Compiled), BodyStyle.None, "visibility"),
Expand Down Expand Up @@ -2099,6 +2113,22 @@ public static IReadOnlyCollection<string> GetSupportedLanguages()
"class", "struct", "interface", "protocol", "namespace", "enum", "object", "heading", "specialization", "class_hook"
];

private static bool IsRustDirectTraitBodyMember(List<SymbolRecord> symbols, int candidateLine)
{
SymbolRecord? innermostContainer = null;
foreach (var symbol in symbols)
{
if (!symbol.BodyStartLine.HasValue || !symbol.BodyEndLine.HasValue)
continue;
if (candidateLine < symbol.BodyStartLine.Value || candidateLine > symbol.BodyEndLine.Value)
continue;
if (innermostContainer == null || symbol.StartLine >= innermostContainer.StartLine)
innermostContainer = symbol;
}

return innermostContainer?.Kind == "protocol";
}

/// <summary>
/// Extract symbols from the given source content.
/// 指定されたソース内容からシンボルを抽出する。
Expand Down Expand Up @@ -2687,6 +2717,14 @@ public static List<SymbolRecord> Extract(long fileId, string? lang, string conte
lineOffset = FindNextSameLineBraceStatementStart(matchLine, absoluteStartColumn + Math.Max(1, match.Length), lang);
continue;
}
if (lang == "rust"
&& pattern.Kind == "property"
&& pattern.BodyStyle == BodyStyle.None
&& pattern.ReturnTypeGroup != null
&& !IsRustDirectTraitBodyMember(symbols, i + 1))
{
break;
}
var rawReturnType = NormalizeCSharpImplicitPartialMethodReturnType(
lang,
pattern,
Expand Down Expand Up @@ -3939,6 +3977,7 @@ public static List<SymbolRecord> Extract(long fileId, string? lang, string conte
ExtractSqlCteSymbols(fileId, lines, symbols);
ExtractSqlDefinerSymbols(fileId, lines, sqlSyntheticSymbolLines, symbols);
ExtractSqlRoutineResultColumnSymbols(fileId, lines, sqlSyntheticSymbolLines, symbols);
ExtractSqlGeneratedColumnSymbols(fileId, lines, sqlSyntheticSymbolLines, symbols);
}
if (IsRazorLanguage(originalLang) || IsRazorFilePath(filePath))
ExtractRazorDirectiveSymbols(fileId, lines, symbols);
Expand Down Expand Up @@ -4046,6 +4085,119 @@ private static int GetLineNumberFromOffset(List<int> lineStarts, int offset)
return ~index;
}

private static void ExtractSqlGeneratedColumnSymbols(long fileId, string[] lines, string[] structuralLines, List<SymbolRecord> symbols)
{
var structuralContent = string.Join('\n', structuralLines);
if (structuralContent.IndexOf("GENERATED", StringComparison.OrdinalIgnoreCase) < 0
&& structuralContent.IndexOf("NEXT VALUE FOR", StringComparison.OrdinalIgnoreCase) < 0
&& structuralContent.IndexOf(" AS ", StringComparison.OrdinalIgnoreCase) < 0)
{
return;
}

var lineStarts = BuildLineStarts(structuralContent);
foreach (Match match in SqlAlterTableAddGeneratedColumnRegex.Matches(structuralContent))
{
var nameGroup = match.Groups["name"];
AddSqlGeneratedColumnSymbol(
fileId,
lines,
lineStarts,
new GroupProxy(nameGroup.Value, nameGroup.Index),
match.Groups["table"].Value,
symbols);
}

foreach (Match tableMatch in SqlCreateTableBodyRegex.Matches(structuralContent))
{
var tableName = tableMatch.Groups["table"].Value;
var bodyGroup = tableMatch.Groups["body"];
foreach (var column in EnumerateSqlColumnDefinitions(bodyGroup.Value, bodyGroup.Index))
{
if (!SqlGeneratedColumnDefinitionMarkerRegex.IsMatch(column.Text))
continue;

var nameMatch = SqlColumnDefinitionNameRegex.Match(column.Text);
if (!nameMatch.Success)
continue;

AddSqlGeneratedColumnSymbol(
fileId,
lines,
lineStarts,
new GroupProxy(nameMatch.Groups["name"].Value, column.StartIndex + nameMatch.Groups["name"].Index),
tableName,
symbols);
}
}
}

private static void AddSqlGeneratedColumnSymbol(
long fileId,
string[] lines,
List<int> lineStarts,
IGroupLike nameGroup,
string rawTableName,
List<SymbolRecord> symbols)
{
var name = NormalizeSqlIdentifierSegment(nameGroup.Value);
if (string.IsNullOrWhiteSpace(name))
return;

var lineNumber = GetLineNumberFromOffset(lineStarts, nameGroup.Index);
AddSymbolRecord(
symbols,
null,
lineNumber,
new SymbolRecord
{
FileId = fileId,
Kind = "property",
SubKind = "generated_column",
Name = name,
Line = lineNumber,
StartLine = lineNumber,
StartColumn = nameGroup.Index - lineStarts[lineNumber - 1],
EndLine = lineNumber,
Signature = lines[lineNumber - 1].Trim(),
ContainerKind = "class",
ContainerName = NormalizeSqlIdentifierSegment(SqlNameResolver.GetLeafName(rawTableName)),
},
lines[lineNumber - 1]);
}

private interface IGroupLike
{
string Value { get; }
int Index { get; }
}

private readonly record struct GroupProxy(string Value, int Index) : IGroupLike;

private readonly record struct SqlColumnDefinitionSlice(string Text, int StartIndex);

private static IEnumerable<SqlColumnDefinitionSlice> EnumerateSqlColumnDefinitions(string body, int bodyStartIndex)
{
var start = 0;
var depth = 0;
for (var i = 0; i <= body.Length; i++)
{
if (i == body.Length || (body[i] == ',' && depth == 0))
{
var text = body[start..i].Trim();
if (text.Length > 0)
yield return new SqlColumnDefinitionSlice(text, bodyStartIndex + start + body[start..i].Length - body[start..i].TrimStart().Length);
start = i + 1;
continue;
}

if (body[i] == '(')
depth++;
else if (body[i] == ')' && depth > 0)
depth--;
}
}

private static string NormalizeSqlIdentifierSegment(string value)
{
if (value.Length >= 2 && value[0] == '[' && value[^1] == ']')
Expand Down
Loading
Loading