From cc226ffa01f91e52ea4dfc9a383045186111e8f2 Mon Sep 17 00:00:00 2001 From: Alexey Kulakov Date: Fri, 17 Oct 2025 11:39:10 +0500 Subject: [PATCH 1/7] Add support for Included columns for PostreSql 12+ --- .../Sql.Drivers.PostgreSql/v12_0/ServerInfoProvider.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Orm/Xtensive.Orm.PostgreSql/Sql.Drivers.PostgreSql/v12_0/ServerInfoProvider.cs b/Orm/Xtensive.Orm.PostgreSql/Sql.Drivers.PostgreSql/v12_0/ServerInfoProvider.cs index 8cbbf91399..781681939f 100644 --- a/Orm/Xtensive.Orm.PostgreSql/Sql.Drivers.PostgreSql/v12_0/ServerInfoProvider.cs +++ b/Orm/Xtensive.Orm.PostgreSql/Sql.Drivers.PostgreSql/v12_0/ServerInfoProvider.cs @@ -2,10 +2,14 @@ // This code is distributed under MIT license terms. // See the License.txt file in the project root for more information. +using Xtensive.Sql.Info; + namespace Xtensive.Sql.Drivers.PostgreSql.v12_0 { internal class ServerInfoProvider : v10_0.ServerInfoProvider { + protected override IndexFeatures GetIndexFeatures() => base.GetIndexFeatures() | IndexFeatures.NonKeyColumns; + // Constructors public ServerInfoProvider(SqlDriver driver) From fc721497cf2f009a9827cc64af3cfd4907d7ce66 Mon Sep 17 00:00:00 2001 From: Alexey Kulakov Date: Fri, 17 Oct 2025 12:09:54 +0500 Subject: [PATCH 2/7] Add proper translation of different sections of index creation --- .../v12_0/Translator.cs | 25 +++++++++++++++++++ .../Sql.Drivers.PostgreSql/v8_0/Translator.cs | 14 ++++++++--- 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/Orm/Xtensive.Orm.PostgreSql/Sql.Drivers.PostgreSql/v12_0/Translator.cs b/Orm/Xtensive.Orm.PostgreSql/Sql.Drivers.PostgreSql/v12_0/Translator.cs index e7e5312b1a..821fa2ac36 100644 --- a/Orm/Xtensive.Orm.PostgreSql/Sql.Drivers.PostgreSql/v12_0/Translator.cs +++ b/Orm/Xtensive.Orm.PostgreSql/Sql.Drivers.PostgreSql/v12_0/Translator.cs @@ -2,10 +2,35 @@ // This code is distributed under MIT license terms. // See the License.txt file in the project root for more information. +using Xtensive.Sql.Compiler; +using Xtensive.Sql.Ddl; + namespace Xtensive.Sql.Drivers.PostgreSql.v12_0 { internal class Translator : v10_0.Translator { + public override void Translate(SqlCompilerContext context, SqlCreateIndex node, CreateIndexSection section) + { + var index = node.Index; + if (!index.IsFullText) { + var output = context.Output; + switch (section) { + case CreateIndexSection.NonkeyColumnsEnter: + _ = output.AppendOpeningPunctuation("INCLUDE ("); + break; + case CreateIndexSection.NonkeyColumnsExit: + _ = output.AppendClosingPunctuation(")"); + break; + default: + base.Translate(context, node, section); + break; + } + } + else { + base.Translate(context, node, section); + } + } + // Constructors public Translator(SqlDriver driver) diff --git a/Orm/Xtensive.Orm.PostgreSql/Sql.Drivers.PostgreSql/v8_0/Translator.cs b/Orm/Xtensive.Orm.PostgreSql/Sql.Drivers.PostgreSql/v8_0/Translator.cs index 76139d1516..458738945e 100644 --- a/Orm/Xtensive.Orm.PostgreSql/Sql.Drivers.PostgreSql/v8_0/Translator.cs +++ b/Orm/Xtensive.Orm.PostgreSql/Sql.Drivers.PostgreSql/v8_0/Translator.cs @@ -294,22 +294,30 @@ public override void Translate(SqlCompilerContext context, SqlCreateIndex node, if (index.IsSpatial) { _ = output.Append(" USING GIST"); } + break; + case CreateIndexSection.ColumnsEnter: _ = output.Append("("); break; - case CreateIndexSection.StorageOptions: + case CreateIndexSection.ColumnsExit: _ = output.Append(")"); + break; + case CreateIndexSection.NonkeyColumnsEnter: + break; + case CreateIndexSection.NonkeyColumnsExit: + break; + case CreateIndexSection.StorageOptions: AppendIndexStorageParameters(output, index); if (!string.IsNullOrEmpty(index.Filegroup)) { _ = output.Append(" TABLESPACE "); TranslateIdentifier(output, index.Filegroup); } - break; - case CreateIndexSection.Exit: break; case CreateIndexSection.Where: _ = output.Append(" WHERE "); break; + case CreateIndexSection.Exit: + break; default: break; ; From a2e9c4f8b8bab92ac19fbf5fe74c70feae919936 Mon Sep 17 00:00:00 2001 From: Alexey Kulakov Date: Fri, 17 Oct 2025 13:48:28 +0500 Subject: [PATCH 3/7] Fix column enter and column exit for fll text indexes --- .../Sql.Drivers.PostgreSql/v8_3/Translator.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Orm/Xtensive.Orm.PostgreSql/Sql.Drivers.PostgreSql/v8_3/Translator.cs b/Orm/Xtensive.Orm.PostgreSql/Sql.Drivers.PostgreSql/v8_3/Translator.cs index fb0fe8279e..7107ca88f6 100644 --- a/Orm/Xtensive.Orm.PostgreSql/Sql.Drivers.PostgreSql/v8_3/Translator.cs +++ b/Orm/Xtensive.Orm.PostgreSql/Sql.Drivers.PostgreSql/v8_3/Translator.cs @@ -31,7 +31,10 @@ public override void Translate(SqlCompilerContext context, SqlCreateIndex node, TranslateIdentifier(output, index.Name); _ = output.Append(" ON "); Translate(context, index.DataTable); - _ = output.Append(" USING gin ("); + _ = output.Append(" USING gin "); + break; + case CreateIndexSection.ColumnsEnter: + _ = output.Append("("); break; case CreateIndexSection.ColumnsExit: // Add actual columns list From de44d58f7577bff27749bc4fd847189f27d78261 Mon Sep 17 00:00:00 2001 From: Alexey Kulakov Date: Fri, 17 Oct 2025 16:56:30 +0500 Subject: [PATCH 4/7] Implement non-key columns extraction --- .../Sql.Drivers.PostgreSql/v12_0/Extractor.cs | 129 +++++++++++++++++- 1 file changed, 128 insertions(+), 1 deletion(-) diff --git a/Orm/Xtensive.Orm.PostgreSql/Sql.Drivers.PostgreSql/v12_0/Extractor.cs b/Orm/Xtensive.Orm.PostgreSql/Sql.Drivers.PostgreSql/v12_0/Extractor.cs index 71418c3b3e..7e98eaddbd 100644 --- a/Orm/Xtensive.Orm.PostgreSql/Sql.Drivers.PostgreSql/v12_0/Extractor.cs +++ b/Orm/Xtensive.Orm.PostgreSql/Sql.Drivers.PostgreSql/v12_0/Extractor.cs @@ -2,6 +2,10 @@ // This code is distributed under MIT license terms. // See the License.txt file in the project root for more information. +using System; +using System.Data.Common; +using System.Linq; +using System.Text.RegularExpressions; using Xtensive.Sql.Model; namespace Xtensive.Sql.Drivers.PostgreSql.v12_0 @@ -13,7 +17,10 @@ protected override void BuildPgCatalogSchema(Schema schema) { base.BuildPgCatalogSchema(schema); var defaultValuesTable = schema.Tables["pg_attrdef"]; - defaultValuesTable.CreateColumn("adbin", new SqlValueType(SqlType.Binary)); + _ = defaultValuesTable.CreateColumn("adbin", new SqlValueType(SqlType.Binary)); + + var indexesTable = schema.Tables["pg_index"]; + CreateInt2Column(indexesTable, "indnkeyatts"); } /// @@ -77,6 +84,126 @@ protected override ISqlCompileUnit BuildExtractTableAndDomainConstraintsQuery(Ex return select; } + /// + protected override ISqlCompileUnit BuildExtractTableIndexesQuery(ExtractionContext context) + { + var tableMap = context.TableMap; + var tableSpacesTable = PgTablespace; + var relationsTable = PgClass; + var indexTable = PgIndex; + var dependencyTable = PgDepend; + + //subselect that index was not created automatically + var subSelect = SqlDml.Select(dependencyTable); + subSelect.Where = dependencyTable["classid"] == PgClassOid && + dependencyTable["objid"] == indexTable["indexrelid"] && + dependencyTable["deptype"] == 'i'; + subSelect.Columns.Add(dependencyTable[0]); + + //not automatically created indexes of our tables + var select = SqlDml.Select(indexTable + .InnerJoin(relationsTable, relationsTable["oid"] == indexTable["indexrelid"]) + .LeftOuterJoin(tableSpacesTable, tableSpacesTable["oid"] == relationsTable["reltablespace"])); + select.Where = SqlDml.In(indexTable["indrelid"], CreateOidRow(tableMap.Keys)) && !SqlDml.Exists(subSelect); + select.Columns.Add(indexTable["indrelid"]); + select.Columns.Add(indexTable["indexrelid"]); + select.Columns.Add(relationsTable["relname"]); + select.Columns.Add(indexTable["indisunique"]); + select.Columns.Add(indexTable["indisclustered"]); + select.Columns.Add(indexTable["indkey"]); + select.Columns.Add(tableSpacesTable["spcname"]); + select.Columns.Add(indexTable["indnatts"]); + select.Columns.Add(indexTable["indnkeyatts"]); + select.Columns.Add(SqlDml.FunctionCall("pg_get_expr", indexTable["indexprs"], indexTable["indrelid"], true), + "indexprstext"); + select.Columns.Add(SqlDml.FunctionCall("pg_get_expr", indexTable["indpred"], indexTable["indrelid"], true), + "indpredtext"); + select.Columns.Add(SqlDml.FunctionCall("pg_get_indexdef", indexTable["indexrelid"]), "inddef"); + AddSpecialIndexQueryColumns(select, tableSpacesTable, relationsTable, indexTable, dependencyTable); + return select; + } + + /// + protected override int ReadTableIndexData(DbDataReader dataReader, ExtractionContext context) + { + var tableMap = context.TableMap; + var tableColumns = context.TableColumnMap; + + var maxColumnNumber = 0; + var tableIdentifier = Convert.ToInt64(dataReader["indrelid"]); + var indexIdentifier = Convert.ToInt64(dataReader["indexrelid"]); + var indexName = dataReader["relname"].ToString(); + var isUnique = dataReader.GetBoolean(dataReader.GetOrdinal("indisunique")); + var indexKey = (short[]) dataReader["indkey"]; + + var tablespaceName = (dataReader["spcname"] != DBNull.Value) ? dataReader["spcname"].ToString() : null; + var filterExpression = (dataReader["indpredtext"] != DBNull.Value) + ? dataReader["indpredtext"].ToString() + : string.Empty; + + var table = tableMap[tableIdentifier]; + + var fullTextRegex = + @"(?<=CREATE INDEX \S+ ON \S+ USING (?:gist|gin)(?:\s|\S)*)to_tsvector\('(\w+)'::regconfig, \(*(?:(?:\s|\)|\(|\|)*(?:\(""(\S+)""\)|'\s')::text)+\)"; + var indexScript = dataReader["inddef"].ToString(); + var matches = Regex.Matches(indexScript, fullTextRegex, RegexOptions.Compiled); + if (matches.Count > 0) { + // Fulltext index + var fullTextIndex = table.CreateFullTextIndex(indexName); + foreach (Match match in matches) { + var columnConfigurationName = match.Groups[1].Value; + foreach (Capture capture in match.Groups[2].Captures) { + var columnName = capture.Value; + var fullTextColumn = fullTextIndex.Columns[columnName] + ?? fullTextIndex.CreateIndexColumn(table.Columns.Single(column => column.Name == columnName)); + if (fullTextColumn.Languages[columnConfigurationName] == null) + fullTextColumn.Languages.Add(new Language(columnConfigurationName)); + } + } + } + else { + //Regular index + var index = table.CreateIndex(indexName); + index.IsBitmap = false; + index.IsUnique = isUnique; + index.Filegroup = tablespaceName; + if (!string.IsNullOrEmpty(filterExpression)) + index.Where = SqlDml.Native(filterExpression); + + // Expression-based index + var some = dataReader["indexprstext"]; + if (some != DBNull.Value) { + context.ExpressionIndexMap[indexIdentifier] = new ExpressionIndexInfo(index, indexKey); + maxColumnNumber = dataReader.GetInt16(dataReader.GetOrdinal("indnatts")); + } + else { + var keyColumnNumber = dataReader.GetInt16(dataReader.GetOrdinal("indnkeyatts")); + for (int j = 0; j < indexKey.Length; j++) { + if (j < keyColumnNumber) { + int colIndex = indexKey[j]; + if (colIndex > 0) { + _ = index.CreateIndexColumn(tableColumns[tableIdentifier][colIndex], true); + } + else { + //column index is 0 + //this means that this index column is an expression + //which is not possible with SqlDom tables + } + } + else { + int colIndex = indexKey[j]; + index.NonkeyColumns.Add(tableColumns[tableIdentifier][colIndex]); + } + } + } + + ReadSpecialIndexProperties(dataReader, index); + } + + return maxColumnNumber; + } + + // Constructors public Extractor(SqlDriver driver) From 7c427912e01ee1c4c23054e2296c197311d908c8 Mon Sep 17 00:00:00 2001 From: Alexey Kulakov Date: Fri, 17 Oct 2025 17:51:02 +0500 Subject: [PATCH 5/7] PostgreSql.ExtractorTest: Change test database structure to have included columns --- .../PostgreSql/ExtractorTest.cs | 21 +++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/Orm/Xtensive.Orm.Tests.Sql/PostgreSql/ExtractorTest.cs b/Orm/Xtensive.Orm.Tests.Sql/PostgreSql/ExtractorTest.cs index 493a3bc8f9..df267166e2 100644 --- a/Orm/Xtensive.Orm.Tests.Sql/PostgreSql/ExtractorTest.cs +++ b/Orm/Xtensive.Orm.Tests.Sql/PostgreSql/ExtractorTest.cs @@ -92,10 +92,23 @@ protected override string GetForeignKeyExtractionCleanUpScript() => protected override string GetIndexExtractionPrepareScript(string tableName) { - return - $"CREATE TABLE \"{tableName}\" (\"column1\" int, \"column2\" int);" + - $"\n CREATE INDEX \"{tableName}_index1_desc_asc\" on \"{tableName}\" (\"column1\" desc, \"column2\" asc);" + - $"\n CREATE UNIQUE INDEX \"{tableName}_index1_u_asc_desc\" on \"{tableName}\" (\"column1\" asc, \"column2\" desc);"; + // CREATE TABLE table1 (column1 int, column2 int); + // CREATE INDEX table1_index1_desc_asc on table1 (column1 desc, column2 asc); + // CREATE UNIQUE INDEX table1_index1_u_asc_desc on table1 (column1 asc, column2 desc); + // CREATE UNIQUE INDEX table1_index_with_included_columns on table1 (column1 asc) include (column2); + if (NonKeyColumnsSupported) { + return + $"CREATE TABLE \"{tableName}\" (\"column1\" int, \"column2\" int);" + + $"\n CREATE INDEX \"{tableName}_index1_desc_asc\" on \"{tableName}\" (\"column1\" desc, \"column2\" asc);" + + $"\n CREATE UNIQUE INDEX \"{tableName}_index1_u_asc_desc\" on \"{tableName}\" (\"column1\" asc, \"column2\" desc);" + + $"\n CREATE UNIQUE INDEX \"{tableName}_index_with_included_columns\" on \"{tableName}\" (\"column1\" asc) include \"column2\");"; + } + else { + return + $"CREATE TABLE \"{tableName}\" (\"column1\" int, \"column2\" int);" + + $"\n CREATE INDEX \"{tableName}_index1_desc_asc\" on \"{tableName}\" (\"column1\" desc, \"column2\" asc);" + + $"\n CREATE UNIQUE INDEX \"{tableName}_index1_u_asc_desc\" on \"{tableName}\" (\"column1\" asc, \"column2\" desc);"; + } } protected override string GetIndexExtractionCleanUpScript(string tableName) => $"drop table \"{tableName}\";"; From 9ccc361d085d3880ffa4e083e1c9706f2f99eac8 Mon Sep 17 00:00:00 2001 From: Alexey Kulakov Date: Mon, 20 Oct 2025 14:03:47 +0500 Subject: [PATCH 6/7] Fix error in sql command and handled Pgsql indexed column drop --- Orm/Xtensive.Orm.Tests.Sql/PostgreSql/ExtractorTest.cs | 2 +- .../Storage/IgnoreRulesValidateTest.cs | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/Orm/Xtensive.Orm.Tests.Sql/PostgreSql/ExtractorTest.cs b/Orm/Xtensive.Orm.Tests.Sql/PostgreSql/ExtractorTest.cs index df267166e2..0555f33c43 100644 --- a/Orm/Xtensive.Orm.Tests.Sql/PostgreSql/ExtractorTest.cs +++ b/Orm/Xtensive.Orm.Tests.Sql/PostgreSql/ExtractorTest.cs @@ -101,7 +101,7 @@ protected override string GetIndexExtractionPrepareScript(string tableName) $"CREATE TABLE \"{tableName}\" (\"column1\" int, \"column2\" int);" + $"\n CREATE INDEX \"{tableName}_index1_desc_asc\" on \"{tableName}\" (\"column1\" desc, \"column2\" asc);" + $"\n CREATE UNIQUE INDEX \"{tableName}_index1_u_asc_desc\" on \"{tableName}\" (\"column1\" asc, \"column2\" desc);" + - $"\n CREATE UNIQUE INDEX \"{tableName}_index_with_included_columns\" on \"{tableName}\" (\"column1\" asc) include \"column2\");"; + $"\n CREATE UNIQUE INDEX \"{tableName}_index_with_included_columns\" on \"{tableName}\" (\"column1\" asc) include (\"column2\");"; } else { return diff --git a/Orm/Xtensive.Orm.Tests/Storage/IgnoreRulesValidateTest.cs b/Orm/Xtensive.Orm.Tests/Storage/IgnoreRulesValidateTest.cs index 144d6cf2c3..487df27c4f 100644 --- a/Orm/Xtensive.Orm.Tests/Storage/IgnoreRulesValidateTest.cs +++ b/Orm/Xtensive.Orm.Tests/Storage/IgnoreRulesValidateTest.cs @@ -344,6 +344,7 @@ public class IgnoreRulesValidateTest private readonly bool createConstraintsWithTable = StorageProviderInfo.Instance.Provider == StorageProvider.Sqlite; private readonly bool noExceptionOnIndexKeyColumnDrop = StorageProviderInfo.Instance.Provider.In(StorageProvider.PostgreSql, StorageProvider.MySql); + private readonly bool noExceptionOnIndexIncludedColumnDrop = StorageProviderInfo.Instance.Provider.In(StorageProvider.PostgreSql); private readonly SqlDriver sqlDriver = TestSqlDriver.Create(GetConnectionInfo()); private Key changedOrderKey; @@ -646,8 +647,13 @@ public void DropIncludedColumnOfIgnoredIndexTest() var ignoreRuleCollection = new IgnoreRuleCollection(); _ = ignoreRuleCollection.IgnoreIndex("IX_Ignored_Index").WhenTable("MyEntity2"); - _ = Assert.Throws( - () => BuildDomain(DomainUpgradeMode.Perform, ignoreRuleCollection, model5Types).Dispose()); + if (noExceptionOnIndexIncludedColumnDrop) { + BuildDomain(DomainUpgradeMode.Perform, ignoreRuleCollection, model6Types).Dispose(); + } + else { + _ = Assert.Throws( + () => BuildDomain(DomainUpgradeMode.Perform, ignoreRuleCollection, model5Types).Dispose()); + } } [Test] From 8c8f6080eb19db8e1c03a77be727d7bb6ec8cbc1 Mon Sep 17 00:00:00 2001 From: Alexey Kulakov Date: Thu, 30 Oct 2025 15:14:35 +0500 Subject: [PATCH 7/7] Improved changelog --- ChangeLog/7.2.0-RC-dev.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/ChangeLog/7.2.0-RC-dev.txt b/ChangeLog/7.2.0-RC-dev.txt index e69de29bb2..eac5c3c135 100644 --- a/ChangeLog/7.2.0-RC-dev.txt +++ b/ChangeLog/7.2.0-RC-dev.txt @@ -0,0 +1 @@ +[postgresql] Support for included columns for indexes starting from PostgreSQL 12 \ No newline at end of file