From 3b5268e1470bf374cf2635c7d958c0e79c6f7b85 Mon Sep 17 00:00:00 2001 From: Vitaliy Filippov Date: Tue, 30 Sep 2025 01:45:44 +0300 Subject: [PATCH 1/6] Deduplicate sequence copying code (#25991) --- ...hard__operation_consistent_copy_tables.cpp | 70 +++++++++++-------- .../schemeshard__operation_copy_table.cpp | 36 +--------- .../schemeshard/schemeshard__operation_part.h | 8 +-- 3 files changed, 45 insertions(+), 69 deletions(-) diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_consistent_copy_tables.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_consistent_copy_tables.cpp index adc9ba3ead1b..4d6ecd72bbe3 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_consistent_copy_tables.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_consistent_copy_tables.cpp @@ -138,24 +138,7 @@ bool CreateConsistentCopyTables( TPath dstPath = TPath::Resolve(dstStr, context.SS); TPath dstParentPath = dstPath.Parent(); - THashSet sequences; - for (const auto& child: srcPath.Base()->GetChildren()) { - auto name = child.first; - auto pathId = child.second; - - TPath childPath = srcPath.Child(name); - if (!childPath.IsSequence() || childPath.IsDeleted()) { - continue; - } - - Y_ABORT_UNLESS(childPath.Base()->PathId == pathId); - - TSequenceInfo::TPtr sequenceInfo = context.SS->Sequences.at(pathId); - const auto& sequenceDesc = sequenceInfo->Description; - const auto& sequenceName = sequenceDesc.GetName(); - - sequences.emplace(sequenceName); - } + THashSet sequences = GetLocalSequences(context, srcPath); if (descr.HasTargetPathTargetState()) { result.push_back(CreateCopyTable( @@ -170,7 +153,6 @@ bool CreateConsistentCopyTables( sequences)); } - TVector sequenceDescriptions; for (const auto& child: srcPath.Base()->GetChildren()) { const auto& name = child.first; const auto& pathId = child.second; @@ -183,9 +165,6 @@ bool CreateConsistentCopyTables( } if (srcIndexPath.IsSequence()) { - TSequenceInfo::TPtr sequenceInfo = context.SS->Sequences.at(pathId); - const auto& sequenceDesc = sequenceInfo->Description; - sequenceDescriptions.push_back(sequenceDesc); continue; } @@ -218,21 +197,50 @@ bool CreateConsistentCopyTables( } } - for (auto&& sequenceDescription : sequenceDescriptions) { - auto scheme = TransactionTemplate( - dstPath.PathString(), - NKikimrSchemeOp::EOperationType::ESchemeOpCreateSequence); - scheme.SetFailOnExist(true); + AddCopySequences(nextId, tx, context, result, srcPath, dstPath.PathString()); + } + + return true; +} + +THashSet GetLocalSequences(TOperationContext& context, const TPath& srcPath) { + THashSet sequences; + for (const auto& [name, pathId] : srcPath.Base()->GetChildren()) { + TPath childPath = srcPath.Child(name); + if (!childPath.IsSequence() || childPath.IsDeleted()) { + continue; + } + + Y_ABORT_UNLESS(childPath.Base()->PathId == pathId); + + TSequenceInfo::TPtr sequenceInfo = context.SS->Sequences.at(pathId); + const auto& sequenceDesc = sequenceInfo->Description; + const auto& sequenceName = sequenceDesc.GetName(); + + sequences.emplace(sequenceName); + } + return sequences; +} + +void AddCopySequences(TOperationId nextId, const TTxTransaction& tx, TOperationContext& context, + TVector& result, const TPath& srcTable, const TString& dstPath) +{ + for (const auto& [subName, subPathId] : srcTable.Base()->GetChildren()) { + TPath subPath = srcTable.Child(subName); + if (subPath.IsSequence() && !subPath.IsDeleted()) { + TSequenceInfo::TPtr sequenceInfo = context.SS->Sequences.at(subPathId); + const auto& sequenceDesc = sequenceInfo->Description; + + auto scheme = TransactionTemplate(dstPath, NKikimrSchemeOp::EOperationType::ESchemeOpCreateSequence); + scheme.SetFailOnExist(tx.GetFailOnExist()); auto* copySequence = scheme.MutableCopySequence(); - copySequence->SetCopyFrom(srcPath.PathString() + "/" + sequenceDescription.GetName()); - *scheme.MutableSequence() = std::move(sequenceDescription); + copySequence->SetCopyFrom(subPath.PathString()); + *scheme.MutableSequence() = sequenceDesc; result.push_back(CreateCopySequence(NextPartId(nextId, result), scheme)); } } - - return true; } TVector CreateConsistentCopyTables(TOperationId nextId, const TTxTransaction& tx, TOperationContext& context) { diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_copy_table.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_copy_table.cpp index 84aaffdf940a..8e5ac40fb6d9 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_copy_table.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_copy_table.cpp @@ -777,24 +777,7 @@ TVector CreateCopyTable(TOperationId nextId, const TTxTrans } } - THashSet sequences; - for (auto& child: srcPath.Base()->GetChildren()) { - auto name = child.first; - auto pathId = child.second; - - TPath childPath = srcPath.Child(name); - if (!childPath.IsSequence() || childPath.IsDeleted()) { - continue; - } - - Y_ABORT_UNLESS(childPath.Base()->PathId == pathId); - - TSequenceInfo::TPtr sequenceInfo = context.SS->Sequences.at(pathId); - const auto& sequenceDesc = sequenceInfo->Description; - const auto& sequenceName = sequenceDesc.GetName(); - - sequences.emplace(sequenceName); - } + THashSet sequences = GetLocalSequences(context, srcPath); TPath workDir = TPath::Resolve(tx.GetWorkingDir(), context.SS); TPath dstPath = workDir.Child(copying.GetName()); @@ -817,7 +800,6 @@ TVector CreateCopyTable(TOperationId nextId, const TTxTrans result.push_back(CreateCopyTable(NextPartId(nextId, result), schema, sequences)); } - TVector sequenceDescriptions; for (auto& child: srcPath.Base()->GetChildren()) { auto name = child.first; auto pathId = child.second; @@ -828,9 +810,6 @@ TVector CreateCopyTable(TOperationId nextId, const TTxTrans } if (childPath.IsSequence()) { - TSequenceInfo::TPtr sequenceInfo = context.SS->Sequences.at(pathId); - const auto& sequenceDesc = sequenceInfo->Description; - sequenceDescriptions.push_back(sequenceDesc); continue; } @@ -884,18 +863,7 @@ TVector CreateCopyTable(TOperationId nextId, const TTxTrans } } - for (auto&& sequenceDescription : sequenceDescriptions) { - auto scheme = TransactionTemplate( - tx.GetWorkingDir() + "/" + copying.GetName(), - NKikimrSchemeOp::EOperationType::ESchemeOpCreateSequence); - scheme.SetFailOnExist(tx.GetFailOnExist()); - - auto* copySequence = scheme.MutableCopySequence(); - copySequence->SetCopyFrom(copying.GetCopyFromTable() + "/" + sequenceDescription.GetName()); - *scheme.MutableSequence() = std::move(sequenceDescription); - - result.push_back(CreateCopySequence(NextPartId(nextId, result), scheme)); - } + AddCopySequences(nextId, tx, context, result, srcPath, dstPath.PathString()); return result; } diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_part.h b/ydb/core/tx/schemeshard/schemeshard__operation_part.h index fc62bebdf82c..a680b5fa9633 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_part.h +++ b/ydb/core/tx/schemeshard/schemeshard__operation_part.h @@ -493,11 +493,11 @@ ISubOperation::TPtr CreateDropTableIndex(TOperationId id, TTxState::ETxState sta ISubOperation::TPtr CreateAlterTableIndex(TOperationId id, const TTxTransaction& tx); ISubOperation::TPtr CreateAlterTableIndex(TOperationId id, TTxState::ETxState state); -bool CreateConsistentCopyTables( - TOperationId nextId, - const TTxTransaction& tx, - TOperationContext& context, +bool CreateConsistentCopyTables(TOperationId nextId, const TTxTransaction& tx, TOperationContext& context, TVector& result); +THashSet GetLocalSequences(TOperationContext& context, const TPath& srcTable); +void AddCopySequences(TOperationId nextId, const TTxTransaction& tx, TOperationContext& context, + TVector& result, const TPath& srcTable, const TString& dstPath); TVector CreateConsistentCopyTables(TOperationId nextId, const TTxTransaction& tx, TOperationContext& context); ISubOperation::TPtr CreateNewOlapStore(TOperationId id, const TTxTransaction& tx); From f6b070498ef69556e765976038aa3e7117bbc290 Mon Sep 17 00:00:00 2001 From: Vitaliy Filippov Date: Tue, 30 Sep 2025 18:21:29 +0300 Subject: [PATCH 2/6] Create a sequence for prefixed vector indexes (#26046) --- ydb/core/base/table_index.h | 1 + .../kqp_indexes_prefixed_vector_ut.cpp | 6 ++ .../kqp/ut/indexes/kqp_indexes_vector_ut.cpp | 6 ++ .../schemeshard__operation_alter_sequence.cpp | 16 +++- ...hard__operation_consistent_copy_tables.cpp | 6 +- .../schemeshard__operation_copy_sequence.cpp | 15 ++- .../schemeshard__operation_copy_table.cpp | 7 +- ...emeshard__operation_create_build_index.cpp | 12 ++- ...eshard__operation_create_indexed_table.cpp | 21 ++++- ...schemeshard__operation_create_sequence.cpp | 21 ++++- .../schemeshard__operation_create_table.cpp | 6 +- .../schemeshard__operation_drop_sequence.cpp | 18 +++- .../schemeshard/schemeshard__operation_part.h | 2 +- .../schemeshard_build_index__progress.cpp | 91 ++++++++++++++++--- .../schemeshard_build_index_tx_base.cpp | 1 + .../tx/schemeshard/schemeshard_info_types.h | 3 + ydb/core/tx/schemeshard/schemeshard_utils.cpp | 1 + 17 files changed, 189 insertions(+), 44 deletions(-) diff --git a/ydb/core/base/table_index.h b/ydb/core/base/table_index.h index ffe3681f0ae2..af55b99a09b3 100644 --- a/ydb/core/base/table_index.h +++ b/ydb/core/base/table_index.h @@ -67,6 +67,7 @@ inline constexpr const char* BuildSuffix1 = "1build"; // Prefix table inline constexpr const char* PrefixTable = "indexImplPrefixTable"; +inline constexpr const char* IdColumnSequence = "__ydb_id_sequence"; inline constexpr const int DefaultKMeansRounds = 3; diff --git a/ydb/core/kqp/ut/indexes/kqp_indexes_prefixed_vector_ut.cpp b/ydb/core/kqp/ut/indexes/kqp_indexes_prefixed_vector_ut.cpp index 6d3d035482a0..8529a8b7800b 100644 --- a/ydb/core/kqp/ut/indexes/kqp_indexes_prefixed_vector_ut.cpp +++ b/ydb/core/kqp/ut/indexes/kqp_indexes_prefixed_vector_ut.cpp @@ -311,6 +311,12 @@ Y_UNIT_TEST_SUITE(KqpPrefixedVectorIndexes) { UNIT_ASSERT_EQUAL(settings.Clusters, 2); } DoPositiveQueriesPrefixedVectorIndexOrderByCosine(session); + + { + const TString dropIndex(Q_("ALTER TABLE `/Root/TestTable` DROP INDEX index")); + auto result = session.ExecuteSchemeQuery(dropIndex).ExtractValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + } } Y_UNIT_TEST_QUAD(OrderByCosineLevel2, Nullable, UseSimilarity) { diff --git a/ydb/core/kqp/ut/indexes/kqp_indexes_vector_ut.cpp b/ydb/core/kqp/ut/indexes/kqp_indexes_vector_ut.cpp index b2ba0b2657c6..a39f45b3e75b 100644 --- a/ydb/core/kqp/ut/indexes/kqp_indexes_vector_ut.cpp +++ b/ydb/core/kqp/ut/indexes/kqp_indexes_vector_ut.cpp @@ -297,6 +297,12 @@ Y_UNIT_TEST_SUITE(KqpVectorIndexes) { UNIT_ASSERT_EQUAL(settings.Clusters, 2); } DoPositiveQueriesVectorIndexOrderByCosine(session); + + { + const TString dropIndex(Q_("ALTER TABLE `/Root/TestTable` DROP INDEX index")); + auto result = session.ExecuteSchemeQuery(dropIndex).ExtractValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + } } Y_UNIT_TEST_QUAD(OrderByCosineLevel2, Nullable, UseSimilarity) { diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_alter_sequence.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_alter_sequence.cpp index a2d8225c30d9..943b6118b610 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_alter_sequence.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_alter_sequence.cpp @@ -429,18 +429,26 @@ class TAlterSequence: public TSubOperation { .IsAtLocalSchemeShard() .IsResolved() .NotDeleted() - .NotUnderDeleting() - .IsCommonSensePath(); + .NotUnderDeleting(); if (checks) { - if (parentPath->IsTable()) { + if (parentPath.Parent()->IsTableIndex()) { + checks.IsInsideTableIndexPath(); + // Only __ydb_id sequence can be altered and only by internal transactions (build_index__progress) + if (name != NTableIndex::NKMeans::IdColumnSequence || !Transaction.GetInternal()) { + result->SetError(NKikimrScheme::EStatus::StatusNameConflict, "sequences are not allowed in indexes"); + return result; + } + } else if (parentPath->IsTable()) { // allow immediately inside a normal table + checks.IsCommonSensePath(); if (parentPath.IsUnderOperation()) { checks.IsUnderTheSameOperation(OperationId.GetTxId()); // allowed only as part of consistent operations } } else { // otherwise don't allow unexpected object types - checks.IsLikeDirectory(); + checks.IsCommonSensePath() + .IsLikeDirectory(); } } diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_consistent_copy_tables.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_consistent_copy_tables.cpp index 4d6ecd72bbe3..264b055ded6f 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_consistent_copy_tables.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_consistent_copy_tables.cpp @@ -191,9 +191,9 @@ bool CreateConsistentCopyTables( Y_ABORT_UNLESS(srcImplTable.Base()->PathId == srcImplTablePathId); TPath dstImplTable = dstIndexPath.Child(srcImplTableName); - result.push_back(CreateCopyTable( - NextPartId(nextId, result), - CopyTableTask(srcImplTable, dstImplTable, descr))); + result.push_back(CreateCopyTable(NextPartId(nextId, result), + CopyTableTask(srcImplTable, dstImplTable, descr), GetLocalSequences(context, srcImplTable))); + AddCopySequences(nextId, tx, context, result, srcImplTable, dstImplTable.PathString()); } } diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_copy_sequence.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_copy_sequence.cpp index f4eb2c8f7667..995c2f9d29c1 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_copy_sequence.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_copy_sequence.cpp @@ -551,18 +551,27 @@ class TCopySequence: public TSubOperation { .IsResolved() .NotDeleted() .NotUnderDeleting() - .IsCommonSensePath() .FailOnRestrictedCreateInTempZone(Transaction.GetAllowCreateInTempDir()); if (checks) { - if (parentPath->IsTable()) { + if (parentPath.Parent()->IsTableIndex()) { + // Only __ydb_id sequence can be created in the prefixed index + if (name != NTableIndex::NKMeans::IdColumnSequence) { + result->SetError(NKikimrScheme::EStatus::StatusNameConflict, "sequences are not allowed in indexes"); + return result; + } + checks.IsUnderTheSameOperation(OperationId.GetTxId()); // allowed only as part of consistent operations + checks.IsInsideTableIndexPath(); + } else if (parentPath->IsTable()) { // allow immediately inside a normal table + checks.IsCommonSensePath(); if (parentPath.IsUnderOperation()) { checks.IsUnderTheSameOperation(OperationId.GetTxId()); // allowed only as part of consistent operations } } else { // otherwise don't allow unexpected object types - checks.IsLikeDirectory(); + checks.IsCommonSensePath() + .IsLikeDirectory(); } } diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_copy_table.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_copy_table.cpp index 8e5ac40fb6d9..82683cc75f5f 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_copy_table.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_copy_table.cpp @@ -399,9 +399,7 @@ class TCopyTable: public TSubOperation { if (checks) { if (parent.Base()->IsTableIndex()) { - checks - .IsInsideTableIndexPath() //copy imp index table as index index table, not a separate one - .NotChildren(); //imp table doesn't have indexes + checks.IsInsideTableIndexPath(); //copy imp index table as index index table, not a separate one } else { checks.IsCommonSensePath(); } @@ -859,7 +857,8 @@ TVector CreateCopyTable(TOperationId nextId, const TTxTrans operation->SetOmitFollowers(copying.GetOmitFollowers()); operation->SetIsBackup(copying.GetIsBackup()); - result.push_back(CreateCopyTable(NextPartId(nextId, result), schema)); + result.push_back(CreateCopyTable(NextPartId(nextId, result), schema, GetLocalSequences(context, implTable))); + AddCopySequences(nextId, tx, context, result, implTable, JoinPath({dstPath.PathString(), name, implTableName})); } } diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_create_build_index.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_create_build_index.cpp index a135183dfbcf..6879d4c21a97 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_create_build_index.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_create_build_index.cpp @@ -117,7 +117,7 @@ TVector CreateBuildIndex(TOperationId opId, const TTxTransa result.push_back(CreateInitializeBuildIndexMainTable(NextPartId(opId, result), outTx)); } - auto createImplTable = [&](NKikimrSchemeOp::TTableDescription&& implTableDesc) { + auto createImplTable = [&](NKikimrSchemeOp::TTableDescription&& implTableDesc, const THashSet& localSequences = {}) { if (indexType != NKikimrSchemeOp::EIndexTypeGlobalUnique) { implTableDesc.MutablePartitionConfig()->SetShadowData(true); } @@ -126,7 +126,7 @@ TVector CreateBuildIndex(TOperationId opId, const TTxTransa *outTx.MutableCreateTable() = std::move(implTableDesc); outTx.SetInternal(tx.GetInternal()); - return CreateInitializeBuildIndexImplTable(NextPartId(opId, result), outTx); + return CreateInitializeBuildIndexImplTable(NextPartId(opId, result), outTx, localSequences); }; if (indexDesc.GetType() == NKikimrSchemeOp::EIndexType::EIndexTypeGlobalVectorKmeansTree) { @@ -145,7 +145,13 @@ TVector CreateBuildIndex(TOperationId opId, const TTxTransa result.push_back(createImplTable(CalcVectorKmeansTreePostingImplTableDesc(tableInfo, tableInfo->PartitionConfig(), indexDataColumns, indexPostingTableDesc))); if (prefixVectorIndex) { const THashSet prefixColumns{indexDesc.GetKeyColumnNames().begin(), indexDesc.GetKeyColumnNames().end() - 1}; - result.push_back(createImplTable(CalcVectorKmeansTreePrefixImplTableDesc(prefixColumns, tableInfo, tableInfo->PartitionConfig(), implTableColumns, indexPrefixTableDesc))); + result.push_back(createImplTable(CalcVectorKmeansTreePrefixImplTableDesc( + prefixColumns, tableInfo, tableInfo->PartitionConfig(), implTableColumns, indexPrefixTableDesc), + THashSet{NTableIndex::NKMeans::IdColumnSequence})); + auto outTx = TransactionTemplate(index.PathString() + "/" + NTableIndex::NKMeans::PrefixTable, NKikimrSchemeOp::EOperationType::ESchemeOpCreateSequence); + outTx.MutableSequence()->SetName(NTableIndex::NKMeans::IdColumnSequence); + outTx.SetInternal(tx.GetInternal()); + result.push_back(CreateNewSequence(NextPartId(opId, result), outTx)); } } else { NKikimrSchemeOp::TTableDescription indexTableDesc; diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_create_indexed_table.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_create_indexed_table.cpp index ea53e715266e..41d3fbac56af 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_create_indexed_table.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_create_indexed_table.cpp @@ -284,7 +284,7 @@ TVector CreateIndexedTable(TOperationId nextId, const TTxTr result.push_back(CreateNewTableIndex(NextPartId(nextId, result), scheme)); } - auto createIndexImplTable = [&] (NKikimrSchemeOp::TTableDescription&& implTableDesc) { + auto createIndexImplTable = [&] (NKikimrSchemeOp::TTableDescription&& implTableDesc, const THashSet& localSequences = {}) { auto scheme = TransactionTemplate( tx.GetWorkingDir() + "/" + baseTableDescription.GetName() + "/" + indexDescription.GetName(), NKikimrSchemeOp::EOperationType::ESchemeOpCreateTable); @@ -294,7 +294,7 @@ TVector CreateIndexedTable(TOperationId nextId, const TTxTr *scheme.MutableCreateTable() = std::move(implTableDesc); - return CreateNewTable(NextPartId(nextId, result), scheme); + return CreateNewTable(NextPartId(nextId, result), scheme, localSequences); }; const auto& implTableColumns = indexes.at(indexDescription.GetName()); @@ -314,7 +314,22 @@ TVector CreateIndexedTable(TOperationId nextId, const TTxTr result.push_back(createIndexImplTable(CalcVectorKmeansTreePostingImplTableDesc(baseTableDescription, baseTableDescription.GetPartitionConfig(), indexDataColumns, userPostingDesc))); if (prefixVectorIndex) { const THashSet prefixColumns{indexDescription.GetKeyColumnNames().begin(), indexDescription.GetKeyColumnNames().end() - 1}; - result.push_back(createIndexImplTable(CalcVectorKmeansTreePrefixImplTableDesc(prefixColumns, baseTableDescription, baseTableDescription.GetPartitionConfig(), implTableColumns, userPrefixDesc))); + result.push_back(createIndexImplTable(CalcVectorKmeansTreePrefixImplTableDesc( + prefixColumns, baseTableDescription, baseTableDescription.GetPartitionConfig(), implTableColumns, userPrefixDesc), + THashSet{NTableIndex::NKMeans::IdColumnSequence})); + // Create the sequence + auto outTx = TransactionTemplate(tx.GetWorkingDir() + "/" + baseTableDescription.GetName() + "/" + + indexDescription.GetName() + "/" + NTableIndex::NKMeans::PrefixTable, + NKikimrSchemeOp::EOperationType::ESchemeOpCreateSequence); + outTx.MutableSequence()->SetName(NTableIndex::NKMeans::IdColumnSequence); + outTx.MutableSequence()->SetMinValue(-0x7FFFFFFFFFFFFFFF); + outTx.MutableSequence()->SetMaxValue(-1); + outTx.MutableSequence()->SetStartValue(NTableIndex::NKMeans::SetPostingParentFlag(1)); + outTx.MutableSequence()->SetRestart(true); + outTx.SetFailOnExist(tx.GetFailOnExist()); + outTx.SetAllowCreateInTempDir(tx.GetAllowCreateInTempDir()); + outTx.SetInternal(tx.GetInternal()); + result.push_back(CreateNewSequence(NextPartId(nextId, result), outTx)); } } else { NKikimrSchemeOp::TTableDescription userIndexDesc; diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_create_sequence.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_create_sequence.cpp index 4240188c541a..893fb71352d7 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_create_sequence.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_create_sequence.cpp @@ -389,19 +389,30 @@ class TCreateSequence : public TSubOperation { .IsResolved() .NotDeleted() .NotUnderDeleting() - .IsCommonSensePath() .FailOnRestrictedCreateInTempZone(Transaction.GetAllowCreateInTempDir()); if (checks) { - if (parentPath->IsTable()) { - checks.NotBackupTable(); + if (parentPath.Parent()->IsTableIndex()) { + // Only __ydb_id sequence can be created in the prefixed index + if (name != NTableIndex::NKMeans::IdColumnSequence) { + result->SetError(NKikimrScheme::EStatus::StatusNameConflict, "sequences are not allowed in indexes"); + return result; + } + checks.IsInsideTableIndexPath() + .IsUnderCreating() + .IsUnderTheSameOperation(OperationId.GetTxId()); // allowed only as part of consistent operations + } else if (parentPath->IsTable()) { + checks + .IsCommonSensePath() + .NotBackupTable(); // allow immediately inside a normal table if (parentPath.IsUnderOperation()) { checks.IsUnderTheSameOperation(OperationId.GetTxId()); // allowed only as part of consistent operations } - } else { + } else if (!Transaction.GetAllowAccessToPrivatePaths()) { // otherwise don't allow unexpected object types - checks.IsLikeDirectory(); + checks.IsCommonSensePath() + .IsLikeDirectory(); } } diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_create_table.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_create_table.cpp index 761d1803f40c..a74417a33085 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_create_table.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_create_table.cpp @@ -827,9 +827,11 @@ ISubOperation::TPtr CreateNewTable(TOperationId id, TTxState::ETxState state) { return MakeSubOperation(id, state); } -ISubOperation::TPtr CreateInitializeBuildIndexImplTable(TOperationId id, const TTxTransaction& tx) { +ISubOperation::TPtr CreateInitializeBuildIndexImplTable(TOperationId id, const TTxTransaction& tx, const THashSet& localSequences) { auto obj = MakeSubOperation(id, tx); - static_cast(obj.Get())->SetAllowShadowDataForBuildIndex(); + TCreateTable *createTable = static_cast(obj.Get()); + createTable->SetAllowShadowDataForBuildIndex(); + createTable->SetLocalSequences(localSequences); return obj; } diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_drop_sequence.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_drop_sequence.cpp index f67b2a397113..84471665f068 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_drop_sequence.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_drop_sequence.cpp @@ -291,19 +291,29 @@ class TDropSequence: public TSubOperation { .NotUnderDomainUpgrade() .IsAtLocalSchemeShard() .IsResolved() - .NotDeleted() - .IsCommonSensePath(); + .NotDeleted(); if (checks) { - if (parent->IsTable()) { + if (parent.Parent()->IsTableIndex()) { + // Only __ydb_id sequence can be present in the prefixed index + if (name != NTableIndex::NKMeans::IdColumnSequence) { + result->SetError(NKikimrScheme::EStatus::StatusNameConflict, "sequences are not allowed in indexes"); + return result; + } + checks.IsInsideTableIndexPath() + .IsUnderDeleting() + .IsUnderTheSameOperation(OperationId.GetTxId()); // allowed only as part of consistent operations + } else if (parent->IsTable()) { // allow immediately inside a normal table if (parent.IsUnderOperation()) { checks.IsUnderTheSameOperation(OperationId.GetTxId()); // allowed only as part of consistent operations } + checks.IsCommonSensePath(); } else { checks .NotUnderDeleting() - .IsLikeDirectory(); + .IsLikeDirectory() + .IsCommonSensePath(); } } diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_part.h b/ydb/core/tx/schemeshard/schemeshard__operation_part.h index a680b5fa9633..735ad3ea535d 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_part.h +++ b/ydb/core/tx/schemeshard/schemeshard__operation_part.h @@ -604,7 +604,7 @@ ISubOperation::TPtr CreateDropSolomon(TOperationId id, TTxState::ETxState state) ISubOperation::TPtr CreateInitializeBuildIndexMainTable(TOperationId id, const TTxTransaction& tx); ISubOperation::TPtr CreateInitializeBuildIndexMainTable(TOperationId id, TTxState::ETxState state); -ISubOperation::TPtr CreateInitializeBuildIndexImplTable(TOperationId id, const TTxTransaction& tx); +ISubOperation::TPtr CreateInitializeBuildIndexImplTable(TOperationId id, const TTxTransaction& tx, const THashSet& localSequences = {}); ISubOperation::TPtr CreateInitializeBuildIndexImplTable(TOperationId id, TTxState::ETxState state); ISubOperation::TPtr CreateFinalizeBuildIndexImplTable(TOperationId id, const TTxTransaction& tx); diff --git a/ydb/core/tx/schemeshard/schemeshard_build_index__progress.cpp b/ydb/core/tx/schemeshard/schemeshard_build_index__progress.cpp index 300b8d3d0200..7c4397284fed 100644 --- a/ydb/core/tx/schemeshard/schemeshard_build_index__progress.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_build_index__progress.cpp @@ -256,7 +256,7 @@ THolder LockPropose( modifyScheme.MutableLockConfig()->SetName(path.LeafName()); modifyScheme.MutableLockConfig()->SetLockTxId(ui64(buildInfo.LockTxId)); - LOG_DEBUG_S((TlsActivationContext->AsActorContext()), NKikimrServices::BUILD_INDEX, + LOG_NOTICE_S((TlsActivationContext->AsActorContext()), NKikimrServices::BUILD_INDEX, "LockPropose " << buildInfo.Id << " " << buildInfo.State << " " << propose->Record.ShortDebugString()); return propose; @@ -283,7 +283,7 @@ THolder CreateIndexPropose( Y_ENSURE(false, "Unknown operation kind while building CreateIndexPropose"); } - LOG_DEBUG_S((TlsActivationContext->AsActorContext()), NKikimrServices::BUILD_INDEX, + LOG_NOTICE_S((TlsActivationContext->AsActorContext()), NKikimrServices::BUILD_INDEX, "CreateIndexPropose " << buildInfo.Id << " " << buildInfo.State << " " << propose->Record.ShortDebugString()); return propose; @@ -311,7 +311,7 @@ THolder DropBuildPropose( modifyScheme.SetOperationType(NKikimrSchemeOp::ESchemeOpDropTable); modifyScheme.MutableDrop()->SetName(path->Name); - LOG_DEBUG_S((TlsActivationContext->AsActorContext()), NKikimrServices::BUILD_INDEX, + LOG_NOTICE_S((TlsActivationContext->AsActorContext()), NKikimrServices::BUILD_INDEX, "DropBuildPropose " << buildInfo.Id << " " << buildInfo.State << " " << propose->Record.ShortDebugString()); return propose; @@ -373,7 +373,7 @@ THolder CreateBuildPropose( policy.SetMinPartitionsCount(32768); policy.SetMaxPartitionsCount(0); - LOG_DEBUG_S((TlsActivationContext->AsActorContext()), NKikimrServices::BUILD_INDEX, + LOG_NOTICE_S((TlsActivationContext->AsActorContext()), NKikimrServices::BUILD_INDEX, "CreateBuildPropose " << buildInfo.Id << " " << buildInfo.State << " " << propose->Record.ShortDebugString()); return propose; @@ -398,7 +398,7 @@ THolder CreateBuildPropose( policy.SetMaxPartitionsCount(0); } - LOG_DEBUG_S((TlsActivationContext->AsActorContext()), NKikimrServices::BUILD_INDEX, + LOG_NOTICE_S((TlsActivationContext->AsActorContext()), NKikimrServices::BUILD_INDEX, "CreateBuildPropose " << buildInfo.Id << " " << buildInfo.State << " " << propose->Record.ShortDebugString()); return propose; @@ -446,7 +446,7 @@ THolder AlterMainTablePropose( } - LOG_DEBUG_S((TlsActivationContext->AsActorContext()), NKikimrServices::BUILD_INDEX, + LOG_NOTICE_S((TlsActivationContext->AsActorContext()), NKikimrServices::BUILD_INDEX, "AlterMainTablePropose " << buildInfo.Id << " " << buildInfo.State << " " << propose->Record.ShortDebugString()); return propose; @@ -474,7 +474,7 @@ THolder ApplyPropose( indexBuild.SetSnapshotTxId(ui64(buildInfo.InitiateTxId)); indexBuild.SetBuildIndexId(ui64(buildInfo.Id)); - LOG_DEBUG_S((TlsActivationContext->AsActorContext()), NKikimrServices::BUILD_INDEX, + LOG_NOTICE_S((TlsActivationContext->AsActorContext()), NKikimrServices::BUILD_INDEX, "ApplyPropose " << buildInfo.Id << " " << buildInfo.State << " " << propose->Record.ShortDebugString()); return propose; @@ -508,7 +508,7 @@ THolder UnlockPropose( } } - LOG_DEBUG_S((TlsActivationContext->AsActorContext()), NKikimrServices::BUILD_INDEX, + LOG_NOTICE_S((TlsActivationContext->AsActorContext()), NKikimrServices::BUILD_INDEX, "UnlockPropose " << buildInfo.Id << " " << buildInfo.State << " " << propose->Record.ShortDebugString()); return propose; @@ -532,7 +532,7 @@ THolder CancelPropose( indexBuild.SetSnapshotTxId(ui64(buildInfo.InitiateTxId)); indexBuild.SetBuildIndexId(ui64(buildInfo.Id)); - LOG_DEBUG_S((TlsActivationContext->AsActorContext()), NKikimrServices::BUILD_INDEX, + LOG_NOTICE_S((TlsActivationContext->AsActorContext()), NKikimrServices::BUILD_INDEX, "CancelPropose " << buildInfo.Id << " " << buildInfo.State << " " << propose->Record.ShortDebugString()); return propose; @@ -559,12 +559,44 @@ THolder DropColumnsPropose( buildInfo.SerializeToProto(ss, settings); - LOG_DEBUG_S((TlsActivationContext->AsActorContext()), NKikimrServices::BUILD_INDEX, + LOG_NOTICE_S((TlsActivationContext->AsActorContext()), NKikimrServices::BUILD_INDEX, "DropColumnsPropose " << buildInfo.Id << " " << buildInfo.State << " " << propose->Record.ShortDebugString()); return propose; } +THolder AlterSequencePropose( + TSchemeShard* ss, const TIndexBuildInfo& buildInfo) +{ + Y_ENSURE(buildInfo.IsBuildPrefixedVectorIndex(), "Unknown operation kind while building AlterSequencePropose"); + + auto propose = MakeHolder(ui64(buildInfo.ApplyTxId), ss->TabletID()); + + NKikimrSchemeOp::TModifyScheme& modifyScheme = *propose->Record.AddTransaction(); + modifyScheme.SetOperationType(NKikimrSchemeOp::ESchemeOpAlterSequence); + modifyScheme.SetInternal(true); + + auto path = TPath::Init(buildInfo.TablePathId, ss); + path.Dive(buildInfo.IndexName); + path.Dive(NTableIndex::NKMeans::PrefixTable); + modifyScheme.SetWorkingDir(path.PathString()); + + // about 2 * TableSize per each prefix, see PrefixIndexDone and SendPrefixKMeansRequest() + ui64 minValue = NTableIndex::NKMeans::SetPostingParentFlag(buildInfo.KMeans.ChildBegin + (2 * buildInfo.KMeans.TableSize) * buildInfo.Shards.size()); + + auto seq = modifyScheme.MutableSequence(); + seq->SetName(NTableIndex::NKMeans::IdColumnSequence); + seq->SetMinValue(-0x7FFFFFFFFFFFFFFF); + seq->SetMaxValue(-1); + seq->SetStartValue(minValue); + seq->SetRestart(true); + + LOG_NOTICE_S((TlsActivationContext->AsActorContext()), NKikimrServices::BUILD_INDEX, + "AlterSequencePropose " << buildInfo.Id << " " << buildInfo.State << " " << propose->Record.ShortDebugString()); + + return propose; +} + using namespace NTabletFlatExecutor; struct TSchemeShard::TIndexBuilder::TTxProgress: public TSchemeShard::TIndexBuilder::TTxBase { @@ -1044,7 +1076,7 @@ struct TSchemeShard::TIndexBuilder::TTxProgress: public TSchemeShard::TIndexBuil } } - bool FillPrefixKMeans(TIndexBuildInfo& buildInfo) { + bool FillPrefixKMeans(TTransactionContext& txc, TIndexBuildInfo& buildInfo) { if (NoShardsAdded(buildInfo)) { AddAllShards(buildInfo); } @@ -1052,6 +1084,15 @@ struct TSchemeShard::TIndexBuilder::TTxProgress: public TSchemeShard::TIndexBuil for (auto& [shardIdx, shardStatus]: buildInfo.Shards) { shardStatus.Index = i++; } + // Set correct start value for the prefix ID sequence + if (!buildInfo.KMeans.AlterPrefixSequenceDone) { + // Alter the sequence + buildInfo.KMeans.AlterPrefixSequenceDone = true; + NIceDb::TNiceDb db{txc.DB}; + ChangeState(BuildId, TIndexBuildInfo::EState::AlterSequence); + Progress(BuildId); + return false; + } return SendToShards(buildInfo, [&](TShardIdx shardIdx) { SendPrefixKMeansRequest(shardIdx, buildInfo); }) && buildInfo.DoneShards.size() == buildInfo.Shards.size(); } @@ -1129,7 +1170,7 @@ struct TSchemeShard::TIndexBuilder::TTxProgress: public TSchemeShard::TIndexBuil return false; } else { bool filled = buildInfo.KMeans.Level == 2 - ? FillPrefixKMeans(buildInfo) + ? FillPrefixKMeans(txc, buildInfo) : FillLocalKMeans(buildInfo); if (!filled) { return false; @@ -1563,6 +1604,29 @@ struct TSchemeShard::TIndexBuilder::TTxProgress: public TSchemeShard::TIndexBuil Progress(BuildId); } break; + case TIndexBuildInfo::EState::AlterSequence: + Y_ENSURE(buildInfo.IsBuildPrefixedVectorIndex()); + Y_ENSURE(buildInfo.KMeans.Level == 2); + if (buildInfo.ApplyTxId == InvalidTxId) { + AllocateTxId(BuildId); + } else if (buildInfo.ApplyTxStatus == NKikimrScheme::StatusSuccess) { + Send(Self->SelfId(), AlterSequencePropose(Self, buildInfo), 0, ui64(BuildId)); + } else if (!buildInfo.ApplyTxDone) { + Send(Self->SelfId(), MakeHolder(ui64(buildInfo.ApplyTxId))); + } else { + buildInfo.ApplyTxId = {}; + buildInfo.ApplyTxStatus = NKikimrScheme::StatusSuccess; + buildInfo.ApplyTxDone = false; + + NIceDb::TNiceDb db(txc.DB); + Self->PersistBuildIndexApplyTxId(db, buildInfo); + Self->PersistBuildIndexApplyTxStatus(db, buildInfo); + Self->PersistBuildIndexApplyTxDone(db, buildInfo); + + ChangeState(BuildId, TIndexBuildInfo::EState::Filling); + Progress(BuildId); + } + break; case TIndexBuildInfo::EState::Applying: if (buildInfo.ApplyTxId == InvalidTxId) { AllocateTxId(BuildId); @@ -2376,6 +2440,7 @@ struct TSchemeShard::TIndexBuilder::TTxReplyCompleted: public TSchemeShard::TInd case TIndexBuildInfo::EState::DropBuild: case TIndexBuildInfo::EState::CreateBuild: case TIndexBuildInfo::EState::LockBuild: + case TIndexBuildInfo::EState::AlterSequence: case TIndexBuildInfo::EState::Applying: case TIndexBuildInfo::EState::Cancellation_Applying: case TIndexBuildInfo::EState::Rejection_Applying: @@ -2538,6 +2603,7 @@ struct TSchemeShard::TIndexBuilder::TTxReplyModify: public TSchemeShard::TIndexB case TIndexBuildInfo::EState::DropBuild: case TIndexBuildInfo::EState::CreateBuild: case TIndexBuildInfo::EState::LockBuild: + case TIndexBuildInfo::EState::AlterSequence: { Y_ENSURE(txId == buildInfo.ApplyTxId); @@ -2685,6 +2751,7 @@ struct TSchemeShard::TIndexBuilder::TTxReplyAllocate: public TSchemeShard::TInde case TIndexBuildInfo::EState::DropBuild: case TIndexBuildInfo::EState::CreateBuild: case TIndexBuildInfo::EState::LockBuild: + case TIndexBuildInfo::EState::AlterSequence: case TIndexBuildInfo::EState::Applying: case TIndexBuildInfo::EState::Cancellation_Applying: case TIndexBuildInfo::EState::Rejection_Applying: diff --git a/ydb/core/tx/schemeshard/schemeshard_build_index_tx_base.cpp b/ydb/core/tx/schemeshard/schemeshard_build_index_tx_base.cpp index 771239e89d89..fb47b456d044 100644 --- a/ydb/core/tx/schemeshard/schemeshard_build_index_tx_base.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_build_index_tx_base.cpp @@ -232,6 +232,7 @@ void TSchemeShard::TIndexBuilder::TTxBase::Fill(NKikimrIndexBuilder::TIndexBuild case TIndexBuildInfo::EState::DropBuild: case TIndexBuildInfo::EState::CreateBuild: case TIndexBuildInfo::EState::LockBuild: + case TIndexBuildInfo::EState::AlterSequence: index.SetState(Ydb::Table::IndexBuildState::STATE_TRANSFERING_DATA); index.SetProgress(indexInfo.CalcProgressPercent()); break; diff --git a/ydb/core/tx/schemeshard/schemeshard_info_types.h b/ydb/core/tx/schemeshard/schemeshard_info_types.h index 980027076feb..038ef9d19cd8 100644 --- a/ydb/core/tx/schemeshard/schemeshard_info_types.h +++ b/ydb/core/tx/schemeshard/schemeshard_info_types.h @@ -3069,6 +3069,7 @@ struct TIndexBuildInfo: public TSimpleRefCount { LockBuild = 47, Applying = 50, Unlocking = 60, + AlterSequence = 61, Done = 200, Cancellation_Applying = 350, @@ -3176,6 +3177,8 @@ struct TIndexBuildInfo: public TSimpleRefCount { EState State = Sample; + bool AlterPrefixSequenceDone = false; + NTableIndex::NKMeans::TClusterId ParentBegin = 0; // included NTableIndex::NKMeans::TClusterId Parent = ParentBegin; diff --git a/ydb/core/tx/schemeshard/schemeshard_utils.cpp b/ydb/core/tx/schemeshard/schemeshard_utils.cpp index 95cd196bdefa..2f8b70354989 100644 --- a/ydb/core/tx/schemeshard/schemeshard_utils.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_utils.cpp @@ -307,6 +307,7 @@ auto CalcVectorKmeansTreePrefixImplTableDescImpl( idColumn->SetType(NTableIndex::NKMeans::ClusterIdTypeName); idColumn->SetTypeId(NSchemeShard::ClusterIdTypeId); idColumn->SetNotNull(true); + idColumn->SetDefaultFromSequence(NKMeans::IdColumnSequence); } implTableDesc.AddKeyColumnNames(NKMeans::IdColumn); From 3ea1a69038d5dd20864836a29a8cf449dd645229 Mon Sep 17 00:00:00 2001 From: Vitaliy Filippov Date: Thu, 2 Oct 2025 10:10:23 +0300 Subject: [PATCH 3/6] Do not crash on new (unknown) build states during index build, add a test for it (#26120) --- .../tx/schemeshard/schemeshard_info_types.cpp | 53 +++++++ .../tx/schemeshard/schemeshard_info_types.h | 24 +++- .../ut_index_build/ut_vector_index_build.cpp | 132 ++++++++++++++++++ 3 files changed, 207 insertions(+), 2 deletions(-) diff --git a/ydb/core/tx/schemeshard/schemeshard_info_types.cpp b/ydb/core/tx/schemeshard/schemeshard_info_types.cpp index 6a1839330cf0..4846faddd8c8 100644 --- a/ydb/core/tx/schemeshard/schemeshard_info_types.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_info_types.cpp @@ -2530,6 +2530,59 @@ void TIndexBuildInfo::AddParent(const TSerializedTableRange& range, TShardIdx sh Cluster2Shards.emplace_hint(itFrom, parentTo, TClusterShards{.From = parentFrom, .Shards = {shard}}); } +bool TIndexBuildInfo::IsValidState(EState value) +{ + switch (value) { + case EState::Invalid: + case EState::AlterMainTable: + case EState::Locking: + case EState::GatheringStatistics: + case EState::Initiating: + case EState::Filling: + case EState::DropBuild: + case EState::CreateBuild: + case EState::LockBuild: + case EState::Applying: + case EState::Unlocking: + case EState::AlterSequence: + case EState::Done: + case EState::Cancellation_Applying: + case EState::Cancellation_Unlocking: + case EState::Cancellation_DroppingColumns: + case EState::Cancelled: + case EState::Rejection_Applying: + case EState::Rejection_Unlocking: + case EState::Rejection_DroppingColumns: + case EState::Rejected: + return true; + } + return false; +} + +bool TIndexBuildInfo::IsValidSubState(ESubState value) +{ + switch (value) { + case ESubState::None: + case ESubState::UniqIndexValidation: + return true; + } + return false; +} + +bool TIndexBuildInfo::IsValidBuildKind(EBuildKind value) +{ + switch (value) { + case EBuildKind::BuildKindUnspecified: + case EBuildKind::BuildSecondaryIndex: + case EBuildKind::BuildVectorIndex: + case EBuildKind::BuildPrefixedVectorIndex: + case EBuildKind::BuildSecondaryUniqueIndex: + case EBuildKind::BuildColumns: + return true; + } + return false; +} + TColumnFamiliesMerger::TColumnFamiliesMerger(NKikimrSchemeOp::TPartitionConfig &container) : Container(container) , DeduplicationById(TPartitionConfigMerger::DeduplicateColumnFamiliesById(Container)) diff --git a/ydb/core/tx/schemeshard/schemeshard_info_types.h b/ydb/core/tx/schemeshard/schemeshard_info_types.h index 038ef9d19cd8..b558b1a0031f 100644 --- a/ydb/core/tx/schemeshard/schemeshard_info_types.h +++ b/ydb/core/tx/schemeshard/schemeshard_info_types.h @@ -3410,6 +3410,10 @@ struct TIndexBuildInfo: public TSimpleRefCount { return result; } + static bool IsValidState(EState value); + static bool IsValidSubState(ESubState value); + static bool IsValidBuildKind(EBuildKind value); + struct TClusterShards { NTableIndex::NKMeans::TClusterId From = std::numeric_limits::max(); std::vector Shards; @@ -3467,18 +3471,34 @@ struct TIndexBuildInfo: public TSimpleRefCount { indexInfo->Id = id; indexInfo->Uid = uid; + indexInfo->Issue = + row.template GetValueOrDefault(); + indexInfo->State = TIndexBuildInfo::EState( row.template GetValue()); + if (!IsValidState(indexInfo->State)) { + indexInfo->IsBroken = true; + indexInfo->AddIssue(TStringBuilder() << "Unknown build state: " << ui32(indexInfo->State)); + indexInfo->State = TIndexBuildInfo::EState::Invalid; + } indexInfo->SubState = TIndexBuildInfo::ESubState( row.template GetValueOrDefault(ui32(TIndexBuildInfo::ESubState::None))); - indexInfo->Issue = - row.template GetValueOrDefault(); + if (!IsValidSubState(indexInfo->SubState)) { + indexInfo->IsBroken = true; + indexInfo->AddIssue(TStringBuilder() << "Unknown build sub-state: " << ui32(indexInfo->SubState)); + indexInfo->SubState = TIndexBuildInfo::ESubState::None; + } // note: please note that here we specify BuildSecondaryIndex as operation default, // because previously this table was dedicated for build secondary index operations only. indexInfo->BuildKind = TIndexBuildInfo::EBuildKind( row.template GetValueOrDefault( ui32(TIndexBuildInfo::EBuildKind::BuildSecondaryIndex))); + if (!IsValidBuildKind(indexInfo->BuildKind)) { + indexInfo->IsBroken = true; + indexInfo->AddIssue(TStringBuilder() << "Unknown build kind: " << ui32(indexInfo->BuildKind)); + indexInfo->BuildKind = TIndexBuildInfo::EBuildKind::BuildKindUnspecified; + } indexInfo->DomainPathId = TPathId(row.template GetValue(), diff --git a/ydb/core/tx/schemeshard/ut_index_build/ut_vector_index_build.cpp b/ydb/core/tx/schemeshard/ut_index_build/ut_vector_index_build.cpp index 51ec81e1a2f7..e29871d20edb 100644 --- a/ydb/core/tx/schemeshard/ut_index_build/ut_vector_index_build.cpp +++ b/ydb/core/tx/schemeshard/ut_index_build/ut_vector_index_build.cpp @@ -1579,6 +1579,138 @@ Y_UNIT_TEST_SUITE(VectorIndexBuildTest) { } } + Y_UNIT_TEST(UnknownState) { + TTestBasicRuntime runtime; + TTestEnv env(runtime); + ui64 txId = 100; + + runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_TRACE); + runtime.SetLogPriority(NKikimrServices::BUILD_INDEX, NLog::PRI_TRACE); + + TestCreateTable(runtime, ++txId, "/MyRoot", R"( + Name: "vectors" + Columns { Name: "id" Type: "Uint64" } + Columns { Name: "embedding" Type: "String" } + KeyColumnNames: [ "id" ] + )"); + env.TestWaitNotification(runtime, txId); + + NYdb::NTable::TGlobalIndexSettings globalIndexSettings; + + std::unique_ptr kmeansTreeSettings; + { + Ydb::Table::KMeansTreeSettings proto; + UNIT_ASSERT(google::protobuf::TextFormat::ParseFromString(R"( + settings { + metric: DISTANCE_COSINE + vector_type: VECTOR_TYPE_FLOAT + vector_dimension: 1024 + } + levels: 5 + clusters: 4 + )", &proto)); + using T = NYdb::NTable::TKMeansTreeSettings; + kmeansTreeSettings = std::make_unique(T::FromProto(proto)); + } + + TBlockEvents kmeansBlocker(runtime, [&](const auto& ) { + return true; + }); + + const ui64 buildIndexTx = ++txId; + AsyncBuildVectorIndex(runtime, buildIndexTx, TTestTxConfig::SchemeShard, "/MyRoot", "/MyRoot/vectors", "index1", {"embedding"}); + + runtime.WaitFor("LocalKMeansRequest", [&]{ return kmeansBlocker.size(); }); + + { + // set unknown State value + TString writeQuery = Sprintf(R"( + ( + (let key '( '('Id (Uint64 '%lu)) ) ) + (let value '('('State (Uint32 '999999)) ) ) + (return (AsList (UpdateRow 'IndexBuild key value) )) + ) + )", buildIndexTx); + NKikimrMiniKQL::TResult result; + TString err; + NKikimrProto::EReplyStatus status = LocalMiniKQL(runtime, TTestTxConfig::SchemeShard, writeQuery, result, err); + UNIT_ASSERT_VALUES_EQUAL_C(status, NKikimrProto::EReplyStatus::OK, err); + } + + Cerr << "... rebooting scheme shard" << Endl; + RebootTablet(runtime, TTestTxConfig::SchemeShard, runtime.AllocateEdgeActor()); + kmeansBlocker.Stop().Unblock(); + + { + auto buildIndexOperation = TestGetBuildIndex(runtime, TTestTxConfig::SchemeShard, "/MyRoot", buildIndexTx); + auto buildIndexHtml = TestGetBuildIndexHtml(runtime, TTestTxConfig::SchemeShard, buildIndexTx); + UNIT_ASSERT_VALUES_EQUAL_C( + buildIndexOperation.GetIndexBuild().GetState(), Ydb::Table::IndexBuildState::STATE_UNSPECIFIED, + buildIndexOperation.DebugString() + ); + UNIT_ASSERT_STRING_CONTAINS(buildIndexOperation.DebugString(), "Unknown build state"); + UNIT_ASSERT_STRING_CONTAINS(buildIndexHtml, "IsBroken: YES"); + } + + { + // set a known State but unknown SubState + TString writeQuery = Sprintf(R"( + ( + (let key '( '('Id (Uint64 '%lu)) ) ) + (let value '('('State (Uint32 '40)) '('SubState (Uint32 '999999)) ) ) + (return (AsList (UpdateRow 'IndexBuild key value) )) + ) + )", buildIndexTx); + NKikimrMiniKQL::TResult result; + TString err; + NKikimrProto::EReplyStatus status = LocalMiniKQL(runtime, TTestTxConfig::SchemeShard, writeQuery, result, err); + UNIT_ASSERT_VALUES_EQUAL_C(status, NKikimrProto::EReplyStatus::OK, err); + } + + Cerr << "... rebooting scheme shard" << Endl; + RebootTablet(runtime, TTestTxConfig::SchemeShard, runtime.AllocateEdgeActor()); + + { + auto buildIndexOperation = TestGetBuildIndex(runtime, TTestTxConfig::SchemeShard, "/MyRoot", buildIndexTx); + auto buildIndexHtml = TestGetBuildIndexHtml(runtime, TTestTxConfig::SchemeShard, buildIndexTx); + UNIT_ASSERT_VALUES_EQUAL_C( + buildIndexOperation.GetIndexBuild().GetState(), Ydb::Table::IndexBuildState::STATE_TRANSFERING_DATA, + buildIndexOperation.DebugString() + ); + UNIT_ASSERT_STRING_CONTAINS(buildIndexOperation.DebugString(), "Unknown build sub-state"); + UNIT_ASSERT_STRING_CONTAINS(buildIndexHtml, "IsBroken: YES"); + } + + { + // set a known SubState but unknown BuildKind + TString writeQuery = Sprintf(R"( + ( + (let key '( '('Id (Uint64 '%lu)) ) ) + (let value '('('SubState (Uint32 '0)) '('BuildKind (Uint32 '999999)) ) ) + (return (AsList (UpdateRow 'IndexBuild key value) )) + ) + )", buildIndexTx); + NKikimrMiniKQL::TResult result; + TString err; + NKikimrProto::EReplyStatus status = LocalMiniKQL(runtime, TTestTxConfig::SchemeShard, writeQuery, result, err); + UNIT_ASSERT_VALUES_EQUAL_C(status, NKikimrProto::EReplyStatus::OK, err); + } + + Cerr << "... rebooting scheme shard" << Endl; + RebootTablet(runtime, TTestTxConfig::SchemeShard, runtime.AllocateEdgeActor()); + + { + auto buildIndexOperation = TestGetBuildIndex(runtime, TTestTxConfig::SchemeShard, "/MyRoot", buildIndexTx); + auto buildIndexHtml = TestGetBuildIndexHtml(runtime, TTestTxConfig::SchemeShard, buildIndexTx); + UNIT_ASSERT_VALUES_EQUAL_C( + buildIndexOperation.GetIndexBuild().GetState(), Ydb::Table::IndexBuildState::STATE_TRANSFERING_DATA, + buildIndexOperation.DebugString() + ); + UNIT_ASSERT_STRING_CONTAINS(buildIndexOperation.DebugString(), "Unknown build kind"); + UNIT_ASSERT_STRING_CONTAINS(buildIndexHtml, "IsBroken: YES"); + } + } + Y_UNIT_TEST(CreateBuildProposeReject) { TTestBasicRuntime runtime; TTestEnv env(runtime); From 5259bd4ce6e727d17c3339bd53f8c4ddba73c8c0 Mon Sep 17 00:00:00 2001 From: Vitaliy Filippov Date: Thu, 2 Oct 2025 11:12:51 +0300 Subject: [PATCH 4/6] Handle new prefixes in prefixed vector index update (#25505) --- ydb/core/base/table_index.cpp | 2 - ydb/core/base/table_index.h | 2 + .../kqp/opt/logical/kqp_opt_log_indexes.cpp | 48 ++- .../effects/kqp_opt_phy_effects_impl.h | 5 + .../effects/kqp_opt_phy_insert_index.cpp | 9 +- .../effects/kqp_opt_phy_upsert_index.cpp | 9 +- .../effects/kqp_opt_phy_vector_index.cpp | 373 ++++++++++++++++-- ydb/core/kqp/provider/yql_kikimr_gateway.h | 4 + ydb/core/kqp/runtime/kqp_vector_actor.cpp | 8 +- .../kqp_indexes_prefixed_vector_ut.cpp | 300 +++++++++++--- 10 files changed, 654 insertions(+), 106 deletions(-) diff --git a/ydb/core/base/table_index.cpp b/ydb/core/base/table_index.cpp index b24e0b607d5e..856dc57abef2 100644 --- a/ydb/core/base/table_index.cpp +++ b/ydb/core/base/table_index.cpp @@ -191,8 +191,6 @@ bool IsBuildImplTable(std::string_view tableName) { namespace NKMeans { -inline constexpr TClusterId PostingParentFlag = (1ull << 63ull); - bool HasPostingParentFlag(TClusterId parent) { return bool(parent & PostingParentFlag); } diff --git a/ydb/core/base/table_index.h b/ydb/core/base/table_index.h index af55b99a09b3..c99bcb392a75 100644 --- a/ydb/core/base/table_index.h +++ b/ydb/core/base/table_index.h @@ -71,6 +71,8 @@ inline constexpr const char* IdColumnSequence = "__ydb_id_sequence"; inline constexpr const int DefaultKMeansRounds = 3; +inline constexpr TClusterId PostingParentFlag = (1ull << 63ull); + bool HasPostingParentFlag(TClusterId parent); void EnsureNoPostingParentFlag(TClusterId parent); TClusterId SetPostingParentFlag(TClusterId parent); diff --git a/ydb/core/kqp/opt/logical/kqp_opt_log_indexes.cpp b/ydb/core/kqp/opt/logical/kqp_opt_log_indexes.cpp index 062c6772db02..a2b8ac055c5c 100644 --- a/ydb/core/kqp/opt/logical/kqp_opt_log_indexes.cpp +++ b/ydb/core/kqp/opt/logical/kqp_opt_log_indexes.cpp @@ -642,7 +642,6 @@ TExprBase DoRewriteTopSortOverKMeansTree( YQL_ENSURE(levelTableDesc->Metadata->Name.EndsWith(NTableIndex::NKMeans::LevelTable)); YQL_ENSURE(postingTableDesc->Metadata->Name.EndsWith(NTableIndex::NKMeans::PostingTable)); - // TODO(mbkkt) It's kind of strange that almost everything here have same position const auto pos = match.Pos(); const auto levelTable = BuildTableMeta(*levelTableDesc->Metadata, pos, ctx); @@ -717,6 +716,33 @@ TExprBase DoRewriteTopSortOverKMeansTree( return TExprBase{read}; } +template +TExprBase FilterLeafRows(const TExprBase& read, TExprContext& ctx, TPositionHandle pos) { + auto leafFlag = Build(ctx, pos) + .Literal() + .Value(std::to_string(NTableIndex::NKMeans::PostingParentFlag)) // "9223372036854775808" + .Build() + .Done(); + auto prefixRowArg = ctx.NewArgument(pos, "prefixRow"); + auto prefixCluster = Build(ctx, pos) + .Struct(prefixRowArg) + .Name().Build(NTableIndex::NKMeans::ParentColumn) + .Done(); + return Build(ctx, pos) + .Input(read) + .Lambda() + .Args({prefixRowArg}) + .Body() + .Predicate() + .Left(prefixCluster) + .Right(leafFlag) + .Build() + .Value(prefixRowArg) + .Build() + .Build() + .Done(); +} + TExprBase DoRewriteTopSortOverPrefixedKMeansTree( const TReadMatch& match, const TCoFlatMap& flatMap, const TExprBase& lambdaArgs, const TExprBase& lambdaBody, const TCoTopBase& top, TExprContext& ctx, const TKqpOptimizeContext& kqpCtx, @@ -732,7 +758,6 @@ TExprBase DoRewriteTopSortOverPrefixedKMeansTree( YQL_ENSURE(postingTableDesc->Metadata->Name.EndsWith(NTableIndex::NKMeans::PostingTable)); YQL_ENSURE(prefixTableDesc->Metadata->Name.EndsWith(NTableIndex::NKMeans::PrefixTable)); - // TODO(mbkkt) It's kind of strange that almost everything here have same position const auto pos = match.Pos(); const auto levelTable = BuildTableMeta(*levelTableDesc->Metadata, pos, ctx); @@ -776,18 +801,29 @@ TExprBase DoRewriteTopSortOverPrefixedKMeansTree( .Lambda(prefixLambda) .Done().Ptr(); + read = Build(ctx, pos) + .Input(read) + .Done().Ptr(); + RemapIdToParent(ctx, pos, read); + auto prefixLeafRows = FilterLeafRows(TExprBase(read), ctx, pos); + auto prefixRootRows = FilterLeafRows(TExprBase(read), ctx, pos); + TKqpStreamLookupSettings settings; settings.Strategy = EStreamLookupStrategyType::LookupRows; - read = Build(ctx, pos) + auto levelRows = Build(ctx, pos) .Table(levelTable) - .LookupKeys(read) + .LookupKeys(prefixRootRows) .Columns(levelColumns) .Settings(settings.BuildNode(ctx, pos)) - .Done().Ptr(); + .Done().Ptr(); + VectorReadLevel(indexDesc, ctx, pos, kqpCtx, levelLambda, top, levelTable, levelColumns, levelRows); - VectorReadLevel(indexDesc, ctx, pos, kqpCtx, levelLambda, top, levelTable, levelColumns, read); + read = Build(ctx, pos) + .Add(levelRows) + .Add(prefixLeafRows) + .Done().Ptr(); VectorReadMain(ctx, pos, postingTable, postingTableDesc->Metadata, mainTable, tableDesc.Metadata, mainColumns, read); diff --git a/ydb/core/kqp/opt/physical/effects/kqp_opt_phy_effects_impl.h b/ydb/core/kqp/opt/physical/effects/kqp_opt_phy_effects_impl.h index de167865da32..95916f6cc9ad 100644 --- a/ydb/core/kqp/opt/physical/effects/kqp_opt_phy_effects_impl.h +++ b/ydb/core/kqp/opt/physical/effects/kqp_opt_phy_effects_impl.h @@ -113,4 +113,9 @@ NYql::NNodes::TExprBase BuildVectorIndexPrefixRows(const NYql::TKikimrTableDescr bool withData, const NYql::TIndexDescription* indexDesc, const NYql::NNodes::TExprBase& inputRows, TVector& indexTableColumns, NYql::TPositionHandle pos, NYql::TExprContext& ctx); +std::pair BuildVectorIndexPrefixRowsWithNew( + const NYql::TKikimrTableDescription& table, const NYql::TKikimrTableDescription& prefixTable, + const NYql::TIndexDescription* indexDesc, const NYql::NNodes::TExprBase& inputRows, + TVector& indexTableColumns, NYql::TPositionHandle pos, NYql::TExprContext& ctx); + } // NKikimr::NKqp::NOpt diff --git a/ydb/core/kqp/opt/physical/effects/kqp_opt_phy_insert_index.cpp b/ydb/core/kqp/opt/physical/effects/kqp_opt_phy_insert_index.cpp index 43069e0f010b..401757c79935 100644 --- a/ydb/core/kqp/opt/physical/effects/kqp_opt_phy_insert_index.cpp +++ b/ydb/core/kqp/opt/physical/effects/kqp_opt_phy_insert_index.cpp @@ -203,7 +203,14 @@ TExprBase KqpBuildInsertIndexStages(TExprBase node, TExprContext& ctx, const TKq // First resolve prefix IDs using StreamLookup const auto& prefixTable = kqpCtx.Tables->ExistingTable(kqpCtx.Cluster, TStringBuilder() << insert.Table().Path().Value() << "/" << indexDesc->Name << "/" << NKikimr::NTableIndex::NKMeans::PrefixTable); - upsertIndexRows = BuildVectorIndexPrefixRows(table, prefixTable, true, indexDesc, upsertIndexRows, indexTableColumns, insert.Pos(), ctx); + if (prefixTable.Metadata->Columns.at(NTableIndex::NKMeans::IdColumn).DefaultKind == NKikimrKqp::TKqpColumnMetadataProto::DEFAULT_KIND_SEQUENCE) { + auto res = BuildVectorIndexPrefixRowsWithNew(table, prefixTable, indexDesc, upsertIndexRows, indexTableColumns, insert.Pos(), ctx); + upsertIndexRows = std::move(res.first); + effects.emplace_back(std::move(res.second)); + } else { + // Handle old prefixed vector index tables without the sequence + upsertIndexRows = BuildVectorIndexPrefixRows(table, prefixTable, true, indexDesc, upsertIndexRows, indexTableColumns, insert.Pos(), ctx); + } } upsertIndexRows = BuildVectorIndexPostingRows(table, insert.Table(), indexDesc->Name, indexTableColumns, upsertIndexRows, true, insert.Pos(), ctx); diff --git a/ydb/core/kqp/opt/physical/effects/kqp_opt_phy_upsert_index.cpp b/ydb/core/kqp/opt/physical/effects/kqp_opt_phy_upsert_index.cpp index e2811873ad95..9ba019dbc8e6 100644 --- a/ydb/core/kqp/opt/physical/effects/kqp_opt_phy_upsert_index.cpp +++ b/ydb/core/kqp/opt/physical/effects/kqp_opt_phy_upsert_index.cpp @@ -918,7 +918,14 @@ TMaybeNode KqpPhyUpsertIndexEffectsImpl(TKqpPhyUpsertIndexMode mode, if (indexDesc->Type == TIndexDescription::EType::GlobalSyncVectorKMeansTree) { if (indexDesc->KeyColumns.size() > 1) { - upsertIndexRows = BuildVectorIndexPrefixRows(table, *prefixTable, true, indexDesc, upsertIndexRows, indexTableColumns, pos, ctx); + if (prefixTable->Metadata->Columns.at(NTableIndex::NKMeans::IdColumn).DefaultKind == NKikimrKqp::TKqpColumnMetadataProto::DEFAULT_KIND_SEQUENCE) { + auto res = BuildVectorIndexPrefixRowsWithNew(table, *prefixTable, indexDesc, upsertIndexRows, indexTableColumns, pos, ctx); + upsertIndexRows = std::move(res.first); + effects.emplace_back(std::move(res.second)); + } else { + // Handle old prefixed vector index tables without the sequence + upsertIndexRows = BuildVectorIndexPrefixRows(table, *prefixTable, true, indexDesc, upsertIndexRows, indexTableColumns, pos, ctx); + } } upsertIndexRows = BuildVectorIndexPostingRows(table, mainTableNode, diff --git a/ydb/core/kqp/opt/physical/effects/kqp_opt_phy_vector_index.cpp b/ydb/core/kqp/opt/physical/effects/kqp_opt_phy_vector_index.cpp index 748198e630f9..b426a0884be2 100644 --- a/ydb/core/kqp/opt/physical/effects/kqp_opt_phy_vector_index.cpp +++ b/ydb/core/kqp/opt/physical/effects/kqp_opt_phy_vector_index.cpp @@ -143,15 +143,31 @@ TVector MakeColumnGetters(const TExprBase& rowArgument, const TVector return columnGetters; } -TExprBase BuildVectorIndexPrefixRows(const TKikimrTableDescription& table, const TKikimrTableDescription& prefixTable, - bool withData, const TIndexDescription* indexDesc, const TExprBase& inputRows, - TVector& indexTableColumns, TPositionHandle pos, TExprContext& ctx) -{ - // This whole method does a very simple thing: - // SELECT i., p.__ydb_id AS __ydb_parent FROM i INNER JOIN prefixTable p ON p.=i. - // To implement it, we use a StreamLookup in Join mode. - // It takes tuples as input and returns tuples as output ("cookie" contains read stats). +TExprBase BuildNth(const TExprBase& lookupArg, const char *n, TPositionHandle pos, TExprContext& ctx) { + return Build(ctx, pos) + .Tuple(lookupArg) + .Index().Value(n).Build() + .Done(); +}; + +struct TVectorIndexPrefixLookup { + TExprBase Node; + TKqpTable PrefixTableNode; + TVector PrefixColumns; + const TStructExprType* PrefixType; + TVector PostingColumns; + TVectorIndexPrefixLookup(TExprBase&& node, TKqpTable&& prefixTableNode): + Node(std::move(node)), + PrefixTableNode(std::move(prefixTableNode)) { + } +}; + +TVectorIndexPrefixLookup BuildVectorIndexPrefixLookup( + const TKikimrTableDescription& table, const TKikimrTableDescription& prefixTable, + bool withData, bool withNulls, const TIndexDescription* indexDesc, const TExprBase& inputRows, + TPositionHandle pos, TExprContext& ctx) +{ TVector prefixColumns(indexDesc->KeyColumns.begin(), indexDesc->KeyColumns.end()-1); THashSet postingColumnsSet; TVector postingColumns; @@ -172,7 +188,15 @@ TExprBase BuildVectorIndexPrefixRows(const TKikimrTableDescription& table, const } TKqpStreamLookupSettings streamLookupSettings; - streamLookupSettings.Strategy = EStreamLookupStrategyType::LookupJoinRows; + streamLookupSettings.Strategy = EStreamLookupStrategyType::LookupSemiJoinRows; + TVector leftColumns = postingColumns; + if (withNulls) { + for (const auto& column : prefixColumns) { + if (postingColumnsSet.emplace(column).second) { + leftColumns.emplace_back(column); + } + } + } TVector prefixItems; for (auto& column : prefixColumns) { @@ -183,17 +207,17 @@ TExprBase BuildVectorIndexPrefixRows(const TKikimrTableDescription& table, const } auto prefixType = ctx.MakeType(prefixItems); - TVector postingItems; - for (auto& column : postingColumns) { + TVector leftItems; + for (auto& column : leftColumns) { auto type = table.GetColumnType(TString(column)); YQL_ENSURE(type, "No index column: " << column); auto itemType = ctx.MakeType(column, type); - postingItems.push_back(itemType); + leftItems.push_back(itemType); } - auto postingType = ctx.MakeType(postingItems); + auto leftType = ctx.MakeType(leftItems); TVector joinItemItems; - joinItemItems.push_back(postingType); + joinItemItems.push_back(leftType); joinItemItems.push_back(ctx.MakeType(prefixType)); auto joinItemType = ctx.MakeType(joinItemItems); auto joinInputType = ctx.MakeType(joinItemType); @@ -226,7 +250,7 @@ TExprBase BuildVectorIndexPrefixRows(const TKikimrTableDescription& table, const // Join StreamLookup takes tuples as input - build them .Body() .Add() - .Add(MakeColumnGetters(rowArg, postingColumns, pos, ctx)) + .Add(MakeColumnGetters(rowArg, leftColumns, pos, ctx)) .Build() .Add() .Input() @@ -248,28 +272,284 @@ TExprBase BuildVectorIndexPrefixRows(const TKikimrTableDescription& table, const .Settings(streamLookupSettings.BuildNode(ctx, pos)) .Done(); + TVectorIndexPrefixLookup res(std::move(lookup), std::move(prefixTableNode)); + res.PrefixColumns = std::move(prefixColumns); + res.PrefixType = prefixType; + res.PostingColumns = std::move(postingColumns); + return res; +} + +TExprBase BuildVectorIndexPrefixRows(const TKikimrTableDescription& table, const TKikimrTableDescription& prefixTable, + bool withData, const TIndexDescription* indexDesc, const TExprBase& inputRows, + TVector& indexTableColumns, TPositionHandle pos, TExprContext& ctx) +{ + // This whole method does a very simple thing: + // SELECT i., p.__ydb_id AS __ydb_parent FROM i INNER JOIN prefixTable p ON p.=i. + // To implement it, we use a StreamLookup in Join mode. + // It takes tuples as input and returns tuples as output ("cookie" contains read stats). + + TVectorIndexPrefixLookup lookup = BuildVectorIndexPrefixLookup(table, prefixTable, withData, false, indexDesc, inputRows, pos, ctx); + // Join StreamLookup returns tuples as output // But we need left row + 1 field of the right row - build it using TCoMap const auto lookupArg = Build(ctx, pos).Name("lookupRow").Done(); - const auto leftRow = Build(ctx, pos) - .Tuple(lookupArg) - .Index().Value("0").Build() + const auto leftRow = BuildNth(lookupArg, "0", pos, ctx); + const auto rightRow = BuildNth(lookupArg, "1", pos, ctx); + + // Filter rows where rightRow is null + + auto mapLambda = Build(ctx, pos) + .Args({lookupArg}) + .Body() + .Predicate() + .Optional(rightRow) + .Build() + .Value() + .Input() + .Add(MakeColumnGetters(leftRow, lookup.PostingColumns, pos, ctx)) + .Add() + .Name().Build(NTableIndex::NKMeans::ParentColumn) + .template Value() + .Struct().Optional(rightRow).Build() + .Name().Build(NTableIndex::NKMeans::IdColumn) + .Build() + .Build() + .Build() + .Build() + .Build() .Done(); - const auto rightRow = Build(ctx, pos) - .Tuple(lookupArg) - .Index().Value("1").Build() + + auto mapStage = Build(ctx, pos) + .Inputs() + .Add(lookup.Node) + .Build() + .Program() + .Args({"rows"}) + .Body() + .Input() + .Input("rows") + .Lambda(mapLambda) + .Build() + .Build() + .Build() + .Settings().Build() + .Done(); + + indexTableColumns = std::move(lookup.PostingColumns); + indexTableColumns.push_back(NTableIndex::NKMeans::ParentColumn); + + return Build(ctx, pos) + .Output() + .Stage(mapStage) + .Index().Build(0) + .Build() + .Done(); +} + +std::pair BuildVectorIndexPrefixRowsWithNew( + const TKikimrTableDescription& table, const TKikimrTableDescription& prefixTable, + const TIndexDescription* indexDesc, const TExprBase& inputRows, + TVector& indexTableColumns, TPositionHandle pos, TExprContext& ctx) +{ + // This method is similar to the previous one, but also handles unknown prefixes. + // StreamLookupJoin is executed in SemiJoin mode in this case, so may + // be null for prefixes missing from the prefix table. + // We feed such rows to KqpCnSequencer, replicate their IDs to the output stream, + // and also separately insert them into the prefix table. + + TVectorIndexPrefixLookup lookup = BuildVectorIndexPrefixLookup(table, prefixTable, true, true, indexDesc, inputRows, pos, ctx); + + const auto lookupArg = Build(ctx, pos).Name("lookupRow").Done(); + const auto leftRow = BuildNth(lookupArg, "0", pos, ctx); + const auto rightRow = BuildNth(lookupArg, "1", pos, ctx); + + const auto newRowsArg = Build(ctx, pos).Name("rows").Done(); + auto newPrefixStream = Build(ctx, pos) + .Input(newRowsArg) + .Lambda() + .Args({lookupArg}) + .Body() + .Predicate() + .Value() + .Optional(rightRow) + .Build() + .Build() + .Value(leftRow) + .Build() + .Build() + .Done(); + + const auto keyArg = Build(ctx, pos).Name("keyArg").Done(); + const auto payloadArg = Build(ctx, pos).Name("payloadArg").Done(); + auto newPrefixDict = Build(ctx, pos) + .Stream(newPrefixStream) + .KeySelector() + .Args(keyArg) + .Body() + .Add(MakeColumnGetters(keyArg, lookup.PrefixColumns, pos, ctx)) + .Build() + .Build() + .PayloadSelector() + .Args(payloadArg) + .Body() + .Build() + .Build() + .Settings() + .Add().Build("One") + .Add().Build("Hashed") + .Build() + .Done(); + + auto newPrefixDictPrecompute = Build(ctx, pos) + .Connection() + .Output() + .Stage() + .Inputs() + .Add(lookup.Node) + .Build() + .Program() + .Args({newRowsArg}) + .Body(newPrefixDict) + .Build() + .Settings().Build() + .Build() + .Index().Build(0) + .Build() + .Build() + .Done(); + + // Feed newPrefixes from dict to KqpCnSequencer + + const auto dictArg = Build(ctx, pos).Name("dict").Done(); + auto newPrefixesStage = Build(ctx, pos) + .Inputs() + .Add(newPrefixDictPrecompute) + .Build() + .Program() + .Args({dictArg}) + .Body() + .Input() + .Dict(dictArg) + .Build() + .Build() + .Build() + .Settings().Build() + .Done(); + + auto listPrefixType = ctx.MakeType(lookup.PrefixType); + auto fullPrefixColumns = lookup.PrefixColumns; + fullPrefixColumns.push_back(NTableIndex::NKMeans::IdColumn); + auto fullPrefixColumnList = BuildColumnsList(fullPrefixColumns, pos, ctx); + + auto cnSequencer = Build(ctx, pos) + .Output() + .Stage(newPrefixesStage) + .Index().Build(0) + .Build() + .Table(lookup.PrefixTableNode) + .Columns(fullPrefixColumnList) + .DefaultConstraintColumns() + .Add(ctx.NewAtom(pos, NTableIndex::NKMeans::IdColumn)) + .Build() + .InputItemType(ExpandType(pos, *listPrefixType, ctx)) + .Done(); + + auto sequencerPrecompute = Build(ctx, pos) + .Connection() + .Output() + .Stage() + .Inputs() + .Add(cnSequencer) + .Build() + .Program() + .Args({"rows"}) + .Body() + .Input("rows") + .Build() + .Build() + .Settings().Build() + .Build() + .Index().Build(0) + .Build() + .Build() + .Done(); + + const auto sequencerArg = Build(ctx, pos).Name("seqRows").Done(); + const auto seqKeyArg = Build(ctx, pos).Name("seqKey").Done(); + const auto seqValueArg = Build(ctx, pos).Name("seqValue").Done(); + auto sequencerDict = Build(ctx, pos) + .List(sequencerArg) + .KeySelector() + .Args({seqKeyArg}) + .Body() + .Add(MakeColumnGetters(seqKeyArg, lookup.PrefixColumns, pos, ctx)) + .Build() + .Build() + .PayloadSelector() + .Args({seqValueArg}) + .Body() + .Struct(seqValueArg) + .Name().Build(NTableIndex::NKMeans::IdColumn) + .Build() + .Build() + .Settings() + .Add().Build("One") + .Add().Build("Hashed") + .Build() .Done(); + auto sequencerDictPrecompute = Build(ctx, pos) + .Connection() + .Output() + .Stage() + .Inputs() + .Add(sequencerPrecompute) + .Build() + .Program() + .Args({sequencerArg}) + .Body() + .Input() + .Add(sequencerDict) + .Build() + .Build() + .Build() + .Settings().Build() + .Build() + .Index().Build(0) + .Build() + .Build() + .Done(); + + // Take output and remap it to the input using a dictionary + + const auto sequencerDictArg = Build(ctx, pos).Name("dict").Done(); + const auto origArg = Build(ctx, pos).Name("origRows").Done(); + const auto fillRowArg = Build(ctx, pos).Name("fillRow").Done(); + const auto leftFillRow = BuildNth(fillRowArg, "0", pos, ctx); + const auto rightFillRow = BuildNth(fillRowArg, "1", pos, ctx); auto mapLambda = Build(ctx, pos) - .Args({lookupArg}) + .Args({fillRowArg}) .Body() - .Add(MakeColumnGetters(leftRow, postingColumns, pos, ctx)) + .Add(MakeColumnGetters(leftFillRow, lookup.PostingColumns, pos, ctx)) .Add() .Name().Build(NTableIndex::NKMeans::ParentColumn) - .template Value() - .Struct().Optional(rightRow).Build() - .Name().Build(NTableIndex::NKMeans::IdColumn) + .template Value() + .Predicate() + .Optional(rightFillRow) + .Build() + .ThenValue() + .Struct().Optional(rightFillRow).Build() + .Name().Build(NTableIndex::NKMeans::IdColumn) + .Build() + .ElseValue() + .Optional() + .Collection(sequencerDictArg) + .Lookup() + .Add(MakeColumnGetters(leftFillRow, lookup.PrefixColumns, pos, ctx)) + .Build() + .Build() + .Build() .Build() .Build() .Build() @@ -277,13 +557,14 @@ TExprBase BuildVectorIndexPrefixRows(const TKikimrTableDescription& table, const auto mapStage = Build(ctx, pos) .Inputs() - .Add(lookup) + .Add(lookup.Node) + .Add(sequencerDictPrecompute) .Build() .Program() - .Args({"rows"}) + .Args({origArg, sequencerDictArg}) .Body() .Input() - .Input("rows") + .Input(origArg) .Lambda(mapLambda) .Build() .Build() @@ -291,15 +572,43 @@ TExprBase BuildVectorIndexPrefixRows(const TKikimrTableDescription& table, const .Settings().Build() .Done(); - postingColumns.push_back(NTableIndex::NKMeans::ParentColumn); - indexTableColumns = std::move(postingColumns); + indexTableColumns = std::move(lookup.PostingColumns); + indexTableColumns.push_back(NTableIndex::NKMeans::ParentColumn); - return Build(ctx, pos) + auto mappedRows = Build(ctx, pos) .Output() .Stage(mapStage) .Index().Build(0) .Build() .Done(); + + auto upsertPrefixStage = Build(ctx, pos) + .Inputs() + .Add(sequencerPrecompute) + .Build() + .Program() + .Args({"rows"}) + .Body() + .Input("rows") + .Build() + .Build() + .Settings().Build() + .Done(); + + auto upsertPrefix = Build(ctx, pos) + .Table(lookup.PrefixTableNode) + .Input() + .Output() + .Stage(upsertPrefixStage) + .Index().Build(0) + .Build() + .Build() + .Columns(fullPrefixColumnList) + .ReturningColumns().Build() + .IsBatch(ctx.NewAtom(pos, "false")) + .Done(); + + return std::make_pair((TExprBase)mappedRows, (TExprBase)upsertPrefix); } } // namespace NKikimr::NKqp::NOpt diff --git a/ydb/core/kqp/provider/yql_kikimr_gateway.h b/ydb/core/kqp/provider/yql_kikimr_gateway.h index 73e07d0a082a..5b0247891468 100644 --- a/ydb/core/kqp/provider/yql_kikimr_gateway.h +++ b/ydb/core/kqp/provider/yql_kikimr_gateway.h @@ -206,6 +206,10 @@ struct TIndexDescription { case EType::GlobalAsync: return false; case EType::GlobalSyncVectorKMeansTree: + if (State != EIndexState::Ready) { + // Do not try to update vector indexes until their build is finished + return false; + } return true; } } diff --git a/ydb/core/kqp/runtime/kqp_vector_actor.cpp b/ydb/core/kqp/runtime/kqp_vector_actor.cpp index f72335724044..bac8f23fcfdf 100644 --- a/ydb/core/kqp/runtime/kqp_vector_actor.cpp +++ b/ydb/core/kqp/runtime/kqp_vector_actor.cpp @@ -185,9 +185,12 @@ class TKqpVectorResolveActor : public NActors::TActorBootstrapped 0) { diff --git a/ydb/core/kqp/ut/indexes/kqp_indexes_prefixed_vector_ut.cpp b/ydb/core/kqp/ut/indexes/kqp_indexes_prefixed_vector_ut.cpp index 8529a8b7800b..c28c75e2df23 100644 --- a/ydb/core/kqp/ut/indexes/kqp_indexes_prefixed_vector_ut.cpp +++ b/ydb/core/kqp/ut/indexes/kqp_indexes_prefixed_vector_ut.cpp @@ -62,7 +62,7 @@ Y_UNIT_TEST_SUITE(KqpPrefixedVectorIndexes) { } } - void DoPositiveQueriesVectorIndex(TSession& session, const TString& mainQuery, const TString& indexQuery, bool covered = false) { + void DoPositiveQueriesVectorIndex(TSession& session, const TString& mainQuery, const TString& indexQuery, bool covered = false, size_t count = 3) { auto toStr = [](const auto& rs) -> TString { TStringBuilder b; for (const auto& r : rs) { @@ -72,12 +72,12 @@ Y_UNIT_TEST_SUITE(KqpPrefixedVectorIndexes) { }; auto mainResults = DoPositiveQueryVectorIndex(session, mainQuery); absl::c_sort(mainResults); - UNIT_ASSERT_EQUAL_C(mainResults.size(), 3, toStr(mainResults)); + UNIT_ASSERT_EQUAL_C(mainResults.size(), count, toStr(mainResults)); UNIT_ASSERT_C(std::unique(mainResults.begin(), mainResults.end()) == mainResults.end(), toStr(mainResults)); auto indexResults = DoPositiveQueryVectorIndex(session, indexQuery, covered); absl::c_sort(indexResults); - UNIT_ASSERT_EQUAL_C(indexResults.size(), 3, toStr(indexResults)); + UNIT_ASSERT_EQUAL_C(indexResults.size(), count, toStr(indexResults)); UNIT_ASSERT_C(std::unique(indexResults.begin(), indexResults.end()) == indexResults.end(), toStr(indexResults)); UNIT_ASSERT_VALUES_EQUAL(mainResults, indexResults); @@ -89,10 +89,9 @@ Y_UNIT_TEST_SUITE(KqpPrefixedVectorIndexes) { std::string_view direction, std::string_view left, std::string_view right, - bool covered = false) { - constexpr std::string_view init = - "$target = \"\x67\x68\x03\";\n" - "$user = \"user_b\";"; + bool covered = false, + std::string_view init = "$target = \"\x67\x68\x03\";\n$user = \"user_b\";", + size_t count = 3) { std::string metric = std::format("Knn::{}({}, {})", function, left, right); // no metric in result { @@ -113,7 +112,7 @@ Y_UNIT_TEST_SUITE(KqpPrefixedVectorIndexes) { ORDER BY {} {} LIMIT 3; )", init, metric, direction))); - DoPositiveQueriesVectorIndex(session, plainQuery, indexQuery, covered); + DoPositiveQueriesVectorIndex(session, plainQuery, indexQuery, covered, count); } // metric in result { @@ -130,7 +129,7 @@ Y_UNIT_TEST_SUITE(KqpPrefixedVectorIndexes) { ORDER BY {} {} LIMIT 3; )", init, metric, metric, direction))); - DoPositiveQueriesVectorIndex(session, plainQuery, indexQuery, covered); + DoPositiveQueriesVectorIndex(session, plainQuery, indexQuery, covered, count); } // metric as result // TODO(mbkkt) fix this behavior too @@ -149,7 +148,7 @@ Y_UNIT_TEST_SUITE(KqpPrefixedVectorIndexes) { ORDER BY m {} LIMIT 3; )", init, metric, direction))); - DoPositiveQueriesVectorIndex(session, plainQuery, indexQuery, covered); + DoPositiveQueriesVectorIndex(session, plainQuery, indexQuery, covered, count); } } @@ -157,23 +156,27 @@ Y_UNIT_TEST_SUITE(KqpPrefixedVectorIndexes) { TSession& session, std::string_view function, std::string_view direction, - bool covered = false) { + bool covered = false, + std::string_view init = "$target = \"\x67\x68\x03\";\n$user = \"user_b\";", + size_t count = 3) { // target is left, member is right - DoPositiveQueriesPrefixedVectorIndexOrderBy(session, function, direction, "$target", "emb", covered); + DoPositiveQueriesPrefixedVectorIndexOrderBy(session, function, direction, "$target", "emb", covered, init, count); // target is right, member is left - DoPositiveQueriesPrefixedVectorIndexOrderBy(session, function, direction, "emb", "$target", covered); + DoPositiveQueriesPrefixedVectorIndexOrderBy(session, function, direction, "emb", "$target", covered, init, count); } - void DoPositiveQueriesPrefixedVectorIndexOrderByCosine(TSession& session, bool covered = false) { + void DoPositiveQueriesPrefixedVectorIndexOrderByCosine(TSession& session, bool covered = false, + std::string_view init = "$target = \"\x67\x68\x03\";\n$user = \"user_b\";", size_t count = 3) { // distance, default direction - DoPositiveQueriesPrefixedVectorIndexOrderBy(session, "CosineDistance", "", covered); + DoPositiveQueriesPrefixedVectorIndexOrderBy(session, "CosineDistance", "", covered, init, count); // distance, asc direction - DoPositiveQueriesPrefixedVectorIndexOrderBy(session, "CosineDistance", "ASC", covered); + DoPositiveQueriesPrefixedVectorIndexOrderBy(session, "CosineDistance", "ASC", covered, init, count); // similarity, desc direction - DoPositiveQueriesPrefixedVectorIndexOrderBy(session, "CosineSimilarity", "DESC", covered); + DoPositiveQueriesPrefixedVectorIndexOrderBy(session, "CosineSimilarity", "DESC", covered, init, count); } - TSession DoCreateTableForPrefixedVectorIndex(TTableClient& db, bool nullable, bool suffixPk = false) { + TSession DoOnlyCreateTableForPrefixedVectorIndex(TTableClient& db, bool nullable, bool suffixPk = false, + bool withIndex = false, bool covered = false) { auto session = db.CreateSession().GetValueSync().GetSession(); { @@ -210,54 +213,74 @@ Y_UNIT_TEST_SUITE(KqpPrefixedVectorIndexes) { .AppendSplitPoints(TValueBuilder{}.BeginTuple().AddElement().OptionalInt64(60).EndTuple().Build()); tableBuilder.SetPartitionAtKeys(partitions); } + if (withIndex) { + TKMeansTreeSettings kmeans; + kmeans.Settings.Metric = TVectorIndexSettings::EMetric::CosineDistance; + kmeans.Settings.VectorType = TVectorIndexSettings::EVectorType::Uint8; + kmeans.Settings.VectorDimension = 2; + kmeans.Clusters = 2; + kmeans.Levels = 2; + std::vector dataColumns; + if (covered) { + dataColumns = {"user", "emb", "data"}; + } + tableBuilder.AddVectorKMeansTreeIndex("index", {"user", "emb"}, dataColumns, kmeans); + } auto result = session.CreateTable("/Root/TestTable", tableBuilder.Build()).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); } - { - const TString query1(Q_(R"( - UPSERT INTO `/Root/TestTable` (pk, user, emb, data) VALUES)" - "( 1, \"user_a\", \"\x03\x30\x03\", \"10\")," - "(11, \"user_a\", \"\x13\x31\x03\", \"11\")," - "(21, \"user_a\", \"\x23\x32\x03\", \"12\")," - "(31, \"user_a\", \"\x53\x33\x03\", \"13\")," - "(41, \"user_a\", \"\x43\x34\x03\", \"14\")," - "(51, \"user_a\", \"\x50\x60\x03\", \"15\")," - "(61, \"user_a\", \"\x61\x61\x03\", \"16\")," - "(71, \"user_a\", \"\x12\x62\x03\", \"17\")," - "(81, \"user_a\", \"\x75\x76\x03\", \"18\")," - "(91, \"user_a\", \"\x76\x76\x03\", \"19\")," - - "( 2, \"user_b\", \"\x03\x30\x03\", \"20\")," - "(12, \"user_b\", \"\x13\x31\x03\", \"21\")," - "(22, \"user_b\", \"\x23\x32\x03\", \"22\")," - "(32, \"user_b\", \"\x53\x33\x03\", \"23\")," - "(42, \"user_b\", \"\x43\x34\x03\", \"24\")," - "(52, \"user_b\", \"\x50\x60\x03\", \"25\")," - "(62, \"user_b\", \"\x61\x61\x03\", \"26\")," - "(72, \"user_b\", \"\x12\x62\x03\", \"27\")," - "(82, \"user_b\", \"\x75\x76\x03\", \"28\")," - "(92, \"user_b\", \"\x76\x76\x03\", \"29\")," - - "( 3, \"user_c\", \"\x03\x30\x03\", \"30\")," - "(13, \"user_c\", \"\x13\x31\x03\", \"31\")," - "(23, \"user_c\", \"\x23\x32\x03\", \"32\")," - "(33, \"user_c\", \"\x53\x33\x03\", \"33\")," - "(43, \"user_c\", \"\x43\x34\x03\", \"34\")," - "(53, \"user_c\", \"\x50\x60\x03\", \"35\")," - "(63, \"user_c\", \"\x61\x61\x03\", \"36\")," - "(73, \"user_c\", \"\x12\x62\x03\", \"37\")," - "(83, \"user_c\", \"\x75\x76\x03\", \"38\")," - "(93, \"user_c\", \"\x76\x76\x03\", \"39\");" - )); - - auto result = session.ExecuteDataQuery( - query1, - TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) - .ExtractValueSync(); - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); - } + return session; + } + + void InsertDataForPrefixedVectorIndex(TSession& session) { + const TString query1(Q_(R"( + UPSERT INTO `/Root/TestTable` (pk, user, emb, data) VALUES)" + "( 1, \"user_a\", \"\x03\x30\x03\", \"10\")," + "(11, \"user_a\", \"\x13\x31\x03\", \"11\")," + "(21, \"user_a\", \"\x23\x32\x03\", \"12\")," + "(31, \"user_a\", \"\x53\x33\x03\", \"13\")," + "(41, \"user_a\", \"\x43\x34\x03\", \"14\")," + "(51, \"user_a\", \"\x50\x60\x03\", \"15\")," + "(61, \"user_a\", \"\x61\x61\x03\", \"16\")," + "(71, \"user_a\", \"\x12\x62\x03\", \"17\")," + "(81, \"user_a\", \"\x75\x76\x03\", \"18\")," + "(91, \"user_a\", \"\x76\x76\x03\", \"19\")," + + "( 2, \"user_b\", \"\x03\x30\x03\", \"20\")," + "(12, \"user_b\", \"\x13\x31\x03\", \"21\")," + "(22, \"user_b\", \"\x23\x32\x03\", \"22\")," + "(32, \"user_b\", \"\x53\x33\x03\", \"23\")," + "(42, \"user_b\", \"\x43\x34\x03\", \"24\")," + "(52, \"user_b\", \"\x50\x60\x03\", \"25\")," + "(62, \"user_b\", \"\x61\x61\x03\", \"26\")," + "(72, \"user_b\", \"\x12\x62\x03\", \"27\")," + "(82, \"user_b\", \"\x75\x76\x03\", \"28\")," + "(92, \"user_b\", \"\x76\x76\x03\", \"29\")," + + "( 3, \"user_c\", \"\x03\x30\x03\", \"30\")," + "(13, \"user_c\", \"\x13\x31\x03\", \"31\")," + "(23, \"user_c\", \"\x23\x32\x03\", \"32\")," + "(33, \"user_c\", \"\x53\x33\x03\", \"33\")," + "(43, \"user_c\", \"\x43\x34\x03\", \"34\")," + "(53, \"user_c\", \"\x50\x60\x03\", \"35\")," + "(63, \"user_c\", \"\x61\x61\x03\", \"36\")," + "(73, \"user_c\", \"\x12\x62\x03\", \"37\")," + "(83, \"user_c\", \"\x75\x76\x03\", \"38\")," + "(93, \"user_c\", \"\x76\x76\x03\", \"39\");" + )); + + auto result = session.ExecuteDataQuery( + query1, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + } + + TSession DoCreateTableForPrefixedVectorIndex(TTableClient& db, bool nullable, bool suffixPk = false) { + auto session = DoOnlyCreateTableForPrefixedVectorIndex(db, nullable, suffixPk); + InsertDataForPrefixedVectorIndex(session); return session; } @@ -567,6 +590,161 @@ Y_UNIT_TEST_SUITE(KqpPrefixedVectorIndexes) { DoTestPrefixedVectorIndexInsert(Returning, Covered); } + Y_UNIT_TEST_QUAD(PrefixedVectorIndexInsertNewPrefix, Nullable, Covered) { + NKikimrConfig::TFeatureFlags featureFlags; + featureFlags.SetEnableVectorIndex(true); + auto setting = NKikimrKqp::TKqpSetting(); + auto serverSettings = TKikimrSettings() + .SetFeatureFlags(featureFlags) + .SetKqpSettings({setting}); + + TKikimrRunner kikimr(serverSettings); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::SEQUENCESHARD, NActors::NLog::PRI_TRACE); + + auto db = kikimr.GetTableClient(); + + auto session = DoCreateTableForPrefixedVectorIndex(db, Nullable); + DoCreatePrefixedVectorIndex(session, false, Covered ? "COVER (user, emb, data)" : "", 2); + + const TString originalPostingTable = ReadTablePartToYson(session, "/Root/TestTable/index/indexImplPostingTable"); + + // Insert to the table with index should succeed + { + TString query1(Q_(R"( + INSERT INTO `/Root/TestTable` (pk, user, emb, data) VALUES + (101, "user_a", "\x03\x29\x03", "101"), + (102, "user_xxx", "\x03\x29\x03", "102"), + (111, "user_yyy", "\x76\x75\x03", "111"), + (112, "user_yyy", "\x76\x75\x03", "112"); + )")); + + auto result = session.ExecuteDataQuery(query1, TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + } + + // Index is updated + const TString postingTable1_ins = ReadTablePartToYson(session, "/Root/TestTable/index/indexImplPostingTable"); + UNIT_ASSERT_STRINGS_UNEQUAL(originalPostingTable, postingTable1_ins); + + // Check that we can now actually find new rows + DoPositiveQueriesPrefixedVectorIndexOrderByCosine(session, Covered, + "$target = \"\x67\x68\x03\";\n$user = \"user_a\";"); + DoPositiveQueriesPrefixedVectorIndexOrderByCosine(session, Covered, + "$target = \"\x67\x68\x03\";\n$user = \"user_b\";"); + DoPositiveQueriesPrefixedVectorIndexOrderByCosine(session, Covered, + "$target = \"\x67\x68\x03\";\n$user = \"user_xxx\";", 1); + DoPositiveQueriesPrefixedVectorIndexOrderByCosine(session, Covered, + "$target = \"\x67\x68\x03\";\n$user = \"user_yyy\";", 2); + + // Check that PKs 1/101, 111/112 are now in same clusters + { + const TString query1(Q_(R"( + SELECT COUNT(DISTINCT __ydb_parent) FROM `/Root/TestTable/index/indexImplPostingTable` + WHERE pk IN (1, 101) + UNION ALL + SELECT COUNT(DISTINCT __ydb_parent) FROM `/Root/TestTable/index/indexImplPostingTable` + WHERE pk IN (111, 112) + ; + )")); + auto result = session.ExecuteDataQuery(query1, TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), "[[1u];[1u]]"); + } + + // Delete one of the new rows + { + auto result = session.ExecuteDataQuery("DELETE FROM `/Root/TestTable` WHERE pk=112;", + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + } + + // Check that PK 112 is not present in the posting table + { + const TString query1(Q_(R"( + SELECT COUNT(*) FROM `/Root/TestTable/index/indexImplPostingTable` + WHERE pk=112; + )")); + auto result = session.ExecuteDataQuery(query1, TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), "[[0u]]"); + } + } + + Y_UNIT_TEST_QUAD(PrefixedVectorEmptyIndexedTableInsert, Nullable, Covered) { + NKikimrConfig::TFeatureFlags featureFlags; + featureFlags.SetEnableVectorIndex(true); + auto setting = NKikimrKqp::TKqpSetting(); + auto serverSettings = TKikimrSettings() + .SetFeatureFlags(featureFlags) + .SetKqpSettings({setting}); + + TKikimrRunner kikimr(serverSettings); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::SEQUENCESHARD, NActors::NLog::PRI_TRACE); + + auto db = kikimr.GetTableClient(); + + auto session = DoOnlyCreateTableForPrefixedVectorIndex(db, Nullable, false, true, Covered); + + const TString originalPostingTable = ReadTablePartToYson(session, "/Root/TestTable/index/indexImplPostingTable"); + UNIT_ASSERT_STRINGS_EQUAL(originalPostingTable, "[]"); + + // Insert to the table with index should succeed + InsertDataForPrefixedVectorIndex(session); + + // Index is updated + const TString postingTable1_ins = ReadTablePartToYson(session, "/Root/TestTable/index/indexImplPostingTable"); + UNIT_ASSERT_STRINGS_UNEQUAL(originalPostingTable, postingTable1_ins); + + // Check that we can now actually find new rows + DoPositiveQueriesPrefixedVectorIndexOrderByCosine(session, Covered, + "$target = \"\x67\x68\x03\";\n$user = \"user_a\";"); + DoPositiveQueriesPrefixedVectorIndexOrderByCosine(session, Covered, + "$target = \"\x67\x68\x03\";\n$user = \"user_b\";"); + } + + // Same as PrefixedVectorEmptyIndexedTableInsert, but the index is created separately after creating the table + Y_UNIT_TEST_QUAD(EmptyPrefixedVectorIndexInsert, Nullable, Covered) { + NKikimrConfig::TFeatureFlags featureFlags; + featureFlags.SetEnableVectorIndex(true); + auto setting = NKikimrKqp::TKqpSetting(); + auto serverSettings = TKikimrSettings() + .SetFeatureFlags(featureFlags) + .SetKqpSettings({setting}); + + TKikimrRunner kikimr(serverSettings); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::SEQUENCESHARD, NActors::NLog::PRI_TRACE); + + auto db = kikimr.GetTableClient(); + + auto session = DoOnlyCreateTableForPrefixedVectorIndex(db, Nullable, false); + DoCreatePrefixedVectorIndex(session, false, Covered ? "COVER (user, emb, data)" : "", 2); + + const TString originalPostingTable = ReadTablePartToYson(session, "/Root/TestTable/index/indexImplPostingTable"); + UNIT_ASSERT_STRINGS_EQUAL(originalPostingTable, "[]"); + + // Insert to the table with index should succeed + InsertDataForPrefixedVectorIndex(session); + + // Index is updated + const TString postingTable1_ins = ReadTablePartToYson(session, "/Root/TestTable/index/indexImplPostingTable"); + UNIT_ASSERT_STRINGS_UNEQUAL(originalPostingTable, postingTable1_ins); + + // Check that we can now actually find new rows + DoPositiveQueriesPrefixedVectorIndexOrderByCosine(session, Covered, + "$target = \"\x67\x68\x03\";\n$user = \"user_a\";"); + DoPositiveQueriesPrefixedVectorIndexOrderByCosine(session, Covered, + "$target = \"\x67\x68\x03\";\n$user = \"user_b\";"); + } + void DoTestPrefixedVectorIndexDelete(const TString& deleteQuery, bool returning, bool covered) { NKikimrConfig::TFeatureFlags featureFlags; featureFlags.SetEnableVectorIndex(true); From 1134464a67c5e44c767fc6962e8bae56a4885926 Mon Sep 17 00:00:00 2001 From: Vitaliy Filippov Date: Thu, 9 Oct 2025 21:32:42 +0300 Subject: [PATCH 5/6] Deduplicate drop index code between operation_drop_index and operation_move_index (#26613) --- .../schemeshard__operation_drop_index.cpp | 20 +++++++++------ .../schemeshard__operation_move_index.cpp | 25 +------------------ .../schemeshard__operation_move_tables.cpp | 19 +------------- .../schemeshard/schemeshard__operation_part.h | 1 + 4 files changed, 15 insertions(+), 50 deletions(-) diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_drop_index.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_drop_index.cpp index d19f39da111f..ede19722ba9e 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_drop_index.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_drop_index.cpp @@ -464,13 +464,17 @@ TVector CreateDropIndex(TOperationId nextId, const TTxTrans result.push_back(CreateDropTableIndexAtMainTable(NextPartId(nextId, result), mainTableIndexDropping)); } - { - auto indexDropping = TransactionTemplate(mainTablePath.PathString(), NKikimrSchemeOp::EOperationType::ESchemeOpDropTableIndex); - auto operation = indexDropping.MutableDrop(); - operation->SetName(ToString(indexPath.Base()->Name)); + AddDropIndex(result, nextId, indexPath); - result.push_back(CreateDropTableIndex(NextPartId(nextId, result), indexDropping)); - } + return result; +} + +ISubOperation::TPtr AddDropIndex(TVector& result, const TOperationId &nextId, const TPath& indexPath) { + auto indexDropping = TransactionTemplate(indexPath.Parent().PathString(), NKikimrSchemeOp::EOperationType::ESchemeOpDropTableIndex); + auto operation = indexDropping.MutableDrop(); + operation->SetName(ToString(indexPath.Base()->Name)); + + result.push_back(CreateDropTableIndex(NextPartId(nextId, result), indexDropping)); for (const auto& [childName, childPathId] : indexPath.Base()->GetChildren()) { TPath child = indexPath.Child(childName); @@ -486,11 +490,11 @@ TVector CreateDropIndex(TOperationId nextId, const TTxTrans result.push_back(CreateDropTable(NextPartId(nextId, result), implTableDropping)); if (auto reject = CascadeDropTableChildren(result, nextId, child)) { - return {reject}; + return reject; } } - return result; + return nullptr; } } diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_move_index.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_move_index.cpp index ebe0f0e1f112..f1ec56ee19c6 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_move_index.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_move_index.cpp @@ -565,30 +565,7 @@ TVector CreateConsistentMoveIndex(TOperationId nextId, cons << "exists, but overwrite flag has not been set"; return {CreateReject(nextId, NKikimrScheme::StatusSchemeError, errStr)}; } - { - auto indexDropping = TransactionTemplate(mainTablePath.PathString(), NKikimrSchemeOp::EOperationType::ESchemeOpDropTableIndex); - auto operation = indexDropping.MutableDrop(); - operation->SetName(dstIndex); - - result.push_back(CreateDropTableIndex(NextPartId(nextId, result), indexDropping)); - } - - for (const auto& [name, pathId]: dstIndexPath.Base()->GetChildren()) { - Y_ABORT_UNLESS(context.SS->PathsById.contains(pathId)); - auto implPath = context.SS->PathsById.at(pathId); - if (implPath->Dropped()) { - continue; - } - - auto implTable = context.SS->PathsById.at(pathId); - Y_ABORT_UNLESS(implTable->IsTable()); - - auto implTableDropping = TransactionTemplate(dstIndexPath.PathString(), NKikimrSchemeOp::EOperationType::ESchemeOpDropTable); - auto operation = implTableDropping.MutableDrop(); - operation->SetName(name); - - result.push_back(CreateDropTable(NextPartId(nextId, result), implTableDropping)); - } + AddDropIndex(result, nextId, dstIndexPath); } } diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_move_tables.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_move_tables.cpp index a5aa9ed7711d..48e609945baf 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_move_tables.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_move_tables.cpp @@ -54,24 +54,7 @@ TVector CreateConsistentMoveTable(TOperationId nextId, cons } } - THashSet sequences; - for (const auto& child: srcPath.Base()->GetChildren()) { - auto name = child.first; - auto pathId = child.second; - - TPath childPath = srcPath.Child(name); - if (!childPath.IsSequence() || childPath.IsDeleted()) { - continue; - } - - Y_ABORT_UNLESS(childPath.Base()->PathId == pathId); - - TSequenceInfo::TPtr sequenceInfo = context.SS->Sequences.at(pathId); - const auto& sequenceDesc = sequenceInfo->Description; - const auto& sequenceName = sequenceDesc.GetName(); - - sequences.emplace(sequenceName); - } + THashSet sequences = GetLocalSequences(context, srcPath); TPath dstPath = TPath::Resolve(dstStr, context.SS); diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_part.h b/ydb/core/tx/schemeshard/schemeshard__operation_part.h index 735ad3ea535d..b0bcb176383e 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_part.h +++ b/ydb/core/tx/schemeshard/schemeshard__operation_part.h @@ -398,6 +398,7 @@ TVector ApplyBuildIndex(TOperationId id, const TTxTransacti TVector CancelBuildIndex(TOperationId id, const TTxTransaction& tx, TOperationContext& context); TVector CreateDropIndex(TOperationId id, const TTxTransaction& tx, TOperationContext& context); +ISubOperation::TPtr AddDropIndex(TVector& result, const TOperationId &nextId, const TPath& indexPath); ISubOperation::TPtr CreateDropTableIndexAtMainTable(TOperationId id, const TTxTransaction& tx); ISubOperation::TPtr CreateDropTableIndexAtMainTable(TOperationId id, TTxState::ETxState state); From 93c7b56183eedc590a8e2a1d2d9094591ba29fe8 Mon Sep 17 00:00:00 2001 From: Vitaliy Filippov Date: Fri, 10 Oct 2025 17:22:16 +0300 Subject: [PATCH 6/6] Fix moving sequences during the new prefixed vector index move/replace (#26614) --- .../schemeshard__operation_move_index.cpp | 1 + .../schemeshard__operation_move_sequence.cpp | 83 +++++++++++++------ .../schemeshard__operation_move_tables.cpp | 19 +++-- .../schemeshard/schemeshard__operation_part.h | 2 + ydb/core/tx/schemeshard/ut_move/ut_move.cpp | 13 ++- 5 files changed, 83 insertions(+), 35 deletions(-) diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_move_index.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_move_index.cpp index f1ec56ee19c6..6d2b311adca6 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_move_index.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_move_index.cpp @@ -583,6 +583,7 @@ TVector CreateConsistentMoveIndex(TOperationId nextId, cons TPath dstImplTable = dstIndexPath.Child(srcImplTableName); result.push_back(CreateMoveTable(NextPartId(nextId, result), MoveTableTask(srcImplTable, dstImplTable))); + AddMoveSequences(nextId, result, srcImplTable, dstImplTable.PathString(), GetLocalSequences(context, srcImplTable)); } return result; diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_move_sequence.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_move_sequence.cpp index 28678f81455c..bd487cd72ad9 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_move_sequence.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_move_sequence.cpp @@ -813,18 +813,28 @@ class TMoveSequence: public TSubOperation { .IsResolved() .NotDeleted() .NotUnderDeleting() - .IsCommonSensePath() .NotAsyncReplicaTable(); if (checks) { - if (srcParentPath->IsTable()) { + if (srcParentPath.Parent()->IsTableIndex()) { + // Only __ydb_id sequence can be created in the prefixed index + if (srcPath.LeafName() != NTableIndex::NKMeans::IdColumnSequence) { + result->SetError(NKikimrScheme::EStatus::StatusNameConflict, "sequences are not allowed in indexes"); + return result; + } + if (srcParentPath.IsUnderOperation()) { + checks.IsUnderTheSameOperation(OperationId.GetTxId()); // allowed only as part of consistent operations + } + } else if (srcParentPath->IsTable()) { // allow immediately inside a normal table + checks.IsCommonSensePath(); if (srcParentPath.IsUnderOperation()) { checks.IsUnderTheSameOperation(OperationId.GetTxId()); // allowed only as part of consistent operations } } else { // otherwise don't allow unexpected object types - checks.IsLikeDirectory(); + checks.IsCommonSensePath() + .IsLikeDirectory(); } } @@ -835,28 +845,6 @@ class TMoveSequence: public TSubOperation { } TPath dstPath = TPath::Resolve(dstPathStr, context.SS); - TPath dstParentPath = dstPath.Parent(); - - { - TPath::TChecker checks = dstParentPath.Check(); - checks - .NotUnderDomainUpgrade() - .IsAtLocalSchemeShard() - .IsResolved(); - - if (dstParentPath.IsUnderOperation()) { - checks - .IsUnderTheSameOperation(OperationId.GetTxId()); - } else { - checks - .NotUnderOperation(); - } - - if (!checks) { - result->SetError(checks.GetStatus(), checks.GetError()); - return result; - } - } const TString acl = Transaction.GetModifyACL().GetDiffACL(); @@ -893,7 +881,6 @@ class TMoveSequence: public TSubOperation { .DepthLimit() .IsValidLeafName(context.UserToken.Get()) .IsTheSameDomain(srcPath) - .DirChildrenLimit() .IsValidACL(acl); } @@ -903,6 +890,50 @@ class TMoveSequence: public TSubOperation { } } + // Parent is probably already modified and inactive, because it's either a regular table or an index + // implementation table which is in the process of moving. "Inactive" paths are only used in move + // operations and mean that the path isn't yet inserted into the child node map of its parent itself. + // Thus, dstParentPath checks are performed on the 'new' (inactive) version. + + // Most checks on dstPath are performed on the 'old' path, but DirChildrenLimit requires dstParentPath + // to be resolved, so we perform it on the 'new' version of the path. + + dstPath = TPath::ResolveWithInactive(OperationId, dstPathStr, context.SS); + TPath dstParentPath = dstPath.Parent(); + + { + TPath::TChecker checks = dstPath.Check(); + + checks + .DirChildrenLimit(); + + if (!checks) { + result->SetError(checks.GetStatus(), checks.GetError()); + return result; + } + } + + { + TPath::TChecker checks = dstParentPath.Check(); + + checks + .NotUnderDomainUpgrade() + .IsAtLocalSchemeShard() + .IsResolved(); + if (dstParentPath.IsUnderOperation()) { + checks + .IsUnderTheSameOperation(OperationId.GetTxId()); + } else { + checks + .NotUnderOperation(); + } + + if (!checks) { + result->SetError(checks.GetStatus(), checks.GetError()); + return result; + } + } + if (!context.SS->CheckApplyIf(Transaction, errStr)) { result->SetError(NKikimrScheme::StatusPreconditionFailed, errStr); return result; diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_move_tables.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_move_tables.cpp index 48e609945baf..12ba21b3a29b 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_move_tables.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_move_tables.cpp @@ -92,23 +92,28 @@ TVector CreateConsistentMoveTable(TOperationId nextId, cons TPath dstImplTable = dstIndexPath.Child(implTableName); result.push_back(CreateMoveTable(NextPartId(nextId, result), MoveTableTask(srcImplTable, dstImplTable))); + AddMoveSequences(nextId, result, srcImplTable, dstImplTable.PathString(), GetLocalSequences(context, srcImplTable)); } } + AddMoveSequences(nextId, result, srcPath, dstPath.PathString(), sequences); + + return result; +} + +void AddMoveSequences(TOperationId nextId, TVector& result, const TPath& srcTable, + const TString& dstPath, const THashSet& sequences) +{ for (const auto& sequence : sequences) { - auto scheme = TransactionTemplate( - dstPath.PathString(), - NKikimrSchemeOp::EOperationType::ESchemeOpMoveSequence); + auto scheme = TransactionTemplate(dstPath, NKikimrSchemeOp::EOperationType::ESchemeOpMoveSequence); scheme.SetFailOnExist(true); auto* moveSequence = scheme.MutableMoveSequence(); - moveSequence->SetSrcPath(srcPath.PathString() + "/" + sequence); - moveSequence->SetDstPath(dstPath.PathString() + "/" + sequence); + moveSequence->SetSrcPath(srcTable.PathString() + "/" + sequence); + moveSequence->SetDstPath(dstPath + "/" + sequence); result.push_back(CreateMoveSequence(NextPartId(nextId, result), scheme)); } - - return result; } } diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_part.h b/ydb/core/tx/schemeshard/schemeshard__operation_part.h index b0bcb176383e..909c91b95671 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_part.h +++ b/ydb/core/tx/schemeshard/schemeshard__operation_part.h @@ -634,6 +634,8 @@ ISubOperation::TPtr CreateAlterLogin(TOperationId id, TTxState::ETxState state); TVector CreateConsistentMoveTable(TOperationId id, const TTxTransaction& tx, TOperationContext& context); TVector CreateConsistentMoveIndex(TOperationId id, const TTxTransaction& tx, TOperationContext& context); +void AddMoveSequences(TOperationId nextId, TVector& result, + const TPath& srcTable, const TString& dstPath, const THashSet& sequences); ISubOperation::TPtr CreateMoveTable(TOperationId id, const TTxTransaction& tx); ISubOperation::TPtr CreateMoveTable(TOperationId id, TTxState::ETxState state); diff --git a/ydb/core/tx/schemeshard/ut_move/ut_move.cpp b/ydb/core/tx/schemeshard/ut_move/ut_move.cpp index 83aa5d0db7d7..ff0ff1203c1d 100644 --- a/ydb/core/tx/schemeshard/ut_move/ut_move.cpp +++ b/ydb/core/tx/schemeshard/ut_move/ut_move.cpp @@ -1059,7 +1059,7 @@ Y_UNIT_TEST_SUITE(TSchemeShardMoveTest) { // Replace again - it previously crashed here when Dec/IncAliveChildren were incorrect - TestBuildVectorIndex(runtime, ++txId, TTestTxConfig::SchemeShard, "/MyRoot", "/MyRoot/Table", "index2", {"embedding"}); + TestBuildVectorIndex(runtime, ++txId, TTestTxConfig::SchemeShard, "/MyRoot", "/MyRoot/Table", "index2", {"prefix", "embedding"}); env.TestWaitNotification(runtime, txId); TestMoveIndex(runtime, ++txId, "/MyRoot/Table", "index2", "index1", true); @@ -1067,11 +1067,20 @@ Y_UNIT_TEST_SUITE(TSchemeShardMoveTest) { TestDescribeResult(DescribePath(runtime, "/MyRoot/Table/index2"), {NLs::PathNotExist}); TestDescribeResult(DescribePath(runtime, "/MyRoot/Table"), {NLs::PathExist, NLs::IndexesCount(1)}); - TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/Table/index1/indexImplPrefixTable"), {NLs::PathNotExist}); + TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/Table/index1/indexImplPrefixTable"), + { NLs::PathExist, NLs::CheckColumns(PrefixTable, {"prefix", IdColumn}, {}, {"prefix", IdColumn}, true) }); TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/Table/index1/indexImplLevelTable"), { NLs::PathExist, NLs::CheckColumns(LevelTable, {ParentColumn, IdColumn, CentroidColumn}, {}, {ParentColumn, IdColumn}, true) }); TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/Table/index1/indexImplPostingTable"), { NLs::PathExist, NLs::CheckColumns(PostingTable, {ParentColumn, "key"}, {}, {ParentColumn, "key"}, true) }); + + // Drop - it also crashed here when the sequence wasn't moved correctly + + TestDropTableIndex(runtime, ++txId, "/MyRoot", R"( + TableName: "Table" + IndexName: "index1" + )"); + env.TestWaitNotification(runtime, txId); }