diff --git a/ydb/core/base/table_index.h b/ydb/core/base/table_index.h index a251122d0b5e..96e467fe2771 100644 --- a/ydb/core/base/table_index.h +++ b/ydb/core/base/table_index.h @@ -72,6 +72,7 @@ inline constexpr const char* BuildSuffix1 = "1build"; // Prefix table inline constexpr const char* PrefixTable = "indexImplPrefixTable"; +inline constexpr const char* IdColumnSequence = "__ydb_id_sequence"; inline constexpr const int DefaultKMeansRounds = 3; diff --git a/ydb/core/kqp/ut/indexes/kqp_indexes_prefixed_vector_ut.cpp b/ydb/core/kqp/ut/indexes/kqp_indexes_prefixed_vector_ut.cpp index 6d3d035482a0..8529a8b7800b 100644 --- a/ydb/core/kqp/ut/indexes/kqp_indexes_prefixed_vector_ut.cpp +++ b/ydb/core/kqp/ut/indexes/kqp_indexes_prefixed_vector_ut.cpp @@ -311,6 +311,12 @@ Y_UNIT_TEST_SUITE(KqpPrefixedVectorIndexes) { UNIT_ASSERT_EQUAL(settings.Clusters, 2); } DoPositiveQueriesPrefixedVectorIndexOrderByCosine(session); + + { + const TString dropIndex(Q_("ALTER TABLE `/Root/TestTable` DROP INDEX index")); + auto result = session.ExecuteSchemeQuery(dropIndex).ExtractValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + } } Y_UNIT_TEST_QUAD(OrderByCosineLevel2, Nullable, UseSimilarity) { diff --git a/ydb/core/kqp/ut/indexes/kqp_indexes_vector_ut.cpp b/ydb/core/kqp/ut/indexes/kqp_indexes_vector_ut.cpp index b2ba0b2657c6..a39f45b3e75b 100644 --- a/ydb/core/kqp/ut/indexes/kqp_indexes_vector_ut.cpp +++ b/ydb/core/kqp/ut/indexes/kqp_indexes_vector_ut.cpp @@ -297,6 +297,12 @@ Y_UNIT_TEST_SUITE(KqpVectorIndexes) { UNIT_ASSERT_EQUAL(settings.Clusters, 2); } DoPositiveQueriesVectorIndexOrderByCosine(session); + + { + const TString dropIndex(Q_("ALTER TABLE `/Root/TestTable` DROP INDEX index")); + auto result = session.ExecuteSchemeQuery(dropIndex).ExtractValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + } } Y_UNIT_TEST_QUAD(OrderByCosineLevel2, Nullable, UseSimilarity) { diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_alter_sequence.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_alter_sequence.cpp index a2d8225c30d9..943b6118b610 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_alter_sequence.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_alter_sequence.cpp @@ -429,18 +429,26 @@ class TAlterSequence: public TSubOperation { .IsAtLocalSchemeShard() .IsResolved() .NotDeleted() - .NotUnderDeleting() - .IsCommonSensePath(); + .NotUnderDeleting(); if (checks) { - if (parentPath->IsTable()) { + if (parentPath.Parent()->IsTableIndex()) { + checks.IsInsideTableIndexPath(); + // Only __ydb_id sequence can be altered and only by internal transactions (build_index__progress) + if (name != NTableIndex::NKMeans::IdColumnSequence || !Transaction.GetInternal()) { + result->SetError(NKikimrScheme::EStatus::StatusNameConflict, "sequences are not allowed in indexes"); + return result; + } + } else if (parentPath->IsTable()) { // allow immediately inside a normal table + checks.IsCommonSensePath(); if (parentPath.IsUnderOperation()) { checks.IsUnderTheSameOperation(OperationId.GetTxId()); // allowed only as part of consistent operations } } else { // otherwise don't allow unexpected object types - checks.IsLikeDirectory(); + checks.IsCommonSensePath() + .IsLikeDirectory(); } } diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_consistent_copy_tables.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_consistent_copy_tables.cpp index cb0e250544a4..62915a388d43 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_consistent_copy_tables.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_consistent_copy_tables.cpp @@ -203,9 +203,9 @@ bool CreateConsistentCopyTables( Y_ABORT_UNLESS(srcImplTable.Base()->PathId == srcImplTablePathId); TPath dstImplTable = dstIndexPath.Child(srcImplTableName); - result.push_back(CreateCopyTable( - NextPartId(nextId, result), - CopyTableTask(srcImplTable, dstImplTable, descr))); + result.push_back(CreateCopyTable(NextPartId(nextId, result), + CopyTableTask(srcImplTable, dstImplTable, descr), GetLocalSequences(context, srcImplTable))); + AddCopySequences(nextId, tx, context, result, srcImplTable, dstImplTable.PathString()); } } diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_copy_sequence.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_copy_sequence.cpp index f4eb2c8f7667..995c2f9d29c1 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_copy_sequence.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_copy_sequence.cpp @@ -551,18 +551,27 @@ class TCopySequence: public TSubOperation { .IsResolved() .NotDeleted() .NotUnderDeleting() - .IsCommonSensePath() .FailOnRestrictedCreateInTempZone(Transaction.GetAllowCreateInTempDir()); if (checks) { - if (parentPath->IsTable()) { + if (parentPath.Parent()->IsTableIndex()) { + // Only __ydb_id sequence can be created in the prefixed index + if (name != NTableIndex::NKMeans::IdColumnSequence) { + result->SetError(NKikimrScheme::EStatus::StatusNameConflict, "sequences are not allowed in indexes"); + return result; + } + checks.IsUnderTheSameOperation(OperationId.GetTxId()); // allowed only as part of consistent operations + checks.IsInsideTableIndexPath(); + } else if (parentPath->IsTable()) { // allow immediately inside a normal table + checks.IsCommonSensePath(); if (parentPath.IsUnderOperation()) { checks.IsUnderTheSameOperation(OperationId.GetTxId()); // allowed only as part of consistent operations } } else { // otherwise don't allow unexpected object types - checks.IsLikeDirectory(); + checks.IsCommonSensePath() + .IsLikeDirectory(); } } diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_copy_table.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_copy_table.cpp index ae1bf70d1a30..48ae9a4d0184 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_copy_table.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_copy_table.cpp @@ -399,9 +399,7 @@ class TCopyTable: public TSubOperation { if (checks) { if (parent.Base()->IsTableIndex()) { - checks - .IsInsideTableIndexPath() //copy imp index table as index index table, not a separate one - .NotChildren(); //imp table doesn't have indexes + checks.IsInsideTableIndexPath(); //copy imp index table as index index table, not a separate one } else { checks.IsCommonSensePath(); } @@ -869,7 +867,8 @@ TVector CreateCopyTable(TOperationId nextId, const TTxTrans operation->SetOmitFollowers(copying.GetOmitFollowers()); operation->SetIsBackup(copying.GetIsBackup()); - result.push_back(CreateCopyTable(NextPartId(nextId, result), schema)); + result.push_back(CreateCopyTable(NextPartId(nextId, result), schema, GetLocalSequences(context, implTable))); + AddCopySequences(nextId, tx, context, result, implTable, JoinPath({dstPath.PathString(), name, implTableName})); } } diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_create_build_index.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_create_build_index.cpp index 133d9670700a..6001f41e26f1 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_create_build_index.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_create_build_index.cpp @@ -140,7 +140,7 @@ TVector CreateBuildIndex(TOperationId opId, const TTxTransa result.push_back(CreateInitializeBuildIndexMainTable(NextPartId(opId, result), outTx)); } - auto createImplTable = [&](NKikimrSchemeOp::TTableDescription&& implTableDesc) { + auto createImplTable = [&](NKikimrSchemeOp::TTableDescription&& implTableDesc, const THashSet& localSequences = {}) { if (GetIndexType(indexDesc) != NKikimrSchemeOp::EIndexTypeGlobalUnique) { implTableDesc.MutablePartitionConfig()->SetShadowData(true); } @@ -149,7 +149,7 @@ TVector CreateBuildIndex(TOperationId opId, const TTxTransa *outTx.MutableCreateTable() = std::move(implTableDesc); outTx.SetInternal(tx.GetInternal()); - return CreateInitializeBuildIndexImplTable(NextPartId(opId, result), outTx); + return CreateInitializeBuildIndexImplTable(NextPartId(opId, result), outTx, localSequences); }; switch (GetIndexType(indexDesc)) { @@ -183,7 +183,13 @@ TVector CreateBuildIndex(TOperationId opId, const TTxTransa result.push_back(createImplTable(CalcVectorKmeansTreePostingImplTableDesc(tableInfo, tableInfo->PartitionConfig(), indexDataColumns, indexPostingTableDesc))); if (prefixVectorIndex) { const THashSet prefixColumns{indexDesc.GetKeyColumnNames().begin(), indexDesc.GetKeyColumnNames().end() - 1}; - result.push_back(createImplTable(CalcVectorKmeansTreePrefixImplTableDesc(prefixColumns, tableInfo, tableInfo->PartitionConfig(), implTableColumns, indexPrefixTableDesc))); + result.push_back(createImplTable(CalcVectorKmeansTreePrefixImplTableDesc( + prefixColumns, tableInfo, tableInfo->PartitionConfig(), implTableColumns, indexPrefixTableDesc), + THashSet{NTableIndex::NKMeans::IdColumnSequence})); + auto outTx = TransactionTemplate(index.PathString() + "/" + NTableIndex::NKMeans::PrefixTable, NKikimrSchemeOp::EOperationType::ESchemeOpCreateSequence); + outTx.MutableSequence()->SetName(NTableIndex::NKMeans::IdColumnSequence); + outTx.SetInternal(tx.GetInternal()); + result.push_back(CreateNewSequence(NextPartId(opId, result), outTx)); } break; } diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_create_indexed_table.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_create_indexed_table.cpp index fc648c9dda86..937cf0d1b2e5 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_create_indexed_table.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_create_indexed_table.cpp @@ -290,7 +290,7 @@ TVector CreateIndexedTable(TOperationId nextId, const TTxTr result.push_back(CreateNewTableIndex(NextPartId(nextId, result), scheme)); } - auto createIndexImplTable = [&] (NKikimrSchemeOp::TTableDescription&& implTableDesc) { + auto createIndexImplTable = [&] (NKikimrSchemeOp::TTableDescription&& implTableDesc, const THashSet& localSequences = {}) { auto scheme = TransactionTemplate( tx.GetWorkingDir() + "/" + baseTableDescription.GetName() + "/" + indexDescription.GetName(), NKikimrSchemeOp::EOperationType::ESchemeOpCreateTable); @@ -300,7 +300,7 @@ TVector CreateIndexedTable(TOperationId nextId, const TTxTr *scheme.MutableCreateTable() = std::move(implTableDesc); - return CreateNewTable(NextPartId(nextId, result), scheme); + return CreateNewTable(NextPartId(nextId, result), scheme, localSequences); }; const auto& implTableColumns = indexes.at(indexDescription.GetName()); @@ -332,7 +332,22 @@ TVector CreateIndexedTable(TOperationId nextId, const TTxTr result.push_back(createIndexImplTable(CalcVectorKmeansTreePostingImplTableDesc(baseTableDescription, baseTableDescription.GetPartitionConfig(), indexDataColumns, userPostingDesc))); if (prefixVectorIndex) { const THashSet prefixColumns{indexDescription.GetKeyColumnNames().begin(), indexDescription.GetKeyColumnNames().end() - 1}; - result.push_back(createIndexImplTable(CalcVectorKmeansTreePrefixImplTableDesc(prefixColumns, baseTableDescription, baseTableDescription.GetPartitionConfig(), implTableColumns, userPrefixDesc))); + result.push_back(createIndexImplTable(CalcVectorKmeansTreePrefixImplTableDesc( + prefixColumns, baseTableDescription, baseTableDescription.GetPartitionConfig(), implTableColumns, userPrefixDesc), + THashSet{NTableIndex::NKMeans::IdColumnSequence})); + // Create the sequence + auto outTx = TransactionTemplate(tx.GetWorkingDir() + "/" + baseTableDescription.GetName() + "/" + + indexDescription.GetName() + "/" + NTableIndex::NKMeans::PrefixTable, + NKikimrSchemeOp::EOperationType::ESchemeOpCreateSequence); + outTx.MutableSequence()->SetName(NTableIndex::NKMeans::IdColumnSequence); + outTx.MutableSequence()->SetMinValue(-0x7FFFFFFFFFFFFFFF); + outTx.MutableSequence()->SetMaxValue(-1); + outTx.MutableSequence()->SetStartValue(NTableIndex::NKMeans::SetPostingParentFlag(1)); + outTx.MutableSequence()->SetRestart(true); + outTx.SetFailOnExist(tx.GetFailOnExist()); + outTx.SetAllowCreateInTempDir(tx.GetAllowCreateInTempDir()); + outTx.SetInternal(tx.GetInternal()); + result.push_back(CreateNewSequence(NextPartId(nextId, result), outTx)); } break; } diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_create_sequence.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_create_sequence.cpp index 4240188c541a..893fb71352d7 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_create_sequence.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_create_sequence.cpp @@ -389,19 +389,30 @@ class TCreateSequence : public TSubOperation { .IsResolved() .NotDeleted() .NotUnderDeleting() - .IsCommonSensePath() .FailOnRestrictedCreateInTempZone(Transaction.GetAllowCreateInTempDir()); if (checks) { - if (parentPath->IsTable()) { - checks.NotBackupTable(); + if (parentPath.Parent()->IsTableIndex()) { + // Only __ydb_id sequence can be created in the prefixed index + if (name != NTableIndex::NKMeans::IdColumnSequence) { + result->SetError(NKikimrScheme::EStatus::StatusNameConflict, "sequences are not allowed in indexes"); + return result; + } + checks.IsInsideTableIndexPath() + .IsUnderCreating() + .IsUnderTheSameOperation(OperationId.GetTxId()); // allowed only as part of consistent operations + } else if (parentPath->IsTable()) { + checks + .IsCommonSensePath() + .NotBackupTable(); // allow immediately inside a normal table if (parentPath.IsUnderOperation()) { checks.IsUnderTheSameOperation(OperationId.GetTxId()); // allowed only as part of consistent operations } - } else { + } else if (!Transaction.GetAllowAccessToPrivatePaths()) { // otherwise don't allow unexpected object types - checks.IsLikeDirectory(); + checks.IsCommonSensePath() + .IsLikeDirectory(); } } diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_create_table.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_create_table.cpp index 2e9062112326..e1a53a8e7383 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_create_table.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_create_table.cpp @@ -821,9 +821,11 @@ ISubOperation::TPtr CreateNewTable(TOperationId id, TTxState::ETxState state) { return MakeSubOperation(id, state); } -ISubOperation::TPtr CreateInitializeBuildIndexImplTable(TOperationId id, const TTxTransaction& tx) { +ISubOperation::TPtr CreateInitializeBuildIndexImplTable(TOperationId id, const TTxTransaction& tx, const THashSet& localSequences) { auto obj = MakeSubOperation(id, tx); - static_cast(obj.Get())->SetAllowShadowDataForBuildIndex(); + TCreateTable *createTable = static_cast(obj.Get()); + createTable->SetAllowShadowDataForBuildIndex(); + createTable->SetLocalSequences(localSequences); return obj; } diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_drop_sequence.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_drop_sequence.cpp index f67b2a397113..84471665f068 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_drop_sequence.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_drop_sequence.cpp @@ -291,19 +291,29 @@ class TDropSequence: public TSubOperation { .NotUnderDomainUpgrade() .IsAtLocalSchemeShard() .IsResolved() - .NotDeleted() - .IsCommonSensePath(); + .NotDeleted(); if (checks) { - if (parent->IsTable()) { + if (parent.Parent()->IsTableIndex()) { + // Only __ydb_id sequence can be present in the prefixed index + if (name != NTableIndex::NKMeans::IdColumnSequence) { + result->SetError(NKikimrScheme::EStatus::StatusNameConflict, "sequences are not allowed in indexes"); + return result; + } + checks.IsInsideTableIndexPath() + .IsUnderDeleting() + .IsUnderTheSameOperation(OperationId.GetTxId()); // allowed only as part of consistent operations + } else if (parent->IsTable()) { // allow immediately inside a normal table if (parent.IsUnderOperation()) { checks.IsUnderTheSameOperation(OperationId.GetTxId()); // allowed only as part of consistent operations } + checks.IsCommonSensePath(); } else { checks .NotUnderDeleting() - .IsLikeDirectory(); + .IsLikeDirectory() + .IsCommonSensePath(); } } diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_part.h b/ydb/core/tx/schemeshard/schemeshard__operation_part.h index a680b5fa9633..735ad3ea535d 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_part.h +++ b/ydb/core/tx/schemeshard/schemeshard__operation_part.h @@ -604,7 +604,7 @@ ISubOperation::TPtr CreateDropSolomon(TOperationId id, TTxState::ETxState state) ISubOperation::TPtr CreateInitializeBuildIndexMainTable(TOperationId id, const TTxTransaction& tx); ISubOperation::TPtr CreateInitializeBuildIndexMainTable(TOperationId id, TTxState::ETxState state); -ISubOperation::TPtr CreateInitializeBuildIndexImplTable(TOperationId id, const TTxTransaction& tx); +ISubOperation::TPtr CreateInitializeBuildIndexImplTable(TOperationId id, const TTxTransaction& tx, const THashSet& localSequences = {}); ISubOperation::TPtr CreateInitializeBuildIndexImplTable(TOperationId id, TTxState::ETxState state); ISubOperation::TPtr CreateFinalizeBuildIndexImplTable(TOperationId id, const TTxTransaction& tx); diff --git a/ydb/core/tx/schemeshard/schemeshard_build_index__progress.cpp b/ydb/core/tx/schemeshard/schemeshard_build_index__progress.cpp index 6744ef652b69..a2cd8a693b21 100644 --- a/ydb/core/tx/schemeshard/schemeshard_build_index__progress.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_build_index__progress.cpp @@ -256,7 +256,7 @@ THolder LockPropose( modifyScheme.MutableLockConfig()->SetName(path.LeafName()); modifyScheme.MutableLockConfig()->SetLockTxId(ui64(buildInfo.LockTxId)); - LOG_DEBUG_S((TlsActivationContext->AsActorContext()), NKikimrServices::BUILD_INDEX, + LOG_NOTICE_S((TlsActivationContext->AsActorContext()), NKikimrServices::BUILD_INDEX, "LockPropose " << buildInfo.Id << " " << buildInfo.State << " " << propose->Record.ShortDebugString()); return propose; @@ -283,7 +283,7 @@ THolder CreateIndexPropose( Y_ENSURE(false, "Unknown operation kind while building CreateIndexPropose"); } - LOG_DEBUG_S((TlsActivationContext->AsActorContext()), NKikimrServices::BUILD_INDEX, + LOG_NOTICE_S((TlsActivationContext->AsActorContext()), NKikimrServices::BUILD_INDEX, "CreateIndexPropose " << buildInfo.Id << " " << buildInfo.State << " " << propose->Record.ShortDebugString()); return propose; @@ -311,7 +311,7 @@ THolder DropBuildPropose( modifyScheme.SetOperationType(NKikimrSchemeOp::ESchemeOpDropTable); modifyScheme.MutableDrop()->SetName(path->Name); - LOG_DEBUG_S((TlsActivationContext->AsActorContext()), NKikimrServices::BUILD_INDEX, + LOG_NOTICE_S((TlsActivationContext->AsActorContext()), NKikimrServices::BUILD_INDEX, "DropBuildPropose " << buildInfo.Id << " " << buildInfo.State << " " << propose->Record.ShortDebugString()); return propose; @@ -373,7 +373,7 @@ THolder CreateBuildPropose( policy.SetMinPartitionsCount(32768); policy.SetMaxPartitionsCount(0); - LOG_DEBUG_S((TlsActivationContext->AsActorContext()), NKikimrServices::BUILD_INDEX, + LOG_NOTICE_S((TlsActivationContext->AsActorContext()), NKikimrServices::BUILD_INDEX, "CreateBuildPropose " << buildInfo.Id << " " << buildInfo.State << " " << propose->Record.ShortDebugString()); return propose; @@ -398,7 +398,7 @@ THolder CreateBuildPropose( policy.SetMaxPartitionsCount(0); } - LOG_DEBUG_S((TlsActivationContext->AsActorContext()), NKikimrServices::BUILD_INDEX, + LOG_NOTICE_S((TlsActivationContext->AsActorContext()), NKikimrServices::BUILD_INDEX, "CreateBuildPropose " << buildInfo.Id << " " << buildInfo.State << " " << propose->Record.ShortDebugString()); return propose; @@ -446,7 +446,7 @@ THolder AlterMainTablePropose( } - LOG_DEBUG_S((TlsActivationContext->AsActorContext()), NKikimrServices::BUILD_INDEX, + LOG_NOTICE_S((TlsActivationContext->AsActorContext()), NKikimrServices::BUILD_INDEX, "AlterMainTablePropose " << buildInfo.Id << " " << buildInfo.State << " " << propose->Record.ShortDebugString()); return propose; @@ -474,7 +474,7 @@ THolder ApplyPropose( indexBuild.SetSnapshotTxId(ui64(buildInfo.InitiateTxId)); indexBuild.SetBuildIndexId(ui64(buildInfo.Id)); - LOG_DEBUG_S((TlsActivationContext->AsActorContext()), NKikimrServices::BUILD_INDEX, + LOG_NOTICE_S((TlsActivationContext->AsActorContext()), NKikimrServices::BUILD_INDEX, "ApplyPropose " << buildInfo.Id << " " << buildInfo.State << " " << propose->Record.ShortDebugString()); return propose; @@ -508,7 +508,7 @@ THolder UnlockPropose( } } - LOG_DEBUG_S((TlsActivationContext->AsActorContext()), NKikimrServices::BUILD_INDEX, + LOG_NOTICE_S((TlsActivationContext->AsActorContext()), NKikimrServices::BUILD_INDEX, "UnlockPropose " << buildInfo.Id << " " << buildInfo.State << " " << propose->Record.ShortDebugString()); return propose; @@ -532,7 +532,7 @@ THolder CancelPropose( indexBuild.SetSnapshotTxId(ui64(buildInfo.InitiateTxId)); indexBuild.SetBuildIndexId(ui64(buildInfo.Id)); - LOG_DEBUG_S((TlsActivationContext->AsActorContext()), NKikimrServices::BUILD_INDEX, + LOG_NOTICE_S((TlsActivationContext->AsActorContext()), NKikimrServices::BUILD_INDEX, "CancelPropose " << buildInfo.Id << " " << buildInfo.State << " " << propose->Record.ShortDebugString()); return propose; @@ -559,12 +559,44 @@ THolder DropColumnsPropose( buildInfo.SerializeToProto(ss, settings); - LOG_DEBUG_S((TlsActivationContext->AsActorContext()), NKikimrServices::BUILD_INDEX, + LOG_NOTICE_S((TlsActivationContext->AsActorContext()), NKikimrServices::BUILD_INDEX, "DropColumnsPropose " << buildInfo.Id << " " << buildInfo.State << " " << propose->Record.ShortDebugString()); return propose; } +THolder AlterSequencePropose( + TSchemeShard* ss, const TIndexBuildInfo& buildInfo) +{ + Y_ENSURE(buildInfo.IsBuildPrefixedVectorIndex(), "Unknown operation kind while building AlterSequencePropose"); + + auto propose = MakeHolder(ui64(buildInfo.ApplyTxId), ss->TabletID()); + + NKikimrSchemeOp::TModifyScheme& modifyScheme = *propose->Record.AddTransaction(); + modifyScheme.SetOperationType(NKikimrSchemeOp::ESchemeOpAlterSequence); + modifyScheme.SetInternal(true); + + auto path = TPath::Init(buildInfo.TablePathId, ss); + path.Dive(buildInfo.IndexName); + path.Dive(NTableIndex::NKMeans::PrefixTable); + modifyScheme.SetWorkingDir(path.PathString()); + + // about 2 * TableSize per each prefix, see PrefixIndexDone and SendPrefixKMeansRequest() + ui64 minValue = NTableIndex::NKMeans::SetPostingParentFlag(buildInfo.KMeans.ChildBegin + (2 * buildInfo.KMeans.TableSize) * buildInfo.Shards.size()); + + auto seq = modifyScheme.MutableSequence(); + seq->SetName(NTableIndex::NKMeans::IdColumnSequence); + seq->SetMinValue(-0x7FFFFFFFFFFFFFFF); + seq->SetMaxValue(-1); + seq->SetStartValue(minValue); + seq->SetRestart(true); + + LOG_NOTICE_S((TlsActivationContext->AsActorContext()), NKikimrServices::BUILD_INDEX, + "AlterSequencePropose " << buildInfo.Id << " " << buildInfo.State << " " << propose->Record.ShortDebugString()); + + return propose; +} + using namespace NTabletFlatExecutor; struct TSchemeShard::TIndexBuilder::TTxProgress: public TSchemeShard::TIndexBuilder::TTxBase { @@ -1074,7 +1106,7 @@ struct TSchemeShard::TIndexBuilder::TTxProgress: public TSchemeShard::TIndexBuil } } - bool FillPrefixKMeans(TIndexBuildInfo& buildInfo) { + bool FillPrefixKMeans(TTransactionContext& txc, TIndexBuildInfo& buildInfo) { if (NoShardsAdded(buildInfo)) { AddAllShards(buildInfo); } @@ -1082,6 +1114,15 @@ struct TSchemeShard::TIndexBuilder::TTxProgress: public TSchemeShard::TIndexBuil for (auto& [shardIdx, shardStatus]: buildInfo.Shards) { shardStatus.Index = i++; } + // Set correct start value for the prefix ID sequence + if (!buildInfo.KMeans.AlterPrefixSequenceDone) { + // Alter the sequence + buildInfo.KMeans.AlterPrefixSequenceDone = true; + NIceDb::TNiceDb db{txc.DB}; + ChangeState(BuildId, TIndexBuildInfo::EState::AlterSequence); + Progress(BuildId); + return false; + } return SendToShards(buildInfo, [&](TShardIdx shardIdx) { SendPrefixKMeansRequest(shardIdx, buildInfo); }) && buildInfo.DoneShards.size() == buildInfo.Shards.size(); } @@ -1159,7 +1200,7 @@ struct TSchemeShard::TIndexBuilder::TTxProgress: public TSchemeShard::TIndexBuil return false; } else { bool filled = buildInfo.KMeans.Level == 2 - ? FillPrefixKMeans(buildInfo) + ? FillPrefixKMeans(txc, buildInfo) : FillLocalKMeans(buildInfo); if (!filled) { return false; @@ -1611,6 +1652,29 @@ struct TSchemeShard::TIndexBuilder::TTxProgress: public TSchemeShard::TIndexBuil Progress(BuildId); } break; + case TIndexBuildInfo::EState::AlterSequence: + Y_ENSURE(buildInfo.IsBuildPrefixedVectorIndex()); + Y_ENSURE(buildInfo.KMeans.Level == 2); + if (buildInfo.ApplyTxId == InvalidTxId) { + AllocateTxId(BuildId); + } else if (buildInfo.ApplyTxStatus == NKikimrScheme::StatusSuccess) { + Send(Self->SelfId(), AlterSequencePropose(Self, buildInfo), 0, ui64(BuildId)); + } else if (!buildInfo.ApplyTxDone) { + Send(Self->SelfId(), MakeHolder(ui64(buildInfo.ApplyTxId))); + } else { + buildInfo.ApplyTxId = {}; + buildInfo.ApplyTxStatus = NKikimrScheme::StatusSuccess; + buildInfo.ApplyTxDone = false; + + NIceDb::TNiceDb db(txc.DB); + Self->PersistBuildIndexApplyTxId(db, buildInfo); + Self->PersistBuildIndexApplyTxStatus(db, buildInfo); + Self->PersistBuildIndexApplyTxDone(db, buildInfo); + + ChangeState(BuildId, TIndexBuildInfo::EState::Filling); + Progress(BuildId); + } + break; case TIndexBuildInfo::EState::Applying: if (buildInfo.ApplyTxId == InvalidTxId) { AllocateTxId(BuildId); @@ -2447,6 +2511,7 @@ struct TSchemeShard::TIndexBuilder::TTxReplyCompleted: public TSchemeShard::TInd case TIndexBuildInfo::EState::DropBuild: case TIndexBuildInfo::EState::CreateBuild: case TIndexBuildInfo::EState::LockBuild: + case TIndexBuildInfo::EState::AlterSequence: case TIndexBuildInfo::EState::Applying: case TIndexBuildInfo::EState::Cancellation_Applying: case TIndexBuildInfo::EState::Rejection_Applying: @@ -2609,6 +2674,7 @@ struct TSchemeShard::TIndexBuilder::TTxReplyModify: public TSchemeShard::TIndexB case TIndexBuildInfo::EState::DropBuild: case TIndexBuildInfo::EState::CreateBuild: case TIndexBuildInfo::EState::LockBuild: + case TIndexBuildInfo::EState::AlterSequence: case TIndexBuildInfo::EState::Applying: case TIndexBuildInfo::EState::Rejection_Applying: { @@ -2739,6 +2805,7 @@ struct TSchemeShard::TIndexBuilder::TTxReplyAllocate: public TSchemeShard::TInde case TIndexBuildInfo::EState::DropBuild: case TIndexBuildInfo::EState::CreateBuild: case TIndexBuildInfo::EState::LockBuild: + case TIndexBuildInfo::EState::AlterSequence: case TIndexBuildInfo::EState::Applying: case TIndexBuildInfo::EState::Cancellation_Applying: case TIndexBuildInfo::EState::Rejection_Applying: diff --git a/ydb/core/tx/schemeshard/schemeshard_build_index_tx_base.cpp b/ydb/core/tx/schemeshard/schemeshard_build_index_tx_base.cpp index 675b2cde48ef..5195091a1b70 100644 --- a/ydb/core/tx/schemeshard/schemeshard_build_index_tx_base.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_build_index_tx_base.cpp @@ -232,6 +232,7 @@ void TSchemeShard::TIndexBuilder::TTxBase::Fill(NKikimrIndexBuilder::TIndexBuild case TIndexBuildInfo::EState::DropBuild: case TIndexBuildInfo::EState::CreateBuild: case TIndexBuildInfo::EState::LockBuild: + case TIndexBuildInfo::EState::AlterSequence: index.SetState(Ydb::Table::IndexBuildState::STATE_TRANSFERING_DATA); index.SetProgress(indexInfo.CalcProgressPercent()); break; diff --git a/ydb/core/tx/schemeshard/schemeshard_info_types.h b/ydb/core/tx/schemeshard/schemeshard_info_types.h index 048d2dfdc067..75b02d78c38c 100644 --- a/ydb/core/tx/schemeshard/schemeshard_info_types.h +++ b/ydb/core/tx/schemeshard/schemeshard_info_types.h @@ -3108,6 +3108,7 @@ struct TIndexBuildInfo: public TSimpleRefCount { LockBuild = 47, Applying = 50, Unlocking = 60, + AlterSequence = 61, Done = 200, Cancellation_Applying = 350, @@ -3218,6 +3219,8 @@ struct TIndexBuildInfo: public TSimpleRefCount { EState State = Sample; + bool AlterPrefixSequenceDone = false; + NTableIndex::NKMeans::TClusterId ParentBegin = 0; // included NTableIndex::NKMeans::TClusterId Parent = ParentBegin; diff --git a/ydb/core/tx/schemeshard/schemeshard_utils.cpp b/ydb/core/tx/schemeshard/schemeshard_utils.cpp index 7ef907b8cd0f..32883fd9d895 100644 --- a/ydb/core/tx/schemeshard/schemeshard_utils.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_utils.cpp @@ -307,6 +307,7 @@ auto CalcVectorKmeansTreePrefixImplTableDescImpl( idColumn->SetType(NTableIndex::NKMeans::ClusterIdTypeName); idColumn->SetTypeId(NSchemeShard::ClusterIdTypeId); idColumn->SetNotNull(true); + idColumn->SetDefaultFromSequence(NKMeans::IdColumnSequence); } implTableDesc.AddKeyColumnNames(NKMeans::IdColumn);