Create a sequence for prefixed vector indexes

ydb/core/base/table_index.h

@@ -72,6 +72,7 @@ inline constexpr const char* BuildSuffix1 = "1build";
 
 // Prefix table
 inline constexpr const char* PrefixTable = "indexImplPrefixTable";
+inline constexpr const char* IdColumnSequence = "__ydb_id_sequence";
 
 inline constexpr const int DefaultKMeansRounds = 3;
 

ydb/core/kqp/ut/indexes/kqp_indexes_prefixed_vector_ut.cpp

@@ -311,6 +311,12 @@ Y_UNIT_TEST_SUITE(KqpPrefixedVectorIndexes) {
             UNIT_ASSERT_EQUAL(settings.Clusters, 2);
         }
         DoPositiveQueriesPrefixedVectorIndexOrderByCosine(session);
+
+        {
+            const TString dropIndex(Q_("ALTER TABLE `/Root/TestTable` DROP INDEX index"));
+            auto result = session.ExecuteSchemeQuery(dropIndex).ExtractValueSync();
+            UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+        }
     }
 
     Y_UNIT_TEST_QUAD(OrderByCosineLevel2, Nullable, UseSimilarity) {

ydb/core/kqp/ut/indexes/kqp_indexes_vector_ut.cpp

@@ -297,6 +297,12 @@ Y_UNIT_TEST_SUITE(KqpVectorIndexes) {
             UNIT_ASSERT_EQUAL(settings.Clusters, 2);
         }
         DoPositiveQueriesVectorIndexOrderByCosine(session);
+
+        {
+            const TString dropIndex(Q_("ALTER TABLE `/Root/TestTable` DROP INDEX index"));
+            auto result = session.ExecuteSchemeQuery(dropIndex).ExtractValueSync();
+            UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+        }
     }
 
     Y_UNIT_TEST_QUAD(OrderByCosineLevel2, Nullable, UseSimilarity) {

ydb/core/tx/schemeshard/schemeshard__operation_alter_sequence.cpp

@@ -429,18 +429,26 @@ class TAlterSequence: public TSubOperation {
                 .IsAtLocalSchemeShard()
                 .IsResolved()
                 .NotDeleted()
-                .NotUnderDeleting()
-                .IsCommonSensePath();
+                .NotUnderDeleting();
 
             if (checks) {
-                if (parentPath->IsTable()) {
+                if (parentPath.Parent()->IsTableIndex()) {
+                    checks.IsInsideTableIndexPath();
+                    // Only __ydb_id sequence can be altered and only by internal transactions (build_index__progress)
+                    if (name != NTableIndex::NKMeans::IdColumnSequence || !Transaction.GetInternal()) {
+                        result->SetError(NKikimrScheme::EStatus::StatusNameConflict, "sequences are not allowed in indexes");
+                        return result;
+                    }
+                } else if (parentPath->IsTable()) {
                     // allow immediately inside a normal table
+                    checks.IsCommonSensePath();
                     if (parentPath.IsUnderOperation()) {
                         checks.IsUnderTheSameOperation(OperationId.GetTxId()); // allowed only as part of consistent operations
                     }
                 } else {
                     // otherwise don't allow unexpected object types
-                    checks.IsLikeDirectory();
+                    checks.IsCommonSensePath()
+                        .IsLikeDirectory();
                 }
             }
 

ydb/core/tx/schemeshard/schemeshard__operation_consistent_copy_tables.cpp

@@ -203,9 +203,9 @@ bool CreateConsistentCopyTables(
                 Y_ABORT_UNLESS(srcImplTable.Base()->PathId == srcImplTablePathId);
                 TPath dstImplTable = dstIndexPath.Child(srcImplTableName);
 
-                result.push_back(CreateCopyTable(
-                                     NextPartId(nextId, result),
-                                     CopyTableTask(srcImplTable, dstImplTable, descr)));
+                result.push_back(CreateCopyTable(NextPartId(nextId, result),
+                    CopyTableTask(srcImplTable, dstImplTable, descr), GetLocalSequences(context, srcImplTable)));
+                AddCopySequences(nextId, tx, context, result, srcImplTable, dstImplTable.PathString());
             }
         }
 

ydb/core/tx/schemeshard/schemeshard__operation_copy_sequence.cpp

@@ -551,18 +551,27 @@ class TCopySequence: public TSubOperation {
                 .IsResolved()
                 .NotDeleted()
                 .NotUnderDeleting()
-                .IsCommonSensePath()
                 .FailOnRestrictedCreateInTempZone(Transaction.GetAllowCreateInTempDir());
 
             if (checks) {
-                if (parentPath->IsTable()) {
+                if (parentPath.Parent()->IsTableIndex()) {
+                    // Only __ydb_id sequence can be created in the prefixed index
+                    if (name != NTableIndex::NKMeans::IdColumnSequence) {
+                        result->SetError(NKikimrScheme::EStatus::StatusNameConflict, "sequences are not allowed in indexes");
+                        return result;
+                    }
+                    checks.IsUnderTheSameOperation(OperationId.GetTxId()); // allowed only as part of consistent operations
+                    checks.IsInsideTableIndexPath();
+                } else if (parentPath->IsTable()) {
                     // allow immediately inside a normal table
+                    checks.IsCommonSensePath();
                     if (parentPath.IsUnderOperation()) {
                         checks.IsUnderTheSameOperation(OperationId.GetTxId()); // allowed only as part of consistent operations
                     }
                 } else {
                     // otherwise don't allow unexpected object types
-                    checks.IsLikeDirectory();
+                    checks.IsCommonSensePath()
+                          .IsLikeDirectory();
                 }
             }
 

ydb/core/tx/schemeshard/schemeshard__operation_copy_table.cpp

@@ -399,9 +399,7 @@ class TCopyTable: public TSubOperation {
 
             if (checks) {
                 if (parent.Base()->IsTableIndex()) {
-                    checks
-                        .IsInsideTableIndexPath() //copy imp index table as index index table, not a separate one
-                        .NotChildren(); //imp table doesn't have indexes
+                    checks.IsInsideTableIndexPath(); //copy imp index table as index index table, not a separate one
                 } else {
                     checks.IsCommonSensePath();
                 }
@@ -869,7 +867,8 @@ TVector<ISubOperation::TPtr> CreateCopyTable(TOperationId nextId, const TTxTrans
             operation->SetOmitFollowers(copying.GetOmitFollowers());
             operation->SetIsBackup(copying.GetIsBackup());
 
-            result.push_back(CreateCopyTable(NextPartId(nextId, result), schema));
+            result.push_back(CreateCopyTable(NextPartId(nextId, result), schema, GetLocalSequences(context, implTable)));
+            AddCopySequences(nextId, tx, context, result, implTable, JoinPath({dstPath.PathString(), name, implTableName}));
         }
     }
 

ydb/core/tx/schemeshard/schemeshard__operation_create_build_index.cpp

@@ -140,7 +140,7 @@ TVector<ISubOperation::TPtr> CreateBuildIndex(TOperationId opId, const TTxTransa
         result.push_back(CreateInitializeBuildIndexMainTable(NextPartId(opId, result), outTx));
     }
 
-    auto createImplTable = [&](NKikimrSchemeOp::TTableDescription&& implTableDesc) {
+    auto createImplTable = [&](NKikimrSchemeOp::TTableDescription&& implTableDesc, const THashSet<TString>& localSequences = {}) {
         if (GetIndexType(indexDesc) != NKikimrSchemeOp::EIndexTypeGlobalUnique) {
             implTableDesc.MutablePartitionConfig()->SetShadowData(true);
         }
@@ -149,7 +149,7 @@ TVector<ISubOperation::TPtr> CreateBuildIndex(TOperationId opId, const TTxTransa
         *outTx.MutableCreateTable() = std::move(implTableDesc);
         outTx.SetInternal(tx.GetInternal());
 
-        return CreateInitializeBuildIndexImplTable(NextPartId(opId, result), outTx);
+        return CreateInitializeBuildIndexImplTable(NextPartId(opId, result), outTx, localSequences);
     };
 
     switch (GetIndexType(indexDesc)) {
@@ -183,7 +183,13 @@ TVector<ISubOperation::TPtr> CreateBuildIndex(TOperationId opId, const TTxTransa
             result.push_back(createImplTable(CalcVectorKmeansTreePostingImplTableDesc(tableInfo, tableInfo->PartitionConfig(), indexDataColumns, indexPostingTableDesc)));
             if (prefixVectorIndex) {
                 const THashSet<TString> prefixColumns{indexDesc.GetKeyColumnNames().begin(), indexDesc.GetKeyColumnNames().end() - 1};
-                result.push_back(createImplTable(CalcVectorKmeansTreePrefixImplTableDesc(prefixColumns, tableInfo, tableInfo->PartitionConfig(), implTableColumns, indexPrefixTableDesc)));
+                result.push_back(createImplTable(CalcVectorKmeansTreePrefixImplTableDesc(
+                    prefixColumns, tableInfo, tableInfo->PartitionConfig(), implTableColumns, indexPrefixTableDesc),
+                    THashSet<TString>{NTableIndex::NKMeans::IdColumnSequence}));
+                auto outTx = TransactionTemplate(index.PathString() + "/" + NTableIndex::NKMeans::PrefixTable, NKikimrSchemeOp::EOperationType::ESchemeOpCreateSequence);
+                outTx.MutableSequence()->SetName(NTableIndex::NKMeans::IdColumnSequence);
+                outTx.SetInternal(tx.GetInternal());
+                result.push_back(CreateNewSequence(NextPartId(opId, result), outTx));
             }
             break;
         }

ydb/core/tx/schemeshard/schemeshard__operation_create_indexed_table.cpp

@@ -290,7 +290,7 @@ TVector<ISubOperation::TPtr> CreateIndexedTable(TOperationId nextId, const TTxTr
             result.push_back(CreateNewTableIndex(NextPartId(nextId, result), scheme));
         }
 
-        auto createIndexImplTable = [&] (NKikimrSchemeOp::TTableDescription&& implTableDesc) {
+        auto createIndexImplTable = [&] (NKikimrSchemeOp::TTableDescription&& implTableDesc, const THashSet<TString>& localSequences = {}) {
             auto scheme = TransactionTemplate(
                 tx.GetWorkingDir() + "/" + baseTableDescription.GetName() + "/" + indexDescription.GetName(),
                 NKikimrSchemeOp::EOperationType::ESchemeOpCreateTable);
@@ -300,7 +300,7 @@ TVector<ISubOperation::TPtr> CreateIndexedTable(TOperationId nextId, const TTxTr
 
             *scheme.MutableCreateTable() = std::move(implTableDesc);
 
-            return CreateNewTable(NextPartId(nextId, result), scheme);
+            return CreateNewTable(NextPartId(nextId, result), scheme, localSequences);
         };
 
         const auto& implTableColumns = indexes.at(indexDescription.GetName());
@@ -332,7 +332,22 @@ TVector<ISubOperation::TPtr> CreateIndexedTable(TOperationId nextId, const TTxTr
                 result.push_back(createIndexImplTable(CalcVectorKmeansTreePostingImplTableDesc(baseTableDescription, baseTableDescription.GetPartitionConfig(), indexDataColumns, userPostingDesc)));
                 if (prefixVectorIndex) {
                     const THashSet<TString> prefixColumns{indexDescription.GetKeyColumnNames().begin(), indexDescription.GetKeyColumnNames().end() - 1};
-                    result.push_back(createIndexImplTable(CalcVectorKmeansTreePrefixImplTableDesc(prefixColumns, baseTableDescription, baseTableDescription.GetPartitionConfig(), implTableColumns, userPrefixDesc)));
+                    result.push_back(createIndexImplTable(CalcVectorKmeansTreePrefixImplTableDesc(
+                        prefixColumns, baseTableDescription, baseTableDescription.GetPartitionConfig(), implTableColumns, userPrefixDesc),
+                        THashSet<TString>{NTableIndex::NKMeans::IdColumnSequence}));
+                    // Create the sequence
+                    auto outTx = TransactionTemplate(tx.GetWorkingDir() + "/" + baseTableDescription.GetName() + "/" +
+                        indexDescription.GetName() + "/" + NTableIndex::NKMeans::PrefixTable,
+                        NKikimrSchemeOp::EOperationType::ESchemeOpCreateSequence);
+                    outTx.MutableSequence()->SetName(NTableIndex::NKMeans::IdColumnSequence);
+                    outTx.MutableSequence()->SetMinValue(-0x7FFFFFFFFFFFFFFF);
+                    outTx.MutableSequence()->SetMaxValue(-1);
+                    outTx.MutableSequence()->SetStartValue(NTableIndex::NKMeans::SetPostingParentFlag(1));
+                    outTx.MutableSequence()->SetRestart(true);
+                    outTx.SetFailOnExist(tx.GetFailOnExist());
+                    outTx.SetAllowCreateInTempDir(tx.GetAllowCreateInTempDir());
+                    outTx.SetInternal(tx.GetInternal());
+                    result.push_back(CreateNewSequence(NextPartId(nextId, result), outTx));
                 }
                 break;
             }

ydb/core/tx/schemeshard/schemeshard__operation_create_sequence.cpp

@@ -389,19 +389,30 @@ class TCreateSequence : public TSubOperation {
                 .IsResolved()
                 .NotDeleted()
                 .NotUnderDeleting()
-                .IsCommonSensePath()
                 .FailOnRestrictedCreateInTempZone(Transaction.GetAllowCreateInTempDir());
 
             if (checks) {
-                if (parentPath->IsTable()) {
-                    checks.NotBackupTable();
+                if (parentPath.Parent()->IsTableIndex()) {
+                    // Only __ydb_id sequence can be created in the prefixed index
+                    if (name != NTableIndex::NKMeans::IdColumnSequence) {
+                        result->SetError(NKikimrScheme::EStatus::StatusNameConflict, "sequences are not allowed in indexes");
+                        return result;
+                    }
+                    checks.IsInsideTableIndexPath()
+                        .IsUnderCreating()
+                        .IsUnderTheSameOperation(OperationId.GetTxId()); // allowed only as part of consistent operations
+                } else if (parentPath->IsTable()) {
+                    checks
+                        .IsCommonSensePath()
+                        .NotBackupTable();
                     // allow immediately inside a normal table
                     if (parentPath.IsUnderOperation()) {
                         checks.IsUnderTheSameOperation(OperationId.GetTxId()); // allowed only as part of consistent operations
                     }
-                } else {
+                } else if (!Transaction.GetAllowAccessToPrivatePaths()) {
                     // otherwise don't allow unexpected object types
-                    checks.IsLikeDirectory();
+                    checks.IsCommonSensePath()
+                          .IsLikeDirectory();
                 }
             }
 

ydb/core/tx/schemeshard/schemeshard__operation_create_table.cpp

@@ -821,9 +821,11 @@ ISubOperation::TPtr CreateNewTable(TOperationId id, TTxState::ETxState state) {
     return MakeSubOperation<TCreateTable>(id, state);
 }
 
-ISubOperation::TPtr CreateInitializeBuildIndexImplTable(TOperationId id, const TTxTransaction& tx) {
+ISubOperation::TPtr CreateInitializeBuildIndexImplTable(TOperationId id, const TTxTransaction& tx, const THashSet<TString>& localSequences) {
     auto obj = MakeSubOperation<TCreateTable>(id, tx);
-    static_cast<TCreateTable*>(obj.Get())->SetAllowShadowDataForBuildIndex();
+    TCreateTable *createTable = static_cast<TCreateTable*>(obj.Get());
+    createTable->SetAllowShadowDataForBuildIndex();
+    createTable->SetLocalSequences(localSequences);
     return obj;
 }
 

ydb/core/tx/schemeshard/schemeshard__operation_drop_sequence.cpp

@@ -291,19 +291,29 @@ class TDropSequence: public TSubOperation {
                 .NotUnderDomainUpgrade()
                 .IsAtLocalSchemeShard()
                 .IsResolved()
-                .NotDeleted()
-                .IsCommonSensePath();
+                .NotDeleted();
 
             if (checks) {
-                if (parent->IsTable()) {
+                if (parent.Parent()->IsTableIndex()) {
+                    // Only __ydb_id sequence can be present in the prefixed index
+                    if (name != NTableIndex::NKMeans::IdColumnSequence) {
+                        result->SetError(NKikimrScheme::EStatus::StatusNameConflict, "sequences are not allowed in indexes");
+                        return result;
+                    }
+                    checks.IsInsideTableIndexPath()
+                        .IsUnderDeleting()
+                        .IsUnderTheSameOperation(OperationId.GetTxId()); // allowed only as part of consistent operations
+                } else if (parent->IsTable()) {
                     // allow immediately inside a normal table
                     if (parent.IsUnderOperation()) {
                         checks.IsUnderTheSameOperation(OperationId.GetTxId()); // allowed only as part of consistent operations
                     }
+                    checks.IsCommonSensePath();
                 } else {
                     checks
                         .NotUnderDeleting()
-                        .IsLikeDirectory();
+                        .IsLikeDirectory()
+                        .IsCommonSensePath();
                 }
             }
 

ydb/core/tx/schemeshard/schemeshard__operation_part.h

@@ -604,7 +604,7 @@ ISubOperation::TPtr CreateDropSolomon(TOperationId id, TTxState::ETxState state)
 ISubOperation::TPtr CreateInitializeBuildIndexMainTable(TOperationId id, const TTxTransaction& tx);
 ISubOperation::TPtr CreateInitializeBuildIndexMainTable(TOperationId id, TTxState::ETxState state);
 
-ISubOperation::TPtr CreateInitializeBuildIndexImplTable(TOperationId id, const TTxTransaction& tx);
+ISubOperation::TPtr CreateInitializeBuildIndexImplTable(TOperationId id, const TTxTransaction& tx, const THashSet<TString>& localSequences = {});
 ISubOperation::TPtr CreateInitializeBuildIndexImplTable(TOperationId id, TTxState::ETxState state);
 
 ISubOperation::TPtr CreateFinalizeBuildIndexImplTable(TOperationId id, const TTxTransaction& tx);