diff --git a/.github/config/muted_ya.txt b/.github/config/muted_ya.txt index 3b903a948d73..afd3f5c805d6 100644 --- a/.github/config/muted_ya.txt +++ b/.github/config/muted_ya.txt @@ -20,6 +20,26 @@ ydb/core/kqp/ut/query KqpAnalyze.AnalyzeTable+ColumnStore ydb/core/kqp/ut/scheme KqpOlapScheme.AddPgColumnWithStore ydb/core/persqueue/ut/ut_with_sdk TopicTimestamp.TimestampRead_40MB_Topic_offset+middle_WithRestart ydb/core/statistics/aggregator/ut AnalyzeColumnshard.AnalyzeRebootColumnShard +ydb/core/sys_view/ut AuthSystemView.AuthGroupMembers_Access +ydb/core/sys_view/ut AuthSystemView.AuthGroups_Access +ydb/core/sys_view/ut AuthSystemView.AuthOwners_Access +ydb/core/sys_view/ut AuthSystemView.AuthPermissions_Access +ydb/core/sys_view/ut SystemView.ShowCreateTableChangefeeds +ydb/core/sys_view/ut SystemView.SystemViewFailOps+EnableRealSystemViewPaths +ydb/core/tablet_flat/ut_large TFlatTableLongTxLarge.LargeDeltaChain +ydb/core/tablet_flat/ut_large unittest.sole chunk +ydb/core/transfer/ut/large TransferLarge.Transfer100KM_10P_LocalRead_TopicAutoPartitioning +ydb/core/transfer/ut/large TransferLarge.Transfer100KM_10P_RowTable_TopicAutoPartitioning +ydb/core/tx/datashard/ut_order DataShardTxOrder.RandomPointsAndRanges +ydb/core/tx/datashard/ut_order unittest.[*/*] chunk +ydb/core/tx/schemeshard/ut_background_cleaning TSchemeshardBackgroundCleaningTest.SchemeshardBackgroundCleaningTestCreateCleanManyTables +ydb/core/tx/schemeshard/ut_base_reboots TTablesWithReboots.CopyIndexedTableWithReboots +ydb/core/tx/schemeshard/ut_base_reboots unittest.[*/*] chunk +ydb/core/tx/schemeshard/ut_cdc_stream_reboots TCdcStreamWithRebootsTests.CreateDropRecreate[TabletReboots] +ydb/core/tx/schemeshard/ut_cdc_stream_reboots unittest.[*/*] chunk +ydb/core/tx/schemeshard/ut_export_reboots_s3 TExportToS3WithRebootsTests.CancelOnOnSingleTopic +ydb/core/tx/schemeshard/ut_export_reboots_s3 TExportToS3WithRebootsTests.CancelOnSingleShardTableWithChangefeed +ydb/core/tx/schemeshard/ut_export_reboots_s3 TExportToS3WithRebootsTests.CancelShouldSucceedOnMultiShardTable ydb/core/tx/schemeshard/ut_export_reboots_s3 TExportToS3WithRebootsTests.CancelShouldSucceedOnSingleShardTableWithUniqueIndex ydb/core/tx/schemeshard/ut_vector_index_build_reboots VectorIndexBuildTestReboots.BaseCase+Prefixed[TabletReboots] ydb/core/tx/schemeshard/ut_vector_index_build_reboots unittest.[*/*] chunk diff --git a/ydb/core/kqp/host/kqp_gateway_proxy.cpp b/ydb/core/kqp/host/kqp_gateway_proxy.cpp index 0066ce269f40..ec466f0ff099 100644 --- a/ydb/core/kqp/host/kqp_gateway_proxy.cpp +++ b/ydb/core/kqp/host/kqp_gateway_proxy.cpp @@ -1339,7 +1339,8 @@ class TKqpGatewayProxy : public IKikimrGateway { op.SetPrefix(settings.Prefix); if (settings.Settings.IncrementalBackupEnabled) { - op.MutableIncrementalBackupConfig(); + auto* config = op.MutableIncrementalBackupConfig(); + config->SetOmitIndexes(settings.Settings.OmitIndexes); } auto errOpt = std::visit( diff --git a/ydb/core/kqp/provider/yql_kikimr_exec.cpp b/ydb/core/kqp/provider/yql_kikimr_exec.cpp index 50e8b87c6547..4d880d550f7b 100644 --- a/ydb/core/kqp/provider/yql_kikimr_exec.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_exec.cpp @@ -1028,6 +1028,13 @@ namespace { "INCREMENTAL_BACKUP_ENABLED must be true or false")); return false; } + } else if (name == "omit_indexes") { + auto value = ToString(setting.Value().Cast().Literal().Cast().Value()); + if (!TryFromString(value, dstSettings.OmitIndexes)) { + ctx.AddError(TIssue(ctx.GetPosition(pos), + "OMIT_INDEXES must be true or false")); + return false; + } } else if (name == "storage") { auto value = ToString(setting.Value().Cast().Literal().Cast().Value()); if (to_lower(value) != "cluster") { diff --git a/ydb/core/kqp/provider/yql_kikimr_gateway.h b/ydb/core/kqp/provider/yql_kikimr_gateway.h index 4801d8176165..a00c3d19d780 100644 --- a/ydb/core/kqp/provider/yql_kikimr_gateway.h +++ b/ydb/core/kqp/provider/yql_kikimr_gateway.h @@ -1003,7 +1003,8 @@ struct TAnalyzeSettings { }; struct TBackupCollectionSettings { - bool IncrementalBackupEnabled; + bool IncrementalBackupEnabled = false; + bool OmitIndexes = false; }; struct TCreateBackupCollectionSettings { diff --git a/ydb/core/kqp/provider/yql_kikimr_type_ann.cpp b/ydb/core/kqp/provider/yql_kikimr_type_ann.cpp index d772d9bed268..9ef67e37baf9 100644 --- a/ydb/core/kqp/provider/yql_kikimr_type_ann.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_type_ann.cpp @@ -2396,6 +2396,7 @@ virtual TStatus HandleCreateTable(TKiCreateTable create, TExprContext& ctx) over const THashSet supportedSettings = { "incremental_backup_enabled", "storage", + "omit_indexes", }; if (!CheckBackupCollectionSettings(node.BackupCollectionSettings(), supportedSettings, ctx)) { diff --git a/ydb/core/protos/flat_scheme_op.proto b/ydb/core/protos/flat_scheme_op.proto index 657ce93ee087..6d3ca02b457b 100644 --- a/ydb/core/protos/flat_scheme_op.proto +++ b/ydb/core/protos/flat_scheme_op.proto @@ -346,6 +346,8 @@ message TTableDescription { optional bool AllowUnderSameOperation = 44 [default = false]; // Create only as-well. Used for CopyTable to create table in desired state instead of default optional EPathState PathState = 46; + // Skip automatic index/impl table copying - indexes will be handled separately + optional bool OmitIndexes = 47 [default = false]; } message TDictionaryEncodingSettings { @@ -1273,6 +1275,11 @@ message TCopyTableConfig { //TTableDescription implemets copying a table in orig optional bool AllowUnderSameOperation = 7 [default = false]; optional NKikimrSchemeOp.EPathState TargetPathTargetState = 8; + + // Map from index name to CDC stream config for incremental backups + // Key: index name (e.g., "age_index", "name_index") + // Value: CDC stream configuration to create on that index's impl table + map IndexImplTableCdcStreams = 9; } message TConsistentTableCopyingConfig { @@ -2302,7 +2309,7 @@ message TBackupCollectionDescription { } message TIncrementalBackupConfig { - + optional bool OmitIndexes = 1 [default = false]; } oneof Entries { @@ -2316,6 +2323,8 @@ message TBackupCollectionDescription { oneof Storage { google.protobuf.Empty Cluster = 7; } + + optional bool OmitIndexes = 9 [default = false]; } message TBackupBackupCollection { diff --git a/ydb/core/tx/datashard/datashard_ut_common_kqp.h b/ydb/core/tx/datashard/datashard_ut_common_kqp.h index 87c7beb34e7c..de1d09e88c80 100644 --- a/ydb/core/tx/datashard/datashard_ut_common_kqp.h +++ b/ydb/core/tx/datashard/datashard_ut_common_kqp.h @@ -3,6 +3,7 @@ #include #include #include +#include namespace NKikimr { namespace NDataShard { @@ -182,6 +183,14 @@ namespace NKqpHelpers { return FormatResult(response); } + inline TString KqpSimpleExecSuccess(TTestActorRuntime& runtime, const TString& query, bool staleRo = false, const TString& database = {}, NYdb::NUt::TTestContext testCtx = NYdb::NUt::TTestContext()) { + auto response = AwaitResponse(runtime, KqpSimpleSend(runtime, query, staleRo, database)); + CTX_UNIT_ASSERT_VALUES_EQUAL_C(response.operation().status(), Ydb::StatusIds::SUCCESS, + "Query failed: " << query << ", status: " << response.operation().status() + << ", issues: " << response.operation().issues()); + return FormatResult(response); + } + inline auto KqpSimpleStaleRoSend(TTestActorRuntime& runtime, const TString& query, const TString& database = {}) { return KqpSimpleSend(runtime, query, true, database); } diff --git a/ydb/core/tx/datashard/datashard_ut_incremental_backup.cpp b/ydb/core/tx/datashard/datashard_ut_incremental_backup.cpp index b9874b431b01..a9fcbc0a1791 100644 --- a/ydb/core/tx/datashard/datashard_ut_incremental_backup.cpp +++ b/ydb/core/tx/datashard/datashard_ut_incremental_backup.cpp @@ -39,6 +39,11 @@ Y_UNIT_TEST_SUITE(IncrementalBackup) { using TCdcStream = TShardedTableOptions::TCdcStream; + struct CdcOperationCounts { + int Deletes = 0; + int Inserts = 0; + }; + static NKikimrPQ::TPQConfig DefaultPQConfig() { NKikimrPQ::TPQConfig pqConfig; pqConfig.SetEnabled(true); @@ -140,14 +145,14 @@ Y_UNIT_TEST_SUITE(IncrementalBackup) { google::protobuf::util::MessageDifferencer md; auto fieldComparator = google::protobuf::util::DefaultFieldComparator(); md.set_field_comparator(&fieldComparator); - + TString diff; google::protobuf::io::StringOutputStream diffStream(&diff); google::protobuf::util::MessageDifferencer::StreamReporter reporter(&diffStream); md.ReportDifferencesTo(&reporter); - + bool isEqual = md.Compare(proto.GetCdcDataChange(), expected.at(i).GetCdcDataChange()); - + UNIT_ASSERT_C(isEqual, "CDC data change mismatch at record " << i << ":\n" << diff << "\nActual: " << proto.GetCdcDataChange().ShortDebugString() @@ -173,7 +178,7 @@ Y_UNIT_TEST_SUITE(IncrementalBackup) { auto& dcKey = *dc.MutableKey(); dcKey.AddTags(1); dcKey.SetData(TSerializedCellVec::Serialize({keyCell})); - + auto& upsert = *dc.MutableUpsert(); upsert.AddTags(2); upsert.SetData(TSerializedCellVec::Serialize({valueCell})); @@ -190,7 +195,7 @@ Y_UNIT_TEST_SUITE(IncrementalBackup) { auto& dcKey = *dc.MutableKey(); dcKey.AddTags(1); dcKey.SetData(TSerializedCellVec::Serialize({keyCell})); - + auto& reset = *dc.MutableReset(); reset.AddTags(2); reset.SetData(TSerializedCellVec::Serialize({valueCell})); @@ -211,6 +216,99 @@ Y_UNIT_TEST_SUITE(IncrementalBackup) { return proto; } + TString FindIncrementalBackupDir(TTestActorRuntime& runtime, const TActorId& sender, const TString& collectionPath) { + auto request = MakeHolder(); + request->Record.MutableDescribePath()->SetPath(collectionPath); + request->Record.MutableDescribePath()->MutableOptions()->SetReturnChildren(true); + runtime.Send(new IEventHandle(MakeTxProxyID(), sender, request.Release())); + auto reply = runtime.GrabEdgeEventRethrow(sender); + + UNIT_ASSERT_EQUAL(reply->Get()->GetRecord().GetStatus(), NKikimrScheme::EStatus::StatusSuccess); + + const auto& pathDescription = reply->Get()->GetRecord().GetPathDescription(); + for (ui32 i = 0; i < pathDescription.ChildrenSize(); ++i) { + const auto& child = pathDescription.GetChildren(i); + if (child.GetName().EndsWith("_incremental")) { + return child.GetName(); + } + } + return ""; + } + + struct TCdcMetadata { + bool IsDelete; + TVector UpdatedColumns; + TVector ErasedColumns; + }; + + TCdcMetadata ParseCdcMetadata(const TString& bytesValue) { + TCdcMetadata result; + result.IsDelete = false; + + // The bytes contain protobuf-encoded CDC metadata + // For Update mode CDC: + // - Updates have \020\000 (indicating value columns present) + // - Deletes have \020\001 (indicating erase operation) + + if (bytesValue.find("\020\001") != TString::npos) { + result.IsDelete = true; + } + + // Parse column tags from the metadata + // Format: \010\020 + for (size_t i = 0; i < bytesValue.size(); ++i) { + if (bytesValue[i] == '\010' && i + 1 < bytesValue.size()) { + ui32 tag = static_cast(bytesValue[i + 1]); + if (i + 2 < bytesValue.size() && bytesValue[i + 2] == '\020') { + ui8 flags = i + 3 < bytesValue.size() ? static_cast(bytesValue[i + 3]) : 0; + if (flags & 1) { + result.ErasedColumns.push_back(tag); + } else { + result.UpdatedColumns.push_back(tag); + } + } + } + } + + return result; + } + + CdcOperationCounts CountCdcOperations(const TString& backup) { + CdcOperationCounts counts; + size_t pos = 0; + + while ((pos = backup.find("bytes_value: \"", pos)) != TString::npos) { + pos += 14; + size_t endPos = backup.find("\"", pos); + if (endPos == TString::npos) break; + + TString metadataStr = backup.substr(pos, endPos - pos); + TString unescaped; + for (size_t i = 0; i < metadataStr.size(); ++i) { + if (metadataStr[i] == '\\' && i + 3 < metadataStr.size()) { + ui8 val = ((metadataStr[i+1] - '0') << 6) | + ((metadataStr[i+2] - '0') << 3) | + (metadataStr[i+3] - '0'); + unescaped += static_cast(val); + i += 3; + } else { + unescaped += metadataStr[i]; + } + } + + auto metadata = ParseCdcMetadata(unescaped); + if (metadata.IsDelete) { + counts.Deletes++; + } else { + counts.Inserts++; + } + + pos = endPos + 1; + } + + return counts; + } + NKikimrChangeExchange::TChangeRecord MakeUpsertPartial(ui32 key, ui32 value, const TVector& tags = {2}) { auto keyCell = TCell::Make(key); auto valueCell = TCell::Make(value); @@ -220,7 +318,7 @@ Y_UNIT_TEST_SUITE(IncrementalBackup) { auto& dcKey = *dc.MutableKey(); dcKey.AddTags(1); dcKey.SetData(TSerializedCellVec::Serialize({keyCell})); - + auto& upsert = *dc.MutableUpsert(); for (auto tag : tags) { upsert.AddTags(tag); @@ -239,7 +337,7 @@ Y_UNIT_TEST_SUITE(IncrementalBackup) { auto& dcKey = *dc.MutableKey(); dcKey.AddTags(1); dcKey.SetData(TSerializedCellVec::Serialize({keyCell})); - + auto& reset = *dc.MutableReset(); for (auto tag : tags) { reset.AddTags(tag); @@ -253,17 +351,17 @@ Y_UNIT_TEST_SUITE(IncrementalBackup) { TString SerializeChangeMetadata(bool isDeleted = false, const TVector>>& columnStates = {}) { NKikimrBackup::TChangeMetadata metadata; metadata.SetIsDeleted(isDeleted); - + for (const auto& [tag, state] : columnStates) { auto* columnState = metadata.AddColumnStates(); columnState->SetTag(tag); columnState->SetIsNull(state.first); columnState->SetIsChanged(state.second); } - + TString binaryData; Y_PROTOBUF_SUPPRESS_NODISCARD metadata.SerializeToString(&binaryData); - + // Convert binary data to hex escape sequences for YDB SQL string literal TString result; for (unsigned char byte : binaryData) { @@ -410,6 +508,87 @@ Y_UNIT_TEST_SUITE(IncrementalBackup) { "{ items { uint32_value: 3 } items { uint32_value: 30 } }"); } + Y_UNIT_TEST(SimpleBackupRestoreWithIndex) { + TPortManager portManager; + TServer::TPtr server = new TServer(TServerSettings(portManager.GetPort(2134), {}, DefaultPQConfig()) + .SetUseRealThreads(false) + .SetDomainName("Root") + .SetEnableBackupService(true) + .SetEnableChangefeedInitialScan(true) + ); + + auto& runtime = *server->GetRuntime(); + const auto edgeActor = runtime.AllocateEdgeActor(); + + SetupLogging(runtime); + InitRoot(server, edgeActor); + + CreateShardedTable(server, edgeActor, "/Root", "TableWithIndex", + TShardedTableOptions() + .Columns({ + {"key", "Uint32", true, false}, + {"value", "Uint32", false, false}, + {"indexed", "Uint32", false, false} + }) + .Indexes({ + {"idx", {"indexed"}, {}, NKikimrSchemeOp::EIndexTypeGlobal} + })); + + ExecSQL(server, edgeActor, R"( + UPSERT INTO `/Root/TableWithIndex` (key, value, indexed) VALUES + (1, 10, 100), + (2, 20, 200), + (3, 30, 300); + )"); + + auto beforeBackup = KqpSimpleExecSuccess(runtime, R"( + SELECT key FROM `/Root/TableWithIndex` VIEW idx WHERE indexed = 200 + )"); + UNIT_ASSERT_C(beforeBackup.find("uint32_value: 2") != TString::npos, + "Index should work before backup: " << beforeBackup); + + ExecSQL(server, edgeActor, R"( + CREATE BACKUP COLLECTION `TestCollection` + ( TABLE `/Root/TableWithIndex` ) + WITH + ( STORAGE = 'cluster' + , INCREMENTAL_BACKUP_ENABLED = 'true' + ); + )", false); + + ExecSQL(server, edgeActor, R"(BACKUP `TestCollection`;)", false); + + // Wait for CDC streams to be fully created (AtTable phase completes async) + SimulateSleep(server, TDuration::Seconds(5)); + + auto expectedData = KqpSimpleExecSuccess(runtime, R"( + SELECT key, value, indexed FROM `/Root/TableWithIndex` ORDER BY key + )"); + + ExecSQL(server, edgeActor, R"(DROP TABLE `/Root/TableWithIndex`;)", false); + runtime.SimulateSleep(TDuration::Seconds(1)); + + ExecSQL(server, edgeActor, R"(RESTORE `TestCollection`;)", false); + runtime.SimulateSleep(TDuration::Seconds(5)); + + auto actualData = KqpSimpleExecSuccess(runtime, R"( + SELECT key, value, indexed FROM `/Root/TableWithIndex` ORDER BY key + )"); + UNIT_ASSERT_VALUES_EQUAL(expectedData, actualData); + + auto afterRestore = KqpSimpleExecSuccess(runtime, R"( + SELECT key FROM `/Root/TableWithIndex` VIEW idx WHERE indexed = 200 + )"); + UNIT_ASSERT_C(afterRestore.find("uint32_value: 2") != TString::npos, + "Index should work after restore: " << afterRestore); + + auto indexImplData = KqpSimpleExecSuccess(runtime, R"( + SELECT COUNT(*) FROM `/Root/TableWithIndex/idx/indexImplTable` + )"); + UNIT_ASSERT_C(indexImplData.find("uint64_value: 3") != TString::npos, + "Index impl table should have 3 rows: " << indexImplData); + } + Y_UNIT_TEST(MultiBackup) { TPortManager portManager; TServer::TPtr server = new TServer(TServerSettings(portManager.GetPort(2134), {}, DefaultPQConfig()) @@ -1228,7 +1407,7 @@ Y_UNIT_TEST_SUITE(IncrementalBackup) { InitRoot(server, edgeActor); // Create a table with multiple shards by using 4 shards - CreateShardedTable(server, edgeActor, "/Root", "MultiShardTable", + CreateShardedTable(server, edgeActor, "/Root", "MultiShardTable", TShardedTableOptions() .Shards(4) .Columns({ @@ -1309,7 +1488,7 @@ Y_UNIT_TEST_SUITE(IncrementalBackup) { // Verify that we have the expected final state: // - Keys 1, 11, 21, 31 deleted by incremental backup // - Keys 2, 12, 22, 32 updated to 200, 1200, 2200, 3200 by incremental backup - UNIT_ASSERT_VALUES_EQUAL(actual, + UNIT_ASSERT_VALUES_EQUAL(actual, "{ items { uint32_value: 2 } items { uint32_value: 200 } }, " "{ items { uint32_value: 12 } items { uint32_value: 1200 } }, " "{ items { uint32_value: 22 } items { uint32_value: 2200 } }, " @@ -1346,7 +1525,7 @@ Y_UNIT_TEST_SUITE(IncrementalBackup) { // Create full backup tables with different sharding // Table with 2 shards - CreateShardedTable(server, edgeActor, "/Root/.backups/collections/ForgedMultiShardCollection/19700101000001Z_full", "Table2Shard", + CreateShardedTable(server, edgeActor, "/Root/.backups/collections/ForgedMultiShardCollection/19700101000001Z_full", "Table2Shard", TShardedTableOptions().Shards(2)); ExecSQL(server, edgeActor, R"( @@ -1356,7 +1535,7 @@ Y_UNIT_TEST_SUITE(IncrementalBackup) { )"); // Table with 3 shards - CreateShardedTable(server, edgeActor, "/Root/.backups/collections/ForgedMultiShardCollection/19700101000001Z_full", "Table3Shard", + CreateShardedTable(server, edgeActor, "/Root/.backups/collections/ForgedMultiShardCollection/19700101000001Z_full", "Table3Shard", TShardedTableOptions().Shards(3)); ExecSQL(server, edgeActor, R"( @@ -1366,12 +1545,12 @@ Y_UNIT_TEST_SUITE(IncrementalBackup) { )"); // Table with 4 shards - CreateShardedTable(server, edgeActor, "/Root/.backups/collections/ForgedMultiShardCollection/19700101000001Z_full", "Table4Shard", + CreateShardedTable(server, edgeActor, "/Root/.backups/collections/ForgedMultiShardCollection/19700101000001Z_full", "Table4Shard", TShardedTableOptions().Shards(4)); ExecSQL(server, edgeActor, R"( UPSERT INTO `/Root/.backups/collections/ForgedMultiShardCollection/19700101000001Z_full/Table4Shard` (key, value) VALUES - (1, 1), (2, 2), (3, 3), (4, 4), (11, 11), (12, 12), (13, 13), (14, 14), + (1, 1), (2, 2), (3, 3), (4, 4), (11, 11), (12, 12), (13, 13), (14, 14), (21, 21), (22, 22), (23, 23), (24, 24), (31, 31), (32, 32), (33, 33), (34, 34) ; )"); @@ -1386,7 +1565,7 @@ Y_UNIT_TEST_SUITE(IncrementalBackup) { // Create incremental backup tables with same sharding as full backup // Table2Shard - 2 shards: delete some keys, update others - CreateShardedTable(server, edgeActor, "/Root/.backups/collections/ForgedMultiShardCollection/19700101000002Z_incremental", "Table2Shard", + CreateShardedTable(server, edgeActor, "/Root/.backups/collections/ForgedMultiShardCollection/19700101000002Z_incremental", "Table2Shard", opts.Shards(2)); auto normalMetadata = SerializeChangeMetadata(false); // Not deleted @@ -1402,7 +1581,7 @@ Y_UNIT_TEST_SUITE(IncrementalBackup) { )"); // Table3Shard - 3 shards: more complex changes across all shards - CreateShardedTable(server, edgeActor, "/Root/.backups/collections/ForgedMultiShardCollection/19700101000002Z_incremental", "Table3Shard", + CreateShardedTable(server, edgeActor, "/Root/.backups/collections/ForgedMultiShardCollection/19700101000002Z_incremental", "Table3Shard", opts.Shards(3)); ExecSQL(server, edgeActor, TStringBuilder() << R"( @@ -1417,7 +1596,7 @@ Y_UNIT_TEST_SUITE(IncrementalBackup) { )"); // Table4Shard - 4 shards: changes in all shards - CreateShardedTable(server, edgeActor, "/Root/.backups/collections/ForgedMultiShardCollection/19700101000002Z_incremental", "Table4Shard", + CreateShardedTable(server, edgeActor, "/Root/.backups/collections/ForgedMultiShardCollection/19700101000002Z_incremental", "Table4Shard", opts.Shards(4)); ExecSQL(server, edgeActor, TStringBuilder() << R"( @@ -1644,7 +1823,7 @@ Y_UNIT_TEST_SUITE(IncrementalBackup) { // === DATABASE STRUCTURE CREATION === // Create orders table - CreateShardedTable(server, edgeActor, "/Root", "orders", + CreateShardedTable(server, edgeActor, "/Root", "orders", TShardedTableOptions().Columns({ {"order_id", "Uint32", true, false}, {"customer_name", "Utf8", false, false}, @@ -1652,7 +1831,7 @@ Y_UNIT_TEST_SUITE(IncrementalBackup) { {"amount", "Uint32", false, false} })); - // Create products table + // Create products table CreateShardedTable(server, edgeActor, "/Root", "products", TShardedTableOptions().Columns({ {"product_id", "Uint32", true, false}, @@ -1675,7 +1854,7 @@ Y_UNIT_TEST_SUITE(IncrementalBackup) { // === CHECKPOINT 1: Initial data + full backup === ExecSQL(server, edgeActor, R"( - UPSERT INTO `/Root/orders` (order_id, customer_name, order_date, amount) VALUES + UPSERT INTO `/Root/orders` (order_id, customer_name, order_date, amount) VALUES (1001, 'Иван Петров', '2024-01-01T10:00:00Z', 2500), (1002, 'Мария Сидорова', '2024-01-01T10:15:00Z', 1800), (1003, 'Алексей Иванов', '2024-01-01T10:30:00Z', 3200); @@ -1691,7 +1870,7 @@ Y_UNIT_TEST_SUITE(IncrementalBackup) { // Check initial data auto initialOrdersCount = KqpSimpleExec(runtime, R"(SELECT COUNT(*) FROM `/Root/orders`)"); auto initialProductsCount = KqpSimpleExec(runtime, R"(SELECT COUNT(*) FROM `/Root/products`)"); - + UNIT_ASSERT_VALUES_EQUAL(initialOrdersCount, "{ items { uint64_value: 3 } }"); UNIT_ASSERT_VALUES_EQUAL(initialProductsCount, "{ items { uint64_value: 3 } }"); @@ -1703,10 +1882,10 @@ Y_UNIT_TEST_SUITE(IncrementalBackup) { SimulateSleep(server, TDuration::Seconds(3)); // === CHECKPOINT 2: Changes + first incremental backup === - + // New orders ExecSQL(server, edgeActor, R"( - UPSERT INTO `/Root/orders` (order_id, customer_name, order_date, amount) VALUES + UPSERT INTO `/Root/orders` (order_id, customer_name, order_date, amount) VALUES (1004, 'Елена Козлова', '2024-01-01T11:00:00Z', 4100), (1005, 'Дмитрий Волков', '2024-01-01T11:15:00Z', 2800); )"); @@ -1731,10 +1910,10 @@ Y_UNIT_TEST_SUITE(IncrementalBackup) { SimulateSleep(server, TDuration::Seconds(5)); // === CHECKPOINT 3: More changes + second incremental backup === - + // Add more orders ExecSQL(server, edgeActor, R"( - UPSERT INTO `/Root/orders` (order_id, customer_name, order_date, amount) VALUES + UPSERT INTO `/Root/orders` (order_id, customer_name, order_date, amount) VALUES (1006, 'Анна Смирнова', '2024-01-01T12:00:00Z', 5200), (1007, 'Сергей Попов', '2024-01-01T12:15:00Z', 1950); )"); @@ -1757,7 +1936,7 @@ Y_UNIT_TEST_SUITE(IncrementalBackup) { SimulateSleep(server, TDuration::Seconds(5)); // === FINAL STATE CHECK === - + // Check final state of main tables auto finalOrdersState = KqpSimpleExec(runtime, R"( SELECT order_id, customer_name FROM `/Root/orders` @@ -1765,7 +1944,7 @@ Y_UNIT_TEST_SUITE(IncrementalBackup) { )"); // Expected orders: 1001, 1003, 1004, 1005, 1006, 1007 (1002 was deleted) - TString expectedFinalOrders = + TString expectedFinalOrders = "{ items { uint32_value: 1001 } items { text_value: \"Иван Петров\" } }, " "{ items { uint32_value: 1003 } items { text_value: \"Алексей Иванов\" } }, " "{ items { uint32_value: 1004 } items { text_value: \"Елена Козлова\" } }, " @@ -1780,7 +1959,7 @@ Y_UNIT_TEST_SUITE(IncrementalBackup) { ORDER BY product_id )"); - TString expectedFinalProducts = + TString expectedFinalProducts = "{ items { uint32_value: 501 } items { text_value: \"Ноутбук Model A\" } }, " "{ items { uint32_value: 502 } items { text_value: \"Мышь YdbTech\" } }, " "{ items { uint32_value: 503 } items { text_value: \"Монитор 24\\\"\" } }, " @@ -1789,7 +1968,7 @@ Y_UNIT_TEST_SUITE(IncrementalBackup) { UNIT_ASSERT_VALUES_EQUAL(finalProductsState, expectedFinalProducts); // === RESTORE TEST === - + // Save expected state before dropping tables auto expectedOrdersForRestore = KqpSimpleExec(runtime, R"( SELECT order_id, customer_name, amount FROM `/Root/orders` ORDER BY order_id @@ -1852,7 +2031,7 @@ Y_UNIT_TEST_SUITE(IncrementalBackup) { ExecSQL(server, edgeActor, R"(DROP BACKUP COLLECTION `TestCollection`;)", false); // Verify collection was deleted by trying to drop it again (should fail) - ExecSQL(server, edgeActor, R"(DROP BACKUP COLLECTION `TestCollection`;)", + ExecSQL(server, edgeActor, R"(DROP BACKUP COLLECTION `TestCollection`;)", false, Ydb::StatusIds::SCHEME_ERROR); } @@ -1875,7 +2054,7 @@ Y_UNIT_TEST_SUITE(IncrementalBackup) { // Test with collection name that could be confused with database name const TString collectionName = "Root"; - + ExecSQL(server, edgeActor, Sprintf(R"( CREATE BACKUP COLLECTION `%s` ( TABLE `/Root/Table` @@ -1893,7 +2072,7 @@ Y_UNIT_TEST_SUITE(IncrementalBackup) { ExecSQL(server, edgeActor, Sprintf(R"(DROP BACKUP COLLECTION `%s`;)", collectionName.c_str()), false); // Verify collection was deleted by trying to drop it again (should fail) - ExecSQL(server, edgeActor, Sprintf(R"(DROP BACKUP COLLECTION `%s`;)", collectionName.c_str()), + ExecSQL(server, edgeActor, Sprintf(R"(DROP BACKUP COLLECTION `%s`;)", collectionName.c_str()), false, Ydb::StatusIds::SCHEME_ERROR); } @@ -1913,7 +2092,7 @@ Y_UNIT_TEST_SUITE(IncrementalBackup) { SetupLogging(runtime); InitRoot(server, edgeActor); - ExecSQL(server, edgeActor, R"(DROP BACKUP COLLECTION `NonExistentCollection`;)", + ExecSQL(server, edgeActor, R"(DROP BACKUP COLLECTION `NonExistentCollection`;)", false, Ydb::StatusIds::SCHEME_ERROR); } @@ -1967,7 +2146,7 @@ Y_UNIT_TEST_SUITE(IncrementalBackup) { // Try to find the incremental backup table by iterating through possible timestamps TString foundIncrementalBackupPath; bool foundIncrementalBackupTable = false; - + // Common incremental backup paths to try (timestamp-based) TVector possiblePaths = { "/Root/.backups/collections/TestCollection/19700101000002Z_incremental/Table", @@ -1987,7 +2166,7 @@ Y_UNIT_TEST_SUITE(IncrementalBackup) { request->Record.MutableDescribePath()->MutableOptions()->SetShowPrivateTable(true); runtime.Send(new IEventHandle(MakeTxProxyID(), edgeActor, request.Release())); auto reply = runtime.GrabEdgeEventRethrow(edgeActor); - + if (reply->Get()->GetRecord().GetStatus() == NKikimrScheme::EStatus::StatusSuccess) { foundIncrementalBackupPath = path; foundIncrementalBackupTable = true; @@ -2006,14 +2185,14 @@ Y_UNIT_TEST_SUITE(IncrementalBackup) { auto reply = runtime.GrabEdgeEventRethrow(edgeActor); UNIT_ASSERT_EQUAL(reply->Get()->GetRecord().GetStatus(), NKikimrScheme::EStatus::StatusSuccess); - + const auto& pathDescription = reply->Get()->GetRecord().GetPathDescription(); UNIT_ASSERT(pathDescription.HasTable()); - + // Verify that incremental backup table has __incremental_backup attribute bool hasIncrementalBackupAttr = false; bool hasAsyncReplicaAttr = false; - + for (const auto& attr : pathDescription.GetUserAttributes()) { Cerr << "Found attribute: " << attr.GetKey() << " = " << attr.GetValue() << Endl; if (attr.GetKey() == "__incremental_backup") { @@ -2023,7 +2202,7 @@ Y_UNIT_TEST_SUITE(IncrementalBackup) { hasAsyncReplicaAttr = true; } } - + // Verify that we have __incremental_backup but NOT __async_replica UNIT_ASSERT_C(hasIncrementalBackupAttr, TStringBuilder() << "Incremental backup table at " << foundIncrementalBackupPath << " must have __incremental_backup attribute"); UNIT_ASSERT_C(!hasAsyncReplicaAttr, TStringBuilder() << "Incremental backup table at " << foundIncrementalBackupPath << " must NOT have __async_replica attribute"); @@ -2088,7 +2267,7 @@ Y_UNIT_TEST_SUITE(IncrementalBackup) { WaitTxNotification(server, edgeActor, AsyncCreateContinuousBackup(server, "/Root", "Table")); - // Test multiple REPLACE operations + // Test multiple REPLACE operations ExecSQL(server, edgeActor, R"( REPLACE INTO `/Root/Table` (key, value) VALUES (1, 100), @@ -2141,7 +2320,7 @@ Y_UNIT_TEST_SUITE(IncrementalBackup) { NKikimrChangeExchange::TChangeRecord firstRecord; UNIT_ASSERT(firstRecord.ParseFromString(records[0].second)); UNIT_ASSERT_C(firstRecord.GetCdcDataChange().HasUpsert(), "First record should be an upsert"); - + // Parse the second record (Reset if EvWrite, Upsert is ProposeTx) NKikimrChangeExchange::TChangeRecord secondRecord; UNIT_ASSERT(secondRecord.ParseFromString(records[1].second)); @@ -2185,8 +2364,8 @@ Y_UNIT_TEST_SUITE(IncrementalBackup) { NKikimrChangeExchange::TChangeRecord parsedRecord; UNIT_ASSERT(parsedRecord.ParseFromString(records[i].second)); const auto& dataChange = parsedRecord.GetCdcDataChange(); - - UNIT_ASSERT_C(dataChange.HasUpsert() || dataChange.HasReset(), + + UNIT_ASSERT_C(dataChange.HasUpsert() || dataChange.HasReset(), "Record should be either upsert or reset operation"); } } @@ -2196,27 +2375,27 @@ Y_UNIT_TEST_SUITE(IncrementalBackup) { auto settings = TServerSettings(portManager.GetPort(2134), {}, DefaultPQConfig()) .SetUseRealThreads(false) .SetDomainName("Root"); - + settings.SetEnableBackupService(true); - + TServer::TPtr server = new TServer(settings); auto& runtime = *server->GetRuntime(); const auto edgeActor = runtime.AllocateEdgeActor(); InitRoot(server, edgeActor); - + ExecSQL(server, edgeActor, R"( CREATE BACKUP COLLECTION `MixedCollection` ( TABLE `/Root/NonExistentTable` ) - WITH ( + WITH ( STORAGE = 'cluster', INCREMENTAL_BACKUP_ENABLED = 'true' ); )", false, Ydb::StatusIds::SUCCESS); ExecSQL(server, edgeActor, R"(BACKUP `MixedCollection` INCREMENTAL;)", false, Ydb::StatusIds::SCHEME_ERROR); - + ExecSQL(server, edgeActor, "SELECT 1;"); } @@ -2309,7 +2488,7 @@ Y_UNIT_TEST_SUITE(IncrementalBackup) { // Query the first incremental backup table auto incrBackup1Result = KqpSimpleExec(runtime, R"( - SELECT key, value, LENGTH(__ydb_incrBackupImpl_changeMetadata) as metadata_len + SELECT key, value, LENGTH(__ydb_incrBackupImpl_changeMetadata) as metadata_len FROM `/Root/.backups/collections/TestCollection/19700101000002Z_incremental/Table` ORDER BY key )"); @@ -2374,6 +2553,1272 @@ Y_UNIT_TEST_SUITE(IncrementalBackup) { "Full backup should contain original value 10"); } + Y_UNIT_TEST_TWIN(BackupMetadataDirectoriesSkippedDuringRestore, WithIncremental) { + TPortManager portManager; + TServer::TPtr server = new TServer(TServerSettings(portManager.GetPort(2134), {}, DefaultPQConfig()) + .SetUseRealThreads(false) + .SetDomainName("Root") + .SetEnableChangefeedInitialScan(true) + .SetEnableBackupService(true) + ); + + auto& runtime = *server->GetRuntime(); + const auto edgeActor = runtime.AllocateEdgeActor(); + + SetupLogging(runtime); + InitRoot(server, edgeActor); + + runtime.GetAppData(0).FeatureFlags.SetEnableSystemNamesProtection(true); + + ExecSQL(server, edgeActor, R"( + CREATE BACKUP COLLECTION `MetaTestCollection` + ( TABLE `/Root/TestTable` + ) + WITH + ( STORAGE = 'cluster' + , INCREMENTAL_BACKUP_ENABLED = ')" + TString(WithIncremental ? "true" : "false") + R"(' + ); + )", false); + + CreateShardedTable(server, edgeActor, + "/Root/.backups/collections/MetaTestCollection/19700101000001Z_full", + "TestTable", SimpleTable()); + + ExecSQL(server, edgeActor, R"( + UPSERT INTO `/Root/.backups/collections/MetaTestCollection/19700101000001Z_full/TestTable` + (key, value) VALUES (1, 10), (2, 20), (3, 30); + )"); + + CreateShardedTable(server, edgeActor, + "/Root/.backups/collections/MetaTestCollection/19700101000001Z_full/__ydb_backup_meta", + "MetaTable1", SimpleTable()); + + ExecSQL(server, edgeActor, R"( + UPSERT INTO `/Root/.backups/collections/MetaTestCollection/19700101000001Z_full/__ydb_backup_meta/MetaTable1` + (key, value) VALUES (100, 1000); + )"); + + if (WithIncremental) { + auto normalMetadata = SerializeChangeMetadata(false); // Not deleted + auto deletedMetadata = SerializeChangeMetadata(true); // Deleted + + CreateShardedTable( + server, edgeActor, + "/Root/.backups/collections/MetaTestCollection/19700101000002Z_incremental", + "TestTable", + SimpleTable() + .AllowSystemColumnNames(true) + .Columns({ + {"key", "Uint32", true, false}, + {"value", "Uint32", false, false}, + {"__ydb_incrBackupImpl_changeMetadata", "String", false, false}})); + + ExecSQL(server, edgeActor, TStringBuilder() << R"( + UPSERT INTO `/Root/.backups/collections/MetaTestCollection/19700101000002Z_incremental/TestTable` + (key, value, __ydb_incrBackupImpl_changeMetadata) VALUES + (1, NULL, ')" << deletedMetadata << R"('), + (4, 40, ')" << normalMetadata << R"('); + )"); + + CreateShardedTable(server, edgeActor, + "/Root/.backups/collections/MetaTestCollection/19700101000002Z_incremental/__ydb_backup_meta", + "MetaTable2", SimpleTable()); + + ExecSQL(server, edgeActor, R"( + UPSERT INTO `/Root/.backups/collections/MetaTestCollection/19700101000002Z_incremental/__ydb_backup_meta/MetaTable2` + (key, value) VALUES (200, 2000); + )"); + } + + auto checkMeta1 = KqpSimpleExec(runtime, R"( + SELECT key, value FROM `/Root/.backups/collections/MetaTestCollection/19700101000001Z_full/__ydb_backup_meta/MetaTable1` + )"); + UNIT_ASSERT_C(checkMeta1.find("uint32_value: 100") != TString::npos, + "MetaTable1 should exist in full backup before restore"); + + if (WithIncremental) { + auto checkMeta2 = KqpSimpleExec(runtime, R"( + SELECT key, value FROM `/Root/.backups/collections/MetaTestCollection/19700101000002Z_incremental/__ydb_backup_meta/MetaTable2` + )"); + UNIT_ASSERT_C(checkMeta2.find("uint32_value: 200") != TString::npos, + "MetaTable2 should exist in incremental backup before restore"); + } + + ExecSQL(server, edgeActor, R"(RESTORE `MetaTestCollection`;)", false); + runtime.SimulateSleep(TDuration::Seconds(10)); + + auto restoredTable = KqpSimpleExec(runtime, R"( + SELECT key, value FROM `/Root/TestTable` ORDER BY key + )"); + + if (!WithIncremental) { + UNIT_ASSERT_C(restoredTable.find("uint32_value: 1") != TString::npos, + "Restored table should contain key 1"); + UNIT_ASSERT_C(restoredTable.find("uint32_value: 2") != TString::npos, + "Restored table should contain key 2"); + UNIT_ASSERT_C(restoredTable.find("uint32_value: 3") != TString::npos, + "Restored table should contain key 3"); + } else { + UNIT_ASSERT_C(restoredTable.find("uint32_value: 2") != TString::npos, + "Restored table should contain key 2"); + UNIT_ASSERT_C(restoredTable.find("uint32_value: 3") != TString::npos, + "Restored table should contain key 3"); + UNIT_ASSERT_C(restoredTable.find("uint32_value: 4") != TString::npos, + "Restored table should contain key 4"); + // Key 1 should NOT be present (deleted by incremental) + UNIT_ASSERT_C(restoredTable.find("uint32_value: 1") == TString::npos, + "Restored table should NOT contain deleted key 1"); + } + + auto tryQueryMetaDir = KqpSimpleExec(runtime, R"( + SELECT key, value FROM `/Root/__ydb_backup_meta/MetaTable1` + )", Ydb::StatusIds::SCHEME_ERROR); // Should fail - path doesn't exist + + UNIT_ASSERT_C(tryQueryMetaDir.empty() || tryQueryMetaDir.find("SCHEME_ERROR") != TString::npos, + "__ydb_backup_meta should NOT be restored to /Root"); + + auto tryQueryMetaTable1 = KqpSimpleExec(runtime, R"( + SELECT key, value FROM `/Root/MetaTable1` + )", Ydb::StatusIds::SCHEME_ERROR); + + UNIT_ASSERT_C(tryQueryMetaTable1.empty() || tryQueryMetaTable1.find("SCHEME_ERROR") != TString::npos, + "MetaTable1 should NOT be restored to /Root"); + + if (WithIncremental) { + auto tryQueryMetaTable2 = KqpSimpleExec(runtime, R"( + SELECT key, value FROM `/Root/MetaTable2` + )", Ydb::StatusIds::SCHEME_ERROR); + + UNIT_ASSERT_C(tryQueryMetaTable2.empty() || tryQueryMetaTable2.find("SCHEME_ERROR") != TString::npos, + "MetaTable2 should NOT be restored to /Root"); + } + + auto verifyMeta1StillInBackup = KqpSimpleExec(runtime, R"( + SELECT key, value FROM `/Root/.backups/collections/MetaTestCollection/19700101000001Z_full/__ydb_backup_meta/MetaTable1` + )"); + UNIT_ASSERT_C(verifyMeta1StillInBackup.find("uint32_value: 100") != TString::npos, + "MetaTable1 should still exist in backup location after restore"); + + if (WithIncremental) { + auto verifyMeta2StillInBackup = KqpSimpleExec(runtime, R"( + SELECT key, value FROM `/Root/.backups/collections/MetaTestCollection/19700101000002Z_incremental/__ydb_backup_meta/MetaTable2` + )"); + UNIT_ASSERT_C(verifyMeta2StillInBackup.find("uint32_value: 200") != TString::npos, + "MetaTable2 should still exist in backup location after restore"); + } + } + + Y_UNIT_TEST(IncrementalBackupWithIndexes) { + TPortManager portManager; + TServer::TPtr server = new TServer(TServerSettings(portManager.GetPort(2134), {}, DefaultPQConfig()) + .SetUseRealThreads(false) + .SetDomainName("Root") + .SetEnableChangefeedInitialScan(true) + .SetEnableBackupService(true) + .SetEnableRealSystemViewPaths(false) + .SetEnableDataColumnForIndexTable(true) + ); + + auto& runtime = *server->GetRuntime(); + const auto edgeActor = runtime.AllocateEdgeActor(); + + SetupLogging(runtime); + InitRoot(server, edgeActor); + + TShardedTableOptions opts; + opts.Columns({ + {"key", "Uint32", true, false}, + {"value", "Uint32", false, false} + }); + opts.Indexes({ + TShardedTableOptions::TIndex{"ByValue", {"value"}, {}, NKikimrSchemeOp::EIndexTypeGlobal} + }); + CreateShardedTable(server, edgeActor, "/Root", "Table", opts); + + ExecSQL(server, edgeActor, R"( + CREATE BACKUP COLLECTION `MyCollection` + ( TABLE `/Root/Table` + ) + WITH + ( STORAGE = 'cluster' + , INCREMENTAL_BACKUP_ENABLED = 'true' + ); + )", false); + + ExecSQL(server, edgeActor, R"(BACKUP `MyCollection`;)", false); + SimulateSleep(server, TDuration::Seconds(1)); + + ExecSQL(server, edgeActor, R"( + UPSERT INTO `/Root/Table` (key, value) VALUES + (1, 100) + , (2, 200) + , (3, 300) + ; + )"); + + ExecSQL(server, edgeActor, R"( + UPSERT INTO `/Root/Table` (key, value) VALUES (2, 250); + )"); + + ExecSQL(server, edgeActor, R"(DELETE FROM `/Root/Table` WHERE key=3;)"); + + ExecSQL(server, edgeActor, R"( + UPSERT INTO `/Root/Table` (key, value) VALUES (4, 400); + )"); + + SimulateSleep(server, TDuration::Seconds(1)); + + ExecSQL(server, edgeActor, R"(BACKUP `MyCollection` INCREMENTAL;)", false); + + SimulateSleep(server, TDuration::Seconds(10)); + + TString incrBackupDir = FindIncrementalBackupDir(runtime, edgeActor, "/Root/.backups/collections/MyCollection"); + UNIT_ASSERT_C(!incrBackupDir.empty(), "Could not find incremental backup directory"); + + TString mainTablePath = TStringBuilder() << "/Root/.backups/collections/MyCollection/" << incrBackupDir << "/Table"; + auto mainTableBackup = KqpSimpleExec(runtime, TStringBuilder() << R"( + SELECT key, value FROM `)" << mainTablePath << R"(` + ORDER BY key + )"); + + UNIT_ASSERT_C(mainTableBackup.find("uint32_value: 2") != TString::npos, + "Main table backup should contain updated key 2"); + UNIT_ASSERT_C(mainTableBackup.find("uint32_value: 250") != TString::npos, + "Main table backup should contain new value 250"); + UNIT_ASSERT_C(mainTableBackup.find("uint32_value: 3") != TString::npos, + "Main table backup should contain deleted key 3"); + UNIT_ASSERT_C(mainTableBackup.find("uint32_value: 4") != TString::npos, + "Main table backup should contain new key 4"); + UNIT_ASSERT_C(mainTableBackup.find("uint32_value: 400") != TString::npos, + "Main table backup should contain new value 400"); + + TString indexBackupPath = TStringBuilder() << "/Root/.backups/collections/MyCollection/" << incrBackupDir << "/__ydb_backup_meta/indexes/Table/ByValue"; + auto indexBackup = KqpSimpleExec(runtime, TStringBuilder() << R"( + SELECT * FROM `)" << indexBackupPath << R"(` + ORDER BY value + )"); + + UNIT_ASSERT_C(indexBackup.find("uint32_value: 200") != TString::npos, + "Index backup should contain old value 200 (deleted)"); + UNIT_ASSERT_C(indexBackup.find("uint32_value: 250") != TString::npos, + "Index backup should contain new value 250"); + UNIT_ASSERT_C(indexBackup.find("uint32_value: 300") != TString::npos, + "Index backup should contain deleted value 300"); + UNIT_ASSERT_C(indexBackup.find("uint32_value: 400") != TString::npos, + "Index backup should contain new value 400"); + } + + Y_UNIT_TEST(IncrementalBackupWithCoveringIndex) { + TPortManager portManager; + TServer::TPtr server = new TServer(TServerSettings(portManager.GetPort(2134), {}, DefaultPQConfig()) + .SetUseRealThreads(false) + .SetDomainName("Root") + .SetEnableChangefeedInitialScan(true) + .SetEnableBackupService(true) + .SetEnableRealSystemViewPaths(false) + .SetEnableDataColumnForIndexTable(true) + ); + + auto& runtime = *server->GetRuntime(); + const auto edgeActor = runtime.AllocateEdgeActor(); + + SetupLogging(runtime); + InitRoot(server, edgeActor); + + TShardedTableOptions opts; + opts.Columns({ + {"key", "Uint32", true, false}, + {"name", "Utf8", false, false}, + {"age", "Uint32", false, false}, + {"salary", "Uint32", false, false} + }); + opts.Indexes({ + TShardedTableOptions::TIndex{"ByAge", {"age"}, {"name"}, NKikimrSchemeOp::EIndexTypeGlobal} + }); + CreateShardedTable(server, edgeActor, "/Root", "Table", opts); + + ExecSQL(server, edgeActor, R"( + CREATE BACKUP COLLECTION `MyCollection` + ( TABLE `/Root/Table` + ) + WITH + ( STORAGE = 'cluster' + , INCREMENTAL_BACKUP_ENABLED = 'true' + ); + )", false); + + ExecSQL(server, edgeActor, R"(BACKUP `MyCollection`;)", false); + SimulateSleep(server, TDuration::Seconds(1)); + + ExecSQL(server, edgeActor, R"( + UPSERT INTO `/Root/Table` (key, name, age, salary) VALUES + (1, 'Alice', 30u, 5000u) + , (2, 'Bob', 25u, 4000u) + ; + )"); + + ExecSQL(server, edgeActor, R"( + UPSERT INTO `/Root/Table` (key, name, age, salary) VALUES (1, 'Alice2', 30u, 5000u); + )"); + + ExecSQL(server, edgeActor, R"( + UPSERT INTO `/Root/Table` (key, name, age, salary) VALUES (1, 'Alice2', 30u, 6000u); + )"); + + ExecSQL(server, edgeActor, R"( + UPSERT INTO `/Root/Table` (key, name, age, salary) VALUES (2, 'Bob', 26u, 4000u); + )"); + + ExecSQL(server, edgeActor, R"(DELETE FROM `/Root/Table` WHERE key=2;)"); + + SimulateSleep(server, TDuration::Seconds(1)); + + ExecSQL(server, edgeActor, R"(BACKUP `MyCollection` INCREMENTAL;)", false); + + SimulateSleep(server, TDuration::Seconds(10)); + + TString incrBackupDir = FindIncrementalBackupDir(runtime, edgeActor, "/Root/.backups/collections/MyCollection"); + UNIT_ASSERT_C(!incrBackupDir.empty(), "Could not find incremental backup directory"); + + TString indexBackupPath = TStringBuilder() << "/Root/.backups/collections/MyCollection/" << incrBackupDir << "/__ydb_backup_meta/indexes/Table/ByAge"; + auto indexBackup = KqpSimpleExec(runtime, TStringBuilder() << R"( + SELECT * FROM `)" << indexBackupPath << R"(` + )"); + + UNIT_ASSERT_C(indexBackup.find("uint32_value: 30") != TString::npos, + "Index backup should contain age 30"); + UNIT_ASSERT_C(indexBackup.find("Alice") != TString::npos, + "Index backup should contain Alice2 from covering column name update"); + UNIT_ASSERT_C(indexBackup.find("uint32_value: 25") != TString::npos, + "Index backup should contain tombstone for age 25"); + UNIT_ASSERT_C(indexBackup.find("uint32_value: 26") != TString::npos, + "Index backup should contain tombstone for age 26"); + UNIT_ASSERT_C(indexBackup.find("null_flag_value: NULL_VALUE") != TString::npos, + "Index backup tombstones should have NULL for covering columns"); + + auto counts = CountCdcOperations(indexBackup); + Cerr << "CDC metadata: " << counts.Deletes << " DELETEs, " << counts.Inserts << " INSERTs" << Endl; + + UNIT_ASSERT_EQUAL_C(counts.Deletes, 2, "Should have 2 DELETE operations (tombstones for age 25 and 26)"); + UNIT_ASSERT_EQUAL_C(counts.Inserts, 1, "Should have 1 INSERT operation (for Alice2)"); + } + + Y_UNIT_TEST(IncrementalBackupMultipleIndexes) { + TPortManager portManager; + TServer::TPtr server = new TServer(TServerSettings(portManager.GetPort(2134), {}, DefaultPQConfig()) + .SetUseRealThreads(false) + .SetDomainName("Root") + .SetEnableChangefeedInitialScan(true) + .SetEnableBackupService(true) + .SetEnableRealSystemViewPaths(false) + .SetEnableDataColumnForIndexTable(true) + ); + + auto& runtime = *server->GetRuntime(); + const auto edgeActor = runtime.AllocateEdgeActor(); + + SetupLogging(runtime); + InitRoot(server, edgeActor); + + TShardedTableOptions opts; + opts.Columns({ + {"key", "Uint32", true, false}, + {"name", "Utf8", false, false}, + {"age", "Uint32", false, false}, + {"city", "Utf8", false, false}, + {"salary", "Uint32", false, false} + }); + opts.Indexes({ + TShardedTableOptions::TIndex{"ByName", {"name"}, {}, NKikimrSchemeOp::EIndexTypeGlobal}, + TShardedTableOptions::TIndex{"ByAge", {"age"}, {"salary"}, NKikimrSchemeOp::EIndexTypeGlobal}, + TShardedTableOptions::TIndex{"ByCity", {"city", "name"}, {}, NKikimrSchemeOp::EIndexTypeGlobal} + }); + CreateShardedTable(server, edgeActor, "/Root", "Table", opts); + + ExecSQL(server, edgeActor, R"( + CREATE BACKUP COLLECTION `MyCollection` + ( TABLE `/Root/Table` + ) + WITH + ( STORAGE = 'cluster' + , INCREMENTAL_BACKUP_ENABLED = 'true' + ); + )", false); + + ExecSQL(server, edgeActor, R"(BACKUP `MyCollection`;)", false); + SimulateSleep(server, TDuration::Seconds(5)); + + ExecSQL(server, edgeActor, R"( + UPSERT INTO `/Root/Table` (key, name, age, city, salary) VALUES + (1, 'Alice', 30u, 'NYC', 5000u) + , (2, 'Bob', 25u, 'LA', 4000u) + ; + )"); + + ExecSQL(server, edgeActor, R"( + UPSERT INTO `/Root/Table` (key, name, age, city, salary) VALUES (1, 'Alice2', 30u, 'NYC', 5000u); + )"); + + ExecSQL(server, edgeActor, R"( + UPSERT INTO `/Root/Table` (key, name, age, city, salary) VALUES (2, 'Bob', 26u, 'LA', 4000u); + )"); + + ExecSQL(server, edgeActor, R"(DELETE FROM `/Root/Table` WHERE key=1;)"); + + ExecSQL(server, edgeActor, R"( + UPSERT INTO `/Root/Table` (key, name, age, city, salary) VALUES (3, 'Carol', 28u, 'SF', 5500u); + )"); + + SimulateSleep(server, TDuration::Seconds(1)); + + ExecSQL(server, edgeActor, R"(BACKUP `MyCollection` INCREMENTAL;)", false); + + SimulateSleep(server, TDuration::Seconds(10)); + + TString incrBackupDir = FindIncrementalBackupDir(runtime, edgeActor, "/Root/.backups/collections/MyCollection"); + UNIT_ASSERT_C(!incrBackupDir.empty(), "Could not find incremental backup directory"); + + TString byNamePath = TStringBuilder() << "/Root/.backups/collections/MyCollection/" << incrBackupDir << "/__ydb_backup_meta/indexes/Table/ByName"; + auto byNameBackup = KqpSimpleExec(runtime, TStringBuilder() << R"( + SELECT * FROM `)" << byNamePath << R"(` + )"); + UNIT_ASSERT_C(byNameBackup.find("Alice") != TString::npos, + "ByName backup should contain Alice (deleted)"); + UNIT_ASSERT_C(byNameBackup.find("Alice2") != TString::npos, + "ByName backup should contain Alice2 (updated)"); + UNIT_ASSERT_C(byNameBackup.find("Carol") != TString::npos, + "ByName backup should contain Carol (new)"); + + TString byAgePath = TStringBuilder() << "/Root/.backups/collections/MyCollection/" << incrBackupDir << "/__ydb_backup_meta/indexes/Table/ByAge"; + auto byAgeBackup = KqpSimpleExec(runtime, TStringBuilder() << R"( + SELECT * FROM `)" << byAgePath << R"(` + )"); + UNIT_ASSERT_C(byAgeBackup.find("uint32_value: 30") != TString::npos, + "ByAge backup should contain age 30 (deleted)"); + UNIT_ASSERT_C(byAgeBackup.find("uint32_value: 25") != TString::npos || + byAgeBackup.find("uint32_value: 26") != TString::npos, + "ByAge backup should contain age change (25 or 26)"); + UNIT_ASSERT_C(byAgeBackup.find("uint32_value: 28") != TString::npos, + "ByAge backup should contain age 28 (new)"); + UNIT_ASSERT_C(byAgeBackup.find("uint32_value: 5500") != TString::npos, + "ByAge backup should contain covered salary 5500"); + + TString byCityPath = TStringBuilder() << "/Root/.backups/collections/MyCollection/" << incrBackupDir << "/__ydb_backup_meta/indexes/Table/ByCity"; + auto byCityBackup = KqpSimpleExec(runtime, TStringBuilder() << R"( + SELECT * FROM `)" << byCityPath << R"(` + )"); + UNIT_ASSERT_C(byCityBackup.find("NYC") != TString::npos, + "ByCity backup should contain NYC"); + UNIT_ASSERT_C(byCityBackup.find("Alice") != TString::npos, + "ByCity backup should contain Alice (part of composite key)"); + UNIT_ASSERT_C(byCityBackup.find("Alice2") != TString::npos, + "ByCity backup should contain Alice2 (updated composite key)"); + UNIT_ASSERT_C(byCityBackup.find("SF") != TString::npos, + "ByCity backup should contain SF (new)"); + UNIT_ASSERT_C(byCityBackup.find("Carol") != TString::npos, + "ByCity backup should contain Carol (new composite key)"); + } + + Y_UNIT_TEST(OmitIndexesIncrementalBackup) { + TPortManager portManager; + TServer::TPtr server = new TServer(TServerSettings(portManager.GetPort(2134), {}, DefaultPQConfig()) + .SetUseRealThreads(false) + .SetDomainName("Root") + .SetEnableChangefeedInitialScan(true) + .SetEnableBackupService(true) + .SetEnableRealSystemViewPaths(false) + .SetEnableDataColumnForIndexTable(true) + ); + + auto& runtime = *server->GetRuntime(); + const auto edgeActor = runtime.AllocateEdgeActor(); + + SetupLogging(runtime); + InitRoot(server, edgeActor); + + TShardedTableOptions opts; + opts.Columns({ + {"key", "Uint32", true, false}, + {"value", "Uint32", false, false} + }); + opts.Indexes({ + TShardedTableOptions::TIndex{"ByValue", {"value"}, {}, NKikimrSchemeOp::EIndexTypeGlobal} + }); + CreateShardedTable(server, edgeActor, "/Root", "Table", opts); + + ExecSQL(server, edgeActor, R"( + CREATE BACKUP COLLECTION `MyCollection` + ( TABLE `/Root/Table` + ) + WITH + ( STORAGE = 'cluster' + , INCREMENTAL_BACKUP_ENABLED = 'true' + , OMIT_INDEXES = 'true' + ); + )", false); + + ExecSQL(server, edgeActor, R"(BACKUP `MyCollection`;)", false); + SimulateSleep(server, TDuration::Seconds(5)); + + ExecSQL(server, edgeActor, R"( + UPSERT INTO `/Root/Table` (key, value) VALUES + (1, 100) + , (2, 200) + , (3, 300) + ; + )"); + + ExecSQL(server, edgeActor, R"( + UPSERT INTO `/Root/Table` (key, value) VALUES (2, 250); + )"); + + ExecSQL(server, edgeActor, R"(DELETE FROM `/Root/Table` WHERE key=3;)"); + + ExecSQL(server, edgeActor, R"(BACKUP `MyCollection` INCREMENTAL;)", false); + + SimulateSleep(server, TDuration::Seconds(5)); + + // Find the incremental backup directory using DescribePath + TString backupDir = FindIncrementalBackupDir(runtime, edgeActor, "/Root/.backups/collections/MyCollection"); + UNIT_ASSERT_C(!backupDir.empty(), "Could not find incremental backup directory"); + + Cerr << "Using backup directory: " << backupDir << Endl; + + // Verify the incremental backup table was created using DescribePath + TString mainTablePath = TStringBuilder() << "/Root/.backups/collections/MyCollection/" << backupDir << "/Table"; + + auto tableRequest = MakeHolder(); + tableRequest->Record.MutableDescribePath()->SetPath(mainTablePath); + tableRequest->Record.MutableDescribePath()->MutableOptions()->SetShowPrivateTable(true); + runtime.Send(new IEventHandle(MakeTxProxyID(), edgeActor, tableRequest.Release())); + auto tableReply = runtime.GrabEdgeEventRethrow(edgeActor); + + UNIT_ASSERT_EQUAL(tableReply->Get()->GetRecord().GetStatus(), NKikimrScheme::EStatus::StatusSuccess); + UNIT_ASSERT(tableReply->Get()->GetRecord().GetPathDescription().HasTable()); + + // Verify the table has the expected schema (including incremental backup metadata column) + bool hasChangeMetadataColumn = false; + for (const auto& col : tableReply->Get()->GetRecord().GetPathDescription().GetTable().GetColumns()) { + if (col.GetName() == "__ydb_incrBackupImpl_changeMetadata") { + hasChangeMetadataColumn = true; + break; + } + } + UNIT_ASSERT_C(hasChangeMetadataColumn, "Incremental backup table should have __ydb_incrBackupImpl_changeMetadata column"); + + // Now verify the actual data + auto mainTableBackup = KqpSimpleExec(runtime, TStringBuilder() << R"( + SELECT key, value FROM `)" << mainTablePath << R"(` + ORDER BY key + )"); + + UNIT_ASSERT_C(mainTableBackup.find("uint32_value: 2") != TString::npos, + "Main table backup should contain updated key 2"); + UNIT_ASSERT_C(mainTableBackup.find("uint32_value: 250") != TString::npos, + "Main table backup should contain updated value"); + + // Verify index backup does NOT exist when OmitIndexes is set + TString indexMetaPath = TStringBuilder() << "/Root/.backups/collections/MyCollection/" << backupDir << "/__ydb_backup_meta"; + + auto indexMetaRequest = MakeHolder(); + indexMetaRequest->Record.MutableDescribePath()->SetPath(indexMetaPath); + runtime.Send(new IEventHandle(MakeTxProxyID(), edgeActor, indexMetaRequest.Release())); + auto indexMetaReply = runtime.GrabEdgeEventRethrow(edgeActor); + + // With OmitIndexes=true, the __ydb_backup_meta directory should not exist + UNIT_ASSERT_C(indexMetaReply->Get()->GetRecord().GetStatus() == NKikimrScheme::EStatus::StatusPathDoesNotExist, + "Index backup metadata directory should NOT exist when OmitIndexes flag is set"); + } + Y_UNIT_TEST(BasicIndexIncrementalRestore) { + TPortManager portManager; + TServer::TPtr server = new TServer(TServerSettings(portManager.GetPort(2134), {}, DefaultPQConfig()) + .SetUseRealThreads(false) + .SetDomainName("Root") + .SetEnableChangefeedInitialScan(true) + .SetEnableBackupService(true) + .SetEnableRealSystemViewPaths(false) + ); + + auto& runtime = *server->GetRuntime(); + const auto edgeActor = runtime.AllocateEdgeActor(); + + SetupLogging(runtime); + InitRoot(server, edgeActor); + + // Create table with one global index + CreateShardedTable(server, edgeActor, "/Root", "TableWithIndex", + TShardedTableOptions() + .Columns({ + {"key", "Uint32", true, false}, + {"value", "Uint32", false, false}, + {"indexed_col", "Uint32", false, false} + }) + .Indexes({ + {"value_index", {"indexed_col"}, {}, NKikimrSchemeOp::EIndexTypeGlobal} + })); + + // Insert data + ExecSQL(server, edgeActor, R"( + UPSERT INTO `/Root/TableWithIndex` (key, value, indexed_col) VALUES + (1, 10, 100), + (2, 20, 200), + (3, 30, 300); + )"); + + // Create backup collection + ExecSQL(server, edgeActor, R"( + CREATE BACKUP COLLECTION `IndexTestCollection` + ( TABLE `/Root/TableWithIndex` + ) + WITH + ( STORAGE = 'cluster' + , INCREMENTAL_BACKUP_ENABLED = 'true' + ); + )", false); + + // Create full backup + ExecSQL(server, edgeActor, R"(BACKUP `IndexTestCollection`;)", false); + SimulateSleep(server, TDuration::Seconds(1)); + + // Modify data + ExecSQL(server, edgeActor, R"( + UPSERT INTO `/Root/TableWithIndex` (key, value, indexed_col) VALUES + (4, 40, 400), + (2, 25, 250); + )"); + + // Create incremental backup + ExecSQL(server, edgeActor, R"(BACKUP `IndexTestCollection` INCREMENTAL;)", false); + SimulateSleep(server, TDuration::Seconds(5)); + + // Capture expected state + auto expectedTable = KqpSimpleExecSuccess(runtime, R"( + SELECT key, value, indexed_col FROM `/Root/TableWithIndex` ORDER BY key + )"); + + auto expectedIndex = KqpSimpleExecSuccess(runtime, R"( + SELECT indexed_col FROM `/Root/TableWithIndex` VIEW value_index WHERE indexed_col > 0 ORDER BY indexed_col + )"); + + // Drop table (this also drops index) + ExecSQL(server, edgeActor, R"(DROP TABLE `/Root/TableWithIndex`;)", false); + + // Restore from backups + ExecSQL(server, edgeActor, R"(RESTORE `IndexTestCollection`;)", false); + runtime.SimulateSleep(TDuration::Seconds(10)); + + // Verify table data + auto actualTable = KqpSimpleExecSuccess(runtime, R"( + SELECT key, value, indexed_col FROM `/Root/TableWithIndex` ORDER BY key + )"); + UNIT_ASSERT_VALUES_EQUAL(expectedTable, actualTable); + + // Verify index works and has correct data + auto actualIndex = KqpSimpleExecSuccess(runtime, R"( + SELECT indexed_col FROM `/Root/TableWithIndex` VIEW value_index WHERE indexed_col > 0 ORDER BY indexed_col + )"); + UNIT_ASSERT_VALUES_EQUAL(expectedIndex, actualIndex); + + // Verify we can query using the index + auto indexQuery = KqpSimpleExecSuccess(runtime, R"( + SELECT key, indexed_col FROM `/Root/TableWithIndex` VIEW value_index WHERE indexed_col = 250 + )"); + UNIT_ASSERT_C(indexQuery.find("uint32_value: 2") != TString::npos, "Should find key=2"); + UNIT_ASSERT_C(indexQuery.find("uint32_value: 250") != TString::npos, "Should find indexed_col=250"); + + // Verify index implementation table was restored correctly + auto indexImplTable = KqpSimpleExecSuccess(runtime, R"( + SELECT indexed_col, key FROM `/Root/TableWithIndex/value_index/indexImplTable` ORDER BY indexed_col + )"); + // Should have 4 rows after incremental: (100,1), (250,2), (300,3), (400,4) + UNIT_ASSERT_C(indexImplTable.find("uint32_value: 100") != TString::npos, "Index table should have indexed_col=100"); + UNIT_ASSERT_C(indexImplTable.find("uint32_value: 250") != TString::npos, "Index table should have indexed_col=250"); + UNIT_ASSERT_C(indexImplTable.find("uint32_value: 300") != TString::npos, "Index table should have indexed_col=300"); + UNIT_ASSERT_C(indexImplTable.find("uint32_value: 400") != TString::npos, "Index table should have indexed_col=400"); + + // Count rows in index impl table + auto indexRowCount = KqpSimpleExecSuccess(runtime, R"( + SELECT COUNT(*) FROM `/Root/TableWithIndex/value_index/indexImplTable` + )"); + UNIT_ASSERT_C(indexRowCount.find("uint64_value: 4") != TString::npos, "Index table should have 4 rows"); + } + + Y_UNIT_TEST(MultipleIndexesIncrementalRestore) { + TPortManager portManager; + TServer::TPtr server = new TServer(TServerSettings(portManager.GetPort(2134), {}, DefaultPQConfig()) + .SetUseRealThreads(false) + .SetDomainName("Root") + .SetEnableChangefeedInitialScan(true) + .SetEnableBackupService(true) + .SetEnableRealSystemViewPaths(false) + ); + + auto& runtime = *server->GetRuntime(); + const auto edgeActor = runtime.AllocateEdgeActor(); + + SetupLogging(runtime); + InitRoot(server, edgeActor); + + // Create table with multiple global indexes + CreateShardedTable(server, edgeActor, "/Root", "MultiIndexTable", + TShardedTableOptions() + .Columns({ + {"key", "Uint32", true, false}, + {"value1", "Uint32", false, false}, + {"value2", "Uint32", false, false}, + {"value3", "Uint32", false, false} + }) + .Indexes({ + {"index1", {"value1"}, {}, NKikimrSchemeOp::EIndexTypeGlobal}, + {"index2", {"value2"}, {}, NKikimrSchemeOp::EIndexTypeGlobal}, + {"index3", {"value3"}, {}, NKikimrSchemeOp::EIndexTypeGlobal} + })); + + // Insert data + ExecSQL(server, edgeActor, R"( + UPSERT INTO `/Root/MultiIndexTable` (key, value1, value2, value3) VALUES + (1, 11, 21, 31), + (2, 12, 22, 32), + (3, 13, 23, 33); + )"); + + // Create backup collection + ExecSQL(server, edgeActor, R"( + CREATE BACKUP COLLECTION `MultiIndexCollection` + ( TABLE `/Root/MultiIndexTable` + ) + WITH + ( STORAGE = 'cluster' + , INCREMENTAL_BACKUP_ENABLED = 'true' + ); + )", false); + + // Create full backup + ExecSQL(server, edgeActor, R"(BACKUP `MultiIndexCollection`;)", false); + // Wait for CDC streams to be fully created and schema versions to stabilize + SimulateSleep(server, TDuration::Seconds(5)); + + // Modify data + ExecSQL(server, edgeActor, R"( + UPSERT INTO `/Root/MultiIndexTable` (key, value1, value2, value3) VALUES + (4, 14, 24, 34); + )"); + + // Create incremental backup + ExecSQL(server, edgeActor, R"(BACKUP `MultiIndexCollection` INCREMENTAL;)", false); + SimulateSleep(server, TDuration::Seconds(5)); + + // Capture expected state for all indexes + auto expectedTable = KqpSimpleExecSuccess(runtime, R"( + SELECT key, value1, value2, value3 FROM `/Root/MultiIndexTable` ORDER BY key + )"); + + // Drop and restore + ExecSQL(server, edgeActor, R"(DROP TABLE `/Root/MultiIndexTable`;)", false); + ExecSQL(server, edgeActor, R"(RESTORE `MultiIndexCollection`;)", false); + runtime.SimulateSleep(TDuration::Seconds(10)); + + // Verify table data + auto actualTable = KqpSimpleExecSuccess(runtime, R"( + SELECT key, value1, value2, value3 FROM `/Root/MultiIndexTable` ORDER BY key + )"); + UNIT_ASSERT_VALUES_EQUAL(expectedTable, actualTable); + + // Verify all indexes work + auto index1Query = KqpSimpleExecSuccess(runtime, R"( + SELECT key FROM `/Root/MultiIndexTable` VIEW index1 WHERE value1 = 14 + )"); + UNIT_ASSERT_C(index1Query.find("uint32_value: 4") != TString::npos, "Index1 should work"); + + auto index2Query = KqpSimpleExecSuccess(runtime, R"( + SELECT key FROM `/Root/MultiIndexTable` VIEW index2 WHERE value2 = 24 + )"); + UNIT_ASSERT_C(index2Query.find("uint32_value: 4") != TString::npos, "Index2 should work"); + + auto index3Query = KqpSimpleExecSuccess(runtime, R"( + SELECT key FROM `/Root/MultiIndexTable` VIEW index3 WHERE value3 = 34 + )"); + UNIT_ASSERT_C(index3Query.find("uint32_value: 4") != TString::npos, "Index3 should work"); + + // Verify all index implementation tables were restored + auto index1ImplCount = KqpSimpleExecSuccess(runtime, R"( + SELECT COUNT(*) FROM `/Root/MultiIndexTable/index1/indexImplTable` + )"); + UNIT_ASSERT_C(index1ImplCount.find("uint64_value: 4") != TString::npos, "Index1 impl table should have 4 rows"); + + auto index2ImplCount = KqpSimpleExecSuccess(runtime, R"( + SELECT COUNT(*) FROM `/Root/MultiIndexTable/index2/indexImplTable` + )"); + UNIT_ASSERT_C(index2ImplCount.find("uint64_value: 4") != TString::npos, "Index2 impl table should have 4 rows"); + + auto index3ImplCount = KqpSimpleExecSuccess(runtime, R"( + SELECT COUNT(*) FROM `/Root/MultiIndexTable/index3/indexImplTable` + )"); + UNIT_ASSERT_C(index3ImplCount.find("uint64_value: 4") != TString::npos, "Index3 impl table should have 4 rows"); + + // Verify index3 impl table data (spot check) + auto index3ImplData = KqpSimpleExecSuccess(runtime, R"( + SELECT value3, key FROM `/Root/MultiIndexTable/index3/indexImplTable` WHERE value3 = 34 + )"); + UNIT_ASSERT_C(index3ImplData.find("uint32_value: 34") != TString::npos, "Index3 impl should have value3=34"); + UNIT_ASSERT_C(index3ImplData.find("uint32_value: 4") != TString::npos, "Index3 impl should have key=4"); + } + + Y_UNIT_TEST(IndexDataVerificationIncrementalRestore) { + TPortManager portManager; + TServer::TPtr server = new TServer(TServerSettings(portManager.GetPort(2134), {}, DefaultPQConfig()) + .SetUseRealThreads(false) + .SetDomainName("Root") + .SetEnableChangefeedInitialScan(true) + .SetEnableBackupService(true) + .SetEnableRealSystemViewPaths(false) + ); + + auto& runtime = *server->GetRuntime(); + const auto edgeActor = runtime.AllocateEdgeActor(); + + SetupLogging(runtime); + InitRoot(server, edgeActor); + + // Create table with index + CreateShardedTable(server, edgeActor, "/Root", "DataVerifyTable", + TShardedTableOptions() + .Shards(2) + .Columns({ + {"key", "Uint32", true, false}, + {"name", "Utf8", false, false}, + {"age", "Uint32", false, false} + }) + .Indexes({ + {"age_index", {"age"}, {}, NKikimrSchemeOp::EIndexTypeGlobal} + })); + + // Insert data across shards + ExecSQL(server, edgeActor, R"( + UPSERT INTO `/Root/DataVerifyTable` (key, name, age) VALUES + (1, 'Alice', 25), + (2, 'Bob', 30), + (11, 'Charlie', 35), + (12, 'David', 40); + )"); + + // Create backup collection + ExecSQL(server, edgeActor, R"( + CREATE BACKUP COLLECTION `DataVerifyCollection` + ( TABLE `/Root/DataVerifyTable` + ) + WITH + ( STORAGE = 'cluster' + , INCREMENTAL_BACKUP_ENABLED = 'true' + ); + )", false); + + // Full backup + ExecSQL(server, edgeActor, R"(BACKUP `DataVerifyCollection`;)", false); + SimulateSleep(server, TDuration::Seconds(1)); + + // Modify: update existing records and add new ones + ExecSQL(server, edgeActor, R"( + UPSERT INTO `/Root/DataVerifyTable` (key, name, age) VALUES + (2, 'Bob', 31), -- update in shard 1 + (12, 'David', 41), -- update in shard 2 + (3, 'Eve', 28), -- new in shard 1 + (13, 'Frank', 45); -- new in shard 2 + )"); + + // Delete some records + ExecSQL(server, edgeActor, R"( + DELETE FROM `/Root/DataVerifyTable` WHERE key IN (1, 11); + )"); + + // Incremental backup + ExecSQL(server, edgeActor, R"(BACKUP `DataVerifyCollection` INCREMENTAL;)", false); + SimulateSleep(server, TDuration::Seconds(5)); + + // Verify index has correct data BEFORE restore + auto beforeRestore = KqpSimpleExecSuccess(runtime, R"( + SELECT key, name, age FROM `/Root/DataVerifyTable` VIEW age_index WHERE age >= 30 ORDER BY age + )"); + + // Drop and restore + ExecSQL(server, edgeActor, R"(DROP TABLE `/Root/DataVerifyTable`;)", false); + ExecSQL(server, edgeActor, R"(RESTORE `DataVerifyCollection`;)", false); + runtime.SimulateSleep(TDuration::Seconds(10)); + + // Verify index has correct data AFTER restore + auto afterRestore = KqpSimpleExecSuccess(runtime, R"( + SELECT key, name, age FROM `/Root/DataVerifyTable` VIEW age_index WHERE age >= 30 ORDER BY age + )"); + + UNIT_ASSERT_VALUES_EQUAL(beforeRestore, afterRestore); + + // Verify specific queries + UNIT_ASSERT_C(afterRestore.find("text_value: \"Bob\"") != TString::npos, "Bob should be present"); + UNIT_ASSERT_C(afterRestore.find("uint32_value: 31") != TString::npos, "Age 31 should be present"); + UNIT_ASSERT_C(afterRestore.find("text_value: \"Alice\"") == TString::npos, "Alice should be deleted"); + UNIT_ASSERT_C(afterRestore.find("text_value: \"Frank\"") != TString::npos, "Frank should be present"); + UNIT_ASSERT_C(afterRestore.find("uint32_value: 45") != TString::npos, "Age 45 should be present"); + + // Verify index implementation table has correct data + // Note: Index impl tables only contain index key columns (age, key), not data columns (name) + auto indexImplData = KqpSimpleExecSuccess(runtime, R"( + SELECT age, key FROM `/Root/DataVerifyTable/age_index/indexImplTable` ORDER BY age + )"); + // Should have: (28, 3), (31, 2), (41, 12), (45, 13) + // Deleted: (25, 1), (35, 11) + UNIT_ASSERT_C(indexImplData.find("uint32_value: 28") != TString::npos, "Index should have age=28"); + UNIT_ASSERT_C(indexImplData.find("uint32_value: 3") != TString::npos, "Index should have key=3 (Eve's key)"); + UNIT_ASSERT_C(indexImplData.find("uint32_value: 31") != TString::npos, "Index should have age=31"); + UNIT_ASSERT_C(indexImplData.find("uint32_value: 2") != TString::npos, "Index should have key=2 (Bob's key)"); + UNIT_ASSERT_C(indexImplData.find("uint32_value: 25") == TString::npos, "Index should NOT have age=25 (Alice deleted)"); + UNIT_ASSERT_C(indexImplData.find("uint32_value: 35") == TString::npos, "Index should NOT have age=35 (Charlie deleted)"); + + auto indexImplCount = KqpSimpleExecSuccess(runtime, R"( + SELECT COUNT(*) FROM `/Root/DataVerifyTable/age_index/indexImplTable` + )"); + UNIT_ASSERT_C(indexImplCount.find("uint64_value: 4") != TString::npos, "Index impl table should have 4 rows"); + } + + Y_UNIT_TEST(MultipleIncrementalBackupsWithIndexes) { + TPortManager portManager; + TServer::TPtr server = new TServer(TServerSettings(portManager.GetPort(2134), {}, DefaultPQConfig()) + .SetUseRealThreads(false) + .SetDomainName("Root") + .SetEnableChangefeedInitialScan(true) + .SetEnableBackupService(true) + .SetEnableRealSystemViewPaths(false) + ); + + auto& runtime = *server->GetRuntime(); + const auto edgeActor = runtime.AllocateEdgeActor(); + + SetupLogging(runtime); + InitRoot(server, edgeActor); + + // Create table with index + CreateShardedTable(server, edgeActor, "/Root", "SequenceTable", + TShardedTableOptions() + .Columns({ + {"key", "Uint32", true, false}, + {"value", "Uint32", false, false}, + {"indexed", "Uint32", false, false} + }) + .Indexes({ + {"idx", {"indexed"}, {}, NKikimrSchemeOp::EIndexTypeGlobal} + })); + + // Initial data + ExecSQL(server, edgeActor, R"( + UPSERT INTO `/Root/SequenceTable` (key, value, indexed) VALUES + (1, 10, 100), + (2, 20, 200); + )"); + + // Create backup collection + ExecSQL(server, edgeActor, R"( + CREATE BACKUP COLLECTION `SequenceCollection` + ( TABLE `/Root/SequenceTable` + ) + WITH + ( STORAGE = 'cluster' + , INCREMENTAL_BACKUP_ENABLED = 'true' + ); + )", false); + + // Full backup + ExecSQL(server, edgeActor, R"(BACKUP `SequenceCollection`;)", false); + SimulateSleep(server, TDuration::Seconds(1)); + + // First incremental: add data + ExecSQL(server, edgeActor, R"( + UPSERT INTO `/Root/SequenceTable` (key, value, indexed) VALUES (3, 30, 300); + )"); + ExecSQL(server, edgeActor, R"(BACKUP `SequenceCollection` INCREMENTAL;)", false); + SimulateSleep(server, TDuration::Seconds(5)); + + // Second incremental: update data + ExecSQL(server, edgeActor, R"( + UPSERT INTO `/Root/SequenceTable` (key, value, indexed) VALUES (2, 25, 250); + )"); + ExecSQL(server, edgeActor, R"(BACKUP `SequenceCollection` INCREMENTAL;)", false); + SimulateSleep(server, TDuration::Seconds(5)); + + // Third incremental: delete and add + ExecSQL(server, edgeActor, R"( + DELETE FROM `/Root/SequenceTable` WHERE key = 1; + )"); + ExecSQL(server, edgeActor, R"( + UPSERT INTO `/Root/SequenceTable` (key, value, indexed) VALUES (4, 40, 400); + )"); + ExecSQL(server, edgeActor, R"(BACKUP `SequenceCollection` INCREMENTAL;)", false); + SimulateSleep(server, TDuration::Seconds(5)); + + // Capture expected state + auto expectedTable = KqpSimpleExecSuccess(runtime, R"( + SELECT key, value, indexed FROM `/Root/SequenceTable` ORDER BY key + )"); + + auto expectedIndex = KqpSimpleExecSuccess(runtime, R"( + SELECT indexed FROM `/Root/SequenceTable` VIEW idx WHERE indexed > 0 ORDER BY indexed + )"); + + // Drop and restore + ExecSQL(server, edgeActor, R"(DROP TABLE `/Root/SequenceTable`;)", false); + ExecSQL(server, edgeActor, R"(RESTORE `SequenceCollection`;)", false); + runtime.SimulateSleep(TDuration::Seconds(15)); + + // Verify + auto actualTable = KqpSimpleExecSuccess(runtime, R"( + SELECT key, value, indexed FROM `/Root/SequenceTable` ORDER BY key + )"); + UNIT_ASSERT_VALUES_EQUAL(expectedTable, actualTable); + + auto actualIndex = KqpSimpleExecSuccess(runtime, R"( + SELECT indexed FROM `/Root/SequenceTable` VIEW idx WHERE indexed > 0 ORDER BY indexed + )"); + UNIT_ASSERT_VALUES_EQUAL(expectedIndex, actualIndex); + + // Verify final state: key 1 deleted, key 2 updated, keys 3 and 4 added + UNIT_ASSERT_C(actualTable.find("uint32_value: 1") == TString::npos, "Key 1 should be deleted"); + UNIT_ASSERT_C(actualTable.find("uint32_value: 25") != TString::npos, "Key 2 should have value 25"); + UNIT_ASSERT_C(actualTable.find("uint32_value: 30") != TString::npos, "Key 3 should exist"); + UNIT_ASSERT_C(actualTable.find("uint32_value: 40") != TString::npos, "Key 4 should exist"); + + // Verify index implementation table reflects all 3 incremental changes + auto indexImplData = KqpSimpleExecSuccess(runtime, R"( + SELECT indexed, key FROM `/Root/SequenceTable/idx/indexImplTable` ORDER BY indexed + )"); + // Final state should be: (250, 2), (300, 3), (400, 4) + // Deleted: (100, 1), (200, 2->old value) + UNIT_ASSERT_C(indexImplData.find("uint32_value: 100") == TString::npos, "Index should NOT have indexed=100 (deleted)"); + UNIT_ASSERT_C(indexImplData.find("uint32_value: 200") == TString::npos, "Index should NOT have indexed=200 (updated)"); + UNIT_ASSERT_C(indexImplData.find("uint32_value: 250") != TString::npos, "Index should have indexed=250 (updated value)"); + UNIT_ASSERT_C(indexImplData.find("uint32_value: 300") != TString::npos, "Index should have indexed=300 (added)"); + UNIT_ASSERT_C(indexImplData.find("uint32_value: 400") != TString::npos, "Index should have indexed=400 (added)"); + + auto indexImplCount = KqpSimpleExecSuccess(runtime, R"( + SELECT COUNT(*) FROM `/Root/SequenceTable/idx/indexImplTable` + )"); + UNIT_ASSERT_C(indexImplCount.find("uint64_value: 3") != TString::npos, "Index impl table should have 3 rows"); + } + + Y_UNIT_TEST(MultipleTablesWithIndexesIncrementalRestore) { + TPortManager portManager; + TServer::TPtr server = new TServer(TServerSettings(portManager.GetPort(2134), {}, DefaultPQConfig()) + .SetUseRealThreads(false) + .SetDomainName("Root") + .SetEnableChangefeedInitialScan(true) + .SetEnableBackupService(true) + .SetEnableRealSystemViewPaths(false) + ); + + auto& runtime = *server->GetRuntime(); + const auto edgeActor = runtime.AllocateEdgeActor(); + + SetupLogging(runtime); + InitRoot(server, edgeActor); + + // Create first table with index + CreateShardedTable(server, edgeActor, "/Root", "Table1", + TShardedTableOptions() + .Columns({ + {"key", "Uint32", true, false}, + {"val1", "Uint32", false, false} + }) + .Indexes({ + {"idx1", {"val1"}, {}, NKikimrSchemeOp::EIndexTypeGlobal} + })); + + // Create second table with different index + CreateShardedTable(server, edgeActor, "/Root", "Table2", + TShardedTableOptions() + .Columns({ + {"key", "Uint32", true, false}, + {"val2", "Uint32", false, false} + }) + .Indexes({ + {"idx2", {"val2"}, {}, NKikimrSchemeOp::EIndexTypeGlobal} + })); + + // Insert data into both tables + ExecSQL(server, edgeActor, R"( + UPSERT INTO `/Root/Table1` (key, val1) VALUES (1, 100), (2, 200); + UPSERT INTO `/Root/Table2` (key, val2) VALUES (1, 1000), (2, 2000); + )"); + + // Create backup collection with both tables + ExecSQL(server, edgeActor, R"( + CREATE BACKUP COLLECTION `MultiTableCollection` + ( TABLE `/Root/Table1` + , TABLE `/Root/Table2` + ) + WITH + ( STORAGE = 'cluster' + , INCREMENTAL_BACKUP_ENABLED = 'true' + ); + )", false); + + // Full backup + ExecSQL(server, edgeActor, R"(BACKUP `MultiTableCollection`;)", false); + SimulateSleep(server, TDuration::Seconds(1)); + + // Modify both tables + ExecSQL(server, edgeActor, R"( + UPSERT INTO `/Root/Table1` (key, val1) VALUES (3, 300); + UPSERT INTO `/Root/Table2` (key, val2) VALUES (3, 3000); + )"); + + // Incremental backup + ExecSQL(server, edgeActor, R"(BACKUP `MultiTableCollection` INCREMENTAL;)", false); + SimulateSleep(server, TDuration::Seconds(5)); + + // Capture expected states + auto expected1 = KqpSimpleExecSuccess(runtime, R"( + SELECT key, val1 FROM `/Root/Table1` ORDER BY key + )"); + auto expected2 = KqpSimpleExecSuccess(runtime, R"( + SELECT key, val2 FROM `/Root/Table2` ORDER BY key + )"); + + // Drop both tables + ExecSQL(server, edgeActor, R"(DROP TABLE `/Root/Table1`;)", false); + ExecSQL(server, edgeActor, R"(DROP TABLE `/Root/Table2`;)", false); + + // Restore + ExecSQL(server, edgeActor, R"(RESTORE `MultiTableCollection`;)", false); + runtime.SimulateSleep(TDuration::Seconds(10)); + + // Verify both tables and indexes + auto actual1 = KqpSimpleExecSuccess(runtime, R"( + SELECT key, val1 FROM `/Root/Table1` ORDER BY key + )"); + auto actual2 = KqpSimpleExecSuccess(runtime, R"( + SELECT key, val2 FROM `/Root/Table2` ORDER BY key + )"); + + UNIT_ASSERT_VALUES_EQUAL(expected1, actual1); + UNIT_ASSERT_VALUES_EQUAL(expected2, actual2); + + // Verify indexes work + auto idx1Query = KqpSimpleExecSuccess(runtime, R"( + SELECT key FROM `/Root/Table1` VIEW idx1 WHERE val1 = 300 + )"); + UNIT_ASSERT_C(idx1Query.find("uint32_value: 3") != TString::npos, "Index idx1 should work"); + + auto idx2Query = KqpSimpleExecSuccess(runtime, R"( + SELECT key FROM `/Root/Table2` VIEW idx2 WHERE val2 = 3000 + )"); + UNIT_ASSERT_C(idx2Query.find("uint32_value: 3") != TString::npos, "Index idx2 should work"); + + // Verify both index implementation tables were restored + auto idx1ImplCount = KqpSimpleExecSuccess(runtime, R"( + SELECT COUNT(*) FROM `/Root/Table1/idx1/indexImplTable` + )"); + UNIT_ASSERT_C(idx1ImplCount.find("uint64_value: 3") != TString::npos, "Table1 index impl should have 3 rows"); + + auto idx2ImplCount = KqpSimpleExecSuccess(runtime, R"( + SELECT COUNT(*) FROM `/Root/Table2/idx2/indexImplTable` + )"); + UNIT_ASSERT_C(idx2ImplCount.find("uint64_value: 3") != TString::npos, "Table2 index impl should have 3 rows"); + + // Verify index impl tables have correct data + auto idx1ImplData = KqpSimpleExecSuccess(runtime, R"( + SELECT val1, key FROM `/Root/Table1/idx1/indexImplTable` WHERE val1 = 300 + )"); + UNIT_ASSERT_C(idx1ImplData.find("uint32_value: 300") != TString::npos, "Table1 index should have val1=300"); + UNIT_ASSERT_C(idx1ImplData.find("uint32_value: 3") != TString::npos, "Table1 index should have key=3"); + + auto idx2ImplData = KqpSimpleExecSuccess(runtime, R"( + SELECT val2, key FROM `/Root/Table2/idx2/indexImplTable` WHERE val2 = 3000 + )"); + UNIT_ASSERT_C(idx2ImplData.find("uint32_value: 3000") != TString::npos, "Table2 index should have val2=3000"); + UNIT_ASSERT_C(idx2ImplData.find("uint32_value: 3") != TString::npos, "Table2 index should have key=3"); + } + + + Y_UNIT_TEST(CdcVersionSync) { + TPortManager portManager; + TServer::TPtr server = new TServer(TServerSettings(portManager.GetPort(2134), {}, DefaultPQConfig()) + .SetUseRealThreads(false) + .SetDomainName("Root") + .SetEnableChangefeedInitialScan(true) + .SetEnableBackupService(true) + .SetEnableRealSystemViewPaths(false) + ); + + auto& runtime = *server->GetRuntime(); + const auto edgeActor = runtime.AllocateEdgeActor(); + + SetupLogging(runtime); + InitRoot(server, edgeActor); + + // Create first table with index + CreateShardedTable(server, edgeActor, "/Root", "Table1", + TShardedTableOptions() + .Columns({ + {"key", "Uint32", true, false}, + {"val1", "Uint32", false, false} + }) + .Indexes({ + {"idx1", {"val1"}, {}, NKikimrSchemeOp::EIndexTypeGlobal} + })); + + // Create second table with different index + CreateShardedTable(server, edgeActor, "/Root", "Table2", + TShardedTableOptions() + .Columns({ + {"key", "Uint32", true, false}, + {"val2", "Uint32", false, false} + }) + .Indexes({ + {"idx2", {"val2"}, {}, NKikimrSchemeOp::EIndexTypeGlobal} + })); + + // Insert data into both tables + ExecSQL(server, edgeActor, R"( + UPSERT INTO `/Root/Table1` (key, val1) VALUES (1, 100), (2, 200); + UPSERT INTO `/Root/Table2` (key, val2) VALUES (1, 1000), (2, 2000); + )"); + + // Create backup collection with both tables + ExecSQL(server, edgeActor, R"( + CREATE BACKUP COLLECTION `MultiTableCollection` + ( TABLE `/Root/Table1` + , TABLE `/Root/Table2` + ) + WITH + ( STORAGE = 'cluster' + , INCREMENTAL_BACKUP_ENABLED = 'true' + ); + )", false); + + // Full backup + ExecSQL(server, edgeActor, R"(BACKUP `MultiTableCollection`;)", false); + SimulateSleep(server, TDuration::Seconds(1)); + + // Modify both tables + ExecSQL(server, edgeActor, R"( + UPSERT INTO `/Root/Table1` (key, val1) VALUES (3, 300); + UPSERT INTO `/Root/Table2` (key, val2) VALUES (3, 3000); + )"); + + // Incremental backup + ExecSQL(server, edgeActor, R"(BACKUP `MultiTableCollection` INCREMENTAL;)", false); + SimulateSleep(server, TDuration::Seconds(5)); + + // Capture expected states + ExecSQL(server, edgeActor, R"( + SELECT key, val1 FROM `/Root/Table1` ORDER BY key + )"); + + ExecSQL(server, edgeActor, R"( + SELECT key, val2 FROM `/Root/Table2` ORDER BY key + )"); + + // Drop both tables + ExecSQL(server, edgeActor, R"(DROP TABLE `/Root/Table1`;)", false); + } + } // Y_UNIT_TEST_SUITE(IncrementalBackup) } // NKikimr diff --git a/ydb/core/tx/datashard/ut_common/datashard_ut_common.cpp b/ydb/core/tx/datashard/ut_common/datashard_ut_common.cpp index 8023e50fbe91..3328988cd8b7 100644 --- a/ydb/core/tx/datashard/ut_common/datashard_ut_common.cpp +++ b/ydb/core/tx/datashard/ut_common/datashard_ut_common.cpp @@ -2099,7 +2099,8 @@ void ExecSQL(Tests::TServer::TPtr server, TActorId sender, const TString &sql, bool dml, - Ydb::StatusIds::StatusCode code) + Ydb::StatusIds::StatusCode code, + NYdb::NUt::TTestContext testCtx) { auto &runtime = *server->GetRuntime(); auto request = MakeSQLRequest(sql, dml); @@ -2107,9 +2108,9 @@ void ExecSQL(Tests::TServer::TPtr server, auto ev = runtime.GrabEdgeEventRethrow(sender); auto& response = ev->Get()->Record; auto& issues = response.GetResponse().GetQueryIssues(); - UNIT_ASSERT_VALUES_EQUAL_C(response.GetYdbStatus(), - code, - issues.empty() ? response.DebugString() : issues.Get(0).DebugString() + CTX_UNIT_ASSERT_VALUES_EQUAL_C(response.GetYdbStatus(), + code, + issues.empty() ? response.DebugString() : issues.Get(0).DebugString() ); } diff --git a/ydb/core/tx/datashard/ut_common/datashard_ut_common.h b/ydb/core/tx/datashard/ut_common/datashard_ut_common.h index fe0d29c29c92..67cb22949056 100644 --- a/ydb/core/tx/datashard/ut_common/datashard_ut_common.h +++ b/ydb/core/tx/datashard/ut_common/datashard_ut_common.h @@ -10,6 +10,8 @@ #include #include #include +#include +#include #include #include @@ -808,7 +810,8 @@ void ExecSQL(Tests::TServer::TPtr server, TActorId sender, const TString &sql, bool dml = true, - Ydb::StatusIds::StatusCode code = Ydb::StatusIds::SUCCESS); + Ydb::StatusIds::StatusCode code = Ydb::StatusIds::SUCCESS, + NYdb::NUt::TTestContext testCtx = NYdb::NUt::TTestContext()); TRowVersion AcquireReadSnapshot(TTestActorRuntime& runtime, const TString& databaseName, ui32 nodeIndex = 0); diff --git a/ydb/core/tx/datashard/ut_common/ya.make b/ydb/core/tx/datashard/ut_common/ya.make index 18099689b44a..a8f922dec6d4 100644 --- a/ydb/core/tx/datashard/ut_common/ya.make +++ b/ydb/core/tx/datashard/ut_common/ya.make @@ -3,6 +3,7 @@ LIBRARY() PEERDIR( contrib/libs/protobuf ydb/core/kqp/ut/common + ydb/library/ut ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/tx/schemeshard/schemeshard__backup_collection_common.cpp b/ydb/core/tx/schemeshard/schemeshard__backup_collection_common.cpp index 28c22af19ece..a44e5e18897c 100644 --- a/ydb/core/tx/schemeshard/schemeshard__backup_collection_common.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__backup_collection_common.cpp @@ -130,6 +130,65 @@ std::optional>> GetBackupRequiredPaths( } } + // Add index backup metadata directories if incremental backup is enabled + bool incrBackupEnabled = bc->Description.HasIncrementalBackupConfig(); + + // Check OmitIndexes from two possible locations: + // 1. Top-level OmitIndexes field (for full backups) + // 2. IncrementalBackupConfig.OmitIndexes (for incremental backups) + bool omitIndexes = bc->Description.GetOmitIndexes() || + (incrBackupEnabled && bc->Description.GetIncrementalBackupConfig().GetOmitIndexes()); + + if (incrBackupEnabled && !omitIndexes) { + for (const auto& item : bc->Description.GetExplicitEntryList().GetEntries()) { + const auto tablePath = TPath::Resolve(item.GetPath(), context.SS); + + // Skip if path is not resolved or not a table + auto checks = tablePath.Check(); + checks.IsResolved().IsTable(); + if (!checks) { + continue; + } + + std::pair paths; + TString err; + if (!TrySplitPathByDb(item.GetPath(), tx.GetWorkingDir(), paths, err)) { + continue; + } + auto& relativeItemPath = paths.second; + + // Check if table has indexes + for (const auto& [childName, childPathId] : tablePath.Base()->GetChildren()) { + auto childPath = context.SS->PathsById.at(childPathId); + + if (childPath->PathType != NKikimrSchemeOp::EPathTypeTableIndex) { + continue; + } + + // Skip deleted indexes + if (childPath->Dropped()) { + continue; + } + + auto indexInfo = context.SS->Indexes.at(childPathId); + if (indexInfo->Type != NKikimrSchemeOp::EIndexTypeGlobal) { + continue; + } + + // Add required PARENT directory path for index backup: + // {targetDir}/__ydb_backup_meta/indexes/{table_path} + // The index name will be the table name created within this directory + TString indexBackupParentPath = JoinPath({ + targetDir, + "__ydb_backup_meta", + "indexes", + relativeItemPath + }); + collectionPaths.emplace(indexBackupParentPath); + } + } + } + return paths; } diff --git a/ydb/core/tx/schemeshard/schemeshard__init.cpp b/ydb/core/tx/schemeshard/schemeshard__init.cpp index 4cb2e5ff16a9..c26815a43e20 100644 --- a/ydb/core/tx/schemeshard/schemeshard__init.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__init.cpp @@ -3673,6 +3673,18 @@ struct TSchemeShard::TTxInit : public TTransactionBase { Y_ABORT_UNLESS(deserializeRes); txState.CdcPathId = TPathId::FromProto(proto.GetTxCopyTableExtraData().GetCdcPathId()); } + } else if (txState.TxType == TTxState::TxCreateCdcStreamAtTable || + txState.TxType == TTxState::TxCreateCdcStreamAtTableWithInitialScan || + txState.TxType == TTxState::TxAlterCdcStreamAtTable || + txState.TxType == TTxState::TxAlterCdcStreamAtTableDropSnapshot || + txState.TxType == TTxState::TxDropCdcStreamAtTable || + txState.TxType == TTxState::TxDropCdcStreamAtTableDropSnapshot) { + if (!extraData.empty()) { + NKikimrSchemeOp::TGenericTxInFlyExtraData proto; + bool deserializeRes = ParseFromStringNoSizeLimit(proto, extraData); + Y_ABORT_UNLESS(deserializeRes); + txState.CdcPathId = TPathId::FromProto(proto.GetTxCopyTableExtraData().GetCdcPathId()); + } } Y_ABORT_UNLESS(txState.TxType != TTxState::TxInvalid); diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_alter_cdc_stream.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_alter_cdc_stream.cpp index 92155f564df4..2e68d878d6ab 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_alter_cdc_stream.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_alter_cdc_stream.cpp @@ -459,6 +459,7 @@ class TAlterCdcStreamAtTable: public TSubOperation { Y_ABORT_UNLESS(!context.SS->FindTx(OperationId)); auto& txState = context.SS->CreateTx(OperationId, txType, tablePath.Base()->PathId); txState.State = TTxState::ConfigureParts; + txState.CdcPathId = streamPath.Base()->PathId; // Store CDC stream PathId for later use tablePath.Base()->PathState = NKikimrSchemeOp::EPathStateAlter; tablePath.Base()->LastTxId = OperationId.GetTxId(); @@ -635,7 +636,7 @@ TVector CreateAlterCdcStream(TOperationId opId, const TTxTr result.push_back(DropLock(NextPartId(opId, result), outTx)); } - if (workingDirPath.IsTableIndex()) { + if (workingDirPath.IsTableIndex() && !streamName.EndsWith("_continuousBackupImpl")) { auto outTx = TransactionTemplate(workingDirPath.Parent().PathString(), NKikimrSchemeOp::EOperationType::ESchemeOpAlterTableIndex); outTx.MutableAlterTableIndex()->SetName(workingDirPath.LeafName()); outTx.MutableAlterTableIndex()->SetState(NKikimrSchemeOp::EIndexState::EIndexStateReady); diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_backup_backup_collection.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_backup_backup_collection.cpp index 261afefbbec5..903f60c871cd 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_backup_backup_collection.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_backup_backup_collection.cpp @@ -2,6 +2,8 @@ #include "schemeshard__op_traits.h" #include "schemeshard__operation_common.h" #include "schemeshard__operation_create_cdc_stream.h" +#include "schemeshard__operation_part.h" +#include "schemeshard_utils.h" #include "schemeshard_impl.h" @@ -64,6 +66,10 @@ TVector CreateBackupBackupCollection(TOperationId opId, con Y_ABORT_UNLESS(context.SS->BackupCollections.contains(bcPath->PathId)); const auto& bc = context.SS->BackupCollections[bcPath->PathId]; bool incrBackupEnabled = bc->Description.HasIncrementalBackupConfig(); + + bool omitIndexes = bc->Description.GetOmitIndexes() || + (incrBackupEnabled && bc->Description.GetIncrementalBackupConfig().GetOmitIndexes()); + TString streamName = NBackup::ToX509String(TlsActivationContext->AsActorContext().Now()) + "_continuousBackupImpl"; for (const auto& item : bc->Description.GetExplicitEntryList().GetEntries()) { @@ -77,18 +83,14 @@ TVector CreateBackupBackupCollection(TOperationId opId, con } auto& relativeItemPath = paths.second; desc.SetDstPath(JoinPath({tx.GetWorkingDir(), tx.GetBackupBackupCollection().GetName(), tx.GetBackupBackupCollection().GetTargetDir(), relativeItemPath})); - desc.SetOmitIndexes(true); + + desc.SetOmitIndexes(omitIndexes); + desc.SetOmitFollowers(true); desc.SetAllowUnderSameOperation(true); if (incrBackupEnabled) { NKikimrSchemeOp::TCreateCdcStream createCdcStreamOp; - createCdcStreamOp.SetTableName(item.GetPath()); - auto& streamDescription = *createCdcStreamOp.MutableStreamDescription(); - streamDescription.SetName(streamName); - streamDescription.SetMode(NKikimrSchemeOp::ECdcStreamModeUpdate); - streamDescription.SetFormat(NKikimrSchemeOp::ECdcStreamFormatProto); - const auto sPath = TPath::Resolve(item.GetPath(), context.SS); { @@ -104,9 +106,73 @@ TVector CreateBackupBackupCollection(TOperationId opId, con } } + createCdcStreamOp.SetTableName(sPath.LeafName()); + auto& streamDescription = *createCdcStreamOp.MutableStreamDescription(); + streamDescription.SetName(streamName); + streamDescription.SetMode(NKikimrSchemeOp::ECdcStreamModeUpdate); + streamDescription.SetFormat(NKikimrSchemeOp::ECdcStreamFormatProto); + NCdc::DoCreateStreamImpl(result, createCdcStreamOp, opId, sPath, false, false); - desc.MutableCreateSrcCdcStream()->CopyFrom(createCdcStreamOp); + + if (incrBackupEnabled && !omitIndexes) { + const auto tablePath = sPath; + + for (const auto& [childName, childPathId] : tablePath.Base()->GetChildren()) { + auto childPath = context.SS->PathsById.at(childPathId); + + if (childPath->PathType != NKikimrSchemeOp::EPathTypeTableIndex) { + continue; + } + + if (childPath->Dropped()) { + continue; + } + + auto indexInfo = context.SS->Indexes.at(childPathId); + if (indexInfo->Type != NKikimrSchemeOp::EIndexTypeGlobal) { + continue; + } + + auto indexPath = TPath::Init(childPathId, context.SS); + Y_ABORT_UNLESS(indexPath.Base()->GetChildren().size() == 1); + auto [implTableName, implTablePathId] = *indexPath.Base()->GetChildren().begin(); + + auto indexTablePath = indexPath.Child(implTableName); + + NKikimrSchemeOp::TCreateCdcStream indexCdcStreamOp; + indexCdcStreamOp.SetTableName(implTableName); + auto& indexStreamDescription = *indexCdcStreamOp.MutableStreamDescription(); + indexStreamDescription.SetName(streamName); + indexStreamDescription.SetMode(NKikimrSchemeOp::ECdcStreamModeUpdate); + indexStreamDescription.SetFormat(NKikimrSchemeOp::ECdcStreamFormatProto); + + NCdc::DoCreateStreamImpl(result, indexCdcStreamOp, opId, indexTablePath, false, false); + (*desc.MutableIndexImplTableCdcStreams())[childName].CopyFrom(indexCdcStreamOp); + } + } + + if (incrBackupEnabled && !omitIndexes) { + // Also invalidate cache for index impl tables + for (const auto& [childName, childPathId] : sPath.Base()->GetChildren()) { + auto childPath = context.SS->PathsById.at(childPathId); + if (childPath->PathType != NKikimrSchemeOp::EPathTypeTableIndex && !childPath->Dropped()) { + auto indexInfo = context.SS->Indexes.find(childPathId); + if (indexInfo != context.SS->Indexes.end() && + indexInfo->second->Type == NKikimrSchemeOp::EIndexTypeGlobal) { + + auto indexPath = TPath::Init(childPathId, context.SS); + for (const auto& [implTableName, implTablePathId] : indexPath.Base()->GetChildren()) { + auto implTablePath = context.SS->PathsById.at(implTablePathId); + if (implTablePath->IsTable()) { + context.SS->ClearDescribePathCaches(implTablePath); + context.OnComplete.PublishToSchemeBoard(opId, implTablePathId); + } + } + } + } + } + } } } @@ -155,6 +221,61 @@ TVector CreateBackupBackupCollection(TOperationId opId, con NCdc::DoCreatePqPart(result, createCdcStreamOp, opId, streamPath, streamName, table, boundaries, false); } + + if (incrBackupEnabled && !omitIndexes) { + for (const auto& item : bc->Description.GetExplicitEntryList().GetEntries()) { + const auto tablePath = TPath::Resolve(item.GetPath(), context.SS); + + // Iterate through table's children to find indexes + for (const auto& [childName, childPathId] : tablePath.Base()->GetChildren()) { + auto childPath = context.SS->PathsById.at(childPathId); + + // Skip non-index children (CDC streams, etc.) + if (childPath->PathType != NKikimrSchemeOp::EPathTypeTableIndex) { + continue; + } + + // Skip deleted indexes + if (childPath->Dropped()) { + continue; + } + + // Get index info and filter for global sync only + auto indexInfo = context.SS->Indexes.at(childPathId); + if (indexInfo->Type != NKikimrSchemeOp::EIndexTypeGlobal) { + continue; + } + + // Get index implementation table (the only child of index) + auto indexPath = TPath::Init(childPathId, context.SS); + Y_ABORT_UNLESS(indexPath.Base()->GetChildren().size() == 1); + auto [implTableName, implTablePathId] = *indexPath.Base()->GetChildren().begin(); + + auto indexTablePath = indexPath.Child(implTableName); + auto indexTable = context.SS->Tables.at(implTablePathId); + + NKikimrSchemeOp::TCreateCdcStream indexCdcStreamOp; + indexCdcStreamOp.SetTableName(implTableName); + auto& indexStreamDescription = *indexCdcStreamOp.MutableStreamDescription(); + indexStreamDescription.SetName(streamName); + indexStreamDescription.SetMode(NKikimrSchemeOp::ECdcStreamModeUpdate); + indexStreamDescription.SetFormat(NKikimrSchemeOp::ECdcStreamFormatProto); + + TVector indexBoundaries; + const auto& indexPartitions = indexTable->GetPartitions(); + indexBoundaries.reserve(indexPartitions.size() - 1); + for (ui32 i = 0; i < indexPartitions.size(); ++i) { + const auto& partition = indexPartitions.at(i); + if (i != indexPartitions.size() - 1) { + indexBoundaries.push_back(partition.EndOfRange); + } + } + + const auto indexStreamPath = indexTablePath.Child(streamName); + NCdc::DoCreatePqPart(result, indexCdcStreamOp, opId, indexStreamPath, streamName, indexTable, indexBoundaries, false); + } + } + } } return result; diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_backup_incremental_backup_collection.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_backup_incremental_backup_collection.cpp index e08dcf9c0e17..6b49d60002e9 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_backup_incremental_backup_collection.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_backup_incremental_backup_collection.cpp @@ -223,6 +223,79 @@ TVector CreateBackupIncrementalBackupCollection(TOperationI streams.push_back(stream); } + // Process indexes if they are not omitted + bool omitIndexes = bc->Description.GetIncrementalBackupConfig().GetOmitIndexes(); + if (!omitIndexes) { + for (const auto& item : bc->Description.GetExplicitEntryList().GetEntries()) { + const auto tablePath = TPath::Resolve(item.GetPath(), context.SS); + auto table = context.SS->Tables.at(tablePath.Base()->PathId); + + std::pair paths; + TString err; + if (!TrySplitPathByDb(item.GetPath(), bcPath.GetDomainPathString(), paths, err)) { + result = {CreateReject(opId, NKikimrScheme::StatusInvalidParameter, err)}; + return result; + } + auto& relativeItemPath = paths.second; + + // Iterate through table's children to find indexes + for (const auto& [childName, childPathId] : tablePath.Base()->GetChildren()) { + auto childPath = context.SS->PathsById.at(childPathId); + + // Skip non-index children (CDC streams, etc.) + if (childPath->PathType != NKikimrSchemeOp::EPathTypeTableIndex) { + continue; + } + + // Skip deleted indexes + if (childPath->Dropped()) { + continue; + } + + // Get index info and filter for global sync only + auto indexInfo = context.SS->Indexes.at(childPathId); + if (indexInfo->Type != NKikimrSchemeOp::EIndexTypeGlobal) { + continue; + } + + // Get index implementation table (single child of index) + auto indexPath = TPath::Init(childPathId, context.SS); + Y_ABORT_UNLESS(indexPath.Base()->GetChildren().size() == 1); + auto [implTableName, implTablePathId] = *indexPath.Base()->GetChildren().begin(); + + // Build relative path to index impl table (relative to working dir) + TString indexImplTableRelPath = JoinPath({relativeItemPath, childName, implTableName}); + + // Create AlterContinuousBackup for index impl table + NKikimrSchemeOp::TModifyScheme modifyScheme; + modifyScheme.SetWorkingDir(tx.GetWorkingDir()); + modifyScheme.SetOperationType(NKikimrSchemeOp::ESchemeOpAlterContinuousBackup); + modifyScheme.SetInternal(true); + + auto& cb = *modifyScheme.MutableAlterContinuousBackup(); + cb.SetTableName(indexImplTableRelPath); // Relative path: table1/index1/indexImplTable + + auto& ib = *cb.MutableTakeIncrementalBackup(); + // Destination: {backup_collection}/{timestamp}_inc/__ydb_backup_meta/indexes/{table_path}/{index_name} + TString dstPath = JoinPath({ + tx.GetBackupIncrementalBackupCollection().GetName(), + tx.GetBackupIncrementalBackupCollection().GetTargetDir(), + "__ydb_backup_meta", + "indexes", + relativeItemPath, // Relative table path (e.g., "table1") + childName // Index name (e.g., "index1") + }); + ib.SetDstPath(dstPath); + + TPathId stream; + if (!CreateAlterContinuousBackup(opId, modifyScheme, context, result, stream)) { + return result; + } + streams.push_back(stream); + } + } + } + CreateLongIncrementalBackupOp(opId, bcPath, result, streams); return result; diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_common_cdc_stream.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_common_cdc_stream.cpp index 2cb60ead5641..fd7a526e9aa7 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_common_cdc_stream.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_common_cdc_stream.cpp @@ -1,4 +1,5 @@ #include "schemeshard__operation_common.h" +#include "schemeshard_cdc_stream_common.h" #include "schemeshard_private.h" #include @@ -9,6 +10,8 @@ namespace NKikimr::NSchemeShard::NCdcStreamState { namespace { +constexpr const char* CONTINUOUS_BACKUP_SUFFIX = "_continuousBackupImpl"; + bool IsExpectedTxType(TTxState::ETxType txType) { switch (txType) { case TTxState::TxCreateCdcStreamAtTable: @@ -24,11 +27,359 @@ bool IsExpectedTxType(TTxState::ETxType txType) { } } +bool IsContinuousBackupStream(const TString& streamName) { + return streamName.EndsWith(CONTINUOUS_BACKUP_SUFFIX); +} + +struct TTableVersionContext { + TPathId PathId; + TPathId ParentPathId; + TPathId GrandParentPathId; + bool IsIndexImplTable = false; + bool IsContinuousBackupStream = false; + bool IsPartOfContinuousBackup = false; +}; + +bool DetectContinuousBackupStream(const TTxState& txState, TOperationContext& context) { + if (!txState.CdcPathId || !context.SS->PathsById.contains(txState.CdcPathId)) { + return false; + } + + auto cdcPath = context.SS->PathsById.at(txState.CdcPathId); + LOG_DEBUG_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, + "Checking CDC stream name" + << ", cdcPathId: " << txState.CdcPathId + << ", streamName: " << cdcPath->Name + << ", at schemeshard: " << context.SS->SelfTabletId()); + + return IsContinuousBackupStream(cdcPath->Name); +} + +bool DetectIndexImplTable(TPathElement::TPtr path, TOperationContext& context, TPathId& outGrandParentPathId) { + const TPathId& parentPathId = path->ParentPathId; + if (!parentPathId || !context.SS->PathsById.contains(parentPathId)) { + return false; + } + + auto parentPath = context.SS->PathsById.at(parentPathId); + if (parentPath->IsTableIndex()) { + outGrandParentPathId = parentPath->ParentPathId; + return true; + } + + return false; +} + +bool HasParentContinuousBackup(const TPathId& grandParentPathId, TOperationContext& context) { + if (!grandParentPathId || !context.SS->PathsById.contains(grandParentPathId)) { + return false; + } + + auto grandParentPath = context.SS->PathsById.at(grandParentPathId); + for (const auto& [childName, childPathId] : grandParentPath->GetChildren()) { + auto childPath = context.SS->PathsById.at(childPathId); + if (childPath->IsCdcStream() && IsContinuousBackupStream(childName)) { + LOG_DEBUG_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, + "Detected continuous backup via parent table CDC stream" + << ", parentTablePathId: " << grandParentPathId + << ", cdcStreamName: " << childName + << ", at schemeshard: " << context.SS->SelfTabletId()); + return true; + } + } + + return false; +} + +TTableVersionContext BuildTableVersionContext( + const TTxState& txState, + TPathElement::TPtr path, + TOperationContext& context) +{ + TTableVersionContext ctx; + ctx.PathId = txState.TargetPathId; + ctx.ParentPathId = path->ParentPathId; + ctx.IsContinuousBackupStream = DetectContinuousBackupStream(txState, context); + ctx.IsIndexImplTable = DetectIndexImplTable(path, context, ctx.GrandParentPathId); + + // Check if impl table is part of continuous backup + if (ctx.IsIndexImplTable) { + ctx.IsPartOfContinuousBackup = HasParentContinuousBackup(ctx.GrandParentPathId, context); + } else { + ctx.IsPartOfContinuousBackup = ctx.IsContinuousBackupStream; + } + + return ctx; +} + +// Synchronizes AlterVersion across index entities without modifying impl table versions. +// Uses lock-free-like helping coordination where each operation helps update sibling index entities. +void HelpSyncSiblingVersions( + const TPathId& myImplTablePathId, + const TPathId& myIndexPathId, + const TPathId& parentTablePathId, + ui64 myVersion, + TOperationId operationId, + TOperationContext& context, + NIceDb::TNiceDb& db) +{ + LOG_DEBUG_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, + "HelpSyncSiblingVersions ENTRY" + << ", myImplTablePathId: " << myImplTablePathId + << ", myIndexPathId: " << myIndexPathId + << ", parentTablePathId: " << parentTablePathId + << ", myVersion: " << myVersion + << ", operationId: " << operationId + << ", at schemeshard: " << context.SS->SelfTabletId()); + + TVector allIndexPathIds; + TVector allImplTablePathIds; + + if (!context.SS->PathsById.contains(parentTablePathId)) { + LOG_WARN_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, + "Parent table not found in PathsById" + << ", parentTablePathId: " << parentTablePathId + << ", at schemeshard: " << context.SS->SelfTabletId()); + return; + } + + auto parentTablePath = context.SS->PathsById.at(parentTablePathId); + + for (const auto& [childName, childPathId] : parentTablePath->GetChildren()) { + auto childPath = context.SS->PathsById.at(childPathId); + + if (!childPath->IsTableIndex() || childPath->Dropped()) { + continue; + } + + allIndexPathIds.push_back(childPathId); + + auto indexPath = context.SS->PathsById.at(childPathId); + Y_ABORT_UNLESS(indexPath->GetChildren().size() == 1); + auto [implTableName, implTablePathId] = *indexPath->GetChildren().begin(); + allImplTablePathIds.push_back(implTablePathId); + + LOG_DEBUG_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, + "Found index and impl table" + << ", indexPathId: " << childPathId + << ", implTablePathId: " << implTablePathId + << ", at schemeshard: " << context.SS->SelfTabletId()); + } + + LOG_DEBUG_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, + "Collected index family" + << ", indexCount: " << allIndexPathIds.size() + << ", implTableCount: " << allImplTablePathIds.size() + << ", at schemeshard: " << context.SS->SelfTabletId()); + + ui64 maxVersion = myVersion; + + for (const auto& indexPathId : allIndexPathIds) { + if (context.SS->Indexes.contains(indexPathId)) { + auto index = context.SS->Indexes.at(indexPathId); + maxVersion = Max(maxVersion, index->AlterVersion); + } + } + + for (const auto& implTablePathId : allImplTablePathIds) { + if (context.SS->Tables.contains(implTablePathId)) { + auto implTable = context.SS->Tables.at(implTablePathId); + maxVersion = Max(maxVersion, implTable->AlterVersion); + } + } + + LOG_DEBUG_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, + "Computed maximum version across all siblings" + << ", myVersion: " << myVersion + << ", maxVersion: " << maxVersion + << ", at schemeshard: " << context.SS->SelfTabletId()); + + // DO NOT update self to catch up - each impl table has already incremented its own version. + // Changing our version based on other operations would cause datashard version mismatches. + + if (context.SS->Indexes.contains(myIndexPathId)) { + auto myIndex = context.SS->Indexes.at(myIndexPathId); + if (myIndex->AlterVersion < maxVersion) { + myIndex->AlterVersion = maxVersion; + context.SS->PersistTableIndexAlterVersion(db, myIndexPathId, myIndex); + + auto myIndexPath = context.SS->PathsById.at(myIndexPathId); + context.SS->ClearDescribePathCaches(myIndexPath); + context.OnComplete.PublishToSchemeBoard(operationId, myIndexPathId); + + LOG_DEBUG_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, + "Updated my index entity" + << ", myIndexPathId: " << myIndexPathId + << ", newVersion: " << maxVersion + << ", at schemeshard: " << context.SS->SelfTabletId()); + } + } + + ui64 indexesUpdated = 0; + for (const auto& indexPathId : allIndexPathIds) { + if (indexPathId == myIndexPathId) { + continue; + } + + if (!context.SS->Indexes.contains(indexPathId)) { + continue; + } + + auto index = context.SS->Indexes.at(indexPathId); + if (index->AlterVersion < maxVersion) { + index->AlterVersion = maxVersion; + context.SS->PersistTableIndexAlterVersion(db, indexPathId, index); + + auto indexPath = context.SS->PathsById.at(indexPathId); + context.SS->ClearDescribePathCaches(indexPath); + context.OnComplete.PublishToSchemeBoard(operationId, indexPathId); + + indexesUpdated++; + + LOG_DEBUG_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, + "Updated sibling index entity" + << ", indexPathId: " << indexPathId + << ", newVersion: " << maxVersion + << ", at schemeshard: " << context.SS->SelfTabletId()); + } + } + + // CRITICAL: DO NOT help update sibling impl tables. + // Impl tables have datashards that expect schema change transactions. + // Bumping AlterVersion without TX_KIND_SCHEME_CHANGED causes "Wrong schema version" errors. + // Each impl table must increment its own version when its CDC operation executes. + + LOG_DEBUG_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, + "HelpSyncSiblingVersions COMPLETE" + << ", maxVersion: " << maxVersion + << ", indexesUpdated: " << indexesUpdated + << ", totalIndexes: " << allIndexPathIds.size() + << ", totalImplTables: " << allImplTablePathIds.size() + << ", NOTE: Sibling impl tables NOT updated (they update themselves)" + << ", at schemeshard: " << context.SS->SelfTabletId()); +} + } // namespace anonymous +// Public functions for version synchronization (used by copy-table and other operations) +void SyncIndexEntityVersion( + const TPathId& indexPathId, + ui64 targetVersion, + TOperationId operationId, + TOperationContext& context, + NIceDb::TNiceDb& db) +{ + LOG_DEBUG_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, + "SyncIndexEntityVersion ENTRY" + << ", indexPathId: " << indexPathId + << ", targetVersion: " << targetVersion + << ", operationId: " << operationId + << ", at schemeshard: " << context.SS->SelfTabletId()); + + if (!context.SS->Indexes.contains(indexPathId)) { + LOG_DEBUG_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, + "SyncIndexEntityVersion EXIT - index not found" + << ", indexPathId: " << indexPathId + << ", at schemeshard: " << context.SS->SelfTabletId()); + return; + } + + auto index = context.SS->Indexes.at(indexPathId); + ui64 oldIndexVersion = index->AlterVersion; + + LOG_DEBUG_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, + "SyncIndexEntityVersion current state" + << ", indexPathId: " << indexPathId + << ", currentIndexVersion: " << oldIndexVersion + << ", targetVersion: " << targetVersion + << ", at schemeshard: " << context.SS->SelfTabletId()); + + // Only update if we're increasing the version (prevent downgrade due to race conditions) + if (targetVersion > oldIndexVersion) { + index->AlterVersion = targetVersion; + + LOG_DEBUG_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, + "SyncIndexEntityVersion UPDATING index->AlterVersion" + << ", indexPathId: " << indexPathId + << ", oldVersion: " << oldIndexVersion + << ", newVersion: " << index->AlterVersion + << ", at schemeshard: " << context.SS->SelfTabletId()); + + context.SS->PersistTableIndexAlterVersion(db, indexPathId, index); + + auto indexPath = context.SS->PathsById.at(indexPathId); + context.SS->ClearDescribePathCaches(indexPath); + context.OnComplete.PublishToSchemeBoard(operationId, indexPathId); + + LOG_DEBUG_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, + "Synced index entity version" + << ", indexPathId: " << indexPathId + << ", oldVersion: " << oldIndexVersion + << ", newVersion: " << index->AlterVersion + << ", at schemeshard: " << context.SS->SelfTabletId()); + } else { + LOG_DEBUG_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, + "Skipping index entity sync - already at higher version" + << ", indexPathId: " << indexPathId + << ", currentVersion: " << oldIndexVersion + << ", targetVersion: " << targetVersion + << ", at schemeshard: " << context.SS->SelfTabletId()); + } +} + +void SyncChildIndexes( + TPathElement::TPtr parentPath, + ui64 targetVersion, + TOperationId operationId, + TOperationContext& context, + NIceDb::TNiceDb& db) +{ + LOG_DEBUG_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, + "SyncChildIndexes ENTRY" + << ", parentPath: " << parentPath->PathId + << ", targetVersion: " << targetVersion + << ", operationId: " << operationId + << ", at schemeshard: " << context.SS->SelfTabletId()); + + for (const auto& [childName, childPathId] : parentPath->GetChildren()) { + auto childPath = context.SS->PathsById.at(childPathId); + + // Skip non-index children and deleted indexes + if (!childPath->IsTableIndex() || childPath->Dropped()) { + continue; + } + + LOG_DEBUG_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, + "SyncChildIndexes processing index" + << ", indexPathId: " << childPathId + << ", indexName: " << childName + << ", targetVersion: " << targetVersion + << ", at schemeshard: " << context.SS->SelfTabletId()); + + NCdcStreamState::SyncIndexEntityVersion(childPathId, targetVersion, operationId, context, db); + + // NOTE: We intentionally do NOT sync the index impl table version here. + // Bumping AlterVersion without sending a TX_KIND_SCHEME transaction to datashards + // causes SCHEME_CHANGED errors because datashards still have the old version. + // The version should only be incremented when there's an actual schema change. + + LOG_DEBUG_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, + "Synced parent index version with parent table" + << ", parentTable: " << parentPath->Name + << ", indexName: " << childName + << ", indexPathId: " << childPathId + << ", newVersion: " << targetVersion + << ", at schemeshard: " << context.SS->SelfTabletId()); + } + + LOG_DEBUG_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, + "SyncChildIndexes EXIT" + << ", parentPath: " << parentPath->PathId + << ", targetVersion: " << targetVersion + << ", at schemeshard: " << context.SS->SelfTabletId()); +} + -// NCdcStreamState::TConfigurePartsAtTable -// TConfigurePartsAtTable::TConfigurePartsAtTable(TOperationId id) : OperationId(id) { @@ -80,8 +431,6 @@ bool TConfigurePartsAtTable::HandleReply(TEvDataShard::TEvProposeTransactionResu } -// NCdcStreamState::TProposeAtTable -// TProposeAtTable::TProposeAtTable(TOperationId id) : OperationId(id) { @@ -124,11 +473,33 @@ bool TProposeAtTable::HandleReply(TEvPrivate::TEvOperationPlan::TPtr& ev, TOpera Y_ABORT_UNLESS(context.SS->Tables.contains(pathId)); auto table = context.SS->Tables.at(pathId); - table->AlterVersion += 1; - NIceDb::TNiceDb db(context.GetDB()); - context.SS->PersistTableAlterVersion(db, pathId, table); + auto versionCtx = BuildTableVersionContext(*txState, path, context); + + bool isIndexImplTableCdc = versionCtx.IsPartOfContinuousBackup && versionCtx.IsIndexImplTable; + + if (isIndexImplTableCdc) { + table->AlterVersion += 1; + ui64 myIncrementedVersion = table->AlterVersion; + + HelpSyncSiblingVersions( + pathId, + versionCtx.ParentPathId, + versionCtx.GrandParentPathId, + myIncrementedVersion, + OperationId, + context, + db); + } else { + table->AlterVersion += 1; + } + + if (versionCtx.IsContinuousBackupStream && !versionCtx.IsIndexImplTable) { + NCdcStreamState::SyncChildIndexes(path, table->AlterVersion, OperationId, context, db); + } + + context.SS->PersistTableAlterVersion(db, pathId, table); context.SS->ClearDescribePathCaches(path); context.OnComplete.PublishToSchemeBoard(OperationId, pathId); @@ -147,8 +518,6 @@ bool TProposeAtTable::HandleReply(TEvDataShard::TEvSchemaChanged::TPtr& ev, TOpe } -// NCdcStreamState::TProposeAtTableDropSnapshot -// bool TProposeAtTableDropSnapshot::HandleReply(TEvPrivate::TEvOperationPlan::TPtr& ev, TOperationContext& context) { TProposeAtTable::HandleReply(ev, context); diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_consistent_copy_tables.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_consistent_copy_tables.cpp index 264b055ded6f..9bc63698b4ee 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_consistent_copy_tables.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_consistent_copy_tables.cpp @@ -8,6 +8,18 @@ #include +static bool ShouldOmitAutomaticIndexProcessing(const NKikimrSchemeOp::TCopyTableConfig& descr) { + if (descr.GetOmitIndexes()) { + return true; // User explicitly wants to skip indexes + } + + if (!descr.GetIndexImplTableCdcStreams().empty()) { + return true; // Incremental backup - manual handling required + } + + return false; // Regular copy - let CreateCopyTable handle indexes automatically +} + static NKikimrSchemeOp::TModifyScheme CopyTableTask(NKikimr::NSchemeShard::TPath& src, NKikimr::NSchemeShard::TPath& dst, const NKikimrSchemeOp::TCopyTableConfig& descr) { using namespace NKikimr::NSchemeShard; @@ -20,6 +32,7 @@ static NKikimrSchemeOp::TModifyScheme CopyTableTask(NKikimr::NSchemeShard::TPath operation->SetOmitFollowers(descr.GetOmitFollowers()); operation->SetIsBackup(descr.GetIsBackup()); operation->SetAllowUnderSameOperation(descr.GetAllowUnderSameOperation()); + operation->SetOmitIndexes(ShouldOmitAutomaticIndexProcessing(descr)); if (descr.HasCreateSrcCdcStream()) { auto* coOp = scheme.MutableCreateCdcStream(); coOp->CopyFrom(descr.GetCreateSrcCdcStream()); @@ -153,6 +166,49 @@ bool CreateConsistentCopyTables( sequences)); } + // Log information about the table being copied + LOG_TRACE_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, + "CreateConsistentCopyTables: Processing table" + << ", srcPath: " << srcPath.PathString() + << ", dstPath: " << dstPath.PathString() + << ", pathId: " << srcPath.Base()->PathId + << ", childrenCount: " << srcPath.Base()->GetChildren().size() + << ", omitIndexes: " << descr.GetOmitIndexes()); + + // Log table info if available + if (context.SS->Tables.contains(srcPath.Base()->PathId)) { + TTableInfo::TPtr tableInfo = context.SS->Tables.at(srcPath.Base()->PathId); + const auto& tableDesc = tableInfo->TableDescription; + LOG_TRACE_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, + "CreateConsistentCopyTables: Table info" + << ", tableIndexesSize: " << tableDesc.TableIndexesSize() + << ", isBackup: " << tableInfo->IsBackup); + + for (size_t i = 0; i < static_cast(tableDesc.TableIndexesSize()); ++i) { + const auto& indexDesc = tableDesc.GetTableIndexes(i); + LOG_TRACE_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, + "CreateConsistentCopyTables: Table has index in description" + << ", indexName: " << indexDesc.GetName() + << ", indexType: " << NKikimrSchemeOp::EIndexType_Name(indexDesc.GetType())); + } + } + + // Log all children + for (const auto& child: srcPath.Base()->GetChildren()) { + const auto& name = child.first; + const auto& pathId = child.second; + TPath childPath = srcPath.Child(name); + + LOG_TRACE_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, + "CreateConsistentCopyTables: Child found" + << ", name: " << name + << ", pathId: " << pathId + << ", isResolved: " << childPath.IsResolved() + << ", isDeleted: " << childPath.IsDeleted() + << ", isSequence: " << childPath.IsSequence() + << ", isTableIndex: " << childPath.IsTableIndex()); + } + for (const auto& child: srcPath.Base()->GetChildren()) { const auto& name = child.first; const auto& pathId = child.second; @@ -161,21 +217,32 @@ bool CreateConsistentCopyTables( TPath dstIndexPath = dstPath.Child(name); if (srcIndexPath.IsDeleted()) { + LOG_TRACE_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, + "CreateConsistentCopyTables: Skipping deleted child: " << name); continue; } if (srcIndexPath.IsSequence()) { + LOG_TRACE_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, + "CreateConsistentCopyTables: Skipping sequence child: " << name); continue; } if (descr.GetOmitIndexes()) { + LOG_TRACE_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, + "CreateConsistentCopyTables: Skipping due to OmitIndexes: " << name); continue; } if (!srcIndexPath.IsTableIndex()) { + LOG_TRACE_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, + "CreateConsistentCopyTables: Skipping non-index child: " << name); continue; } + LOG_TRACE_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, + "CreateConsistentCopyTables: Creating index copy operation for: " << name); + Y_ABORT_UNLESS(srcIndexPath.Base()->PathId == pathId); TTableIndexInfo::TPtr indexInfo = context.SS->Indexes.at(pathId); auto scheme = CreateIndexTask(indexInfo, dstIndexPath); @@ -191,8 +258,27 @@ bool CreateConsistentCopyTables( Y_ABORT_UNLESS(srcImplTable.Base()->PathId == srcImplTablePathId); TPath dstImplTable = dstIndexPath.Child(srcImplTableName); + LOG_TRACE_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, + "CreateConsistentCopyTables: Creating index impl table copy" + << ", srcImplTable: " << srcImplTable.PathString() + << ", dstImplTable: " << dstImplTable.PathString()); + + // Check if we have CDC stream info for this index impl table in the descriptor + NKikimrSchemeOp::TCopyTableConfig indexDescr; + indexDescr.CopyFrom(descr); + + auto it = descr.GetIndexImplTableCdcStreams().find(name); + if (it != descr.GetIndexImplTableCdcStreams().end()) { + // CDC stream Impl was already created in the backup operation before copying + // Store the CDC info so the copy operation creates AtTable and PQ parts + indexDescr.MutableCreateSrcCdcStream()->CopyFrom(it->second); + } else { + // No CDC stream for this index impl table, clear it + indexDescr.ClearCreateSrcCdcStream(); + } + result.push_back(CreateCopyTable(NextPartId(nextId, result), - CopyTableTask(srcImplTable, dstImplTable, descr), GetLocalSequences(context, srcImplTable))); + CopyTableTask(srcImplTable, dstImplTable, indexDescr), GetLocalSequences(context, srcImplTable))); AddCopySequences(nextId, tx, context, result, srcImplTable, dstImplTable.PathString()); } } diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_copy_table.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_copy_table.cpp index 82683cc75f5f..fb45b19bec23 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_copy_table.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_copy_table.cpp @@ -224,7 +224,10 @@ class TPropose: public TSubOperationState { srcTable->AlterVersion += 1; - context.SS->PersistTableAlterVersion(db, srcPathId, table); + context.SS->PersistTableAlterVersion(db, srcPathId, srcTable); + + // Sync child indexes to match the new version + NCdcStreamState::SyncChildIndexes(srcPath, srcTable->AlterVersion, OperationId, context, db); context.SS->ClearDescribePathCaches(srcPath); context.OnComplete.PublishToSchemeBoard(OperationId, srcPathId); @@ -393,9 +396,13 @@ class TCopyTable: public TSubOperation { .IsResolved() .NotDeleted() .NotUnderDeleting() - .IsTable() - .NotUnderTheSameOperation(OperationId.GetTxId()) - .NotUnderOperation(); + .IsTable(); + + if (!Transaction.GetCreateTable().GetAllowUnderSameOperation()) { + checks + .NotUnderTheSameOperation(OperationId.GetTxId()) + .NotUnderOperation(); + } if (checks) { if (parent.Base()->IsTableIndex()) { @@ -798,6 +805,8 @@ TVector CreateCopyTable(TOperationId nextId, const TTxTrans result.push_back(CreateCopyTable(NextPartId(nextId, result), schema, sequences)); } + // Process indexes: always create index structure, but skip impl table copies if OmitIndexes is set + // (impl tables are handled separately by CreateConsistentCopyTables for incremental backups with CDC) for (auto& child: srcPath.Base()->GetChildren()) { auto name = child.first; auto pathId = child.second; @@ -842,6 +851,11 @@ TVector CreateCopyTable(TOperationId nextId, const TTxTrans result.push_back(CreateNewTableIndex(NextPartId(nextId, result), schema)); } + // Skip impl table copies if OmitIndexes is set (handled by CreateConsistentCopyTables for incremental backups) + if (copying.GetOmitIndexes()) { + continue; + } + for (const auto& [implTableName, implTablePathId] : childPath.Base()->GetChildren()) { TPath implTable = childPath.Child(implTableName); Y_ABORT_UNLESS(implTable.Base()->PathId == implTablePathId); diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_create_cdc_stream.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_create_cdc_stream.cpp index 211bb3654e0d..2ade3854dede 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_create_cdc_stream.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_create_cdc_stream.cpp @@ -322,6 +322,13 @@ class TNewCdcStream: public TSubOperation { Y_ABORT_UNLESS(!context.SS->FindTx(OperationId)); auto& txState = context.SS->CreateTx(OperationId, TTxState::TxCreateCdcStream, streamPath.Base()->PathId); + txState.CdcPathId = streamPath.Base()->PathId; + LOG_DEBUG_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, + "DoNewStream: Set CdcPathId" + << ", operationId: " << OperationId + << ", cdcPathId: " << streamPath.Base()->PathId + << ", streamName: " << streamPath.Base()->Name + << ", at schemeshard: " << context.SS->SelfTabletId()); txState.State = TTxState::Propose; streamPath.Base()->PathState = NKikimrSchemeOp::EPathStateCreate; @@ -582,6 +589,18 @@ class TNewCdcStreamAtTable: public TSubOperation { Y_ABORT_UNLESS(!context.SS->FindTx(OperationId)); auto& txState = context.SS->CreateTx(OperationId, txType, tablePath.Base()->PathId); txState.State = TTxState::ConfigureParts; + + // Set CdcPathId for continuous backup detection + auto streamPath = tablePath.Child(streamName); + if (streamPath.IsResolved()) { + txState.CdcPathId = streamPath.Base()->PathId; + LOG_DEBUG_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, + "TNewCdcStreamAtTable: Set CdcPathId" + << ", operationId: " << OperationId + << ", cdcPathId: " << streamPath.Base()->PathId + << ", streamName: " << streamName + << ", at schemeshard: " << context.SS->SelfTabletId()); + } tablePath.Base()->PathState = NKikimrSchemeOp::EPathStateAlter; tablePath.Base()->LastTxId = OperationId.GetTxId(); @@ -984,7 +1003,7 @@ TVector CreateNewCdcStream(TOperationId opId, const TTxTran DoCreateLock(result, opId, workingDirPath, tablePath); } - if (workingDirPath.IsTableIndex()) { + if (workingDirPath.IsTableIndex() && !streamName.EndsWith("_continuousBackupImpl")) { auto outTx = TransactionTemplate(workingDirPath.Parent().PathString(), NKikimrSchemeOp::EOperationType::ESchemeOpAlterTableIndex); outTx.MutableAlterTableIndex()->SetName(workingDirPath.LeafName()); outTx.MutableAlterTableIndex()->SetState(NKikimrSchemeOp::EIndexState::EIndexStateReady); diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_create_restore_incremental_backup.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_create_restore_incremental_backup.cpp index 521cc4868509..bc3282a08e66 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_create_restore_incremental_backup.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_create_restore_incremental_backup.cpp @@ -417,8 +417,12 @@ class TNewRestoreFromAtTable : public TSubOperationWithContext { .NotDeleted() .IsTable() .NotAsyncReplicaTable() - .NotUnderDeleting() - .IsCommonSensePath(); + .NotUnderDeleting(); + + // Allow restoring to private paths (e.g., index implementation tables) + if (!dstTablePath.IsInsideTableIndexPath(false)) { + checks.IsCommonSensePath(); + } if (!checks) { result->SetError(checks.GetStatus(), checks.GetError()); @@ -558,8 +562,12 @@ bool CreateRestoreMultipleIncrementalBackups( .IsResolved() .NotDeleted() .IsTable() - .NotUnderDeleting() - .IsCommonSensePath(); + .NotUnderDeleting(); + + // Allow restoring to private paths (e.g., index implementation tables) + if (!dstTablePath.IsInsideTableIndexPath(false)) { + checks.IsCommonSensePath(); + } } else { checks .FailOnExist(TPathElement::EPathType::EPathTypeTable, false); diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_drop_cdc_stream.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_drop_cdc_stream.cpp index 593de4172b0a..11b82835a7ba 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_drop_cdc_stream.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_drop_cdc_stream.cpp @@ -191,6 +191,7 @@ class TDropCdcStream: public TSubOperation { Y_ABORT_UNLESS(!context.SS->FindTx(OperationId)); auto& txState = context.SS->CreateTx(OperationId, TTxState::TxDropCdcStream, streamPath.Base()->PathId); + txState.CdcPathId = streamPath.Base()->PathId; txState.State = TTxState::Propose; txState.MinStep = TStepId(1); @@ -583,7 +584,15 @@ void DoDropStream( result.push_back(DropLock(NextPartId(opId, result), outTx)); } - if (workingDirPath.IsTableIndex()) { + bool hasContinuousBackupStream = false; + for (const auto& streamPath : streamPaths) { + if (streamPath.Base()->Name.EndsWith("_continuousBackupImpl")) { + hasContinuousBackupStream = true; + break; + } + } + + if (workingDirPath.IsTableIndex() && !hasContinuousBackupStream) { auto outTx = TransactionTemplate(workingDirPath.Parent().PathString(), NKikimrSchemeOp::EOperationType::ESchemeOpAlterTableIndex); outTx.MutableAlterTableIndex()->SetName(workingDirPath.LeafName()); outTx.MutableAlterTableIndex()->SetState(NKikimrSchemeOp::EIndexState::EIndexStateReady); diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_incremental_restore_finalize.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_incremental_restore_finalize.cpp index 36ff97d1c9e4..c5107d17d5d7 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_incremental_restore_finalize.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_incremental_restore_finalize.cpp @@ -3,6 +3,8 @@ #include "schemeshard__operation_base.h" #include "schemeshard__operation_common.h" +#include + #define LOG_I(stream) LOG_INFO_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "[" << context.SS->TabletID() << "] " << stream) #define LOG_N(stream) LOG_NOTICE_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "[" << context.SS->TabletID() << "] " << stream) #define LOG_W(stream) LOG_WARN_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "[" << context.SS->TabletID() << "] " << stream) @@ -13,6 +15,8 @@ class TIncrementalRestoreFinalizeOp: public TSubOperationWithContext { TTxState::ETxState NextState(TTxState::ETxState state) const override { switch(state) { case TTxState::Waiting: + return TTxState::ConfigureParts; + case TTxState::ConfigureParts: return TTxState::Propose; case TTxState::Propose: return TTxState::Done; @@ -24,6 +28,8 @@ class TIncrementalRestoreFinalizeOp: public TSubOperationWithContext { TSubOperationState::TPtr SelectStateFunc(TTxState::ETxState state) override { switch(state) { case TTxState::Waiting: + case TTxState::ConfigureParts: + return MakeHolder(OperationId, Transaction); case TTxState::Propose: return MakeHolder(OperationId, Transaction); case TTxState::Done: @@ -33,6 +39,172 @@ class TIncrementalRestoreFinalizeOp: public TSubOperationWithContext { } } + class TConfigureParts: public TSubOperationState { + private: + TOperationId OperationId; + TTxTransaction Transaction; + + TString DebugHint() const override { + return TStringBuilder() + << "TIncrementalRestoreFinalize TConfigureParts" + << " operationId: " << OperationId; + } + + public: + TConfigureParts(TOperationId id, const TTxTransaction& tx) + : OperationId(id), Transaction(tx) + { + IgnoreMessages(DebugHint(), {TEvHive::TEvCreateTabletReply::EventType}); + } + + bool HandleReply(TEvDataShard::TEvProposeTransactionResult::TPtr& ev, TOperationContext& context) override { + TTabletId ssId = context.SS->SelfTabletId(); + + LOG_INFO_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, + DebugHint() << " HandleReply TEvProposeTransactionResult" + << ", at schemeshard: " << ssId + << ", message: " << ev->Get()->Record.ShortDebugString()); + + return NTableState::CollectProposeTransactionResults(OperationId, ev, context); + } + + bool ProgressState(TOperationContext& context) override { + TTabletId ssId = context.SS->SelfTabletId(); + + LOG_INFO_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, + DebugHint() << " ProgressState" + << ", at schemeshard: " << ssId); + + TTxState* txState = context.SS->FindTx(OperationId); + Y_ABORT_UNLESS(txState); + + const auto& finalize = Transaction.GetIncrementalRestoreFinalize(); + + // Collect all index impl tables that need schema version updates + THashSet implTablesToUpdate; + CollectIndexImplTables(finalize, context, implTablesToUpdate); + + if (implTablesToUpdate.empty()) { + LOG_I(DebugHint() << " No index impl tables to update, skipping ConfigureParts"); + return true; + } + + // Prepare AlterData for each table and add shards to txState + NIceDb::TNiceDb db(context.GetDB()); + txState->ClearShardsInProgress(); + + for (const auto& tablePathId : implTablesToUpdate) { + if (!context.SS->Tables.contains(tablePathId)) { + LOG_W(DebugHint() << " Table not found: " << tablePathId); + continue; + } + + auto table = context.SS->Tables.at(tablePathId); + + // Create AlterData if it doesn't exist + if (!table->AlterData) { + // Create minimal AlterData just to bump schema version + auto alterData = MakeIntrusive(); + alterData->AlterVersion = table->AlterVersion + 1; + alterData->NextColumnId = table->NextColumnId; + alterData->Columns = table->Columns; + alterData->KeyColumnIds = table->KeyColumnIds; + alterData->IsBackup = table->IsBackup; + alterData->IsRestore = table->IsRestore; + alterData->TableDescriptionFull = table->TableDescription; + + table->PrepareAlter(alterData); + } else { + // Increment AlterVersion if AlterData already exists + table->AlterData->AlterVersion = table->AlterVersion + 1; + } + + LOG_I(DebugHint() << " Preparing ALTER for table " << tablePathId + << " version: " << table->AlterVersion << " -> " << table->AlterData->AlterVersion); + + // Add all shards of this table to txState + for (const auto& shard : table->GetPartitions()) { + auto shardIdx = shard.ShardIdx; + if (!txState->ShardsInProgress.contains(shardIdx)) { + txState->Shards.emplace_back(shardIdx, ETabletType::DataShard, TTxState::ConfigureParts); + txState->ShardsInProgress.insert(shardIdx); + + LOG_I(DebugHint() << " Added shard " << shardIdx + << " (tablet: " << context.SS->ShardInfos[shardIdx].TabletID << ") to txState"); + } + } + } + + context.SS->PersistTxState(db, OperationId); + + // Send ALTER TABLE transactions to all datashards + for (const auto& shard : txState->Shards) { + auto shardIdx = shard.Idx; + auto datashardId = context.SS->ShardInfos[shardIdx].TabletID; + + LOG_I(DebugHint() << " Propose ALTER to datashard " << datashardId + << " shardIdx: " << shardIdx << " txid: " << OperationId); + + const auto seqNo = context.SS->StartRound(*txState); + + // Find which table this shard belongs to + TPathId tablePathId; + for (const auto& pathId : implTablesToUpdate) { + auto table = context.SS->Tables.at(pathId); + for (const auto& partition : table->GetPartitions()) { + if (partition.ShardIdx == shardIdx) { + tablePathId = pathId; + break; + } + } + if (tablePathId) break; + } + + if (!tablePathId) { + LOG_W(DebugHint() << " Could not find table for shard " << shardIdx); + continue; + } + + const auto txBody = context.SS->FillAlterTableTxBody(tablePathId, shardIdx, seqNo); + auto event = context.SS->MakeDataShardProposal(tablePathId, OperationId, txBody, context.Ctx); + context.OnComplete.BindMsgToPipe(OperationId, datashardId, shardIdx, event.Release()); + } + + txState->UpdateShardsInProgress(); + return false; + } + + private: + void CollectIndexImplTables(const NKikimrSchemeOp::TIncrementalRestoreFinalize& finalize, + TOperationContext& context, + THashSet& implTables) { + for (const auto& tablePath : finalize.GetTargetTablePaths()) { + // Check if this path looks like an index implementation table + TString indexImplTableSuffix = TString("/") + NTableIndex::ImplTable; + if (!tablePath.Contains(indexImplTableSuffix)) { + continue; + } + + TPath path = TPath::Resolve(tablePath, context.SS); + if (!path.IsResolved()) { + LOG_W("CollectIndexImplTables: Table not resolved: " << tablePath); + continue; + } + + if (path.Base()->PathType != NKikimrSchemeOp::EPathType::EPathTypeTable) { + continue; + } + + TPathId implTablePathId = path.Base()->PathId; + if (context.SS->Tables.contains(implTablePathId)) { + implTables.insert(implTablePathId); + LOG_I("CollectIndexImplTables: Found index impl table: " << tablePath + << " pathId: " << implTablePathId); + } + } + } + }; + class TFinalizationPropose: public TSubOperationState { private: TOperationId OperationId; @@ -60,6 +232,9 @@ class TIncrementalRestoreFinalizeOp: public TSubOperationWithContext { const auto& finalize = Transaction.GetIncrementalRestoreFinalize(); + // Sync schema versions for restored indexes before releasing path states + SyncIndexSchemaVersions(finalize, context); + // Release all affected path states to EPathStateNoChanges TVector pathsToNormalize; CollectPathsToNormalize(finalize, context, pathsToNormalize); @@ -96,6 +271,76 @@ class TIncrementalRestoreFinalizeOp: public TSubOperationWithContext { } private: + void SyncIndexSchemaVersions(const NKikimrSchemeOp::TIncrementalRestoreFinalize& finalize, + TOperationContext& context) { + LOG_I("SyncIndexSchemaVersions: Starting schema version sync for restored indexes"); + LOG_I("SyncIndexSchemaVersions: Processing " << finalize.GetTargetTablePaths().size() << " target table paths"); + + NIceDb::TNiceDb db(context.GetDB()); + + // Iterate through all target table paths and finalize their alters + for (const auto& tablePath : finalize.GetTargetTablePaths()) { + // Check if this path looks like an index implementation table + TString indexImplTableSuffix = TString("/") + NTableIndex::ImplTable; + if (!tablePath.Contains(indexImplTableSuffix)) { + continue; + } + + TPath path = TPath::Resolve(tablePath, context.SS); + if (!path.IsResolved()) { + LOG_W("SyncIndexSchemaVersions: Table not resolved: " << tablePath); + continue; + } + + if (path.Base()->PathType != NKikimrSchemeOp::EPathType::EPathTypeTable) { + continue; + } + + TPathId implTablePathId = path.Base()->PathId; + if (!context.SS->Tables.contains(implTablePathId)) { + LOG_W("SyncIndexSchemaVersions: Table not found: " << implTablePathId); + continue; + } + + auto table = context.SS->Tables.at(implTablePathId); + if (!table->AlterData) { + LOG_W("SyncIndexSchemaVersions: No AlterData for table: " << implTablePathId); + continue; + } + + // Finalize the alter - this commits AlterData to the main table state + LOG_I("SyncIndexSchemaVersions: Finalizing ALTER for table " << implTablePathId + << " version: " << table->AlterVersion << " -> " << table->AlterData->AlterVersion); + + table->FinishAlter(); + context.SS->PersistTableAltered(db, implTablePathId, table); + + // Clear describe path caches and publish to scheme board + context.SS->ClearDescribePathCaches(path.Base()); + context.OnComplete.PublishToSchemeBoard(OperationId, implTablePathId); + + LOG_I("SyncIndexSchemaVersions: Finalized schema version for: " << tablePath); + + // Also update the parent index version + TPath indexPath = path.Parent(); + if (indexPath.IsResolved() && indexPath.Base()->PathType == NKikimrSchemeOp::EPathTypeTableIndex) { + TPathId indexPathId = indexPath.Base()->PathId; + if (context.SS->Indexes.contains(indexPathId)) { + auto oldVersion = context.SS->Indexes[indexPathId]->AlterVersion; + context.SS->Indexes[indexPathId]->AlterVersion += 1; + context.SS->PersistTableIndexAlterVersion(db, indexPathId, context.SS->Indexes[indexPathId]); + + LOG_I("SyncIndexSchemaVersions: Index AlterVersion incremented from " + << oldVersion << " to " << context.SS->Indexes[indexPathId]->AlterVersion); + + context.OnComplete.PublishToSchemeBoard(OperationId, indexPathId); + } + } + } + + LOG_I("SyncIndexSchemaVersions: Finished schema version sync"); + } + void CollectPathsToNormalize(const NKikimrSchemeOp::TIncrementalRestoreFinalize& finalize, TOperationContext& context, TVector& pathsToNormalize) { diff --git a/ydb/core/tx/schemeshard/schemeshard_cdc_stream_common.h b/ydb/core/tx/schemeshard/schemeshard_cdc_stream_common.h index 8158a02fe4e7..64cfebd4ec1e 100644 --- a/ydb/core/tx/schemeshard/schemeshard_cdc_stream_common.h +++ b/ydb/core/tx/schemeshard/schemeshard_cdc_stream_common.h @@ -10,6 +10,7 @@ struct TPathId; namespace NSchemeShard { struct TOperationContext; +struct TTxState; } // namespace NSchemeShard @@ -31,4 +32,23 @@ void CheckSrcDirOnPropose( bool isInsideTableIndexPath, TTxId op = InvalidTxId); -} // namespace NKikimr::NSchemeShard::NCdc +} // namespace NKikimr::NSchemeShard::NCdcStreamAtTable + +namespace NKikimr::NSchemeShard::NCdcStreamState { + +// Synchronize child index versions when parent table version is updated for continuous backup +void SyncIndexEntityVersion( + const TPathId& indexPathId, + ui64 targetVersion, + TOperationId operationId, + TOperationContext& context, + NIceDb::TNiceDb& db); + +void SyncChildIndexes( + TPathElement::TPtr parentPath, + ui64 targetVersion, + TOperationId operationId, + TOperationContext& context, + NIceDb::TNiceDb& db); + +} // namespace NKikimr::NSchemeShard::NCdcStreamState diff --git a/ydb/core/tx/schemeshard/schemeshard_impl.cpp b/ydb/core/tx/schemeshard/schemeshard_impl.cpp index 3095d0f2cafc..26030bc5a1ae 100644 --- a/ydb/core/tx/schemeshard/schemeshard_impl.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_impl.cpp @@ -1959,6 +1959,12 @@ void TSchemeShard::PersistTableIndexAlterData(NIceDb::TNiceDb& db, const TPathId } } +void TSchemeShard::PersistTableIndexAlterVersion(NIceDb::TNiceDb& db, const TPathId& pathId, const TTableIndexInfo::TPtr indexInfo) { + db.Table().Key(pathId.LocalPathId).Update( + NIceDb::TUpdate(indexInfo->AlterVersion) + ); +} + void TSchemeShard::PersistCdcStream(NIceDb::TNiceDb& db, const TPathId& pathId) { Y_ABORT_UNLESS(PathsById.contains(pathId)); auto path = PathsById.at(pathId); @@ -2643,6 +2649,16 @@ void TSchemeShard::PersistTxState(NIceDb::TNiceDb& db, const TOperationId opId) txState.CdcPathId.ToProto(proto.MutableTxCopyTableExtraData()->MutableCdcPathId()); bool serializeRes = proto.SerializeToString(&extraData); Y_ABORT_UNLESS(serializeRes); + } else if (txState.TxType == TTxState::TxCreateCdcStreamAtTable || + txState.TxType == TTxState::TxCreateCdcStreamAtTableWithInitialScan || + txState.TxType == TTxState::TxAlterCdcStreamAtTable || + txState.TxType == TTxState::TxAlterCdcStreamAtTableDropSnapshot || + txState.TxType == TTxState::TxDropCdcStreamAtTable || + txState.TxType == TTxState::TxDropCdcStreamAtTableDropSnapshot) { + NKikimrSchemeOp::TGenericTxInFlyExtraData proto; + txState.CdcPathId.ToProto(proto.MutableTxCopyTableExtraData()->MutableCdcPathId()); + bool serializeRes = proto.SerializeToString(&extraData); + Y_ABORT_UNLESS(serializeRes); } db.Table().Key(opId.GetTxId(), opId.GetSubTxId()).Update( diff --git a/ydb/core/tx/schemeshard/schemeshard_impl.h b/ydb/core/tx/schemeshard/schemeshard_impl.h index 2e4a0f33d378..eefab470fb6c 100644 --- a/ydb/core/tx/schemeshard/schemeshard_impl.h +++ b/ydb/core/tx/schemeshard/schemeshard_impl.h @@ -724,6 +724,7 @@ class TSchemeShard // table index void PersistTableIndex(NIceDb::TNiceDb& db, const TPathId& pathId); void PersistTableIndexAlterData(NIceDb::TNiceDb& db, const TPathId& pathId); + void PersistTableIndexAlterVersion(NIceDb::TNiceDb& db, const TPathId& pathId, const TTableIndexInfo::TPtr indexInfo); // cdc stream void PersistCdcStream(NIceDb::TNiceDb& db, const TPathId& pathId); @@ -1192,6 +1193,36 @@ class TSchemeShard void Handle(TEvDataShard::TEvIncrementalRestoreResponse::TPtr& ev, const TActorContext& ctx); void CreateIncrementalRestoreOperation(const TPathId& backupCollectionPathId, ui64 operationId, const TString& backupName, const TActorContext& ctx); + void DiscoverAndCreateIndexRestoreOperations( + const TPathId& backupCollectionPathId, + ui64 operationId, + const TString& backupName, + const TPath& bcPath, + const TBackupCollectionInfo::TPtr& backupCollectionInfo, + const TActorContext& ctx); + + void DiscoverIndexesRecursive( + ui64 operationId, + const TString& backupName, + const TPath& bcPath, + const TBackupCollectionInfo::TPtr& backupCollectionInfo, + const TPath& currentPath, + const TString& accumulatedRelativePath, + const TActorContext& ctx); + + void CreateSingleIndexRestoreOperation( + ui64 operationId, + const TString& backupName, + const TPath& bcPath, + const TString& relativeTablePath, + const TString& indexName, + const TString& targetTablePath, + const TActorContext& ctx); + + TString FindTargetTablePath( + const TBackupCollectionInfo::TPtr& backupCollectionInfo, + const TString& relativeTablePath); + void Handle(TEvDataShard::TEvProposeTransactionAttachResult::TPtr& ev, const TActorContext& ctx); void Handle(TEvTabletPipe::TEvClientConnected::TPtr &ev, const TActorContext &ctx); diff --git a/ydb/core/tx/schemeshard/schemeshard_incremental_restore_scan.cpp b/ydb/core/tx/schemeshard/schemeshard_incremental_restore_scan.cpp index e3fc2fbe4e08..b9ea78d25772 100644 --- a/ydb/core/tx/schemeshard/schemeshard_incremental_restore_scan.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_incremental_restore_scan.cpp @@ -1,6 +1,7 @@ #include "schemeshard_impl.h" #include "schemeshard_utils.h" +#include #include #include @@ -244,6 +245,22 @@ class TSchemeShard::TTxProgressIncrementalRestore : public NTabletFlatExecutor:: for (const auto& tablePath : op.GetTablePathList()) { finalize.AddTargetTablePaths(tablePath); } + + // Also collect index implementation tables that are in incoming restore state + // These are restored separately but need to be finalized together with main tables + for (auto& [pathId, pathInfo] : Self->PathsById) { + if (pathInfo->PathState == NKikimrSchemeOp::EPathState::EPathStateIncomingIncrementalRestore) { + TString pathString = TPath::Init(pathId, Self).PathString(); + // Check if this is an index implementation table under one of our restored tables + for (const auto& tablePath : op.GetTablePathList()) { + TString indexImplTableSuffix = TString("/") + NTableIndex::ImplTable; + if (pathString.StartsWith(tablePath + "/") && pathString.Contains(indexImplTableSuffix)) { + finalize.AddTargetTablePaths(pathString); + break; + } + } + } + } } else { // For simple operations, collect paths directly from affected paths for (auto& [pathId, pathInfo] : Self->PathsById) { @@ -594,10 +611,271 @@ void TSchemeShard::CreateIncrementalRestoreOperation( LOG_W("Incremental backup path not found: " << incrBackupPathStr); } } - + + // Discover and create index restore operations in parallel + DiscoverAndCreateIndexRestoreOperations( + backupCollectionPathId, + operationId, + backupName, + bcPath, + backupCollectionInfo, + ctx + ); + LOG_I("Created separate restore operations for incremental backup: " << backupName); } +TString TSchemeShard::FindTargetTablePath( + const TBackupCollectionInfo::TPtr& backupCollectionInfo, + const TString& relativeTablePath) { + + // Map backup relative path to restore target path using backup collection's ExplicitEntryList + for (const auto& item : backupCollectionInfo->Description.GetExplicitEntryList().GetEntries()) { + if (item.GetType() != NKikimrSchemeOp::TBackupCollectionDescription_TBackupEntry_EType_ETypeTable) { + continue; + } + + // Extract the relative part of the item path + // Item path is like /Root/db/table1, we need to extract the relative part + TString itemPath = item.GetPath(); + + // Only accept exact matches or suffixes preceded by path separator + // to avoid false matches (e.g. "/Root/FooBar" should not match "Bar") + if (itemPath == relativeTablePath || itemPath.EndsWith("/" + relativeTablePath)) { + return itemPath; + } + } + + return {}; +} + +void TSchemeShard::DiscoverIndexesRecursive( + ui64 operationId, + const TString& backupName, + const TPath& bcPath, + const TBackupCollectionInfo::TPtr& backupCollectionInfo, + const TPath& currentPath, + const TString& accumulatedRelativePath, + const TActorContext& ctx) { + + // Try to find target table for current accumulated path + TString targetTablePath = FindTargetTablePath(backupCollectionInfo, accumulatedRelativePath); + + if (!targetTablePath.empty()) { + // Found target table, children are indexes + LOG_I("Found table mapping: " << accumulatedRelativePath << " -> " << targetTablePath); + + for (const auto& [indexName, indexDirPathId] : currentPath.Base()->GetChildren()) { + CreateSingleIndexRestoreOperation( + operationId, + backupName, + bcPath, + accumulatedRelativePath, + indexName, + targetTablePath, + ctx + ); + } + } else { + // Not a table yet, descend into children to build up the path + for (const auto& [childName, childPathId] : currentPath.Base()->GetChildren()) { + auto childPath = TPath::Init(childPathId, this); + TString newRelativePath = accumulatedRelativePath.empty() + ? childName + : accumulatedRelativePath + "/" + childName; + + DiscoverIndexesRecursive( + operationId, + backupName, + bcPath, + backupCollectionInfo, + childPath, + newRelativePath, + ctx + ); + } + } +} + +void TSchemeShard::DiscoverAndCreateIndexRestoreOperations( + const TPathId& /*backupCollectionPathId*/, + ui64 operationId, + const TString& backupName, + const TPath& bcPath, + const TBackupCollectionInfo::TPtr& backupCollectionInfo, + const TActorContext& ctx) { + + // Check if indexes were backed up (OmitIndexes flag) + bool omitIndexes = backupCollectionInfo->Description.GetIncrementalBackupConfig().GetOmitIndexes(); + if (omitIndexes) { + LOG_I("Indexes were omitted in backup, skipping index restore"); + return; + } + + // Path to index metadata: {backup}/__ydb_backup_meta/indexes + TString indexMetaBasePath = JoinPath({ + bcPath.PathString(), + backupName + "_incremental", + "__ydb_backup_meta", + "indexes" + }); + + const TPath& indexMetaPath = TPath::Resolve(indexMetaBasePath, this); + if (!indexMetaPath.IsResolved()) { + LOG_I("No index metadata found at: " << indexMetaBasePath << " (this is normal if no indexes were backed up)"); + return; + } + + LOG_I("Discovering indexes for restore at: " << indexMetaBasePath); + + // Start recursive discovery from the indexes root with empty accumulated path + DiscoverIndexesRecursive( + operationId, + backupName, + bcPath, + backupCollectionInfo, + indexMetaPath, + "", // Start with empty accumulated path + ctx + ); +} + +void TSchemeShard::CreateSingleIndexRestoreOperation( + ui64 operationId, + const TString& backupName, + const TPath& bcPath, + const TString& relativeTablePath, + const TString& indexName, + const TString& targetTablePath, + const TActorContext& ctx) { + + LOG_I("CreateSingleIndexRestoreOperation: table=" << targetTablePath + << " index=" << indexName + << " relativeTablePath=" << relativeTablePath); + + // Validate target table exists + const TPath targetTablePathObj = TPath::Resolve(targetTablePath, this); + if (!targetTablePathObj.IsResolved() || !targetTablePathObj.Base()->IsTable()) { + LOG_W("Target table not found or invalid: " << targetTablePath); + return; + } + + // Find the index and its impl table + TPathId indexPathId; + TPathId indexImplTablePathId; + bool indexFound = false; + + for (const auto& [childName, childPathId] : targetTablePathObj.Base()->GetChildren()) { + if (childName == indexName) { + auto childPath = PathsById.at(childPathId); + if (childPath->PathType == NKikimrSchemeOp::EPathTypeTableIndex) { + indexPathId = childPathId; + + // Get index info to verify it's a global index + auto indexInfoIt = Indexes.find(indexPathId); + if (indexInfoIt == Indexes.end()) { + LOG_W("Index info not found for pathId: " << indexPathId); + return; + } + + auto indexInfo = indexInfoIt->second; + if (indexInfo->Type != NKikimrSchemeOp::EIndexTypeGlobal) { + LOG_I("Skipping non-global index: " << indexName << " (type=" << indexInfo->Type << ")"); + return; + } + + // Get index impl table (single child of index) + auto indexPath = TPath::Init(indexPathId, this); + if (indexPath.Base()->GetChildren().size() == 1) { + auto [implTableName, implTablePathId] = *indexPath.Base()->GetChildren().begin(); + indexImplTablePathId = implTablePathId; + indexFound = true; + LOG_I("Found global index '" << indexName << "' with impl table: " << implTableName); + break; + } else { + LOG_W("Index '" << indexName << "' has unexpected number of children: " + << indexPath.Base()->GetChildren().size()); + return; + } + } + } + } + + if (!indexFound) { + LOG_W("Index '" << indexName << "' not found on table " << targetTablePath + << " - skipping (index may have been dropped)"); + return; + } + + // Source: {backup}/__ydb_backup_meta/indexes/{table}/{index} + TString srcIndexBackupPath = JoinPath({ + bcPath.PathString(), + backupName + "_incremental", + "__ydb_backup_meta", + "indexes", + relativeTablePath, + indexName + }); + + const TPath& srcBackupPath = TPath::Resolve(srcIndexBackupPath, this); + if (!srcBackupPath.IsResolved()) { + LOG_W("Index backup not found at: " << srcIndexBackupPath); + return; + } + + // Destination: {table}/{index}/indexImplTable + auto indexImplTablePath = TPath::Init(indexImplTablePathId, this); + TString dstIndexImplPath = indexImplTablePath.PathString(); + + LOG_I("Creating index restore operation: " << srcIndexBackupPath << " -> " << dstIndexImplPath); + + // Create restore request (SAME structure as table restore) + auto indexRequest = MakeHolder(); + auto& indexRecord = indexRequest->Record; + + TTxId indexTxId = GetCachedTxId(ctx); + indexRecord.SetTxId(ui64(indexTxId)); + + auto& indexTx = *indexRecord.AddTransaction(); + indexTx.SetOperationType(NKikimrSchemeOp::ESchemeOpRestoreMultipleIncrementalBackups); + indexTx.SetInternal(true); + indexTx.SetWorkingDir(bcPath.PathString()); + + auto& indexRestore = *indexTx.MutableRestoreMultipleIncrementalBackups(); + indexRestore.AddSrcTablePaths(srcIndexBackupPath); + indexRestore.SetDstTablePath(dstIndexImplPath); + + // Track this operation as part of incremental restore + TOperationId indexRestoreOpId(indexTxId, 0); + IncrementalRestoreOperationToState[indexRestoreOpId] = operationId; + TxIdToIncrementalRestore[indexTxId] = operationId; + + auto stateIt = IncrementalRestoreStates.find(operationId); + if (stateIt != IncrementalRestoreStates.end()) { + // Add to in-progress operations (will be tracked alongside table operations) + stateIt->second.InProgressOperations.insert(indexRestoreOpId); + + // Track expected shards for this index impl table + auto& indexOpState = stateIt->second.TableOperations[indexRestoreOpId]; + indexOpState.OperationId = indexRestoreOpId; + + if (Tables.contains(indexImplTablePathId)) { + auto indexImplTable = Tables.at(indexImplTablePathId); + for (const auto& [shardIdx, partitionIdx] : indexImplTable->GetShard2PartitionIdx()) { + indexOpState.ExpectedShards.insert(shardIdx); + stateIt->second.InvolvedShards.insert(shardIdx); + } + LOG_I("Index operation " << indexRestoreOpId << " expects " << indexOpState.ExpectedShards.size() << " shards"); + } + + LOG_I("Tracking index operation " << indexRestoreOpId << " for incremental restore " << operationId); + } + + // Send the request (parallel with table operations) + LOG_I("Sending index restore operation for: " << dstIndexImplPath); + Send(SelfId(), indexRequest.Release()); +} + // Notification function for operation completion void TSchemeShard::NotifyIncrementalRestoreOperationCompleted(const TOperationId& operationId, const TActorContext& ctx) { // Find which incremental restore this operation belongs to diff --git a/ydb/core/tx/schemeshard/schemeshard_path.cpp b/ydb/core/tx/schemeshard/schemeshard_path.cpp index f9ced80da942..b6eabe943c0c 100644 --- a/ydb/core/tx/schemeshard/schemeshard_path.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_path.cpp @@ -3,6 +3,7 @@ #include "schemeshard_system_names.h" #include "schemeshard_impl.h" +#include #include #include @@ -1722,13 +1723,6 @@ bool TPath::IsInsideTableIndexPath(bool failOnUnresolved) const { return false; } - ++item; - for (; item != Elements.rend(); ++item) { - if (!(*item)->IsDirectory() && !(*item)->IsSubDomainRoot()) { - return false; - } - } - return true; } @@ -1870,7 +1864,22 @@ bool TPath::IsValidLeafName(const NACLib::TUserToken* userToken, TString& explai } if (AppData()->FeatureFlags.GetEnableSystemNamesProtection()) { - if (!CheckReservedName(leaf, AppData(), userToken, explain)) { + TPathCreationContext context; + context.IsSystemUser = NSchemeShard::IsSystemUser(userToken); + context.IsAdministrator = NKikimr::IsAdministrator(AppData(), userToken); + + if (IsBackupServiceReservedName(leaf)) { + TPath parentPath = Parent(); + while (parentPath.IsResolved() && !parentPath.Base()->IsRoot()) { + if (parentPath.Base()->IsBackupCollection()) { + context.IsInsideBackupCollection = true; + break; + } + parentPath = parentPath.Parent(); + } + } + + if (!CheckReservedName(leaf, context, explain)) { return false; } } else if (leaf == NSysView::SysPathName) { diff --git a/ydb/core/tx/schemeshard/schemeshard_system_names.cpp b/ydb/core/tx/schemeshard/schemeshard_system_names.cpp index 3d4b2b4e9ec7..532a3c74095d 100644 --- a/ydb/core/tx/schemeshard/schemeshard_system_names.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_system_names.cpp @@ -48,7 +48,14 @@ const TVector ReservedNamesExceptions = { ".STD", }; -bool CheckReservedNameImpl(const TString& name, bool isSystemUser, bool isAdministrator, TString& explain) { +// Special prefix for backup service metadata +const TString BackupServicePrefix = "__ydb_backup_"; + +bool IsBackupServiceReservedName(const TString& name) { + return name.StartsWith(BackupServicePrefix); +} + +bool CheckReservedNameImpl(const TString& name, const TPathCreationContext& context, TString& explain) { // System reserved names can't be created by ordinary users. // They can only be created: // - by the system itself @@ -57,10 +64,10 @@ bool CheckReservedNameImpl(const TString& name, bool isSystemUser, bool isAdmini auto it = std::find(ReservedNames.begin(), ReservedNames.end(), name); return (it != ReservedNames.end()); }(); - if (nameIsReserved && !(isSystemUser || isAdministrator)) { + if (nameIsReserved && !(context.IsSystemUser || context.IsAdministrator)) { explain += TStringBuilder() << "path part '" << name << "', name is reserved by the system: '" << name << "'" - << "(subject: system user " << isSystemUser << ", cluster admin " << isAdministrator << ")"; + << "(subject: system user " << context.IsSystemUser << ", cluster admin " << context.IsAdministrator << ")"; return false; } @@ -73,6 +80,20 @@ bool CheckReservedNameImpl(const TString& name, bool isSystemUser, bool isAdmini return true; } + // Special handling for backup service reserved names + // These names have a dedicated prefix and are only allowed inside backup collections + if (IsBackupServiceReservedName(name)) { + if (context.IsInsideBackupCollection) { + // Allowed: backup service metadata inside backup collections + return true; + } + + explain += TStringBuilder() + << "path part '" << name << "' uses backup service reserved prefix '" << BackupServicePrefix << "'. " + << "These names are reserved for backup metadata and can only be created inside backup collections"; + return false; + } + // Names that aren't reserved but start with a reserved prefix can't be created at all, // not even by admins or the system. // Such names must be explicitly added to the ReservedNames list before creation is possible. @@ -93,11 +114,21 @@ bool IsSystemUser(const NACLib::TUserToken* userToken) { } bool CheckReservedName(const TString& name, const NACLib::TUserToken* userToken, const TVector& allowedSids, TString& explain) { - return CheckReservedNameImpl(name, IsSystemUser(userToken), IsTokenAllowed(userToken, allowedSids), explain); + TPathCreationContext context; + context.IsSystemUser = IsSystemUser(userToken); + context.IsAdministrator = IsTokenAllowed(userToken, allowedSids); + return CheckReservedNameImpl(name, context, explain); } bool CheckReservedName(const TString& name, const TAppData* appData, const NACLib::TUserToken* userToken, TString& explain) { - return CheckReservedNameImpl(name, IsSystemUser(userToken), IsAdministrator(appData, userToken), explain); + TPathCreationContext context; + context.IsSystemUser = IsSystemUser(userToken); + context.IsAdministrator = NKikimr::IsAdministrator(appData, userToken); + return CheckReservedNameImpl(name, context, explain); +} + +bool CheckReservedName(const TString& name, const TPathCreationContext& context, TString& explain) { + return CheckReservedNameImpl(name, context, explain); } } // namespace NKikimr::NSchemeShard diff --git a/ydb/core/tx/schemeshard/schemeshard_system_names.h b/ydb/core/tx/schemeshard/schemeshard_system_names.h index b9d217327029..e8abb0553d3d 100644 --- a/ydb/core/tx/schemeshard/schemeshard_system_names.h +++ b/ydb/core/tx/schemeshard/schemeshard_system_names.h @@ -11,7 +11,18 @@ class TUserToken; namespace NKikimr::NSchemeShard { +struct TPathCreationContext { + bool IsSystemUser = false; + bool IsAdministrator = false; + bool IsInsideBackupCollection = false; +}; + bool CheckReservedName(const TString& name, const NACLib::TUserToken* userToken, const TVector& adminSids, TString& explain); bool CheckReservedName(const TString& name, const TAppData* appData, const NACLib::TUserToken* userToken, TString& explain); +bool IsSystemUser(const NACLib::TUserToken* userToken); +bool IsBackupServiceReservedName(const TString& name); + +bool CheckReservedName(const TString& name, const TPathCreationContext& context, TString& explain); + } // namespace NKikimr::NSchemeShard diff --git a/ydb/core/tx/schemeshard/ut_backup_collection/ut_backup_collection.cpp b/ydb/core/tx/schemeshard/ut_backup_collection/ut_backup_collection.cpp index e296402668fd..1b4521e464da 100644 --- a/ydb/core/tx/schemeshard/ut_backup_collection/ut_backup_collection.cpp +++ b/ydb/core/tx/schemeshard/ut_backup_collection/ut_backup_collection.cpp @@ -85,10 +85,10 @@ Y_UNIT_TEST_SUITE(TBackupCollectionTests) { auto transaction = modifyTx->Record.AddTransaction(); transaction->SetWorkingDir(workingDir); transaction->SetOperationType(NKikimrSchemeOp::EOperationType::ESchemeOpBackupBackupCollection); - + bool parseOk = ::google::protobuf::TextFormat::ParseFromString(request, transaction->MutableBackupBackupCollection()); UNIT_ASSERT(parseOk); - + AsyncSend(runtime, TTestTxConfig::SchemeShard, modifyTx.release(), 0); } @@ -99,14 +99,14 @@ Y_UNIT_TEST_SUITE(TBackupCollectionTests) { void AsyncBackupIncrementalBackupCollection(TTestBasicRuntime& runtime, ui64 txId, const TString& workingDir, const TString& request) { TActorId sender = runtime.AllocateEdgeActor(); - + auto request2 = MakeHolder(txId, TTestTxConfig::SchemeShard); auto transaction = request2->Record.AddTransaction(); transaction->SetOperationType(NKikimrSchemeOp::EOperationType::ESchemeOpBackupIncrementalBackupCollection); transaction->SetWorkingDir(workingDir); bool parseOk = ::google::protobuf::TextFormat::ParseFromString(request, transaction->MutableBackupIncrementalBackupCollection()); UNIT_ASSERT(parseOk); - + AsyncSend(runtime, TTestTxConfig::SchemeShard, request2.Release(), 0, sender); } @@ -589,8 +589,8 @@ Y_UNIT_TEST_SUITE(TBackupCollectionTests) { AsyncBackupBackupCollection(runtime, ++txId, "/MyRoot", R"(Name: ".backups/collections/)" DEFAULT_NAME_1 R"(")"); - TestDropBackupCollection(runtime, ++txId, "/MyRoot/.backups/collections", - "Name: \"" DEFAULT_NAME_1 "\"", + TestDropBackupCollection(runtime, ++txId, "/MyRoot/.backups/collections", + "Name: \"" DEFAULT_NAME_1 "\"", {NKikimrScheme::StatusPreconditionFailed}); env.TestWaitNotification(runtime, txId); @@ -616,8 +616,8 @@ Y_UNIT_TEST_SUITE(TBackupCollectionTests) { SetupLogging(runtime); PrepareDirs(runtime, env, txId); - TestDropBackupCollection(runtime, ++txId, "/MyRoot/.backups/collections", - "Name: \"NonExistentCollection\"", + TestDropBackupCollection(runtime, ++txId, "/MyRoot/.backups/collections", + "Name: \"NonExistentCollection\"", {NKikimrScheme::StatusPathDoesNotExist}); env.TestWaitNotification(runtime, txId); @@ -801,7 +801,7 @@ Y_UNIT_TEST_SUITE(TBackupCollectionTests) { R"(Name: ".backups/collections/)" DEFAULT_NAME_1 R"(")"); env.TestWaitNotification(runtime, txId); - TestDropBackupCollection(runtime, ++txId, "/MyRoot/.backups/collections", + TestDropBackupCollection(runtime, ++txId, "/MyRoot/.backups/collections", "Name: \"NonExistentCollection\"", {NKikimrScheme::StatusPathDoesNotExist}); env.TestWaitNotification(runtime, txId); @@ -860,7 +860,7 @@ Y_UNIT_TEST_SUITE(TBackupCollectionTests) { if (i > 0) { runtime.AdvanceCurrentTime(TDuration::Seconds(1)); } - + TestBackupBackupCollection(runtime, ++txId, "/MyRoot", R"(Name: ".backups/collections/)" DEFAULT_NAME_1 R"(")"); env.TestWaitNotification(runtime, txId); @@ -886,18 +886,18 @@ Y_UNIT_TEST_SUITE(TBackupCollectionTests) { SetupLogging(runtime); PrepareDirs(runtime, env, txId); - TestDropBackupCollection(runtime, ++txId, "/MyRoot/.backups/collections", - "Name: \"\"", + TestDropBackupCollection(runtime, ++txId, "/MyRoot/.backups/collections", + "Name: \"\"", {NKikimrScheme::StatusInvalidParameter}); env.TestWaitNotification(runtime, txId); - TestDropBackupCollection(runtime, ++txId, "/NonExistent/path", - "Name: \"test\"", + TestDropBackupCollection(runtime, ++txId, "/NonExistent/path", + "Name: \"test\"", {NKikimrScheme::StatusPathDoesNotExist}); env.TestWaitNotification(runtime, txId); - TestDropBackupCollection(runtime, ++txId, "/MyRoot", - "Name: \"test\"", + TestDropBackupCollection(runtime, ++txId, "/MyRoot", + "Name: \"test\"", {NKikimrScheme::StatusSchemeError}); env.TestWaitNotification(runtime, txId); } @@ -910,15 +910,15 @@ Y_UNIT_TEST_SUITE(TBackupCollectionTests) { SetupLogging(runtime); PrepareDirs(runtime, env, txId); - TestCreateBackupCollection(runtime, ++txId, "/MyRoot/.backups/collections/", + TestCreateBackupCollection(runtime, ++txId, "/MyRoot/.backups/collections/", DefaultCollectionSettingsWithName("Collection1")); env.TestWaitNotification(runtime, txId); - TestCreateBackupCollection(runtime, ++txId, "/MyRoot/.backups/collections/", + TestCreateBackupCollection(runtime, ++txId, "/MyRoot/.backups/collections/", DefaultCollectionSettingsWithName("Collection2")); env.TestWaitNotification(runtime, txId); - TestCreateBackupCollection(runtime, ++txId, "/MyRoot/.backups/collections/", + TestCreateBackupCollection(runtime, ++txId, "/MyRoot/.backups/collections/", DefaultCollectionSettingsWithName("Collection3")); env.TestWaitNotification(runtime, txId); @@ -939,7 +939,7 @@ Y_UNIT_TEST_SUITE(TBackupCollectionTests) { env.TestWaitNotification(runtime, txId); } - + Y_UNIT_TEST(DropCollectionVerifyLocalDatabaseCleanup) { TTestBasicRuntime runtime; TTestEnv env(runtime, TTestEnvOptions().EnableBackupService(true)); @@ -959,7 +959,7 @@ Y_UNIT_TEST_SUITE(TBackupCollectionTests) { } Cluster: {} )"; - TestCreateBackupCollection(runtime, ++txId, "/MyRoot/.backups/collections/", + TestCreateBackupCollection(runtime, ++txId, "/MyRoot/.backups/collections/", localDbCollectionSettings); env.TestWaitNotification(runtime, txId); @@ -973,17 +973,17 @@ Y_UNIT_TEST_SUITE(TBackupCollectionTests) { TestBackupBackupCollection(runtime, ++txId, "/MyRoot", R"(Name: ".backups/collections/LocalDbTestCollection")"); - env.TestWaitNotification(runtime, txId); TestDropBackupCollection(runtime, ++txId, "/MyRoot/.backups/collections", + env.TestWaitNotification(runtime, txId); TestDropBackupCollection(runtime, ++txId, "/MyRoot/.backups/collections", "Name: \"LocalDbTestCollection\""); env.TestWaitNotification(runtime, txId); RebootTablet(runtime, TTestTxConfig::SchemeShard, runtime.AllocateEdgeActor()); - TestDescribeResult(DescribePath(runtime, "/MyRoot/.backups/collections/LocalDbTestCollection"), + TestDescribeResult(DescribePath(runtime, "/MyRoot/.backups/collections/LocalDbTestCollection"), {NLs::PathNotExist}); ui64 schemeshardTabletId = TTestTxConfig::SchemeShard; - + bool backupCollectionTableClean = true; try { auto result = LocalMiniKQL(runtime, schemeshardTabletId, R"( @@ -996,7 +996,7 @@ Y_UNIT_TEST_SUITE(TBackupCollectionTests) { )) ) )"); - + auto& value = result.GetValue(); if (value.GetStruct(0).GetOptional().HasOptional()) { backupCollectionTableClean = false; @@ -1006,7 +1006,7 @@ Y_UNIT_TEST_SUITE(TBackupCollectionTests) { backupCollectionTableClean = false; Cerr << "ERROR: Failed to query BackupCollection table" << Endl; } - + UNIT_ASSERT_C(backupCollectionTableClean, "BackupCollection table not properly cleaned up"); bool incrementalRestoreOperationsClean = true; @@ -1021,7 +1021,7 @@ Y_UNIT_TEST_SUITE(TBackupCollectionTests) { )) ) )"); - + auto& value = result.GetValue(); if (value.GetStruct(0).GetOptional().HasOptional()) { incrementalRestoreOperationsClean = false; @@ -1031,7 +1031,7 @@ Y_UNIT_TEST_SUITE(TBackupCollectionTests) { incrementalRestoreOperationsClean = false; Cerr << "ERROR: Failed to query IncrementalRestoreOperations table" << Endl; } - + UNIT_ASSERT_C(incrementalRestoreOperationsClean, "IncrementalRestoreOperations table not properly cleaned up"); bool incrementalRestoreStateClean = true; @@ -1046,7 +1046,7 @@ Y_UNIT_TEST_SUITE(TBackupCollectionTests) { )) ) )"); - + auto& value = result.GetValue(); if (value.GetStruct(0).GetOptional().HasOptional()) { incrementalRestoreStateClean = false; @@ -1056,7 +1056,7 @@ Y_UNIT_TEST_SUITE(TBackupCollectionTests) { incrementalRestoreStateClean = false; Cerr << "ERROR: Failed to query IncrementalRestoreState table" << Endl; } - + UNIT_ASSERT_C(incrementalRestoreStateClean, "IncrementalRestoreState table not properly cleaned up"); bool incrementalRestoreShardProgressClean = true; @@ -1071,7 +1071,7 @@ Y_UNIT_TEST_SUITE(TBackupCollectionTests) { )) ) )"); - + auto& value = result.GetValue(); if (value.GetStruct(0).GetOptional().HasOptional()) { incrementalRestoreShardProgressClean = false; @@ -1081,7 +1081,7 @@ Y_UNIT_TEST_SUITE(TBackupCollectionTests) { incrementalRestoreShardProgressClean = false; Cerr << "ERROR: Failed to query IncrementalRestoreShardProgress table" << Endl; } - + UNIT_ASSERT_C(incrementalRestoreShardProgressClean, "IncrementalRestoreShardProgress table not properly cleaned up"); Cerr << "SUCCESS: All LocalDB tables properly cleaned up after DROP BACKUP COLLECTION" << Endl; @@ -1097,7 +1097,7 @@ Y_UNIT_TEST_SUITE(TBackupCollectionTests) { } Cluster: {} )"; - TestCreateBackupCollection(runtime, ++txId, "/MyRoot/.backups/collections/", + TestCreateBackupCollection(runtime, ++txId, "/MyRoot/.backups/collections/", recreateCollectionSettings); env.TestWaitNotification(runtime, txId); } @@ -1139,22 +1139,22 @@ Y_UNIT_TEST_SUITE(TBackupCollectionTests) { ui64 backupTxId = txId; // This shows that active operation protection IS implemented - TestDropBackupCollection(runtime, ++txId, "/MyRoot/.backups/collections", - "Name: \"ActiveOpTestCollection\"", + TestDropBackupCollection(runtime, ++txId, "/MyRoot/.backups/collections", + "Name: \"ActiveOpTestCollection\"", {NKikimrScheme::StatusPreconditionFailed}); // CORRECT: System properly rejects this env.TestWaitNotification(runtime, txId); env.TestWaitNotification(runtime, backupTxId); // VERIFICATION: Collection should still exist since drop was properly rejected - TestDescribeResult(DescribePath(runtime, "/MyRoot/.backups/collections/ActiveOpTestCollection"), + TestDescribeResult(DescribePath(runtime, "/MyRoot/.backups/collections/ActiveOpTestCollection"), {NLs::PathExist}); - TestDropBackupCollection(runtime, ++txId, "/MyRoot/.backups/collections", + TestDropBackupCollection(runtime, ++txId, "/MyRoot/.backups/collections", "Name: \"ActiveOpTestCollection\""); env.TestWaitNotification(runtime, txId); - TestDescribeResult(DescribePath(runtime, "/MyRoot/.backups/collections/ActiveOpTestCollection"), + TestDescribeResult(DescribePath(runtime, "/MyRoot/.backups/collections/ActiveOpTestCollection"), {NLs::PathNotExist}); } @@ -1179,7 +1179,7 @@ Y_UNIT_TEST_SUITE(TBackupCollectionTests) { IncrementalBackupConfig: {} )"; - TestCreateBackupCollection(runtime, ++txId, "/MyRoot/.backups/collections/", + TestCreateBackupCollection(runtime, ++txId, "/MyRoot/.backups/collections/", collectionSettingsWithIncremental); env.TestWaitNotification(runtime, txId); @@ -1202,11 +1202,11 @@ Y_UNIT_TEST_SUITE(TBackupCollectionTests) { TestDescribeResult(DescribePath(runtime, "/MyRoot/TestTable"), {NLs::PathExist, NLs::IsTable}); - + TestDescribeResult(DescribePath(runtime, "/MyRoot/.backups/collections/" DEFAULT_NAME_1), {NLs::PathExist, NLs::IsBackupCollection}); - TestDropBackupCollection(runtime, ++txId, "/MyRoot/.backups/collections", + TestDropBackupCollection(runtime, ++txId, "/MyRoot/.backups/collections", "Name: \"" DEFAULT_NAME_1 "\""); env.TestWaitNotification(runtime, txId); @@ -1263,7 +1263,7 @@ Y_UNIT_TEST_SUITE(TBackupCollectionTests) { auto describeResult = DescribePath(runtime, "/MyRoot/Table1", true, true); TVector cdcStreamNames; - + // Check table description for CDC streams (this is where they are actually stored) if (describeResult.GetPathDescription().HasTable()) { const auto& tableDesc = describeResult.GetPathDescription().GetTable(); @@ -1278,18 +1278,18 @@ Y_UNIT_TEST_SUITE(TBackupCollectionTests) { } } } - + UNIT_ASSERT_C(!cdcStreamNames.empty(), "Expected to find CDC streams with '_continuousBackupImpl' suffix after incremental backup"); - + for (const auto& streamName : cdcStreamNames) { - UNIT_ASSERT_C(streamName.size() >= 15 + TString("_continuousBackupImpl").size(), + UNIT_ASSERT_C(streamName.size() >= 15 + TString("_continuousBackupImpl").size(), "CDC stream name should have timestamp prefix: " + streamName); - + TString prefix = streamName.substr(0, streamName.size() - TString("_continuousBackupImpl").size()); UNIT_ASSERT_C(prefix.EndsWith("Z"), "CDC stream timestamp should end with 'Z': " + prefix); } - TestDropBackupCollection(runtime, ++txId, "/MyRoot/.backups/collections", + TestDropBackupCollection(runtime, ++txId, "/MyRoot/.backups/collections", "Name: \"" DEFAULT_NAME_1 "\""); env.TestWaitNotification(runtime, txId); @@ -1298,7 +1298,7 @@ Y_UNIT_TEST_SUITE(TBackupCollectionTests) { auto describeAfter = DescribePath(runtime, "/MyRoot/Table1", true, true); TVector remainingCdcStreams; - + // Check table description for remaining CDC streams if (describeAfter.GetPathDescription().HasTable()) { const auto& tableDesc = describeAfter.GetPathDescription().GetTable(); @@ -1313,12 +1313,12 @@ Y_UNIT_TEST_SUITE(TBackupCollectionTests) { } } } - - UNIT_ASSERT_C(remainingCdcStreams.empty(), + + UNIT_ASSERT_C(remainingCdcStreams.empty(), "Incremental backup CDC streams with '_continuousBackupImpl' suffix should be cleaned up after dropping backup collection"); // During incremental backup, CDC streams are created under the source table // They should be properly cleaned up when the backup collection is dropped - + TestDescribeResult(DescribePath(runtime, "/MyRoot/Table1"), {NLs::PathExist, NLs::IsTable}); @@ -1328,13 +1328,13 @@ Y_UNIT_TEST_SUITE(TBackupCollectionTests) { TestDescribeResult(DescribePath(runtime, "/MyRoot/.backups/collections/" DEFAULT_NAME_1), {NLs::PathNotExist}); - + TestDescribeResult(DescribePath(runtime, "/MyRoot/Table1"), {NLs::PathExist, NLs::IsTable}); auto describeAfterReboot = DescribePath(runtime, "/MyRoot/Table1", true, true); TVector cdcStreamsAfterReboot; - + if (describeAfterReboot.GetPathDescription().HasTable()) { const auto& tableDesc = describeAfterReboot.GetPathDescription().GetTable(); if (tableDesc.CdcStreamsSize() > 0) { @@ -1348,8 +1348,8 @@ Y_UNIT_TEST_SUITE(TBackupCollectionTests) { } } } - - UNIT_ASSERT_C(cdcStreamsAfterReboot.empty(), + + UNIT_ASSERT_C(cdcStreamsAfterReboot.empty(), "Incremental backup CDC streams with '_continuousBackupImpl' suffix should remain cleaned up after restart"); // The implementation properly handles CDC stream cleanup during backup collection drop @@ -1382,7 +1382,7 @@ Y_UNIT_TEST_SUITE(TBackupCollectionTests) { env.TestWaitNotification(runtime, txId); } - TestDropBackupCollection(runtime, ++txId, "/MyRoot/.backups/collections", + TestDropBackupCollection(runtime, ++txId, "/MyRoot/.backups/collections", "Name: \"" DEFAULT_NAME_1 "\""); env.TestWaitNotification(runtime, txId); @@ -1420,12 +1420,12 @@ Y_UNIT_TEST_SUITE(TBackupCollectionTests) { env.TestWaitNotification(runtime, txId); // Start first drop operation asynchronously - AsyncDropBackupCollection(runtime, ++txId, "/MyRoot/.backups/collections", + AsyncDropBackupCollection(runtime, ++txId, "/MyRoot/.backups/collections", "Name: \"" DEFAULT_NAME_1 "\""); // Immediately try second drop operation (should fail) - TestDropBackupCollection(runtime, ++txId, "/MyRoot/.backups/collections", - "Name: \"" DEFAULT_NAME_1 "\"", + TestDropBackupCollection(runtime, ++txId, "/MyRoot/.backups/collections", + "Name: \"" DEFAULT_NAME_1 "\"", {NKikimrScheme::StatusMultipleModifications}); // Expect concurrent operation error env.TestWaitNotification(runtime, txId - 1); @@ -1661,4 +1661,890 @@ Y_UNIT_TEST_SUITE(TBackupCollectionTests) { TestGetIncrementalBackup(runtime, backupId, "/MyRoot", Ydb::StatusIds::NOT_FOUND); } + + Y_UNIT_TEST(BackupServiceDirectoryValidation) { + TTestBasicRuntime runtime; + TTestEnv env(runtime, TTestEnvOptions().EnableBackupService(true)); + SetupLogging(runtime); + + // Enable system names protection feature + runtime.GetAppData().FeatureFlags.SetEnableSystemNamesProtection(true); + + ui64 txId = 100; + + PrepareDirs(runtime, env, txId); + + // Create a backup collection + TestCreateBackupCollection(runtime, ++txId, "/MyRoot/.backups/collections", R"( + Name: "TestCollection" + ExplicitEntryList { + Entries { + Type: ETypeTable + Path: "/MyRoot/Table1" + } + } + )"); + env.TestWaitNotification(runtime, txId); + + // Try to create __ydb_backup_meta outside backup collection (should fail - reserved name) + TestMkDir(runtime, ++txId, "/MyRoot", "__ydb_backup_meta", {NKikimrScheme::StatusSchemeError}); + + // Verify we can't create directories with reserved backup service prefix outside backup context + TestMkDir(runtime, ++txId, "/MyRoot", "__ydb_backup_test", {NKikimrScheme::StatusSchemeError}); + + // But we CAN create __ydb_backup_meta inside a backup collection (should succeed) + TestMkDir(runtime, ++txId, "/MyRoot/.backups/collections/TestCollection", "__ydb_backup_meta"); + env.TestWaitNotification(runtime, txId); + + // Verify it was created + TestLs(runtime, "/MyRoot/.backups/collections/TestCollection/__ydb_backup_meta", false, NLs::PathExist); + } + + Y_UNIT_TEST(SingleTableWithGlobalSyncIndex) { + TTestBasicRuntime runtime; + TTestEnv env(runtime, TTestEnvOptions().EnableBackupService(true)); + ui64 txId = 100; + + SetupLogging(runtime); + PrepareDirs(runtime, env, txId); + + // Create incremental backup collection + TString collectionSettings = R"( + Name: ")" DEFAULT_NAME_1 R"(" + ExplicitEntryList { + Entries { + Type: ETypeTable + Path: "/MyRoot/TableWithIndex" + } + } + Cluster: {} + IncrementalBackupConfig: {} + )"; + + TestCreateBackupCollection(runtime, ++txId, "/MyRoot/.backups/collections/", collectionSettings); + env.TestWaitNotification(runtime, txId); + + // Create table with one global sync covering index + TestCreateIndexedTable(runtime, ++txId, "/MyRoot", R"( + TableDescription { + Name: "TableWithIndex" + Columns { Name: "key" Type: "Uint32" } + Columns { Name: "value" Type: "Utf8" } + KeyColumnNames: ["key"] + } + IndexDescription { + Name: "ValueIndex" + KeyColumnNames: ["value"] + Type: EIndexTypeGlobal + } + )"); + env.TestWaitNotification(runtime, txId); + + // Execute full backup + TestBackupBackupCollection(runtime, ++txId, "/MyRoot", + R"(Name: ".backups/collections/)" DEFAULT_NAME_1 R"(")"); + env.TestWaitNotification(runtime, txId); + + // Verify CDC stream exists on main table + auto mainTableDesc = DescribePrivatePath(runtime, "/MyRoot/TableWithIndex", true, true); + UNIT_ASSERT(mainTableDesc.GetPathDescription().HasTable()); + + const auto& tableDesc = mainTableDesc.GetPathDescription().GetTable(); + bool foundMainTableCdc = false; + TString mainTableCdcName; + + for (size_t i = 0; i < tableDesc.CdcStreamsSize(); ++i) { + const auto& cdcStream = tableDesc.GetCdcStreams(i); + if (cdcStream.GetName().EndsWith("_continuousBackupImpl")) { + foundMainTableCdc = true; + mainTableCdcName = cdcStream.GetName(); + Cerr << "Found main table CDC stream: " << mainTableCdcName << Endl; + break; + } + } + UNIT_ASSERT_C(foundMainTableCdc, "Main table should have CDC stream with '_continuousBackupImpl' suffix"); + + // Verify CDC stream exists on index implementation table + auto indexDesc = DescribePrivatePath(runtime, "/MyRoot/TableWithIndex/ValueIndex", true, true); + UNIT_ASSERT(indexDesc.GetPathDescription().HasTableIndex()); + + // Get index implementation table (first child of index) + UNIT_ASSERT_VALUES_EQUAL(indexDesc.GetPathDescription().ChildrenSize(), 1); + TString indexImplTableName = indexDesc.GetPathDescription().GetChildren(0).GetName(); + + auto indexImplTableDesc = DescribePrivatePath(runtime, + "/MyRoot/TableWithIndex/ValueIndex/" + indexImplTableName, true, true); + UNIT_ASSERT(indexImplTableDesc.GetPathDescription().HasTable()); + + const auto& indexTableDesc = indexImplTableDesc.GetPathDescription().GetTable(); + bool foundIndexCdc = false; + TString indexCdcName; + + for (size_t i = 0; i < indexTableDesc.CdcStreamsSize(); ++i) { + const auto& cdcStream = indexTableDesc.GetCdcStreams(i); + if (cdcStream.GetName().EndsWith("_continuousBackupImpl")) { + foundIndexCdc = true; + indexCdcName = cdcStream.GetName(); + Cerr << "Found index CDC stream: " << indexCdcName << Endl; + break; + } + } + UNIT_ASSERT_C(foundIndexCdc, "Index implementation table should have CDC stream with '_continuousBackupImpl' suffix"); + + // Verify CDC stream names match pattern and use same timestamp + UNIT_ASSERT_VALUES_EQUAL(mainTableCdcName, indexCdcName); + UNIT_ASSERT_C(mainTableCdcName.Contains("Z") && mainTableCdcName.EndsWith("_continuousBackupImpl"), + "CDC stream name should have X.509 timestamp format (YYYYMMDDHHMMSSZ_continuousBackupImpl)"); + } + + Y_UNIT_TEST(SingleTableWithMultipleGlobalSyncIndexes) { + TTestBasicRuntime runtime; + TTestEnv env(runtime, TTestEnvOptions().EnableBackupService(true)); + ui64 txId = 100; + + SetupLogging(runtime); + PrepareDirs(runtime, env, txId); + + // Create incremental backup collection + TString collectionSettings = R"( + Name: ")" DEFAULT_NAME_1 R"(" + ExplicitEntryList { + Entries { + Type: ETypeTable + Path: "/MyRoot/TableWithMultipleIndexes" + } + } + Cluster: {} + IncrementalBackupConfig: {} + )"; + + TestCreateBackupCollection(runtime, ++txId, "/MyRoot/.backups/collections/", collectionSettings); + env.TestWaitNotification(runtime, txId); + + // Create table with multiple global sync indexes + TestCreateIndexedTable(runtime, ++txId, "/MyRoot", R"( + TableDescription { + Name: "TableWithMultipleIndexes" + Columns { Name: "key" Type: "Uint32" } + Columns { Name: "value1" Type: "Utf8" } + Columns { Name: "value2" Type: "Uint64" } + KeyColumnNames: ["key"] + } + IndexDescription { + Name: "Value1Index" + KeyColumnNames: ["value1"] + Type: EIndexTypeGlobal + } + IndexDescription { + Name: "Value2Index" + KeyColumnNames: ["value2"] + DataColumnNames: ["value1"] + Type: EIndexTypeGlobal + } + )"); + env.TestWaitNotification(runtime, txId); + + // Execute full backup + TestBackupBackupCollection(runtime, ++txId, "/MyRoot", + R"(Name: ".backups/collections/)" DEFAULT_NAME_1 R"(")"); + env.TestWaitNotification(runtime, txId); + + // Verify CDC stream on main table + auto mainTableDesc = DescribePrivatePath(runtime, "/MyRoot/TableWithMultipleIndexes", true, true); + const auto& tableDesc = mainTableDesc.GetPathDescription().GetTable(); + + TString mainCdcName; + for (size_t i = 0; i < tableDesc.CdcStreamsSize(); ++i) { + const auto& cdcStream = tableDesc.GetCdcStreams(i); + if (cdcStream.GetName().EndsWith("_continuousBackupImpl")) { + mainCdcName = cdcStream.GetName(); + break; + } + } + UNIT_ASSERT_C(!mainCdcName.empty(), "Main table should have CDC stream"); + + // Verify CDC streams on both indexes + TVector indexNames = {"Value1Index", "Value2Index"}; + TVector indexCdcNames; + + for (const auto& indexName : indexNames) { + auto indexDesc = DescribePrivatePath(runtime, + "/MyRoot/TableWithMultipleIndexes/" + indexName, true, true); + UNIT_ASSERT_VALUES_EQUAL(indexDesc.GetPathDescription().ChildrenSize(), 1); + TString indexImplTableName = indexDesc.GetPathDescription().GetChildren(0).GetName(); + + auto indexImplTableDesc = DescribePrivatePath(runtime, + "/MyRoot/TableWithMultipleIndexes/" + indexName + "/" + indexImplTableName, true, true); + const auto& indexTableDesc = indexImplTableDesc.GetPathDescription().GetTable(); + + bool foundCdc = false; + for (size_t i = 0; i < indexTableDesc.CdcStreamsSize(); ++i) { + const auto& cdcStream = indexTableDesc.GetCdcStreams(i); + if (cdcStream.GetName().EndsWith("_continuousBackupImpl")) { + indexCdcNames.push_back(cdcStream.GetName()); + foundCdc = true; + Cerr << "Found CDC stream on " << indexName << ": " << cdcStream.GetName() << Endl; + break; + } + } + UNIT_ASSERT_C(foundCdc, "Index " + indexName + " should have CDC stream"); + } + + // Verify all streams use the same timestamp + UNIT_ASSERT_VALUES_EQUAL(indexCdcNames.size(), 2); + UNIT_ASSERT_VALUES_EQUAL(mainCdcName, indexCdcNames[0]); + UNIT_ASSERT_VALUES_EQUAL(mainCdcName, indexCdcNames[1]); + } + + Y_UNIT_TEST(TableWithMixedIndexTypes) { + TTestBasicRuntime runtime; + TTestEnv env(runtime, TTestEnvOptions().EnableBackupService(true)); + ui64 txId = 100; + + SetupLogging(runtime); + PrepareDirs(runtime, env, txId); + + // Create incremental backup collection + TString collectionSettings = R"( + Name: ")" DEFAULT_NAME_1 R"(" + ExplicitEntryList { + Entries { + Type: ETypeTable + Path: "/MyRoot/TableWithMixedIndexes" + } + } + Cluster: {} + IncrementalBackupConfig: {} + )"; + + TestCreateBackupCollection(runtime, ++txId, "/MyRoot/.backups/collections/", collectionSettings); + env.TestWaitNotification(runtime, txId); + + // Create table with global sync + async indexes + TestCreateIndexedTable(runtime, ++txId, "/MyRoot", R"( + TableDescription { + Name: "TableWithMixedIndexes" + Columns { Name: "key" Type: "Uint32" } + Columns { Name: "value1" Type: "Utf8" } + Columns { Name: "value2" Type: "Uint64" } + KeyColumnNames: ["key"] + } + IndexDescription { + Name: "SyncIndex" + KeyColumnNames: ["value1"] + Type: EIndexTypeGlobal + } + IndexDescription { + Name: "AsyncIndex" + KeyColumnNames: ["value2"] + Type: EIndexTypeGlobalAsync + } + )"); + env.TestWaitNotification(runtime, txId); + + // Execute full backup + TestBackupBackupCollection(runtime, ++txId, "/MyRoot", + R"(Name: ".backups/collections/)" DEFAULT_NAME_1 R"(")"); + env.TestWaitNotification(runtime, txId); + + // Verify CDC stream on main table + auto mainTableDesc = DescribePrivatePath(runtime, "/MyRoot/TableWithMixedIndexes", true, true); + const auto& tableDesc = mainTableDesc.GetPathDescription().GetTable(); + + bool foundMainCdc = false; + for (size_t i = 0; i < tableDesc.CdcStreamsSize(); ++i) { + if (tableDesc.GetCdcStreams(i).GetName().EndsWith("_continuousBackupImpl")) { + foundMainCdc = true; + break; + } + } + UNIT_ASSERT_C(foundMainCdc, "Main table should have CDC stream"); + + // Verify CDC stream on global sync index ONLY + auto syncIndexDesc = DescribePrivatePath(runtime, "/MyRoot/TableWithMixedIndexes/SyncIndex", true, true); + UNIT_ASSERT_VALUES_EQUAL(syncIndexDesc.GetPathDescription().ChildrenSize(), 1); + TString syncImplTableName = syncIndexDesc.GetPathDescription().GetChildren(0).GetName(); + + auto syncImplTableDesc = DescribePrivatePath(runtime, + "/MyRoot/TableWithMixedIndexes/SyncIndex/" + syncImplTableName, true, true); + const auto& syncTableDesc = syncImplTableDesc.GetPathDescription().GetTable(); + + bool foundSyncCdc = false; + for (size_t i = 0; i < syncTableDesc.CdcStreamsSize(); ++i) { + if (syncTableDesc.GetCdcStreams(i).GetName().EndsWith("_continuousBackupImpl")) { + foundSyncCdc = true; + Cerr << "Found CDC stream on SyncIndex (expected)" << Endl; + break; + } + } + UNIT_ASSERT_C(foundSyncCdc, "Global sync index should have CDC stream"); + + // Verify NO CDC stream on async index + auto asyncIndexDesc = DescribePrivatePath(runtime, "/MyRoot/TableWithMixedIndexes/AsyncIndex", true, true); + UNIT_ASSERT_VALUES_EQUAL(asyncIndexDesc.GetPathDescription().ChildrenSize(), 1); + TString asyncImplTableName = asyncIndexDesc.GetPathDescription().GetChildren(0).GetName(); + + auto asyncImplTableDesc = DescribePrivatePath(runtime, + "/MyRoot/TableWithMixedIndexes/AsyncIndex/" + asyncImplTableName, true, true); + const auto& asyncTableDesc = asyncImplTableDesc.GetPathDescription().GetTable(); + + bool foundAsyncCdc = false; + for (size_t i = 0; i < asyncTableDesc.CdcStreamsSize(); ++i) { + if (asyncTableDesc.GetCdcStreams(i).GetName().EndsWith("_continuousBackupImpl")) { + foundAsyncCdc = true; + break; + } + } + UNIT_ASSERT_C(!foundAsyncCdc, "Async index should NOT have CDC stream"); + } + + Y_UNIT_TEST(MultipleTablesWithIndexes) { + TTestBasicRuntime runtime; + TTestEnv env(runtime, TTestEnvOptions().EnableBackupService(true)); + ui64 txId = 100; + + SetupLogging(runtime); + PrepareDirs(runtime, env, txId); + + // Create incremental backup collection with 2 tables + TString collectionSettings = R"( + Name: ")" DEFAULT_NAME_1 R"(" + ExplicitEntryList { + Entries { + Type: ETypeTable + Path: "/MyRoot/Table1" + } + Entries { + Type: ETypeTable + Path: "/MyRoot/Table2" + } + } + Cluster: {} + IncrementalBackupConfig: {} + )"; + + TestCreateBackupCollection(runtime, ++txId, "/MyRoot/.backups/collections/", collectionSettings); + env.TestWaitNotification(runtime, txId); + + // Create Table1 with index + TestCreateIndexedTable(runtime, ++txId, "/MyRoot", R"( + TableDescription { + Name: "Table1" + Columns { Name: "key" Type: "Uint32" } + Columns { Name: "value" Type: "Utf8" } + KeyColumnNames: ["key"] + } + IndexDescription { + Name: "Index1" + KeyColumnNames: ["value"] + Type: EIndexTypeGlobal + } + )"); + env.TestWaitNotification(runtime, txId); + + // Create Table2 with index + TestCreateIndexedTable(runtime, ++txId, "/MyRoot", R"( + TableDescription { + Name: "Table2" + Columns { Name: "key" Type: "Uint32" } + Columns { Name: "data" Type: "Utf8" } + KeyColumnNames: ["key"] + } + IndexDescription { + Name: "Index2" + KeyColumnNames: ["data"] + Type: EIndexTypeGlobal + } + )"); + env.TestWaitNotification(runtime, txId); + + // Execute full backup + TestBackupBackupCollection(runtime, ++txId, "/MyRoot", + R"(Name: ".backups/collections/)" DEFAULT_NAME_1 R"(")"); + env.TestWaitNotification(runtime, txId); + + // Verify CDC streams on both main tables + TVector tables = {"Table1", "Table2"}; + TVector indexes = {"Index1", "Index2"}; + + for (size_t tableIdx = 0; tableIdx < tables.size(); ++tableIdx) { + const auto& tableName = tables[tableIdx]; + const auto& indexName = indexes[tableIdx]; + + // Check main table CDC + auto mainTableDesc = DescribePrivatePath(runtime, "/MyRoot/" + tableName, true, true); + const auto& tableDesc = mainTableDesc.GetPathDescription().GetTable(); + + bool foundMainCdc = false; + for (size_t i = 0; i < tableDesc.CdcStreamsSize(); ++i) { + if (tableDesc.GetCdcStreams(i).GetName().EndsWith("_continuousBackupImpl")) { + foundMainCdc = true; + Cerr << "Found CDC stream on " << tableName << Endl; + break; + } + } + UNIT_ASSERT_C(foundMainCdc, tableName + " should have CDC stream"); + + // Check index CDC + auto indexDesc = DescribePrivatePath(runtime, "/MyRoot/" + tableName + "/" + indexName, true, true); + UNIT_ASSERT_VALUES_EQUAL(indexDesc.GetPathDescription().ChildrenSize(), 1); + TString indexImplTableName = indexDesc.GetPathDescription().GetChildren(0).GetName(); + + auto indexImplTableDesc = DescribePrivatePath(runtime, + "/MyRoot/" + tableName + "/" + indexName + "/" + indexImplTableName, true, true); + const auto& indexTableDesc = indexImplTableDesc.GetPathDescription().GetTable(); + + bool foundIndexCdc = false; + for (size_t i = 0; i < indexTableDesc.CdcStreamsSize(); ++i) { + if (indexTableDesc.GetCdcStreams(i).GetName().EndsWith("_continuousBackupImpl")) { + foundIndexCdc = true; + Cerr << "Found CDC stream on " << indexName << Endl; + break; + } + } + UNIT_ASSERT_C(foundIndexCdc, indexName + " should have CDC stream"); + } + } + + Y_UNIT_TEST(IncrementalBackupWithIndexes) { + TTestBasicRuntime runtime; + TTestEnv env(runtime, TTestEnvOptions().EnableBackupService(true)); + ui64 txId = 100; + + SetupLogging(runtime); + PrepareDirs(runtime, env, txId); + + // Create incremental backup collection + TString collectionSettings = R"( + Name: ")" DEFAULT_NAME_1 R"(" + ExplicitEntryList { + Entries { + Type: ETypeTable + Path: "/MyRoot/TableForIncremental" + } + } + Cluster: {} + IncrementalBackupConfig: {} + )"; + + TestCreateBackupCollection(runtime, ++txId, "/MyRoot/.backups/collections/", collectionSettings); + env.TestWaitNotification(runtime, txId); + + // Create table with global sync index + TestCreateIndexedTable(runtime, ++txId, "/MyRoot", R"( + TableDescription { + Name: "TableForIncremental" + Columns { Name: "key" Type: "Uint32" } + Columns { Name: "value" Type: "Utf8" } + KeyColumnNames: ["key"] + } + IndexDescription { + Name: "ValueIndex" + KeyColumnNames: ["value"] + Type: EIndexTypeGlobal + } + )"); + env.TestWaitNotification(runtime, txId); + + // Execute full backup (creates CDC streams) + TestBackupBackupCollection(runtime, ++txId, "/MyRoot", + R"(Name: ".backups/collections/)" DEFAULT_NAME_1 R"(")"); + env.TestWaitNotification(runtime, txId); + + // Verify CDC streams were created for both main table and index + auto mainTableDesc = DescribePrivatePath(runtime, "/MyRoot/TableForIncremental", true, true); + const auto& tableDesc = mainTableDesc.GetPathDescription().GetTable(); + + bool foundMainCdc = false; + for (size_t i = 0; i < tableDesc.CdcStreamsSize(); ++i) { + if (tableDesc.GetCdcStreams(i).GetName().EndsWith("_continuousBackupImpl")) { + foundMainCdc = true; + Cerr << "Found CDC stream on main table" << Endl; + break; + } + } + UNIT_ASSERT_C(foundMainCdc, "Main table should have CDC stream after full backup"); + + // Verify CDC stream on index + auto indexDesc = DescribePrivatePath(runtime, "/MyRoot/TableForIncremental/ValueIndex", true, true); + UNIT_ASSERT_VALUES_EQUAL(indexDesc.GetPathDescription().ChildrenSize(), 1); + TString indexImplTableName = indexDesc.GetPathDescription().GetChildren(0).GetName(); + + auto indexImplTableDesc = DescribePrivatePath(runtime, + "/MyRoot/TableForIncremental/ValueIndex/" + indexImplTableName, true, true); + const auto& indexTableDesc = indexImplTableDesc.GetPathDescription().GetTable(); + + bool foundIndexCdc = false; + for (size_t i = 0; i < indexTableDesc.CdcStreamsSize(); ++i) { + if (indexTableDesc.GetCdcStreams(i).GetName().EndsWith("_continuousBackupImpl")) { + foundIndexCdc = true; + Cerr << "Found CDC stream on index implementation table" << Endl; + break; + } + } + UNIT_ASSERT_C(foundIndexCdc, "Index implementation table should have CDC stream after full backup"); + + runtime.AdvanceCurrentTime(TDuration::Seconds(1)); + + // Execute incremental backup (rotates CDC, creates backup tables) + TestBackupIncrementalBackupCollection(runtime, ++txId, "/MyRoot", + R"(Name: ".backups/collections/)" DEFAULT_NAME_1 R"(")"); + env.TestWaitNotification(runtime, txId); + + // Verify backup collection structure + TestDescribeResult(DescribePath(runtime, "/MyRoot/.backups/collections/" DEFAULT_NAME_1), { + NLs::PathExist, + NLs::IsBackupCollection, + NLs::ChildrenCount(2), // full + incremental + }); + + // Find the incremental backup directory (should end with "_incremental") + auto collectionDesc = DescribePath(runtime, "/MyRoot/.backups/collections/" DEFAULT_NAME_1, true, true); + TString incrBackupDir; + for (size_t i = 0; i < collectionDesc.GetPathDescription().ChildrenSize(); ++i) { + const auto& child = collectionDesc.GetPathDescription().GetChildren(i); + Cerr << "Child: " << child.GetName() << " PathState: " << child.GetPathState() << Endl; + if (child.GetName().EndsWith("_incremental")) { + incrBackupDir = child.GetName(); + break; + } + } + UNIT_ASSERT_C(!incrBackupDir.empty(), "Should find incremental backup directory"); + + // Verify backup table for main table exists + TestDescribeResult(DescribePath(runtime, + "/MyRoot/.backups/collections/" DEFAULT_NAME_1 "/" + incrBackupDir + "/TableForIncremental"), { + NLs::PathExist, + NLs::IsTable, + }); + + // Verify index backup table exists in __ydb_backup_meta/indexes/TableForIncremental/ValueIndex + TString indexBackupPath = "/MyRoot/.backups/collections/" DEFAULT_NAME_1 "/" + incrBackupDir + + "/__ydb_backup_meta/indexes/TableForIncremental/ValueIndex"; + TestDescribeResult(DescribePath(runtime, indexBackupPath), { + NLs::PathExist, + NLs::IsTable, + }); + + Cerr << "SUCCESS: Full backup created CDC streams for both main table and index" << Endl; + Cerr << " Incremental backup created backup tables for both main table and index" << Endl; + Cerr << " Index backup table verified at: " << indexBackupPath << Endl; + } + + Y_UNIT_TEST(OmitIndexesFlag) { + TTestBasicRuntime runtime; + TTestEnv env(runtime, TTestEnvOptions().EnableBackupService(true)); + ui64 txId = 100; + + SetupLogging(runtime); + PrepareDirs(runtime, env, txId); + + // Create incremental backup collection WITH OmitIndexes flag set + TString collectionSettings = R"( + Name: ")" DEFAULT_NAME_1 R"(" + ExplicitEntryList { + Entries { + Type: ETypeTable + Path: "/MyRoot/TableWithIndex" + } + } + Cluster: {} + IncrementalBackupConfig { + OmitIndexes: true + } + )"; + + TestCreateBackupCollection(runtime, ++txId, "/MyRoot/.backups/collections/", collectionSettings); + env.TestWaitNotification(runtime, txId); + + // Create table with global sync index + TestCreateIndexedTable(runtime, ++txId, "/MyRoot", R"( + TableDescription { + Name: "TableWithIndex" + Columns { Name: "key" Type: "Uint32" } + Columns { Name: "value" Type: "Utf8" } + KeyColumnNames: ["key"] + } + IndexDescription { + Name: "ValueIndex" + KeyColumnNames: ["value"] + Type: EIndexTypeGlobal + } + )"); + env.TestWaitNotification(runtime, txId); + + // Execute full backup + TestBackupBackupCollection(runtime, ++txId, "/MyRoot", + R"(Name: ".backups/collections/)" DEFAULT_NAME_1 R"(")"); + env.TestWaitNotification(runtime, txId); + + // Verify CDC stream exists on main table + auto mainTableDesc = DescribePrivatePath(runtime, "/MyRoot/TableWithIndex", true, true); + const auto& tableDesc = mainTableDesc.GetPathDescription().GetTable(); + + bool foundMainCdc = false; + for (size_t i = 0; i < tableDesc.CdcStreamsSize(); ++i) { + if (tableDesc.GetCdcStreams(i).GetName().EndsWith("_continuousBackupImpl")) { + foundMainCdc = true; + Cerr << "Found CDC stream on main table (expected)" << Endl; + break; + } + } + UNIT_ASSERT_C(foundMainCdc, "Main table should have CDC stream even with OmitIndexes=true"); + + // Verify NO CDC stream on index (because OmitIndexes is true) + auto indexDesc = DescribePrivatePath(runtime, "/MyRoot/TableWithIndex/ValueIndex", true, true); + UNIT_ASSERT_VALUES_EQUAL(indexDesc.GetPathDescription().ChildrenSize(), 1); + TString indexImplTableName = indexDesc.GetPathDescription().GetChildren(0).GetName(); + + auto indexImplTableDesc = DescribePrivatePath(runtime, + "/MyRoot/TableWithIndex/ValueIndex/" + indexImplTableName, true, true); + const auto& indexTableDesc = indexImplTableDesc.GetPathDescription().GetTable(); + + bool foundIndexCdc = false; + for (size_t i = 0; i < indexTableDesc.CdcStreamsSize(); ++i) { + if (indexTableDesc.GetCdcStreams(i).GetName().EndsWith("_continuousBackupImpl")) { + foundIndexCdc = true; + break; + } + } + UNIT_ASSERT_C(!foundIndexCdc, "Index should NOT have CDC stream when OmitIndexes=true"); + + Cerr << "SUCCESS: OmitIndexes flag works correctly - main table has CDC, index does not" << Endl; + } + + Y_UNIT_TEST(BackupWithIndexes) { + TTestBasicRuntime runtime; + TTestEnv env(runtime, TTestEnvOptions().EnableBackupService(true)); + SetupLogging(runtime); + ui64 txId = 100; + + // Create table with index + TestCreateIndexedTable(runtime, ++txId, "/MyRoot", R"( + TableDescription { + Name: "TableWithIndex" + Columns { Name: "key" Type: "Uint64" } + Columns { Name: "value" Type: "Utf8" } + KeyColumnNames: ["key"] + } + IndexDescription { + Name: "ValueIndex" + KeyColumnNames: ["value"] + } + )"); + env.TestWaitNotification(runtime, txId); + + // Verify source table has the index + TestDescribeResult(DescribePath(runtime, "/MyRoot/TableWithIndex"), { + NLs::PathExist, + NLs::IndexesCount(1) + }); + + PrepareDirs(runtime, env, txId); + + // Create backup collection with OmitIndexes = false (explicitly request indexes) + TestCreateBackupCollection(runtime, ++txId, "/MyRoot/.backups/collections", R"( + Name: "CollectionWithIndex" + ExplicitEntryList { + Entries { + Type: ETypeTable + Path: "/MyRoot/TableWithIndex" + } + } + OmitIndexes: false + )"); + env.TestWaitNotification(runtime, txId); + + // Backup the table (indexes should be included) + TestBackupBackupCollection(runtime, ++txId, "/MyRoot", + R"(Name: ".backups/collections/CollectionWithIndex")"); + env.TestWaitNotification(runtime, txId); + + // Verify backup collection has children (the backup directory) + TestDescribeResult(DescribePath(runtime, "/MyRoot/.backups/collections/CollectionWithIndex"), { + NLs::PathExist, + NLs::IsBackupCollection, + NLs::ChildrenCount(1) + }); + + // Get the backup directory and verify its structure contains index + auto backupDesc = DescribePath(runtime, "/MyRoot/.backups/collections/CollectionWithIndex"); + UNIT_ASSERT(backupDesc.GetPathDescription().ChildrenSize() == 1); + TString backupDirName = backupDesc.GetPathDescription().GetChildren(0).GetName(); + + // Verify backup directory has the table (indexes are stored under the table) + TString backupPath = "/MyRoot/.backups/collections/CollectionWithIndex/" + backupDirName; + auto backupContentDesc = DescribePath(runtime, backupPath); + + // The backup should contain 1 child (the table; indexes are children of the table) + UNIT_ASSERT_C(backupContentDesc.GetPathDescription().ChildrenSize() == 1, + "Backup should contain 1 table, got " << backupContentDesc.GetPathDescription().ChildrenSize()); + + // Verify the table HAS indexes in the backup (check via TableIndexesSize) + UNIT_ASSERT_VALUES_EQUAL(backupContentDesc.GetPathDescription().GetChildren(0).GetName(), "TableWithIndex"); + + auto tableDesc = DescribePath(runtime, backupPath + "/TableWithIndex"); + UNIT_ASSERT(tableDesc.GetPathDescription().HasTable()); + UNIT_ASSERT_VALUES_EQUAL(tableDesc.GetPathDescription().GetTable().TableIndexesSize(), 1); + + // Verify ChildrenExist flag is set (index exists as child, even if not in Children list) + UNIT_ASSERT(tableDesc.GetPathDescription().GetSelf().GetChildrenExist()); + } + + Y_UNIT_TEST(BackupWithIndexesOmit) { + TTestBasicRuntime runtime; + TTestEnv env(runtime, TTestEnvOptions().EnableBackupService(true)); + SetupLogging(runtime); + ui64 txId = 100; + + // Create table with index + TestCreateIndexedTable(runtime, ++txId, "/MyRoot", R"( + TableDescription { + Name: "TableWithIndex" + Columns { Name: "key" Type: "Uint64" } + Columns { Name: "value" Type: "Utf8" } + KeyColumnNames: ["key"] + } + IndexDescription { + Name: "ValueIndex" + KeyColumnNames: ["value"] + } + )"); + env.TestWaitNotification(runtime, txId); + + // Verify source table has the index + TestDescribeResult(DescribePath(runtime, "/MyRoot/TableWithIndex"), { + NLs::PathExist, + NLs::IndexesCount(1) + }); + + PrepareDirs(runtime, env, txId); + + // Create backup collection with OmitIndexes = true (at collection level) + TestCreateBackupCollection(runtime, ++txId, "/MyRoot/.backups/collections", R"( + Name: "CollectionWithoutIndex" + ExplicitEntryList { + Entries { + Type: ETypeTable + Path: "/MyRoot/TableWithIndex" + } + } + OmitIndexes: true + )"); + env.TestWaitNotification(runtime, txId); + + // Backup the table (indexes should be omitted) + TestBackupBackupCollection(runtime, ++txId, "/MyRoot", + R"(Name: ".backups/collections/CollectionWithoutIndex")"); + env.TestWaitNotification(runtime, txId); + + // Verify backup collection has children (the backup directory) + TestDescribeResult(DescribePath(runtime, "/MyRoot/.backups/collections/CollectionWithoutIndex"), { + NLs::PathExist, + NLs::IsBackupCollection, + NLs::ChildrenCount(1) + }); + + // Get the backup directory and verify its structure does NOT contain index + auto backupDesc = DescribePath(runtime, "/MyRoot/.backups/collections/CollectionWithoutIndex"); + UNIT_ASSERT(backupDesc.GetPathDescription().ChildrenSize() == 1); + TString backupDirName = backupDesc.GetPathDescription().GetChildren(0).GetName(); + + // Verify backup directory has only the table (no index children when omitted) + TString backupPath = "/MyRoot/.backups/collections/CollectionWithoutIndex/" + backupDirName; + auto backupContentDesc = DescribePath(runtime, backupPath); + + // The backup should contain 1 child (the table), without index children + UNIT_ASSERT_C(backupContentDesc.GetPathDescription().ChildrenSize() == 1, + "Backup should contain only table without index, got " << backupContentDesc.GetPathDescription().ChildrenSize()); + + // Verify the table exists but has NO indexes (omitted via OmitIndexes: true) + UNIT_ASSERT_VALUES_EQUAL(backupContentDesc.GetPathDescription().GetChildren(0).GetName(), "TableWithIndex"); + + auto tableDesc = DescribePath(runtime, backupPath + "/TableWithIndex"); + UNIT_ASSERT(tableDesc.GetPathDescription().HasTable()); + + // When indexes are omitted, TableIndexesSize should be 0 + UNIT_ASSERT_VALUES_EQUAL(tableDesc.GetPathDescription().GetTable().TableIndexesSize(), 0); + + // Verify ChildrenExist is false (no index children) + UNIT_ASSERT(!tableDesc.GetPathDescription().GetSelf().GetChildrenExist()); + } + + Y_UNIT_TEST(BackupWithIndexesDefault) { + TTestBasicRuntime runtime; + TTestEnv env(runtime, TTestEnvOptions().EnableBackupService(true)); + SetupLogging(runtime); + ui64 txId = 100; + + // Create table with index + TestCreateIndexedTable(runtime, ++txId, "/MyRoot", R"( + TableDescription { + Name: "TableWithIndex" + Columns { Name: "key" Type: "Uint64" } + Columns { Name: "value" Type: "Utf8" } + KeyColumnNames: ["key"] + } + IndexDescription { + Name: "ValueIndex" + KeyColumnNames: ["value"] + } + )"); + env.TestWaitNotification(runtime, txId); + + // Verify source table has the index + TestDescribeResult(DescribePath(runtime, "/MyRoot/TableWithIndex"), { + NLs::PathExist, + NLs::IndexesCount(1) + }); + + PrepareDirs(runtime, env, txId); + + // Create backup collection without specifying OmitIndexes (default behavior) + TestCreateBackupCollection(runtime, ++txId, "/MyRoot/.backups/collections", R"( + Name: "CollectionDefaultBehavior" + ExplicitEntryList { + Entries { + Type: ETypeTable + Path: "/MyRoot/TableWithIndex" + } + } + )"); + env.TestWaitNotification(runtime, txId); + + // Backup the table (default behavior: OmitIndexes not specified, should default to false) + TestBackupBackupCollection(runtime, ++txId, "/MyRoot", + R"(Name: ".backups/collections/CollectionDefaultBehavior")"); + env.TestWaitNotification(runtime, txId); + + // Verify backup collection has children (the backup directory) + TestDescribeResult(DescribePath(runtime, "/MyRoot/.backups/collections/CollectionDefaultBehavior"), { + NLs::PathExist, + NLs::IsBackupCollection, + NLs::ChildrenCount(1) + }); + + // Get the backup directory and verify its structure + auto backupDesc = DescribePath(runtime, "/MyRoot/.backups/collections/CollectionDefaultBehavior"); + UNIT_ASSERT(backupDesc.GetPathDescription().ChildrenSize() == 1); + TString backupDirName = backupDesc.GetPathDescription().GetChildren(0).GetName(); + + // Verify backup directory structure + TString backupPath = "/MyRoot/.backups/collections/CollectionDefaultBehavior/" + backupDirName; + auto backupContentDesc = DescribePath(runtime, backupPath); + + // The backup should contain 1 child (the table; indexes are children of the table) + UNIT_ASSERT_C(backupContentDesc.GetPathDescription().ChildrenSize() == 1, + "Backup should contain 1 table, got " << backupContentDesc.GetPathDescription().ChildrenSize()); + + // Verify the table HAS indexes in the backup by default (check via TableIndexesSize) + UNIT_ASSERT_VALUES_EQUAL(backupContentDesc.GetPathDescription().GetChildren(0).GetName(), "TableWithIndex"); + + auto tableDesc = DescribePath(runtime, backupPath + "/TableWithIndex"); + UNIT_ASSERT(tableDesc.GetPathDescription().HasTable()); + UNIT_ASSERT_VALUES_EQUAL(tableDesc.GetPathDescription().GetTable().TableIndexesSize(), 1); + + // Verify ChildrenExist flag is set by default (index exists as child) + UNIT_ASSERT(tableDesc.GetPathDescription().GetSelf().GetChildrenExist()); + } } // TBackupCollectionTests diff --git a/ydb/core/tx/schemeshard/ut_base/ut_base.cpp b/ydb/core/tx/schemeshard/ut_base/ut_base.cpp index 03d0862c4ec9..8d2c799c8030 100644 --- a/ydb/core/tx/schemeshard/ut_base/ut_base.cpp +++ b/ydb/core/tx/schemeshard/ut_base/ut_base.cpp @@ -11926,9 +11926,12 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { auto backupDirName = descr.GetChildren(0).GetName().c_str(); + // When WithIncremental=true, __ydb_backup_meta directory is created for index backups + // So we expect 3 children: Table1, DirA, and __ydb_backup_meta + // When WithIncremental=false, we expect 2 children: Table1 and DirA TestDescribeResult(DescribePath(runtime, Sprintf("/MyRoot/.backups/collections/MyCollection1/%s", backupDirName)), { NLs::PathExist, - NLs::ChildrenCount(2), + NLs::ChildrenCount(WithIncremental ? 3 : 2), NLs::Finished, }); @@ -11992,9 +11995,10 @@ Y_UNIT_TEST_SUITE(TSchemeShardTest) { } } + // Incremental backup directory contains Table1, DirA, and __ydb_backup_meta TestDescribeResult(DescribePath(runtime, Sprintf("/MyRoot/.backups/collections/MyCollection1/%s", incrBackupDirName)), { NLs::PathExist, - NLs::ChildrenCount(2), + NLs::ChildrenCount(3), NLs::Finished, }); diff --git a/ydb/core/tx/schemeshard/ut_consistent_copy_tables/ut_consistent_copy_tables.cpp b/ydb/core/tx/schemeshard/ut_consistent_copy_tables/ut_consistent_copy_tables.cpp new file mode 100644 index 000000000000..d8b6953e171d --- /dev/null +++ b/ydb/core/tx/schemeshard/ut_consistent_copy_tables/ut_consistent_copy_tables.cpp @@ -0,0 +1,267 @@ +#include +#include + +using namespace NSchemeShardUT_Private; + +Y_UNIT_TEST_SUITE(TSchemeShardConsistentCopyTablesTest) { + void SetupLogging(TTestActorRuntimeBase& runtime) { + runtime.SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + } + + // Priority 1 Test 1: Regular consistent copy with global sync index + // This test would have caught the OmitIndexes bug + Y_UNIT_TEST(ConsistentCopyTableWithGlobalSyncIndex) { + TTestBasicRuntime runtime; + TTestEnv env(runtime); + ui64 txId = 100; + + SetupLogging(runtime); + + // 1. Create table with global sync index + TestCreateIndexedTable(runtime, ++txId, "/MyRoot", R"( + TableDescription { + Name: "TableWithIndex" + Columns { Name: "key" Type: "Uint32" } + Columns { Name: "value" Type: "Utf8" } + KeyColumnNames: ["key"] + } + IndexDescription { + Name: "ValueIndex" + KeyColumnNames: ["value"] + Type: EIndexTypeGlobal + } + )"); + env.TestWaitNotification(runtime, txId); + + // 2. Perform consistent copy (NOT backup - this is the critical test case) + TestConsistentCopyTables(runtime, ++txId, "/MyRoot", R"( + CopyTableDescriptions { + SrcPath: "/MyRoot/TableWithIndex" + DstPath: "/MyRoot/TableWithIndexCopy" + } + )"); + env.TestWaitNotification(runtime, txId); + + // 3. Verify ALL components exist + // Main table + TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/TableWithIndexCopy"), + {NLs::PathExist}); + + // Index structure + auto indexDesc = DescribePrivatePath(runtime, "/MyRoot/TableWithIndexCopy/ValueIndex", true, true); + UNIT_ASSERT(indexDesc.GetPathDescription().HasTableIndex()); + UNIT_ASSERT_VALUES_EQUAL(indexDesc.GetPathDescription().GetTableIndex().GetState(), + NKikimrSchemeOp::EIndexStateReady); + + // CRITICAL: Verify index impl table exists (THIS WOULD HAVE FAILED WITH THE BUG) + UNIT_ASSERT_C(indexDesc.GetPathDescription().ChildrenSize() == 1, + "Index should have exactly one impl table child"); + + TString indexImplTableName = indexDesc.GetPathDescription().GetChildren(0).GetName(); + Cerr << "Index impl table name: " << indexImplTableName << Endl; + + auto indexImplTableDesc = DescribePrivatePath(runtime, + "/MyRoot/TableWithIndexCopy/ValueIndex/" + indexImplTableName, true, true); + UNIT_ASSERT_C(indexImplTableDesc.GetPathDescription().HasTable(), + "Index impl table should exist - this is what the bug broke!"); + } + + // Priority 1 Test 2: OmitIndexes flag should be respected + Y_UNIT_TEST(ConsistentCopyWithOmitIndexesTrueSkipsIndexes) { + TTestBasicRuntime runtime; + TTestEnv env(runtime); + ui64 txId = 100; + + SetupLogging(runtime); + + // 1. Create table with global sync index + TestCreateIndexedTable(runtime, ++txId, "/MyRoot", R"( + TableDescription { + Name: "TableWithIndex" + Columns { Name: "key" Type: "Uint32" } + Columns { Name: "value" Type: "Utf8" } + KeyColumnNames: ["key"] + } + IndexDescription { + Name: "ValueIndex" + KeyColumnNames: ["value"] + Type: EIndexTypeGlobal + } + )"); + env.TestWaitNotification(runtime, txId); + + // 2. Copy with OmitIndexes=true (explicit user request) + TestConsistentCopyTables(runtime, ++txId, "/MyRoot", R"( + CopyTableDescriptions { + SrcPath: "/MyRoot/TableWithIndex" + DstPath: "/MyRoot/TableCopy" + OmitIndexes: true + } + )"); + env.TestWaitNotification(runtime, txId); + + // 3. Verify main table exists + TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/TableCopy"), + {NLs::PathExist}); + + // 4. Verify index does NOT exist (user requested to omit it) + TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/TableCopy/ValueIndex"), + {NLs::PathNotExist}); + } + + // Priority 1 Test 3: Incremental backup path should still work + // (Ensure fix didn't break the use case it was designed for) + Y_UNIT_TEST(IncrementalBackupIndexesContinuesToWork) { + TTestBasicRuntime runtime; + TTestEnv env(runtime, TTestEnvOptions().EnableBackupService(true)); + ui64 txId = 100; + + SetupLogging(runtime); + + // Prepare backup directories + TestMkDir(runtime, ++txId, "/MyRoot", ".backups"); + env.TestWaitNotification(runtime, txId); + TestMkDir(runtime, ++txId, "/MyRoot/.backups", "collections"); + env.TestWaitNotification(runtime, txId); + + // Create incremental backup collection + TString collectionSettings = R"( + Name: "TestCollection" + ExplicitEntryList { + Entries { + Type: ETypeTable + Path: "/MyRoot/TableWithIndex" + } + } + Cluster: {} + IncrementalBackupConfig: {} + )"; + + TestCreateBackupCollection(runtime, ++txId, "/MyRoot/.backups/collections/", collectionSettings); + env.TestWaitNotification(runtime, txId); + + // Create table with one global sync index + TestCreateIndexedTable(runtime, ++txId, "/MyRoot", R"( + TableDescription { + Name: "TableWithIndex" + Columns { Name: "key" Type: "Uint32" } + Columns { Name: "value" Type: "Utf8" } + KeyColumnNames: ["key"] + } + IndexDescription { + Name: "ValueIndex" + KeyColumnNames: ["value"] + Type: EIndexTypeGlobal + } + )"); + env.TestWaitNotification(runtime, txId); + + // Execute full backup + TestBackupBackupCollection(runtime, ++txId, "/MyRoot", + R"(Name: ".backups/collections/TestCollection")"); + env.TestWaitNotification(runtime, txId); + + // Verify CDC stream exists on main table + auto mainTableDesc = DescribePrivatePath(runtime, "/MyRoot/TableWithIndex", true, true); + UNIT_ASSERT(mainTableDesc.GetPathDescription().HasTable()); + + const auto& tableDesc = mainTableDesc.GetPathDescription().GetTable(); + bool foundMainTableCdc = false; + + for (size_t i = 0; i < tableDesc.CdcStreamsSize(); ++i) { + const auto& cdcStream = tableDesc.GetCdcStreams(i); + if (cdcStream.GetName().EndsWith("_continuousBackupImpl")) { + foundMainTableCdc = true; + break; + } + } + UNIT_ASSERT_C(foundMainTableCdc, "Main table should have CDC stream for incremental backup"); + + // Verify CDC stream exists on index implementation table + auto indexDesc = DescribePrivatePath(runtime, "/MyRoot/TableWithIndex/ValueIndex", true, true); + UNIT_ASSERT(indexDesc.GetPathDescription().HasTableIndex()); + + // Get index implementation table + UNIT_ASSERT_VALUES_EQUAL(indexDesc.GetPathDescription().ChildrenSize(), 1); + TString indexImplTableName = indexDesc.GetPathDescription().GetChildren(0).GetName(); + + auto indexImplTableDesc = DescribePrivatePath(runtime, + "/MyRoot/TableWithIndex/ValueIndex/" + indexImplTableName, true, true); + UNIT_ASSERT(indexImplTableDesc.GetPathDescription().HasTable()); + + const auto& indexTableDesc = indexImplTableDesc.GetPathDescription().GetTable(); + bool foundIndexCdc = false; + + for (size_t i = 0; i < indexTableDesc.CdcStreamsSize(); ++i) { + const auto& cdcStream = indexTableDesc.GetCdcStreams(i); + if (cdcStream.GetName().EndsWith("_continuousBackupImpl")) { + foundIndexCdc = true; + break; + } + } + UNIT_ASSERT_C(foundIndexCdc, + "Index impl table should have CDC stream for incremental backup"); + } + + // Priority 2 Test 1: Multiple indexes + Y_UNIT_TEST(ConsistentCopyTableWithMultipleIndexes) { + TTestBasicRuntime runtime; + TTestEnv env(runtime); + ui64 txId = 100; + + SetupLogging(runtime); + + // 1. Create table with multiple global indexes + TestCreateIndexedTable(runtime, ++txId, "/MyRoot", R"( + TableDescription { + Name: "TableWithMultipleIndexes" + Columns { Name: "key" Type: "Uint32" } + Columns { Name: "value1" Type: "Utf8" } + Columns { Name: "value2" Type: "Utf8" } + KeyColumnNames: ["key"] + } + IndexDescription { + Name: "ValueIndex1" + KeyColumnNames: ["value1"] + Type: EIndexTypeGlobal + } + IndexDescription { + Name: "ValueIndex2" + KeyColumnNames: ["value2"] + Type: EIndexTypeGlobal + } + )"); + env.TestWaitNotification(runtime, txId); + + // 2. Perform consistent copy + TestConsistentCopyTables(runtime, ++txId, "/MyRoot", R"( + CopyTableDescriptions { + SrcPath: "/MyRoot/TableWithMultipleIndexes" + DstPath: "/MyRoot/TableCopy" + } + )"); + env.TestWaitNotification(runtime, txId); + + // 3. Verify both indexes and their impl tables exist + TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/TableCopy"), + {NLs::PathExist}); + + // Check first index + auto index1Desc = DescribePrivatePath(runtime, "/MyRoot/TableCopy/ValueIndex1", true, true); + UNIT_ASSERT(index1Desc.GetPathDescription().HasTableIndex()); + UNIT_ASSERT_VALUES_EQUAL(index1Desc.GetPathDescription().ChildrenSize(), 1); + TString implTable1Name = index1Desc.GetPathDescription().GetChildren(0).GetName(); + TestDescribeResult(DescribePrivatePath(runtime, + "/MyRoot/TableCopy/ValueIndex1/" + implTable1Name), + {NLs::PathExist}); + + // Check second index + auto index2Desc = DescribePrivatePath(runtime, "/MyRoot/TableCopy/ValueIndex2", true, true); + UNIT_ASSERT(index2Desc.GetPathDescription().HasTableIndex()); + UNIT_ASSERT_VALUES_EQUAL(index2Desc.GetPathDescription().ChildrenSize(), 1); + TString implTable2Name = index2Desc.GetPathDescription().GetChildren(0).GetName(); + TestDescribeResult(DescribePrivatePath(runtime, + "/MyRoot/TableCopy/ValueIndex2/" + implTable2Name), + {NLs::PathExist}); + } +} diff --git a/ydb/core/tx/schemeshard/ut_consistent_copy_tables/ya.make b/ydb/core/tx/schemeshard/ut_consistent_copy_tables/ya.make new file mode 100644 index 000000000000..26690e2d9b8d --- /dev/null +++ b/ydb/core/tx/schemeshard/ut_consistent_copy_tables/ya.make @@ -0,0 +1,22 @@ +UNITTEST_FOR(ydb/core/tx/schemeshard) + +FORK_SUBTESTS() + +TIMEOUT(600) + +SIZE(MEDIUM) + +PEERDIR( + ydb/core/kqp/ut/common + ydb/core/tx/schemeshard/ut_helpers + yql/essentials/sql/pg + yql/essentials/parser/pg_wrapper +) + +SRCS( + ut_consistent_copy_tables.cpp +) + +YQL_LAST_ABI_VERSION() + +END() diff --git a/ydb/core/tx/schemeshard/ya.make b/ydb/core/tx/schemeshard/ya.make index 5fad82895524..e36bb17095ce 100644 --- a/ydb/core/tx/schemeshard/ya.make +++ b/ydb/core/tx/schemeshard/ya.make @@ -12,6 +12,7 @@ RECURSE_FOR_TESTS( ut_cdc_stream_reboots ut_column_build ut_compaction + ut_consistent_copy_tables ut_continuous_backup ut_continuous_backup_reboots ut_shred diff --git a/ydb/core/util/source_location.h b/ydb/core/util/source_location.h index 1d2fb03bd289..635dad07f444 100644 --- a/ydb/core/util/source_location.h +++ b/ydb/core/util/source_location.h @@ -11,7 +11,7 @@ using TSourceLocation = std::source_location; constexpr inline bool HasSourceLocation = true; -} // namespace NCompat +} // namespace NKikimr::NCompat #else namespace NKikimr::NCompat { @@ -30,15 +30,19 @@ struct TSourceLocation { constexpr uint_least32_t line() const noexcept { return 0; } + + constexpr const char* function_name() const noexcept { + return ""; + } }; constexpr inline bool HasSourceLocation = false; -} // namespace NCompat +} // namespace NKikimr::NCompat #endif namespace NKikimr::NUtil { TString TrimSourceFileName(const char* fileName); -} // namespace NKikimrNUtil +} // namespace NKikimr::NUtil diff --git a/ydb/library/ut/README.md b/ydb/library/ut/README.md new file mode 100644 index 000000000000..9c0abb7282be --- /dev/null +++ b/ydb/library/ut/README.md @@ -0,0 +1,44 @@ +# Unit Test Context Helpers + +This library provides helpers for passing test context through helper functions in unit tests. + +## Problem + +When using helper functions in unit tests that contain assertions, if an assertion fails, the error message usually points to the line inside the helper function. This makes it difficult to know which call to the helper function caused the failure, especially if the helper is called multiple times. + +## Solution + +`TTestContext` captures the source location of the call site. By passing `TTestContext` to your helper functions and using `CTX_UNIT_*` macros, you can report errors with the location of the call site. + +## Usage + +1. Include the header: + ```cpp + #include + ``` + +2. Define your helper function to accept `TTestContext` as a default argument: + ```cpp + void MyHelper(int expected, int actual, NYdb::NUt::TTestContext testCtx = NYdb::NUt::TTestContext()) { + ``` + +3. Call the helper as usual: + ```cpp + Y_UNIT_TEST(MyTest) { + MyHelper(1, 1); // Success + MyHelper(1, 2); // Failure reported at this line + } + ``` + +## Future Improvements + +After `library/cpp/testing/unittest` extension we won't need any custom `CTX_UNIT` redefinition (we just use customization point and replace original `UNIT_FAIL_IMPL` with context aware version). + +## Macros + +- `CTX_UNIT_ASSERT(condition)` +- `CTX_UNIT_ASSERT_C(condition, message)` +- `CTX_UNIT_ASSERT_VALUES_EQUAL(a, b)` +- `CTX_UNIT_ASSERT_VALUES_EQUAL_C(a, b, message)` +- `CTX_UNIT_ASSERT_VALUES_UNEQUAL(a, b)` +- `CTX_UNIT_ASSERT_VALUES_UNEQUAL_C(a, b, message)` diff --git a/ydb/library/ut/ut.h b/ydb/library/ut/ut.h new file mode 100644 index 000000000000..8efd4f4993db --- /dev/null +++ b/ydb/library/ut/ut.h @@ -0,0 +1,104 @@ +#pragma once + +#include + +#include + +namespace NYdb::NUt { +class TTestContextBase { +public: + TTestContextBase(const NKikimr::NCompat::TSourceLocation &loc = + NKikimr::NCompat::TSourceLocation::current()) + : Location_(loc) {} + + virtual ~TTestContextBase() = default; + virtual TString Format() const = 0; + +protected: + TString FormatLocation() const { + return Sprintf("%s:%d, %s", Location_.file_name(), Location_.line(), + Location_.function_name()); + } + + NKikimr::NCompat::TSourceLocation Location_; +}; + +// Simple default context - just location +struct TTestContext : public TTestContextBase { + TTestContext(const NKikimr::NCompat::TSourceLocation &loc = + NKikimr::NCompat::TSourceLocation::current()) + : TTestContextBase(loc) {} + + TString Format() const override { return FormatLocation(); } +}; +} // namespace NYdb::NUt + +// Context-aware fail implementation +// TODO: After library/cpp/testing/unittest extension we won't need any custom +// CTX_UNIT redefinition (we just use customization point and replace original +// UNIT_FAIL_IMPL with context aware version) +#define CTX_UNIT_FAIL_IMPL(R, M) \ + do { \ + ::TStringBuilder locationInfo; \ + if constexpr ( \ + requires { testCtx; } && \ + std::derived_from && \ + requires { \ + { testCtx.Format() } -> std::convertible_to; \ + }) { \ + locationInfo << testCtx.Format(); \ + } else { \ + locationInfo << __LOCATION__ << ", " << __PRETTY_FUNCTION__; \ + } \ + ::NUnitTest::NPrivate::RaiseError( \ + R, ::TStringBuilder() << R << " at " << locationInfo << ": " << M, \ + true); \ + } while (false) + +#define CTX_UNIT_ASSERT_C(A, C) \ + do { \ + if (!(A)) { \ + CTX_UNIT_FAIL_IMPL( \ + "assertion failed", \ + Sprintf("(%s) %s", #A, (::TStringBuilder() << C).data())); \ + } \ + } while (false) + +#define CTX_UNIT_ASSERT(A) CTX_UNIT_ASSERT_C(A, "") + +// values with context +#define CTX_UNIT_ASSERT_VALUES_EQUAL_IMPL(A, B, C, EQflag, EQstr, NEQstr) \ + do { \ + /* NOLINTBEGIN(bugprone-reserved-identifier, \ + * readability-identifier-naming) */ \ + TString _as; \ + TString _bs; \ + TString _asInd; \ + TString _bsInd; \ + bool _usePlainDiff; \ + if (!::NUnitTest::NPrivate::CompareAndMakeStrings( \ + A, B, _as, _asInd, _bs, _bsInd, _usePlainDiff, EQflag)) { \ + auto &&failMsg = Sprintf("(%s %s %s) failed: (%s %s %s) %s", #A, EQstr, \ + #B, _as.data(), NEQstr, _bs.data(), \ + (::TStringBuilder() << C).data()); \ + if (EQflag && !_usePlainDiff) { \ + failMsg += ", with diff:\n"; \ + failMsg += ::NUnitTest::ColoredDiff(_asInd, _bsInd); \ + } \ + CTX_UNIT_FAIL_IMPL("assertion failed", failMsg); \ + } \ + /* NOLINTEND(bugprone-reserved-identifier, readability-identifier-naming) \ + */ \ + } while (false) + +#define CTX_UNIT_ASSERT_VALUES_EQUAL_C(A, B, C) \ + CTX_UNIT_ASSERT_VALUES_EQUAL_IMPL(A, B, C, true, "==", "!=") + +#define CTX_UNIT_ASSERT_VALUES_UNEQUAL_C(A, B, C) \ + CTX_UNIT_ASSERT_VALUES_EQUAL_IMPL(A, B, C, false, "!=", "==") + +#define CTX_UNIT_ASSERT_VALUES_EQUAL(A, B) \ + CTX_UNIT_ASSERT_VALUES_EQUAL_C(A, B, "") +#define CTX_UNIT_ASSERT_VALUES_UNEQUAL(A, B) \ + CTX_UNIT_ASSERT_VALUES_UNEQUAL_C(A, B, "") diff --git a/ydb/library/ut/ya.make b/ydb/library/ut/ya.make new file mode 100644 index 000000000000..7d1c1518bc4b --- /dev/null +++ b/ydb/library/ut/ya.make @@ -0,0 +1,12 @@ +LIBRARY() + +SRCS( + ut.h +) + +PEERDIR( + library/cpp/testing/unittest + ydb/core/util +) + +END()