From de5c8d5f46a181d7f86b8064fb5df6180013b810 Mon Sep 17 00:00:00 2001 From: Daniil Timizhev Date: Sat, 25 Oct 2025 03:43:24 +0300 Subject: [PATCH] Impl methods to detect BulkUpsert with DEFAULT columns and a flag to disable (#27596) Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- ydb/core/kqp/ut/scheme/kqp_constraints_ut.cpp | 9 ++++--- ydb/core/protos/feature_flags.proto | 1 + ydb/core/testlib/basics/feature_flags.h | 1 + .../tx/tx_proxy/upload_rows_common_impl.h | 24 +++++++++++++++++++ ydb/core/tx/tx_proxy/upload_rows_counters.cpp | 1 + ydb/core/tx/tx_proxy/upload_rows_counters.h | 6 +++++ 6 files changed, 39 insertions(+), 3 deletions(-) diff --git a/ydb/core/kqp/ut/scheme/kqp_constraints_ut.cpp b/ydb/core/kqp/ut/scheme/kqp_constraints_ut.cpp index 207cfec405b8..2c703460a259 100644 --- a/ydb/core/kqp/ut/scheme/kqp_constraints_ut.cpp +++ b/ydb/core/kqp/ut/scheme/kqp_constraints_ut.cpp @@ -782,12 +782,15 @@ Y_UNIT_TEST_SUITE(KqpConstraints) { TKikimrRunner kikimr(TKikimrSettings() .SetUseRealThreads(false) .SetEnableAddColumsWithDefaults(true) + .SetDisableMissingDefaultColumnsInBulkUpsert(true) .SetWithSampleTables(false)); auto db = kikimr.RunCall([&] { return kikimr.GetQueryClient(); } ); auto session = kikimr.RunCall([&] { return db.GetSession().GetValueSync().GetSession(); } ); auto querySession = kikimr.RunCall([&] { return db.GetSession().GetValueSync().GetSession(); } ); + auto tableClient = kikimr.RunCall([&] { return kikimr.GetTableClient(); } ); + auto& runtime = *kikimr.GetTestServer().GetRuntime(); { @@ -860,7 +863,7 @@ Y_UNIT_TEST_SUITE(KqpConstraints) { auto alterQuery = R"( ALTER TABLE `/Root/AddNonColumnDoesnotReturnInternalError` - ADD COLUMN Value3 Int32 NOT NULL DEFAULT 7; + ADD COLUMN Value3 Int32 DEFAULT 7; )"; auto alterFuture = kikimr.RunInThreadPool([&] { return session.ExecuteQuery(alterQuery, TTxControl::NoTx()).GetValueSync(); }); @@ -924,8 +927,8 @@ Y_UNIT_TEST_SUITE(KqpConstraints) { auto result = runtime.WaitFuture(alterFuture); fCompareTable(R"([ - [1u;"Changed";"Updated";7]; - [2u;"New";"text";7] + [1u;"Changed";"Updated";[7]]; + [2u;"New";"text";[7]] ])"); } diff --git a/ydb/core/protos/feature_flags.proto b/ydb/core/protos/feature_flags.proto index ad247ef67c5f..cd21062a5e04 100644 --- a/ydb/core/protos/feature_flags.proto +++ b/ydb/core/protos/feature_flags.proto @@ -213,4 +213,5 @@ message TFeatureFlags { optional bool EnableTopicCompactificationByKey = 196 [default = true]; optional bool EnableCompactionOverloadDetection = 197 [default = true]; reserved 198; // DisableColumnShardBulkUpsertRequireAllColumns + optional bool DisableMissingDefaultColumnsInBulkUpsert = 215 [default = false]; } diff --git a/ydb/core/testlib/basics/feature_flags.h b/ydb/core/testlib/basics/feature_flags.h index bff8e3fc2a8b..6cd5344bfc2f 100644 --- a/ydb/core/testlib/basics/feature_flags.h +++ b/ydb/core/testlib/basics/feature_flags.h @@ -78,6 +78,7 @@ class TTestFeatureFlagsHolder { FEATURE_FLAG_SETTER(EnablePermissionsExport) FEATURE_FLAG_SETTER(EnableLocalDBBtreeIndex) FEATURE_FLAG_SETTER(EnableSharedMetadataAccessorCache) + FEATURE_FLAG_SETTER(DisableMissingDefaultColumnsInBulkUpsert) #undef FEATURE_FLAG_SETTER }; diff --git a/ydb/core/tx/tx_proxy/upload_rows_common_impl.h b/ydb/core/tx/tx_proxy/upload_rows_common_impl.h index 5487335bb858..f0112ccddb5c 100644 --- a/ydb/core/tx/tx_proxy/upload_rows_common_impl.h +++ b/ydb/core/tx/tx_proxy/upload_rows_common_impl.h @@ -407,6 +407,7 @@ class TUploadRowsBase : public TActorBootstrapped columnByName; THashSet keyColumnsLeft; THashSet notNullColumnsLeft = entry.NotNullColumns; + THashSet defaultColumnsLeft; SrcColumns.reserve(entry.Columns.size()); THashSet HasInternalConversion; @@ -426,6 +427,10 @@ class TUploadRowsBase : public TActorBootstrappedFeatureFlags.GetDisableMissingDefaultColumnsInBulkUpsert()) { + return TConclusionStatus::Fail(Sprintf("Missing default columns: %s", JoinSeq(", ", defaultColumnsLeft).c_str())); + } + + UploadCounters.OnMissingDefaultColumns(); + LOG_WARN_S(ctx, NKikimrServices::RPC_REQUEST, "Missing default columns: " << JoinSeq(", ", defaultColumnsLeft).c_str()); + } + TConclusionStatus res = TConclusionStatus::Success(); if (isColumnTable && HasAppData() && AppDataVerified().ColumnShardConfig.GetBulkUpsertRequireAllColumns()) { res = CheckRequiredColumns(entry, *reqColumns); diff --git a/ydb/core/tx/tx_proxy/upload_rows_counters.cpp b/ydb/core/tx/tx_proxy/upload_rows_counters.cpp index 9e39f37298cf..16fbd5ed4acc 100644 --- a/ydb/core/tx/tx_proxy/upload_rows_counters.cpp +++ b/ydb/core/tx/tx_proxy/upload_rows_counters.cpp @@ -99,5 +99,6 @@ TUploadCounters::TUploadCounters() WrittenBytes = TBase::GetDeriviative("Replies/WrittenBytes"); FailedBytes = TBase::GetDeriviative("Replies/FailedBytes"); RequestsBytes = TBase::GetDeriviative("Requests/Bytes"); + MissingDefaultColumnsCount = TBase::GetDeriviative("MissingDefaultColumns/Count"); } } diff --git a/ydb/core/tx/tx_proxy/upload_rows_counters.h b/ydb/core/tx/tx_proxy/upload_rows_counters.h index b71e94b3ab24..76d6cea85134 100644 --- a/ydb/core/tx/tx_proxy/upload_rows_counters.h +++ b/ydb/core/tx/tx_proxy/upload_rows_counters.h @@ -76,6 +76,8 @@ class TUploadCounters: public NColumnShard::TCommonCountersOwner { NMonitoring::TDynamicCounters::TCounterPtr FailedBytes; NMonitoring::TDynamicCounters::TCounterPtr RequestsBytes; + NMonitoring::TDynamicCounters::TCounterPtr MissingDefaultColumnsCount; + THashMap CodesCount; NMonitoring::TDynamicCounters::TCounterPtr GetCodeCounter(const TUploadStatus& status); @@ -142,6 +144,10 @@ class TUploadCounters: public NColumnShard::TCommonCountersOwner { PackageSizeCountByRecords->Collect(rowsCount); RequestsBytes->Add(requestBytes); } + + void OnMissingDefaultColumns() { + MissingDefaultColumnsCount->Inc(); + } }; } // namespace NKikimr