From fcc7c289da73331af24a0454504ec290c5253b80 Mon Sep 17 00:00:00 2001 From: Alexander Kotov Date: Wed, 22 Oct 2025 18:26:29 +0300 Subject: [PATCH 01/36] [*] refactoring --- .../pqtablet/partition/partition.cpp | 21 ++++++++++++------- .../persqueue/pqtablet/partition/partition.h | 4 +++- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/ydb/core/persqueue/pqtablet/partition/partition.cpp b/ydb/core/persqueue/pqtablet/partition/partition.cpp index bc994f2e9253..54357951c7b0 100644 --- a/ydb/core/persqueue/pqtablet/partition/partition.cpp +++ b/ydb/core/persqueue/pqtablet/partition/partition.cpp @@ -2664,14 +2664,14 @@ TPartition::EProcessResult TPartition::PreProcessUserActionOrTransaction(TSimple return EProcessResult::Continue; } t->Predicate.ConstructInPlace(true); - return PreProcessImmediateTx(t->ProposeTransaction->GetRecord()); + return PreProcessImmediateTx(*t); } else if (t->Tx) { // Distributed TX if (t->Predicate.Defined()) { // Predicate defined - either failed previously or Tx created with predicate defined. ReplyToProposeOrPredicate(t, true); return EProcessResult::Continue; } - result = BeginTransaction(*t->Tx, t->Predicate, t->Message); + result = BeginTransactionData(*t); if (t->Predicate.Defined()) { ReplyToProposeOrPredicate(t, true); } @@ -2680,7 +2680,7 @@ TPartition::EProcessResult TPartition::PreProcessUserActionOrTransaction(TSimple if (!FirstEvent) { return EProcessResult::Blocked; } - t->Predicate = BeginTransaction(*t->ProposeConfig); + t->Predicate = BeginTransactionConfig(*t); ChangingConfig = true; PendingPartitionConfig = GetPartitionConfig(t->ProposeConfig->Config); //Y_VERIFY_DEBUG_S(PendingPartitionConfig, "Partition " << Partition << " config not found"); @@ -2739,9 +2739,12 @@ bool TPartition::ExecUserActionOrTransaction(TSimpleSharedPtr& t, return true; } -TPartition::EProcessResult TPartition::BeginTransaction(const TEvPQ::TEvTxCalcPredicate& tx, - TMaybe& predicateOut, TString& issueMsg) +TPartition::EProcessResult TPartition::BeginTransactionData(TTransaction& t) { + const TEvPQ::TEvTxCalcPredicate& tx = *t.Tx; + TMaybe& predicateOut = t.Predicate; + TString& issueMsg = t.Message; + if (tx.ForcePredicateFalse) { predicateOut = false; return EProcessResult::Continue; @@ -2846,8 +2849,10 @@ TPartition::EProcessResult TPartition::BeginTransaction(const TEvPQ::TEvTxCalcPr return EProcessResult::Continue; } -bool TPartition::BeginTransaction(const TEvPQ::TEvProposePartitionConfig& event) +bool TPartition::BeginTransactionConfig(TTransaction& t) { + const TEvPQ::TEvProposePartitionConfig& event = *t.ProposeConfig; + ChangeConfig = MakeSimpleShared(TopicConverter, event.Config); @@ -3259,8 +3264,10 @@ void TPartition::ChangePlanStepAndTxId(ui64 step, ui64 txId) TxIdHasChanged = true; } -TPartition::EProcessResult TPartition::PreProcessImmediateTx(const NKikimrPQ::TEvProposeTransaction& tx) +TPartition::EProcessResult TPartition::PreProcessImmediateTx(TTransaction& t) { + const NKikimrPQ::TEvProposeTransaction& tx = t.ProposeTransaction->GetRecord(); + if (AffectedUsers.size() >= MAX_USERS) { return EProcessResult::Blocked; } diff --git a/ydb/core/persqueue/pqtablet/partition/partition.h b/ydb/core/persqueue/pqtablet/partition/partition.h index 751cbcf79fb4..e1d808516211 100644 --- a/ydb/core/persqueue/pqtablet/partition/partition.h +++ b/ydb/core/persqueue/pqtablet/partition/partition.h @@ -441,6 +441,7 @@ class TPartition : public TBaseTabletActor { THolder MakeCommitDone(ui64 step, ui64 txId); bool BeginTransaction(const TEvPQ::TEvProposePartitionConfig& event); + bool BeginTransactionConfig(TTransaction& t); void CommitTransaction(TSimpleSharedPtr& t); void RollbackTransaction(TSimpleSharedPtr& t); @@ -822,7 +823,7 @@ class TPartition : public TBaseTabletActor { void CommitUserAct(TEvPQ::TEvSetClientInfo& act); - [[nodiscard]] EProcessResult PreProcessImmediateTx(const NKikimrPQ::TEvProposeTransaction& tx); + [[nodiscard]] EProcessResult PreProcessImmediateTx(TTransaction& t); void ExecImmediateTx(TTransaction& tx); EProcessResult PreProcessRequest(TRegisterMessageGroupMsg& msg); @@ -837,6 +838,7 @@ class TPartition : public TBaseTabletActor { [[nodiscard]] EProcessResult BeginTransaction(const TEvPQ::TEvTxCalcPredicate& event, TMaybe& predicate, TString& issueMsg); + [[nodiscard]] EProcessResult BeginTransactionData(TTransaction& t); EProcessResult ApplyWriteInfoResponse(TTransaction& tx); From 8d9be2b278c7195729e44c2a024a606b81e2f483 Mon Sep 17 00:00:00 2001 From: Alexander Kotov Date: Wed, 22 Oct 2025 18:58:47 +0300 Subject: [PATCH 02/36] [/] unused code --- .../pqtablet/partition/partition.cpp | 20 ++++++++----------- .../persqueue/pqtablet/partition/partition.h | 12 ++--------- 2 files changed, 10 insertions(+), 22 deletions(-) diff --git a/ydb/core/persqueue/pqtablet/partition/partition.cpp b/ydb/core/persqueue/pqtablet/partition/partition.cpp index 54357951c7b0..ac7e98466f9e 100644 --- a/ydb/core/persqueue/pqtablet/partition/partition.cpp +++ b/ydb/core/persqueue/pqtablet/partition/partition.cpp @@ -2721,13 +2721,12 @@ bool TPartition::ExecUserActionOrTransaction(TSimpleSharedPtr& t, case ECommitState::Committed: break; } - const auto& ctx = ActorContext(); if (t->ChangeConfig) { PQ_ENSURE(!ChangeConfig); PQ_ENSURE(ChangingConfig); ChangeConfig = t->ChangeConfig; SendChangeConfigReply = t->SendReply; - BeginChangePartitionConfig(ChangeConfig->Config, ctx); + BeginChangePartitionConfig(ChangeConfig->Config); } else if (t->ProposeConfig) { PQ_ENSURE(ChangingConfig); ChangeConfig = MakeSimpleShared(TopicConverter, @@ -2956,7 +2955,6 @@ void TPartition::CommitWriteOperations(TTransaction& t) void TPartition::CommitTransaction(TSimpleSharedPtr& t) { - const auto& ctx = ActorContext(); if (t->Tx) { PQ_ENSURE(t->Predicate.Defined() && *t->Predicate); @@ -3001,7 +2999,7 @@ void TPartition::CommitTransaction(TSimpleSharedPtr& t) } else if (t->ProposeConfig) { PQ_ENSURE(t->Predicate.Defined() && *t->Predicate); - BeginChangePartitionConfig(t->ProposeConfig->Config, ctx); + BeginChangePartitionConfig(t->ProposeConfig->Config); ExecChangePartitionConfig(); ChangePlanStepAndTxId(t->ProposeConfig->Step, t->ProposeConfig->TxId); @@ -3033,8 +3031,7 @@ void TPartition::RollbackTransaction(TSimpleSharedPtr& t) } } -void TPartition::BeginChangePartitionConfig(const NKikimrPQ::TPQTabletConfig& config, - const TActorContext& ctx) +void TPartition::BeginChangePartitionConfig(const NKikimrPQ::TPQTabletConfig& config) { TSet hasReadRule; for (auto&& [consumer, info] : UsersInfoStorage->ViewAll()) { @@ -3057,7 +3054,7 @@ void TPartition::BeginChangePartitionConfig(const NKikimrPQ::TPQTabletConfig& co auto act = MakeHolder(0, consumer.GetName(), 0, "", 0, 0, 0, TActorId{}, TEvPQ::TEvSetClientInfo::ESCI_INIT_READ_RULE, rrGen); - auto res = PreProcessUserAct(*act, ctx); + auto res = PreProcessUserAct(*act); ChangeConfigActs.emplace_back(std::move(act)); PQ_ENSURE(res == EProcessResult::Continue); @@ -3070,7 +3067,7 @@ void TPartition::BeginChangePartitionConfig(const NKikimrPQ::TPQTabletConfig& co auto act = MakeHolder(0, consumer, 0, "", 0, 0, 0, TActorId{}, TEvPQ::TEvSetClientInfo::ESCI_DROP_READ_RULE, 0); - auto res = PreProcessUserAct(*act, ctx); + auto res = PreProcessUserAct(*act); PQ_ENSURE(res == EProcessResult::Continue); ChangeConfigActs.emplace_back(std::move(act)); } @@ -3402,7 +3399,7 @@ TPartition::EProcessResult TPartition::PreProcessUserActionOrTransaction(TSimple if (AffectedUsers.size() >= MAX_USERS) { return EProcessResult::Blocked; } - return PreProcessUserAct(*act, ActorContext()); + return PreProcessUserAct(*act); } bool TPartition::ExecUserActionOrTransaction( @@ -3463,9 +3460,8 @@ bool TPartition::ExecUserActionOrTransaction(TMessage& msg, TEvKeyValue::TEvRequ return true; } -TPartition::EProcessResult TPartition::PreProcessUserAct( - TEvPQ::TEvSetClientInfo& act, const TActorContext& -) { +TPartition::EProcessResult TPartition::PreProcessUserAct(TEvPQ::TEvSetClientInfo& act) +{ PQ_ENSURE(!KVWriteInProgress); const TString& user = act.ClientId; diff --git a/ydb/core/persqueue/pqtablet/partition/partition.h b/ydb/core/persqueue/pqtablet/partition/partition.h index e1d808516211..a94a9a5a5275 100644 --- a/ydb/core/persqueue/pqtablet/partition/partition.h +++ b/ydb/core/persqueue/pqtablet/partition/partition.h @@ -440,15 +440,13 @@ class TPartition : public TBaseTabletActor { const TString& reason); THolder MakeCommitDone(ui64 step, ui64 txId); - bool BeginTransaction(const TEvPQ::TEvProposePartitionConfig& event); bool BeginTransactionConfig(TTransaction& t); void CommitTransaction(TSimpleSharedPtr& t); void RollbackTransaction(TSimpleSharedPtr& t); - void BeginChangePartitionConfig(const NKikimrPQ::TPQTabletConfig& config, - const TActorContext& ctx); + void BeginChangePartitionConfig(const NKikimrPQ::TPQTabletConfig& config); void ExecChangePartitionConfig(); void OnProcessTxsAndUserActsWriteComplete(const TActorContext& ctx); @@ -808,18 +806,14 @@ class TPartition : public TBaseTabletActor { //TSimpleSharedPtr& GetCurrentTransaction(); EProcessResult PreProcessUserActionOrTransaction(TSimpleSharedPtr& event); - EProcessResult PreProcessUserActionOrTransaction(TSimpleSharedPtr& event); EProcessResult PreProcessUserActionOrTransaction(TSimpleSharedPtr& tx); EProcessResult PreProcessUserActionOrTransaction(TMessage& msg); bool ExecUserActionOrTransaction(TSimpleSharedPtr& event, TEvKeyValue::TEvRequest* request); - - bool ExecUserActionOrTransaction(TSimpleSharedPtr& event, - TEvKeyValue::TEvRequest* request); bool ExecUserActionOrTransaction(TSimpleSharedPtr& tx, TEvKeyValue::TEvRequest* request); bool ExecUserActionOrTransaction(TMessage& msg, TEvKeyValue::TEvRequest* request); - [[nodiscard]] EProcessResult PreProcessUserAct(TEvPQ::TEvSetClientInfo& act, const TActorContext& ctx); + [[nodiscard]] EProcessResult PreProcessUserAct(TEvPQ::TEvSetClientInfo& act); void CommitUserAct(TEvPQ::TEvSetClientInfo& act); @@ -836,8 +830,6 @@ class TPartition : public TBaseTabletActor { void ExecRequest(TSplitMessageGroupMsg& msg, ProcessParameters& parameters); bool ExecRequest(TWriteMsg& msg, ProcessParameters& parameters, TEvKeyValue::TEvRequest* request); - [[nodiscard]] EProcessResult BeginTransaction(const TEvPQ::TEvTxCalcPredicate& event, - TMaybe& predicate, TString& issueMsg); [[nodiscard]] EProcessResult BeginTransactionData(TTransaction& t); EProcessResult ApplyWriteInfoResponse(TTransaction& tx); From d7dc79f2bd3cda80c116841ec06453b7fbfde2d6 Mon Sep 17 00:00:00 2001 From: Alexander Kotov Date: Wed, 22 Oct 2025 20:16:04 +0300 Subject: [PATCH 03/36] [+] affectedSourceIdsAndConsumers --- .../pqtablet/partition/partition.cpp | 44 +++++++++++++------ .../persqueue/pqtablet/partition/partition.h | 42 +++++++++++++----- .../pqtablet/partition/partition_write.cpp | 27 +++++++++--- 3 files changed, 82 insertions(+), 31 deletions(-) diff --git a/ydb/core/persqueue/pqtablet/partition/partition.cpp b/ydb/core/persqueue/pqtablet/partition/partition.cpp index ac7e98466f9e..028b97e0094c 100644 --- a/ydb/core/persqueue/pqtablet/partition/partition.cpp +++ b/ydb/core/persqueue/pqtablet/partition/partition.cpp @@ -1549,7 +1549,11 @@ void TPartition::WriteInfoResponseHandler( ProcessTxsAndUserActs(ctx); } -TPartition::EProcessResult TPartition::ApplyWriteInfoResponse(TTransaction& tx) { +TPartition::EProcessResult TPartition::ApplyWriteInfoResponse(TTransaction& tx, + TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers) +{ + Y_UNUSED(affectedSourceIdsAndConsumers); + bool isImmediate = (tx.ProposeTransaction != nullptr); PQ_ENSURE(tx.WriteInfo); PQ_ENSURE(!tx.WriteInfoApplied); @@ -2338,9 +2342,6 @@ void TPartition::ContinueProcessTxsAndUserActs(const TActorContext&) BatchingState = ETxBatchingState::Finishing; return; } - auto visitor = [this](auto& event) { - return this->PreProcessUserActionOrTransaction(event); - }; while (CanProcessUserActionAndTransactionEvents() && !UserActionAndTransactionEvents.empty()) { if (ChangingConfig) { BatchingState = ETxBatchingState::Finishing; @@ -2350,6 +2351,9 @@ void TPartition::ContinueProcessTxsAndUserActs(const TActorContext&) if (TMessage* msg = std::get_if(&front.Event); msg && msg->WaitPreviousWriteSpan) { msg->WaitPreviousWriteSpan.End(); } + auto visitor = [this, &front](auto& event) { + return this->PreProcessUserActionOrTransaction(event, front.AffectedSourceIdsAndConsumers); + }; switch (std::visit(visitor, front.Event)) { case EProcessResult::Continue: MoveUserActOrTxToCommitState(); @@ -2639,8 +2643,11 @@ void TPartition::AnswerCurrentReplies(const TActorContext& ctx) Replies.clear(); } -TPartition::EProcessResult TPartition::PreProcessUserActionOrTransaction(TSimpleSharedPtr& t) +TPartition::EProcessResult TPartition::PreProcessUserActionOrTransaction(TSimpleSharedPtr& t, + TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers) { + Y_UNUSED(affectedSourceIdsAndConsumers); + auto span = t->CalcPredicateSpan.CreateChild(TWilsonTopic::TopicTopLevel, "Topic.Partition.PreProcess", NWilson::EFlags::AUTO_END); @@ -2651,7 +2658,7 @@ TPartition::EProcessResult TPartition::PreProcessUserActionOrTransaction(TSimple return EProcessResult::NotReady; } if (t->WriteInfo && !t->WriteInfoApplied) { //Recieved write info but not applied - result = ApplyWriteInfoResponse(*t); + result = ApplyWriteInfoResponse(*t, affectedSourceIdsAndConsumers); if (!t->WriteInfoApplied) { // Tried to apply write info but couldn't - TX must be blocked. PQ_LOG_TX_D("The TxId " << t->GetTxId() << " must be blocked"); PQ_ENSURE(result != EProcessResult::Continue); @@ -2671,7 +2678,7 @@ TPartition::EProcessResult TPartition::PreProcessUserActionOrTransaction(TSimple ReplyToProposeOrPredicate(t, true); return EProcessResult::Continue; } - result = BeginTransactionData(*t); + result = BeginTransactionData(*t, affectedSourceIdsAndConsumers); if (t->Predicate.Defined()) { ReplyToProposeOrPredicate(t, true); } @@ -2738,8 +2745,11 @@ bool TPartition::ExecUserActionOrTransaction(TSimpleSharedPtr& t, return true; } -TPartition::EProcessResult TPartition::BeginTransactionData(TTransaction& t) +TPartition::EProcessResult TPartition::BeginTransactionData(TTransaction& t, + TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers) { + Y_UNUSED(affectedSourceIdsAndConsumers); + const TEvPQ::TEvTxCalcPredicate& tx = *t.Tx; TMaybe& predicateOut = t.Predicate; TString& issueMsg = t.Message; @@ -3394,11 +3404,15 @@ void TPartition::ExecImmediateTx(TTransaction& t) return; } -TPartition::EProcessResult TPartition::PreProcessUserActionOrTransaction(TSimpleSharedPtr& act) +TPartition::EProcessResult TPartition::PreProcessUserActionOrTransaction(TSimpleSharedPtr& act, + TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers) { + Y_UNUSED(affectedSourceIdsAndConsumers); + if (AffectedUsers.size() >= MAX_USERS) { return EProcessResult::Blocked; } + return PreProcessUserAct(*act); } @@ -3409,7 +3423,8 @@ bool TPartition::ExecUserActionOrTransaction( return true; } -TPartition::EProcessResult TPartition::PreProcessUserActionOrTransaction(TMessage& msg) +TPartition::EProcessResult TPartition::PreProcessUserActionOrTransaction(TMessage& msg, + TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers) { if (WriteCycleSize >= MAX_WRITE_CYCLE_SIZE) { return EProcessResult::Blocked; @@ -3417,16 +3432,17 @@ TPartition::EProcessResult TPartition::PreProcessUserActionOrTransaction(TMessag auto result = EProcessResult::Continue; if (msg.IsWrite()) { - result = PreProcessRequest(msg.GetWrite()); + result = PreProcessRequest(msg.GetWrite(), affectedSourceIdsAndConsumers); } else if (msg.IsRegisterMessageGroup()) { - result = PreProcessRequest(msg.GetRegisterMessageGroup()); + result = PreProcessRequest(msg.GetRegisterMessageGroup(), affectedSourceIdsAndConsumers); } else if (msg.IsDeregisterMessageGroup()) { - result = PreProcessRequest(msg.GetDeregisterMessageGroup()); + result = PreProcessRequest(msg.GetDeregisterMessageGroup(), affectedSourceIdsAndConsumers); } else if (msg.IsSplitMessageGroup()) { - result = PreProcessRequest(msg.GetSplitMessageGroup()); + result = PreProcessRequest(msg.GetSplitMessageGroup(), affectedSourceIdsAndConsumers); } else { PQ_ENSURE(msg.IsOwnership()); } + return result; } diff --git a/ydb/core/persqueue/pqtablet/partition/partition.h b/ydb/core/persqueue/pqtablet/partition/partition.h index a94a9a5a5275..5a316a46e18f 100644 --- a/ydb/core/persqueue/pqtablet/partition/partition.h +++ b/ydb/core/persqueue/pqtablet/partition/partition.h @@ -805,9 +805,19 @@ class TPartition : public TBaseTabletActor { // template T& GetCurrentEvent(); //TSimpleSharedPtr& GetCurrentTransaction(); - EProcessResult PreProcessUserActionOrTransaction(TSimpleSharedPtr& event); - EProcessResult PreProcessUserActionOrTransaction(TSimpleSharedPtr& tx); - EProcessResult PreProcessUserActionOrTransaction(TMessage& msg); + struct TAffectedSourceIdsAndConsumers { + TVector TxWriteSourcesIds; + TVector WriteSourcesIds; + TVector TxReadConsumers; + TVector ReadConsumers; + }; + + EProcessResult PreProcessUserActionOrTransaction(TSimpleSharedPtr& event, + TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers); + EProcessResult PreProcessUserActionOrTransaction(TSimpleSharedPtr& tx, + TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers); + EProcessResult PreProcessUserActionOrTransaction(TMessage& msg, + TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers); bool ExecUserActionOrTransaction(TSimpleSharedPtr& event, TEvKeyValue::TEvRequest* request); bool ExecUserActionOrTransaction(TSimpleSharedPtr& tx, TEvKeyValue::TEvRequest* request); @@ -820,19 +830,25 @@ class TPartition : public TBaseTabletActor { [[nodiscard]] EProcessResult PreProcessImmediateTx(TTransaction& t); void ExecImmediateTx(TTransaction& tx); - EProcessResult PreProcessRequest(TRegisterMessageGroupMsg& msg); - EProcessResult PreProcessRequest(TDeregisterMessageGroupMsg& msg); - EProcessResult PreProcessRequest(TSplitMessageGroupMsg& msg); - EProcessResult PreProcessRequest(TWriteMsg& msg); + EProcessResult PreProcessRequest(TRegisterMessageGroupMsg& msg, + TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers); + EProcessResult PreProcessRequest(TDeregisterMessageGroupMsg& msg, + TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers); + EProcessResult PreProcessRequest(TSplitMessageGroupMsg& msg, + TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers); + EProcessResult PreProcessRequest(TWriteMsg& msg, + TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers); void ExecRequest(TRegisterMessageGroupMsg& msg, ProcessParameters& parameters); void ExecRequest(TDeregisterMessageGroupMsg& msg, ProcessParameters& parameters); void ExecRequest(TSplitMessageGroupMsg& msg, ProcessParameters& parameters); bool ExecRequest(TWriteMsg& msg, ProcessParameters& parameters, TEvKeyValue::TEvRequest* request); - [[nodiscard]] EProcessResult BeginTransactionData(TTransaction& t); + [[nodiscard]] EProcessResult BeginTransactionData(TTransaction& t, + TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers); - EProcessResult ApplyWriteInfoResponse(TTransaction& tx); + EProcessResult ApplyWriteInfoResponse(TTransaction& tx, + TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers); bool FirstEvent = true; bool HaveWriteMsg = false; @@ -858,9 +874,6 @@ class TPartition : public TBaseTabletActor { // user actions and transactions // struct TUserActionAndTransactionEvent { - std::variant, // user actions - TSimpleSharedPtr, // distributed transaction or update config - TMessage> Event; TUserActionAndTransactionEvent(TSimpleSharedPtr&& transaction) : Event(std::move(transaction)) {} @@ -870,6 +883,11 @@ class TPartition : public TBaseTabletActor { TUserActionAndTransactionEvent(TMessage&& message) : Event(std::move(message)) {} + + std::variant, // user actions + TSimpleSharedPtr, // distributed transaction or update config + TMessage> Event; + TAffectedSourceIdsAndConsumers AffectedSourceIdsAndConsumers; }; std::deque UserActionAndTransactionEvents; diff --git a/ydb/core/persqueue/pqtablet/partition/partition_write.cpp b/ydb/core/persqueue/pqtablet/partition/partition_write.cpp index bbb1cd9b9e2c..ebccc68f45d2 100644 --- a/ydb/core/persqueue/pqtablet/partition/partition_write.cpp +++ b/ydb/core/persqueue/pqtablet/partition/partition_write.cpp @@ -896,7 +896,11 @@ void TPartition::CancelOneWriteOnWrite(const TActorContext& ctx, StartProcessChangeOwnerRequests(ctx); } -TPartition::EProcessResult TPartition::PreProcessRequest(TRegisterMessageGroupMsg& msg) { +TPartition::EProcessResult TPartition::PreProcessRequest(TRegisterMessageGroupMsg& msg, + TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers) +{ + Y_UNUSED(affectedSourceIdsAndConsumers); + if (!CanWrite()) { ScheduleReplyError(msg.Cookie, false, InactivePartitionErrorCode, TStringBuilder() << "Write to inactive partition " << Partition.OriginalPartitionId); @@ -928,7 +932,11 @@ void TPartition::ExecRequest(TRegisterMessageGroupMsg& msg, ProcessParameters& p parameters.SourceIdBatch.RegisterSourceId(body.SourceId, body.SeqNo, parameters.CurOffset, CurrentTimestamp, std::move(keyRange)); } -TPartition::EProcessResult TPartition::PreProcessRequest(TDeregisterMessageGroupMsg& msg) { +TPartition::EProcessResult TPartition::PreProcessRequest(TDeregisterMessageGroupMsg& msg, + TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers) +{ + Y_UNUSED(affectedSourceIdsAndConsumers); + if (!CanWrite()) { ScheduleReplyError(msg.Cookie, false, InactivePartitionErrorCode, TStringBuilder() << "Write to inactive partition " << Partition.OriginalPartitionId); @@ -951,7 +959,11 @@ void TPartition::ExecRequest(TDeregisterMessageGroupMsg& msg, ProcessParameters& } -TPartition::EProcessResult TPartition::PreProcessRequest(TSplitMessageGroupMsg& msg) { +TPartition::EProcessResult TPartition::PreProcessRequest(TSplitMessageGroupMsg& msg, + TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers) +{ + Y_UNUSED(affectedSourceIdsAndConsumers); + if (!CanWrite()) { ScheduleReplyError(msg.Cookie, false, InactivePartitionErrorCode, TStringBuilder() << "Write to inactive partition " << Partition.OriginalPartitionId); @@ -975,8 +987,8 @@ TPartition::EProcessResult TPartition::PreProcessRequest(TSplitMessageGroupMsg& } WriteAffectedSourcesIds.insert(body.SourceId); } - return EProcessResult::Continue; + return EProcessResult::Continue; } @@ -996,7 +1008,11 @@ void TPartition::ExecRequest(TSplitMessageGroupMsg& msg, ProcessParameters& para } } -TPartition::EProcessResult TPartition::PreProcessRequest(TWriteMsg& p) { +TPartition::EProcessResult TPartition::PreProcessRequest(TWriteMsg& p, + TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers) +{ + Y_UNUSED(affectedSourceIdsAndConsumers); + if (!CanWrite()) { WriteInflightSize -= p.Msg.Data.size(); ScheduleReplyError(p.Cookie, false, InactivePartitionErrorCode, @@ -1023,6 +1039,7 @@ TPartition::EProcessResult TPartition::PreProcessRequest(TWriteMsg& p) { } } WriteAffectedSourcesIds.insert(p.Msg.SourceId); + return EProcessResult::Continue; } From d2531fd45a5def23e4f1e353aa9bf69d2ec405a8 Mon Sep 17 00:00:00 2001 From: Alexander Kotov Date: Thu, 23 Oct 2025 12:15:46 +0300 Subject: [PATCH 04/36] [+] use WriteSourcesIds --- .../pqtablet/partition/partition.cpp | 20 ++++++++++++++++--- .../persqueue/pqtablet/partition/partition.h | 3 +++ .../pqtablet/partition/partition_write.cpp | 12 +++++------ 3 files changed, 25 insertions(+), 10 deletions(-) diff --git a/ydb/core/persqueue/pqtablet/partition/partition.cpp b/ydb/core/persqueue/pqtablet/partition/partition.cpp index 028b97e0094c..0aa4ee2b4560 100644 --- a/ydb/core/persqueue/pqtablet/partition/partition.cpp +++ b/ydb/core/persqueue/pqtablet/partition/partition.cpp @@ -1552,8 +1552,6 @@ void TPartition::WriteInfoResponseHandler( TPartition::EProcessResult TPartition::ApplyWriteInfoResponse(TTransaction& tx, TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers) { - Y_UNUSED(affectedSourceIdsAndConsumers); - bool isImmediate = (tx.ProposeTransaction != nullptr); PQ_ENSURE(tx.WriteInfo); PQ_ENSURE(!tx.WriteInfoApplied); @@ -1580,7 +1578,7 @@ TPartition::EProcessResult TPartition::ApplyWriteInfoResponse(TTransaction& tx, break; } if (isImmediate) { - WriteAffectedSourcesIds.insert(s.first); + affectedSourceIdsAndConsumers.WriteSourcesIds.push_back(s.first); } else { if (WriteAffectedSourcesIds.contains(s.first)) { PQ_LOG_TX_D("WriteAffectedSourcesIds contains SourceId " << s.first << ". TxId " << tx.GetTxId()); @@ -2382,8 +2380,21 @@ void TPartition::ContinueProcessTxsAndUserActs(const TActorContext&) } +void TPartition::AppendAffectedSourceIdsAndConsumers(const TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers) +{ + AppendWriteAffectedSourceIds(affectedSourceIdsAndConsumers); +} + +void TPartition::AppendWriteAffectedSourceIds(const TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers) +{ + for (const auto& sourceId : affectedSourceIdsAndConsumers.WriteSourcesIds) { + WriteAffectedSourcesIds.insert(sourceId); + } +} + void TPartition::MoveUserActOrTxToCommitState() { auto& front = UserActionAndTransactionEvents.front(); + AppendAffectedSourceIdsAndConsumers(front.AffectedSourceIdsAndConsumers); UserActionAndTxPendingCommit.push_back(std::move(front)); UserActionAndTransactionEvents.pop_front(); } @@ -3486,8 +3497,10 @@ TPartition::EProcessResult TPartition::PreProcessUserAct(TEvPQ::TEvSetClientInfo return EProcessResult::Blocked; } } + WriteKeysSizeEstimate += 1; SetOffsetAffectedConsumers.insert(user); + return EProcessResult::Continue; } @@ -4435,4 +4448,5 @@ void TPartition::ResetDetailedMetrics() { BytesWrittenPerPartition.Reset(); MessagesWrittenPerPartition.Reset(); } + } // namespace NKikimr::NPQ diff --git a/ydb/core/persqueue/pqtablet/partition/partition.h b/ydb/core/persqueue/pqtablet/partition/partition.h index 5a316a46e18f..e21b52759f16 100644 --- a/ydb/core/persqueue/pqtablet/partition/partition.h +++ b/ydb/core/persqueue/pqtablet/partition/partition.h @@ -812,6 +812,9 @@ class TPartition : public TBaseTabletActor { TVector ReadConsumers; }; + void AppendAffectedSourceIdsAndConsumers(const TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers); + void AppendWriteAffectedSourceIds(const TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers); + EProcessResult PreProcessUserActionOrTransaction(TSimpleSharedPtr& event, TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers); EProcessResult PreProcessUserActionOrTransaction(TSimpleSharedPtr& tx, diff --git a/ydb/core/persqueue/pqtablet/partition/partition_write.cpp b/ydb/core/persqueue/pqtablet/partition/partition_write.cpp index ebccc68f45d2..0af562b6290f 100644 --- a/ydb/core/persqueue/pqtablet/partition/partition_write.cpp +++ b/ydb/core/persqueue/pqtablet/partition/partition_write.cpp @@ -899,8 +899,6 @@ void TPartition::CancelOneWriteOnWrite(const TActorContext& ctx, TPartition::EProcessResult TPartition::PreProcessRequest(TRegisterMessageGroupMsg& msg, TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers) { - Y_UNUSED(affectedSourceIdsAndConsumers); - if (!CanWrite()) { ScheduleReplyError(msg.Cookie, false, InactivePartitionErrorCode, TStringBuilder() << "Write to inactive partition " << Partition.OriginalPartitionId); @@ -915,7 +913,7 @@ TPartition::EProcessResult TPartition::PreProcessRequest(TRegisterMessageGroupMs if (TxAffectedSourcesIds.contains(msg.Body.SourceId)) { return EProcessResult::Blocked; } - WriteAffectedSourcesIds.insert(msg.Body.SourceId); + affectedSourceIdsAndConsumers.WriteSourcesIds.push_back(msg.Body.SourceId); return EProcessResult::Continue; } @@ -950,7 +948,7 @@ TPartition::EProcessResult TPartition::PreProcessRequest(TDeregisterMessageGroup if (TxAffectedSourcesIds.contains(msg.Body.SourceId)) { return EProcessResult::Blocked; } - WriteAffectedSourcesIds.insert(msg.Body.SourceId); + affectedSourceIdsAndConsumers.WriteSourcesIds.push_back(msg.Body.SourceId); return EProcessResult::Continue; } @@ -979,13 +977,13 @@ TPartition::EProcessResult TPartition::PreProcessRequest(TSplitMessageGroupMsg& if (TxAffectedSourcesIds.contains(body.SourceId)) { return EProcessResult::Blocked; } - WriteAffectedSourcesIds.insert(body.SourceId); + affectedSourceIdsAndConsumers.WriteSourcesIds.push_back(body.SourceId); } for (auto& body : msg.Deregistrations) { if (TxAffectedSourcesIds.contains(body.SourceId)) { return EProcessResult::Blocked; } - WriteAffectedSourcesIds.insert(body.SourceId); + affectedSourceIdsAndConsumers.WriteSourcesIds.push_back(body.SourceId); } return EProcessResult::Continue; @@ -1038,7 +1036,7 @@ TPartition::EProcessResult TPartition::PreProcessRequest(TWriteMsg& p, return EProcessResult::Blocked; } } - WriteAffectedSourcesIds.insert(p.Msg.SourceId); + affectedSourceIdsAndConsumers.WriteSourcesIds.push_back(p.Msg.SourceId); return EProcessResult::Continue; } From 67ff3341c29621d5560bea54dc342e0176bc04fc Mon Sep 17 00:00:00 2001 From: Alexander Kotov Date: Thu, 23 Oct 2025 12:48:58 +0300 Subject: [PATCH 05/36] [+] use TxWriteSourcesIds --- .../pqtablet/partition/partition.cpp | 23 +++++++++++++++---- .../persqueue/pqtablet/partition/partition.h | 1 + 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/ydb/core/persqueue/pqtablet/partition/partition.cpp b/ydb/core/persqueue/pqtablet/partition/partition.cpp index 0aa4ee2b4560..7e593997548b 100644 --- a/ydb/core/persqueue/pqtablet/partition/partition.cpp +++ b/ydb/core/persqueue/pqtablet/partition/partition.cpp @@ -1570,22 +1570,23 @@ TPartition::EProcessResult TPartition::ApplyWriteInfoResponse(TTransaction& tx, EProcessResult ret = EProcessResult::Continue; const auto& knownSourceIds = SourceIdStorage.GetInMemorySourceIds(); - THashSet txSourceIds; - for (auto& s : srcIdInfo) { + TVector txSourceIds; + for (const auto& s : srcIdInfo) { if (TxAffectedSourcesIds.contains(s.first)) { PQ_LOG_TX_D("TxAffectedSourcesIds contains SourceId " << s.first << ". TxId " << tx.GetTxId()); ret = EProcessResult::Blocked; break; } if (isImmediate) { - affectedSourceIdsAndConsumers.WriteSourcesIds.push_back(s.first); + txSourceIds.push_back(s.first); + LOG_D("TxId " << tx.GetTxId() << " affect SourceId " << s.first); } else { if (WriteAffectedSourcesIds.contains(s.first)) { PQ_LOG_TX_D("WriteAffectedSourcesIds contains SourceId " << s.first << ". TxId " << tx.GetTxId()); ret = EProcessResult::Blocked; break; } - txSourceIds.insert(s.first); + txSourceIds.push_back(s.first); LOG_D("TxId " << tx.GetTxId() << " affect SourceId " << s.first); } @@ -1619,10 +1620,14 @@ TPartition::EProcessResult TPartition::ApplyWriteInfoResponse(TTransaction& tx, } } } + if (ret == EProcessResult::Continue && tx.Predicate.GetOrElse(true)) { - TxAffectedSourcesIds.insert(txSourceIds.begin(), txSourceIds.end()); + auto& sourceIds = + (isImmediate ? affectedSourceIdsAndConsumers.WriteSourcesIds : affectedSourceIdsAndConsumers.TxWriteSourcesIds); + sourceIds = std::move(txSourceIds); tx.WriteInfoApplied = true; + WriteKeysSizeEstimate += tx.WriteInfo->BodyKeys.size(); WriteKeysSizeEstimate += tx.WriteInfo->SrcIdInfo.size(); } @@ -2382,9 +2387,17 @@ void TPartition::ContinueProcessTxsAndUserActs(const TActorContext&) void TPartition::AppendAffectedSourceIdsAndConsumers(const TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers) { + AppendTxWriteAffectedSourceIds(affectedSourceIdsAndConsumers); AppendWriteAffectedSourceIds(affectedSourceIdsAndConsumers); } +void TPartition::AppendTxWriteAffectedSourceIds(const TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers) +{ + for (const auto& sourceId : affectedSourceIdsAndConsumers.TxWriteSourcesIds) { + TxAffectedSourcesIds.insert(sourceId); + } +} + void TPartition::AppendWriteAffectedSourceIds(const TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers) { for (const auto& sourceId : affectedSourceIdsAndConsumers.WriteSourcesIds) { diff --git a/ydb/core/persqueue/pqtablet/partition/partition.h b/ydb/core/persqueue/pqtablet/partition/partition.h index e21b52759f16..9f3b38cbfcf2 100644 --- a/ydb/core/persqueue/pqtablet/partition/partition.h +++ b/ydb/core/persqueue/pqtablet/partition/partition.h @@ -813,6 +813,7 @@ class TPartition : public TBaseTabletActor { }; void AppendAffectedSourceIdsAndConsumers(const TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers); + void AppendTxWriteAffectedSourceIds(const TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers); void AppendWriteAffectedSourceIds(const TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers); EProcessResult PreProcessUserActionOrTransaction(TSimpleSharedPtr& event, From 02cb16ae9ef9b0e17b8c271cb5c9abee08b48daf Mon Sep 17 00:00:00 2001 From: Alexander Kotov Date: Thu, 23 Oct 2025 12:52:37 +0300 Subject: [PATCH 06/36] [/] unused code --- ydb/core/persqueue/pqtablet/partition/partition.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/ydb/core/persqueue/pqtablet/partition/partition.cpp b/ydb/core/persqueue/pqtablet/partition/partition.cpp index 7e593997548b..1148dcef0432 100644 --- a/ydb/core/persqueue/pqtablet/partition/partition.cpp +++ b/ydb/core/persqueue/pqtablet/partition/partition.cpp @@ -2670,8 +2670,6 @@ void TPartition::AnswerCurrentReplies(const TActorContext& ctx) TPartition::EProcessResult TPartition::PreProcessUserActionOrTransaction(TSimpleSharedPtr& t, TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers) { - Y_UNUSED(affectedSourceIdsAndConsumers); - auto span = t->CalcPredicateSpan.CreateChild(TWilsonTopic::TopicTopLevel, "Topic.Partition.PreProcess", NWilson::EFlags::AUTO_END); From 3169fb136186ebd94afab891fbadb6df55afda61 Mon Sep 17 00:00:00 2001 From: Alexander Kotov Date: Thu, 23 Oct 2025 13:06:57 +0300 Subject: [PATCH 07/36] [*] refactoring --- ydb/core/persqueue/pqtablet/partition/partition.cpp | 7 +++++-- ydb/core/persqueue/pqtablet/partition/partition.h | 3 ++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/ydb/core/persqueue/pqtablet/partition/partition.cpp b/ydb/core/persqueue/pqtablet/partition/partition.cpp index 1148dcef0432..67c8da629d44 100644 --- a/ydb/core/persqueue/pqtablet/partition/partition.cpp +++ b/ydb/core/persqueue/pqtablet/partition/partition.cpp @@ -2693,7 +2693,7 @@ TPartition::EProcessResult TPartition::PreProcessUserActionOrTransaction(TSimple return EProcessResult::Continue; } t->Predicate.ConstructInPlace(true); - return PreProcessImmediateTx(*t); + return PreProcessImmediateTx(*t, affectedSourceIdsAndConsumers); } else if (t->Tx) { // Distributed TX if (t->Predicate.Defined()) { // Predicate defined - either failed previously or Tx created with predicate defined. @@ -3293,8 +3293,11 @@ void TPartition::ChangePlanStepAndTxId(ui64 step, ui64 txId) TxIdHasChanged = true; } -TPartition::EProcessResult TPartition::PreProcessImmediateTx(TTransaction& t) +TPartition::EProcessResult TPartition::PreProcessImmediateTx(TTransaction& t, + TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers) { + Y_UNUSED(affectedSourceIdsAndConsumers); + const NKikimrPQ::TEvProposeTransaction& tx = t.ProposeTransaction->GetRecord(); if (AffectedUsers.size() >= MAX_USERS) { diff --git a/ydb/core/persqueue/pqtablet/partition/partition.h b/ydb/core/persqueue/pqtablet/partition/partition.h index 9f3b38cbfcf2..2f939cd7fa54 100644 --- a/ydb/core/persqueue/pqtablet/partition/partition.h +++ b/ydb/core/persqueue/pqtablet/partition/partition.h @@ -831,7 +831,8 @@ class TPartition : public TBaseTabletActor { void CommitUserAct(TEvPQ::TEvSetClientInfo& act); - [[nodiscard]] EProcessResult PreProcessImmediateTx(TTransaction& t); + [[nodiscard]] EProcessResult PreProcessImmediateTx(TTransaction& t, + TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers); void ExecImmediateTx(TTransaction& tx); EProcessResult PreProcessRequest(TRegisterMessageGroupMsg& msg, From e5e7539e77d638bfd43f8fdfa4362d8597c7a2ac Mon Sep 17 00:00:00 2001 From: Alexander Kotov Date: Thu, 23 Oct 2025 13:27:23 +0300 Subject: [PATCH 08/36] [+] use TxReadConsumers --- .../persqueue/pqtablet/partition/partition.cpp | 16 ++++++++++++---- .../persqueue/pqtablet/partition/partition.h | 1 + 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/ydb/core/persqueue/pqtablet/partition/partition.cpp b/ydb/core/persqueue/pqtablet/partition/partition.cpp index 67c8da629d44..4c752cc4453d 100644 --- a/ydb/core/persqueue/pqtablet/partition/partition.cpp +++ b/ydb/core/persqueue/pqtablet/partition/partition.cpp @@ -2389,6 +2389,7 @@ void TPartition::AppendAffectedSourceIdsAndConsumers(const TAffectedSourceIdsAnd { AppendTxWriteAffectedSourceIds(affectedSourceIdsAndConsumers); AppendWriteAffectedSourceIds(affectedSourceIdsAndConsumers); + AppendTxReadAffectedConsumers(affectedSourceIdsAndConsumers); } void TPartition::AppendTxWriteAffectedSourceIds(const TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers) @@ -2405,6 +2406,13 @@ void TPartition::AppendWriteAffectedSourceIds(const TAffectedSourceIdsAndConsume } } +void TPartition::AppendTxReadAffectedConsumers(const TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers) +{ + for (const auto& consumer : affectedSourceIdsAndConsumers.TxReadConsumers) { + TxAffectedConsumers.insert(consumer); + } +} + void TPartition::MoveUserActOrTxToCommitState() { auto& front = UserActionAndTransactionEvents.front(); AppendAffectedSourceIdsAndConsumers(front.AffectedSourceIdsAndConsumers); @@ -2781,9 +2789,9 @@ TPartition::EProcessResult TPartition::BeginTransactionData(TTransaction& t, return EProcessResult::Continue; } - THashSet consumers; + TVector consumers; bool result = true; - for (auto& operation : tx.Operations) { + for (const auto& operation : tx.Operations) { const TString& consumer = operation.GetConsumer(); if (TxAffectedConsumers.contains(consumer)) { PQ_LOG_TX_D("TxAffectedConsumers contains consumer " << consumer << ". TxId " << tx.TxId); @@ -2868,13 +2876,13 @@ TPartition::EProcessResult TPartition::BeginTransactionData(TTransaction& t, } break; } - consumers.insert(consumer); + consumers.push_back(consumer); PQ_LOG_TX_D("TxId " << tx.TxId << " affect consumer " << consumer); } } if (result) { - TxAffectedConsumers.insert(consumers.begin(), consumers.end()); + affectedSourceIdsAndConsumers.TxReadConsumers = std::move(consumers); } predicateOut = result; return EProcessResult::Continue; diff --git a/ydb/core/persqueue/pqtablet/partition/partition.h b/ydb/core/persqueue/pqtablet/partition/partition.h index 2f939cd7fa54..01f61bef5a73 100644 --- a/ydb/core/persqueue/pqtablet/partition/partition.h +++ b/ydb/core/persqueue/pqtablet/partition/partition.h @@ -815,6 +815,7 @@ class TPartition : public TBaseTabletActor { void AppendAffectedSourceIdsAndConsumers(const TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers); void AppendTxWriteAffectedSourceIds(const TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers); void AppendWriteAffectedSourceIds(const TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers); + void AppendTxReadAffectedConsumers(const TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers); EProcessResult PreProcessUserActionOrTransaction(TSimpleSharedPtr& event, TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers); From fc3c9d8629b7360133baed04d1bc3b4baff2d84e Mon Sep 17 00:00:00 2001 From: Alexander Kotov Date: Thu, 23 Oct 2025 17:47:51 +0300 Subject: [PATCH 09/36] [+] use ReadConsumers --- .../pqtablet/partition/partition.cpp | 62 +++++++++++-------- .../persqueue/pqtablet/partition/partition.h | 8 ++- .../pqtablet/partition/partition_write.cpp | 6 -- 3 files changed, 41 insertions(+), 35 deletions(-) diff --git a/ydb/core/persqueue/pqtablet/partition/partition.cpp b/ydb/core/persqueue/pqtablet/partition/partition.cpp index 4c752cc4453d..f302b2384ee2 100644 --- a/ydb/core/persqueue/pqtablet/partition/partition.cpp +++ b/ydb/core/persqueue/pqtablet/partition/partition.cpp @@ -2390,6 +2390,7 @@ void TPartition::AppendAffectedSourceIdsAndConsumers(const TAffectedSourceIdsAnd AppendTxWriteAffectedSourceIds(affectedSourceIdsAndConsumers); AppendWriteAffectedSourceIds(affectedSourceIdsAndConsumers); AppendTxReadAffectedConsumers(affectedSourceIdsAndConsumers); + AppendReadAffectedConsumers(affectedSourceIdsAndConsumers); } void TPartition::AppendTxWriteAffectedSourceIds(const TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers) @@ -2413,6 +2414,13 @@ void TPartition::AppendTxReadAffectedConsumers(const TAffectedSourceIdsAndConsum } } +void TPartition::AppendReadAffectedConsumers(const TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers) +{ + for (const auto& consumer : affectedSourceIdsAndConsumers.ReadConsumers) { + SetOffsetAffectedConsumers.insert(consumer); + } +} + void TPartition::MoveUserActOrTxToCommitState() { auto& front = UserActionAndTransactionEvents.front(); AppendAffectedSourceIdsAndConsumers(front.AffectedSourceIdsAndConsumers); @@ -2427,9 +2435,6 @@ void TPartition::ProcessCommitQueue() { if (!PersistRequest) { PersistRequest = MakeHolder(); } - auto visitor = [this, request = PersistRequest.Get()](auto& event) { - return this->ExecUserActionOrTransaction(event, request); - }; while (!UserActionAndTxPendingCommit.empty()) { auto& front = UserActionAndTxPendingCommit.front(); auto state = ECommitState::Committed; @@ -2446,6 +2451,9 @@ void TPartition::ProcessCommitQueue() { } auto event = std::move(front.Event); UserActionAndTxPendingCommit.pop_front(); + auto visitor = [this, request = PersistRequest.Get()](auto& event) { + return this->ExecUserActionOrTransaction(event, request); + }; std::visit(visitor, event); } if (UserActionAndTxPendingCommit.empty()) { @@ -2739,7 +2747,8 @@ TPartition::EProcessResult TPartition::PreProcessUserActionOrTransaction(TSimple return result; } -bool TPartition::ExecUserActionOrTransaction(TSimpleSharedPtr& t, TEvKeyValue::TEvRequest*) +bool TPartition::ExecUserActionOrTransaction(TSimpleSharedPtr& t, + TEvKeyValue::TEvRequest*) { auto span = t->CommitSpan.CreateChild(TWilsonTopic::TopicTopLevel, "Topic.Partition.Process", @@ -2778,8 +2787,6 @@ bool TPartition::ExecUserActionOrTransaction(TSimpleSharedPtr& t, TPartition::EProcessResult TPartition::BeginTransactionData(TTransaction& t, TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers) { - Y_UNUSED(affectedSourceIdsAndConsumers); - const TEvPQ::TEvTxCalcPredicate& tx = *t.Tx; TMaybe& predicateOut = t.Predicate; TString& issueMsg = t.Message; @@ -3078,7 +3085,7 @@ void TPartition::BeginChangePartitionConfig(const NKikimrPQ::TPQTabletConfig& co hasReadRule.insert(consumer); } - for (auto& consumer : config.GetConsumers()) { + for (const auto& consumer : config.GetConsumers()) { auto& userInfo = GetOrCreatePendingUser(consumer.GetName(), 0); TInstant ts = TInstant::MilliSeconds(consumer.GetReadFromTimestampsMs()); @@ -3094,21 +3101,22 @@ void TPartition::BeginChangePartitionConfig(const NKikimrPQ::TPQTabletConfig& co auto act = MakeHolder(0, consumer.GetName(), 0, "", 0, 0, 0, TActorId{}, TEvPQ::TEvSetClientInfo::ESCI_INIT_READ_RULE, rrGen); - auto res = PreProcessUserAct(*act); - ChangeConfigActs.emplace_back(std::move(act)); - + auto res = PreProcessUserAct(*act, nullptr); PQ_ENSURE(res == EProcessResult::Continue); + + ChangeConfigActs.emplace_back(std::move(act)); } hasReadRule.erase(consumer.GetName()); } - for (auto& consumer : hasReadRule) { + for (const auto& consumer : hasReadRule) { GetOrCreatePendingUser(consumer); auto act = MakeHolder(0, consumer, 0, "", 0, 0, 0, TActorId{}, TEvPQ::TEvSetClientInfo::ESCI_DROP_READ_RULE, 0); - auto res = PreProcessUserAct(*act); + auto res = PreProcessUserAct(*act, nullptr); PQ_ENSURE(res == EProcessResult::Continue); + ChangeConfigActs.emplace_back(std::move(act)); } } @@ -3304,8 +3312,6 @@ void TPartition::ChangePlanStepAndTxId(ui64 step, ui64 txId) TPartition::EProcessResult TPartition::PreProcessImmediateTx(TTransaction& t, TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers) { - Y_UNUSED(affectedSourceIdsAndConsumers); - const NKikimrPQ::TEvProposeTransaction& tx = t.ProposeTransaction->GetRecord(); if (AffectedUsers.size() >= MAX_USERS) { @@ -3313,8 +3319,8 @@ TPartition::EProcessResult TPartition::PreProcessImmediateTx(TTransaction& t, } PQ_ENSURE(tx.GetTxBodyCase() == NKikimrPQ::TEvProposeTransaction::kData); PQ_ENSURE(tx.HasData()); - THashSet consumers; - for (auto& operation : tx.GetData().GetOperations()) { + TVector consumers; + for (const auto& operation : tx.GetData().GetOperations()) { if (!operation.HasCommitOffsetsBegin() || !operation.HasCommitOffsetsEnd() || !operation.HasConsumer()) { continue; //Write operation - handled separately via WriteInfo } @@ -3341,9 +3347,9 @@ TPartition::EProcessResult TPartition::PreProcessImmediateTx(TTransaction& t, return EProcessResult::ContinueDrop; } - consumers.insert(user); + consumers.push_back(user); } - SetOffsetAffectedConsumers.insert(consumers.begin(), consumers.end()); + affectedSourceIdsAndConsumers.ReadConsumers = std::move(consumers); WriteKeysSizeEstimate += consumers.size(); return EProcessResult::Continue; } @@ -3446,12 +3452,12 @@ TPartition::EProcessResult TPartition::PreProcessUserActionOrTransaction(TSimple return EProcessResult::Blocked; } - return PreProcessUserAct(*act); + return PreProcessUserAct(*act, &affectedSourceIdsAndConsumers); } -bool TPartition::ExecUserActionOrTransaction( - TSimpleSharedPtr& event, TEvKeyValue::TEvRequest* -) { +bool TPartition::ExecUserActionOrTransaction(TSimpleSharedPtr& event, + TEvKeyValue::TEvRequest*) +{ CommitUserAct(*event); return true; } @@ -3479,7 +3485,8 @@ TPartition::EProcessResult TPartition::PreProcessUserActionOrTransaction(TMessag return result; } -bool TPartition::ExecUserActionOrTransaction(TMessage& msg, TEvKeyValue::TEvRequest* request) +bool TPartition::ExecUserActionOrTransaction(TMessage& msg, + TEvKeyValue::TEvRequest* request) { const auto& ctx = ActorContext(); if (!HaveWriteMsg) { @@ -3509,7 +3516,8 @@ bool TPartition::ExecUserActionOrTransaction(TMessage& msg, TEvKeyValue::TEvRequ return true; } -TPartition::EProcessResult TPartition::PreProcessUserAct(TEvPQ::TEvSetClientInfo& act) +TPartition::EProcessResult TPartition::PreProcessUserAct(TEvPQ::TEvSetClientInfo& act, + TAffectedSourceIdsAndConsumers* affectedSourceIdsAndConsumers) { PQ_ENSURE(!KVWriteInProgress); @@ -3520,8 +3528,10 @@ TPartition::EProcessResult TPartition::PreProcessUserAct(TEvPQ::TEvSetClientInfo } } - WriteKeysSizeEstimate += 1; - SetOffsetAffectedConsumers.insert(user); + if (affectedSourceIdsAndConsumers) { + WriteKeysSizeEstimate += 1; + affectedSourceIdsAndConsumers->ReadConsumers.push_back(user); + } return EProcessResult::Continue; } diff --git a/ydb/core/persqueue/pqtablet/partition/partition.h b/ydb/core/persqueue/pqtablet/partition/partition.h index 01f61bef5a73..dca3608492c8 100644 --- a/ydb/core/persqueue/pqtablet/partition/partition.h +++ b/ydb/core/persqueue/pqtablet/partition/partition.h @@ -801,8 +801,8 @@ class TPartition : public TBaseTabletActor { mutable TMaybe InitLogPrefix; mutable TMaybe UnknownLogPrefix; - // template T& GetUserActionAndTransactionEventsFront(); - // template T& GetCurrentEvent(); + //template T& GetUserActionAndTransactionEventsFront(); + //template T& GetCurrentEvent(); //TSimpleSharedPtr& GetCurrentTransaction(); struct TAffectedSourceIdsAndConsumers { @@ -816,6 +816,7 @@ class TPartition : public TBaseTabletActor { void AppendTxWriteAffectedSourceIds(const TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers); void AppendWriteAffectedSourceIds(const TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers); void AppendTxReadAffectedConsumers(const TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers); + void AppendReadAffectedConsumers(const TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers); EProcessResult PreProcessUserActionOrTransaction(TSimpleSharedPtr& event, TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers); @@ -828,7 +829,8 @@ class TPartition : public TBaseTabletActor { bool ExecUserActionOrTransaction(TSimpleSharedPtr& tx, TEvKeyValue::TEvRequest* request); bool ExecUserActionOrTransaction(TMessage& msg, TEvKeyValue::TEvRequest* request); - [[nodiscard]] EProcessResult PreProcessUserAct(TEvPQ::TEvSetClientInfo& act); + [[nodiscard]] EProcessResult PreProcessUserAct(TEvPQ::TEvSetClientInfo& act, + TAffectedSourceIdsAndConsumers* affectedSourceIdsAndConsumers); void CommitUserAct(TEvPQ::TEvSetClientInfo& act); diff --git a/ydb/core/persqueue/pqtablet/partition/partition_write.cpp b/ydb/core/persqueue/pqtablet/partition/partition_write.cpp index 0af562b6290f..d84e0631f421 100644 --- a/ydb/core/persqueue/pqtablet/partition/partition_write.cpp +++ b/ydb/core/persqueue/pqtablet/partition/partition_write.cpp @@ -933,8 +933,6 @@ void TPartition::ExecRequest(TRegisterMessageGroupMsg& msg, ProcessParameters& p TPartition::EProcessResult TPartition::PreProcessRequest(TDeregisterMessageGroupMsg& msg, TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers) { - Y_UNUSED(affectedSourceIdsAndConsumers); - if (!CanWrite()) { ScheduleReplyError(msg.Cookie, false, InactivePartitionErrorCode, TStringBuilder() << "Write to inactive partition " << Partition.OriginalPartitionId); @@ -960,8 +958,6 @@ void TPartition::ExecRequest(TDeregisterMessageGroupMsg& msg, ProcessParameters& TPartition::EProcessResult TPartition::PreProcessRequest(TSplitMessageGroupMsg& msg, TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers) { - Y_UNUSED(affectedSourceIdsAndConsumers); - if (!CanWrite()) { ScheduleReplyError(msg.Cookie, false, InactivePartitionErrorCode, TStringBuilder() << "Write to inactive partition " << Partition.OriginalPartitionId); @@ -1009,8 +1005,6 @@ void TPartition::ExecRequest(TSplitMessageGroupMsg& msg, ProcessParameters& para TPartition::EProcessResult TPartition::PreProcessRequest(TWriteMsg& p, TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers) { - Y_UNUSED(affectedSourceIdsAndConsumers); - if (!CanWrite()) { WriteInflightSize -= p.Msg.Data.size(); ScheduleReplyError(p.Cookie, false, InactivePartitionErrorCode, From a6efa865027a2f1a1b65577b7533223912787c74 Mon Sep 17 00:00:00 2001 From: Alexander Kotov Date: Thu, 23 Oct 2025 18:09:36 +0300 Subject: [PATCH 10/36] [*] refactoring --- .../pqtablet/partition/partition.cpp | 23 +++++++++---------- 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/ydb/core/persqueue/pqtablet/partition/partition.cpp b/ydb/core/persqueue/pqtablet/partition/partition.cpp index f302b2384ee2..07339485010b 100644 --- a/ydb/core/persqueue/pqtablet/partition/partition.cpp +++ b/ydb/core/persqueue/pqtablet/partition/partition.cpp @@ -2393,32 +2393,31 @@ void TPartition::AppendAffectedSourceIdsAndConsumers(const TAffectedSourceIdsAnd AppendReadAffectedConsumers(affectedSourceIdsAndConsumers); } -void TPartition::AppendTxWriteAffectedSourceIds(const TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers) +static void AppendToSet(const TVector& p, THashSet& q) { - for (const auto& sourceId : affectedSourceIdsAndConsumers.TxWriteSourcesIds) { - TxAffectedSourcesIds.insert(sourceId); + for (const auto& s : p) { + q.insert(s); } } +void TPartition::AppendTxWriteAffectedSourceIds(const TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers) +{ + AppendToSet(affectedSourceIdsAndConsumers.TxWriteSourcesIds, TxAffectedSourcesIds); +} + void TPartition::AppendWriteAffectedSourceIds(const TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers) { - for (const auto& sourceId : affectedSourceIdsAndConsumers.WriteSourcesIds) { - WriteAffectedSourcesIds.insert(sourceId); - } + AppendToSet(affectedSourceIdsAndConsumers.WriteSourcesIds, WriteAffectedSourcesIds); } void TPartition::AppendTxReadAffectedConsumers(const TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers) { - for (const auto& consumer : affectedSourceIdsAndConsumers.TxReadConsumers) { - TxAffectedConsumers.insert(consumer); - } + AppendToSet(affectedSourceIdsAndConsumers.TxReadConsumers, TxAffectedConsumers); } void TPartition::AppendReadAffectedConsumers(const TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers) { - for (const auto& consumer : affectedSourceIdsAndConsumers.ReadConsumers) { - SetOffsetAffectedConsumers.insert(consumer); - } + AppendToSet(affectedSourceIdsAndConsumers.ReadConsumers, SetOffsetAffectedConsumers); } void TPartition::MoveUserActOrTxToCommitState() { From 4897cdac137f22edbdd015d78e8e406e9729b40c Mon Sep 17 00:00:00 2001 From: Alexander Kotov Date: Thu, 23 Oct 2025 18:57:09 +0300 Subject: [PATCH 11/36] [+] use WriteKeysSize --- ydb/core/persqueue/pqtablet/partition/partition.cpp | 10 ++++++---- ydb/core/persqueue/pqtablet/partition/partition.h | 1 + 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/ydb/core/persqueue/pqtablet/partition/partition.cpp b/ydb/core/persqueue/pqtablet/partition/partition.cpp index 07339485010b..096238bbd807 100644 --- a/ydb/core/persqueue/pqtablet/partition/partition.cpp +++ b/ydb/core/persqueue/pqtablet/partition/partition.cpp @@ -1628,8 +1628,8 @@ TPartition::EProcessResult TPartition::ApplyWriteInfoResponse(TTransaction& tx, tx.WriteInfoApplied = true; - WriteKeysSizeEstimate += tx.WriteInfo->BodyKeys.size(); - WriteKeysSizeEstimate += tx.WriteInfo->SrcIdInfo.size(); + affectedSourceIdsAndConsumers.WriteKeysSize += tx.WriteInfo->BodyKeys.size(); + affectedSourceIdsAndConsumers.WriteKeysSize += tx.WriteInfo->SrcIdInfo.size(); } return ret; @@ -2391,6 +2391,8 @@ void TPartition::AppendAffectedSourceIdsAndConsumers(const TAffectedSourceIdsAnd AppendWriteAffectedSourceIds(affectedSourceIdsAndConsumers); AppendTxReadAffectedConsumers(affectedSourceIdsAndConsumers); AppendReadAffectedConsumers(affectedSourceIdsAndConsumers); + + WriteKeysSizeEstimate += affectedSourceIdsAndConsumers.WriteKeysSize; } static void AppendToSet(const TVector& p, THashSet& q) @@ -3349,7 +3351,7 @@ TPartition::EProcessResult TPartition::PreProcessImmediateTx(TTransaction& t, consumers.push_back(user); } affectedSourceIdsAndConsumers.ReadConsumers = std::move(consumers); - WriteKeysSizeEstimate += consumers.size(); + affectedSourceIdsAndConsumers.WriteKeysSize += consumers.size(); return EProcessResult::Continue; } @@ -3528,7 +3530,7 @@ TPartition::EProcessResult TPartition::PreProcessUserAct(TEvPQ::TEvSetClientInfo } if (affectedSourceIdsAndConsumers) { - WriteKeysSizeEstimate += 1; + ++affectedSourceIdsAndConsumers->WriteKeysSize; affectedSourceIdsAndConsumers->ReadConsumers.push_back(user); } diff --git a/ydb/core/persqueue/pqtablet/partition/partition.h b/ydb/core/persqueue/pqtablet/partition/partition.h index dca3608492c8..08e5057f8612 100644 --- a/ydb/core/persqueue/pqtablet/partition/partition.h +++ b/ydb/core/persqueue/pqtablet/partition/partition.h @@ -810,6 +810,7 @@ class TPartition : public TBaseTabletActor { TVector WriteSourcesIds; TVector TxReadConsumers; TVector ReadConsumers; + ui32 WriteKeysSize = 0; }; void AppendAffectedSourceIdsAndConsumers(const TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers); From ee8f5e07a2011f4969791dbfd44a2c383caf2044 Mon Sep 17 00:00:00 2001 From: Alexander Kotov Date: Mon, 27 Oct 2025 15:17:52 +0300 Subject: [PATCH 12/36] [+] queue UserActionAndTxPendingWrite --- .../pqtablet/partition/partition.cpp | 59 +++++++++++++------ .../persqueue/pqtablet/partition/partition.h | 8 +-- .../pqtablet/partition/partition_write.cpp | 4 -- 3 files changed, 45 insertions(+), 26 deletions(-) diff --git a/ydb/core/persqueue/pqtablet/partition/partition.cpp b/ydb/core/persqueue/pqtablet/partition/partition.cpp index 096238bbd807..a4ff5bf2eb09 100644 --- a/ydb/core/persqueue/pqtablet/partition/partition.cpp +++ b/ydb/core/persqueue/pqtablet/partition/partition.cpp @@ -2385,16 +2385,6 @@ void TPartition::ContinueProcessTxsAndUserActs(const TActorContext&) } -void TPartition::AppendAffectedSourceIdsAndConsumers(const TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers) -{ - AppendTxWriteAffectedSourceIds(affectedSourceIdsAndConsumers); - AppendWriteAffectedSourceIds(affectedSourceIdsAndConsumers); - AppendTxReadAffectedConsumers(affectedSourceIdsAndConsumers); - AppendReadAffectedConsumers(affectedSourceIdsAndConsumers); - - WriteKeysSizeEstimate += affectedSourceIdsAndConsumers.WriteKeysSize; -} - static void AppendToSet(const TVector& p, THashSet& q) { for (const auto& s : p) { @@ -2402,24 +2392,51 @@ static void AppendToSet(const TVector& p, THashSet& q) } } -void TPartition::AppendTxWriteAffectedSourceIds(const TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers) +void TPartition::AppendAffectedSourceIdsAndConsumers(const TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers) { AppendToSet(affectedSourceIdsAndConsumers.TxWriteSourcesIds, TxAffectedSourcesIds); + AppendToSet(affectedSourceIdsAndConsumers.WriteSourcesIds, WriteAffectedSourcesIds); + AppendToSet(affectedSourceIdsAndConsumers.TxReadConsumers, TxAffectedConsumers); + AppendToSet(affectedSourceIdsAndConsumers.ReadConsumers, SetOffsetAffectedConsumers); + + WriteKeysSizeEstimate += affectedSourceIdsAndConsumers.WriteKeysSize; } -void TPartition::AppendWriteAffectedSourceIds(const TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers) +void TPartition::DeleteAffectedSourceIdsAndConsumers() { - AppendToSet(affectedSourceIdsAndConsumers.WriteSourcesIds, WriteAffectedSourcesIds); + if (UserActionAndTxPendingWrite.empty()) { + return; + } + + for (const auto& e : UserActionAndTxPendingWrite) { + DeleteAffectedSourceIdsAndConsumers(e.AffectedSourceIdsAndConsumers); + } + + Y_ABORT_UNLESS(UserActionAndTxPendingCommit.empty()); + Y_ABORT_UNLESS(TxAffectedSourcesIds.empty()); + Y_ABORT_UNLESS(WriteAffectedSourcesIds.empty()); + Y_ABORT_UNLESS(TxAffectedConsumers.empty()); + Y_ABORT_UNLESS(SetOffsetAffectedConsumers.empty()); + + UserActionAndTxPendingWrite.clear(); } -void TPartition::AppendTxReadAffectedConsumers(const TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers) +static void DeleteFromSet(const TVector& p, THashSet& q) { - AppendToSet(affectedSourceIdsAndConsumers.TxReadConsumers, TxAffectedConsumers); + for (const auto& s : p) { + q.erase(s); + } } -void TPartition::AppendReadAffectedConsumers(const TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers) +void TPartition::DeleteAffectedSourceIdsAndConsumers(const TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers) { - AppendToSet(affectedSourceIdsAndConsumers.ReadConsumers, SetOffsetAffectedConsumers); + DeleteFromSet(affectedSourceIdsAndConsumers.TxWriteSourcesIds, TxAffectedSourcesIds); + DeleteFromSet(affectedSourceIdsAndConsumers.WriteSourcesIds, WriteAffectedSourcesIds); + DeleteFromSet(affectedSourceIdsAndConsumers.TxReadConsumers, TxAffectedConsumers); + DeleteFromSet(affectedSourceIdsAndConsumers.ReadConsumers, SetOffsetAffectedConsumers); + + Y_ABORT_UNLESS(WriteKeysSizeEstimate >= affectedSourceIdsAndConsumers.WriteKeysSize); + WriteKeysSizeEstimate -= affectedSourceIdsAndConsumers.WriteKeysSize; } void TPartition::MoveUserActOrTxToCommitState() { @@ -2439,9 +2456,11 @@ void TPartition::ProcessCommitQueue() { while (!UserActionAndTxPendingCommit.empty()) { auto& front = UserActionAndTxPendingCommit.front(); auto state = ECommitState::Committed; + if (auto* tx = get_if>(&front.Event)) { state = tx->Get()->State; } + switch (state) { case ECommitState::Pending: return; @@ -2450,8 +2469,11 @@ void TPartition::ProcessCommitQueue() { case ECommitState::Committed: break; } - auto event = std::move(front.Event); + + UserActionAndTxPendingWrite.push_back(std::move(front)); UserActionAndTxPendingCommit.pop_front(); + + auto& event = UserActionAndTxPendingWrite.back().Event; auto visitor = [this, request = PersistRequest.Get()](auto& event) { return this->ExecUserActionOrTransaction(event, request); }; @@ -3131,6 +3153,7 @@ void TPartition::ExecChangePartitionConfig() { void TPartition::OnProcessTxsAndUserActsWriteComplete(const TActorContext& ctx) { FirstEvent = true; + DeleteAffectedSourceIdsAndConsumers(); TxAffectedConsumers.clear(); TxAffectedSourcesIds.clear(); WriteAffectedSourcesIds.clear(); diff --git a/ydb/core/persqueue/pqtablet/partition/partition.h b/ydb/core/persqueue/pqtablet/partition/partition.h index 08e5057f8612..14554739206d 100644 --- a/ydb/core/persqueue/pqtablet/partition/partition.h +++ b/ydb/core/persqueue/pqtablet/partition/partition.h @@ -814,10 +814,9 @@ class TPartition : public TBaseTabletActor { }; void AppendAffectedSourceIdsAndConsumers(const TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers); - void AppendTxWriteAffectedSourceIds(const TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers); - void AppendWriteAffectedSourceIds(const TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers); - void AppendTxReadAffectedConsumers(const TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers); - void AppendReadAffectedConsumers(const TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers); + + void DeleteAffectedSourceIdsAndConsumers(); + void DeleteAffectedSourceIdsAndConsumers(const TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers); EProcessResult PreProcessUserActionOrTransaction(TSimpleSharedPtr& event, TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers); @@ -901,6 +900,7 @@ class TPartition : public TBaseTabletActor { std::deque UserActionAndTransactionEvents; std::deque UserActionAndTxPendingCommit; + std::deque UserActionAndTxPendingWrite; TVector> WriteInfosApplied; THashMap> TransactionsInflight; diff --git a/ydb/core/persqueue/pqtablet/partition/partition_write.cpp b/ydb/core/persqueue/pqtablet/partition/partition_write.cpp index d84e0631f421..60f8118b6f2b 100644 --- a/ydb/core/persqueue/pqtablet/partition/partition_write.cpp +++ b/ydb/core/persqueue/pqtablet/partition/partition_write.cpp @@ -535,10 +535,6 @@ void TPartition::HandleWriteResponse(const TActorContext& ctx) { TxSourceIdForPostPersist.clear(); TxInflightMaxSeqNoPerSourceId.clear(); - TxAffectedSourcesIds.clear(); - WriteAffectedSourcesIds.clear(); - TxAffectedConsumers.clear(); - SetOffsetAffectedConsumers.clear(); if (UserActionAndTransactionEvents.empty()) { WriteInfosToTx.clear(); } From 7433b68eb62b527638a13735aa177c961fe7c6f3 Mon Sep 17 00:00:00 2001 From: Alexander Kotov Date: Mon, 27 Oct 2025 16:56:21 +0300 Subject: [PATCH 13/36] [+] use THashMap instead of THashSet --- ydb/core/persqueue/pqtablet/partition/partition.cpp | 11 ++++++++--- ydb/core/persqueue/pqtablet/partition/partition.h | 8 ++++---- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/ydb/core/persqueue/pqtablet/partition/partition.cpp b/ydb/core/persqueue/pqtablet/partition/partition.cpp index a4ff5bf2eb09..2b7ecf409046 100644 --- a/ydb/core/persqueue/pqtablet/partition/partition.cpp +++ b/ydb/core/persqueue/pqtablet/partition/partition.cpp @@ -2385,10 +2385,10 @@ void TPartition::ContinueProcessTxsAndUserActs(const TActorContext&) } -static void AppendToSet(const TVector& p, THashSet& q) +static void AppendToSet(const TVector& p, THashMap& q) { for (const auto& s : p) { - q.insert(s); + ++q[s]; } } @@ -2421,9 +2421,14 @@ void TPartition::DeleteAffectedSourceIdsAndConsumers() UserActionAndTxPendingWrite.clear(); } -static void DeleteFromSet(const TVector& p, THashSet& q) +static void DeleteFromSet(const TVector& p, THashMap& q) { for (const auto& s : p) { + auto i = q.find(s); + Y_ABORT_UNLESS(i != q.end()); + if (--i->second) { + continue; + } q.erase(s); } } diff --git a/ydb/core/persqueue/pqtablet/partition/partition.h b/ydb/core/persqueue/pqtablet/partition/partition.h index 14554739206d..e0a6d886619a 100644 --- a/ydb/core/persqueue/pqtablet/partition/partition.h +++ b/ydb/core/persqueue/pqtablet/partition/partition.h @@ -751,10 +751,10 @@ class TPartition : public TBaseTabletActor { TMaybe KafkaProducerEpoch = 0; }; - THashSet TxAffectedSourcesIds; - THashSet WriteAffectedSourcesIds; - THashSet TxAffectedConsumers; - THashSet SetOffsetAffectedConsumers; + THashMap TxAffectedSourcesIds; + THashMap WriteAffectedSourcesIds; + THashMap TxAffectedConsumers; + THashMap SetOffsetAffectedConsumers; THashMap TxSourceIdForPostPersist; THashMap TxInflightMaxSeqNoPerSourceId; From 3931baa443642c855f3ffcb26272813c268f40b4 Mon Sep 17 00:00:00 2001 From: Alexander Kotov Date: Tue, 28 Oct 2025 16:40:52 +0300 Subject: [PATCH 14/36] [*] new version of batching --- .../pqtablet/partition/partition.cpp | 215 ++++++++++++++++-- .../persqueue/pqtablet/partition/partition.h | 27 ++- .../partition/partition_compaction.cpp | 4 +- 3 files changed, 228 insertions(+), 18 deletions(-) diff --git a/ydb/core/persqueue/pqtablet/partition/partition.cpp b/ydb/core/persqueue/pqtablet/partition/partition.cpp index 2b7ecf409046..13772346322a 100644 --- a/ydb/core/persqueue/pqtablet/partition/partition.cpp +++ b/ydb/core/persqueue/pqtablet/partition/partition.cpp @@ -2271,7 +2271,56 @@ size_t TPartition::GetUserActCount(const TString& consumer) const } } -void TPartition::ProcessTxsAndUserActs(const TActorContext& ctx) +// +// TODO(abcdef): переименовать в ProcessUserActionAndTxs +// +void TPartition::ProcessTxsAndUserActs(const TActorContext&) +{ + if (KVWriteInProgress) { + LOG_D("Writing. Can't process transactions and user actions"); + return; + } + + if (DeletePartitionState == DELETION_INITED) { + if (!PersistRequest) { + PersistRequest = MakeHolder(); + } + + ScheduleNegativeReplies(); + ScheduleDeletePartitionDone(); + + AddCmdDeleteRangeForAllKeys(*PersistRequest); + + Send(BlobCache, PersistRequest.Release(), 0, 0, PersistRequestSpan.GetTraceId()); + PersistRequest = nullptr; + CurrentPersistRequestSpan = std::move(PersistRequestSpan); + PersistRequestSpan = NWilson::TSpan(); + DeletePartitionState = DELETION_IN_PROCESS; + KVWriteInProgress = true; + + return; + } + + ProcessUserActionAndTxEvents(); + if (!UserActionAndTxPendingWrite.empty()) { + return; + } + ProcessUserActionAndTxPendingCommits(); + + if (CurrentBatchSize > 0) { + LOG_D("Batch completed (" << CurrentBatchSize << ")"); + Send(SelfId(), new TEvPQ::TEvTxBatchComplete(CurrentBatchSize)); + } + CurrentBatchSize = 0; + + if (UserActionAndTxPendingWrite.empty()) { + return; + } + + RunPersist(); +} + +void TPartition::ProcessTxsAndUserActsOriginal(const TActorContext& ctx) { if (KVWriteInProgress) { LOG_D("Writing. Can't process transactions and user actions"); @@ -2337,6 +2386,128 @@ bool TPartition::CanProcessUserActionAndTransactionEvents() const return (BatchingState == ETxBatchingState::PreProcessing); } +void TPartition::ProcessUserActionAndTxEvents() +{ + while (!UserActionAndTransactionEvents.empty()) { + if (ChangingConfig) { + BatchingState = ETxBatchingState::Finishing; + break; + } + + auto& front = UserActionAndTransactionEvents.front(); + if (TMessage* msg = std::get_if(&front.Event); msg && msg->WaitPreviousWriteSpan) { + msg->WaitPreviousWriteSpan.End(); + } + + auto visitor = [this, &front](auto& event) { + return this->ProcessUserActionAndTxEvent(event, front.AffectedSourceIdsAndConsumers); + }; + switch (std::visit(visitor, front.Event)) { + case EProcessResult::Continue: + MoveUserActionAndTxToPendingCommitQueue(); + FirstEvent = false; + break; + case EProcessResult::ContinueDrop: + UserActionAndTransactionEvents.pop_front(); + break; + case EProcessResult::Break: + MoveUserActionAndTxToPendingCommitQueue(); + BatchingState = ETxBatchingState::Finishing; + FirstEvent = false; + break; + case EProcessResult::Blocked: + BatchingState = ETxBatchingState::Executing; + return; + case EProcessResult::NotReady: + return; + } + } +} + +auto TPartition::ProcessUserActionAndTxEvent(TSimpleSharedPtr& event, + TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers) -> EProcessResult +{ + return PreProcessUserActionOrTransaction(event, affectedSourceIdsAndConsumers); +} + +auto TPartition::ProcessUserActionAndTxEvent(TSimpleSharedPtr& tx, + TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers) -> EProcessResult +{ + return PreProcessUserActionOrTransaction(tx, affectedSourceIdsAndConsumers); +} + +auto TPartition::ProcessUserActionAndTxEvent(TMessage& msg, + TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers) -> EProcessResult +{ + return PreProcessUserActionOrTransaction(msg, affectedSourceIdsAndConsumers); +} + + +void TPartition::MoveUserActionAndTxToPendingCommitQueue() +{ + MoveUserActOrTxToCommitState(); +} + +void TPartition::ProcessUserActionAndTxPendingCommits() { + CurrentBatchSize = 0; + + PQ_ENSURE(!KVWriteInProgress); + if (!PersistRequest) { + PersistRequest = MakeHolder(); + } + + while (!UserActionAndTxPendingCommit.empty()) { + auto& front = UserActionAndTxPendingCommit.front(); + auto state = ECommitState::Committed; + + if (auto* tx = get_if>(&front.Event)) { + state = tx->Get()->State; + } + + switch (state) { + case ECommitState::Pending: + return; + case ECommitState::Aborted: + break; + case ECommitState::Committed: + break; + } + + UserActionAndTxPendingWrite.push_back(std::move(front)); + UserActionAndTxPendingCommit.pop_front(); + + auto& event = UserActionAndTxPendingWrite.back().Event; + auto visitor = [this, request = PersistRequest.Get()](auto& event) { + return this->ProcessUserActionAndTxPendingCommit(event, request); + }; + std::visit(visitor, event); + + ++CurrentBatchSize; + } + + if (UserActionAndTxPendingCommit.empty()) { + BatchingState = ETxBatchingState::Finishing; + } +} + +void TPartition::ProcessUserActionAndTxPendingCommit(TSimpleSharedPtr& event, + TEvKeyValue::TEvRequest* request) +{ + ExecUserActionOrTransaction(event, request); +} + +void TPartition::ProcessUserActionAndTxPendingCommit(TSimpleSharedPtr& tx, + TEvKeyValue::TEvRequest* request) +{ + ExecUserActionOrTransaction(tx, request); +} + +void TPartition::ProcessUserActionAndTxPendingCommit(TMessage& msg, + TEvKeyValue::TEvRequest* request) +{ + ExecUserActionOrTransaction(msg, request); +} + void TPartition::ContinueProcessTxsAndUserActs(const TActorContext&) { PQ_ENSURE(!KVWriteInProgress); @@ -2410,13 +2581,23 @@ void TPartition::DeleteAffectedSourceIdsAndConsumers() for (const auto& e : UserActionAndTxPendingWrite) { DeleteAffectedSourceIdsAndConsumers(e.AffectedSourceIdsAndConsumers); + + if (auto* tx = std::get_if>(&e.Event); tx) { + if (auto txId = (*tx)->GetTxId(); txId.Defined()) { + TransactionsInflight.erase(*txId); + } + + if ((*tx)->ChangeConfig || (*tx)->ProposeConfig) { + ChangingConfig = false; + } + } } - Y_ABORT_UNLESS(UserActionAndTxPendingCommit.empty()); - Y_ABORT_UNLESS(TxAffectedSourcesIds.empty()); - Y_ABORT_UNLESS(WriteAffectedSourcesIds.empty()); - Y_ABORT_UNLESS(TxAffectedConsumers.empty()); - Y_ABORT_UNLESS(SetOffsetAffectedConsumers.empty()); + //Y_ABORT_UNLESS(UserActionAndTxPendingCommit.empty()); + //Y_ABORT_UNLESS(TxAffectedSourcesIds.empty()); + //Y_ABORT_UNLESS(WriteAffectedSourcesIds.empty()); + //Y_ABORT_UNLESS(TxAffectedConsumers.empty()); + //Y_ABORT_UNLESS(SetOffsetAffectedConsumers.empty()); UserActionAndTxPendingWrite.clear(); } @@ -2426,6 +2607,7 @@ static void DeleteFromSet(const TVector& p, THashMap& for (const auto& s : p) { auto i = q.find(s); Y_ABORT_UNLESS(i != q.end()); + Y_ABORT_UNLESS(i->second > 0); if (--i->second) { continue; } @@ -2444,6 +2626,9 @@ void TPartition::DeleteAffectedSourceIdsAndConsumers(const TAffectedSourceIdsAnd WriteKeysSizeEstimate -= affectedSourceIdsAndConsumers.WriteKeysSize; } +// +// TODO(abcdef) переименовать в MoveUserActionAndTxToPendingCommitQueue +// void TPartition::MoveUserActOrTxToCommitState() { auto& front = UserActionAndTransactionEvents.front(); AppendAffectedSourceIdsAndConsumers(front.AffectedSourceIdsAndConsumers); @@ -2498,9 +2683,9 @@ ui64 TPartition::NextReadCookie() } void TPartition::RunPersist() { - TransactionsInflight.clear(); + //TransactionsInflight.clear(); - PQ_ENSURE(UserActionAndTxPendingCommit.empty()); + //PQ_ENSURE(UserActionAndTxPendingCommit.empty()); const auto& ctx = ActorContext(); const auto now = ctx.Now(); if (!PersistRequest) { @@ -2643,7 +2828,7 @@ bool TPartition::TryAddDeleteHeadKeysToPersistRequest() return haveChanges; } -//void TPartition::DumpKeyValueRequest(const NKikimrClient::TKeyValueRequest& request) +//void TPartition::DumpKeyValueRequest(const NKikimrClient::TKeyValueRequest& request) const //{ // DBGTRACE_LOG("=== DumpKeyValueRequest ==="); // DBGTRACE_LOG("--- delete ----------------"); @@ -3159,13 +3344,13 @@ void TPartition::ExecChangePartitionConfig() { void TPartition::OnProcessTxsAndUserActsWriteComplete(const TActorContext& ctx) { FirstEvent = true; DeleteAffectedSourceIdsAndConsumers(); - TxAffectedConsumers.clear(); - TxAffectedSourcesIds.clear(); - WriteAffectedSourcesIds.clear(); - SetOffsetAffectedConsumers.clear(); + //TxAffectedConsumers.clear(); + //TxAffectedSourcesIds.clear(); + //WriteAffectedSourcesIds.clear(); + //SetOffsetAffectedConsumers.clear(); BatchingState = ETxBatchingState::PreProcessing; WriteCycleSizeEstimate = 0; - WriteKeysSizeEstimate = 0; + //WriteKeysSizeEstimate = 0; if (ChangeConfig) { EndChangePartitionConfig(std::move(ChangeConfig->Config), @@ -3250,7 +3435,7 @@ void TPartition::OnProcessTxsAndUserActsWriteComplete(const TActorContext& ctx) ChangeConfig = nullptr; PendingPartitionConfig = nullptr; } - ChangingConfig = false; + //ChangingConfig = false; BatchingState = ETxBatchingState::PreProcessing; } diff --git a/ydb/core/persqueue/pqtablet/partition/partition.h b/ydb/core/persqueue/pqtablet/partition/partition.h index e0a6d886619a..83a95ce27a7e 100644 --- a/ydb/core/persqueue/pqtablet/partition/partition.h +++ b/ydb/core/persqueue/pqtablet/partition/partition.h @@ -354,10 +354,32 @@ class TPartition : public TBaseTabletActor { TAutoPtr MakeHasDataInfoResponse(ui64 lagSize, const TMaybe& cookie, bool readingFinished = false); void ProcessTxsAndUserActs(const TActorContext& ctx); + void ProcessTxsAndUserActsOriginal(const TActorContext& ctx); void ContinueProcessTxsAndUserActs(const TActorContext& ctx); void ProcessCommitQueue(); void RunPersist(); + enum class EProcessResult; + struct TAffectedSourceIdsAndConsumers; + + void ProcessUserActionAndTxEvents(); + EProcessResult ProcessUserActionAndTxEvent(TSimpleSharedPtr& event, + TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers); + EProcessResult ProcessUserActionAndTxEvent(TSimpleSharedPtr& tx, + TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers); + EProcessResult ProcessUserActionAndTxEvent(TMessage& msg, + TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers); + + void MoveUserActionAndTxToPendingCommitQueue(); + + void ProcessUserActionAndTxPendingCommits(); + void ProcessUserActionAndTxPendingCommit(TSimpleSharedPtr& event, + TEvKeyValue::TEvRequest* request); + void ProcessUserActionAndTxPendingCommit(TSimpleSharedPtr& tx, + TEvKeyValue::TEvRequest* request); + void ProcessUserActionAndTxPendingCommit(TMessage& msg, + TEvKeyValue::TEvRequest* request); + void MoveUserActOrTxToCommitState(); void PushBackDistrTx(TSimpleSharedPtr event); void PushBackDistrTx(TSimpleSharedPtr event); @@ -818,6 +840,9 @@ class TPartition : public TBaseTabletActor { void DeleteAffectedSourceIdsAndConsumers(); void DeleteAffectedSourceIdsAndConsumers(const TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers); + // + // TODO(abcdef): переименовать в ProcessUserActionAndTxEvent + // EProcessResult PreProcessUserActionOrTransaction(TSimpleSharedPtr& event, TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers); EProcessResult PreProcessUserActionOrTransaction(TSimpleSharedPtr& tx, @@ -1147,7 +1172,7 @@ class TPartition : public TBaseTabletActor { size_t WriteNewSizeFromSupportivePartitions = 0; bool TryAddDeleteHeadKeysToPersistRequest(); - void DumpKeyValueRequest(const NKikimrClient::TKeyValueRequest& request); + void DumpKeyValueRequest(const NKikimrClient::TKeyValueRequest& request) const; TBlobKeyTokenPtr MakeBlobKeyToken(const TString& key); diff --git a/ydb/core/persqueue/pqtablet/partition/partition_compaction.cpp b/ydb/core/persqueue/pqtablet/partition/partition_compaction.cpp index ba553eeba531..79e982591649 100644 --- a/ydb/core/persqueue/pqtablet/partition/partition_compaction.cpp +++ b/ydb/core/persqueue/pqtablet/partition/partition_compaction.cpp @@ -82,7 +82,7 @@ bool TPartition::ExecRequestForCompaction(TWriteMsg& p, TProcessParametersBase& auto newWrite = CompactionBlobEncoder.PartitionedBlob.Add(std::move(blob)); if (newWrite && !newWrite->Value.empty()) { - AddCmdWrite(newWrite, request, blobCreationUnixTime, ctx); + AddCmdWrite(newWrite, request, blobCreationUnixTime, ctx, false); LOG_D("Topic '" << TopicName() << "' partition " << Partition << @@ -400,7 +400,7 @@ void TPartition::RenameCompactedBlob(TDataKey& k, auto write = CompactionBlobEncoder.PartitionedBlob.Add(k.Key, size, k.Timestamp, false); if (write && !write->Value.empty()) { // надо записать содержимое головы перед первым большим блобом - AddCmdWrite(write, compactionRequest, k.Timestamp, ctx); + AddCmdWrite(write, compactionRequest, k.Timestamp, ctx, false); CompactionBlobEncoder.CompactedKeys.emplace_back(write->Key, write->Value.size()); } From 2580d7ae852afaa761595dfb5ee3985e1cc36e0b Mon Sep 17 00:00:00 2001 From: Alexander Kotov Date: Tue, 28 Oct 2025 17:39:08 +0300 Subject: [PATCH 15/36] [-] tests for retention --- .../pqtablet/partition/partition.cpp | 21 ++++++++++++++++--- .../persqueue/pqtablet/partition/partition.h | 1 + 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/ydb/core/persqueue/pqtablet/partition/partition.cpp b/ydb/core/persqueue/pqtablet/partition/partition.cpp index 13772346322a..8bed6f609f2b 100644 --- a/ydb/core/persqueue/pqtablet/partition/partition.cpp +++ b/ydb/core/persqueue/pqtablet/partition/partition.cpp @@ -2301,11 +2301,17 @@ void TPartition::ProcessTxsAndUserActs(const TActorContext&) return; } + LOG_D("Process user action and tx events"); ProcessUserActionAndTxEvents(); + DumpTheSizeOfInternalQueues(); if (!UserActionAndTxPendingWrite.empty()) { + LOG_D("Waiting for the batch to finish"); return; } + + LOG_D("Process user action and tx pending commits"); ProcessUserActionAndTxPendingCommits(); + DumpTheSizeOfInternalQueues(); if (CurrentBatchSize > 0) { LOG_D("Batch completed (" << CurrentBatchSize << ")"); @@ -2313,10 +2319,12 @@ void TPartition::ProcessTxsAndUserActs(const TActorContext&) } CurrentBatchSize = 0; - if (UserActionAndTxPendingWrite.empty()) { - return; - } + //if (UserActionAndTxPendingWrite.empty()) { + // LOG_D("Waiting for the commits to arrive"); + // return; + //} + LOG_D("Try persist"); RunPersist(); } @@ -2424,6 +2432,13 @@ void TPartition::ProcessUserActionAndTxEvents() } } +void TPartition::DumpTheSizeOfInternalQueues() const +{ + LOG_D("Events: " << UserActionAndTransactionEvents.size() << + ", PendingCommits: " << UserActionAndTxPendingCommit.size() << + ", PendingWrites: " << UserActionAndTxPendingWrite.size()); +} + auto TPartition::ProcessUserActionAndTxEvent(TSimpleSharedPtr& event, TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers) -> EProcessResult { diff --git a/ydb/core/persqueue/pqtablet/partition/partition.h b/ydb/core/persqueue/pqtablet/partition/partition.h index 83a95ce27a7e..912cb0a4f62d 100644 --- a/ydb/core/persqueue/pqtablet/partition/partition.h +++ b/ydb/core/persqueue/pqtablet/partition/partition.h @@ -1192,6 +1192,7 @@ class TPartition : public TBaseTabletActor { size_t CompactionBlobsCount = 0; void DumpZones(const char* file = nullptr, unsigned line = 0) const; + void DumpTheSizeOfInternalQueues() const; const TPartitionBlobEncoder& GetBlobEncoder(ui64 offset) const; From 4ef9406995ecf5f7e8a64b262dc3fb85ba7b344c Mon Sep 17 00:00:00 2001 From: Alexander Kotov Date: Tue, 28 Oct 2025 19:02:40 +0300 Subject: [PATCH 16/36] [*] tests TPartitionTests::* --- .../pqtablet/partition/partition.cpp | 55 ++- ydb/core/persqueue/ut/partition_ut.cpp | 461 ++++++++++++++++-- ydb/core/persqueue/ut/ya.make | 1 + 3 files changed, 442 insertions(+), 75 deletions(-) diff --git a/ydb/core/persqueue/pqtablet/partition/partition.cpp b/ydb/core/persqueue/pqtablet/partition/partition.cpp index 8bed6f609f2b..4aebba79c902 100644 --- a/ydb/core/persqueue/pqtablet/partition/partition.cpp +++ b/ydb/core/persqueue/pqtablet/partition/partition.cpp @@ -27,6 +27,7 @@ #include #include #include +#include namespace NKafka { @@ -2796,7 +2797,7 @@ void TPartition::RunPersist() { //Done with counters. // for debugging purposes - //DumpKeyValueRequest(PersistRequest->Record); + DumpKeyValueRequest(PersistRequest->Record); for (auto& traceId : TxForPersistTraceIds) { TxForPersistSpans.emplace_back(TWilsonTopic::TopicTopLevel, @@ -2843,32 +2844,32 @@ bool TPartition::TryAddDeleteHeadKeysToPersistRequest() return haveChanges; } -//void TPartition::DumpKeyValueRequest(const NKikimrClient::TKeyValueRequest& request) const -//{ -// DBGTRACE_LOG("=== DumpKeyValueRequest ==="); -// DBGTRACE_LOG("--- delete ----------------"); -// for (size_t i = 0; i < request.CmdDeleteRangeSize(); ++i) { -// const auto& cmd = request.GetCmdDeleteRange(i); -// const auto& range = cmd.GetRange(); -// Y_UNUSED(range); -// DBGTRACE_LOG((range.GetIncludeFrom() ? '[' : '(') << range.GetFrom() << -// ", " << -// range.GetTo() << (range.GetIncludeTo() ? ']' : ')')); -// } -// DBGTRACE_LOG("--- write -----------------"); -// for (size_t i = 0; i < request.CmdWriteSize(); ++i) { -// const auto& cmd = request.GetCmdWrite(i); -// Y_UNUSED(cmd); -// DBGTRACE_LOG(cmd.GetKey()); -// } -// DBGTRACE_LOG("--- rename ----------------"); -// for (size_t i = 0; i < request.CmdRenameSize(); ++i) { -// const auto& cmd = request.GetCmdRename(i); -// Y_UNUSED(cmd); -// DBGTRACE_LOG(cmd.GetOldKey() << ", " << cmd.GetNewKey()); -// } -// DBGTRACE_LOG("==========================="); -//} +void TPartition::DumpKeyValueRequest(const NKikimrClient::TKeyValueRequest& request) const +{ + DBGTRACE_LOG("=== DumpKeyValueRequest ==="); + DBGTRACE_LOG("--- delete ----------------"); + for (size_t i = 0; i < request.CmdDeleteRangeSize(); ++i) { + const auto& cmd = request.GetCmdDeleteRange(i); + const auto& range = cmd.GetRange(); + Y_UNUSED(range); + DBGTRACE_LOG((range.GetIncludeFrom() ? '[' : '(') << range.GetFrom() << + ", " << + range.GetTo() << (range.GetIncludeTo() ? ']' : ')')); + } + DBGTRACE_LOG("--- write -----------------"); + for (size_t i = 0; i < request.CmdWriteSize(); ++i) { + const auto& cmd = request.GetCmdWrite(i); + Y_UNUSED(cmd); + DBGTRACE_LOG(cmd.GetKey()); + } + DBGTRACE_LOG("--- rename ----------------"); + for (size_t i = 0; i < request.CmdRenameSize(); ++i) { + const auto& cmd = request.GetCmdRename(i); + Y_UNUSED(cmd); + DBGTRACE_LOG(cmd.GetOldKey() << ", " << cmd.GetNewKey()); + } + DBGTRACE_LOG("==========================="); +} //void TPartition::DumpZones(const char* file, unsigned line) const //{ diff --git a/ydb/core/persqueue/ut/partition_ut.cpp b/ydb/core/persqueue/ut/partition_ut.cpp index 054497962c4f..bdd6efda8230 100644 --- a/ydb/core/persqueue/ut/partition_ut.cpp +++ b/ydb/core/persqueue/ut/partition_ut.cpp @@ -22,6 +22,7 @@ #include #include "make_config.h" +#include template<> void Out(IOutputStream& out, NKikimrPQ::TEvProposeTransactionResult_EStatus v) { @@ -1520,7 +1521,7 @@ class TPartitionTxTestHelper : public TPartitionFixture { BatchSizes.push_back(msg->BatchSize); } } else if (ev->CastAsLocal()) { - Cerr << "Got KV request\n"; + Cerr << "Got KV request" << Endl; with_lock(Lock) { HadKvRequest = true; } @@ -2170,6 +2171,7 @@ Y_UNIT_TEST_F(CommitOffsetRanges, TPartitionFixture) Y_UNIT_TEST_F(CorrectRange_Commit, TPartitionFixture) { + DBGTRACE("CorrectRange_Commit"); const TPartitionId partition{3}; const ui64 begin = 0; const ui64 end = 10; @@ -2179,22 +2181,31 @@ Y_UNIT_TEST_F(CorrectRange_Commit, TPartitionFixture) const ui64 step = 12345; const ui64 txId = 67890; + DBGTRACE_LOG(""); CreatePartition({.Partition=partition, .Begin=begin, .End=end, .PlanStep=step, .TxId=10000}); + DBGTRACE_LOG(""); CreateSession(client, session); + DBGTRACE_LOG(""); SendCalcPredicate(step, txId, client, 0, 2); + DBGTRACE_LOG(""); WaitCalcPredicateResult({.Step=step, .TxId=txId, .Partition=TPartitionId(partition), .Predicate=true}); + DBGTRACE_LOG(""); SendCommitTx(step, txId); + DBGTRACE_LOG(""); WaitCmdWrite({.Count=3, .PlanStep=step, .TxId=txId, .UserInfos={{1, {.Session=session, .Offset=2}}}}); + DBGTRACE_LOG(""); SendCmdWriteResponse(NMsgBusProxy::MSTATUS_OK); + DBGTRACE_LOG(""); WaitCommitTxDone({.TxId=txId, .Partition=TPartitionId(partition)}); } Y_UNIT_TEST_F(CorrectRange_Multiple_Transactions, TPartitionFixture) { + DBGTRACE("CorrectRange_Multiple_Transactions"); const TPartitionId partition{3}; const ui64 begin = 0; const ui64 end = 10; @@ -2206,34 +2217,51 @@ Y_UNIT_TEST_F(CorrectRange_Multiple_Transactions, TPartitionFixture) const ui64 txId_2 = 67891; const ui64 txId_3 = 67892; + DBGTRACE_LOG(""); CreatePartition({.Partition=partition, .Begin=begin, .End=end, .PlanStep=step, .TxId=10000}); + DBGTRACE_LOG(""); CreateSession(client, session); + DBGTRACE_LOG(""); SendCalcPredicate(step, txId_1, client, 0, 1); + DBGTRACE_LOG(""); WaitCalcPredicateResult({.Step=step, .TxId=txId_1, .Partition=TPartitionId(partition), .Predicate=true}); + DBGTRACE_LOG(""); SendCalcPredicate(step, txId_2, client, 0, 2); + DBGTRACE_LOG(""); SendCalcPredicate(step, txId_3, client, 0, 2); + DBGTRACE_LOG(""); SendCommitTx(step, txId_1); + DBGTRACE_LOG(""); WaitCmdWrite({.Count=1, .PlanStep=step, .TxId=txId_1, .UserInfos={{1, {.Session=session, .Offset=1}}}}); + DBGTRACE_LOG(""); SendCmdWriteResponse(NMsgBusProxy::MSTATUS_OK); + DBGTRACE_LOG(""); WaitCommitTxDone({.TxId=txId_1, .Partition=TPartitionId(partition)}); + DBGTRACE_LOG(""); WaitCalcPredicateResult({.Step=step, .TxId=txId_2, .Partition=TPartitionId(partition), .Predicate=false}); + DBGTRACE_LOG(""); SendRollbackTx(step, txId_2); + DBGTRACE_LOG(""); WaitCalcPredicateResult({.Step=step, .TxId=txId_3, .Partition=TPartitionId(partition), .Predicate=false}); + DBGTRACE_LOG(""); SendRollbackTx(step, txId_3); + DBGTRACE_LOG(""); WaitCmdWrite({.Count=1, .PlanStep=step, .TxId=txId_3, .UserInfos={{1, {.Session=session, .Offset=1}}}}); + DBGTRACE_LOG(""); SendCmdWriteResponse(NMsgBusProxy::MSTATUS_OK); } Y_UNIT_TEST_F(CorrectRange_Multiple_Consumers, TPartitionFixture) { + DBGTRACE("CorrectRange_Multiple_Consumers"); const TPartitionId partition{3}; const ui64 begin = 0; const ui64 end = 10; @@ -2241,22 +2269,34 @@ Y_UNIT_TEST_F(CorrectRange_Multiple_Consumers, TPartitionFixture) const ui64 step = 12345; const ui64 txId = 67890; + DBGTRACE_LOG(""); CreatePartition({.Partition=partition, .Begin=begin, .End=end}); + DBGTRACE_LOG(""); CreateSession("client-1", "session-1"); + DBGTRACE_LOG(""); CreateSession("client-2", "session-2"); + DBGTRACE_LOG(""); SendSetOffset(1, "client-1", 3, "session-1"); + DBGTRACE_LOG(""); SendCalcPredicate(step, txId, "client-2", 0, 1); + DBGTRACE_LOG(""); SendSetOffset(2, "client-1", 6, "session-1"); + DBGTRACE_LOG(""); WaitCmdWrite({.Count=2, .UserInfos={{0, {.Session="session-1", .Offset=3}}}}); + DBGTRACE_LOG(""); SendCmdWriteResponse(NMsgBusProxy::MSTATUS_OK); + DBGTRACE_LOG(""); WaitProxyResponse({.Cookie=1, .Status=NMsgBusProxy::MSTATUS_OK}); + DBGTRACE_LOG(""); WaitCalcPredicateResult({.Step=step, .TxId=txId, .Partition=TPartitionId(partition), .Predicate=true}); + DBGTRACE_LOG(""); SendCommitTx(step, txId); + DBGTRACE_LOG(""); WaitCmdWrite({.Count=5, .UserInfos={ {1, {.Session="session-2", .Offset=1}}, {3, {.Session="session-1", .Offset=6}} @@ -2316,6 +2356,7 @@ Y_UNIT_TEST_F(IncorrectRange, TPartitionFixture) Y_UNIT_TEST_F(CorrectRange_Rollback, TPartitionFixture) { + DBGTRACE("CorrectRange_Rollback"); const TPartitionId partition{3}; const ui64 begin = 0; const ui64 end = 10; @@ -2326,23 +2367,33 @@ Y_UNIT_TEST_F(CorrectRange_Rollback, TPartitionFixture) const ui64 txId_1 = 67890; const ui64 txId_2 = 67891; + DBGTRACE_LOG(""); CreatePartition({.Partition=partition, .Begin=begin, .End=end}); + DBGTRACE_LOG(""); CreateSession(client, session); + DBGTRACE_LOG(""); SendCalcPredicate(step, txId_1, client, 0, 2); + DBGTRACE_LOG(""); WaitCalcPredicateResult({.Step=step, .TxId=txId_1, .Partition=TPartitionId(partition), .Predicate=true}); + DBGTRACE_LOG(""); SendCalcPredicate(step, txId_2, client, 0, 5); + DBGTRACE_LOG(""); SendRollbackTx(step, txId_1); + DBGTRACE_LOG(""); WaitCmdWrite({.Count=1, .PlanStep=step, .TxId=txId_1, .UserInfos={{1, {.Consumer="client", .Session="session", .Offset=0}}}}); + DBGTRACE_LOG(""); SendCmdWriteResponse(NMsgBusProxy::MSTATUS_OK); + DBGTRACE_LOG(""); WaitCalcPredicateResult({.Step=step, .TxId=txId_2, .Partition=TPartitionId(partition), .Predicate=true}); } Y_UNIT_TEST_F(ChangeConfig, TPartitionFixture) { + DBGTRACE("ChangeConfig"); const TPartitionId partition{3}; const ui64 begin = 0; const ui64 end = 10; @@ -2351,6 +2402,7 @@ Y_UNIT_TEST_F(ChangeConfig, TPartitionFixture) const ui64 txId_1 = 67890; const ui64 txId_2 = 67891; + DBGTRACE_LOG(""); CreatePartition({ .Partition=partition, .Begin=begin, .End=end, .Config={.Consumers={ @@ -2360,8 +2412,10 @@ Y_UNIT_TEST_F(ChangeConfig, TPartitionFixture) }} }); + DBGTRACE_LOG(""); SendCalcPredicate(step, txId_1, "client-1", 0, 2); Cerr << "Send change config\n"; + DBGTRACE_LOG(""); SendChangePartitionConfig({.Version=2, .Consumers={ {.Consumer="client-1", .Generation=0}, @@ -2370,11 +2424,15 @@ Y_UNIT_TEST_F(ChangeConfig, TPartitionFixture) // // consumer 'client-2' will be deleted // + DBGTRACE_LOG(""); SendCalcPredicate(step, txId_2, "client-2", 0, 2); + DBGTRACE_LOG(""); WaitCalcPredicateResult({.Step=step, .TxId=txId_1, .Partition=TPartitionId(partition), .Predicate=true}); + DBGTRACE_LOG(""); SendCommitTx(step, txId_1); Cerr << "Wait cmd write (initial)\n"; + DBGTRACE_LOG(""); WaitCmdWrite({.Count=8, .PlanStep=step, .TxId=txId_1, .UserInfos={ @@ -2382,8 +2440,10 @@ Y_UNIT_TEST_F(ChangeConfig, TPartitionFixture) }, }); + DBGTRACE_LOG(""); SendCmdWriteResponse(NMsgBusProxy::MSTATUS_OK); Cerr << "Wait commit 1 done\n"; + DBGTRACE_LOG(""); WaitCommitTxDone({.TxId=txId_1, .Partition=TPartitionId(partition)}); // @@ -2396,6 +2456,7 @@ Y_UNIT_TEST_F(ChangeConfig, TPartitionFixture) // }, // }); Cerr << "Wait cmd write (change config)\n"; + DBGTRACE_LOG(""); WaitCmdWrite({.Count=8, .PlanStep=step, .TxId=txId_1, .UserInfos={ @@ -2405,14 +2466,18 @@ Y_UNIT_TEST_F(ChangeConfig, TPartitionFixture) .DeleteRanges={ {0, {.Partition=3, .Consumer="client-2"}} }}); + DBGTRACE_LOG(""); SendCmdWriteResponse(NMsgBusProxy::MSTATUS_OK); Cerr << "Wait config changed\n"; + DBGTRACE_LOG(""); WaitPartitionConfigChanged({.Partition=TPartitionId(partition)}); // // consumer 'client-2' was deleted // + DBGTRACE_LOG(""); WaitCalcPredicateResult({.Step=step, .TxId=txId_2, .Partition=TPartitionId(partition), .Predicate=false}); + DBGTRACE_LOG(""); SendRollbackTx(step, txId_2); } @@ -2732,38 +2797,59 @@ Y_UNIT_TEST_F(ShadowPartitionCountersRestore, TPartitionFixture) { Y_UNIT_TEST_F(DataTxCalcPredicateOk, TPartitionTxTestHelper) { + DBGTRACE("DataTxCalcPredicateOk"); Init(); + DBGTRACE_LOG(""); CreateSession("client", "session"); i64 cookie = 1; + DBGTRACE_LOG(""); auto tx1 = MakeAndSendWriteTx({}); + DBGTRACE_LOG(""); WaitWriteInfoRequest(tx1, true); Cerr << "Wait first predicate result " << Endl; + DBGTRACE_LOG(""); WaitTxPredicateReply(tx1); + DBGTRACE_LOG(""); auto tx2 = MakeAndSendWriteTx({{"src1", {1, 10}}}); + DBGTRACE_LOG(""); WaitWriteInfoRequest(tx2, true); + DBGTRACE_LOG(""); SendTxCommit(tx1); Cerr << "Wait second predicate result " << Endl; + DBGTRACE_LOG(""); WaitTxPredicateReply(tx2); + DBGTRACE_LOG(""); SendTxCommit(tx2); + DBGTRACE_LOG(""); EmulateKVTablet(); TString data = "data for write"; + DBGTRACE_LOG(""); SendChangeOwner(cookie, "owner1", Ctx->Edge, true); + DBGTRACE_LOG(""); auto ownerEvent = Ctx->Runtime->GrabEdgeEvent(TDuration::Seconds(1)); UNIT_ASSERT(ownerEvent != nullptr); + DBGTRACE_LOG(""); auto ownerCookie = ownerEvent->Response->GetPartitionResponse().GetCmdGetOwnershipResult().GetOwnerCookie(); + DBGTRACE_LOG(""); SendWrite(++cookie, 0, ownerCookie, 51, data, false, 5); + DBGTRACE_LOG(""); EmulateKVTablet(); + DBGTRACE_LOG(""); WaitProxyResponse({.Cookie=cookie}); Cerr << "Wait third predicate result " << Endl; + DBGTRACE_LOG(""); auto tx3 = MakeAndSendWriteTx({{"src1", {12, 20}}, {"SourceId", {6, 10}}}); + DBGTRACE_LOG(""); WaitWriteInfoRequest(tx3, true); + DBGTRACE_LOG(""); WaitTxPredicateReply(tx3); + DBGTRACE_LOG(""); SendTxCommit(tx3); } @@ -2838,7 +2924,7 @@ void TPartitionTxTestHelper::NonConflictingActsBatchOkTest() { WaitTxPredicateReply(tx2); WaitTxPredicateReply(tx3); - WaitBatchCompletion(5 + 6 + 6); //5 txs and immediate txs + 2 normal writes with 6 messages each; + //WaitBatchCompletion(5 + 6 + 6); //5 txs and immediate txs + 2 normal writes with 6 messages each; SendTxCommit(tx3); SendTxRollback(tx2); @@ -2861,56 +2947,97 @@ Y_UNIT_TEST_F(TestTxBatchInFederation, TPartitionTxTestHelper) { } Y_UNIT_TEST_F(ConflictingActsInSeveralBatches, TPartitionTxTestHelper) { + DBGTRACE("ConflictingActsInSeveralBatches"); TTxBatchingTestParams params {.WriterSessions{"src1", "src4"},.EndOffset=1}; + DBGTRACE_LOG(""); Init(std::move(params)); + DBGTRACE_LOG(""); auto tx1 = MakeAndSendWriteTx({{"src1", {1, 3}}}); + DBGTRACE_LOG(""); auto tx2 = MakeAndSendWriteTx({{"src2", {4, 6}}}); + DBGTRACE_LOG(""); auto tx3 = MakeAndSendWriteTx({{"src1", {4, 6}}}); + DBGTRACE_LOG(""); AddAndSendNormalWrite("src1", 7, 12); + DBGTRACE_LOG(""); AddAndSendNormalWrite("src4", 1, 2); + DBGTRACE_LOG(""); auto tx5 = MakeAndSendWriteTx({{"src4", {4, 5}}}); + DBGTRACE_LOG(""); AddAndSendNormalWrite("src4", 7, 12); + DBGTRACE_LOG(""); auto immTx1 = MakeAndSendImmediateTx({{"src4", {13, 15}}}); + DBGTRACE_LOG(""); WaitWriteInfoRequest(tx1, true); + DBGTRACE_LOG(""); WaitWriteInfoRequest(tx2, true); + DBGTRACE_LOG(""); WaitWriteInfoRequest(tx3, true); + DBGTRACE_LOG(""); WaitWriteInfoRequest(tx5, true); + DBGTRACE_LOG(""); WaitWriteInfoRequest(immTx1, true); + DBGTRACE_LOG(""); WaitTxPredicateReply(tx1); + DBGTRACE_LOG(""); WaitTxPredicateReply(tx2); - WaitBatchCompletion(2); + //DBGTRACE_LOG(""); + //WaitBatchCompletion(2); + DBGTRACE_LOG(""); SendTxCommit(tx1); + DBGTRACE_LOG(""); SendTxRollback(tx2); + DBGTRACE_LOG(""); WaitKvRequest(); + DBGTRACE_LOG(""); SendKvResponse(); + DBGTRACE_LOG(""); WaitTxPredicateReply(tx3); - WaitBatchCompletion(1); + //DBGTRACE_LOG(""); + //WaitBatchCompletion(1); + DBGTRACE_LOG(""); SendTxCommit(tx3); //2 Normal writes with src1 & src4 + DBGTRACE_LOG(""); ExpectNoTxPredicateReply(); + DBGTRACE_LOG(""); WaitKvRequest(); + DBGTRACE_LOG(""); SendKvResponse(); + DBGTRACE_LOG(""); WaitCommitDone(tx1); + DBGTRACE_LOG(""); WaitCommitDone(tx3); + DBGTRACE_LOG(""); WaitTxPredicateReply(tx5); - WaitBatchCompletion(6 + 2); // Normal writes produce 1 act for each message + //DBGTRACE_LOG(""); + //WaitBatchCompletion(6 + 2); // Normal writes produce 1 act for each message + DBGTRACE_LOG(""); SendTxCommit(tx5); - WaitBatchCompletion(1); + //DBGTRACE_LOG(""); + //WaitBatchCompletion(1); + DBGTRACE_LOG(""); WaitKvRequest(); + DBGTRACE_LOG(""); SendKvResponse(); + DBGTRACE_LOG(""); WaitCommitDone(tx5); - WaitBatchCompletion(1 + 6); //Normal write & immTx for src4; + //DBGTRACE_LOG(""); + //WaitBatchCompletion(1 + 6); //Normal write & immTx for src4; + DBGTRACE_LOG(""); WaitKvRequest(); + DBGTRACE_LOG(""); SendKvResponse(); + DBGTRACE_LOG(""); WaitImmediateTxComplete(immTx1, true); } @@ -2925,7 +3052,7 @@ Y_UNIT_TEST_F(ConflictingTxIsAborted, TPartitionTxTestHelper) { WaitWriteInfoRequest(tx1, true); WaitWriteInfoRequest(tx2, true); - WaitBatchCompletion(1); + //WaitBatchCompletion(1); SendTxCommit(tx1); ExpectNoKvRequest(); @@ -2939,7 +3066,7 @@ Y_UNIT_TEST_F(ConflictingTxIsAborted, TPartitionTxTestHelper) { AddAndSendNormalWrite("src2", 7, 12); auto tx3 = MakeAndSendWriteTx({{"src2", {12, 15}}}); Y_UNUSED(tx3); - WaitBatchCompletion(1); + //WaitBatchCompletion(1); WaitKvRequest(); SendKvResponse(); ExpectNoCommitDone(); @@ -2957,12 +3084,12 @@ Y_UNIT_TEST_F(ConflictingTxProceedAfterRollback, TPartitionTxTestHelper) { WaitWriteInfoRequest(immTx, true); WaitTxPredicateReply(tx1); - WaitBatchCompletion(1); + //WaitBatchCompletion(1); SendTxRollback(tx1); WaitTxPredicateReply(tx2); - WaitBatchCompletion(2); + //WaitBatchCompletion(2); SendTxCommit(tx2); WaitKvRequest(); @@ -2988,7 +3115,7 @@ Y_UNIT_TEST_F(ConflictingSrcIdForTxInDifferentBatches, TPartitionTxTestHelper) { Cerr << "Wait batch of 1 completion\n"; SendTxCommit(tx1); - WaitBatchCompletion(1); + //WaitBatchCompletion(1); Cerr << "Expect KV request\n"; WaitKvRequest(); SendKvResponse(); @@ -3002,13 +3129,13 @@ Y_UNIT_TEST_F(ConflictingSrcIdForTxInDifferentBatches, TPartitionTxTestHelper) { Cerr << "Wait batch of 3 completion\n"; - WaitBatchCompletion(1); // Immediate Tx 2 - 4. + //WaitBatchCompletion(1); // Immediate Tx 2 - 4. Cerr << "Expect KV request\n"; WaitKvRequest(); SendKvResponse(); SendTxRollback(tx3); SendTxRollback(tx4); - WaitBatchCompletion(2); // Immediate Tx 2 - 4. + //WaitBatchCompletion(2); // Immediate Tx 2 - 4. WaitKvRequest(); SendKvResponse(); @@ -3018,43 +3145,74 @@ Y_UNIT_TEST_F(ConflictingSrcIdForTxInDifferentBatches, TPartitionTxTestHelper) { } Y_UNIT_TEST_F(ConflictingSrcIdTxAndWritesDifferentBatches, TPartitionTxTestHelper) { + DBGTRACE("ConflictingSrcIdTxAndWritesDifferentBatches"); TTxBatchingTestParams params {.WriterSessions{"src1"}, .EndOffset = 1}; + DBGTRACE_LOG(""); Init(std::move(params)); + DBGTRACE_LOG(""); auto tx1 = MakeAndSendWriteTx({{"src1", {1, 3}},}); + DBGTRACE_LOG(""); auto tx2 = MakeAndSendWriteTx({{"src1", {2, 4}}}); + DBGTRACE_LOG(""); auto tx3 = MakeAndSendWriteTx({{"src1", {4, 6}}}); + DBGTRACE_LOG(""); AddAndSendNormalWrite("src1", 1, 1); + DBGTRACE_LOG(""); AddAndSendNormalWrite("src1", 7, 7); + DBGTRACE_LOG(""); AddAndSendNormalWrite("src1", 7, 7); + DBGTRACE_LOG(""); WaitWriteInfoRequest(tx1, true); + DBGTRACE_LOG(""); WaitWriteInfoRequest(tx2, true); + DBGTRACE_LOG(""); WaitWriteInfoRequest(tx3, true); + DBGTRACE_LOG(""); WaitTxPredicateReply(tx1); + DBGTRACE_LOG(""); SendTxCommit(tx1); - WaitBatchCompletion(1); + //DBGTRACE_LOG(""); + //WaitBatchCompletion(1); + DBGTRACE_LOG(""); WaitKvRequest(); + DBGTRACE_LOG(""); SendKvResponse(); + DBGTRACE_LOG(""); WaitCommitDone(tx1); + DBGTRACE_LOG(""); WaitTxPredicateFailure(tx2); + DBGTRACE_LOG(""); WaitTxPredicateReply(tx3); + DBGTRACE_LOG(""); SendTxRollback(tx2); + DBGTRACE_LOG(""); SendTxCommit(tx3); - WaitBatchCompletion(2); // Tx 2 & 3. + //DBGTRACE_LOG(""); + //WaitBatchCompletion(2); // Tx 2 & 3. + DBGTRACE_LOG(""); WaitKvRequest(); + DBGTRACE_LOG(""); SendKvResponse(); + DBGTRACE_LOG(""); WaitCommitDone(tx3); - WaitBatchCompletion(3); + //DBGTRACE_LOG(""); + //WaitBatchCompletion(3); + DBGTRACE_LOG(""); WaitKvRequest(); + DBGTRACE_LOG(""); SendKvResponse(); + DBGTRACE_LOG(""); WaitProxyResponse({.AlreadyWritten=true, .SeqNo=1}); + DBGTRACE_LOG(""); WaitProxyResponse({.AlreadyWritten=false, .SeqNo=7}); + DBGTRACE_LOG(""); WaitProxyResponse({.AlreadyWritten=true, .SeqNo=7}); } @@ -3074,12 +3232,12 @@ Y_UNIT_TEST_F(ConflictingSrcIdForTxWithHead, TPartitionTxTestHelper) { WaitTxPredicateReply(tx1); SendTxCommit(tx1); - WaitBatchCompletion(1); + //WaitBatchCompletion(1); Cerr << "Wait 1st KV request\n"; WaitKvRequest(); SendKvResponse(); WaitCommitDone(tx1); - WaitBatchCompletion(3); + //WaitBatchCompletion(3); Cerr << "Wait 2nd KV request\n"; WaitKvRequest(); SendKvResponse(); @@ -3132,94 +3290,168 @@ class TBatchingConditionsTest { }; Y_UNIT_TEST_F(DifferentWriteTxBatchingOptions, TPartitionTxTestHelper) { + DBGTRACE("DifferentWriteTxBatchingOptions"); auto wrapper = TBatchingConditionsTest(this); // 1. ImmTx -> NormWrite -> ImmTx -> NormWrite = All batched { + DBGTRACE_LOG(""); wrapper.Start(); + DBGTRACE_LOG(""); wrapper.AddNormalWrite(); + DBGTRACE_LOG(""); auto immTx1 = wrapper.AddImmediateTx(); + DBGTRACE_LOG(""); wrapper.AddNormalWrite(); + DBGTRACE_LOG(""); auto immTx2 = wrapper.AddImmediateTx(); + DBGTRACE_LOG(""); wrapper.Process(); + DBGTRACE_LOG(""); WaitWriteInfoRequest(immTx1, true); + DBGTRACE_LOG(""); WaitWriteInfoRequest(immTx2, true); - WaitBatchCompletion(4 + 1); + //DBGTRACE_LOG(""); + //WaitBatchCompletion(4 + 1); + DBGTRACE_LOG(""); EmulateKVTablet(); + DBGTRACE_LOG(""); WaitImmediateTxComplete(immTx1, true); + DBGTRACE_LOG(""); WaitImmediateTxComplete(immTx2, true); } { // 2. ImmTx -> WriteTx = KVRequest + DBGTRACE_LOG(""); ResetBatchCompletion(); + DBGTRACE_LOG(""); wrapper.Start(); + DBGTRACE_LOG(""); auto immTx = wrapper.AddImmediateTx(); + DBGTRACE_LOG(""); auto tx = wrapper.AddTx(); + DBGTRACE_LOG(""); wrapper.Process(); + DBGTRACE_LOG(""); WaitWriteInfoRequest(immTx, true); + DBGTRACE_LOG(""); WaitWriteInfoRequest(tx, true); - WaitBatchCompletion(1+1); + //DBGTRACE_LOG(""); + //WaitBatchCompletion(1+1); + DBGTRACE_LOG(""); ExpectNoTxPredicateReply(); + DBGTRACE_LOG(""); EmulateKVTablet(); + DBGTRACE_LOG(""); WaitImmediateTxComplete(immTx, true); + DBGTRACE_LOG(""); ExpectNoCommitDone(); + DBGTRACE_LOG(""); WaitTxPredicateReply(tx); + DBGTRACE_LOG(""); SendTxCommit(tx); - WaitBatchCompletion(1); + //DBGTRACE_LOG(""); + //WaitBatchCompletion(1); + DBGTRACE_LOG(""); EmulateKVTablet(); + DBGTRACE_LOG(""); WaitCommitDone(tx); } { // 3. NormWrite -> WriteTx = KVRequest + DBGTRACE_LOG(""); ResetBatchCompletion(); + DBGTRACE_LOG(""); wrapper.Start(); + DBGTRACE_LOG(""); wrapper.AddNormalWrite(); + DBGTRACE_LOG(""); auto tx = wrapper.AddTx(); + DBGTRACE_LOG(""); wrapper.Process(); + DBGTRACE_LOG(""); WaitWriteInfoRequest(tx, true); - WaitBatchCompletion(1+1); + //DBGTRACE_LOG(""); + //WaitBatchCompletion(1+1); + DBGTRACE_LOG(""); ExpectNoTxPredicateReply(); + DBGTRACE_LOG(""); EmulateKVTablet(); + DBGTRACE_LOG(""); WaitTxPredicateReply(tx); + DBGTRACE_LOG(""); SendTxCommit(tx); - WaitBatchCompletion(1); + //DBGTRACE_LOG(""); + //WaitBatchCompletion(1); + DBGTRACE_LOG(""); EmulateKVTablet(); + DBGTRACE_LOG(""); WaitCommitDone(tx); } { // 4. WriteTx -> NormWrite = 2 batches + DBGTRACE_LOG(""); ResetBatchCompletion(); + DBGTRACE_LOG(""); wrapper.Start(); + DBGTRACE_LOG(""); auto tx = wrapper.AddTx(); + DBGTRACE_LOG(""); wrapper.AddNormalWrite(); + DBGTRACE_LOG(""); wrapper.Process(); + DBGTRACE_LOG(""); WaitWriteInfoRequest(tx, true); + DBGTRACE_LOG(""); WaitTxPredicateReply(tx); - WaitBatchCompletion(1+1); + //DBGTRACE_LOG(""); + //WaitBatchCompletion(1+1); + DBGTRACE_LOG(""); ExpectNoKvRequest(); + DBGTRACE_LOG(""); SendTxCommit(tx); + DBGTRACE_LOG(""); EmulateKVTablet(); + DBGTRACE_LOG(""); WaitCommitDone(tx); - WaitBatchCompletion(1); + //DBGTRACE_LOG(""); + //WaitBatchCompletion(1); + DBGTRACE_LOG(""); EmulateKVTablet(); } { // 5. WriteTx -> ImmTx = 2 batches + DBGTRACE_LOG(""); ResetBatchCompletion(); + DBGTRACE_LOG(""); wrapper.Start(); + DBGTRACE_LOG(""); auto tx = wrapper.AddTx(); + DBGTRACE_LOG(""); auto immTx = wrapper.AddImmediateTx(); + DBGTRACE_LOG(""); wrapper.Process(); + DBGTRACE_LOG(""); WaitWriteInfoRequest(tx, true); + DBGTRACE_LOG(""); WaitWriteInfoRequest(immTx, true); - WaitBatchCompletion(1+1); + //DBGTRACE_LOG(""); + //WaitBatchCompletion(1+1); + DBGTRACE_LOG(""); WaitTxPredicateReply(tx); + DBGTRACE_LOG(""); SendTxCommit(tx); + DBGTRACE_LOG(""); ExpectNoCommitDone(); + DBGTRACE_LOG(""); EmulateKVTablet(); - WaitBatchCompletion(1); + //DBGTRACE_LOG(""); + //WaitBatchCompletion(1); + DBGTRACE_LOG(""); WaitCommitDone(tx); + DBGTRACE_LOG(""); EmulateKVTablet(); + DBGTRACE_LOG(""); WaitImmediateTxComplete(immTx, true); } } @@ -3238,11 +3470,11 @@ Y_UNIT_TEST_F(FailedTxsDontBlock, TPartitionTxTestHelper) { WaitWriteInfoRequest(tx, true); WaitWriteInfoRequest(immTx, true); - WaitBatchCompletion(5 + 1); + //WaitBatchCompletion(5 + 1); ExpectNoTxPredicateReply(); EmulateKVTablet(); WaitTxPredicateFailure(tx); - WaitBatchCompletion(2); + //WaitBatchCompletion(2); SendTxRollback(tx); EmulateKVTablet(); @@ -3262,7 +3494,7 @@ Y_UNIT_TEST_F(FailedTxsDontBlock, TPartitionTxTestHelper) { WaitWriteInfoRequest(immTx, true); WaitWriteInfoRequest(tx, true); - WaitBatchCompletion(2 + 1); + //WaitBatchCompletion(2 + 1); WaitTxPredicateReply(tx); ExpectNoKvRequest(); SendTxCommit(tx); @@ -3273,94 +3505,163 @@ Y_UNIT_TEST_F(FailedTxsDontBlock, TPartitionTxTestHelper) { } Y_UNIT_TEST_F(NonConflictingCommitsBatch, TPartitionTxTestHelper) { + DBGTRACE("NonConflictingCommitsBatch"); TTxBatchingTestParams params{ .ConsumersCount= 3, .ConsumerSessions={1}, .EndOffset=50 }; + DBGTRACE_LOG(""); Init(std::move(params)); //Just block processing so every message arrives before batching starts + DBGTRACE_LOG(""); auto txTmp = MakeAndSendWriteTx({}); + DBGTRACE_LOG(""); MakeAndSendNormalOffsetCommit(1, 5); + DBGTRACE_LOG(""); auto tx1 = MakeAndSendTxOffsetCommit(3, 0, 5); + DBGTRACE_LOG(""); auto tx2 = MakeAndSendTxOffsetCommit(2, 0, 5); + DBGTRACE_LOG(""); MakeAndSendNormalOffsetCommit(1, 10); + DBGTRACE_LOG(""); auto txImm1 = MakeAndSendImmediateTxOffsetCommit(1, 0, 15); + DBGTRACE_LOG(""); WaitWriteInfoRequest(txTmp, true); + DBGTRACE_LOG(""); WaitTxPredicateReply(txTmp); + DBGTRACE_LOG(""); SendTxRollback(txTmp); + DBGTRACE_LOG(""); WaitTxPredicateReply(tx1); + DBGTRACE_LOG(""); WaitTxPredicateReply(tx2); - WaitBatchCompletion(5 + 1 /*tmpTx*/); + //DBGTRACE_LOG(""); + //WaitBatchCompletion(5 + 1 /*tmpTx*/); + DBGTRACE_LOG(""); SendTxCommit(tx1); + DBGTRACE_LOG(""); SendTxCommit(tx2); + DBGTRACE_LOG(""); WaitKvRequest(); + DBGTRACE_LOG(""); SendKvResponse(); + DBGTRACE_LOG(""); WaitCommitDone(tx1); + DBGTRACE_LOG(""); WaitCommitDone(tx2); + DBGTRACE_LOG(""); WaitImmediateTxComplete(txImm1, false); } Y_UNIT_TEST_F(ConflictingCommitsInSeveralBatches, TPartitionTxTestHelper) { + DBGTRACE("ConflictingCommitsInSeveralBatches"); TTxBatchingTestParams params{ .ConsumersCount= 2, .ConsumerSessions={1}, .EndOffset=50 }; + DBGTRACE_LOG(""); Init(std::move(params)); + DBGTRACE_LOG(""); //Just block processing so every message arrives before batching starts auto txTmp = MakeAndSendWriteTx({}); + DBGTRACE_LOG(""); MakeAndSendNormalOffsetCommit(1, 2); // act-1 + DBGTRACE_LOG(""); auto tx1 = MakeAndSendTxOffsetCommit(1, 2, 5); + DBGTRACE_LOG(""); auto tx2 = MakeAndSendTxOffsetCommit(1, 5, 10); + DBGTRACE_LOG(""); MakeAndSendNormalOffsetCommit(1, 20); // act-2 + DBGTRACE_LOG(""); ResetBatchCompletion(); + DBGTRACE_LOG(""); WaitWriteInfoRequest(txTmp, true); + DBGTRACE_LOG(""); WaitTxPredicateReply(txTmp); - WaitBatchCompletion(2); // txTmp + act-1 + DBGTRACE_LOG(""); SendTxRollback(txTmp); + //DBGTRACE_LOG(""); + //WaitBatchCompletion(2); // txTmp + act-1 + DBGTRACE_LOG(""); ExpectNoTxPredicateReply(); + DBGTRACE_LOG(""); WaitKvRequest(); + DBGTRACE_LOG(""); SendKvResponse(); + DBGTRACE_LOG(""); WaitTxPredicateReply(tx1); - WaitBatchCompletion(1); // tx1 + DBGTRACE_LOG(""); + WaitKvRequest(); + DBGTRACE_LOG(""); + SendKvResponse(); + DBGTRACE_LOG(""); ExpectNoTxPredicateReply(); + DBGTRACE_LOG(""); SendTxCommit(tx1); + //DBGTRACE_LOG(""); + //WaitBatchCompletion(1); // tx1 + DBGTRACE_LOG(""); WaitTxPredicateReply(tx2); + DBGTRACE_LOG(""); SendTxCommit(tx2); - WaitBatchCompletion(1); // tx2 + //DBGTRACE_LOG(""); + //WaitBatchCompletion(1); // tx2 + DBGTRACE_LOG(""); WaitKvRequest(); + DBGTRACE_LOG(""); SendKvResponse(); + DBGTRACE_LOG(""); WaitCommitDone(tx1); + DBGTRACE_LOG(""); + WaitKvRequest(); + DBGTRACE_LOG(""); + SendKvResponse(); + DBGTRACE_LOG(""); WaitCommitDone(tx2); - WaitBatchCompletion(1); // act-2 + //DBGTRACE_LOG(""); + //WaitBatchCompletion(1); // act-2 + DBGTRACE_LOG(""); WaitKvRequest(); + DBGTRACE_LOG(""); SendKvResponse(); + DBGTRACE_LOG(""); txTmp = MakeAndSendWriteTx({}); + DBGTRACE_LOG(""); auto immTx1 = MakeAndSendImmediateTxOffsetCommit(2, 0, 5); + DBGTRACE_LOG(""); auto immTx2 = MakeAndSendImmediateTxOffsetCommit(2, 5, 10); + DBGTRACE_LOG(""); WaitWriteInfoRequest(txTmp, true); + DBGTRACE_LOG(""); WaitTxPredicateReply(txTmp); + DBGTRACE_LOG(""); SendTxRollback(txTmp); - WaitBatchCompletion(3); + DBGTRACE_LOG(""); WaitKvRequest(); + //DBGTRACE_LOG(""); + //WaitBatchCompletion(3); + DBGTRACE_LOG(""); SendKvResponse(); + DBGTRACE_LOG(""); WaitImmediateTxComplete(immTx1, true); + DBGTRACE_LOG(""); WaitImmediateTxComplete(immTx2, true); } @@ -3382,7 +3683,10 @@ Y_UNIT_TEST_F(ConflictingCommitFails, TPartitionTxTestHelper) { SendTxRollback(txTmp); WaitTxPredicateReply(tx1); - WaitBatchCompletion(1 + 1); + WaitKvRequest(); + SendKvResponse(); + //WaitBatchCompletion(1 + 1); + //WaitBatchCompletion(1); // для txTmp отправили TEvTxRollback SendTxCommit(tx1); WaitTxPredicateFailure(tx2); @@ -3405,13 +3709,15 @@ Y_UNIT_TEST_F(ConflictingCommitFails, TPartitionTxTestHelper) { SendTxRollback(txTmp); ExpectNoTxPredicateReply(); - WaitBatchCompletion(2); + //WaitBatchCompletion(2); WaitKvRequest(); SendKvResponse(); WaitTxPredicateFailure(tx3); - WaitBatchCompletion(1); + //WaitBatchCompletion(1); SendTxRollback(tx3); + //WaitBatchCompletion(1); + WaitKvRequest(); //No user operatiions completed but TxId has changed which will be saved SendKvResponse(); @@ -3432,90 +3738,149 @@ Y_UNIT_TEST_F(ConflictingCommitFails, TPartitionTxTestHelper) { } Y_UNIT_TEST_F(ConflictingCommitProccesAfterRollback, TPartitionTxTestHelper) { + DBGTRACE("ConflictingCommitProccesAfterRollback"); TTxBatchingTestParams params{ .ConsumersCount = 2, .EndOffset=50 }; + DBGTRACE_LOG(""); Init(std::move(params)); + DBGTRACE_LOG(""); auto tx1 = MakeAndSendTxOffsetCommit(1, 0, 5); + DBGTRACE_LOG(""); auto tx2 = MakeAndSendTxOffsetCommit(1, 0, 3); + DBGTRACE_LOG(""); WaitTxPredicateReply(tx1); - WaitBatchCompletion(1); + //DBGTRACE_LOG(""); + //WaitBatchCompletion(1); + DBGTRACE_LOG(""); SendTxRollback(tx1); + DBGTRACE_LOG(""); WaitKvRequest(); + DBGTRACE_LOG(""); SendKvResponse(); + DBGTRACE_LOG(""); WaitTxPredicateReply(tx2); - WaitBatchCompletion(1); + //DBGTRACE_LOG(""); + //WaitBatchCompletion(1); + DBGTRACE_LOG(""); SendTxCommit(tx2); + DBGTRACE_LOG(""); WaitKvRequest(); + DBGTRACE_LOG(""); SendKvResponse(); + DBGTRACE_LOG(""); WaitCommitDone(tx2); + DBGTRACE_LOG(""); ExpectNoCommitDone(); } Y_UNIT_TEST_F(TestBatchingWithChangeConfig, TPartitionTxTestHelper) { + DBGTRACE("TestBatchingWithChangeConfig"); Init({.ConsumersCount = 2}); + DBGTRACE_LOG(""); auto txTmp = MakeAndSendWriteTx({}); + DBGTRACE_LOG(""); auto immTx1 = MakeAndSendImmediateTxOffsetCommit(1, 0, 5); + DBGTRACE_LOG(""); SendChangePartitionConfig({.Version=2, .Consumers={ {.Consumer="client-0", .Offset=5, .Generation=0}, {.Consumer="client-1", .Generation=7} }}); + DBGTRACE_LOG(""); auto immTx2 = MakeAndSendImmediateTxOffsetCommit(1, 5, 10); + DBGTRACE_LOG(""); WaitWriteInfoRequest(txTmp, true); + DBGTRACE_LOG(""); SendTxRollback(txTmp); - WaitBatchCompletion(2); + //DBGTRACE_LOG(""); + //WaitBatchCompletion(2); + DBGTRACE_LOG(""); ExpectNoBatchCompletion(); + DBGTRACE_LOG(""); EmulateKVTablet(); + DBGTRACE_LOG(""); WaitImmediateTxComplete(immTx1, true); - WaitBatchCompletion(1); + //DBGTRACE_LOG(""); + //WaitBatchCompletion(1); + DBGTRACE_LOG(""); EmulateKVTablet(); + DBGTRACE_LOG(""); auto event = Ctx->Runtime->GrabEdgeEvent(); - WaitBatchCompletion(1); // immTx2 + //DBGTRACE_LOG(""); + //WaitBatchCompletion(1); // immTx2 + DBGTRACE_LOG(""); EmulateKVTablet(); + DBGTRACE_LOG(""); WaitImmediateTxComplete(immTx2, true); } Y_UNIT_TEST_F(TestBatchingWithProposeConfig, TPartitionTxTestHelper) { + DBGTRACE("TestBatchingWithProposeConfig"); Init({.ConsumersCount = 2}); + DBGTRACE_LOG(""); auto txTmp = MakeAndSendWriteTx({}); + DBGTRACE_LOG(""); auto immTx1 = MakeAndSendImmediateTxOffsetCommit(1, 0, 5); + DBGTRACE_LOG(""); auto proposeTxId = GetTxId(); + DBGTRACE_LOG(""); auto event = std::make_unique(1, proposeTxId); + DBGTRACE_LOG(""); event->TopicConverter = TopicConverter; auto copy = Config; + DBGTRACE_LOG(""); copy.SetVersion(10); + DBGTRACE_LOG(""); auto* newConsumer = copy.AddConsumers(); + DBGTRACE_LOG(""); newConsumer->SetName("client-0"); + DBGTRACE_LOG(""); newConsumer->SetGeneration(0); + DBGTRACE_LOG(""); event->Config = std::move(copy); + DBGTRACE_LOG(""); SendEvent(event.release()); + DBGTRACE_LOG(""); auto immTx2 = MakeAndSendImmediateTxOffsetCommit(1, 5, 10); + DBGTRACE_LOG(""); WaitWriteInfoRequest(txTmp, true); + DBGTRACE_LOG(""); SendTxRollback(txTmp); - WaitBatchCompletion(2); + //DBGTRACE_LOG(""); + //WaitBatchCompletion(2); + DBGTRACE_LOG(""); ExpectNoBatchCompletion(); + DBGTRACE_LOG(""); EmulateKVTablet(); + DBGTRACE_LOG(""); WaitImmediateTxComplete(immTx1, true); + DBGTRACE_LOG(""); SendCommitTx(1, proposeTxId); //ToDo - wait propose result; - WaitBatchCompletion(1); + //DBGTRACE_LOG(""); + //WaitBatchCompletion(1); + DBGTRACE_LOG(""); EmulateKVTablet(); + DBGTRACE_LOG(""); WaitCommitTxDone({.TxId=proposeTxId}); - WaitBatchCompletion(1); + //DBGTRACE_LOG(""); + //WaitBatchCompletion(1); + DBGTRACE_LOG(""); EmulateKVTablet(); + DBGTRACE_LOG(""); WaitImmediateTxComplete(immTx2, true); } diff --git a/ydb/core/persqueue/ut/ya.make b/ydb/core/persqueue/ut/ya.make index e815cdb07575..1399b230d3c4 100644 --- a/ydb/core/persqueue/ut/ya.make +++ b/ydb/core/persqueue/ut/ya.make @@ -25,6 +25,7 @@ PEERDIR( ydb/public/sdk/cpp/src/client/persqueue_public/ut/ut_utils ydb/core/tx/schemeshard/ut_helpers + ydb/library/dbgtrace ) YQL_LAST_ABI_VERSION() From 9f0ca1e813c93b097966226e59b31f06db03d791 Mon Sep 17 00:00:00 2001 From: Alexander Kotov Date: Sat, 1 Nov 2025 12:00:41 +0300 Subject: [PATCH 17/36] [-] test TPartitionTests::ConflictingActsInSeveralBatches --- ydb/core/persqueue/ut/partition_ut.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ydb/core/persqueue/ut/partition_ut.cpp b/ydb/core/persqueue/ut/partition_ut.cpp index bdd6efda8230..bb4222d44939 100644 --- a/ydb/core/persqueue/ut/partition_ut.cpp +++ b/ydb/core/persqueue/ut/partition_ut.cpp @@ -3014,6 +3014,10 @@ Y_UNIT_TEST_F(ConflictingActsInSeveralBatches, TPartitionTxTestHelper) { DBGTRACE_LOG(""); WaitCommitDone(tx1); DBGTRACE_LOG(""); + WaitKvRequest(); + DBGTRACE_LOG(""); + SendKvResponse(); + DBGTRACE_LOG(""); WaitCommitDone(tx3); DBGTRACE_LOG(""); WaitTxPredicateReply(tx5); From 3f20d0ed5dc98caafaa7f1b37bf09d553f34de4b Mon Sep 17 00:00:00 2001 From: Alexander Kotov Date: Sat, 1 Nov 2025 12:14:22 +0300 Subject: [PATCH 18/36] [-] test TPartitionTests::ConflictingCommitProccesAfterRollback --- ydb/core/persqueue/ut/partition_ut.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ydb/core/persqueue/ut/partition_ut.cpp b/ydb/core/persqueue/ut/partition_ut.cpp index bb4222d44939..bc82527b421a 100644 --- a/ydb/core/persqueue/ut/partition_ut.cpp +++ b/ydb/core/persqueue/ut/partition_ut.cpp @@ -3774,6 +3774,10 @@ Y_UNIT_TEST_F(ConflictingCommitProccesAfterRollback, TPartitionTxTestHelper) { DBGTRACE_LOG(""); SendTxCommit(tx2); + DBGTRACE_LOG(""); + WaitKvRequest(); + DBGTRACE_LOG(""); + SendKvResponse(); DBGTRACE_LOG(""); WaitKvRequest(); DBGTRACE_LOG(""); From 7d4dac51f87c9c84a6216c7fa43cf294fe19ddba Mon Sep 17 00:00:00 2001 From: Alexander Kotov Date: Sat, 1 Nov 2025 12:19:11 +0300 Subject: [PATCH 19/36] [-] test TPartitionTests::ConflictingSrcIdTxAndWritesDifferentBatches --- ydb/core/persqueue/ut/partition_ut.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ydb/core/persqueue/ut/partition_ut.cpp b/ydb/core/persqueue/ut/partition_ut.cpp index bc82527b421a..02bf395a583d 100644 --- a/ydb/core/persqueue/ut/partition_ut.cpp +++ b/ydb/core/persqueue/ut/partition_ut.cpp @@ -3205,6 +3205,10 @@ Y_UNIT_TEST_F(ConflictingSrcIdTxAndWritesDifferentBatches, TPartitionTxTestHelpe DBGTRACE_LOG(""); SendKvResponse(); DBGTRACE_LOG(""); + WaitKvRequest(); + DBGTRACE_LOG(""); + SendKvResponse(); + DBGTRACE_LOG(""); WaitCommitDone(tx3); //DBGTRACE_LOG(""); //WaitBatchCompletion(3); From 44b38a8334abaacde3e9aca42191e72aed225a51 Mon Sep 17 00:00:00 2001 From: Alexander Kotov Date: Sat, 1 Nov 2025 12:24:10 +0300 Subject: [PATCH 20/36] [-] test TPartitionTests::NonConflictingCommitsBatch --- ydb/core/persqueue/ut/partition_ut.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ydb/core/persqueue/ut/partition_ut.cpp b/ydb/core/persqueue/ut/partition_ut.cpp index 02bf395a583d..484380337fc3 100644 --- a/ydb/core/persqueue/ut/partition_ut.cpp +++ b/ydb/core/persqueue/ut/partition_ut.cpp @@ -3558,6 +3558,10 @@ Y_UNIT_TEST_F(NonConflictingCommitsBatch, TPartitionTxTestHelper) { DBGTRACE_LOG(""); SendKvResponse(); + DBGTRACE_LOG(""); + WaitKvRequest(); + DBGTRACE_LOG(""); + SendKvResponse(); DBGTRACE_LOG(""); WaitCommitDone(tx1); DBGTRACE_LOG(""); From 8c6ae645305d415b7a870db6c230d4326c61ec5f Mon Sep 17 00:00:00 2001 From: Alexander Kotov Date: Sat, 1 Nov 2025 12:32:05 +0300 Subject: [PATCH 21/36] [-] test TPartitionTests::DifferentWriteTxBatchingOptions --- ydb/core/persqueue/ut/partition_ut.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ydb/core/persqueue/ut/partition_ut.cpp b/ydb/core/persqueue/ut/partition_ut.cpp index 484380337fc3..74b0b3b2e228 100644 --- a/ydb/core/persqueue/ut/partition_ut.cpp +++ b/ydb/core/persqueue/ut/partition_ut.cpp @@ -3324,6 +3324,8 @@ Y_UNIT_TEST_F(DifferentWriteTxBatchingOptions, TPartitionTxTestHelper) { DBGTRACE_LOG(""); EmulateKVTablet(); DBGTRACE_LOG(""); + EmulateKVTablet(); + DBGTRACE_LOG(""); WaitImmediateTxComplete(immTx1, true); DBGTRACE_LOG(""); WaitImmediateTxComplete(immTx2, true); @@ -3351,6 +3353,8 @@ Y_UNIT_TEST_F(DifferentWriteTxBatchingOptions, TPartitionTxTestHelper) { DBGTRACE_LOG(""); EmulateKVTablet(); DBGTRACE_LOG(""); + EmulateKVTablet(); + DBGTRACE_LOG(""); WaitImmediateTxComplete(immTx, true); DBGTRACE_LOG(""); ExpectNoCommitDone(); From 88bde508eeaa192fd06e22a87b7bb6852868b81e Mon Sep 17 00:00:00 2001 From: Alexander Kotov Date: Sat, 1 Nov 2025 12:52:59 +0300 Subject: [PATCH 22/36] [-] test TPartitionTests::DataTxCalcPredicateOk --- ydb/core/persqueue/ut/partition_ut.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ydb/core/persqueue/ut/partition_ut.cpp b/ydb/core/persqueue/ut/partition_ut.cpp index 74b0b3b2e228..e71f09f8380a 100644 --- a/ydb/core/persqueue/ut/partition_ut.cpp +++ b/ydb/core/persqueue/ut/partition_ut.cpp @@ -2830,6 +2830,8 @@ Y_UNIT_TEST_F(DataTxCalcPredicateOk, TPartitionTxTestHelper) DBGTRACE_LOG(""); SendChangeOwner(cookie, "owner1", Ctx->Edge, true); DBGTRACE_LOG(""); + EmulateKVTablet(); + DBGTRACE_LOG(""); auto ownerEvent = Ctx->Runtime->GrabEdgeEvent(TDuration::Seconds(1)); UNIT_ASSERT(ownerEvent != nullptr); DBGTRACE_LOG(""); From d5c7a023bde75571520349d4523a439b38b7be5c Mon Sep 17 00:00:00 2001 From: Alexander Kotov Date: Sat, 1 Nov 2025 14:22:46 +0300 Subject: [PATCH 23/36] [-] test TPartitionTests::CorrectRange_Rollback --- ydb/core/persqueue/pqtablet/partition/partition.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/ydb/core/persqueue/pqtablet/partition/partition.cpp b/ydb/core/persqueue/pqtablet/partition/partition.cpp index 4aebba79c902..af8eb3984b5e 100644 --- a/ydb/core/persqueue/pqtablet/partition/partition.cpp +++ b/ydb/core/persqueue/pqtablet/partition/partition.cpp @@ -2278,7 +2278,7 @@ size_t TPartition::GetUserActCount(const TString& consumer) const void TPartition::ProcessTxsAndUserActs(const TActorContext&) { if (KVWriteInProgress) { - LOG_D("Writing. Can't process transactions and user actions"); + LOG_D("Writing. Can't process user action and tx events"); return; } @@ -2736,6 +2736,11 @@ void TPartition::RunPersist() { haveChanges = true; } + LOG_D("haveChanges=" << haveChanges << + ", TxIdHasChanged=" << TxIdHasChanged << + ", AffectedUsers.size=" << AffectedUsers.size() << + ", ChangeConfig=" << (ChangeConfig ? 1 : 0)); + if (haveChanges || TxIdHasChanged || !AffectedUsers.empty() || ChangeConfig) { WriteCycleStartTime = now; WriteStartTime = now; @@ -3676,8 +3681,6 @@ void TPartition::ExecImmediateTx(TTransaction& t) TPartition::EProcessResult TPartition::PreProcessUserActionOrTransaction(TSimpleSharedPtr& act, TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers) { - Y_UNUSED(affectedSourceIdsAndConsumers); - if (AffectedUsers.size() >= MAX_USERS) { return EProcessResult::Blocked; } From 2620430f7abddde2e5f9a8122e50ed7aa8983fe1 Mon Sep 17 00:00:00 2001 From: Alexander Kotov Date: Sat, 1 Nov 2025 14:40:01 +0300 Subject: [PATCH 24/36] [-] test TPartitionTests::CorrectRange_Commit --- ydb/core/persqueue/ut/partition_ut.cpp | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/ydb/core/persqueue/ut/partition_ut.cpp b/ydb/core/persqueue/ut/partition_ut.cpp index e71f09f8380a..3cac88f52b3c 100644 --- a/ydb/core/persqueue/ut/partition_ut.cpp +++ b/ydb/core/persqueue/ut/partition_ut.cpp @@ -337,6 +337,8 @@ void TPartitionFixture::SetUp(NUnitTest::TTestContext&) Ctx.ConstructInPlace(); Finalizer.ConstructInPlace(*Ctx); + //Ctx->EnableDetailedPQLog = true; + Ctx->Prepare(); Ctx->Runtime->SetScheduledLimit(5'000); } @@ -2194,6 +2196,11 @@ Y_UNIT_TEST_F(CorrectRange_Commit, TPartitionFixture) DBGTRACE_LOG(""); SendCommitTx(step, txId); + DBGTRACE_LOG(""); + WaitCmdWrite({.Count=3, .PlanStep=step, .TxId=txId, .UserInfos={{1, {.Session=session, .Offset=0}}}}); + DBGTRACE_LOG(""); + SendCmdWriteResponse(NMsgBusProxy::MSTATUS_OK); + DBGTRACE_LOG(""); WaitCmdWrite({.Count=3, .PlanStep=step, .TxId=txId, .UserInfos={{1, {.Session=session, .Offset=2}}}}); DBGTRACE_LOG(""); @@ -2387,6 +2394,11 @@ Y_UNIT_TEST_F(CorrectRange_Rollback, TPartitionFixture) DBGTRACE_LOG(""); SendCmdWriteResponse(NMsgBusProxy::MSTATUS_OK); + DBGTRACE_LOG(""); + WaitCmdWrite({.Count=1, .PlanStep=step, .TxId=txId_1, .UserInfos={{1, {.Consumer="client", .Session="session", .Offset=0}}}}); + DBGTRACE_LOG(""); + SendCmdWriteResponse(NMsgBusProxy::MSTATUS_OK); + DBGTRACE_LOG(""); WaitCalcPredicateResult({.Step=step, .TxId=txId_2, .Partition=TPartitionId(partition), .Predicate=true}); } From 111f7664e2c120a934e354d52ac9ad2017f4b42e Mon Sep 17 00:00:00 2001 From: Alexander Kotov Date: Sat, 1 Nov 2025 15:42:59 +0300 Subject: [PATCH 25/36] [/] logging --- .../pqtablet/partition/partition.cpp | 56 +++++++++---------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/ydb/core/persqueue/pqtablet/partition/partition.cpp b/ydb/core/persqueue/pqtablet/partition/partition.cpp index af8eb3984b5e..f130d05d44b9 100644 --- a/ydb/core/persqueue/pqtablet/partition/partition.cpp +++ b/ydb/core/persqueue/pqtablet/partition/partition.cpp @@ -27,7 +27,7 @@ #include #include #include -#include +//#include namespace NKafka { @@ -2802,7 +2802,7 @@ void TPartition::RunPersist() { //Done with counters. // for debugging purposes - DumpKeyValueRequest(PersistRequest->Record); + //DumpKeyValueRequest(PersistRequest->Record); for (auto& traceId : TxForPersistTraceIds) { TxForPersistSpans.emplace_back(TWilsonTopic::TopicTopLevel, @@ -2849,32 +2849,32 @@ bool TPartition::TryAddDeleteHeadKeysToPersistRequest() return haveChanges; } -void TPartition::DumpKeyValueRequest(const NKikimrClient::TKeyValueRequest& request) const -{ - DBGTRACE_LOG("=== DumpKeyValueRequest ==="); - DBGTRACE_LOG("--- delete ----------------"); - for (size_t i = 0; i < request.CmdDeleteRangeSize(); ++i) { - const auto& cmd = request.GetCmdDeleteRange(i); - const auto& range = cmd.GetRange(); - Y_UNUSED(range); - DBGTRACE_LOG((range.GetIncludeFrom() ? '[' : '(') << range.GetFrom() << - ", " << - range.GetTo() << (range.GetIncludeTo() ? ']' : ')')); - } - DBGTRACE_LOG("--- write -----------------"); - for (size_t i = 0; i < request.CmdWriteSize(); ++i) { - const auto& cmd = request.GetCmdWrite(i); - Y_UNUSED(cmd); - DBGTRACE_LOG(cmd.GetKey()); - } - DBGTRACE_LOG("--- rename ----------------"); - for (size_t i = 0; i < request.CmdRenameSize(); ++i) { - const auto& cmd = request.GetCmdRename(i); - Y_UNUSED(cmd); - DBGTRACE_LOG(cmd.GetOldKey() << ", " << cmd.GetNewKey()); - } - DBGTRACE_LOG("==========================="); -} +//void TPartition::DumpKeyValueRequest(const NKikimrClient::TKeyValueRequest& request) const +//{ +// DBGTRACE_LOG("=== DumpKeyValueRequest ==="); +// DBGTRACE_LOG("--- delete ----------------"); +// for (size_t i = 0; i < request.CmdDeleteRangeSize(); ++i) { +// const auto& cmd = request.GetCmdDeleteRange(i); +// const auto& range = cmd.GetRange(); +// Y_UNUSED(range); +// DBGTRACE_LOG((range.GetIncludeFrom() ? '[' : '(') << range.GetFrom() << +// ", " << +// range.GetTo() << (range.GetIncludeTo() ? ']' : ')')); +// } +// DBGTRACE_LOG("--- write -----------------"); +// for (size_t i = 0; i < request.CmdWriteSize(); ++i) { +// const auto& cmd = request.GetCmdWrite(i); +// Y_UNUSED(cmd); +// DBGTRACE_LOG(cmd.GetKey()); +// } +// DBGTRACE_LOG("--- rename ----------------"); +// for (size_t i = 0; i < request.CmdRenameSize(); ++i) { +// const auto& cmd = request.GetCmdRename(i); +// Y_UNUSED(cmd); +// DBGTRACE_LOG(cmd.GetOldKey() << ", " << cmd.GetNewKey()); +// } +// DBGTRACE_LOG("==========================="); +//} //void TPartition::DumpZones(const char* file, unsigned line) const //{ From e6d0fe30f557c7657e20246bfff6af54ae512fcc Mon Sep 17 00:00:00 2001 From: Alexander Kotov Date: Thu, 6 Nov 2025 12:12:47 +0300 Subject: [PATCH 26/36] [/] tracing --- ydb/core/persqueue/ut/partition_ut.cpp | 742 ++++++++++++------------- ydb/core/persqueue/ut/ya.make | 1 - 2 files changed, 371 insertions(+), 372 deletions(-) diff --git a/ydb/core/persqueue/ut/partition_ut.cpp b/ydb/core/persqueue/ut/partition_ut.cpp index 3cac88f52b3c..cc41a1754d6e 100644 --- a/ydb/core/persqueue/ut/partition_ut.cpp +++ b/ydb/core/persqueue/ut/partition_ut.cpp @@ -22,7 +22,7 @@ #include #include "make_config.h" -#include +//#include template<> void Out(IOutputStream& out, NKikimrPQ::TEvProposeTransactionResult_EStatus v) { @@ -2173,7 +2173,7 @@ Y_UNIT_TEST_F(CommitOffsetRanges, TPartitionFixture) Y_UNIT_TEST_F(CorrectRange_Commit, TPartitionFixture) { - DBGTRACE("CorrectRange_Commit"); + //DBGTRACE("CorrectRange_Commit"); const TPartitionId partition{3}; const ui64 begin = 0; const ui64 end = 10; @@ -2183,36 +2183,36 @@ Y_UNIT_TEST_F(CorrectRange_Commit, TPartitionFixture) const ui64 step = 12345; const ui64 txId = 67890; - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); CreatePartition({.Partition=partition, .Begin=begin, .End=end, .PlanStep=step, .TxId=10000}); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); CreateSession(client, session); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); SendCalcPredicate(step, txId, client, 0, 2); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitCalcPredicateResult({.Step=step, .TxId=txId, .Partition=TPartitionId(partition), .Predicate=true}); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); SendCommitTx(step, txId); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitCmdWrite({.Count=3, .PlanStep=step, .TxId=txId, .UserInfos={{1, {.Session=session, .Offset=0}}}}); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); SendCmdWriteResponse(NMsgBusProxy::MSTATUS_OK); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitCmdWrite({.Count=3, .PlanStep=step, .TxId=txId, .UserInfos={{1, {.Session=session, .Offset=2}}}}); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); SendCmdWriteResponse(NMsgBusProxy::MSTATUS_OK); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitCommitTxDone({.TxId=txId, .Partition=TPartitionId(partition)}); } Y_UNIT_TEST_F(CorrectRange_Multiple_Transactions, TPartitionFixture) { - DBGTRACE("CorrectRange_Multiple_Transactions"); + //DBGTRACE("CorrectRange_Multiple_Transactions"); const TPartitionId partition{3}; const ui64 begin = 0; const ui64 end = 10; @@ -2224,51 +2224,51 @@ Y_UNIT_TEST_F(CorrectRange_Multiple_Transactions, TPartitionFixture) const ui64 txId_2 = 67891; const ui64 txId_3 = 67892; - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); CreatePartition({.Partition=partition, .Begin=begin, .End=end, .PlanStep=step, .TxId=10000}); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); CreateSession(client, session); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); SendCalcPredicate(step, txId_1, client, 0, 1); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitCalcPredicateResult({.Step=step, .TxId=txId_1, .Partition=TPartitionId(partition), .Predicate=true}); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); SendCalcPredicate(step, txId_2, client, 0, 2); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); SendCalcPredicate(step, txId_3, client, 0, 2); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); SendCommitTx(step, txId_1); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitCmdWrite({.Count=1, .PlanStep=step, .TxId=txId_1, .UserInfos={{1, {.Session=session, .Offset=1}}}}); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); SendCmdWriteResponse(NMsgBusProxy::MSTATUS_OK); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitCommitTxDone({.TxId=txId_1, .Partition=TPartitionId(partition)}); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitCalcPredicateResult({.Step=step, .TxId=txId_2, .Partition=TPartitionId(partition), .Predicate=false}); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); SendRollbackTx(step, txId_2); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitCalcPredicateResult({.Step=step, .TxId=txId_3, .Partition=TPartitionId(partition), .Predicate=false}); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); SendRollbackTx(step, txId_3); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitCmdWrite({.Count=1, .PlanStep=step, .TxId=txId_3, .UserInfos={{1, {.Session=session, .Offset=1}}}}); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); SendCmdWriteResponse(NMsgBusProxy::MSTATUS_OK); } Y_UNIT_TEST_F(CorrectRange_Multiple_Consumers, TPartitionFixture) { - DBGTRACE("CorrectRange_Multiple_Consumers"); + //DBGTRACE("CorrectRange_Multiple_Consumers"); const TPartitionId partition{3}; const ui64 begin = 0; const ui64 end = 10; @@ -2276,34 +2276,34 @@ Y_UNIT_TEST_F(CorrectRange_Multiple_Consumers, TPartitionFixture) const ui64 step = 12345; const ui64 txId = 67890; - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); CreatePartition({.Partition=partition, .Begin=begin, .End=end}); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); CreateSession("client-1", "session-1"); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); CreateSession("client-2", "session-2"); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); SendSetOffset(1, "client-1", 3, "session-1"); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); SendCalcPredicate(step, txId, "client-2", 0, 1); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); SendSetOffset(2, "client-1", 6, "session-1"); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitCmdWrite({.Count=2, .UserInfos={{0, {.Session="session-1", .Offset=3}}}}); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); SendCmdWriteResponse(NMsgBusProxy::MSTATUS_OK); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitProxyResponse({.Cookie=1, .Status=NMsgBusProxy::MSTATUS_OK}); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitCalcPredicateResult({.Step=step, .TxId=txId, .Partition=TPartitionId(partition), .Predicate=true}); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); SendCommitTx(step, txId); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitCmdWrite({.Count=5, .UserInfos={ {1, {.Session="session-2", .Offset=1}}, {3, {.Session="session-1", .Offset=6}} @@ -2363,7 +2363,7 @@ Y_UNIT_TEST_F(IncorrectRange, TPartitionFixture) Y_UNIT_TEST_F(CorrectRange_Rollback, TPartitionFixture) { - DBGTRACE("CorrectRange_Rollback"); + //DBGTRACE("CorrectRange_Rollback"); const TPartitionId partition{3}; const ui64 begin = 0; const ui64 end = 10; @@ -2374,38 +2374,38 @@ Y_UNIT_TEST_F(CorrectRange_Rollback, TPartitionFixture) const ui64 txId_1 = 67890; const ui64 txId_2 = 67891; - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); CreatePartition({.Partition=partition, .Begin=begin, .End=end}); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); CreateSession(client, session); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); SendCalcPredicate(step, txId_1, client, 0, 2); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitCalcPredicateResult({.Step=step, .TxId=txId_1, .Partition=TPartitionId(partition), .Predicate=true}); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); SendCalcPredicate(step, txId_2, client, 0, 5); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); SendRollbackTx(step, txId_1); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitCmdWrite({.Count=1, .PlanStep=step, .TxId=txId_1, .UserInfos={{1, {.Consumer="client", .Session="session", .Offset=0}}}}); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); SendCmdWriteResponse(NMsgBusProxy::MSTATUS_OK); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitCmdWrite({.Count=1, .PlanStep=step, .TxId=txId_1, .UserInfos={{1, {.Consumer="client", .Session="session", .Offset=0}}}}); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); SendCmdWriteResponse(NMsgBusProxy::MSTATUS_OK); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitCalcPredicateResult({.Step=step, .TxId=txId_2, .Partition=TPartitionId(partition), .Predicate=true}); } Y_UNIT_TEST_F(ChangeConfig, TPartitionFixture) { - DBGTRACE("ChangeConfig"); + //DBGTRACE("ChangeConfig"); const TPartitionId partition{3}; const ui64 begin = 0; const ui64 end = 10; @@ -2414,7 +2414,7 @@ Y_UNIT_TEST_F(ChangeConfig, TPartitionFixture) const ui64 txId_1 = 67890; const ui64 txId_2 = 67891; - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); CreatePartition({ .Partition=partition, .Begin=begin, .End=end, .Config={.Consumers={ @@ -2424,10 +2424,10 @@ Y_UNIT_TEST_F(ChangeConfig, TPartitionFixture) }} }); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); SendCalcPredicate(step, txId_1, "client-1", 0, 2); Cerr << "Send change config\n"; - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); SendChangePartitionConfig({.Version=2, .Consumers={ {.Consumer="client-1", .Generation=0}, @@ -2436,15 +2436,15 @@ Y_UNIT_TEST_F(ChangeConfig, TPartitionFixture) // // consumer 'client-2' will be deleted // - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); SendCalcPredicate(step, txId_2, "client-2", 0, 2); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitCalcPredicateResult({.Step=step, .TxId=txId_1, .Partition=TPartitionId(partition), .Predicate=true}); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); SendCommitTx(step, txId_1); Cerr << "Wait cmd write (initial)\n"; - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitCmdWrite({.Count=8, .PlanStep=step, .TxId=txId_1, .UserInfos={ @@ -2452,10 +2452,10 @@ Y_UNIT_TEST_F(ChangeConfig, TPartitionFixture) }, }); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); SendCmdWriteResponse(NMsgBusProxy::MSTATUS_OK); Cerr << "Wait commit 1 done\n"; - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitCommitTxDone({.TxId=txId_1, .Partition=TPartitionId(partition)}); // @@ -2468,7 +2468,7 @@ Y_UNIT_TEST_F(ChangeConfig, TPartitionFixture) // }, // }); Cerr << "Wait cmd write (change config)\n"; - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitCmdWrite({.Count=8, .PlanStep=step, .TxId=txId_1, .UserInfos={ @@ -2478,18 +2478,18 @@ Y_UNIT_TEST_F(ChangeConfig, TPartitionFixture) .DeleteRanges={ {0, {.Partition=3, .Consumer="client-2"}} }}); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); SendCmdWriteResponse(NMsgBusProxy::MSTATUS_OK); Cerr << "Wait config changed\n"; - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitPartitionConfigChanged({.Partition=TPartitionId(partition)}); // // consumer 'client-2' was deleted // - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitCalcPredicateResult({.Step=step, .TxId=txId_2, .Partition=TPartitionId(partition), .Predicate=false}); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); SendRollbackTx(step, txId_2); } @@ -2809,61 +2809,61 @@ Y_UNIT_TEST_F(ShadowPartitionCountersRestore, TPartitionFixture) { Y_UNIT_TEST_F(DataTxCalcPredicateOk, TPartitionTxTestHelper) { - DBGTRACE("DataTxCalcPredicateOk"); + //DBGTRACE("DataTxCalcPredicateOk"); Init(); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); CreateSession("client", "session"); i64 cookie = 1; - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); auto tx1 = MakeAndSendWriteTx({}); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitWriteInfoRequest(tx1, true); Cerr << "Wait first predicate result " << Endl; - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitTxPredicateReply(tx1); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); auto tx2 = MakeAndSendWriteTx({{"src1", {1, 10}}}); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitWriteInfoRequest(tx2, true); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); SendTxCommit(tx1); Cerr << "Wait second predicate result " << Endl; - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitTxPredicateReply(tx2); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); SendTxCommit(tx2); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); EmulateKVTablet(); TString data = "data for write"; - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); SendChangeOwner(cookie, "owner1", Ctx->Edge, true); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); EmulateKVTablet(); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); auto ownerEvent = Ctx->Runtime->GrabEdgeEvent(TDuration::Seconds(1)); UNIT_ASSERT(ownerEvent != nullptr); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); auto ownerCookie = ownerEvent->Response->GetPartitionResponse().GetCmdGetOwnershipResult().GetOwnerCookie(); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); SendWrite(++cookie, 0, ownerCookie, 51, data, false, 5); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); EmulateKVTablet(); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitProxyResponse({.Cookie=cookie}); Cerr << "Wait third predicate result " << Endl; - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); auto tx3 = MakeAndSendWriteTx({{"src1", {12, 20}}, {"SourceId", {6, 10}}}); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitWriteInfoRequest(tx3, true); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitTxPredicateReply(tx3); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); SendTxCommit(tx3); } @@ -2961,101 +2961,101 @@ Y_UNIT_TEST_F(TestTxBatchInFederation, TPartitionTxTestHelper) { } Y_UNIT_TEST_F(ConflictingActsInSeveralBatches, TPartitionTxTestHelper) { - DBGTRACE("ConflictingActsInSeveralBatches"); + //DBGTRACE("ConflictingActsInSeveralBatches"); TTxBatchingTestParams params {.WriterSessions{"src1", "src4"},.EndOffset=1}; - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); Init(std::move(params)); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); auto tx1 = MakeAndSendWriteTx({{"src1", {1, 3}}}); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); auto tx2 = MakeAndSendWriteTx({{"src2", {4, 6}}}); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); auto tx3 = MakeAndSendWriteTx({{"src1", {4, 6}}}); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); AddAndSendNormalWrite("src1", 7, 12); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); AddAndSendNormalWrite("src4", 1, 2); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); auto tx5 = MakeAndSendWriteTx({{"src4", {4, 5}}}); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); AddAndSendNormalWrite("src4", 7, 12); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); auto immTx1 = MakeAndSendImmediateTx({{"src4", {13, 15}}}); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitWriteInfoRequest(tx1, true); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitWriteInfoRequest(tx2, true); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitWriteInfoRequest(tx3, true); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitWriteInfoRequest(tx5, true); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitWriteInfoRequest(immTx1, true); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitTxPredicateReply(tx1); - DBGTRACE_LOG(""); - WaitTxPredicateReply(tx2); //DBGTRACE_LOG(""); + WaitTxPredicateReply(tx2); + ////DBGTRACE_LOG(""); //WaitBatchCompletion(2); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); SendTxCommit(tx1); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); SendTxRollback(tx2); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitKvRequest(); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); SendKvResponse(); - DBGTRACE_LOG(""); - WaitTxPredicateReply(tx3); //DBGTRACE_LOG(""); + WaitTxPredicateReply(tx3); + ////DBGTRACE_LOG(""); //WaitBatchCompletion(1); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); SendTxCommit(tx3); //2 Normal writes with src1 & src4 - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); ExpectNoTxPredicateReply(); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitKvRequest(); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); SendKvResponse(); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitCommitDone(tx1); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitKvRequest(); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); SendKvResponse(); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitCommitDone(tx3); - DBGTRACE_LOG(""); - WaitTxPredicateReply(tx5); //DBGTRACE_LOG(""); + WaitTxPredicateReply(tx5); + ////DBGTRACE_LOG(""); //WaitBatchCompletion(6 + 2); // Normal writes produce 1 act for each message - DBGTRACE_LOG(""); - SendTxCommit(tx5); //DBGTRACE_LOG(""); + SendTxCommit(tx5); + ////DBGTRACE_LOG(""); //WaitBatchCompletion(1); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitKvRequest(); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); SendKvResponse(); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitCommitDone(tx5); - //DBGTRACE_LOG(""); + ////DBGTRACE_LOG(""); //WaitBatchCompletion(1 + 6); //Normal write & immTx for src4; - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitKvRequest(); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); SendKvResponse(); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitImmediateTxComplete(immTx1, true); } @@ -3163,78 +3163,78 @@ Y_UNIT_TEST_F(ConflictingSrcIdForTxInDifferentBatches, TPartitionTxTestHelper) { } Y_UNIT_TEST_F(ConflictingSrcIdTxAndWritesDifferentBatches, TPartitionTxTestHelper) { - DBGTRACE("ConflictingSrcIdTxAndWritesDifferentBatches"); + //DBGTRACE("ConflictingSrcIdTxAndWritesDifferentBatches"); TTxBatchingTestParams params {.WriterSessions{"src1"}, .EndOffset = 1}; - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); Init(std::move(params)); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); auto tx1 = MakeAndSendWriteTx({{"src1", {1, 3}},}); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); auto tx2 = MakeAndSendWriteTx({{"src1", {2, 4}}}); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); auto tx3 = MakeAndSendWriteTx({{"src1", {4, 6}}}); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); AddAndSendNormalWrite("src1", 1, 1); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); AddAndSendNormalWrite("src1", 7, 7); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); AddAndSendNormalWrite("src1", 7, 7); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitWriteInfoRequest(tx1, true); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitWriteInfoRequest(tx2, true); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitWriteInfoRequest(tx3, true); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitTxPredicateReply(tx1); - DBGTRACE_LOG(""); - SendTxCommit(tx1); //DBGTRACE_LOG(""); + SendTxCommit(tx1); + ////DBGTRACE_LOG(""); //WaitBatchCompletion(1); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitKvRequest(); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); SendKvResponse(); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitCommitDone(tx1); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitTxPredicateFailure(tx2); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitTxPredicateReply(tx3); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); SendTxRollback(tx2); - DBGTRACE_LOG(""); - SendTxCommit(tx3); //DBGTRACE_LOG(""); + SendTxCommit(tx3); + ////DBGTRACE_LOG(""); //WaitBatchCompletion(2); // Tx 2 & 3. - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitKvRequest(); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); SendKvResponse(); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitKvRequest(); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); SendKvResponse(); - DBGTRACE_LOG(""); - WaitCommitDone(tx3); //DBGTRACE_LOG(""); + WaitCommitDone(tx3); + ////DBGTRACE_LOG(""); //WaitBatchCompletion(3); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitKvRequest(); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); SendKvResponse(); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitProxyResponse({.AlreadyWritten=true, .SeqNo=1}); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitProxyResponse({.AlreadyWritten=false, .SeqNo=7}); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitProxyResponse({.AlreadyWritten=true, .SeqNo=7}); } @@ -3312,172 +3312,172 @@ class TBatchingConditionsTest { }; Y_UNIT_TEST_F(DifferentWriteTxBatchingOptions, TPartitionTxTestHelper) { - DBGTRACE("DifferentWriteTxBatchingOptions"); + //DBGTRACE("DifferentWriteTxBatchingOptions"); auto wrapper = TBatchingConditionsTest(this); // 1. ImmTx -> NormWrite -> ImmTx -> NormWrite = All batched { - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); wrapper.Start(); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); wrapper.AddNormalWrite(); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); auto immTx1 = wrapper.AddImmediateTx(); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); wrapper.AddNormalWrite(); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); auto immTx2 = wrapper.AddImmediateTx(); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); wrapper.Process(); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitWriteInfoRequest(immTx1, true); - DBGTRACE_LOG(""); - WaitWriteInfoRequest(immTx2, true); //DBGTRACE_LOG(""); + WaitWriteInfoRequest(immTx2, true); + ////DBGTRACE_LOG(""); //WaitBatchCompletion(4 + 1); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); EmulateKVTablet(); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); EmulateKVTablet(); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitImmediateTxComplete(immTx1, true); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitImmediateTxComplete(immTx2, true); } { // 2. ImmTx -> WriteTx = KVRequest - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); ResetBatchCompletion(); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); wrapper.Start(); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); auto immTx = wrapper.AddImmediateTx(); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); auto tx = wrapper.AddTx(); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); wrapper.Process(); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitWriteInfoRequest(immTx, true); - DBGTRACE_LOG(""); - WaitWriteInfoRequest(tx, true); //DBGTRACE_LOG(""); + WaitWriteInfoRequest(tx, true); + ////DBGTRACE_LOG(""); //WaitBatchCompletion(1+1); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); ExpectNoTxPredicateReply(); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); EmulateKVTablet(); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); EmulateKVTablet(); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitImmediateTxComplete(immTx, true); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); ExpectNoCommitDone(); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitTxPredicateReply(tx); - DBGTRACE_LOG(""); - SendTxCommit(tx); //DBGTRACE_LOG(""); + SendTxCommit(tx); + ////DBGTRACE_LOG(""); //WaitBatchCompletion(1); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); EmulateKVTablet(); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitCommitDone(tx); } { // 3. NormWrite -> WriteTx = KVRequest - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); ResetBatchCompletion(); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); wrapper.Start(); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); wrapper.AddNormalWrite(); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); auto tx = wrapper.AddTx(); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); wrapper.Process(); - DBGTRACE_LOG(""); - WaitWriteInfoRequest(tx, true); //DBGTRACE_LOG(""); + WaitWriteInfoRequest(tx, true); + ////DBGTRACE_LOG(""); //WaitBatchCompletion(1+1); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); ExpectNoTxPredicateReply(); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); EmulateKVTablet(); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitTxPredicateReply(tx); - DBGTRACE_LOG(""); - SendTxCommit(tx); //DBGTRACE_LOG(""); + SendTxCommit(tx); + ////DBGTRACE_LOG(""); //WaitBatchCompletion(1); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); EmulateKVTablet(); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitCommitDone(tx); } { // 4. WriteTx -> NormWrite = 2 batches - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); ResetBatchCompletion(); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); wrapper.Start(); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); auto tx = wrapper.AddTx(); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); wrapper.AddNormalWrite(); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); wrapper.Process(); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitWriteInfoRequest(tx, true); - DBGTRACE_LOG(""); - WaitTxPredicateReply(tx); //DBGTRACE_LOG(""); + WaitTxPredicateReply(tx); + ////DBGTRACE_LOG(""); //WaitBatchCompletion(1+1); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); ExpectNoKvRequest(); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); SendTxCommit(tx); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); EmulateKVTablet(); - DBGTRACE_LOG(""); - WaitCommitDone(tx); //DBGTRACE_LOG(""); + WaitCommitDone(tx); + ////DBGTRACE_LOG(""); //WaitBatchCompletion(1); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); EmulateKVTablet(); } { // 5. WriteTx -> ImmTx = 2 batches - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); ResetBatchCompletion(); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); wrapper.Start(); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); auto tx = wrapper.AddTx(); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); auto immTx = wrapper.AddImmediateTx(); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); wrapper.Process(); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitWriteInfoRequest(tx, true); - DBGTRACE_LOG(""); - WaitWriteInfoRequest(immTx, true); //DBGTRACE_LOG(""); + WaitWriteInfoRequest(immTx, true); + ////DBGTRACE_LOG(""); //WaitBatchCompletion(1+1); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitTxPredicateReply(tx); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); SendTxCommit(tx); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); ExpectNoCommitDone(); - DBGTRACE_LOG(""); - EmulateKVTablet(); //DBGTRACE_LOG(""); + EmulateKVTablet(); + ////DBGTRACE_LOG(""); //WaitBatchCompletion(1); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitCommitDone(tx); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); EmulateKVTablet(); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitImmediateTxComplete(immTx, true); } } @@ -3531,167 +3531,167 @@ Y_UNIT_TEST_F(FailedTxsDontBlock, TPartitionTxTestHelper) { } Y_UNIT_TEST_F(NonConflictingCommitsBatch, TPartitionTxTestHelper) { - DBGTRACE("NonConflictingCommitsBatch"); + //DBGTRACE("NonConflictingCommitsBatch"); TTxBatchingTestParams params{ .ConsumersCount= 3, .ConsumerSessions={1}, .EndOffset=50 }; - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); Init(std::move(params)); //Just block processing so every message arrives before batching starts - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); auto txTmp = MakeAndSendWriteTx({}); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); MakeAndSendNormalOffsetCommit(1, 5); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); auto tx1 = MakeAndSendTxOffsetCommit(3, 0, 5); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); auto tx2 = MakeAndSendTxOffsetCommit(2, 0, 5); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); MakeAndSendNormalOffsetCommit(1, 10); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); auto txImm1 = MakeAndSendImmediateTxOffsetCommit(1, 0, 15); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitWriteInfoRequest(txTmp, true); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitTxPredicateReply(txTmp); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); SendTxRollback(txTmp); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitTxPredicateReply(tx1); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitTxPredicateReply(tx2); - //DBGTRACE_LOG(""); + ////DBGTRACE_LOG(""); //WaitBatchCompletion(5 + 1 /*tmpTx*/); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); SendTxCommit(tx1); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); SendTxCommit(tx2); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitKvRequest(); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); SendKvResponse(); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitKvRequest(); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); SendKvResponse(); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitCommitDone(tx1); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitCommitDone(tx2); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitImmediateTxComplete(txImm1, false); } Y_UNIT_TEST_F(ConflictingCommitsInSeveralBatches, TPartitionTxTestHelper) { - DBGTRACE("ConflictingCommitsInSeveralBatches"); + //DBGTRACE("ConflictingCommitsInSeveralBatches"); TTxBatchingTestParams params{ .ConsumersCount= 2, .ConsumerSessions={1}, .EndOffset=50 }; - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); Init(std::move(params)); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); //Just block processing so every message arrives before batching starts auto txTmp = MakeAndSendWriteTx({}); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); MakeAndSendNormalOffsetCommit(1, 2); // act-1 - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); auto tx1 = MakeAndSendTxOffsetCommit(1, 2, 5); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); auto tx2 = MakeAndSendTxOffsetCommit(1, 5, 10); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); MakeAndSendNormalOffsetCommit(1, 20); // act-2 - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); ResetBatchCompletion(); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitWriteInfoRequest(txTmp, true); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitTxPredicateReply(txTmp); - DBGTRACE_LOG(""); - SendTxRollback(txTmp); //DBGTRACE_LOG(""); + SendTxRollback(txTmp); + ////DBGTRACE_LOG(""); //WaitBatchCompletion(2); // txTmp + act-1 - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); ExpectNoTxPredicateReply(); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitKvRequest(); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); SendKvResponse(); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitTxPredicateReply(tx1); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitKvRequest(); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); SendKvResponse(); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); ExpectNoTxPredicateReply(); - DBGTRACE_LOG(""); - SendTxCommit(tx1); //DBGTRACE_LOG(""); + SendTxCommit(tx1); + ////DBGTRACE_LOG(""); //WaitBatchCompletion(1); // tx1 - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitTxPredicateReply(tx2); - DBGTRACE_LOG(""); - SendTxCommit(tx2); //DBGTRACE_LOG(""); + SendTxCommit(tx2); + ////DBGTRACE_LOG(""); //WaitBatchCompletion(1); // tx2 - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitKvRequest(); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); SendKvResponse(); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitCommitDone(tx1); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitKvRequest(); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); SendKvResponse(); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitCommitDone(tx2); - //DBGTRACE_LOG(""); + ////DBGTRACE_LOG(""); //WaitBatchCompletion(1); // act-2 - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitKvRequest(); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); SendKvResponse(); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); txTmp = MakeAndSendWriteTx({}); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); auto immTx1 = MakeAndSendImmediateTxOffsetCommit(2, 0, 5); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); auto immTx2 = MakeAndSendImmediateTxOffsetCommit(2, 5, 10); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitWriteInfoRequest(txTmp, true); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitTxPredicateReply(txTmp); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); SendTxRollback(txTmp); - DBGTRACE_LOG(""); - WaitKvRequest(); //DBGTRACE_LOG(""); + WaitKvRequest(); + ////DBGTRACE_LOG(""); //WaitBatchCompletion(3); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); SendKvResponse(); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitImmediateTxComplete(immTx1, true); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitImmediateTxComplete(immTx2, true); } @@ -3768,153 +3768,153 @@ Y_UNIT_TEST_F(ConflictingCommitFails, TPartitionTxTestHelper) { } Y_UNIT_TEST_F(ConflictingCommitProccesAfterRollback, TPartitionTxTestHelper) { - DBGTRACE("ConflictingCommitProccesAfterRollback"); + //DBGTRACE("ConflictingCommitProccesAfterRollback"); TTxBatchingTestParams params{ .ConsumersCount = 2, .EndOffset=50 }; - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); Init(std::move(params)); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); auto tx1 = MakeAndSendTxOffsetCommit(1, 0, 5); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); auto tx2 = MakeAndSendTxOffsetCommit(1, 0, 3); - DBGTRACE_LOG(""); - WaitTxPredicateReply(tx1); //DBGTRACE_LOG(""); + WaitTxPredicateReply(tx1); + ////DBGTRACE_LOG(""); //WaitBatchCompletion(1); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); SendTxRollback(tx1); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitKvRequest(); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); SendKvResponse(); - DBGTRACE_LOG(""); - WaitTxPredicateReply(tx2); //DBGTRACE_LOG(""); + WaitTxPredicateReply(tx2); + ////DBGTRACE_LOG(""); //WaitBatchCompletion(1); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); SendTxCommit(tx2); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitKvRequest(); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); SendKvResponse(); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitKvRequest(); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); SendKvResponse(); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitCommitDone(tx2); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); ExpectNoCommitDone(); } Y_UNIT_TEST_F(TestBatchingWithChangeConfig, TPartitionTxTestHelper) { - DBGTRACE("TestBatchingWithChangeConfig"); + //DBGTRACE("TestBatchingWithChangeConfig"); Init({.ConsumersCount = 2}); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); auto txTmp = MakeAndSendWriteTx({}); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); auto immTx1 = MakeAndSendImmediateTxOffsetCommit(1, 0, 5); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); SendChangePartitionConfig({.Version=2, .Consumers={ {.Consumer="client-0", .Offset=5, .Generation=0}, {.Consumer="client-1", .Generation=7} }}); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); auto immTx2 = MakeAndSendImmediateTxOffsetCommit(1, 5, 10); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitWriteInfoRequest(txTmp, true); - DBGTRACE_LOG(""); - SendTxRollback(txTmp); //DBGTRACE_LOG(""); + SendTxRollback(txTmp); + ////DBGTRACE_LOG(""); //WaitBatchCompletion(2); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); ExpectNoBatchCompletion(); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); EmulateKVTablet(); - DBGTRACE_LOG(""); - WaitImmediateTxComplete(immTx1, true); //DBGTRACE_LOG(""); + WaitImmediateTxComplete(immTx1, true); + ////DBGTRACE_LOG(""); //WaitBatchCompletion(1); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); EmulateKVTablet(); - DBGTRACE_LOG(""); - auto event = Ctx->Runtime->GrabEdgeEvent(); //DBGTRACE_LOG(""); + auto event = Ctx->Runtime->GrabEdgeEvent(); + ////DBGTRACE_LOG(""); //WaitBatchCompletion(1); // immTx2 - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); EmulateKVTablet(); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitImmediateTxComplete(immTx2, true); } Y_UNIT_TEST_F(TestBatchingWithProposeConfig, TPartitionTxTestHelper) { - DBGTRACE("TestBatchingWithProposeConfig"); + //DBGTRACE("TestBatchingWithProposeConfig"); Init({.ConsumersCount = 2}); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); auto txTmp = MakeAndSendWriteTx({}); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); auto immTx1 = MakeAndSendImmediateTxOffsetCommit(1, 0, 5); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); auto proposeTxId = GetTxId(); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); auto event = std::make_unique(1, proposeTxId); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); event->TopicConverter = TopicConverter; auto copy = Config; - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); copy.SetVersion(10); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); auto* newConsumer = copy.AddConsumers(); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); newConsumer->SetName("client-0"); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); newConsumer->SetGeneration(0); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); event->Config = std::move(copy); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); SendEvent(event.release()); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); auto immTx2 = MakeAndSendImmediateTxOffsetCommit(1, 5, 10); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitWriteInfoRequest(txTmp, true); - DBGTRACE_LOG(""); - SendTxRollback(txTmp); //DBGTRACE_LOG(""); + SendTxRollback(txTmp); + ////DBGTRACE_LOG(""); //WaitBatchCompletion(2); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); ExpectNoBatchCompletion(); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); EmulateKVTablet(); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitImmediateTxComplete(immTx1, true); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); SendCommitTx(1, proposeTxId); //ToDo - wait propose result; - //DBGTRACE_LOG(""); + ////DBGTRACE_LOG(""); //WaitBatchCompletion(1); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); EmulateKVTablet(); - DBGTRACE_LOG(""); - WaitCommitTxDone({.TxId=proposeTxId}); //DBGTRACE_LOG(""); + WaitCommitTxDone({.TxId=proposeTxId}); + ////DBGTRACE_LOG(""); //WaitBatchCompletion(1); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); EmulateKVTablet(); - DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); WaitImmediateTxComplete(immTx2, true); } diff --git a/ydb/core/persqueue/ut/ya.make b/ydb/core/persqueue/ut/ya.make index 1399b230d3c4..e815cdb07575 100644 --- a/ydb/core/persqueue/ut/ya.make +++ b/ydb/core/persqueue/ut/ya.make @@ -25,7 +25,6 @@ PEERDIR( ydb/public/sdk/cpp/src/client/persqueue_public/ut/ut_utils ydb/core/tx/schemeshard/ut_helpers - ydb/library/dbgtrace ) YQL_LAST_ABI_VERSION() From 3cfda904de34e1535edf35a90fb38aa23458ce87 Mon Sep 17 00:00:00 2001 From: Alexander Kotov Date: Thu, 6 Nov 2025 15:15:39 +0300 Subject: [PATCH 27/36] [-] ChangeConfig --- .../pqtablet/partition/partition.cpp | 24 +++++++++---------- .../persqueue/pqtablet/partition/partition.h | 2 +- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/ydb/core/persqueue/pqtablet/partition/partition.cpp b/ydb/core/persqueue/pqtablet/partition/partition.cpp index f130d05d44b9..80bd69e4f642 100644 --- a/ydb/core/persqueue/pqtablet/partition/partition.cpp +++ b/ydb/core/persqueue/pqtablet/partition/partition.cpp @@ -1393,7 +1393,15 @@ void TPartition::ProcessPendingEvent(std::unique_ptr ev, con } } + auto txIter = TransactionsInflight.begin(); + if (txIter->second->ProposeConfig) { + Y_ABORT_UNLESS(!ChangeConfig); + ChangeConfig = + MakeSimpleShared(TopicConverter, + txIter->second->ProposeConfig->Config); + PendingPartitionConfig = GetPartitionConfig(ChangeConfig->Config); + } if (ChangeConfig) { PQ_ENSURE(TransactionsInflight.size() == 1)("Step", ev->Step)("TxId", ev->TxId); PendingExplicitMessageGroups = ev->ExplicitMessageGroups; @@ -2959,7 +2967,7 @@ TPartition::EProcessResult TPartition::PreProcessUserActionOrTransaction(TSimple if (!FirstEvent) { return EProcessResult::Blocked; } - t->Predicate = BeginTransactionConfig(*t); + t->Predicate = BeginTransactionConfig(); ChangingConfig = true; PendingPartitionConfig = GetPartitionConfig(t->ProposeConfig->Config); //Y_VERIFY_DEBUG_S(PendingPartitionConfig, "Partition " << Partition << " config not found"); @@ -3008,10 +3016,7 @@ bool TPartition::ExecUserActionOrTransaction(TSimpleSharedPtr& t, SendChangeConfigReply = t->SendReply; BeginChangePartitionConfig(ChangeConfig->Config); } else if (t->ProposeConfig) { - PQ_ENSURE(ChangingConfig); - ChangeConfig = MakeSimpleShared(TopicConverter, - t->ProposeConfig->Config); - PendingPartitionConfig = GetPartitionConfig(ChangeConfig->Config); + PQ_ENSURE(ChangeConfig); SendChangeConfigReply = false; } CommitTransaction(t); @@ -3129,15 +3134,8 @@ TPartition::EProcessResult TPartition::BeginTransactionData(TTransaction& t, return EProcessResult::Continue; } -bool TPartition::BeginTransactionConfig(TTransaction& t) +bool TPartition::BeginTransactionConfig() { - const TEvPQ::TEvProposePartitionConfig& event = *t.ProposeConfig; - - ChangeConfig = - MakeSimpleShared(TopicConverter, - event.Config); - PendingPartitionConfig = GetPartitionConfig(ChangeConfig->Config); - SendChangeConfigReply = false; return true; } diff --git a/ydb/core/persqueue/pqtablet/partition/partition.h b/ydb/core/persqueue/pqtablet/partition/partition.h index 912cb0a4f62d..c46d067aea40 100644 --- a/ydb/core/persqueue/pqtablet/partition/partition.h +++ b/ydb/core/persqueue/pqtablet/partition/partition.h @@ -462,7 +462,7 @@ class TPartition : public TBaseTabletActor { const TString& reason); THolder MakeCommitDone(ui64 step, ui64 txId); - bool BeginTransactionConfig(TTransaction& t); + bool BeginTransactionConfig(); void CommitTransaction(TSimpleSharedPtr& t); void RollbackTransaction(TSimpleSharedPtr& t); From fbc6cc73518c0befc12e27b2a7f154b28c582d4b Mon Sep 17 00:00:00 2001 From: Alexander Kotov Date: Thu, 6 Nov 2025 17:18:12 +0300 Subject: [PATCH 28/36] [-] test TPartitionTests::TestBatchingWithProposeConfig --- .../pqtablet/partition/partition.cpp | 56 +- ydb/core/persqueue/ut/partition_ut.cpp | 744 +++++++++--------- ydb/core/persqueue/ut/ya.make | 1 + 3 files changed, 401 insertions(+), 400 deletions(-) diff --git a/ydb/core/persqueue/pqtablet/partition/partition.cpp b/ydb/core/persqueue/pqtablet/partition/partition.cpp index 80bd69e4f642..0ff41ba61b18 100644 --- a/ydb/core/persqueue/pqtablet/partition/partition.cpp +++ b/ydb/core/persqueue/pqtablet/partition/partition.cpp @@ -27,7 +27,7 @@ #include #include #include -//#include +#include namespace NKafka { @@ -2810,7 +2810,7 @@ void TPartition::RunPersist() { //Done with counters. // for debugging purposes - //DumpKeyValueRequest(PersistRequest->Record); + DumpKeyValueRequest(PersistRequest->Record); for (auto& traceId : TxForPersistTraceIds) { TxForPersistSpans.emplace_back(TWilsonTopic::TopicTopLevel, @@ -2857,32 +2857,32 @@ bool TPartition::TryAddDeleteHeadKeysToPersistRequest() return haveChanges; } -//void TPartition::DumpKeyValueRequest(const NKikimrClient::TKeyValueRequest& request) const -//{ -// DBGTRACE_LOG("=== DumpKeyValueRequest ==="); -// DBGTRACE_LOG("--- delete ----------------"); -// for (size_t i = 0; i < request.CmdDeleteRangeSize(); ++i) { -// const auto& cmd = request.GetCmdDeleteRange(i); -// const auto& range = cmd.GetRange(); -// Y_UNUSED(range); -// DBGTRACE_LOG((range.GetIncludeFrom() ? '[' : '(') << range.GetFrom() << -// ", " << -// range.GetTo() << (range.GetIncludeTo() ? ']' : ')')); -// } -// DBGTRACE_LOG("--- write -----------------"); -// for (size_t i = 0; i < request.CmdWriteSize(); ++i) { -// const auto& cmd = request.GetCmdWrite(i); -// Y_UNUSED(cmd); -// DBGTRACE_LOG(cmd.GetKey()); -// } -// DBGTRACE_LOG("--- rename ----------------"); -// for (size_t i = 0; i < request.CmdRenameSize(); ++i) { -// const auto& cmd = request.GetCmdRename(i); -// Y_UNUSED(cmd); -// DBGTRACE_LOG(cmd.GetOldKey() << ", " << cmd.GetNewKey()); -// } -// DBGTRACE_LOG("==========================="); -//} +void TPartition::DumpKeyValueRequest(const NKikimrClient::TKeyValueRequest& request) const +{ + DBGTRACE_LOG("=== DumpKeyValueRequest ==="); + DBGTRACE_LOG("--- delete ----------------"); + for (size_t i = 0; i < request.CmdDeleteRangeSize(); ++i) { + const auto& cmd = request.GetCmdDeleteRange(i); + const auto& range = cmd.GetRange(); + Y_UNUSED(range); + DBGTRACE_LOG((range.GetIncludeFrom() ? '[' : '(') << range.GetFrom() << + ", " << + range.GetTo() << (range.GetIncludeTo() ? ']' : ')')); + } + DBGTRACE_LOG("--- write -----------------"); + for (size_t i = 0; i < request.CmdWriteSize(); ++i) { + const auto& cmd = request.GetCmdWrite(i); + Y_UNUSED(cmd); + DBGTRACE_LOG(cmd.GetKey()); + } + DBGTRACE_LOG("--- rename ----------------"); + for (size_t i = 0; i < request.CmdRenameSize(); ++i) { + const auto& cmd = request.GetCmdRename(i); + Y_UNUSED(cmd); + DBGTRACE_LOG(cmd.GetOldKey() << ", " << cmd.GetNewKey()); + } + DBGTRACE_LOG("==========================="); +} //void TPartition::DumpZones(const char* file, unsigned line) const //{ diff --git a/ydb/core/persqueue/ut/partition_ut.cpp b/ydb/core/persqueue/ut/partition_ut.cpp index cc41a1754d6e..605a12d68868 100644 --- a/ydb/core/persqueue/ut/partition_ut.cpp +++ b/ydb/core/persqueue/ut/partition_ut.cpp @@ -22,7 +22,7 @@ #include #include "make_config.h" -//#include +#include template<> void Out(IOutputStream& out, NKikimrPQ::TEvProposeTransactionResult_EStatus v) { @@ -2173,7 +2173,7 @@ Y_UNIT_TEST_F(CommitOffsetRanges, TPartitionFixture) Y_UNIT_TEST_F(CorrectRange_Commit, TPartitionFixture) { - //DBGTRACE("CorrectRange_Commit"); + DBGTRACE("CorrectRange_Commit"); const TPartitionId partition{3}; const ui64 begin = 0; const ui64 end = 10; @@ -2183,36 +2183,36 @@ Y_UNIT_TEST_F(CorrectRange_Commit, TPartitionFixture) const ui64 step = 12345; const ui64 txId = 67890; - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); CreatePartition({.Partition=partition, .Begin=begin, .End=end, .PlanStep=step, .TxId=10000}); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); CreateSession(client, session); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); SendCalcPredicate(step, txId, client, 0, 2); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitCalcPredicateResult({.Step=step, .TxId=txId, .Partition=TPartitionId(partition), .Predicate=true}); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); SendCommitTx(step, txId); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitCmdWrite({.Count=3, .PlanStep=step, .TxId=txId, .UserInfos={{1, {.Session=session, .Offset=0}}}}); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); SendCmdWriteResponse(NMsgBusProxy::MSTATUS_OK); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitCmdWrite({.Count=3, .PlanStep=step, .TxId=txId, .UserInfos={{1, {.Session=session, .Offset=2}}}}); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); SendCmdWriteResponse(NMsgBusProxy::MSTATUS_OK); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitCommitTxDone({.TxId=txId, .Partition=TPartitionId(partition)}); } Y_UNIT_TEST_F(CorrectRange_Multiple_Transactions, TPartitionFixture) { - //DBGTRACE("CorrectRange_Multiple_Transactions"); + DBGTRACE("CorrectRange_Multiple_Transactions"); const TPartitionId partition{3}; const ui64 begin = 0; const ui64 end = 10; @@ -2224,51 +2224,51 @@ Y_UNIT_TEST_F(CorrectRange_Multiple_Transactions, TPartitionFixture) const ui64 txId_2 = 67891; const ui64 txId_3 = 67892; - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); CreatePartition({.Partition=partition, .Begin=begin, .End=end, .PlanStep=step, .TxId=10000}); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); CreateSession(client, session); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); SendCalcPredicate(step, txId_1, client, 0, 1); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitCalcPredicateResult({.Step=step, .TxId=txId_1, .Partition=TPartitionId(partition), .Predicate=true}); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); SendCalcPredicate(step, txId_2, client, 0, 2); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); SendCalcPredicate(step, txId_3, client, 0, 2); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); SendCommitTx(step, txId_1); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitCmdWrite({.Count=1, .PlanStep=step, .TxId=txId_1, .UserInfos={{1, {.Session=session, .Offset=1}}}}); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); SendCmdWriteResponse(NMsgBusProxy::MSTATUS_OK); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitCommitTxDone({.TxId=txId_1, .Partition=TPartitionId(partition)}); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitCalcPredicateResult({.Step=step, .TxId=txId_2, .Partition=TPartitionId(partition), .Predicate=false}); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); SendRollbackTx(step, txId_2); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitCalcPredicateResult({.Step=step, .TxId=txId_3, .Partition=TPartitionId(partition), .Predicate=false}); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); SendRollbackTx(step, txId_3); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitCmdWrite({.Count=1, .PlanStep=step, .TxId=txId_3, .UserInfos={{1, {.Session=session, .Offset=1}}}}); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); SendCmdWriteResponse(NMsgBusProxy::MSTATUS_OK); } Y_UNIT_TEST_F(CorrectRange_Multiple_Consumers, TPartitionFixture) { - //DBGTRACE("CorrectRange_Multiple_Consumers"); + DBGTRACE("CorrectRange_Multiple_Consumers"); const TPartitionId partition{3}; const ui64 begin = 0; const ui64 end = 10; @@ -2276,34 +2276,34 @@ Y_UNIT_TEST_F(CorrectRange_Multiple_Consumers, TPartitionFixture) const ui64 step = 12345; const ui64 txId = 67890; - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); CreatePartition({.Partition=partition, .Begin=begin, .End=end}); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); CreateSession("client-1", "session-1"); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); CreateSession("client-2", "session-2"); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); SendSetOffset(1, "client-1", 3, "session-1"); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); SendCalcPredicate(step, txId, "client-2", 0, 1); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); SendSetOffset(2, "client-1", 6, "session-1"); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitCmdWrite({.Count=2, .UserInfos={{0, {.Session="session-1", .Offset=3}}}}); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); SendCmdWriteResponse(NMsgBusProxy::MSTATUS_OK); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitProxyResponse({.Cookie=1, .Status=NMsgBusProxy::MSTATUS_OK}); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitCalcPredicateResult({.Step=step, .TxId=txId, .Partition=TPartitionId(partition), .Predicate=true}); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); SendCommitTx(step, txId); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitCmdWrite({.Count=5, .UserInfos={ {1, {.Session="session-2", .Offset=1}}, {3, {.Session="session-1", .Offset=6}} @@ -2363,7 +2363,7 @@ Y_UNIT_TEST_F(IncorrectRange, TPartitionFixture) Y_UNIT_TEST_F(CorrectRange_Rollback, TPartitionFixture) { - //DBGTRACE("CorrectRange_Rollback"); + DBGTRACE("CorrectRange_Rollback"); const TPartitionId partition{3}; const ui64 begin = 0; const ui64 end = 10; @@ -2374,38 +2374,38 @@ Y_UNIT_TEST_F(CorrectRange_Rollback, TPartitionFixture) const ui64 txId_1 = 67890; const ui64 txId_2 = 67891; - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); CreatePartition({.Partition=partition, .Begin=begin, .End=end}); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); CreateSession(client, session); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); SendCalcPredicate(step, txId_1, client, 0, 2); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitCalcPredicateResult({.Step=step, .TxId=txId_1, .Partition=TPartitionId(partition), .Predicate=true}); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); SendCalcPredicate(step, txId_2, client, 0, 5); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); SendRollbackTx(step, txId_1); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitCmdWrite({.Count=1, .PlanStep=step, .TxId=txId_1, .UserInfos={{1, {.Consumer="client", .Session="session", .Offset=0}}}}); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); SendCmdWriteResponse(NMsgBusProxy::MSTATUS_OK); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitCmdWrite({.Count=1, .PlanStep=step, .TxId=txId_1, .UserInfos={{1, {.Consumer="client", .Session="session", .Offset=0}}}}); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); SendCmdWriteResponse(NMsgBusProxy::MSTATUS_OK); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitCalcPredicateResult({.Step=step, .TxId=txId_2, .Partition=TPartitionId(partition), .Predicate=true}); } Y_UNIT_TEST_F(ChangeConfig, TPartitionFixture) { - //DBGTRACE("ChangeConfig"); + DBGTRACE("ChangeConfig"); const TPartitionId partition{3}; const ui64 begin = 0; const ui64 end = 10; @@ -2414,7 +2414,7 @@ Y_UNIT_TEST_F(ChangeConfig, TPartitionFixture) const ui64 txId_1 = 67890; const ui64 txId_2 = 67891; - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); CreatePartition({ .Partition=partition, .Begin=begin, .End=end, .Config={.Consumers={ @@ -2424,10 +2424,10 @@ Y_UNIT_TEST_F(ChangeConfig, TPartitionFixture) }} }); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); SendCalcPredicate(step, txId_1, "client-1", 0, 2); Cerr << "Send change config\n"; - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); SendChangePartitionConfig({.Version=2, .Consumers={ {.Consumer="client-1", .Generation=0}, @@ -2436,15 +2436,15 @@ Y_UNIT_TEST_F(ChangeConfig, TPartitionFixture) // // consumer 'client-2' will be deleted // - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); SendCalcPredicate(step, txId_2, "client-2", 0, 2); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitCalcPredicateResult({.Step=step, .TxId=txId_1, .Partition=TPartitionId(partition), .Predicate=true}); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); SendCommitTx(step, txId_1); Cerr << "Wait cmd write (initial)\n"; - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitCmdWrite({.Count=8, .PlanStep=step, .TxId=txId_1, .UserInfos={ @@ -2452,10 +2452,10 @@ Y_UNIT_TEST_F(ChangeConfig, TPartitionFixture) }, }); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); SendCmdWriteResponse(NMsgBusProxy::MSTATUS_OK); Cerr << "Wait commit 1 done\n"; - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitCommitTxDone({.TxId=txId_1, .Partition=TPartitionId(partition)}); // @@ -2468,7 +2468,7 @@ Y_UNIT_TEST_F(ChangeConfig, TPartitionFixture) // }, // }); Cerr << "Wait cmd write (change config)\n"; - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitCmdWrite({.Count=8, .PlanStep=step, .TxId=txId_1, .UserInfos={ @@ -2478,18 +2478,18 @@ Y_UNIT_TEST_F(ChangeConfig, TPartitionFixture) .DeleteRanges={ {0, {.Partition=3, .Consumer="client-2"}} }}); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); SendCmdWriteResponse(NMsgBusProxy::MSTATUS_OK); Cerr << "Wait config changed\n"; - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitPartitionConfigChanged({.Partition=TPartitionId(partition)}); // // consumer 'client-2' was deleted // - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitCalcPredicateResult({.Step=step, .TxId=txId_2, .Partition=TPartitionId(partition), .Predicate=false}); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); SendRollbackTx(step, txId_2); } @@ -2809,61 +2809,61 @@ Y_UNIT_TEST_F(ShadowPartitionCountersRestore, TPartitionFixture) { Y_UNIT_TEST_F(DataTxCalcPredicateOk, TPartitionTxTestHelper) { - //DBGTRACE("DataTxCalcPredicateOk"); + DBGTRACE("DataTxCalcPredicateOk"); Init(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); CreateSession("client", "session"); i64 cookie = 1; - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); auto tx1 = MakeAndSendWriteTx({}); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitWriteInfoRequest(tx1, true); Cerr << "Wait first predicate result " << Endl; - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitTxPredicateReply(tx1); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); auto tx2 = MakeAndSendWriteTx({{"src1", {1, 10}}}); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitWriteInfoRequest(tx2, true); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); SendTxCommit(tx1); Cerr << "Wait second predicate result " << Endl; - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitTxPredicateReply(tx2); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); SendTxCommit(tx2); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); EmulateKVTablet(); TString data = "data for write"; - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); SendChangeOwner(cookie, "owner1", Ctx->Edge, true); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); EmulateKVTablet(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); auto ownerEvent = Ctx->Runtime->GrabEdgeEvent(TDuration::Seconds(1)); UNIT_ASSERT(ownerEvent != nullptr); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); auto ownerCookie = ownerEvent->Response->GetPartitionResponse().GetCmdGetOwnershipResult().GetOwnerCookie(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); SendWrite(++cookie, 0, ownerCookie, 51, data, false, 5); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); EmulateKVTablet(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitProxyResponse({.Cookie=cookie}); Cerr << "Wait third predicate result " << Endl; - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); auto tx3 = MakeAndSendWriteTx({{"src1", {12, 20}}, {"SourceId", {6, 10}}}); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitWriteInfoRequest(tx3, true); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitTxPredicateReply(tx3); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); SendTxCommit(tx3); } @@ -2961,101 +2961,101 @@ Y_UNIT_TEST_F(TestTxBatchInFederation, TPartitionTxTestHelper) { } Y_UNIT_TEST_F(ConflictingActsInSeveralBatches, TPartitionTxTestHelper) { - //DBGTRACE("ConflictingActsInSeveralBatches"); + DBGTRACE("ConflictingActsInSeveralBatches"); TTxBatchingTestParams params {.WriterSessions{"src1", "src4"},.EndOffset=1}; - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); Init(std::move(params)); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); auto tx1 = MakeAndSendWriteTx({{"src1", {1, 3}}}); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); auto tx2 = MakeAndSendWriteTx({{"src2", {4, 6}}}); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); auto tx3 = MakeAndSendWriteTx({{"src1", {4, 6}}}); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); AddAndSendNormalWrite("src1", 7, 12); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); AddAndSendNormalWrite("src4", 1, 2); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); auto tx5 = MakeAndSendWriteTx({{"src4", {4, 5}}}); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); AddAndSendNormalWrite("src4", 7, 12); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); auto immTx1 = MakeAndSendImmediateTx({{"src4", {13, 15}}}); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitWriteInfoRequest(tx1, true); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitWriteInfoRequest(tx2, true); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitWriteInfoRequest(tx3, true); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitWriteInfoRequest(tx5, true); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitWriteInfoRequest(immTx1, true); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitTxPredicateReply(tx1); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitTxPredicateReply(tx2); - ////DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); //WaitBatchCompletion(2); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); SendTxCommit(tx1); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); SendTxRollback(tx2); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitKvRequest(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); SendKvResponse(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitTxPredicateReply(tx3); - ////DBGTRACE_LOG(""); - //WaitBatchCompletion(1); //DBGTRACE_LOG(""); + //WaitBatchCompletion(1); + DBGTRACE_LOG(""); SendTxCommit(tx3); //2 Normal writes with src1 & src4 - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); ExpectNoTxPredicateReply(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitKvRequest(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); SendKvResponse(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitCommitDone(tx1); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitKvRequest(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); SendKvResponse(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitCommitDone(tx3); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitTxPredicateReply(tx5); - ////DBGTRACE_LOG(""); - //WaitBatchCompletion(6 + 2); // Normal writes produce 1 act for each message //DBGTRACE_LOG(""); + //WaitBatchCompletion(6 + 2); // Normal writes produce 1 act for each message + DBGTRACE_LOG(""); SendTxCommit(tx5); - ////DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); //WaitBatchCompletion(1); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitKvRequest(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); SendKvResponse(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitCommitDone(tx5); - ////DBGTRACE_LOG(""); - //WaitBatchCompletion(1 + 6); //Normal write & immTx for src4; //DBGTRACE_LOG(""); + //WaitBatchCompletion(1 + 6); //Normal write & immTx for src4; + DBGTRACE_LOG(""); WaitKvRequest(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); SendKvResponse(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitImmediateTxComplete(immTx1, true); } @@ -3163,78 +3163,78 @@ Y_UNIT_TEST_F(ConflictingSrcIdForTxInDifferentBatches, TPartitionTxTestHelper) { } Y_UNIT_TEST_F(ConflictingSrcIdTxAndWritesDifferentBatches, TPartitionTxTestHelper) { - //DBGTRACE("ConflictingSrcIdTxAndWritesDifferentBatches"); + DBGTRACE("ConflictingSrcIdTxAndWritesDifferentBatches"); TTxBatchingTestParams params {.WriterSessions{"src1"}, .EndOffset = 1}; - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); Init(std::move(params)); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); auto tx1 = MakeAndSendWriteTx({{"src1", {1, 3}},}); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); auto tx2 = MakeAndSendWriteTx({{"src1", {2, 4}}}); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); auto tx3 = MakeAndSendWriteTx({{"src1", {4, 6}}}); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); AddAndSendNormalWrite("src1", 1, 1); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); AddAndSendNormalWrite("src1", 7, 7); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); AddAndSendNormalWrite("src1", 7, 7); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitWriteInfoRequest(tx1, true); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitWriteInfoRequest(tx2, true); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitWriteInfoRequest(tx3, true); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitTxPredicateReply(tx1); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); SendTxCommit(tx1); - ////DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); //WaitBatchCompletion(1); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitKvRequest(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); SendKvResponse(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitCommitDone(tx1); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitTxPredicateFailure(tx2); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitTxPredicateReply(tx3); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); SendTxRollback(tx2); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); SendTxCommit(tx3); - ////DBGTRACE_LOG(""); - //WaitBatchCompletion(2); // Tx 2 & 3. //DBGTRACE_LOG(""); + //WaitBatchCompletion(2); // Tx 2 & 3. + DBGTRACE_LOG(""); WaitKvRequest(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); SendKvResponse(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitKvRequest(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); SendKvResponse(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitCommitDone(tx3); - ////DBGTRACE_LOG(""); - //WaitBatchCompletion(3); //DBGTRACE_LOG(""); + //WaitBatchCompletion(3); + DBGTRACE_LOG(""); WaitKvRequest(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); SendKvResponse(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitProxyResponse({.AlreadyWritten=true, .SeqNo=1}); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitProxyResponse({.AlreadyWritten=false, .SeqNo=7}); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitProxyResponse({.AlreadyWritten=true, .SeqNo=7}); } @@ -3312,172 +3312,172 @@ class TBatchingConditionsTest { }; Y_UNIT_TEST_F(DifferentWriteTxBatchingOptions, TPartitionTxTestHelper) { - //DBGTRACE("DifferentWriteTxBatchingOptions"); + DBGTRACE("DifferentWriteTxBatchingOptions"); auto wrapper = TBatchingConditionsTest(this); // 1. ImmTx -> NormWrite -> ImmTx -> NormWrite = All batched { - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); wrapper.Start(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); wrapper.AddNormalWrite(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); auto immTx1 = wrapper.AddImmediateTx(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); wrapper.AddNormalWrite(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); auto immTx2 = wrapper.AddImmediateTx(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); wrapper.Process(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitWriteInfoRequest(immTx1, true); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitWriteInfoRequest(immTx2, true); - ////DBGTRACE_LOG(""); - //WaitBatchCompletion(4 + 1); //DBGTRACE_LOG(""); + //WaitBatchCompletion(4 + 1); + DBGTRACE_LOG(""); EmulateKVTablet(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); EmulateKVTablet(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitImmediateTxComplete(immTx1, true); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitImmediateTxComplete(immTx2, true); } { // 2. ImmTx -> WriteTx = KVRequest - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); ResetBatchCompletion(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); wrapper.Start(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); auto immTx = wrapper.AddImmediateTx(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); auto tx = wrapper.AddTx(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); wrapper.Process(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitWriteInfoRequest(immTx, true); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitWriteInfoRequest(tx, true); - ////DBGTRACE_LOG(""); - //WaitBatchCompletion(1+1); //DBGTRACE_LOG(""); + //WaitBatchCompletion(1+1); + DBGTRACE_LOG(""); ExpectNoTxPredicateReply(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); EmulateKVTablet(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); EmulateKVTablet(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitImmediateTxComplete(immTx, true); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); ExpectNoCommitDone(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitTxPredicateReply(tx); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); SendTxCommit(tx); - ////DBGTRACE_LOG(""); - //WaitBatchCompletion(1); //DBGTRACE_LOG(""); + //WaitBatchCompletion(1); + DBGTRACE_LOG(""); EmulateKVTablet(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitCommitDone(tx); } { // 3. NormWrite -> WriteTx = KVRequest - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); ResetBatchCompletion(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); wrapper.Start(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); wrapper.AddNormalWrite(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); auto tx = wrapper.AddTx(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); wrapper.Process(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitWriteInfoRequest(tx, true); - ////DBGTRACE_LOG(""); - //WaitBatchCompletion(1+1); //DBGTRACE_LOG(""); + //WaitBatchCompletion(1+1); + DBGTRACE_LOG(""); ExpectNoTxPredicateReply(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); EmulateKVTablet(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitTxPredicateReply(tx); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); SendTxCommit(tx); - ////DBGTRACE_LOG(""); - //WaitBatchCompletion(1); //DBGTRACE_LOG(""); + //WaitBatchCompletion(1); + DBGTRACE_LOG(""); EmulateKVTablet(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitCommitDone(tx); } { // 4. WriteTx -> NormWrite = 2 batches - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); ResetBatchCompletion(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); wrapper.Start(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); auto tx = wrapper.AddTx(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); wrapper.AddNormalWrite(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); wrapper.Process(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitWriteInfoRequest(tx, true); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitTxPredicateReply(tx); - ////DBGTRACE_LOG(""); - //WaitBatchCompletion(1+1); //DBGTRACE_LOG(""); + //WaitBatchCompletion(1+1); + DBGTRACE_LOG(""); ExpectNoKvRequest(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); SendTxCommit(tx); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); EmulateKVTablet(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitCommitDone(tx); - ////DBGTRACE_LOG(""); - //WaitBatchCompletion(1); //DBGTRACE_LOG(""); + //WaitBatchCompletion(1); + DBGTRACE_LOG(""); EmulateKVTablet(); } { // 5. WriteTx -> ImmTx = 2 batches - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); ResetBatchCompletion(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); wrapper.Start(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); auto tx = wrapper.AddTx(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); auto immTx = wrapper.AddImmediateTx(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); wrapper.Process(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitWriteInfoRequest(tx, true); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitWriteInfoRequest(immTx, true); - ////DBGTRACE_LOG(""); - //WaitBatchCompletion(1+1); //DBGTRACE_LOG(""); + //WaitBatchCompletion(1+1); + DBGTRACE_LOG(""); WaitTxPredicateReply(tx); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); SendTxCommit(tx); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); ExpectNoCommitDone(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); EmulateKVTablet(); - ////DBGTRACE_LOG(""); - //WaitBatchCompletion(1); //DBGTRACE_LOG(""); + //WaitBatchCompletion(1); + DBGTRACE_LOG(""); WaitCommitDone(tx); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); EmulateKVTablet(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitImmediateTxComplete(immTx, true); } } @@ -3531,167 +3531,167 @@ Y_UNIT_TEST_F(FailedTxsDontBlock, TPartitionTxTestHelper) { } Y_UNIT_TEST_F(NonConflictingCommitsBatch, TPartitionTxTestHelper) { - //DBGTRACE("NonConflictingCommitsBatch"); + DBGTRACE("NonConflictingCommitsBatch"); TTxBatchingTestParams params{ .ConsumersCount= 3, .ConsumerSessions={1}, .EndOffset=50 }; - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); Init(std::move(params)); //Just block processing so every message arrives before batching starts - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); auto txTmp = MakeAndSendWriteTx({}); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); MakeAndSendNormalOffsetCommit(1, 5); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); auto tx1 = MakeAndSendTxOffsetCommit(3, 0, 5); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); auto tx2 = MakeAndSendTxOffsetCommit(2, 0, 5); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); MakeAndSendNormalOffsetCommit(1, 10); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); auto txImm1 = MakeAndSendImmediateTxOffsetCommit(1, 0, 15); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitWriteInfoRequest(txTmp, true); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitTxPredicateReply(txTmp); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); SendTxRollback(txTmp); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitTxPredicateReply(tx1); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitTxPredicateReply(tx2); - ////DBGTRACE_LOG(""); - //WaitBatchCompletion(5 + 1 /*tmpTx*/); //DBGTRACE_LOG(""); + //WaitBatchCompletion(5 + 1 /*tmpTx*/); + DBGTRACE_LOG(""); SendTxCommit(tx1); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); SendTxCommit(tx2); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitKvRequest(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); SendKvResponse(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitKvRequest(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); SendKvResponse(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitCommitDone(tx1); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitCommitDone(tx2); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitImmediateTxComplete(txImm1, false); } Y_UNIT_TEST_F(ConflictingCommitsInSeveralBatches, TPartitionTxTestHelper) { - //DBGTRACE("ConflictingCommitsInSeveralBatches"); + DBGTRACE("ConflictingCommitsInSeveralBatches"); TTxBatchingTestParams params{ .ConsumersCount= 2, .ConsumerSessions={1}, .EndOffset=50 }; - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); Init(std::move(params)); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); //Just block processing so every message arrives before batching starts auto txTmp = MakeAndSendWriteTx({}); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); MakeAndSendNormalOffsetCommit(1, 2); // act-1 - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); auto tx1 = MakeAndSendTxOffsetCommit(1, 2, 5); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); auto tx2 = MakeAndSendTxOffsetCommit(1, 5, 10); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); MakeAndSendNormalOffsetCommit(1, 20); // act-2 - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); ResetBatchCompletion(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitWriteInfoRequest(txTmp, true); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitTxPredicateReply(txTmp); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); SendTxRollback(txTmp); - ////DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); //WaitBatchCompletion(2); // txTmp + act-1 - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); ExpectNoTxPredicateReply(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitKvRequest(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); SendKvResponse(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitTxPredicateReply(tx1); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitKvRequest(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); SendKvResponse(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); ExpectNoTxPredicateReply(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); SendTxCommit(tx1); - ////DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); //WaitBatchCompletion(1); // tx1 - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitTxPredicateReply(tx2); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); SendTxCommit(tx2); - ////DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); //WaitBatchCompletion(1); // tx2 - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitKvRequest(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); SendKvResponse(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitCommitDone(tx1); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitKvRequest(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); SendKvResponse(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitCommitDone(tx2); - ////DBGTRACE_LOG(""); - //WaitBatchCompletion(1); // act-2 //DBGTRACE_LOG(""); + //WaitBatchCompletion(1); // act-2 + DBGTRACE_LOG(""); WaitKvRequest(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); SendKvResponse(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); txTmp = MakeAndSendWriteTx({}); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); auto immTx1 = MakeAndSendImmediateTxOffsetCommit(2, 0, 5); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); auto immTx2 = MakeAndSendImmediateTxOffsetCommit(2, 5, 10); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitWriteInfoRequest(txTmp, true); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitTxPredicateReply(txTmp); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); SendTxRollback(txTmp); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitKvRequest(); - ////DBGTRACE_LOG(""); - //WaitBatchCompletion(3); //DBGTRACE_LOG(""); + //WaitBatchCompletion(3); + DBGTRACE_LOG(""); SendKvResponse(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitImmediateTxComplete(immTx1, true); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitImmediateTxComplete(immTx2, true); } @@ -3768,153 +3768,153 @@ Y_UNIT_TEST_F(ConflictingCommitFails, TPartitionTxTestHelper) { } Y_UNIT_TEST_F(ConflictingCommitProccesAfterRollback, TPartitionTxTestHelper) { - //DBGTRACE("ConflictingCommitProccesAfterRollback"); + DBGTRACE("ConflictingCommitProccesAfterRollback"); TTxBatchingTestParams params{ .ConsumersCount = 2, .EndOffset=50 }; - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); Init(std::move(params)); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); auto tx1 = MakeAndSendTxOffsetCommit(1, 0, 5); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); auto tx2 = MakeAndSendTxOffsetCommit(1, 0, 3); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitTxPredicateReply(tx1); - ////DBGTRACE_LOG(""); + //DBGTRACE_LOG(""); //WaitBatchCompletion(1); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); SendTxRollback(tx1); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitKvRequest(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); SendKvResponse(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitTxPredicateReply(tx2); - ////DBGTRACE_LOG(""); - //WaitBatchCompletion(1); //DBGTRACE_LOG(""); + //WaitBatchCompletion(1); + DBGTRACE_LOG(""); SendTxCommit(tx2); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitKvRequest(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); SendKvResponse(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitKvRequest(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); SendKvResponse(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitCommitDone(tx2); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); ExpectNoCommitDone(); } Y_UNIT_TEST_F(TestBatchingWithChangeConfig, TPartitionTxTestHelper) { - //DBGTRACE("TestBatchingWithChangeConfig"); + DBGTRACE("TestBatchingWithChangeConfig"); Init({.ConsumersCount = 2}); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); auto txTmp = MakeAndSendWriteTx({}); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); auto immTx1 = MakeAndSendImmediateTxOffsetCommit(1, 0, 5); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); SendChangePartitionConfig({.Version=2, .Consumers={ {.Consumer="client-0", .Offset=5, .Generation=0}, {.Consumer="client-1", .Generation=7} }}); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); auto immTx2 = MakeAndSendImmediateTxOffsetCommit(1, 5, 10); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitWriteInfoRequest(txTmp, true); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); SendTxRollback(txTmp); - ////DBGTRACE_LOG(""); - //WaitBatchCompletion(2); //DBGTRACE_LOG(""); + //WaitBatchCompletion(2); + DBGTRACE_LOG(""); ExpectNoBatchCompletion(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); EmulateKVTablet(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitImmediateTxComplete(immTx1, true); - ////DBGTRACE_LOG(""); - //WaitBatchCompletion(1); //DBGTRACE_LOG(""); + //WaitBatchCompletion(1); + DBGTRACE_LOG(""); EmulateKVTablet(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); auto event = Ctx->Runtime->GrabEdgeEvent(); - ////DBGTRACE_LOG(""); - //WaitBatchCompletion(1); // immTx2 //DBGTRACE_LOG(""); + //WaitBatchCompletion(1); // immTx2 + DBGTRACE_LOG(""); EmulateKVTablet(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitImmediateTxComplete(immTx2, true); } Y_UNIT_TEST_F(TestBatchingWithProposeConfig, TPartitionTxTestHelper) { - //DBGTRACE("TestBatchingWithProposeConfig"); + DBGTRACE("TestBatchingWithProposeConfig"); Init({.ConsumersCount = 2}); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); auto txTmp = MakeAndSendWriteTx({}); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); auto immTx1 = MakeAndSendImmediateTxOffsetCommit(1, 0, 5); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); auto proposeTxId = GetTxId(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); auto event = std::make_unique(1, proposeTxId); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); event->TopicConverter = TopicConverter; auto copy = Config; - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); copy.SetVersion(10); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); auto* newConsumer = copy.AddConsumers(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); newConsumer->SetName("client-0"); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); newConsumer->SetGeneration(0); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); event->Config = std::move(copy); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); SendEvent(event.release()); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); auto immTx2 = MakeAndSendImmediateTxOffsetCommit(1, 5, 10); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitWriteInfoRequest(txTmp, true); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); SendTxRollback(txTmp); - ////DBGTRACE_LOG(""); - //WaitBatchCompletion(2); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); + WaitBatchCompletion(2); + DBGTRACE_LOG(""); ExpectNoBatchCompletion(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); EmulateKVTablet(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitImmediateTxComplete(immTx1, true); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); SendCommitTx(1, proposeTxId); //ToDo - wait propose result; - ////DBGTRACE_LOG(""); - //WaitBatchCompletion(1); //DBGTRACE_LOG(""); + //WaitBatchCompletion(1); + DBGTRACE_LOG(""); EmulateKVTablet(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitCommitTxDone({.TxId=proposeTxId}); - ////DBGTRACE_LOG(""); - //WaitBatchCompletion(1); //DBGTRACE_LOG(""); + //WaitBatchCompletion(1); + DBGTRACE_LOG(""); EmulateKVTablet(); - //DBGTRACE_LOG(""); + DBGTRACE_LOG(""); WaitImmediateTxComplete(immTx2, true); } diff --git a/ydb/core/persqueue/ut/ya.make b/ydb/core/persqueue/ut/ya.make index e815cdb07575..1399b230d3c4 100644 --- a/ydb/core/persqueue/ut/ya.make +++ b/ydb/core/persqueue/ut/ya.make @@ -25,6 +25,7 @@ PEERDIR( ydb/public/sdk/cpp/src/client/persqueue_public/ut/ut_utils ydb/core/tx/schemeshard/ut_helpers + ydb/library/dbgtrace ) YQL_LAST_ABI_VERSION() From 83859c18e6065b1e057035504a1735b9d6281305 Mon Sep 17 00:00:00 2001 From: Alexander Kotov Date: Thu, 6 Nov 2025 18:47:15 +0300 Subject: [PATCH 29/36] [-] test TPartitionTests::TestBatchingWithChangeConfig --- .../pqtablet/partition/partition.cpp | 251 ++++++++++-------- .../persqueue/pqtablet/partition/partition.h | 2 + ydb/core/persqueue/ut/partition_ut.cpp | 8 +- 3 files changed, 146 insertions(+), 115 deletions(-) diff --git a/ydb/core/persqueue/pqtablet/partition/partition.cpp b/ydb/core/persqueue/pqtablet/partition/partition.cpp index 0ff41ba61b18..0c5c075f55f1 100644 --- a/ydb/core/persqueue/pqtablet/partition/partition.cpp +++ b/ydb/core/persqueue/pqtablet/partition/partition.cpp @@ -123,7 +123,7 @@ static const TDuration WAKE_TIMEOUT = TDuration::Seconds(5); static const TDuration UPDATE_AVAIL_SIZE_INTERVAL = TDuration::MilliSeconds(100); static const TDuration MIN_UPDATE_COUNTERS_DELAY = TDuration::MilliSeconds(300); static const ui32 MAX_USERS = 1000; -static const ui32 MAX_KEYS = 10000; +//static const ui32 MAX_KEYS = 10000; static const ui32 MAX_TXS = 1000; static const ui32 MAX_WRITE_CYCLE_SIZE = 16_MB; @@ -2337,66 +2337,66 @@ void TPartition::ProcessTxsAndUserActs(const TActorContext&) RunPersist(); } -void TPartition::ProcessTxsAndUserActsOriginal(const TActorContext& ctx) -{ - if (KVWriteInProgress) { - LOG_D("Writing. Can't process transactions and user actions"); - return; - } - if (DeletePartitionState == DELETION_INITED) { - if (!PersistRequest) { - PersistRequest = MakeHolder(); - } - ScheduleNegativeReplies(); - ScheduleDeletePartitionDone(); - - AddCmdDeleteRangeForAllKeys(*PersistRequest); - - ctx.Send(BlobCache, PersistRequest.Release(), 0, 0, PersistRequestSpan.GetTraceId()); - PersistRequest = nullptr; - CurrentPersistRequestSpan = std::move(PersistRequestSpan); - PersistRequestSpan = NWilson::TSpan(); - DeletePartitionState = DELETION_IN_PROCESS; - KVWriteInProgress = true; - - return; - } - LOG_D("Batching state before ContinueProcessTxsAndUserActs: " << (int)BatchingState); - if (CanProcessUserActionAndTransactionEvents()) { - ContinueProcessTxsAndUserActs(ctx); - } - // Still preprocessing? Waiting for something - if (CanProcessUserActionAndTransactionEvents()) { - LOG_D("Still preprocessing - waiting for something"); - return; - } - LOG_D("Batching state after ContinueProcessTxsAndUserActs: " << (int)BatchingState); - - // Preprocessing complete; - if (CurrentBatchSize > 0) { - LOG_D("Batch completed (" << CurrentBatchSize << ")"); - Send(SelfId(), new TEvPQ::TEvTxBatchComplete(CurrentBatchSize)); - } - CurrentBatchSize = 0; - - if (UserActionAndTxPendingCommit.empty()) { - // Processing stopped and nothing to commit - finalize - BatchingState = ETxBatchingState::Finishing; - } else { - // Process commit queue - ProcessCommitQueue(); - } - // BatchingState can go to Finishing in ContinueProcessTxsAndUserActs. Therefore, it is necessary to check - // the size of the UserActionAndTxPendingCommit queue here. - if (!UserActionAndTxPendingCommit.empty()) { - // Still pending for come commits - LOG_D("Still pending for come commits"); - return; - } - LOG_D("Try persist"); - // Here we have an empty UserActionAndTxPendingCommit queue and BatchingState is equal to Finishing. - RunPersist(); -} +//void TPartition::ProcessTxsAndUserActsOriginal(const TActorContext& ctx) +//{ +// if (KVWriteInProgress) { +// LOG_D("Writing. Can't process transactions and user actions"); +// return; +// } +// if (DeletePartitionState == DELETION_INITED) { +// if (!PersistRequest) { +// PersistRequest = MakeHolder(); +// } +// ScheduleNegativeReplies(); +// ScheduleDeletePartitionDone(); +// +// AddCmdDeleteRangeForAllKeys(*PersistRequest); +// +// ctx.Send(BlobCache, PersistRequest.Release(), 0, 0, PersistRequestSpan.GetTraceId()); +// PersistRequest = nullptr; +// CurrentPersistRequestSpan = std::move(PersistRequestSpan); +// PersistRequestSpan = NWilson::TSpan(); +// DeletePartitionState = DELETION_IN_PROCESS; +// KVWriteInProgress = true; +// +// return; +// } +// LOG_D("Batching state before ContinueProcessTxsAndUserActs: " << (int)BatchingState); +// if (CanProcessUserActionAndTransactionEvents()) { +// ContinueProcessTxsAndUserActs(ctx); +// } +// // Still preprocessing? Waiting for something +// if (CanProcessUserActionAndTransactionEvents()) { +// LOG_D("Still preprocessing - waiting for something"); +// return; +// } +// LOG_D("Batching state after ContinueProcessTxsAndUserActs: " << (int)BatchingState); +// +// // Preprocessing complete; +// if (CurrentBatchSize > 0) { +// LOG_D("Batch completed (" << CurrentBatchSize << ")"); +// Send(SelfId(), new TEvPQ::TEvTxBatchComplete(CurrentBatchSize)); +// } +// CurrentBatchSize = 0; +// +// if (UserActionAndTxPendingCommit.empty()) { +// // Processing stopped and nothing to commit - finalize +// BatchingState = ETxBatchingState::Finishing; +// } else { +// // Process commit queue +// ProcessCommitQueue(); +// } +// // BatchingState can go to Finishing in ContinueProcessTxsAndUserActs. Therefore, it is necessary to check +// // the size of the UserActionAndTxPendingCommit queue here. +// if (!UserActionAndTxPendingCommit.empty()) { +// // Still pending for come commits +// LOG_D("Still pending for come commits"); +// return; +// } +// LOG_D("Try persist"); +// // Here we have an empty UserActionAndTxPendingCommit queue and BatchingState is equal to Finishing. +// RunPersist(); +//} bool TPartition::CanProcessUserActionAndTransactionEvents() const { @@ -2445,24 +2445,43 @@ void TPartition::DumpTheSizeOfInternalQueues() const { LOG_D("Events: " << UserActionAndTransactionEvents.size() << ", PendingCommits: " << UserActionAndTxPendingCommit.size() << - ", PendingWrites: " << UserActionAndTxPendingWrite.size()); + ", PendingWrites: " << UserActionAndTxPendingWrite.size() << + ", FirstEvent: " << FirstEvent); +} + +TString GetTransactionType(const TTransaction& tx) +{ + if (tx.Tx) { + return "Tx"; + } else if (tx.ProposeTransaction) { + return "ImmediateTx"; + } else if (tx.ProposeConfig) { + return "ProposeConfig"; + } else if (tx.ChangeConfig) { + return "ChangeConfig"; + } else { + return "???"; + } } auto TPartition::ProcessUserActionAndTxEvent(TSimpleSharedPtr& event, TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers) -> EProcessResult { + LOG_D("TPartition::ProcessUserActionAndTxEvent(TEvPQ::TEvSetClientInfo)"); return PreProcessUserActionOrTransaction(event, affectedSourceIdsAndConsumers); } auto TPartition::ProcessUserActionAndTxEvent(TSimpleSharedPtr& tx, TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers) -> EProcessResult { + LOG_D("TPartition::ProcessUserActionAndTxEvent(TTransaction[" << GetTransactionType(*tx) << "])"); return PreProcessUserActionOrTransaction(tx, affectedSourceIdsAndConsumers); } auto TPartition::ProcessUserActionAndTxEvent(TMessage& msg, TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers) -> EProcessResult { + LOG_D("TPartition::ProcessUserActionAndTxEvent(TMessage)"); return PreProcessUserActionOrTransaction(msg, affectedSourceIdsAndConsumers); } @@ -2517,68 +2536,71 @@ void TPartition::ProcessUserActionAndTxPendingCommits() { void TPartition::ProcessUserActionAndTxPendingCommit(TSimpleSharedPtr& event, TEvKeyValue::TEvRequest* request) { + LOG_D("TPartition::ProcessUserActionAndTxPendingCommit(TEvPQ::TEvSetClientInfo)"); ExecUserActionOrTransaction(event, request); } void TPartition::ProcessUserActionAndTxPendingCommit(TSimpleSharedPtr& tx, TEvKeyValue::TEvRequest* request) { + LOG_D("TPartition::ProcessUserActionAndTxPendingCommit(TTransaction[" << GetTransactionType(*tx) << "])"); ExecUserActionOrTransaction(tx, request); } void TPartition::ProcessUserActionAndTxPendingCommit(TMessage& msg, TEvKeyValue::TEvRequest* request) { + LOG_D("TPartition::ProcessUserActionAndTxPendingCommit(TMessage)"); ExecUserActionOrTransaction(msg, request); } -void TPartition::ContinueProcessTxsAndUserActs(const TActorContext&) -{ - PQ_ENSURE(!KVWriteInProgress); - - if (WriteCycleSizeEstimate >= MAX_WRITE_CYCLE_SIZE || WriteKeysSizeEstimate >= MAX_KEYS) { - BatchingState = ETxBatchingState::Finishing; - return; - } - while (CanProcessUserActionAndTransactionEvents() && !UserActionAndTransactionEvents.empty()) { - if (ChangingConfig) { - BatchingState = ETxBatchingState::Finishing; - break; - } - auto& front = UserActionAndTransactionEvents.front(); - if (TMessage* msg = std::get_if(&front.Event); msg && msg->WaitPreviousWriteSpan) { - msg->WaitPreviousWriteSpan.End(); - } - auto visitor = [this, &front](auto& event) { - return this->PreProcessUserActionOrTransaction(event, front.AffectedSourceIdsAndConsumers); - }; - switch (std::visit(visitor, front.Event)) { - case EProcessResult::Continue: - MoveUserActOrTxToCommitState(); - FirstEvent = false; - break; - case EProcessResult::ContinueDrop: - UserActionAndTransactionEvents.pop_front(); - break; - case EProcessResult::Break: - MoveUserActOrTxToCommitState(); - BatchingState = ETxBatchingState::Finishing; - FirstEvent = false; - break; - case EProcessResult::Blocked: - BatchingState = ETxBatchingState::Executing; - return; - case EProcessResult::NotReady: - return; - } - CurrentBatchSize += 1; - } - if (UserActionAndTransactionEvents.empty()) { - BatchingState = ETxBatchingState::Executing; - return; - } - -} +//void TPartition::ContinueProcessTxsAndUserActs(const TActorContext&) +//{ +// PQ_ENSURE(!KVWriteInProgress); +// +// if (WriteCycleSizeEstimate >= MAX_WRITE_CYCLE_SIZE || WriteKeysSizeEstimate >= MAX_KEYS) { +// BatchingState = ETxBatchingState::Finishing; +// return; +// } +// while (CanProcessUserActionAndTransactionEvents() && !UserActionAndTransactionEvents.empty()) { +// if (ChangingConfig) { +// BatchingState = ETxBatchingState::Finishing; +// break; +// } +// auto& front = UserActionAndTransactionEvents.front(); +// if (TMessage* msg = std::get_if(&front.Event); msg && msg->WaitPreviousWriteSpan) { +// msg->WaitPreviousWriteSpan.End(); +// } +// auto visitor = [this, &front](auto& event) { +// return this->PreProcessUserActionOrTransaction(event, front.AffectedSourceIdsAndConsumers); +// }; +// switch (std::visit(visitor, front.Event)) { +// case EProcessResult::Continue: +// MoveUserActOrTxToCommitState(); +// FirstEvent = false; +// break; +// case EProcessResult::ContinueDrop: +// UserActionAndTransactionEvents.pop_front(); +// break; +// case EProcessResult::Break: +// MoveUserActOrTxToCommitState(); +// BatchingState = ETxBatchingState::Finishing; +// FirstEvent = false; +// break; +// case EProcessResult::Blocked: +// BatchingState = ETxBatchingState::Executing; +// return; +// case EProcessResult::NotReady: +// return; +// } +// CurrentBatchSize += 1; +// } +// if (UserActionAndTransactionEvents.empty()) { +// BatchingState = ETxBatchingState::Executing; +// return; +// } +// +//} static void AppendToSet(const TVector& p, THashMap& q) { @@ -2964,7 +2986,8 @@ TPartition::EProcessResult TPartition::PreProcessUserActionOrTransaction(TSimple } return result; } else if (t->ProposeConfig) { - if (!FirstEvent) { + if (HasPendingCommitsOrPendingWrites()) { + LOG_D("Wait until the operation with the config becomes the first in the queue"); return EProcessResult::Blocked; } t->Predicate = BeginTransactionConfig(); @@ -2977,7 +3000,8 @@ TPartition::EProcessResult TPartition::PreProcessUserActionOrTransaction(TSimple PQ_ENSURE(t->ChangeConfig); PQ_ENSURE(!ChangeConfig && !ChangingConfig); - if (!FirstEvent) { + if (HasPendingCommitsOrPendingWrites()) { + LOG_D("Wait until the operation with the config becomes the first in the queue"); return EProcessResult::Blocked; } ChangingConfig = true; @@ -2989,6 +3013,11 @@ TPartition::EProcessResult TPartition::PreProcessUserActionOrTransaction(TSimple return result; } +bool TPartition::HasPendingCommitsOrPendingWrites() const +{ + return !UserActionAndTxPendingCommit.empty() || !UserActionAndTxPendingWrite.empty(); +} + bool TPartition::ExecUserActionOrTransaction(TSimpleSharedPtr& t, TEvKeyValue::TEvRequest*) { diff --git a/ydb/core/persqueue/pqtablet/partition/partition.h b/ydb/core/persqueue/pqtablet/partition/partition.h index c46d067aea40..16a34e05ad68 100644 --- a/ydb/core/persqueue/pqtablet/partition/partition.h +++ b/ydb/core/persqueue/pqtablet/partition/partition.h @@ -903,6 +903,8 @@ class TPartition : public TBaseTabletActor { void BeginAppendHeadWithNewWrites(const TActorContext& ctx); void EndAppendHeadWithNewWrites(const TActorContext& ctx); + bool HasPendingCommitsOrPendingWrites() const; + // // user actions and transactions // diff --git a/ydb/core/persqueue/ut/partition_ut.cpp b/ydb/core/persqueue/ut/partition_ut.cpp index 605a12d68868..22cd126a0cce 100644 --- a/ydb/core/persqueue/ut/partition_ut.cpp +++ b/ydb/core/persqueue/ut/partition_ut.cpp @@ -3833,16 +3833,16 @@ Y_UNIT_TEST_F(TestBatchingWithChangeConfig, TPartitionTxTestHelper) { WaitWriteInfoRequest(txTmp, true); DBGTRACE_LOG(""); SendTxRollback(txTmp); - //DBGTRACE_LOG(""); - //WaitBatchCompletion(2); + DBGTRACE_LOG(""); + WaitBatchCompletion(2); DBGTRACE_LOG(""); ExpectNoBatchCompletion(); DBGTRACE_LOG(""); EmulateKVTablet(); DBGTRACE_LOG(""); WaitImmediateTxComplete(immTx1, true); - //DBGTRACE_LOG(""); - //WaitBatchCompletion(1); + DBGTRACE_LOG(""); + WaitBatchCompletion(1); DBGTRACE_LOG(""); EmulateKVTablet(); DBGTRACE_LOG(""); From 3ea7953e087eb9a2172163b32ee47ed8e9c4622c Mon Sep 17 00:00:00 2001 From: Alexander Kotov Date: Thu, 6 Nov 2025 18:55:28 +0300 Subject: [PATCH 30/36] [-] test TPartitionTests::ChangeConfig --- ydb/core/persqueue/ut/partition_ut.cpp | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/ydb/core/persqueue/ut/partition_ut.cpp b/ydb/core/persqueue/ut/partition_ut.cpp index 22cd126a0cce..f74ba6c56199 100644 --- a/ydb/core/persqueue/ut/partition_ut.cpp +++ b/ydb/core/persqueue/ut/partition_ut.cpp @@ -2445,6 +2445,16 @@ Y_UNIT_TEST_F(ChangeConfig, TPartitionFixture) SendCommitTx(step, txId_1); Cerr << "Wait cmd write (initial)\n"; DBGTRACE_LOG(""); + WaitCmdWrite({.Count=8, + .PlanStep=step, .TxId=txId_1, + .UserInfos={ + {1, {.Consumer="client-1", .Session="session-1", .Offset=0}}, + }, + }); + + DBGTRACE_LOG(""); + SendCmdWriteResponse(NMsgBusProxy::MSTATUS_OK); + DBGTRACE_LOG(""); WaitCmdWrite({.Count=8, .PlanStep=step, .TxId=txId_1, .UserInfos={ From bcc7ab4e6c146a56778c3ec0f260e25152999578 Mon Sep 17 00:00:00 2001 From: Alexander Kotov Date: Thu, 6 Nov 2025 18:59:56 +0300 Subject: [PATCH 31/36] [-] test TPartitionTests::CorrectRange_Multiple_Consumers --- ydb/core/persqueue/ut/partition_ut.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ydb/core/persqueue/ut/partition_ut.cpp b/ydb/core/persqueue/ut/partition_ut.cpp index f74ba6c56199..0edfc0c755cf 100644 --- a/ydb/core/persqueue/ut/partition_ut.cpp +++ b/ydb/core/persqueue/ut/partition_ut.cpp @@ -2303,6 +2303,14 @@ Y_UNIT_TEST_F(CorrectRange_Multiple_Consumers, TPartitionFixture) DBGTRACE_LOG(""); SendCommitTx(step, txId); + DBGTRACE_LOG(""); + WaitCmdWrite({.Count=5, .UserInfos={ + {1, {.Session="session-2", .Offset=0}}, + {3, {.Session="session-1", .Offset=0}} + }}); + DBGTRACE_LOG(""); + SendCmdWriteResponse(NMsgBusProxy::MSTATUS_OK); + DBGTRACE_LOG(""); WaitCmdWrite({.Count=5, .UserInfos={ {1, {.Session="session-2", .Offset=1}}, From 7a71477e561676be65f8802e0067cc49a4e50580 Mon Sep 17 00:00:00 2001 From: Alexander Kotov Date: Thu, 6 Nov 2025 19:21:30 +0300 Subject: [PATCH 32/36] [-] test TPartitionTests::CorrectRange_Multiple_Transactions --- ydb/core/persqueue/ut/partition_ut.cpp | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/ydb/core/persqueue/ut/partition_ut.cpp b/ydb/core/persqueue/ut/partition_ut.cpp index 0edfc0c755cf..475d4036e962 100644 --- a/ydb/core/persqueue/ut/partition_ut.cpp +++ b/ydb/core/persqueue/ut/partition_ut.cpp @@ -2242,6 +2242,11 @@ Y_UNIT_TEST_F(CorrectRange_Multiple_Transactions, TPartitionFixture) DBGTRACE_LOG(""); SendCommitTx(step, txId_1); + DBGTRACE_LOG(""); + WaitCmdWrite({.Count=1, .PlanStep=step, .TxId=txId_1, .UserInfos={{1, {.Session=session, .Offset=0}}}}); + DBGTRACE_LOG(""); + SendCmdWriteResponse(NMsgBusProxy::MSTATUS_OK); + DBGTRACE_LOG(""); WaitCmdWrite({.Count=1, .PlanStep=step, .TxId=txId_1, .UserInfos={{1, {.Session=session, .Offset=1}}}}); DBGTRACE_LOG(""); @@ -2260,6 +2265,11 @@ Y_UNIT_TEST_F(CorrectRange_Multiple_Transactions, TPartitionFixture) DBGTRACE_LOG(""); SendRollbackTx(step, txId_3); + DBGTRACE_LOG(""); + WaitCmdWrite({.Count=1, .PlanStep=step, .TxId=txId_2, .UserInfos={{1, {.Session=session, .Offset=1}}}}); + DBGTRACE_LOG(""); + SendCmdWriteResponse(NMsgBusProxy::MSTATUS_OK); + DBGTRACE_LOG(""); WaitCmdWrite({.Count=1, .PlanStep=step, .TxId=txId_3, .UserInfos={{1, {.Session=session, .Offset=1}}}}); DBGTRACE_LOG(""); From 8a87522b8c9ab7b60a025d2b0da6dd88fa8b1645 Mon Sep 17 00:00:00 2001 From: Alexander Kotov Date: Thu, 6 Nov 2025 19:38:24 +0300 Subject: [PATCH 33/36] [/] tracing --- .../pqtablet/partition/partition.cpp | 55 ++- ydb/core/persqueue/ut/partition_ut.cpp | 379 ------------------ ydb/core/persqueue/ut/ya.make | 1 - 3 files changed, 27 insertions(+), 408 deletions(-) diff --git a/ydb/core/persqueue/pqtablet/partition/partition.cpp b/ydb/core/persqueue/pqtablet/partition/partition.cpp index 0c5c075f55f1..bdc3631e5ded 100644 --- a/ydb/core/persqueue/pqtablet/partition/partition.cpp +++ b/ydb/core/persqueue/pqtablet/partition/partition.cpp @@ -27,7 +27,6 @@ #include #include #include -#include namespace NKafka { @@ -2832,7 +2831,7 @@ void TPartition::RunPersist() { //Done with counters. // for debugging purposes - DumpKeyValueRequest(PersistRequest->Record); + //DumpKeyValueRequest(PersistRequest->Record); for (auto& traceId : TxForPersistTraceIds) { TxForPersistSpans.emplace_back(TWilsonTopic::TopicTopLevel, @@ -2879,32 +2878,32 @@ bool TPartition::TryAddDeleteHeadKeysToPersistRequest() return haveChanges; } -void TPartition::DumpKeyValueRequest(const NKikimrClient::TKeyValueRequest& request) const -{ - DBGTRACE_LOG("=== DumpKeyValueRequest ==="); - DBGTRACE_LOG("--- delete ----------------"); - for (size_t i = 0; i < request.CmdDeleteRangeSize(); ++i) { - const auto& cmd = request.GetCmdDeleteRange(i); - const auto& range = cmd.GetRange(); - Y_UNUSED(range); - DBGTRACE_LOG((range.GetIncludeFrom() ? '[' : '(') << range.GetFrom() << - ", " << - range.GetTo() << (range.GetIncludeTo() ? ']' : ')')); - } - DBGTRACE_LOG("--- write -----------------"); - for (size_t i = 0; i < request.CmdWriteSize(); ++i) { - const auto& cmd = request.GetCmdWrite(i); - Y_UNUSED(cmd); - DBGTRACE_LOG(cmd.GetKey()); - } - DBGTRACE_LOG("--- rename ----------------"); - for (size_t i = 0; i < request.CmdRenameSize(); ++i) { - const auto& cmd = request.GetCmdRename(i); - Y_UNUSED(cmd); - DBGTRACE_LOG(cmd.GetOldKey() << ", " << cmd.GetNewKey()); - } - DBGTRACE_LOG("==========================="); -} +//void TPartition::DumpKeyValueRequest(const NKikimrClient::TKeyValueRequest& request) const +//{ +// DBGTRACE_LOG("=== DumpKeyValueRequest ==="); +// DBGTRACE_LOG("--- delete ----------------"); +// for (size_t i = 0; i < request.CmdDeleteRangeSize(); ++i) { +// const auto& cmd = request.GetCmdDeleteRange(i); +// const auto& range = cmd.GetRange(); +// Y_UNUSED(range); +// DBGTRACE_LOG((range.GetIncludeFrom() ? '[' : '(') << range.GetFrom() << +// ", " << +// range.GetTo() << (range.GetIncludeTo() ? ']' : ')')); +// } +// DBGTRACE_LOG("--- write -----------------"); +// for (size_t i = 0; i < request.CmdWriteSize(); ++i) { +// const auto& cmd = request.GetCmdWrite(i); +// Y_UNUSED(cmd); +// DBGTRACE_LOG(cmd.GetKey()); +// } +// DBGTRACE_LOG("--- rename ----------------"); +// for (size_t i = 0; i < request.CmdRenameSize(); ++i) { +// const auto& cmd = request.GetCmdRename(i); +// Y_UNUSED(cmd); +// DBGTRACE_LOG(cmd.GetOldKey() << ", " << cmd.GetNewKey()); +// } +// DBGTRACE_LOG("==========================="); +//} //void TPartition::DumpZones(const char* file, unsigned line) const //{ diff --git a/ydb/core/persqueue/ut/partition_ut.cpp b/ydb/core/persqueue/ut/partition_ut.cpp index 475d4036e962..259bbea34fd5 100644 --- a/ydb/core/persqueue/ut/partition_ut.cpp +++ b/ydb/core/persqueue/ut/partition_ut.cpp @@ -22,7 +22,6 @@ #include #include "make_config.h" -#include template<> void Out(IOutputStream& out, NKikimrPQ::TEvProposeTransactionResult_EStatus v) { @@ -2173,7 +2172,6 @@ Y_UNIT_TEST_F(CommitOffsetRanges, TPartitionFixture) Y_UNIT_TEST_F(CorrectRange_Commit, TPartitionFixture) { - DBGTRACE("CorrectRange_Commit"); const TPartitionId partition{3}; const ui64 begin = 0; const ui64 end = 10; @@ -2183,36 +2181,25 @@ Y_UNIT_TEST_F(CorrectRange_Commit, TPartitionFixture) const ui64 step = 12345; const ui64 txId = 67890; - DBGTRACE_LOG(""); CreatePartition({.Partition=partition, .Begin=begin, .End=end, .PlanStep=step, .TxId=10000}); - DBGTRACE_LOG(""); CreateSession(client, session); - DBGTRACE_LOG(""); SendCalcPredicate(step, txId, client, 0, 2); - DBGTRACE_LOG(""); WaitCalcPredicateResult({.Step=step, .TxId=txId, .Partition=TPartitionId(partition), .Predicate=true}); - DBGTRACE_LOG(""); SendCommitTx(step, txId); - DBGTRACE_LOG(""); WaitCmdWrite({.Count=3, .PlanStep=step, .TxId=txId, .UserInfos={{1, {.Session=session, .Offset=0}}}}); - DBGTRACE_LOG(""); SendCmdWriteResponse(NMsgBusProxy::MSTATUS_OK); - DBGTRACE_LOG(""); WaitCmdWrite({.Count=3, .PlanStep=step, .TxId=txId, .UserInfos={{1, {.Session=session, .Offset=2}}}}); - DBGTRACE_LOG(""); SendCmdWriteResponse(NMsgBusProxy::MSTATUS_OK); - DBGTRACE_LOG(""); WaitCommitTxDone({.TxId=txId, .Partition=TPartitionId(partition)}); } Y_UNIT_TEST_F(CorrectRange_Multiple_Transactions, TPartitionFixture) { - DBGTRACE("CorrectRange_Multiple_Transactions"); const TPartitionId partition{3}; const ui64 begin = 0; const ui64 end = 10; @@ -2224,61 +2211,40 @@ Y_UNIT_TEST_F(CorrectRange_Multiple_Transactions, TPartitionFixture) const ui64 txId_2 = 67891; const ui64 txId_3 = 67892; - DBGTRACE_LOG(""); CreatePartition({.Partition=partition, .Begin=begin, .End=end, .PlanStep=step, .TxId=10000}); - DBGTRACE_LOG(""); CreateSession(client, session); - DBGTRACE_LOG(""); SendCalcPredicate(step, txId_1, client, 0, 1); - DBGTRACE_LOG(""); WaitCalcPredicateResult({.Step=step, .TxId=txId_1, .Partition=TPartitionId(partition), .Predicate=true}); - DBGTRACE_LOG(""); SendCalcPredicate(step, txId_2, client, 0, 2); - DBGTRACE_LOG(""); SendCalcPredicate(step, txId_3, client, 0, 2); - DBGTRACE_LOG(""); SendCommitTx(step, txId_1); - DBGTRACE_LOG(""); WaitCmdWrite({.Count=1, .PlanStep=step, .TxId=txId_1, .UserInfos={{1, {.Session=session, .Offset=0}}}}); - DBGTRACE_LOG(""); SendCmdWriteResponse(NMsgBusProxy::MSTATUS_OK); - DBGTRACE_LOG(""); WaitCmdWrite({.Count=1, .PlanStep=step, .TxId=txId_1, .UserInfos={{1, {.Session=session, .Offset=1}}}}); - DBGTRACE_LOG(""); SendCmdWriteResponse(NMsgBusProxy::MSTATUS_OK); - DBGTRACE_LOG(""); WaitCommitTxDone({.TxId=txId_1, .Partition=TPartitionId(partition)}); - DBGTRACE_LOG(""); WaitCalcPredicateResult({.Step=step, .TxId=txId_2, .Partition=TPartitionId(partition), .Predicate=false}); - DBGTRACE_LOG(""); SendRollbackTx(step, txId_2); - DBGTRACE_LOG(""); WaitCalcPredicateResult({.Step=step, .TxId=txId_3, .Partition=TPartitionId(partition), .Predicate=false}); - DBGTRACE_LOG(""); SendRollbackTx(step, txId_3); - DBGTRACE_LOG(""); WaitCmdWrite({.Count=1, .PlanStep=step, .TxId=txId_2, .UserInfos={{1, {.Session=session, .Offset=1}}}}); - DBGTRACE_LOG(""); SendCmdWriteResponse(NMsgBusProxy::MSTATUS_OK); - DBGTRACE_LOG(""); WaitCmdWrite({.Count=1, .PlanStep=step, .TxId=txId_3, .UserInfos={{1, {.Session=session, .Offset=1}}}}); - DBGTRACE_LOG(""); SendCmdWriteResponse(NMsgBusProxy::MSTATUS_OK); } Y_UNIT_TEST_F(CorrectRange_Multiple_Consumers, TPartitionFixture) { - DBGTRACE("CorrectRange_Multiple_Consumers"); const TPartitionId partition{3}; const ui64 begin = 0; const ui64 end = 10; @@ -2286,42 +2252,28 @@ Y_UNIT_TEST_F(CorrectRange_Multiple_Consumers, TPartitionFixture) const ui64 step = 12345; const ui64 txId = 67890; - DBGTRACE_LOG(""); CreatePartition({.Partition=partition, .Begin=begin, .End=end}); - DBGTRACE_LOG(""); CreateSession("client-1", "session-1"); - DBGTRACE_LOG(""); CreateSession("client-2", "session-2"); - DBGTRACE_LOG(""); SendSetOffset(1, "client-1", 3, "session-1"); - DBGTRACE_LOG(""); SendCalcPredicate(step, txId, "client-2", 0, 1); - DBGTRACE_LOG(""); SendSetOffset(2, "client-1", 6, "session-1"); - DBGTRACE_LOG(""); WaitCmdWrite({.Count=2, .UserInfos={{0, {.Session="session-1", .Offset=3}}}}); - DBGTRACE_LOG(""); SendCmdWriteResponse(NMsgBusProxy::MSTATUS_OK); - DBGTRACE_LOG(""); WaitProxyResponse({.Cookie=1, .Status=NMsgBusProxy::MSTATUS_OK}); - DBGTRACE_LOG(""); WaitCalcPredicateResult({.Step=step, .TxId=txId, .Partition=TPartitionId(partition), .Predicate=true}); - DBGTRACE_LOG(""); SendCommitTx(step, txId); - DBGTRACE_LOG(""); WaitCmdWrite({.Count=5, .UserInfos={ {1, {.Session="session-2", .Offset=0}}, {3, {.Session="session-1", .Offset=0}} }}); - DBGTRACE_LOG(""); SendCmdWriteResponse(NMsgBusProxy::MSTATUS_OK); - DBGTRACE_LOG(""); WaitCmdWrite({.Count=5, .UserInfos={ {1, {.Session="session-2", .Offset=1}}, {3, {.Session="session-1", .Offset=6}} @@ -2381,7 +2333,6 @@ Y_UNIT_TEST_F(IncorrectRange, TPartitionFixture) Y_UNIT_TEST_F(CorrectRange_Rollback, TPartitionFixture) { - DBGTRACE("CorrectRange_Rollback"); const TPartitionId partition{3}; const ui64 begin = 0; const ui64 end = 10; @@ -2392,38 +2343,26 @@ Y_UNIT_TEST_F(CorrectRange_Rollback, TPartitionFixture) const ui64 txId_1 = 67890; const ui64 txId_2 = 67891; - DBGTRACE_LOG(""); CreatePartition({.Partition=partition, .Begin=begin, .End=end}); - DBGTRACE_LOG(""); CreateSession(client, session); - DBGTRACE_LOG(""); SendCalcPredicate(step, txId_1, client, 0, 2); - DBGTRACE_LOG(""); WaitCalcPredicateResult({.Step=step, .TxId=txId_1, .Partition=TPartitionId(partition), .Predicate=true}); - DBGTRACE_LOG(""); SendCalcPredicate(step, txId_2, client, 0, 5); - DBGTRACE_LOG(""); SendRollbackTx(step, txId_1); - DBGTRACE_LOG(""); WaitCmdWrite({.Count=1, .PlanStep=step, .TxId=txId_1, .UserInfos={{1, {.Consumer="client", .Session="session", .Offset=0}}}}); - DBGTRACE_LOG(""); SendCmdWriteResponse(NMsgBusProxy::MSTATUS_OK); - DBGTRACE_LOG(""); WaitCmdWrite({.Count=1, .PlanStep=step, .TxId=txId_1, .UserInfos={{1, {.Consumer="client", .Session="session", .Offset=0}}}}); - DBGTRACE_LOG(""); SendCmdWriteResponse(NMsgBusProxy::MSTATUS_OK); - DBGTRACE_LOG(""); WaitCalcPredicateResult({.Step=step, .TxId=txId_2, .Partition=TPartitionId(partition), .Predicate=true}); } Y_UNIT_TEST_F(ChangeConfig, TPartitionFixture) { - DBGTRACE("ChangeConfig"); const TPartitionId partition{3}; const ui64 begin = 0; const ui64 end = 10; @@ -2432,7 +2371,6 @@ Y_UNIT_TEST_F(ChangeConfig, TPartitionFixture) const ui64 txId_1 = 67890; const ui64 txId_2 = 67891; - DBGTRACE_LOG(""); CreatePartition({ .Partition=partition, .Begin=begin, .End=end, .Config={.Consumers={ @@ -2442,10 +2380,8 @@ Y_UNIT_TEST_F(ChangeConfig, TPartitionFixture) }} }); - DBGTRACE_LOG(""); SendCalcPredicate(step, txId_1, "client-1", 0, 2); Cerr << "Send change config\n"; - DBGTRACE_LOG(""); SendChangePartitionConfig({.Version=2, .Consumers={ {.Consumer="client-1", .Generation=0}, @@ -2454,15 +2390,11 @@ Y_UNIT_TEST_F(ChangeConfig, TPartitionFixture) // // consumer 'client-2' will be deleted // - DBGTRACE_LOG(""); SendCalcPredicate(step, txId_2, "client-2", 0, 2); - DBGTRACE_LOG(""); WaitCalcPredicateResult({.Step=step, .TxId=txId_1, .Partition=TPartitionId(partition), .Predicate=true}); - DBGTRACE_LOG(""); SendCommitTx(step, txId_1); Cerr << "Wait cmd write (initial)\n"; - DBGTRACE_LOG(""); WaitCmdWrite({.Count=8, .PlanStep=step, .TxId=txId_1, .UserInfos={ @@ -2470,9 +2402,7 @@ Y_UNIT_TEST_F(ChangeConfig, TPartitionFixture) }, }); - DBGTRACE_LOG(""); SendCmdWriteResponse(NMsgBusProxy::MSTATUS_OK); - DBGTRACE_LOG(""); WaitCmdWrite({.Count=8, .PlanStep=step, .TxId=txId_1, .UserInfos={ @@ -2480,10 +2410,8 @@ Y_UNIT_TEST_F(ChangeConfig, TPartitionFixture) }, }); - DBGTRACE_LOG(""); SendCmdWriteResponse(NMsgBusProxy::MSTATUS_OK); Cerr << "Wait commit 1 done\n"; - DBGTRACE_LOG(""); WaitCommitTxDone({.TxId=txId_1, .Partition=TPartitionId(partition)}); // @@ -2496,7 +2424,6 @@ Y_UNIT_TEST_F(ChangeConfig, TPartitionFixture) // }, // }); Cerr << "Wait cmd write (change config)\n"; - DBGTRACE_LOG(""); WaitCmdWrite({.Count=8, .PlanStep=step, .TxId=txId_1, .UserInfos={ @@ -2506,18 +2433,14 @@ Y_UNIT_TEST_F(ChangeConfig, TPartitionFixture) .DeleteRanges={ {0, {.Partition=3, .Consumer="client-2"}} }}); - DBGTRACE_LOG(""); SendCmdWriteResponse(NMsgBusProxy::MSTATUS_OK); Cerr << "Wait config changed\n"; - DBGTRACE_LOG(""); WaitPartitionConfigChanged({.Partition=TPartitionId(partition)}); // // consumer 'client-2' was deleted // - DBGTRACE_LOG(""); WaitCalcPredicateResult({.Step=step, .TxId=txId_2, .Partition=TPartitionId(partition), .Predicate=false}); - DBGTRACE_LOG(""); SendRollbackTx(step, txId_2); } @@ -2837,61 +2760,39 @@ Y_UNIT_TEST_F(ShadowPartitionCountersRestore, TPartitionFixture) { Y_UNIT_TEST_F(DataTxCalcPredicateOk, TPartitionTxTestHelper) { - DBGTRACE("DataTxCalcPredicateOk"); Init(); - DBGTRACE_LOG(""); CreateSession("client", "session"); i64 cookie = 1; - DBGTRACE_LOG(""); auto tx1 = MakeAndSendWriteTx({}); - DBGTRACE_LOG(""); WaitWriteInfoRequest(tx1, true); Cerr << "Wait first predicate result " << Endl; - DBGTRACE_LOG(""); WaitTxPredicateReply(tx1); - DBGTRACE_LOG(""); auto tx2 = MakeAndSendWriteTx({{"src1", {1, 10}}}); - DBGTRACE_LOG(""); WaitWriteInfoRequest(tx2, true); - DBGTRACE_LOG(""); SendTxCommit(tx1); Cerr << "Wait second predicate result " << Endl; - DBGTRACE_LOG(""); WaitTxPredicateReply(tx2); - DBGTRACE_LOG(""); SendTxCommit(tx2); - DBGTRACE_LOG(""); EmulateKVTablet(); TString data = "data for write"; - DBGTRACE_LOG(""); SendChangeOwner(cookie, "owner1", Ctx->Edge, true); - DBGTRACE_LOG(""); EmulateKVTablet(); - DBGTRACE_LOG(""); auto ownerEvent = Ctx->Runtime->GrabEdgeEvent(TDuration::Seconds(1)); UNIT_ASSERT(ownerEvent != nullptr); - DBGTRACE_LOG(""); auto ownerCookie = ownerEvent->Response->GetPartitionResponse().GetCmdGetOwnershipResult().GetOwnerCookie(); - DBGTRACE_LOG(""); SendWrite(++cookie, 0, ownerCookie, 51, data, false, 5); - DBGTRACE_LOG(""); EmulateKVTablet(); - DBGTRACE_LOG(""); WaitProxyResponse({.Cookie=cookie}); Cerr << "Wait third predicate result " << Endl; - DBGTRACE_LOG(""); auto tx3 = MakeAndSendWriteTx({{"src1", {12, 20}}, {"SourceId", {6, 10}}}); - DBGTRACE_LOG(""); WaitWriteInfoRequest(tx3, true); - DBGTRACE_LOG(""); WaitTxPredicateReply(tx3); - DBGTRACE_LOG(""); SendTxCommit(tx3); } @@ -2989,101 +2890,58 @@ Y_UNIT_TEST_F(TestTxBatchInFederation, TPartitionTxTestHelper) { } Y_UNIT_TEST_F(ConflictingActsInSeveralBatches, TPartitionTxTestHelper) { - DBGTRACE("ConflictingActsInSeveralBatches"); TTxBatchingTestParams params {.WriterSessions{"src1", "src4"},.EndOffset=1}; - DBGTRACE_LOG(""); Init(std::move(params)); - DBGTRACE_LOG(""); auto tx1 = MakeAndSendWriteTx({{"src1", {1, 3}}}); - DBGTRACE_LOG(""); auto tx2 = MakeAndSendWriteTx({{"src2", {4, 6}}}); - DBGTRACE_LOG(""); auto tx3 = MakeAndSendWriteTx({{"src1", {4, 6}}}); - DBGTRACE_LOG(""); AddAndSendNormalWrite("src1", 7, 12); - DBGTRACE_LOG(""); AddAndSendNormalWrite("src4", 1, 2); - DBGTRACE_LOG(""); auto tx5 = MakeAndSendWriteTx({{"src4", {4, 5}}}); - DBGTRACE_LOG(""); AddAndSendNormalWrite("src4", 7, 12); - DBGTRACE_LOG(""); auto immTx1 = MakeAndSendImmediateTx({{"src4", {13, 15}}}); - DBGTRACE_LOG(""); WaitWriteInfoRequest(tx1, true); - DBGTRACE_LOG(""); WaitWriteInfoRequest(tx2, true); - DBGTRACE_LOG(""); WaitWriteInfoRequest(tx3, true); - DBGTRACE_LOG(""); WaitWriteInfoRequest(tx5, true); - DBGTRACE_LOG(""); WaitWriteInfoRequest(immTx1, true); - DBGTRACE_LOG(""); WaitTxPredicateReply(tx1); - DBGTRACE_LOG(""); WaitTxPredicateReply(tx2); - //DBGTRACE_LOG(""); //WaitBatchCompletion(2); - DBGTRACE_LOG(""); SendTxCommit(tx1); - DBGTRACE_LOG(""); SendTxRollback(tx2); - DBGTRACE_LOG(""); WaitKvRequest(); - DBGTRACE_LOG(""); SendKvResponse(); - DBGTRACE_LOG(""); WaitTxPredicateReply(tx3); - //DBGTRACE_LOG(""); //WaitBatchCompletion(1); - DBGTRACE_LOG(""); SendTxCommit(tx3); //2 Normal writes with src1 & src4 - DBGTRACE_LOG(""); ExpectNoTxPredicateReply(); - DBGTRACE_LOG(""); WaitKvRequest(); - DBGTRACE_LOG(""); SendKvResponse(); - DBGTRACE_LOG(""); WaitCommitDone(tx1); - DBGTRACE_LOG(""); WaitKvRequest(); - DBGTRACE_LOG(""); SendKvResponse(); - DBGTRACE_LOG(""); WaitCommitDone(tx3); - DBGTRACE_LOG(""); WaitTxPredicateReply(tx5); - //DBGTRACE_LOG(""); //WaitBatchCompletion(6 + 2); // Normal writes produce 1 act for each message - DBGTRACE_LOG(""); SendTxCommit(tx5); - //DBGTRACE_LOG(""); //WaitBatchCompletion(1); - DBGTRACE_LOG(""); WaitKvRequest(); - DBGTRACE_LOG(""); SendKvResponse(); - DBGTRACE_LOG(""); WaitCommitDone(tx5); - //DBGTRACE_LOG(""); //WaitBatchCompletion(1 + 6); //Normal write & immTx for src4; - DBGTRACE_LOG(""); WaitKvRequest(); - DBGTRACE_LOG(""); SendKvResponse(); - DBGTRACE_LOG(""); WaitImmediateTxComplete(immTx1, true); } @@ -3191,78 +3049,45 @@ Y_UNIT_TEST_F(ConflictingSrcIdForTxInDifferentBatches, TPartitionTxTestHelper) { } Y_UNIT_TEST_F(ConflictingSrcIdTxAndWritesDifferentBatches, TPartitionTxTestHelper) { - DBGTRACE("ConflictingSrcIdTxAndWritesDifferentBatches"); TTxBatchingTestParams params {.WriterSessions{"src1"}, .EndOffset = 1}; - DBGTRACE_LOG(""); Init(std::move(params)); - DBGTRACE_LOG(""); auto tx1 = MakeAndSendWriteTx({{"src1", {1, 3}},}); - DBGTRACE_LOG(""); auto tx2 = MakeAndSendWriteTx({{"src1", {2, 4}}}); - DBGTRACE_LOG(""); auto tx3 = MakeAndSendWriteTx({{"src1", {4, 6}}}); - DBGTRACE_LOG(""); AddAndSendNormalWrite("src1", 1, 1); - DBGTRACE_LOG(""); AddAndSendNormalWrite("src1", 7, 7); - DBGTRACE_LOG(""); AddAndSendNormalWrite("src1", 7, 7); - DBGTRACE_LOG(""); WaitWriteInfoRequest(tx1, true); - DBGTRACE_LOG(""); WaitWriteInfoRequest(tx2, true); - DBGTRACE_LOG(""); WaitWriteInfoRequest(tx3, true); - DBGTRACE_LOG(""); WaitTxPredicateReply(tx1); - DBGTRACE_LOG(""); SendTxCommit(tx1); - //DBGTRACE_LOG(""); //WaitBatchCompletion(1); - DBGTRACE_LOG(""); WaitKvRequest(); - DBGTRACE_LOG(""); SendKvResponse(); - DBGTRACE_LOG(""); WaitCommitDone(tx1); - DBGTRACE_LOG(""); WaitTxPredicateFailure(tx2); - DBGTRACE_LOG(""); WaitTxPredicateReply(tx3); - DBGTRACE_LOG(""); SendTxRollback(tx2); - DBGTRACE_LOG(""); SendTxCommit(tx3); - //DBGTRACE_LOG(""); //WaitBatchCompletion(2); // Tx 2 & 3. - DBGTRACE_LOG(""); WaitKvRequest(); - DBGTRACE_LOG(""); SendKvResponse(); - DBGTRACE_LOG(""); WaitKvRequest(); - DBGTRACE_LOG(""); SendKvResponse(); - DBGTRACE_LOG(""); WaitCommitDone(tx3); - //DBGTRACE_LOG(""); //WaitBatchCompletion(3); - DBGTRACE_LOG(""); WaitKvRequest(); - DBGTRACE_LOG(""); SendKvResponse(); - DBGTRACE_LOG(""); WaitProxyResponse({.AlreadyWritten=true, .SeqNo=1}); - DBGTRACE_LOG(""); WaitProxyResponse({.AlreadyWritten=false, .SeqNo=7}); - DBGTRACE_LOG(""); WaitProxyResponse({.AlreadyWritten=true, .SeqNo=7}); } @@ -3340,172 +3165,96 @@ class TBatchingConditionsTest { }; Y_UNIT_TEST_F(DifferentWriteTxBatchingOptions, TPartitionTxTestHelper) { - DBGTRACE("DifferentWriteTxBatchingOptions"); auto wrapper = TBatchingConditionsTest(this); // 1. ImmTx -> NormWrite -> ImmTx -> NormWrite = All batched { - DBGTRACE_LOG(""); wrapper.Start(); - DBGTRACE_LOG(""); wrapper.AddNormalWrite(); - DBGTRACE_LOG(""); auto immTx1 = wrapper.AddImmediateTx(); - DBGTRACE_LOG(""); wrapper.AddNormalWrite(); - DBGTRACE_LOG(""); auto immTx2 = wrapper.AddImmediateTx(); - DBGTRACE_LOG(""); wrapper.Process(); - DBGTRACE_LOG(""); WaitWriteInfoRequest(immTx1, true); - DBGTRACE_LOG(""); WaitWriteInfoRequest(immTx2, true); - //DBGTRACE_LOG(""); //WaitBatchCompletion(4 + 1); - DBGTRACE_LOG(""); EmulateKVTablet(); - DBGTRACE_LOG(""); EmulateKVTablet(); - DBGTRACE_LOG(""); WaitImmediateTxComplete(immTx1, true); - DBGTRACE_LOG(""); WaitImmediateTxComplete(immTx2, true); } { // 2. ImmTx -> WriteTx = KVRequest - DBGTRACE_LOG(""); ResetBatchCompletion(); - DBGTRACE_LOG(""); wrapper.Start(); - DBGTRACE_LOG(""); auto immTx = wrapper.AddImmediateTx(); - DBGTRACE_LOG(""); auto tx = wrapper.AddTx(); - DBGTRACE_LOG(""); wrapper.Process(); - DBGTRACE_LOG(""); WaitWriteInfoRequest(immTx, true); - DBGTRACE_LOG(""); WaitWriteInfoRequest(tx, true); - //DBGTRACE_LOG(""); //WaitBatchCompletion(1+1); - DBGTRACE_LOG(""); ExpectNoTxPredicateReply(); - DBGTRACE_LOG(""); EmulateKVTablet(); - DBGTRACE_LOG(""); EmulateKVTablet(); - DBGTRACE_LOG(""); WaitImmediateTxComplete(immTx, true); - DBGTRACE_LOG(""); ExpectNoCommitDone(); - DBGTRACE_LOG(""); WaitTxPredicateReply(tx); - DBGTRACE_LOG(""); SendTxCommit(tx); - //DBGTRACE_LOG(""); //WaitBatchCompletion(1); - DBGTRACE_LOG(""); EmulateKVTablet(); - DBGTRACE_LOG(""); WaitCommitDone(tx); } { // 3. NormWrite -> WriteTx = KVRequest - DBGTRACE_LOG(""); ResetBatchCompletion(); - DBGTRACE_LOG(""); wrapper.Start(); - DBGTRACE_LOG(""); wrapper.AddNormalWrite(); - DBGTRACE_LOG(""); auto tx = wrapper.AddTx(); - DBGTRACE_LOG(""); wrapper.Process(); - DBGTRACE_LOG(""); WaitWriteInfoRequest(tx, true); - //DBGTRACE_LOG(""); //WaitBatchCompletion(1+1); - DBGTRACE_LOG(""); ExpectNoTxPredicateReply(); - DBGTRACE_LOG(""); EmulateKVTablet(); - DBGTRACE_LOG(""); WaitTxPredicateReply(tx); - DBGTRACE_LOG(""); SendTxCommit(tx); - //DBGTRACE_LOG(""); //WaitBatchCompletion(1); - DBGTRACE_LOG(""); EmulateKVTablet(); - DBGTRACE_LOG(""); WaitCommitDone(tx); } { // 4. WriteTx -> NormWrite = 2 batches - DBGTRACE_LOG(""); ResetBatchCompletion(); - DBGTRACE_LOG(""); wrapper.Start(); - DBGTRACE_LOG(""); auto tx = wrapper.AddTx(); - DBGTRACE_LOG(""); wrapper.AddNormalWrite(); - DBGTRACE_LOG(""); wrapper.Process(); - DBGTRACE_LOG(""); WaitWriteInfoRequest(tx, true); - DBGTRACE_LOG(""); WaitTxPredicateReply(tx); - //DBGTRACE_LOG(""); //WaitBatchCompletion(1+1); - DBGTRACE_LOG(""); ExpectNoKvRequest(); - DBGTRACE_LOG(""); SendTxCommit(tx); - DBGTRACE_LOG(""); EmulateKVTablet(); - DBGTRACE_LOG(""); WaitCommitDone(tx); - //DBGTRACE_LOG(""); //WaitBatchCompletion(1); - DBGTRACE_LOG(""); EmulateKVTablet(); } { // 5. WriteTx -> ImmTx = 2 batches - DBGTRACE_LOG(""); ResetBatchCompletion(); - DBGTRACE_LOG(""); wrapper.Start(); - DBGTRACE_LOG(""); auto tx = wrapper.AddTx(); - DBGTRACE_LOG(""); auto immTx = wrapper.AddImmediateTx(); - DBGTRACE_LOG(""); wrapper.Process(); - DBGTRACE_LOG(""); WaitWriteInfoRequest(tx, true); - DBGTRACE_LOG(""); WaitWriteInfoRequest(immTx, true); - //DBGTRACE_LOG(""); //WaitBatchCompletion(1+1); - DBGTRACE_LOG(""); WaitTxPredicateReply(tx); - DBGTRACE_LOG(""); SendTxCommit(tx); - DBGTRACE_LOG(""); ExpectNoCommitDone(); - DBGTRACE_LOG(""); EmulateKVTablet(); - //DBGTRACE_LOG(""); //WaitBatchCompletion(1); - DBGTRACE_LOG(""); WaitCommitDone(tx); - DBGTRACE_LOG(""); EmulateKVTablet(); - DBGTRACE_LOG(""); WaitImmediateTxComplete(immTx, true); } } @@ -3559,167 +3308,100 @@ Y_UNIT_TEST_F(FailedTxsDontBlock, TPartitionTxTestHelper) { } Y_UNIT_TEST_F(NonConflictingCommitsBatch, TPartitionTxTestHelper) { - DBGTRACE("NonConflictingCommitsBatch"); TTxBatchingTestParams params{ .ConsumersCount= 3, .ConsumerSessions={1}, .EndOffset=50 }; - DBGTRACE_LOG(""); Init(std::move(params)); //Just block processing so every message arrives before batching starts - DBGTRACE_LOG(""); auto txTmp = MakeAndSendWriteTx({}); - DBGTRACE_LOG(""); MakeAndSendNormalOffsetCommit(1, 5); - DBGTRACE_LOG(""); auto tx1 = MakeAndSendTxOffsetCommit(3, 0, 5); - DBGTRACE_LOG(""); auto tx2 = MakeAndSendTxOffsetCommit(2, 0, 5); - DBGTRACE_LOG(""); MakeAndSendNormalOffsetCommit(1, 10); - DBGTRACE_LOG(""); auto txImm1 = MakeAndSendImmediateTxOffsetCommit(1, 0, 15); - DBGTRACE_LOG(""); WaitWriteInfoRequest(txTmp, true); - DBGTRACE_LOG(""); WaitTxPredicateReply(txTmp); - DBGTRACE_LOG(""); SendTxRollback(txTmp); - DBGTRACE_LOG(""); WaitTxPredicateReply(tx1); - DBGTRACE_LOG(""); WaitTxPredicateReply(tx2); - //DBGTRACE_LOG(""); //WaitBatchCompletion(5 + 1 /*tmpTx*/); - DBGTRACE_LOG(""); SendTxCommit(tx1); - DBGTRACE_LOG(""); SendTxCommit(tx2); - DBGTRACE_LOG(""); WaitKvRequest(); - DBGTRACE_LOG(""); SendKvResponse(); - DBGTRACE_LOG(""); WaitKvRequest(); - DBGTRACE_LOG(""); SendKvResponse(); - DBGTRACE_LOG(""); WaitCommitDone(tx1); - DBGTRACE_LOG(""); WaitCommitDone(tx2); - DBGTRACE_LOG(""); WaitImmediateTxComplete(txImm1, false); } Y_UNIT_TEST_F(ConflictingCommitsInSeveralBatches, TPartitionTxTestHelper) { - DBGTRACE("ConflictingCommitsInSeveralBatches"); TTxBatchingTestParams params{ .ConsumersCount= 2, .ConsumerSessions={1}, .EndOffset=50 }; - DBGTRACE_LOG(""); Init(std::move(params)); - DBGTRACE_LOG(""); //Just block processing so every message arrives before batching starts auto txTmp = MakeAndSendWriteTx({}); - DBGTRACE_LOG(""); MakeAndSendNormalOffsetCommit(1, 2); // act-1 - DBGTRACE_LOG(""); auto tx1 = MakeAndSendTxOffsetCommit(1, 2, 5); - DBGTRACE_LOG(""); auto tx2 = MakeAndSendTxOffsetCommit(1, 5, 10); - DBGTRACE_LOG(""); MakeAndSendNormalOffsetCommit(1, 20); // act-2 - DBGTRACE_LOG(""); ResetBatchCompletion(); - DBGTRACE_LOG(""); WaitWriteInfoRequest(txTmp, true); - DBGTRACE_LOG(""); WaitTxPredicateReply(txTmp); - DBGTRACE_LOG(""); SendTxRollback(txTmp); - //DBGTRACE_LOG(""); //WaitBatchCompletion(2); // txTmp + act-1 - DBGTRACE_LOG(""); ExpectNoTxPredicateReply(); - DBGTRACE_LOG(""); WaitKvRequest(); - DBGTRACE_LOG(""); SendKvResponse(); - DBGTRACE_LOG(""); WaitTxPredicateReply(tx1); - DBGTRACE_LOG(""); WaitKvRequest(); - DBGTRACE_LOG(""); SendKvResponse(); - DBGTRACE_LOG(""); ExpectNoTxPredicateReply(); - DBGTRACE_LOG(""); SendTxCommit(tx1); - //DBGTRACE_LOG(""); //WaitBatchCompletion(1); // tx1 - DBGTRACE_LOG(""); WaitTxPredicateReply(tx2); - DBGTRACE_LOG(""); SendTxCommit(tx2); - //DBGTRACE_LOG(""); //WaitBatchCompletion(1); // tx2 - DBGTRACE_LOG(""); WaitKvRequest(); - DBGTRACE_LOG(""); SendKvResponse(); - DBGTRACE_LOG(""); WaitCommitDone(tx1); - DBGTRACE_LOG(""); WaitKvRequest(); - DBGTRACE_LOG(""); SendKvResponse(); - DBGTRACE_LOG(""); WaitCommitDone(tx2); - //DBGTRACE_LOG(""); //WaitBatchCompletion(1); // act-2 - DBGTRACE_LOG(""); WaitKvRequest(); - DBGTRACE_LOG(""); SendKvResponse(); - DBGTRACE_LOG(""); txTmp = MakeAndSendWriteTx({}); - DBGTRACE_LOG(""); auto immTx1 = MakeAndSendImmediateTxOffsetCommit(2, 0, 5); - DBGTRACE_LOG(""); auto immTx2 = MakeAndSendImmediateTxOffsetCommit(2, 5, 10); - DBGTRACE_LOG(""); WaitWriteInfoRequest(txTmp, true); - DBGTRACE_LOG(""); WaitTxPredicateReply(txTmp); - DBGTRACE_LOG(""); SendTxRollback(txTmp); - DBGTRACE_LOG(""); WaitKvRequest(); - //DBGTRACE_LOG(""); //WaitBatchCompletion(3); - DBGTRACE_LOG(""); SendKvResponse(); - DBGTRACE_LOG(""); WaitImmediateTxComplete(immTx1, true); - DBGTRACE_LOG(""); WaitImmediateTxComplete(immTx2, true); } @@ -3796,153 +3478,92 @@ Y_UNIT_TEST_F(ConflictingCommitFails, TPartitionTxTestHelper) { } Y_UNIT_TEST_F(ConflictingCommitProccesAfterRollback, TPartitionTxTestHelper) { - DBGTRACE("ConflictingCommitProccesAfterRollback"); TTxBatchingTestParams params{ .ConsumersCount = 2, .EndOffset=50 }; - DBGTRACE_LOG(""); Init(std::move(params)); - DBGTRACE_LOG(""); auto tx1 = MakeAndSendTxOffsetCommit(1, 0, 5); - DBGTRACE_LOG(""); auto tx2 = MakeAndSendTxOffsetCommit(1, 0, 3); - DBGTRACE_LOG(""); WaitTxPredicateReply(tx1); - //DBGTRACE_LOG(""); //WaitBatchCompletion(1); - DBGTRACE_LOG(""); SendTxRollback(tx1); - DBGTRACE_LOG(""); WaitKvRequest(); - DBGTRACE_LOG(""); SendKvResponse(); - DBGTRACE_LOG(""); WaitTxPredicateReply(tx2); - //DBGTRACE_LOG(""); //WaitBatchCompletion(1); - DBGTRACE_LOG(""); SendTxCommit(tx2); - DBGTRACE_LOG(""); WaitKvRequest(); - DBGTRACE_LOG(""); SendKvResponse(); - DBGTRACE_LOG(""); WaitKvRequest(); - DBGTRACE_LOG(""); SendKvResponse(); - DBGTRACE_LOG(""); WaitCommitDone(tx2); - DBGTRACE_LOG(""); ExpectNoCommitDone(); } Y_UNIT_TEST_F(TestBatchingWithChangeConfig, TPartitionTxTestHelper) { - DBGTRACE("TestBatchingWithChangeConfig"); Init({.ConsumersCount = 2}); - DBGTRACE_LOG(""); auto txTmp = MakeAndSendWriteTx({}); - DBGTRACE_LOG(""); auto immTx1 = MakeAndSendImmediateTxOffsetCommit(1, 0, 5); - DBGTRACE_LOG(""); SendChangePartitionConfig({.Version=2, .Consumers={ {.Consumer="client-0", .Offset=5, .Generation=0}, {.Consumer="client-1", .Generation=7} }}); - DBGTRACE_LOG(""); auto immTx2 = MakeAndSendImmediateTxOffsetCommit(1, 5, 10); - DBGTRACE_LOG(""); WaitWriteInfoRequest(txTmp, true); - DBGTRACE_LOG(""); SendTxRollback(txTmp); - DBGTRACE_LOG(""); WaitBatchCompletion(2); - DBGTRACE_LOG(""); ExpectNoBatchCompletion(); - DBGTRACE_LOG(""); EmulateKVTablet(); - DBGTRACE_LOG(""); WaitImmediateTxComplete(immTx1, true); - DBGTRACE_LOG(""); WaitBatchCompletion(1); - DBGTRACE_LOG(""); EmulateKVTablet(); - DBGTRACE_LOG(""); auto event = Ctx->Runtime->GrabEdgeEvent(); - //DBGTRACE_LOG(""); //WaitBatchCompletion(1); // immTx2 - DBGTRACE_LOG(""); EmulateKVTablet(); - DBGTRACE_LOG(""); WaitImmediateTxComplete(immTx2, true); } Y_UNIT_TEST_F(TestBatchingWithProposeConfig, TPartitionTxTestHelper) { - DBGTRACE("TestBatchingWithProposeConfig"); Init({.ConsumersCount = 2}); - DBGTRACE_LOG(""); auto txTmp = MakeAndSendWriteTx({}); - DBGTRACE_LOG(""); auto immTx1 = MakeAndSendImmediateTxOffsetCommit(1, 0, 5); - DBGTRACE_LOG(""); auto proposeTxId = GetTxId(); - DBGTRACE_LOG(""); auto event = std::make_unique(1, proposeTxId); - DBGTRACE_LOG(""); event->TopicConverter = TopicConverter; auto copy = Config; - DBGTRACE_LOG(""); copy.SetVersion(10); - DBGTRACE_LOG(""); auto* newConsumer = copy.AddConsumers(); - DBGTRACE_LOG(""); newConsumer->SetName("client-0"); - DBGTRACE_LOG(""); newConsumer->SetGeneration(0); - DBGTRACE_LOG(""); event->Config = std::move(copy); - DBGTRACE_LOG(""); SendEvent(event.release()); - DBGTRACE_LOG(""); auto immTx2 = MakeAndSendImmediateTxOffsetCommit(1, 5, 10); - DBGTRACE_LOG(""); WaitWriteInfoRequest(txTmp, true); - DBGTRACE_LOG(""); SendTxRollback(txTmp); - DBGTRACE_LOG(""); WaitBatchCompletion(2); - DBGTRACE_LOG(""); ExpectNoBatchCompletion(); - DBGTRACE_LOG(""); EmulateKVTablet(); - DBGTRACE_LOG(""); WaitImmediateTxComplete(immTx1, true); - DBGTRACE_LOG(""); SendCommitTx(1, proposeTxId); //ToDo - wait propose result; - //DBGTRACE_LOG(""); //WaitBatchCompletion(1); - DBGTRACE_LOG(""); EmulateKVTablet(); - DBGTRACE_LOG(""); WaitCommitTxDone({.TxId=proposeTxId}); - //DBGTRACE_LOG(""); //WaitBatchCompletion(1); - DBGTRACE_LOG(""); EmulateKVTablet(); - DBGTRACE_LOG(""); WaitImmediateTxComplete(immTx2, true); } diff --git a/ydb/core/persqueue/ut/ya.make b/ydb/core/persqueue/ut/ya.make index 1399b230d3c4..e815cdb07575 100644 --- a/ydb/core/persqueue/ut/ya.make +++ b/ydb/core/persqueue/ut/ya.make @@ -25,7 +25,6 @@ PEERDIR( ydb/public/sdk/cpp/src/client/persqueue_public/ut/ut_utils ydb/core/tx/schemeshard/ut_helpers - ydb/library/dbgtrace ) YQL_LAST_ABI_VERSION() From 8cc1be0400efc6917f8a4e2c2e468e5f57d36e0b Mon Sep 17 00:00:00 2001 From: Alexander Kotov Date: Thu, 6 Nov 2025 19:48:11 +0300 Subject: [PATCH 34/36] [/] unused code --- .../pqtablet/partition/partition.cpp | 206 +----------------- .../persqueue/pqtablet/partition/partition.h | 14 +- ydb/core/persqueue/ut/partition_ut.cpp | 2 - 3 files changed, 8 insertions(+), 214 deletions(-) diff --git a/ydb/core/persqueue/pqtablet/partition/partition.cpp b/ydb/core/persqueue/pqtablet/partition/partition.cpp index bdc3631e5ded..3ae296f94118 100644 --- a/ydb/core/persqueue/pqtablet/partition/partition.cpp +++ b/ydb/core/persqueue/pqtablet/partition/partition.cpp @@ -1395,7 +1395,7 @@ void TPartition::ProcessPendingEvent(std::unique_ptr ev, con auto txIter = TransactionsInflight.begin(); if (txIter->second->ProposeConfig) { - Y_ABORT_UNLESS(!ChangeConfig); + PQ_ENSURE(!ChangeConfig); ChangeConfig = MakeSimpleShared(TopicConverter, txIter->second->ProposeConfig->Config); @@ -2279,9 +2279,6 @@ size_t TPartition::GetUserActCount(const TString& consumer) const } } -// -// TODO(abcdef): переименовать в ProcessUserActionAndTxs -// void TPartition::ProcessTxsAndUserActs(const TActorContext&) { if (KVWriteInProgress) { @@ -2327,81 +2324,10 @@ void TPartition::ProcessTxsAndUserActs(const TActorContext&) } CurrentBatchSize = 0; - //if (UserActionAndTxPendingWrite.empty()) { - // LOG_D("Waiting for the commits to arrive"); - // return; - //} - LOG_D("Try persist"); RunPersist(); } -//void TPartition::ProcessTxsAndUserActsOriginal(const TActorContext& ctx) -//{ -// if (KVWriteInProgress) { -// LOG_D("Writing. Can't process transactions and user actions"); -// return; -// } -// if (DeletePartitionState == DELETION_INITED) { -// if (!PersistRequest) { -// PersistRequest = MakeHolder(); -// } -// ScheduleNegativeReplies(); -// ScheduleDeletePartitionDone(); -// -// AddCmdDeleteRangeForAllKeys(*PersistRequest); -// -// ctx.Send(BlobCache, PersistRequest.Release(), 0, 0, PersistRequestSpan.GetTraceId()); -// PersistRequest = nullptr; -// CurrentPersistRequestSpan = std::move(PersistRequestSpan); -// PersistRequestSpan = NWilson::TSpan(); -// DeletePartitionState = DELETION_IN_PROCESS; -// KVWriteInProgress = true; -// -// return; -// } -// LOG_D("Batching state before ContinueProcessTxsAndUserActs: " << (int)BatchingState); -// if (CanProcessUserActionAndTransactionEvents()) { -// ContinueProcessTxsAndUserActs(ctx); -// } -// // Still preprocessing? Waiting for something -// if (CanProcessUserActionAndTransactionEvents()) { -// LOG_D("Still preprocessing - waiting for something"); -// return; -// } -// LOG_D("Batching state after ContinueProcessTxsAndUserActs: " << (int)BatchingState); -// -// // Preprocessing complete; -// if (CurrentBatchSize > 0) { -// LOG_D("Batch completed (" << CurrentBatchSize << ")"); -// Send(SelfId(), new TEvPQ::TEvTxBatchComplete(CurrentBatchSize)); -// } -// CurrentBatchSize = 0; -// -// if (UserActionAndTxPendingCommit.empty()) { -// // Processing stopped and nothing to commit - finalize -// BatchingState = ETxBatchingState::Finishing; -// } else { -// // Process commit queue -// ProcessCommitQueue(); -// } -// // BatchingState can go to Finishing in ContinueProcessTxsAndUserActs. Therefore, it is necessary to check -// // the size of the UserActionAndTxPendingCommit queue here. -// if (!UserActionAndTxPendingCommit.empty()) { -// // Still pending for come commits -// LOG_D("Still pending for come commits"); -// return; -// } -// LOG_D("Try persist"); -// // Here we have an empty UserActionAndTxPendingCommit queue and BatchingState is equal to Finishing. -// RunPersist(); -//} - -bool TPartition::CanProcessUserActionAndTransactionEvents() const -{ - return (BatchingState == ETxBatchingState::PreProcessing); -} - void TPartition::ProcessUserActionAndTxEvents() { while (!UserActionAndTransactionEvents.empty()) { @@ -2421,7 +2347,6 @@ void TPartition::ProcessUserActionAndTxEvents() switch (std::visit(visitor, front.Event)) { case EProcessResult::Continue: MoveUserActionAndTxToPendingCommitQueue(); - FirstEvent = false; break; case EProcessResult::ContinueDrop: UserActionAndTransactionEvents.pop_front(); @@ -2429,7 +2354,6 @@ void TPartition::ProcessUserActionAndTxEvents() case EProcessResult::Break: MoveUserActionAndTxToPendingCommitQueue(); BatchingState = ETxBatchingState::Finishing; - FirstEvent = false; break; case EProcessResult::Blocked: BatchingState = ETxBatchingState::Executing; @@ -2444,8 +2368,7 @@ void TPartition::DumpTheSizeOfInternalQueues() const { LOG_D("Events: " << UserActionAndTransactionEvents.size() << ", PendingCommits: " << UserActionAndTxPendingCommit.size() << - ", PendingWrites: " << UserActionAndTxPendingWrite.size() << - ", FirstEvent: " << FirstEvent); + ", PendingWrites: " << UserActionAndTxPendingWrite.size()); } TString GetTransactionType(const TTransaction& tx) @@ -2484,12 +2407,6 @@ auto TPartition::ProcessUserActionAndTxEvent(TMessage& msg, return PreProcessUserActionOrTransaction(msg, affectedSourceIdsAndConsumers); } - -void TPartition::MoveUserActionAndTxToPendingCommitQueue() -{ - MoveUserActOrTxToCommitState(); -} - void TPartition::ProcessUserActionAndTxPendingCommits() { CurrentBatchSize = 0; @@ -2553,54 +2470,6 @@ void TPartition::ProcessUserActionAndTxPendingCommit(TMessage& msg, ExecUserActionOrTransaction(msg, request); } -//void TPartition::ContinueProcessTxsAndUserActs(const TActorContext&) -//{ -// PQ_ENSURE(!KVWriteInProgress); -// -// if (WriteCycleSizeEstimate >= MAX_WRITE_CYCLE_SIZE || WriteKeysSizeEstimate >= MAX_KEYS) { -// BatchingState = ETxBatchingState::Finishing; -// return; -// } -// while (CanProcessUserActionAndTransactionEvents() && !UserActionAndTransactionEvents.empty()) { -// if (ChangingConfig) { -// BatchingState = ETxBatchingState::Finishing; -// break; -// } -// auto& front = UserActionAndTransactionEvents.front(); -// if (TMessage* msg = std::get_if(&front.Event); msg && msg->WaitPreviousWriteSpan) { -// msg->WaitPreviousWriteSpan.End(); -// } -// auto visitor = [this, &front](auto& event) { -// return this->PreProcessUserActionOrTransaction(event, front.AffectedSourceIdsAndConsumers); -// }; -// switch (std::visit(visitor, front.Event)) { -// case EProcessResult::Continue: -// MoveUserActOrTxToCommitState(); -// FirstEvent = false; -// break; -// case EProcessResult::ContinueDrop: -// UserActionAndTransactionEvents.pop_front(); -// break; -// case EProcessResult::Break: -// MoveUserActOrTxToCommitState(); -// BatchingState = ETxBatchingState::Finishing; -// FirstEvent = false; -// break; -// case EProcessResult::Blocked: -// BatchingState = ETxBatchingState::Executing; -// return; -// case EProcessResult::NotReady: -// return; -// } -// CurrentBatchSize += 1; -// } -// if (UserActionAndTransactionEvents.empty()) { -// BatchingState = ETxBatchingState::Executing; -// return; -// } -// -//} - static void AppendToSet(const TVector& p, THashMap& q) { for (const auto& s : p) { @@ -2638,21 +2507,15 @@ void TPartition::DeleteAffectedSourceIdsAndConsumers() } } - //Y_ABORT_UNLESS(UserActionAndTxPendingCommit.empty()); - //Y_ABORT_UNLESS(TxAffectedSourcesIds.empty()); - //Y_ABORT_UNLESS(WriteAffectedSourcesIds.empty()); - //Y_ABORT_UNLESS(TxAffectedConsumers.empty()); - //Y_ABORT_UNLESS(SetOffsetAffectedConsumers.empty()); - UserActionAndTxPendingWrite.clear(); } -static void DeleteFromSet(const TVector& p, THashMap& q) +void TPartition::DeleteFromSet(const TVector& p, THashMap& q) const { for (const auto& s : p) { auto i = q.find(s); - Y_ABORT_UNLESS(i != q.end()); - Y_ABORT_UNLESS(i->second > 0); + PQ_ENSURE(i != q.end()); + PQ_ENSURE(i->second > 0); if (--i->second) { continue; } @@ -2667,58 +2530,17 @@ void TPartition::DeleteAffectedSourceIdsAndConsumers(const TAffectedSourceIdsAnd DeleteFromSet(affectedSourceIdsAndConsumers.TxReadConsumers, TxAffectedConsumers); DeleteFromSet(affectedSourceIdsAndConsumers.ReadConsumers, SetOffsetAffectedConsumers); - Y_ABORT_UNLESS(WriteKeysSizeEstimate >= affectedSourceIdsAndConsumers.WriteKeysSize); + PQ_ENSURE(WriteKeysSizeEstimate >= affectedSourceIdsAndConsumers.WriteKeysSize); WriteKeysSizeEstimate -= affectedSourceIdsAndConsumers.WriteKeysSize; } -// -// TODO(abcdef) переименовать в MoveUserActionAndTxToPendingCommitQueue -// -void TPartition::MoveUserActOrTxToCommitState() { +void TPartition::MoveUserActionAndTxToPendingCommitQueue() { auto& front = UserActionAndTransactionEvents.front(); AppendAffectedSourceIdsAndConsumers(front.AffectedSourceIdsAndConsumers); UserActionAndTxPendingCommit.push_back(std::move(front)); UserActionAndTransactionEvents.pop_front(); } -void TPartition::ProcessCommitQueue() { - CurrentBatchSize = 0; - - PQ_ENSURE(!KVWriteInProgress); - if (!PersistRequest) { - PersistRequest = MakeHolder(); - } - while (!UserActionAndTxPendingCommit.empty()) { - auto& front = UserActionAndTxPendingCommit.front(); - auto state = ECommitState::Committed; - - if (auto* tx = get_if>(&front.Event)) { - state = tx->Get()->State; - } - - switch (state) { - case ECommitState::Pending: - return; - case ECommitState::Aborted: - break; - case ECommitState::Committed: - break; - } - - UserActionAndTxPendingWrite.push_back(std::move(front)); - UserActionAndTxPendingCommit.pop_front(); - - auto& event = UserActionAndTxPendingWrite.back().Event; - auto visitor = [this, request = PersistRequest.Get()](auto& event) { - return this->ExecUserActionOrTransaction(event, request); - }; - std::visit(visitor, event); - } - if (UserActionAndTxPendingCommit.empty()) { - BatchingState = ETxBatchingState::Finishing; - } -} - ui64 TPartition::NextReadCookie() { if (Cookie == Max()) { @@ -2728,9 +2550,6 @@ ui64 TPartition::NextReadCookie() } void TPartition::RunPersist() { - //TransactionsInflight.clear(); - - //PQ_ENSURE(UserActionAndTxPendingCommit.empty()); const auto& ctx = ActorContext(); const auto now = ctx.Now(); if (!PersistRequest) { @@ -2765,11 +2584,6 @@ void TPartition::RunPersist() { haveChanges = true; } - LOG_D("haveChanges=" << haveChanges << - ", TxIdHasChanged=" << TxIdHasChanged << - ", AffectedUsers.size=" << AffectedUsers.size() << - ", ChangeConfig=" << (ChangeConfig ? 1 : 0)); - if (haveChanges || TxIdHasChanged || !AffectedUsers.empty() || ChangeConfig) { WriteCycleStartTime = now; WriteStartTime = now; @@ -3389,15 +3203,9 @@ void TPartition::ExecChangePartitionConfig() { } void TPartition::OnProcessTxsAndUserActsWriteComplete(const TActorContext& ctx) { - FirstEvent = true; DeleteAffectedSourceIdsAndConsumers(); - //TxAffectedConsumers.clear(); - //TxAffectedSourcesIds.clear(); - //WriteAffectedSourcesIds.clear(); - //SetOffsetAffectedConsumers.clear(); BatchingState = ETxBatchingState::PreProcessing; WriteCycleSizeEstimate = 0; - //WriteKeysSizeEstimate = 0; if (ChangeConfig) { EndChangePartitionConfig(std::move(ChangeConfig->Config), diff --git a/ydb/core/persqueue/pqtablet/partition/partition.h b/ydb/core/persqueue/pqtablet/partition/partition.h index 16a34e05ad68..21dd34f14e05 100644 --- a/ydb/core/persqueue/pqtablet/partition/partition.h +++ b/ydb/core/persqueue/pqtablet/partition/partition.h @@ -354,9 +354,6 @@ class TPartition : public TBaseTabletActor { TAutoPtr MakeHasDataInfoResponse(ui64 lagSize, const TMaybe& cookie, bool readingFinished = false); void ProcessTxsAndUserActs(const TActorContext& ctx); - void ProcessTxsAndUserActsOriginal(const TActorContext& ctx); - void ContinueProcessTxsAndUserActs(const TActorContext& ctx); - void ProcessCommitQueue(); void RunPersist(); enum class EProcessResult; @@ -380,7 +377,6 @@ class TPartition : public TBaseTabletActor { void ProcessUserActionAndTxPendingCommit(TMessage& msg, TEvKeyValue::TEvRequest* request); - void MoveUserActOrTxToCommitState(); void PushBackDistrTx(TSimpleSharedPtr event); void PushBackDistrTx(TSimpleSharedPtr event); void PushFrontDistrTx(TSimpleSharedPtr event); @@ -823,10 +819,6 @@ class TPartition : public TBaseTabletActor { mutable TMaybe InitLogPrefix; mutable TMaybe UnknownLogPrefix; - //template T& GetUserActionAndTransactionEventsFront(); - //template T& GetCurrentEvent(); - //TSimpleSharedPtr& GetCurrentTransaction(); - struct TAffectedSourceIdsAndConsumers { TVector TxWriteSourcesIds; TVector WriteSourcesIds; @@ -839,10 +831,8 @@ class TPartition : public TBaseTabletActor { void DeleteAffectedSourceIdsAndConsumers(); void DeleteAffectedSourceIdsAndConsumers(const TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers); + void DeleteFromSet(const TVector& p, THashMap& q) const; - // - // TODO(abcdef): переименовать в ProcessUserActionAndTxEvent - // EProcessResult PreProcessUserActionOrTransaction(TSimpleSharedPtr& event, TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers); EProcessResult PreProcessUserActionOrTransaction(TSimpleSharedPtr& tx, @@ -883,7 +873,6 @@ class TPartition : public TBaseTabletActor { EProcessResult ApplyWriteInfoResponse(TTransaction& tx, TAffectedSourceIdsAndConsumers& affectedSourceIdsAndConsumers); - bool FirstEvent = true; bool HaveWriteMsg = false; bool HaveCheckDisk = false; bool HaveDrop = false; @@ -1213,7 +1202,6 @@ class TPartition : public TBaseTabletActor { TDeque TxForPersistTraceIds; TDeque TxForPersistSpans; - bool CanProcessUserActionAndTransactionEvents() const; ui64 GetCompactedBlobSizeLowerBound() const; bool CompactRequestedBlob(const TRequestedBlob& requestedBlob, diff --git a/ydb/core/persqueue/ut/partition_ut.cpp b/ydb/core/persqueue/ut/partition_ut.cpp index 259bbea34fd5..4661513d9b1b 100644 --- a/ydb/core/persqueue/ut/partition_ut.cpp +++ b/ydb/core/persqueue/ut/partition_ut.cpp @@ -336,8 +336,6 @@ void TPartitionFixture::SetUp(NUnitTest::TTestContext&) Ctx.ConstructInPlace(); Finalizer.ConstructInPlace(*Ctx); - //Ctx->EnableDetailedPQLog = true; - Ctx->Prepare(); Ctx->Runtime->SetScheduledLimit(5'000); } From 64ae1184c5d87e688469bd3c622053d65c6f7f60 Mon Sep 17 00:00:00 2001 From: Alexander Kotov Date: Thu, 6 Nov 2025 20:44:58 +0300 Subject: [PATCH 35/36] [/] unused code --- ydb/core/persqueue/pqtablet/partition/partition.cpp | 10 ---------- ydb/core/persqueue/pqtablet/partition/partition.h | 9 --------- 2 files changed, 19 deletions(-) diff --git a/ydb/core/persqueue/pqtablet/partition/partition.cpp b/ydb/core/persqueue/pqtablet/partition/partition.cpp index 3ae296f94118..feeae5b843cb 100644 --- a/ydb/core/persqueue/pqtablet/partition/partition.cpp +++ b/ydb/core/persqueue/pqtablet/partition/partition.cpp @@ -2332,7 +2332,6 @@ void TPartition::ProcessUserActionAndTxEvents() { while (!UserActionAndTransactionEvents.empty()) { if (ChangingConfig) { - BatchingState = ETxBatchingState::Finishing; break; } @@ -2353,10 +2352,8 @@ void TPartition::ProcessUserActionAndTxEvents() break; case EProcessResult::Break: MoveUserActionAndTxToPendingCommitQueue(); - BatchingState = ETxBatchingState::Finishing; break; case EProcessResult::Blocked: - BatchingState = ETxBatchingState::Executing; return; case EProcessResult::NotReady: return; @@ -2443,10 +2440,6 @@ void TPartition::ProcessUserActionAndTxPendingCommits() { ++CurrentBatchSize; } - - if (UserActionAndTxPendingCommit.empty()) { - BatchingState = ETxBatchingState::Finishing; - } } void TPartition::ProcessUserActionAndTxPendingCommit(TSimpleSharedPtr& event, @@ -3204,7 +3197,6 @@ void TPartition::ExecChangePartitionConfig() { void TPartition::OnProcessTxsAndUserActsWriteComplete(const TActorContext& ctx) { DeleteAffectedSourceIdsAndConsumers(); - BatchingState = ETxBatchingState::PreProcessing; WriteCycleSizeEstimate = 0; if (ChangeConfig) { @@ -3290,8 +3282,6 @@ void TPartition::OnProcessTxsAndUserActsWriteComplete(const TActorContext& ctx) ChangeConfig = nullptr; PendingPartitionConfig = nullptr; } - //ChangingConfig = false; - BatchingState = ETxBatchingState::PreProcessing; } void TPartition::EndChangePartitionConfig(NKikimrPQ::TPQTabletConfig&& config, diff --git a/ydb/core/persqueue/pqtablet/partition/partition.h b/ydb/core/persqueue/pqtablet/partition/partition.h index 21dd34f14e05..4815845fa064 100644 --- a/ydb/core/persqueue/pqtablet/partition/partition.h +++ b/ydb/core/persqueue/pqtablet/partition/partition.h @@ -939,15 +939,6 @@ class TPartition : public TBaseTabletActor { TMessageQueue Responses; ui64 CurrentBatchSize = 0; - enum class ETxBatchingState{ - PreProcessing, - Executing, - Finishing - }; - ETxBatchingState BatchingState = ETxBatchingState::PreProcessing; - // - // - // std::deque> UpdateUserInfoTimestamp; bool ReadingTimestamp; TString ReadingForUser; From e6124584ea807ae5d97662b7351d9621ddc0a9d7 Mon Sep 17 00:00:00 2001 From: Alexander Kotov Date: Thu, 6 Nov 2025 21:11:40 +0300 Subject: [PATCH 36/36] [+] writing cycle limits --- ydb/core/persqueue/pqtablet/partition/partition.cpp | 9 +++++++-- ydb/core/persqueue/pqtablet/partition/partition.h | 2 ++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/ydb/core/persqueue/pqtablet/partition/partition.cpp b/ydb/core/persqueue/pqtablet/partition/partition.cpp index feeae5b843cb..c9e50cefd075 100644 --- a/ydb/core/persqueue/pqtablet/partition/partition.cpp +++ b/ydb/core/persqueue/pqtablet/partition/partition.cpp @@ -122,7 +122,7 @@ static const TDuration WAKE_TIMEOUT = TDuration::Seconds(5); static const TDuration UPDATE_AVAIL_SIZE_INTERVAL = TDuration::MilliSeconds(100); static const TDuration MIN_UPDATE_COUNTERS_DELAY = TDuration::MilliSeconds(300); static const ui32 MAX_USERS = 1000; -//static const ui32 MAX_KEYS = 10000; +static const ui32 MAX_KEYS = 10000; static const ui32 MAX_TXS = 1000; static const ui32 MAX_WRITE_CYCLE_SIZE = 16_MB; @@ -2404,6 +2404,11 @@ auto TPartition::ProcessUserActionAndTxEvent(TMessage& msg, return PreProcessUserActionOrTransaction(msg, affectedSourceIdsAndConsumers); } +bool TPartition::WritingCycleDoesNotExceedTheLimits() const +{ + return WriteCycleSizeEstimate < MAX_WRITE_CYCLE_SIZE && WriteKeysSizeEstimate < MAX_KEYS; +} + void TPartition::ProcessUserActionAndTxPendingCommits() { CurrentBatchSize = 0; @@ -2412,7 +2417,7 @@ void TPartition::ProcessUserActionAndTxPendingCommits() { PersistRequest = MakeHolder(); } - while (!UserActionAndTxPendingCommit.empty()) { + while (!UserActionAndTxPendingCommit.empty() && WritingCycleDoesNotExceedTheLimits()) { auto& front = UserActionAndTxPendingCommit.front(); auto state = ECommitState::Committed; diff --git a/ydb/core/persqueue/pqtablet/partition/partition.h b/ydb/core/persqueue/pqtablet/partition/partition.h index 4815845fa064..7bc679a681aa 100644 --- a/ydb/core/persqueue/pqtablet/partition/partition.h +++ b/ydb/core/persqueue/pqtablet/partition/partition.h @@ -377,6 +377,8 @@ class TPartition : public TBaseTabletActor { void ProcessUserActionAndTxPendingCommit(TMessage& msg, TEvKeyValue::TEvRequest* request); + bool WritingCycleDoesNotExceedTheLimits() const; + void PushBackDistrTx(TSimpleSharedPtr event); void PushBackDistrTx(TSimpleSharedPtr event); void PushFrontDistrTx(TSimpleSharedPtr event);