From 0616fc13dcc3b0645578cd6316a406658b3d51f6 Mon Sep 17 00:00:00 2001 From: kungasc Date: Tue, 21 May 2024 18:02:43 +0300 Subject: [PATCH 1/2] Fix DB.CalculateReadSize --- .../engine/minikql/minikql_engine_host.cpp | 4 +- ydb/core/tablet_flat/flat_database.cpp | 6 + ydb/core/tablet_flat/flat_database.h | 1 + ydb/core/tablet_flat/flat_dbase_sz_env.h | 30 ++++- ydb/core/tablet_flat/flat_executor_ut.cpp | 106 ++++++++++++++++++ 5 files changed, 139 insertions(+), 8 deletions(-) diff --git a/ydb/core/engine/minikql/minikql_engine_host.cpp b/ydb/core/engine/minikql/minikql_engine_host.cpp index 00eee7d23eac..d0f09d1c03d9 100644 --- a/ydb/core/engine/minikql/minikql_engine_host.cpp +++ b/ydb/core/engine/minikql/minikql_engine_host.cpp @@ -73,7 +73,7 @@ bool TEngineHost::IsValidKey(TKeyDesc& key) const { return NMiniKQL::IsValidKey(Scheme, localTableId, key); } ui64 TEngineHost::CalculateReadSize(const TVector& keys) const { - NTable::TSizeEnv env; + auto env = Db.CreateSizeEnv(); for (const TKeyDesc* ki : keys) { DoCalculateReadSize(*ki, env); @@ -120,7 +120,7 @@ ui64 TEngineHost::CalculateResultSize(const TKeyDesc& key) const { if (key.Range.Point) { return Db.EstimateRowSize(localTid); } else { - NTable::TSizeEnv env; + auto env = Db.CreateSizeEnv(); DoCalculateReadSize(key, env); ui64 size = env.GetSize(); diff --git a/ydb/core/tablet_flat/flat_database.cpp b/ydb/core/tablet_flat/flat_database.cpp index fca12cc0c845..e6290f1a0314 100644 --- a/ydb/core/tablet_flat/flat_database.cpp +++ b/ydb/core/tablet_flat/flat_database.cpp @@ -270,6 +270,12 @@ TSelectRowVersionResult TDatabase::SelectRowVersion( return Require(table)->SelectRowVersion(key, Env, readFlags, visible, observer); } +TSizeEnv TDatabase::CreateSizeEnv() +{ + return TSizeEnv(Env); +} + + void TDatabase::CalculateReadSize(TSizeEnv& env, ui32 table, TRawVals minKey, TRawVals maxKey, TTagsRef tags, ui64 flg, ui64 items, ui64 bytes, EDirection direction, TRowVersion snapshot) diff --git a/ydb/core/tablet_flat/flat_database.h b/ydb/core/tablet_flat/flat_database.h index f305e01242b0..0e75fedec0f9 100644 --- a/ydb/core/tablet_flat/flat_database.h +++ b/ydb/core/tablet_flat/flat_database.h @@ -141,6 +141,7 @@ class TDatabase { EDirection direction = EDirection::Forward, TRowVersion snapshot = TRowVersion::Max()); + TSizeEnv CreateSizeEnv(); void CalculateReadSize(TSizeEnv& env, ui32 table, TRawVals minKey, TRawVals maxKey, TTagsRef tags, ui64 readFlags, ui64 itemsLimit, ui64 bytesLimit, EDirection direction = EDirection::Forward, diff --git a/ydb/core/tablet_flat/flat_dbase_sz_env.h b/ydb/core/tablet_flat/flat_dbase_sz_env.h index 9a5d36538aab..90877d8f67dd 100644 --- a/ydb/core/tablet_flat/flat_dbase_sz_env.h +++ b/ydb/core/tablet_flat/flat_dbase_sz_env.h @@ -11,6 +11,11 @@ namespace NTable { struct TSizeEnv : public IPages { using TInfo = NTabletFlatExecutor::TPrivatePageCache::TInfo; + TSizeEnv(IPages* env) + : Env(env) + { + } + TResult Locate(const TMemTable*, ui64, ui32) noexcept override { Y_ABORT("IPages::Locate(TMemTable*, ...) shouldn't be used here"); @@ -20,14 +25,28 @@ namespace NTable { { auto *partStore = CheckedCast(part); - return { true, Touch(partStore->Locate(lob, ref), ref) }; + Touch(partStore->Locate(lob, ref), ref); + + return { true, nullptr }; } - const TSharedData* TryGetPage(const TPart* part, TPageId page, TGroupId groupId) override + const TSharedData* TryGetPage(const TPart* part, TPageId pageId, TGroupId groupId) override { auto *partStore = CheckedCast(part); - return Touch(partStore->PageCollections.at(groupId.Index).Get(), page); + auto info = partStore->PageCollections.at(groupId.Index).Get(); + auto type = EPage(info->PageCollection->Page(pageId).Type); + + if (type != EPage::FlatIndex) { // do not count flat index pages + Touch(partStore->PageCollections.at(groupId.Index).Get(), pageId); + } + + if (type == EPage::FlatIndex || type == EPage::BTreeIndex) { + // need page data to continue counting + return Env->TryGetPage(part, pageId, groupId); + } + + return nullptr; } ui64 GetSize() const { @@ -35,17 +54,16 @@ namespace NTable { } private: - const TSharedData* Touch(TInfo *info, TPageId page) noexcept + void Touch(TInfo *info, TPageId page) noexcept { if (Touched[info].insert(page).second) { Pages++; Bytes += info->PageCollection->Page(page).Size; } - - return nullptr; } private: + IPages* Env; THashMap> Touched; ui64 Pages = 0; ui64 Bytes = 0; diff --git a/ydb/core/tablet_flat/flat_executor_ut.cpp b/ydb/core/tablet_flat/flat_executor_ut.cpp index 839597319c13..e31ad4ae535c 100644 --- a/ydb/core/tablet_flat/flat_executor_ut.cpp +++ b/ydb/core/tablet_flat/flat_executor_ut.cpp @@ -1,3 +1,4 @@ +#include "flat_dbase_sz_env.h" #include "flat_executor_ut_common.h" namespace NKikimr { @@ -5008,6 +5009,41 @@ Y_UNIT_TEST_SUITE(TFlatTableSnapshotWithCommits) { Y_UNIT_TEST_SUITE(TFlatTableExecutorIndexLoading) { + struct TTxCalculateReadSize : public ITransaction { + ui32 Attempt = 0; + TVector& ReadSizes; + ui64 MinKey, MaxKey; + + TTxCalculateReadSize(TVector& readSizes, ui64 minKey, ui64 maxKey) + : ReadSizes(readSizes) + , MinKey(minKey) + , MaxKey(maxKey) + { + ReadSizes.clear(); + } + + + bool Execute(TTransactionContext &txc, const TActorContext &) override + { + UNIT_ASSERT_LE(++Attempt, 10); + + const auto minKey = NScheme::TInt64::TInstance(MinKey); + const auto maxKey = NScheme::TInt64::TInstance(MaxKey); + const TVector tags{ { TRowsModel::ColumnKeyId, TRowsModel::ColumnValueId } }; + + auto sizeEnv = txc.DB.CreateSizeEnv(); + txc.DB.CalculateReadSize(sizeEnv, TRowsModel::TableId, { minKey }, { maxKey }, tags, 0, 0, 0); + ReadSizes.push_back(sizeEnv.GetSize()); + + return txc.DB.Precharge(TRowsModel::TableId, { minKey }, { maxKey }, tags, 0, 0, 0); + } + + void Complete(const TActorContext &ctx) override + { + ctx.Send(ctx.SelfID, new NFake::TEvReturn); + } + }; + struct TTxPrechargeAndSeek : public ITransaction { ui32 Attempt = 0; bool Pinned = false; @@ -5056,6 +5092,76 @@ Y_UNIT_TEST_SUITE(TFlatTableExecutorIndexLoading) { env->DispatchEvents(options); } + Y_UNIT_TEST(CalculateReadSize_FlatIndex) { + TMyEnvBase env; + TRowsModel rows; + const ui32 rowsCount = 1024; + + auto &appData = env->GetAppData(); + appData.FeatureFlags.SetEnableLocalDBBtreeIndex(false); + appData.FeatureFlags.SetEnableLocalDBFlatIndex(true); + + env.FireTablet(env.Edge, env.Tablet, [&env](const TActorId &tablet, TTabletStorageInfo *info) { + return new TTestFlatTablet(env.Edge, tablet, info); + }); + env.WaitForWakeUp(); + ZeroSharedCache(env); + + env.SendSync(rows.MakeScheme(new TCompactionPolicy(), false)); + + env.SendSync(rows.MakeRows(rowsCount, 10*1024)); + + env.SendSync(new NFake::TEvCompact(TRowsModel::TableId)); + env.WaitFor(); + + TVector sizes; + + env.SendSync(new NFake::TEvExecute{ new TTxCalculateReadSize(sizes, 0, 1) }); + UNIT_ASSERT_VALUES_EQUAL(sizes, (TVector{20566, 20566})); + + env.SendSync(new NFake::TEvExecute{ new TTxCalculateReadSize(sizes, 100, 200) }); + UNIT_ASSERT_VALUES_EQUAL(sizes, (TVector{1048866, 1048866})); + + env.SendSync(new NFake::TEvExecute{ new TTxCalculateReadSize(sizes, 300, 700) }); + UNIT_ASSERT_VALUES_EQUAL(sizes, (TVector{4133766, 4133766})); + } + + Y_UNIT_TEST(CalculateReadSize_BTreeIndex) { + TMyEnvBase env; + TRowsModel rows; + const ui32 rowsCount = 1024; + + auto &appData = env->GetAppData(); + appData.FeatureFlags.SetEnableLocalDBBtreeIndex(true); + appData.FeatureFlags.SetEnableLocalDBFlatIndex(false); + + env.FireTablet(env.Edge, env.Tablet, [&env](const TActorId &tablet, TTabletStorageInfo *info) { + return new TTestFlatTablet(env.Edge, tablet, info); + }); + env.WaitForWakeUp(); + ZeroSharedCache(env); + + auto policy = MakeIntrusive(); + policy->MinBTreeIndexNodeSize = 128; + env.SendSync(rows.MakeScheme(std::move(policy))); + + env.SendSync(rows.MakeRows(rowsCount, 10*1024)); + + env.SendSync(new NFake::TEvCompact(TRowsModel::TableId)); + env.WaitFor(); + + TVector sizes; + + env.SendSync(new NFake::TEvExecute{ new TTxCalculateReadSize(sizes, 0, 1) }); + UNIT_ASSERT_VALUES_EQUAL(sizes, (TVector{102, 1090, 1784, 2478, 23044, 23044})); + + env.SendSync(new NFake::TEvExecute{ new TTxCalculateReadSize(sizes, 100, 200) }); + UNIT_ASSERT_VALUES_EQUAL(sizes, (TVector{102, 1090, 2478, 8030, 1056896, 1056896})); + + env.SendSync(new NFake::TEvExecute{ new TTxCalculateReadSize(sizes, 300, 700) }); + UNIT_ASSERT_VALUES_EQUAL(sizes, (TVector{102, 1090, 4560, 25033, 4158799, 4158799})); + } + Y_UNIT_TEST(PrechargeAndSeek_FlatIndex) { TMyEnvBase env; TRowsModel rows; From d7b7f5b6c871da3ec81df55d9d7f9f688a6253c8 Mon Sep 17 00:00:00 2001 From: kungasc Date: Wed, 22 May 2024 10:49:38 +0300 Subject: [PATCH 2/2] cr: don't count b-tree index either --- ydb/core/tablet_flat/flat_dbase_sz_env.h | 23 ++++++++++++----------- ydb/core/tablet_flat/flat_executor_ut.cpp | 6 +++--- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/ydb/core/tablet_flat/flat_dbase_sz_env.h b/ydb/core/tablet_flat/flat_dbase_sz_env.h index 90877d8f67dd..02f8f172c6bd 100644 --- a/ydb/core/tablet_flat/flat_dbase_sz_env.h +++ b/ydb/core/tablet_flat/flat_dbase_sz_env.h @@ -25,7 +25,7 @@ namespace NTable { { auto *partStore = CheckedCast(part); - Touch(partStore->Locate(lob, ref), ref); + AddPageSize(partStore->Locate(lob, ref), ref); return { true, nullptr }; } @@ -37,16 +37,17 @@ namespace NTable { auto info = partStore->PageCollections.at(groupId.Index).Get(); auto type = EPage(info->PageCollection->Page(pageId).Type); - if (type != EPage::FlatIndex) { // do not count flat index pages - Touch(partStore->PageCollections.at(groupId.Index).Get(), pageId); + switch (type) { + case EPage::FlatIndex: + case EPage::BTreeIndex: + // need index pages to continue counting + // do not count index + // if these pages are not in memory, data won't be counted in precharge + return Env->TryGetPage(part, pageId, groupId); + default: + AddPageSize(partStore->PageCollections.at(groupId.Index).Get(), pageId); + return nullptr; } - - if (type == EPage::FlatIndex || type == EPage::BTreeIndex) { - // need page data to continue counting - return Env->TryGetPage(part, pageId, groupId); - } - - return nullptr; } ui64 GetSize() const { @@ -54,7 +55,7 @@ namespace NTable { } private: - void Touch(TInfo *info, TPageId page) noexcept + void AddPageSize(TInfo *info, TPageId page) noexcept { if (Touched[info].insert(page).second) { Pages++; diff --git a/ydb/core/tablet_flat/flat_executor_ut.cpp b/ydb/core/tablet_flat/flat_executor_ut.cpp index e31ad4ae535c..ad7a7e308a3d 100644 --- a/ydb/core/tablet_flat/flat_executor_ut.cpp +++ b/ydb/core/tablet_flat/flat_executor_ut.cpp @@ -5153,13 +5153,13 @@ Y_UNIT_TEST_SUITE(TFlatTableExecutorIndexLoading) { TVector sizes; env.SendSync(new NFake::TEvExecute{ new TTxCalculateReadSize(sizes, 0, 1) }); - UNIT_ASSERT_VALUES_EQUAL(sizes, (TVector{102, 1090, 1784, 2478, 23044, 23044})); + UNIT_ASSERT_VALUES_EQUAL(sizes, (TVector{0, 0, 0, 0, 20566, 20566})); env.SendSync(new NFake::TEvExecute{ new TTxCalculateReadSize(sizes, 100, 200) }); - UNIT_ASSERT_VALUES_EQUAL(sizes, (TVector{102, 1090, 2478, 8030, 1056896, 1056896})); + UNIT_ASSERT_VALUES_EQUAL(sizes, (TVector{0, 0, 0, 0, 1048866, 1048866})); env.SendSync(new NFake::TEvExecute{ new TTxCalculateReadSize(sizes, 300, 700) }); - UNIT_ASSERT_VALUES_EQUAL(sizes, (TVector{102, 1090, 4560, 25033, 4158799, 4158799})); + UNIT_ASSERT_VALUES_EQUAL(sizes, (TVector{0, 0, 0, 0, 4133766, 4133766})); } Y_UNIT_TEST(PrechargeAndSeek_FlatIndex) {