From 159892830e0f7352f259d3447e4ad3a333a022f0 Mon Sep 17 00:00:00 2001 From: adi_holden Date: Thu, 29 Feb 2024 16:28:24 +0200 Subject: [PATCH 1/3] bug(tiering): add prints to find crash flow Signed-off-by: adi_holden --- src/server/tiered_storage.cc | 5 ++++- src/server/tiered_storage_test.cc | 24 ++++++++++++++++++++++-- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/src/server/tiered_storage.cc b/src/server/tiered_storage.cc index 7264d7e3b00..bc27f75b629 100644 --- a/src/server/tiered_storage.cc +++ b/src/server/tiered_storage.cc @@ -402,6 +402,7 @@ void TieredStorage::FinishIoRequest(int io_res, InflightWriteRequest* req) { } PrimeIterator TieredStorage::Load(DbIndex db_index, PrimeIterator it, string_view key) { + VLOG(2) << "Load:" << key; PrimeValue* entry = &it->second; CHECK(entry->IsExternal()); DCHECK_EQ(entry->ObjType(), OBJ_STRING); @@ -469,7 +470,7 @@ bool TieredStorage::PrepareForOffload(DbIndex db_index, PrimeIterator it) { return false; } - VLOG(2) << "ScheduleOffload:" << it->first.ToString(); + VLOG(2) << "PrepareForOffload:" << it->first.ToString(); bin_record.pending_entries.insert(it->first.AsRef()); it->second.SetIoPending(true); @@ -480,6 +481,7 @@ bool TieredStorage::PrepareForOffload(DbIndex db_index, PrimeIterator it) { } void TieredStorage::CancelOffload(DbIndex db_index, PrimeIterator it) { + VLOG(2) << "PrepareForOffload:" << it->first.ToString(); size_t blob_len = it->second.Size(); if (blob_len > kMaxSmallBin) { return; @@ -521,6 +523,7 @@ error_code TieredStorage::ScheduleOffload(DbIndex db_index, PrimeIterator it) { } error_code TieredStorage::ScheduleOffloadInternal(DbIndex db_index, PrimeIterator it) { + VLOG(2) << "PrepareForOffload:" << it->first.ToString(); size_t blob_len = it->second.Size(); if (blob_len > kMaxSmallBin) { diff --git a/src/server/tiered_storage_test.cc b/src/server/tiered_storage_test.cc index 1b702426f5c..5752e2b4464 100644 --- a/src/server/tiered_storage_test.cc +++ b/src/server/tiered_storage_test.cc @@ -16,7 +16,8 @@ using absl::SetFlag; using absl::StrCat; ABSL_DECLARE_FLAG(string, tiered_prefix); - +ABSL_DECLARE_FLAG(uint32_t, tiered_storage_max_pending_writes); +ABSL_DECLARE_FLAG(float, tiered_offload_threshold); namespace dfly { class TieredStorageTest : public BaseFamilyTest { @@ -50,7 +51,7 @@ bool TieredStorageTest::WaitUntilTieredEntriesGT(size_t value, int db_index) { auto tiered_entries = GetMetrics().db_stats[db_index].tiered_entries; return tiered_entries > value; }; - return WaitUntilCondition(std::move(cb)); + return WaitUntilCondition(std::move(cb), 2000ms); } bool TieredStorageTest::WaitUntilTieredEntriesEQ(size_t value, int db_index) { @@ -318,4 +319,23 @@ TEST_F(TieredStorageTest, GetValueValidation) { EXPECT_EQ(m.db_stats[0].tiered_entries, 0); } +TEST_F(TieredStorageTest, CrashBug) { + shard_set->TEST_EnableHeartBeat(); + max_memory_limit = 300000; + absl::FlagSaver fs; + // Disable writing to disk + absl::SetFlag(&FLAGS_tiered_storage_max_pending_writes, 0); + + const int kKeyNum = 100; + FillExternalKeys(kKeyNum); + usleep(20000); + Metrics m = GetMetrics(); + EXPECT_EQ(m.db_stats[0].tiered_entries, 0); + + absl::SetFlag(&FLAGS_tiered_storage_max_pending_writes, 1); + absl::SetFlag(&FLAGS_tiered_offload_threshold, 0); + FillExternalKeys(kKeyNum); + EXPECT_TRUE(WaitUntilTieredEntriesGT(0)); +} + } // namespace dfly From 25ba94b5dda445cc5dd2db3ab4330974a8a259a5 Mon Sep 17 00:00:00 2001 From: adi_holden Date: Thu, 29 Feb 2024 16:35:54 +0200 Subject: [PATCH 2/3] bug(tiering): add prints to find crash flow Signed-off-by: adi_holden --- src/server/tiered_storage.cc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/server/tiered_storage.cc b/src/server/tiered_storage.cc index bc27f75b629..e96534ca38a 100644 --- a/src/server/tiered_storage.cc +++ b/src/server/tiered_storage.cc @@ -338,12 +338,12 @@ void TieredStorage::Free(PrimeIterator it, DbTableStats* stats) { alloc_.Free(offset, len); } else { uint32_t offs_page = offset / kBlockLen; - auto it = page_refcnt_.find(offs_page); - CHECK(it != page_refcnt_.end()) << offs_page; - CHECK_GT(it->second, 0u); - if (--it->second == 0) { + auto page_it = page_refcnt_.find(offs_page); + CHECK(page_it != page_refcnt_.end()) << offs_page << "," << it->first.ToString(); + CHECK_GT(page_it->second, 0u); + if (--page_it->second == 0) { alloc_.Free(offs_page * kBlockLen, kBlockLen); - page_refcnt_.erase(it); + page_refcnt_.erase(page_it); } } From 9db7299d9cd06f46f84cb68c367d379d0c175261 Mon Sep 17 00:00:00 2001 From: adi_holden Date: Wed, 6 Mar 2024 13:56:50 +0200 Subject: [PATCH 3/3] type Signed-off-by: adi_holden --- src/server/tiered_storage.cc | 2 +- src/server/tiered_storage_test.cc | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/server/tiered_storage.cc b/src/server/tiered_storage.cc index e96534ca38a..584bc881613 100644 --- a/src/server/tiered_storage.cc +++ b/src/server/tiered_storage.cc @@ -481,7 +481,7 @@ bool TieredStorage::PrepareForOffload(DbIndex db_index, PrimeIterator it) { } void TieredStorage::CancelOffload(DbIndex db_index, PrimeIterator it) { - VLOG(2) << "PrepareForOffload:" << it->first.ToString(); + VLOG(2) << "CancelOffload:" << it->first.ToString(); size_t blob_len = it->second.Size(); if (blob_len > kMaxSmallBin) { return; diff --git a/src/server/tiered_storage_test.cc b/src/server/tiered_storage_test.cc index 5752e2b4464..b2ff10e5168 100644 --- a/src/server/tiered_storage_test.cc +++ b/src/server/tiered_storage_test.cc @@ -51,7 +51,7 @@ bool TieredStorageTest::WaitUntilTieredEntriesGT(size_t value, int db_index) { auto tiered_entries = GetMetrics().db_stats[db_index].tiered_entries; return tiered_entries > value; }; - return WaitUntilCondition(std::move(cb), 2000ms); + return WaitUntilCondition(std::move(cb)); } bool TieredStorageTest::WaitUntilTieredEntriesEQ(size_t value, int db_index) { @@ -319,7 +319,7 @@ TEST_F(TieredStorageTest, GetValueValidation) { EXPECT_EQ(m.db_stats[0].tiered_entries, 0); } -TEST_F(TieredStorageTest, CrashBug) { +TEST_F(TieredStorageTest, BackgroundOffload) { shard_set->TEST_EnableHeartBeat(); max_memory_limit = 300000; absl::FlagSaver fs;