From c009190dcbda62242e8a92407c68217704c3b67b Mon Sep 17 00:00:00 2001 From: Cheng Chang Date: Fri, 15 Jan 2021 22:44:25 -0800 Subject: [PATCH 1/5] be able to ignore blob and WAL related VersionEdits in older versions --- db/version_edit.cc | 57 ++++++++++++++++++--- db/version_edit_test.cc | 110 +++++++++++++++++++++++++++++++++++----- 2 files changed, 145 insertions(+), 22 deletions(-) diff --git a/db/version_edit.cc b/db/version_edit.cc index ddaadc58dac..da6c1d593c7 100644 --- a/db/version_edit.cc +++ b/db/version_edit.cc @@ -217,22 +217,30 @@ bool VersionEdit::EncodeTo(std::string* dst) const { for (const auto& blob_file_addition : blob_file_additions_) { PutVarint32(dst, kBlobFileAddition); - blob_file_addition.EncodeTo(dst); + std::string encoded; + blob_file_addition.EncodeTo(&encoded); + PutLengthPrefixedSlice(dst, encoded); } for (const auto& blob_file_garbage : blob_file_garbages_) { PutVarint32(dst, kBlobFileGarbage); - blob_file_garbage.EncodeTo(dst); + std::string encoded; + blob_file_garbage.EncodeTo(&encoded); + PutLengthPrefixedSlice(dst, encoded); } for (const auto& wal_addition : wal_additions_) { PutVarint32(dst, kWalAddition); - wal_addition.EncodeTo(dst); + std::string encoded; + wal_addition.EncodeTo(&encoded); + PutLengthPrefixedSlice(dst, encoded); } if (!wal_deletion_.IsEmpty()) { PutVarint32(dst, kWalDeletion); - wal_deletion_.EncodeTo(dst); + std::string encoded; + wal_deletion_.EncodeTo(&encoded); + PutLengthPrefixedSlice(dst, encoded); } // 0 is default and does not need to be explicitly written @@ -375,6 +383,11 @@ const char* VersionEdit::DecodeNewFile4From(Slice* input) { Status VersionEdit::DecodeFrom(const Slice& src) { Clear(); +#ifndef NDEBUG + bool ignore_ignorable_tags = false; + TEST_SYNC_POINT_CALLBACK("VersionEdit::EncodeTo:IgnoreIgnorableTags", + &ignore_ignorable_tags); +#endif Slice input = src; const char* msg = nullptr; uint32_t tag = 0; @@ -385,6 +398,12 @@ Status VersionEdit::DecodeFrom(const Slice& src) { Slice str; InternalKey key; while (msg == nullptr && GetVarint32(&input, &tag)) { +#ifndef NDEBUG + if (ignore_ignorable_tags && tag > kTagSafeIgnoreMask) { + printf("tag = %d\n", tag); + tag = kTagSafeIgnoreMask; + } +#endif switch (tag) { case kDbId: if (GetLengthPrefixedSlice(&input, &str)) { @@ -543,8 +562,13 @@ Status VersionEdit::DecodeFrom(const Slice& src) { } case kBlobFileAddition: { + Slice encoded; + if (!GetLengthPrefixedSlice(&input, &encoded)) { + msg = "BlobFileAddition not prefixed by length"; + break; + } BlobFileAddition blob_file_addition; - const Status s = blob_file_addition.DecodeFrom(&input); + const Status s = blob_file_addition.DecodeFrom(&encoded); if (!s.ok()) { return s; } @@ -554,8 +578,13 @@ Status VersionEdit::DecodeFrom(const Slice& src) { } case kBlobFileGarbage: { + Slice encoded; + if (!GetLengthPrefixedSlice(&input, &encoded)) { + msg = "BlobFileGarbage not prefixed by length"; + break; + } BlobFileGarbage blob_file_garbage; - const Status s = blob_file_garbage.DecodeFrom(&input); + const Status s = blob_file_garbage.DecodeFrom(&encoded); if (!s.ok()) { return s; } @@ -565,8 +594,14 @@ Status VersionEdit::DecodeFrom(const Slice& src) { } case kWalAddition: { + Slice encoded; + if (!GetLengthPrefixedSlice(&input, &encoded)) { + msg = "WalAddition not prefixed by length"; + break; + } + WalAddition wal_addition; - const Status s = wal_addition.DecodeFrom(&input); + const Status s = wal_addition.DecodeFrom(&encoded); if (!s.ok()) { return s; } @@ -576,8 +611,14 @@ Status VersionEdit::DecodeFrom(const Slice& src) { } case kWalDeletion: { + Slice encoded; + if (!GetLengthPrefixedSlice(&input, &encoded)) { + msg = "WalDeletion not prefixed by length"; + break; + } + WalDeletion wal_deletion; - const Status s = wal_deletion.DecodeFrom(&input); + const Status s = wal_deletion.DecodeFrom(&encoded); if (!s.ok()) { return s; } diff --git a/db/version_edit_test.cc b/db/version_edit_test.cc index a0869b3c76f..cc8ac25eedb 100644 --- a/db/version_edit_test.cc +++ b/db/version_edit_test.cc @@ -324,14 +324,22 @@ TEST_F(VersionEditTest, AddWalEncodeDecode) { TestEncodeDecode(edit); } +static std::string PrefixEncodedWalAdditionWithLength( + const std::string& encoded) { + std::string ret; + PutVarint32(&ret, Tag::kWalAddition); + PutLengthPrefixedSlice(&ret, encoded); + return ret; +} + TEST_F(VersionEditTest, AddWalDecodeBadLogNumber) { std::string encoded; - PutVarint32(&encoded, Tag::kWalAddition); { // No log number. + std::string encoded_edit = PrefixEncodedWalAdditionWithLength(encoded); VersionEdit edit; - Status s = edit.DecodeFrom(encoded); + Status s = edit.DecodeFrom(encoded_edit); ASSERT_TRUE(s.IsCorruption()); ASSERT_TRUE(s.ToString().find("Error decoding WAL log number") != std::string::npos) @@ -345,8 +353,10 @@ TEST_F(VersionEditTest, AddWalDecodeBadLogNumber) { unsigned char* ptr = reinterpret_cast(&c); *ptr = 128; encoded.append(1, c); + + std::string encoded_edit = PrefixEncodedWalAdditionWithLength(encoded); VersionEdit edit; - Status s = edit.DecodeFrom(encoded); + Status s = edit.DecodeFrom(encoded_edit); ASSERT_TRUE(s.IsCorruption()); ASSERT_TRUE(s.ToString().find("Error decoding WAL log number") != std::string::npos) @@ -358,14 +368,14 @@ TEST_F(VersionEditTest, AddWalDecodeBadTag) { constexpr WalNumber kLogNumber = 100; constexpr uint64_t kSizeInBytes = 100; - std::string encoded_without_tag; - PutVarint32(&encoded_without_tag, Tag::kWalAddition); - PutVarint64(&encoded_without_tag, kLogNumber); + std::string encoded; + PutVarint64(&encoded, kLogNumber); { // No tag. + std::string encoded_edit = PrefixEncodedWalAdditionWithLength(encoded); VersionEdit edit; - Status s = edit.DecodeFrom(encoded_without_tag); + Status s = edit.DecodeFrom(encoded_edit); ASSERT_TRUE(s.IsCorruption()); ASSERT_TRUE(s.ToString().find("Error decoding tag") != std::string::npos) << s.ToString(); @@ -373,12 +383,15 @@ TEST_F(VersionEditTest, AddWalDecodeBadTag) { { // Only has size tag, no terminate tag. - std::string encoded_with_size = encoded_without_tag; + std::string encoded_with_size = encoded; PutVarint32(&encoded_with_size, static_cast(WalAdditionTag::kSyncedSize)); PutVarint64(&encoded_with_size, kSizeInBytes); + + std::string encoded_edit = + PrefixEncodedWalAdditionWithLength(encoded_with_size); VersionEdit edit; - Status s = edit.DecodeFrom(encoded_with_size); + Status s = edit.DecodeFrom(encoded_edit); ASSERT_TRUE(s.IsCorruption()); ASSERT_TRUE(s.ToString().find("Error decoding tag") != std::string::npos) << s.ToString(); @@ -386,11 +399,14 @@ TEST_F(VersionEditTest, AddWalDecodeBadTag) { { // Only has terminate tag. - std::string encoded_with_terminate = encoded_without_tag; + std::string encoded_with_terminate = encoded; PutVarint32(&encoded_with_terminate, static_cast(WalAdditionTag::kTerminate)); + + std::string encoded_edit = + PrefixEncodedWalAdditionWithLength(encoded_with_terminate); VersionEdit edit; - ASSERT_OK(edit.DecodeFrom(encoded_with_terminate)); + ASSERT_OK(edit.DecodeFrom(encoded_edit)); auto& wal_addition = edit.GetWalAdditions()[0]; ASSERT_EQ(wal_addition.GetLogNumber(), kLogNumber); ASSERT_FALSE(wal_addition.GetMetadata().HasSyncedSize()); @@ -401,15 +417,15 @@ TEST_F(VersionEditTest, AddWalDecodeNoSize) { constexpr WalNumber kLogNumber = 100; std::string encoded; - PutVarint32(&encoded, Tag::kWalAddition); PutVarint64(&encoded, kLogNumber); PutVarint32(&encoded, static_cast(WalAdditionTag::kSyncedSize)); // No real size after the size tag. { // Without terminate tag. + std::string encoded_edit = PrefixEncodedWalAdditionWithLength(encoded); VersionEdit edit; - Status s = edit.DecodeFrom(encoded); + Status s = edit.DecodeFrom(encoded_edit); ASSERT_TRUE(s.IsCorruption()); ASSERT_TRUE(s.ToString().find("Error decoding WAL file size") != std::string::npos) @@ -419,8 +435,10 @@ TEST_F(VersionEditTest, AddWalDecodeNoSize) { { // With terminate tag. PutVarint32(&encoded, static_cast(WalAdditionTag::kTerminate)); + + std::string encoded_edit = PrefixEncodedWalAdditionWithLength(encoded); VersionEdit edit; - Status s = edit.DecodeFrom(encoded); + Status s = edit.DecodeFrom(encoded_edit); ASSERT_TRUE(s.IsCorruption()); // The terminate tag is misunderstood as the size. ASSERT_TRUE(s.ToString().find("Error decoding tag") != std::string::npos) @@ -515,6 +533,70 @@ TEST_F(VersionEditTest, FullHistoryTsLow) { TestEncodeDecode(edit); } +// Tests that if RocksDB is downgraded, the new types of VersionEdits +// that have a tag larger than kTagSafeIgnoreMask can be safely ignored. +TEST_F(VersionEditTest, IgnorableTags) { + SyncPoint::GetInstance()->SetCallBack( + "VersionEdit::EncodeTo:IgnoreIgnorableTags", [&](void* arg) { + bool* ignore = static_cast(arg); + *ignore = true; + }); + SyncPoint::GetInstance()->EnableProcessing(); + + constexpr uint64_t kPrevLogNumber = 100; + constexpr uint64_t kLogNumber = 200; + constexpr uint64_t kNextFileNumber = 300; + constexpr uint64_t kColumnFamilyId = 400; + + VersionEdit edit; + // Add some ignorable entries. + for (int i = 0; i < 2; i++) { + edit.AddWal(i + 1, WalMetadata(i + 2)); + } + edit.SetDBId("db_id"); + // Add unignorable entries. + edit.SetPrevLogNumber(kPrevLogNumber); + edit.SetLogNumber(kLogNumber); + // Add more ignorable entries. + edit.DeleteWalsBefore(100); + for (int i = 0; i < 2; i++) { + edit.AddBlobFile(i + 1, i + 2, i + 3, "checksum_method", "checksum_value"); + } + // Add unignorable entry. + edit.SetNextFile(kNextFileNumber); + // Add more ignorable entries. + for (int i = 0; i < 2; i++) { + edit.AddBlobFileGarbage(i + 1, i + 2, i + 3); + } + edit.SetFullHistoryTsLow("ts"); + // Add unignorable entry. + edit.SetColumnFamily(kColumnFamilyId); + + std::string encoded; + ASSERT_TRUE(edit.EncodeTo(&encoded)); + + VersionEdit decoded; + ASSERT_OK(decoded.DecodeFrom(encoded)); + + // Check that all ignorable entries are ignored. + ASSERT_FALSE(decoded.HasDbId()); + ASSERT_FALSE(decoded.HasFullHistoryTsLow()); + ASSERT_TRUE(decoded.GetBlobFileAdditions().empty()); + ASSERT_TRUE(decoded.GetBlobFileGarbages().empty()); + ASSERT_FALSE(decoded.IsWalAddition()); + ASSERT_FALSE(decoded.IsWalDeletion()); + ASSERT_TRUE(decoded.GetWalAdditions().empty()); + ASSERT_TRUE(decoded.GetWalDeletion().IsEmpty()); + + // Check that unignorable entries are still present. + ASSERT_EQ(edit.GetPrevLogNumber(), kPrevLogNumber); + ASSERT_EQ(edit.GetLogNumber(), kLogNumber); + ASSERT_EQ(edit.GetNextFile(), kNextFileNumber); + ASSERT_EQ(edit.GetColumnFamily(), kColumnFamilyId); + + SyncPoint::GetInstance()->DisableProcessing(); +} + } // namespace ROCKSDB_NAMESPACE int main(int argc, char** argv) { From 1a2f02ee34d008cfcc69695104d24e7bb83c1bf4 Mon Sep 17 00:00:00 2001 From: Cheng Chang Date: Fri, 15 Jan 2021 23:37:14 -0800 Subject: [PATCH 2/5] remove changes to blob related edits --- db/version_edit.cc | 22 ++++------------------ db/version_edit_test.cc | 8 -------- 2 files changed, 4 insertions(+), 26 deletions(-) diff --git a/db/version_edit.cc b/db/version_edit.cc index da6c1d593c7..634f8e593fc 100644 --- a/db/version_edit.cc +++ b/db/version_edit.cc @@ -217,16 +217,12 @@ bool VersionEdit::EncodeTo(std::string* dst) const { for (const auto& blob_file_addition : blob_file_additions_) { PutVarint32(dst, kBlobFileAddition); - std::string encoded; - blob_file_addition.EncodeTo(&encoded); - PutLengthPrefixedSlice(dst, encoded); + blob_file_addition.EncodeTo(dst); } for (const auto& blob_file_garbage : blob_file_garbages_) { PutVarint32(dst, kBlobFileGarbage); - std::string encoded; - blob_file_garbage.EncodeTo(&encoded); - PutLengthPrefixedSlice(dst, encoded); + blob_file_garbage.EncodeTo(dst); } for (const auto& wal_addition : wal_additions_) { @@ -562,13 +558,8 @@ Status VersionEdit::DecodeFrom(const Slice& src) { } case kBlobFileAddition: { - Slice encoded; - if (!GetLengthPrefixedSlice(&input, &encoded)) { - msg = "BlobFileAddition not prefixed by length"; - break; - } BlobFileAddition blob_file_addition; - const Status s = blob_file_addition.DecodeFrom(&encoded); + const Status s = blob_file_addition.DecodeFrom(&input); if (!s.ok()) { return s; } @@ -578,13 +569,8 @@ Status VersionEdit::DecodeFrom(const Slice& src) { } case kBlobFileGarbage: { - Slice encoded; - if (!GetLengthPrefixedSlice(&input, &encoded)) { - msg = "BlobFileGarbage not prefixed by length"; - break; - } BlobFileGarbage blob_file_garbage; - const Status s = blob_file_garbage.DecodeFrom(&encoded); + const Status s = blob_file_garbage.DecodeFrom(&input); if (!s.ok()) { return s; } diff --git a/db/version_edit_test.cc b/db/version_edit_test.cc index cc8ac25eedb..1dba29310e4 100644 --- a/db/version_edit_test.cc +++ b/db/version_edit_test.cc @@ -559,15 +559,9 @@ TEST_F(VersionEditTest, IgnorableTags) { edit.SetLogNumber(kLogNumber); // Add more ignorable entries. edit.DeleteWalsBefore(100); - for (int i = 0; i < 2; i++) { - edit.AddBlobFile(i + 1, i + 2, i + 3, "checksum_method", "checksum_value"); - } // Add unignorable entry. edit.SetNextFile(kNextFileNumber); // Add more ignorable entries. - for (int i = 0; i < 2; i++) { - edit.AddBlobFileGarbage(i + 1, i + 2, i + 3); - } edit.SetFullHistoryTsLow("ts"); // Add unignorable entry. edit.SetColumnFamily(kColumnFamilyId); @@ -581,8 +575,6 @@ TEST_F(VersionEditTest, IgnorableTags) { // Check that all ignorable entries are ignored. ASSERT_FALSE(decoded.HasDbId()); ASSERT_FALSE(decoded.HasFullHistoryTsLow()); - ASSERT_TRUE(decoded.GetBlobFileAdditions().empty()); - ASSERT_TRUE(decoded.GetBlobFileGarbages().empty()); ASSERT_FALSE(decoded.IsWalAddition()); ASSERT_FALSE(decoded.IsWalDeletion()); ASSERT_TRUE(decoded.GetWalAdditions().empty()); From 65f6c2eaa15a276a01d16a7ce1ff6f3d3eef9d6c Mon Sep 17 00:00:00 2001 From: Cheng Chang Date: Tue, 19 Jan 2021 12:52:03 -0800 Subject: [PATCH 3/5] use new tags --- db/version_edit.cc | 26 ++++++++++++++++++++++++-- db/version_edit.h | 2 ++ db/version_edit_test.cc | 2 +- 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/db/version_edit.cc b/db/version_edit.cc index 634f8e593fc..b7261c6f3fa 100644 --- a/db/version_edit.cc +++ b/db/version_edit.cc @@ -226,14 +226,14 @@ bool VersionEdit::EncodeTo(std::string* dst) const { } for (const auto& wal_addition : wal_additions_) { - PutVarint32(dst, kWalAddition); + PutVarint32(dst, kNewWalAddition); std::string encoded; wal_addition.EncodeTo(&encoded); PutLengthPrefixedSlice(dst, encoded); } if (!wal_deletion_.IsEmpty()) { - PutVarint32(dst, kWalDeletion); + PutVarint32(dst, kNewWalDeletion); std::string encoded; wal_deletion_.EncodeTo(&encoded); PutLengthPrefixedSlice(dst, encoded); @@ -580,6 +580,17 @@ Status VersionEdit::DecodeFrom(const Slice& src) { } case kWalAddition: { + WalAddition wal_addition; + const Status s = wal_addition.DecodeFrom(&input); + if (!s.ok()) { + return s; + } + + wal_additions_.emplace_back(std::move(wal_addition)); + break; + } + + case kNewWalAddition: { Slice encoded; if (!GetLengthPrefixedSlice(&input, &encoded)) { msg = "WalAddition not prefixed by length"; @@ -597,6 +608,17 @@ Status VersionEdit::DecodeFrom(const Slice& src) { } case kWalDeletion: { + WalDeletion wal_deletion; + const Status s = wal_deletion.DecodeFrom(&input); + if (!s.ok()) { + return s; + } + + wal_deletion_ = std::move(wal_deletion); + break; + } + + case kNewWalDeletion: { Slice encoded; if (!GetLengthPrefixedSlice(&input, &encoded)) { msg = "WalDeletion not prefixed by length"; diff --git a/db/version_edit.h b/db/version_edit.h index 6b045878bc0..c52be0f832b 100644 --- a/db/version_edit.h +++ b/db/version_edit.h @@ -62,6 +62,8 @@ enum Tag : uint32_t { kWalAddition, kWalDeletion, kFullHistoryTsLow, + kNewWalAddition, + kNewWalDeletion, }; enum NewFileCustomTag : uint32_t { diff --git a/db/version_edit_test.cc b/db/version_edit_test.cc index 1dba29310e4..5eaced78544 100644 --- a/db/version_edit_test.cc +++ b/db/version_edit_test.cc @@ -327,7 +327,7 @@ TEST_F(VersionEditTest, AddWalEncodeDecode) { static std::string PrefixEncodedWalAdditionWithLength( const std::string& encoded) { std::string ret; - PutVarint32(&ret, Tag::kWalAddition); + PutVarint32(&ret, Tag::kNewWalAddition); PutLengthPrefixedSlice(&ret, encoded); return ret; } From b76199d958c4b4aca88d5d5836d81dba2615d03c Mon Sep 17 00:00:00 2001 From: Cheng Chang Date: Tue, 19 Jan 2021 13:01:46 -0800 Subject: [PATCH 4/5] remove printf --- db/version_edit.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/db/version_edit.cc b/db/version_edit.cc index b7261c6f3fa..f5e4de50090 100644 --- a/db/version_edit.cc +++ b/db/version_edit.cc @@ -396,7 +396,6 @@ Status VersionEdit::DecodeFrom(const Slice& src) { while (msg == nullptr && GetVarint32(&input, &tag)) { #ifndef NDEBUG if (ignore_ignorable_tags && tag > kTagSafeIgnoreMask) { - printf("tag = %d\n", tag); tag = kTagSafeIgnoreMask; } #endif From 160f99f7cb093d09f0866bba5df23c64d3b89a01 Mon Sep 17 00:00:00 2001 From: Cheng Chang Date: Tue, 19 Jan 2021 15:50:36 -0800 Subject: [PATCH 5/5] rename tags --- db/version_edit.cc | 8 ++++---- db/version_edit.h | 4 ++-- db/version_edit_test.cc | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/db/version_edit.cc b/db/version_edit.cc index f5e4de50090..284b65f7183 100644 --- a/db/version_edit.cc +++ b/db/version_edit.cc @@ -226,14 +226,14 @@ bool VersionEdit::EncodeTo(std::string* dst) const { } for (const auto& wal_addition : wal_additions_) { - PutVarint32(dst, kNewWalAddition); + PutVarint32(dst, kWalAddition2); std::string encoded; wal_addition.EncodeTo(&encoded); PutLengthPrefixedSlice(dst, encoded); } if (!wal_deletion_.IsEmpty()) { - PutVarint32(dst, kNewWalDeletion); + PutVarint32(dst, kWalDeletion2); std::string encoded; wal_deletion_.EncodeTo(&encoded); PutLengthPrefixedSlice(dst, encoded); @@ -589,7 +589,7 @@ Status VersionEdit::DecodeFrom(const Slice& src) { break; } - case kNewWalAddition: { + case kWalAddition2: { Slice encoded; if (!GetLengthPrefixedSlice(&input, &encoded)) { msg = "WalAddition not prefixed by length"; @@ -617,7 +617,7 @@ Status VersionEdit::DecodeFrom(const Slice& src) { break; } - case kNewWalDeletion: { + case kWalDeletion2: { Slice encoded; if (!GetLengthPrefixedSlice(&input, &encoded)) { msg = "WalDeletion not prefixed by length"; diff --git a/db/version_edit.h b/db/version_edit.h index c52be0f832b..a80543a0da9 100644 --- a/db/version_edit.h +++ b/db/version_edit.h @@ -62,8 +62,8 @@ enum Tag : uint32_t { kWalAddition, kWalDeletion, kFullHistoryTsLow, - kNewWalAddition, - kNewWalDeletion, + kWalAddition2, + kWalDeletion2, }; enum NewFileCustomTag : uint32_t { diff --git a/db/version_edit_test.cc b/db/version_edit_test.cc index 5eaced78544..43ae6840fb1 100644 --- a/db/version_edit_test.cc +++ b/db/version_edit_test.cc @@ -327,7 +327,7 @@ TEST_F(VersionEditTest, AddWalEncodeDecode) { static std::string PrefixEncodedWalAdditionWithLength( const std::string& encoded) { std::string ret; - PutVarint32(&ret, Tag::kNewWalAddition); + PutVarint32(&ret, Tag::kWalAddition2); PutLengthPrefixedSlice(&ret, encoded); return ret; }