Upgrade RocksDB to 6.4 (#61)

Summary: Update the code to base off RocksDB 6.4.x. There are a few internal interface changes and files moving around from 5.18 to 6.4. titandb_bench is also updated to base off the version in 6.4.x. Test Plan: Travis
tikv · Aug 27, 2019 · c99cf9d · c99cf9d
1 parent 19ea115
commit c99cf9d
Show file tree

Hide file tree

Showing 36 changed files with 880 additions and 219 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -9,7 +9,7 @@ if (NOT ROCKSDB_GIT_REPO)
 endif()
 
 if (NOT ROCKSDB_GIT_BRANCH)
-  set(ROCKSDB_GIT_BRANCH "tikv-3.0")
+  set(ROCKSDB_GIT_BRANCH "6.4.tikv")
 endif()
 
 if (NOT DEFINED ROCKSDB_DIR)

diff --git a/README.md b/README.md
@@ -42,5 +42,5 @@ ctest -R titan
 bash scripts/format-diff.sh
 ```
 
-## Compatibility
-Currently Titan is only compatible with RocksDB 5.18.
+## Compatibility with RocksDB
+Current version of Titan is developed and tested with RocksDB 6.4.
diff --git a/cmake/rocksdb_flags.cmake b/cmake/rocksdb_flags.cmake
@@ -32,7 +32,7 @@ endif()
 
 option(WITH_ZLIB "build with zlib" OFF)
 if (WITH_ZLIB)
-  find_package(zlib REQUIRED)
+  find_package(ZLIB REQUIRED)
   add_definitions(-DZLIB)
   include_directories(${ZLIB_INCLUDE_DIR})
 endif()

diff --git a/include/titan/options.h b/include/titan/options.h
@@ -3,8 +3,8 @@
 #include <map>
 #include <unordered_map>
 
+#include "logging/logging.h"
 #include "rocksdb/options.h"
-#include "util/logging.h"
 
 namespace rocksdb {
 namespace titandb {

diff --git a/src/blob_file_cache.cc b/src/blob_file_cache.cc
@@ -1,7 +1,7 @@
 #include "blob_file_cache.h"
 
+#include "file/filename.h"
 #include "util.h"
-#include "util/filename.h"
 
 namespace rocksdb {
 namespace titandb {

diff --git a/src/blob_file_iterator.cc b/src/blob_file_iterator.cc
@@ -19,7 +19,10 @@ BlobFileIterator::~BlobFileIterator() {}
 bool BlobFileIterator::Init() {
   Slice slice;
   char header_buf[BlobFileHeader::kEncodedLength];
-  status_ = file_->Read(0, BlobFileHeader::kEncodedLength, &slice, header_buf);
+  // With for_compaction=true, rate_limiter is enabled. Since BlobFileIterator
+  // is only used for GC, we always set for_compaction to true.
+  status_ = file_->Read(0, BlobFileHeader::kEncodedLength, &slice, header_buf,
+                        true /*for_compaction*/);
   if (!status_.ok()) {
     return false;
   }
@@ -29,8 +32,11 @@ bool BlobFileIterator::Init() {
     return false;
   }
   char footer_buf[BlobFileFooter::kEncodedLength];
+  // With for_compaction=true, rate_limiter is enabled. Since BlobFileIterator
+  // is only used for GC, we always set for_compaction to true.
   status_ = file_->Read(file_size_ - BlobFileFooter::kEncodedLength,
-                        BlobFileFooter::kEncodedLength, &slice, footer_buf);
+                        BlobFileFooter::kEncodedLength, &slice, footer_buf,
+                        true /*for_compaction*/);
   if (!status_.ok()) return false;
   BlobFileFooter blob_file_footer;
   status_ = blob_file_footer.DecodeFrom(&slice);
@@ -74,8 +80,10 @@ void BlobFileIterator::IterateForPrev(uint64_t offset) {
   FixedSlice<kBlobHeaderSize> header_buffer;
   iterate_offset_ = BlobFileHeader::kEncodedLength;
   for (; iterate_offset_ < offset; iterate_offset_ += total_length) {
+    // With for_compaction=true, rate_limiter is enabled. Since BlobFileIterator
+    // is only used for GC, we always set for_compaction to true.
     status_ = file_->Read(iterate_offset_, kBlobHeaderSize, &header_buffer,
-                          header_buffer.get());
+                          header_buffer.get(), true /*for_compaction*/);
     if (!status_.ok()) return;
     status_ = decoder_.DecodeHeader(&header_buffer);
     if (!status_.ok()) return;
@@ -88,17 +96,21 @@ void BlobFileIterator::IterateForPrev(uint64_t offset) {
 
 void BlobFileIterator::GetBlobRecord() {
   FixedSlice<kBlobHeaderSize> header_buffer;
+  // With for_compaction=true, rate_limiter is enabled. Since BlobFileIterator
+  // is only used for GC, we always set for_compaction to true.
   status_ = file_->Read(iterate_offset_, kBlobHeaderSize, &header_buffer,
-                        header_buffer.get());
+                        header_buffer.get(), true /*for_compaction*/);
   if (!status_.ok()) return;
   status_ = decoder_.DecodeHeader(&header_buffer);
   if (!status_.ok()) return;
 
   Slice record_slice;
   auto record_size = decoder_.GetRecordSize();
   buffer_.resize(record_size);
+  // With for_compaction=true, rate_limiter is enabled. Since BlobFileIterator
+  // is only used for GC, we always set for_compaction to true.
   status_ = file_->Read(iterate_offset_ + kBlobHeaderSize, record_size,
-                        &record_slice, buffer_.data());
+                        &record_slice, buffer_.data(), true /*for_compaction*/);
   if (status_.ok()) {
     status_ =
         decoder_.DecodeRecord(&record_slice, &cur_blob_record_, &uncompressed_);

diff --git a/src/blob_file_iterator.h b/src/blob_file_iterator.h
@@ -14,6 +14,7 @@
 namespace rocksdb {
 namespace titandb {
 
+// Used by GC job for iterate through blob file.
 class BlobFileIterator {
  public:
   const uint64_t kMinReadaheadSize = 4 << 10;

diff --git a/src/blob_file_iterator_test.cc b/src/blob_file_iterator_test.cc
@@ -2,11 +2,12 @@
 
 #include <cinttypes>
 
+#include "file/filename.h"
+#include "test_util/testharness.h"
+
 #include "blob_file_builder.h"
 #include "blob_file_cache.h"
 #include "blob_file_reader.h"
-#include "util/filename.h"
-#include "util/testharness.h"
 
 namespace rocksdb {
 namespace titandb {

diff --git a/src/blob_file_reader.cc b/src/blob_file_reader.cc
@@ -6,10 +6,10 @@
 
 #include <inttypes.h>
 
+#include "file/filename.h"
+#include "test_util/sync_point.h"
 #include "util/crc32c.h"
-#include "util/filename.h"
 #include "util/string_util.h"
-#include "util/sync_point.h"
 
 #include "titan_stats.h"
 
@@ -28,12 +28,9 @@ Status NewBlobFileReader(uint64_t file_number, uint64_t readahead_size,
   if (readahead_size > 0) {
     file = NewReadaheadRandomAccessFile(std::move(file), readahead_size);
   }
-  // Currently only `BlobGCJob` will call `NewBlobFileReader()`. We set
-  // `for_compaction=true` in this case to enable rate limiter.
   result->reset(new RandomAccessFileReader(
       std::move(file), file_name, nullptr /*env*/, nullptr /*stats*/,
-      0 /*hist_type*/, nullptr /*file_read_hist*/, env_options.rate_limiter,
-      true /*for compaction*/));
+      0 /*hist_type*/, nullptr /*file_read_hist*/, env_options.rate_limiter));
   return s;
 }
 

diff --git a/src/blob_file_size_collector_test.cc b/src/blob_file_size_collector_test.cc
@@ -1,5 +1,6 @@
+#include "test_util/testharness.h"
+
 #include "blob_file_size_collector.h"
-#include "util/testharness.h"
 
 namespace rocksdb {
 namespace titandb {
@@ -50,8 +51,9 @@ class BlobFileSizeCollectorTest : public testing::Test {
     CompressionOptions compression_opts;
     TableBuilderOptions options(cf_ioptions_, cf_moptions_,
                                 cf_ioptions_.internal_comparator, &collectors_,
-                                kNoCompression, compression_opts, nullptr,
-                                false, kDefaultColumnFamilyName, 0);
+                                kNoCompression, 0 /*sample_for_compression*/,
+                                compression_opts, false /*skip_filters*/,
+                                kDefaultColumnFamilyName, 0 /*level*/);
     result->reset(table_factory_->NewTableBuilder(options, 0, file));
     ASSERT_TRUE(*result);
   }

diff --git a/src/blob_file_test.cc b/src/blob_file_test.cc
@@ -1,8 +1,9 @@
+#include "file/filename.h"
+#include "test_util/testharness.h"
+
 #include "blob_file_builder.h"
 #include "blob_file_cache.h"
 #include "blob_file_reader.h"
-#include "util/filename.h"
-#include "util/testharness.h"
 
 namespace rocksdb {
 namespace titandb {

diff --git a/src/blob_format.cc b/src/blob_format.cc
@@ -1,7 +1,7 @@
 #include "blob_format.h"
 
+#include "test_util/sync_point.h"
 #include "util/crc32c.h"
-#include "util/sync_point.h"
 
 namespace rocksdb {
 namespace titandb {
@@ -40,7 +40,7 @@ void BlobEncoder::EncodeRecord(const BlobRecord& record) {
 
   CompressionType compression;
   record.EncodeTo(&record_buffer_);
-  record_ = Compress(compression_ctx_, record_buffer_, &compressed_buffer_,
+  record_ = Compress(compression_info_, record_buffer_, &compressed_buffer_,
                      &compression);
 
   assert(record_.size() < std::numeric_limits<uint32_t>::max());
@@ -82,7 +82,8 @@ Status BlobDecoder::DecodeRecord(Slice* src, BlobRecord* record,
     return DecodeInto(input, record);
   }
   UncompressionContext ctx(compression_);
-  Status s = Uncompress(ctx, input, buffer);
+  UncompressionInfo info(ctx, UncompressionDict::GetEmptyDict(), compression_);
+  Status s = Uncompress(info, input, buffer);
   if (!s.ok()) {
     return s;
   }

diff --git a/src/blob_format.h b/src/blob_format.h
@@ -32,7 +32,11 @@ struct BlobRecord {
 
 class BlobEncoder {
  public:
-  BlobEncoder(CompressionType compression) : compression_ctx_(compression) {}
+  BlobEncoder(CompressionType compression)
+      : compression_ctx_(compression),
+        compression_info_(compression_opt_, compression_ctx_,
+                          CompressionDict::GetEmptyDict(), compression,
+                          0 /*sample_for_compression*/) {}
 
   void EncodeRecord(const BlobRecord& record);
 
@@ -46,7 +50,9 @@ class BlobEncoder {
   Slice record_;
   std::string record_buffer_;
   std::string compressed_buffer_;
+  CompressionOptions compression_opt_;
   CompressionContext compression_ctx_;
+  CompressionInfo compression_info_;
 };
 
 class BlobDecoder {

diff --git a/src/blob_format_test.cc b/src/blob_format_test.cc
@@ -1,7 +1,8 @@
+#include "test_util/testharness.h"
+
 #include "blob_format.h"
 #include "testutil.h"
 #include "util.h"
-#include "util/testharness.h"
 
 namespace rocksdb {
 namespace titandb {

diff --git a/src/blob_gc_job.cc b/src/blob_gc_job.cc
@@ -3,6 +3,8 @@
 #endif
 #include <inttypes.h>
 
+#include <memory>
+
 #include "blob_gc_job.h"
 
 namespace rocksdb {
@@ -491,7 +493,7 @@ Status BlobGCJob::InstallOutputBlobFiles() {
       }
     }
   } else {
-    std::vector<unique_ptr<BlobFileHandle>> handles;
+    std::vector<std::unique_ptr<BlobFileHandle>> handles;
     std::string to_delete_files;
     for (auto& builder : this->blob_file_builders_) {
       if (!to_delete_files.empty()) {
@@ -565,7 +567,8 @@ Status BlobGCJob::DeleteInputBlobFiles() {
   VersionEdit edit;
   edit.SetColumnFamilyID(blob_gc_->column_family_handle()->GetID());
   for (const auto& file : blob_gc_->sampled_inputs()) {
-    ROCKS_LOG_INFO(db_options_.info_log, "Titan add obsolete file [%llu]",
+    ROCKS_LOG_INFO(db_options_.info_log,
+                   "Titan add obsolete file [%" PRIu64 "]",
                    file->file_number());
     metrics_.blob_db_gc_num_files++;
     edit.DeleteBlobFile(file->file_number(), obsolete_sequence);

diff --git a/src/blob_gc_job.h b/src/blob_gc_job.h
@@ -4,7 +4,7 @@
 #include "blob_file_iterator.h"
 #include "blob_file_manager.h"
 #include "blob_gc.h"
-#include "db/db_impl.h"
+#include "db/db_impl/db_impl.h"
 #include "rocksdb/statistics.h"
 #include "rocksdb/status.h"
 #include "titan/options.h"

diff --git a/src/blob_gc_job_test.cc b/src/blob_gc_job_test.cc
@@ -1,9 +1,9 @@
-#include "blob_gc_job.h"
+#include "rocksdb/convenience.h"
+#include "test_util/testharness.h"
 
+#include "blob_gc_job.h"
 #include "blob_gc_picker.h"
 #include "db_impl.h"
-#include "rocksdb/convenience.h"
-#include "util/testharness.h"
 
 namespace rocksdb {
 namespace titandb {

diff --git a/src/blob_gc_picker.cc b/src/blob_gc_picker.cc
@@ -1,5 +1,11 @@
 #include "blob_gc_picker.h"
 
+#ifndef __STDC_FORMAT_MACROS
+#define __STDC_FORMAT_MACROS
+#endif
+
+#include <inttypes.h>
+
 namespace rocksdb {
 namespace titandb {
 
@@ -16,9 +22,6 @@ std::unique_ptr<BlobGC> BasicBlobGCPicker::PickBlobGC(
 
   uint64_t batch_size = 0;
   uint64_t estimate_output_size = 0;
-  //  ROCKS_LOG_INFO(db_options_.info_log, "blob file num:%lu gc score:%lu",
-  //                 blob_storage->NumBlobFiles(),
-  //                 blob_storage->gc_score().size());
   bool stop_picking = false;
   bool maybe_continue_next_time = false;
   uint64_t next_gc_size = 0;
@@ -30,23 +33,11 @@ std::unique_ptr<BlobGC> BasicBlobGCPicker::PickBlobGC(
       // or this file had been GCed
       continue;
     }
-
-    //    ROCKS_LOG_INFO(db_options_.info_log,
-    //                   "file number:%lu score:%f being_gc:%d pending:%d, "
-    //                   "size:%lu discard:%lu mark_for_gc:%d
-    //                   mark_for_sample:%d", blob_file->file_number_,
-    //                   gc_score.score, blob_file->being_gc,
-    //                   blob_file->pending, blob_file->file_size_,
-    //                   blob_file->discardable_size_,
-    //                   blob_file->marked_for_gc_,
-    //                   blob_file->marked_for_sample);
-
     if (!CheckBlobFile(blob_file.get())) {
-      ROCKS_LOG_INFO(db_options_.info_log, "file number:%lu no need gc",
+      ROCKS_LOG_INFO(db_options_.info_log, "Blob file %" PRIu64 "  no need gc",
                      blob_file->file_number());
       continue;
     }
-
     if (!stop_picking) {
       blob_files.push_back(blob_file.get());
       batch_size += blob_file->file_size();

diff --git a/src/blob_gc_picker.h b/src/blob_gc_picker.h
@@ -2,14 +2,15 @@
 
 #include <memory>
 
+#include "db/column_family.h"
+#include "db/write_callback.h"
+#include "file/filename.h"
+#include "rocksdb/status.h"
+
 #include "blob_file_manager.h"
 #include "blob_format.h"
 #include "blob_gc.h"
 #include "blob_storage.h"
-#include "db/column_family.h"
-#include "db/write_callback.h"
-#include "rocksdb/status.h"
-#include "util/filename.h"
 
 namespace rocksdb {
 namespace titandb {

diff --git a/src/blob_gc_picker_test.cc b/src/blob_gc_picker_test.cc
@@ -1,11 +1,12 @@
 #include "blob_gc_picker.h"
 
+#include "file/filename.h"
+#include "test_util/testharness.h"
+
 #include "blob_file_builder.h"
 #include "blob_file_cache.h"
 #include "blob_file_iterator.h"
 #include "blob_file_reader.h"
-#include "util/filename.h"
-#include "util/testharness.h"
 
 namespace rocksdb {
 namespace titandb {