Exporting compaction stats in the form of a map

Summary: Currently the compaction stats are printed to stdout. We want to export the compaction stats in a map format so that the upper layer apps (e.g., MySQL) could present the stats in any format required by the them. Closes #1477 Differential Revision: D4149836 Pulled By: maysamyabandeh fbshipit-source-id: b3df19f
facebook · Nov 12, 2016 · 361010d · 361010d
1 parent 672300f
commit 361010d
Show file tree

Hide file tree

Showing 8 changed files with 422 additions and 137 deletions.
diff --git a/db/db_impl.cc b/db/db_impl.cc
@@ -5292,6 +5292,24 @@ bool DBImpl::GetProperty(ColumnFamilyHandle* column_family,
   return false;
 }
 
+bool DBImpl::GetMapProperty(ColumnFamilyHandle* column_family,
+                            const Slice& property,
+                            std::map<std::string, double>* value) {
+  const DBPropertyInfo* property_info = GetPropertyInfo(property);
+  value->clear();
+  auto cfd = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family)->cfd();
+  if (property_info == nullptr) {
+    return false;
+  } else if (property_info->handle_map) {
+    InstrumentedMutexLock l(&mutex_);
+    return cfd->internal_stats()->GetMapProperty(*property_info, property,
+                                                 value);
+  }
+  // If we reach this point it means that handle_map is not provided for the
+  // requested property
+  return false;
+}
+
 bool DBImpl::GetIntProperty(ColumnFamilyHandle* column_family,
                             const Slice& property, uint64_t* value) {
   const DBPropertyInfo* property_info = GetPropertyInfo(property);

diff --git a/db/db_impl.h b/db/db_impl.h
@@ -13,6 +13,7 @@
 #include <functional>
 #include <limits>
 #include <list>
+#include <map>
 #include <queue>
 #include <set>
 #include <string>
@@ -123,6 +124,10 @@ class DBImpl : public DB {
   using DB::GetProperty;
   virtual bool GetProperty(ColumnFamilyHandle* column_family,
                            const Slice& property, std::string* value) override;
+  using DB::GetMapProperty;
+  virtual bool GetMapProperty(ColumnFamilyHandle* column_family,
+                              const Slice& property,
+                              std::map<std::string, double>* value) override;
   using DB::GetIntProperty;
   virtual bool GetIntProperty(ColumnFamilyHandle* column_family,
                               const Slice& property, uint64_t* value) override;

diff --git a/db/db_test.cc b/db/db_test.cc
@@ -2740,6 +2740,12 @@ class ModelDB : public DB {
                               const Slice& property, uint64_t* value) override {
     return false;
   }
+  using DB::GetMapProperty;
+  virtual bool GetMapProperty(ColumnFamilyHandle* column_family,
+                              const Slice& property,
+                              std::map<std::string, double>* value) override {
+    return false;
+  }
   using DB::GetAggregatedIntProperty;
   virtual bool GetAggregatedIntProperty(const Slice& property,
                                         uint64_t* value) override {

diff --git a/db/db_test2.cc b/db/db_test2.cc
@@ -2085,8 +2085,26 @@ TEST_F(DBTest2, AutomaticCompactionOverlapManualCompaction) {
   cro.target_level = 2;
   ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr));
 
+  auto get_stat = [](std::string level_str, LevelStatType type,
+                     std::map<std::string, double> props) {
+    auto prop_str =
+        level_str + "." +
+        InternalStats::compaction_level_stats.at(type).property_name.c_str();
+    auto prop_item = props.find(prop_str);
+    return prop_item == props.end() ? 0 : prop_item->second;
+  };
+
   // Trivial move 2 files to L2
   ASSERT_EQ("0,0,2", FilesPerLevel());
+  // Also test that the stats GetMapProperty API reporting the same result
+  {
+    std::map<std::string, double> prop;
+    ASSERT_TRUE(dbfull()->GetMapProperty("rocksdb.cfstats", &prop));
+    ASSERT_EQ(0, get_stat("L0", LevelStatType::NUM_FILES, prop));
+    ASSERT_EQ(0, get_stat("L1", LevelStatType::NUM_FILES, prop));
+    ASSERT_EQ(2, get_stat("L2", LevelStatType::NUM_FILES, prop));
+    ASSERT_EQ(2, get_stat("Sum", LevelStatType::NUM_FILES, prop));
+  }
 
   // While the compaction is running, we will create 2 new files that
   // can fit in L2, these 2 files will be moved to L2 and overlap with
@@ -2113,6 +2131,13 @@ TEST_F(DBTest2, AutomaticCompactionOverlapManualCompaction) {
   ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr));
 
   rocksdb::SyncPoint::GetInstance()->DisableProcessing();
+
+  // Test that the stats GetMapProperty API reporting 1 file in L2
+  {
+    std::map<std::string, double> prop;
+    ASSERT_TRUE(dbfull()->GetMapProperty("rocksdb.cfstats", &prop));
+    ASSERT_EQ(1, get_stat("L2", LevelStatType::NUM_FILES, prop));
+  }
 }
 
 TEST_F(DBTest2, ManualCompactionOverlapManualCompaction) {

diff --git a/db/internal_stats.cc b/db/internal_stats.cc
diff --git a/db/internal_stats.h b/db/internal_stats.h
@@ -9,10 +9,11 @@
 //
 
 #pragma once
-#include "db/version_set.h"
-
-#include <vector>
+#include <map>
 #include <string>
+#include <vector>
+
+#include "db/version_set.h"
 
 class ColumnFamilyData;
 
@@ -41,13 +42,47 @@ struct DBPropertyInfo {
   //      holding db mutex, which is only supported for int properties.
   bool (InternalStats::*handle_int)(uint64_t* value, DBImpl* db,
                                     Version* version);
+  bool (InternalStats::*handle_map)(
+      std::map<std::string, double>* compaction_stats);
 };
 
 extern const DBPropertyInfo* GetPropertyInfo(const Slice& property);
 
 #ifndef ROCKSDB_LITE
+enum class LevelStatType {
+  INVALID = 0,
+  NUM_FILES,
+  COMPACTED_FILES,
+  SIZE_MB,
+  SCORE,
+  READ_GB,
+  RN_GB,
+  RNP1_GB,
+  WRITE_GB,
+  W_NEW_GB,
+  MOVED_GB,
+  WRITE_AMP,
+  READ_MBPS,
+  WRITE_MBPS,
+  COMP_SEC,
+  COMP_COUNT,
+  AVG_SEC,
+  KEY_IN,
+  KEY_DROP,
+  TOTAL  // total number of types
+};
+
+struct LevelStat {
+  // This what will be L?.property_name in the flat map returned to the user
+  std::string property_name;
+  // This will be what we will print in the header in the cli
+  std::string header_name;
+};
+
 class InternalStats {
  public:
+  static const std::map<LevelStatType, LevelStat> compaction_level_stats;
+
   enum InternalCFStatsType {
     LEVEL0_SLOWDOWN_TOTAL,
     LEVEL0_SLOWDOWN_WITH_COMPACTION,
@@ -221,6 +256,10 @@ class InternalStats {
   bool GetStringProperty(const DBPropertyInfo& property_info,
                          const Slice& property, std::string* value);
 
+  bool GetMapProperty(const DBPropertyInfo& property_info,
+                      const Slice& property,
+                      std::map<std::string, double>* value);
+
   bool GetIntProperty(const DBPropertyInfo& property_info, uint64_t* value,
                       DBImpl* db);
 
@@ -233,6 +272,10 @@ class InternalStats {
 
  private:
   void DumpDBStats(std::string* value);
+  void DumpCFMapStats(std::map<std::string, double>* cf_stats);
+  int DumpCFMapStats(
+      std::map<int, std::map<LevelStatType, double>>* level_stats,
+      CompactionStats* compaction_stats_sum);
   void DumpCFStats(std::string* value);
 
   // Per-DB stats
@@ -313,6 +356,7 @@ class InternalStats {
   bool HandleCompressionRatioAtLevelPrefix(std::string* value, Slice suffix);
   bool HandleLevelStats(std::string* value, Slice suffix);
   bool HandleStats(std::string* value, Slice suffix);
+  bool HandleCFMapStats(std::map<std::string, double>* compaction_stats);
   bool HandleCFStats(std::string* value, Slice suffix);
   bool HandleDBStats(std::string* value, Slice suffix);
   bool HandleSsTables(std::string* value, Slice suffix);
@@ -448,6 +492,12 @@ class InternalStats {
     return false;
   }
 
+  bool GetMapProperty(const DBPropertyInfo& property_info,
+                      const Slice& property,
+                      std::map<std::string, double>* value) {
+    return false;
+  }
+
   bool GetIntProperty(const DBPropertyInfo& property_info, uint64_t* value,
                       DBImpl* db) const {
     return false;

diff --git a/include/rocksdb/db.h b/include/rocksdb/db.h
@@ -11,6 +11,7 @@
 
 #include <stdint.h>
 #include <stdio.h>
+#include <map>
 #include <memory>
 #include <string>
 #include <unordered_map>
@@ -372,6 +373,10 @@ class DB {
     //      family stats per-level over db's lifetime ("L<n>"), aggregated over
     //      db's lifetime ("Sum"), and aggregated over the interval since the
     //      last retrieval ("Int").
+    //  It could also be used to return the stats in the format of the map.
+    //  In this case there will a pair of string to array of double for
+    //  each level as well as for "Sum". "Int" stats will not be affected
+    //  when this form of stats are retrived.
     static const std::string kCFStats;
 
     //  "rocksdb.dbstats" - returns a multi-line string with general database
@@ -511,6 +516,13 @@ class DB {
   virtual bool GetProperty(const Slice& property, std::string* value) {
     return GetProperty(DefaultColumnFamily(), property, value);
   }
+  virtual bool GetMapProperty(ColumnFamilyHandle* column_family,
+                              const Slice& property,
+                              std::map<std::string, double>* value) = 0;
+  virtual bool GetMapProperty(const Slice& property,
+                              std::map<std::string, double>* value) {
+    return GetMapProperty(DefaultColumnFamily(), property, value);
+  }
 
   // Similar to GetProperty(), but only works for a subset of properties whose
   // return value is an integer. Return the value by integer. Supported

diff --git a/include/rocksdb/utilities/stackable_db.h b/include/rocksdb/utilities/stackable_db.h
@@ -3,6 +3,7 @@
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
 
 #pragma once
+#include <map>
 #include <string>
 #include "rocksdb/db.h"
 
@@ -133,11 +134,17 @@ class StackableDB : public DB {
     return db_->ReleaseSnapshot(snapshot);
   }
 
+  using DB::GetMapProperty;
   using DB::GetProperty;
   virtual bool GetProperty(ColumnFamilyHandle* column_family,
                            const Slice& property, std::string* value) override {
     return db_->GetProperty(column_family, property, value);
   }
+  virtual bool GetMapProperty(ColumnFamilyHandle* column_family,
+                              const Slice& property,
+                              std::map<std::string, double>* value) override {
+    return db_->GetMapProperty(column_family, property, value);
+  }
 
   using DB::GetIntProperty;
   virtual bool GetIntProperty(ColumnFamilyHandle* column_family,