feat: Add a feature which support partitioned index filter (OpenAtomF…

…oundation#2601)

* add a feature which support partitioned index filter

* Name of variable change to enable-partitioned-index-filters from enable_partitioned_index_filters

---------

Co-authored-by: Vachel <vachel@example.com>

conf/pika.conf

@@ -386,6 +386,12 @@ thread-migrate-keys-num : 64
 # The slot number of pika when used with codis.
 default-slot-num : 1024
 
+# enable-partitioned-index-filters [yes | no]
+# When `cache-index-and-filter-blocks` is enabled, `pin_l0_filter_and_index_blocks_in_cache` 
+# and `cache-index-and-filter-blocks` is suggested to be enabled
+# https://github.com/facebook/rocksdb/wiki/Partitioned-Index-Filters
+# enable-partitioned-index-filters: default no
+
 # whether or not index and filter blocks is stored in block cache
 # cache-index-and-filter-blocks: no
 

include/pika_conf.h

@@ -299,6 +299,10 @@ class PikaConf : public pstd::BaseConf {
     std::shared_lock l(rwlock_);
     return share_block_cache_;
   }
+  bool enable_partitioned_index_filters() {
+    std::shared_lock l(rwlock_);
+    return enable_partitioned_index_filters_;
+  }
   bool cache_index_and_filter_blocks() {
     std::shared_lock l(rwlock_);
     return cache_index_and_filter_blocks_;
@@ -866,6 +870,7 @@ class PikaConf : public pstd::BaseConf {
   int64_t block_cache_ = 0;
   int64_t num_shard_bits_ = 0;
   bool share_block_cache_ = false;
+  bool enable_partitioned_index_filters_ = false;
   bool cache_index_and_filter_blocks_ = false;
   bool pin_l0_filter_and_index_blocks_in_cache_ = false;
   bool optimize_filters_for_hits_ = false;

src/pika_admin.cc

@@ -1721,6 +1721,12 @@ void ConfigCmd::ConfigGet(std::string& ret) {
     EncodeString(&config_body, g_pika_conf->share_block_cache() ? "yes" : "no");
   }
 
+  if (pstd::stringmatch(pattern.data(), "enable-partitioned-index-filters", 1) != 0) {
+    elements += 2;
+    EncodeString(&config_body, "enable-partitioned-index-filters");
+    EncodeString(&config_body, g_pika_conf->enable_partitioned_index_filters() ? "yes" : "no");
+  }
+
   if (pstd::stringmatch(pattern.data(), "cache-index-and-filter-blocks", 1) != 0) {
     elements += 2;
     EncodeString(&config_body, "cache-index-and-filter-blocks");

src/pika_conf.cc

@@ -455,6 +455,10 @@ int PikaConf::Load() {
   GetConfStr("share-block-cache", &sbc);
   share_block_cache_ = sbc == "yes";
 
+  std::string epif;
+  GetConfStr("enable-partitioned-index-filters", &epif);
+  enable_partitioned_index_filters_ = epif == "yes";
+
   std::string ciafb;
   GetConfStr("cache-index-and-filter-blocks", &ciafb);
   cache_index_and_filter_blocks_ = ciafb == "yes";

src/pika_server.cc

@@ -1398,6 +1398,17 @@ void PikaServer::InitStorageOptions() {
   // for column-family options
   storage_options_.options.ttl = g_pika_conf->rocksdb_ttl_second();
   storage_options_.options.periodic_compaction_seconds = g_pika_conf->rocksdb_periodic_compaction_second();
+
+  // For Partitioned Index Filters
+  if (g_pika_conf->enable_partitioned_index_filters()) {
+    storage_options_.table_options.index_type = rocksdb::BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch;
+    storage_options_.table_options.filter_policy.reset(rocksdb::NewBloomFilterPolicy(10, false));
+    storage_options_.table_options.partition_filters = true;
+    storage_options_.table_options.metadata_block_size = 4096;
+    storage_options_.table_options.cache_index_and_filter_blocks_with_high_priority = true;
+    storage_options_.table_options.pin_top_level_index_and_filter = true; 
+    storage_options_.table_options.optimize_filters_for_memory = true;
+  }
 }
 
 storage::Status PikaServer::RewriteStorageOptions(const storage::OptionType& option_type,