Merge pull request #15786 from markhpc/wip-bluestore-cache-behavior

os/bluestore: cap rocksdb cache size

Reviewed-by: Sage Weil <sage@redhat.com>
Reviewed-by: Igor Fedotov <ifedotov@mirantis.com>

src/common/config_opts.h

@@ -970,8 +970,8 @@ OPTION(kinetic_use_ssl, OPT_BOOL, false) // whether to secure kinetic traffic wi
 OPTION(rocksdb_separate_wal_dir, OPT_BOOL, false) // use $path.wal for wal
 SAFE_OPTION(rocksdb_db_paths, OPT_STR, "")   // path,size( path,size)*
 OPTION(rocksdb_log_to_ceph_log, OPT_BOOL, true)  // log to ceph log
-OPTION(rocksdb_cache_size, OPT_U64, 128*1024*1024)  // default rocksdb cache size
-OPTION(rocksdb_cache_row_ratio, OPT_FLOAT, .2)   // ratio of cache for row (vs block)
+OPTION(rocksdb_cache_size, OPT_U64, 128*1024*1024)  // rocksdb cache size (unless set by bluestore/etc)
+OPTION(rocksdb_cache_row_ratio, OPT_FLOAT, 0)   // ratio of cache for row (vs block)
 OPTION(rocksdb_cache_shard_bits, OPT_INT, 4)  // rocksdb block cache shard bits, 4 bit -> 16 shards
 OPTION(rocksdb_cache_type, OPT_STR, "lru") // 'lru' or 'clock'
 OPTION(rocksdb_block_size, OPT_INT, 4*1024)  // default rocksdb block size
@@ -1164,8 +1164,9 @@ OPTION(bluestore_cache_type, OPT_STR, "2q")   // lru, 2q
 OPTION(bluestore_2q_cache_kin_ratio, OPT_DOUBLE, .5)    // kin page slot size / max page slot size
 OPTION(bluestore_2q_cache_kout_ratio, OPT_DOUBLE, .5)   // number of kout page slot / total number of page slot
 OPTION(bluestore_cache_size, OPT_U64, 3*1024*1024*1024)
-OPTION(bluestore_cache_meta_ratio, OPT_DOUBLE, .7)
-OPTION(bluestore_cache_kv_ratio, OPT_DOUBLE, .2)
+OPTION(bluestore_cache_meta_ratio, OPT_DOUBLE, .01)
+OPTION(bluestore_cache_kv_ratio, OPT_DOUBLE, .99)
+OPTION(bluestore_cache_kv_max, OPT_U64, 512*1024*1024) // limit the maximum amont of cache for the kv store
 OPTION(bluestore_kvbackend, OPT_STR, "rocksdb")
 OPTION(bluestore_allocator, OPT_STR, "bitmap")     // stupid | bitmap
 OPTION(bluestore_freelist_blocks_per_key, OPT_INT, 128)

src/kv/RocksDBStore.cc

@@ -298,27 +298,36 @@ int RocksDBStore::do_open(ostream &out, bool create_if_missing)
   }
 
   // caches
-  if (!cache_size) {
+  if (!set_cache_flag) {
     cache_size = g_conf->rocksdb_cache_size;
   }
   uint64_t row_cache_size = cache_size * g_conf->rocksdb_cache_row_ratio;
   uint64_t block_cache_size = cache_size - row_cache_size;
-  if (g_conf->rocksdb_cache_type == "lru") {
-    bbt_opts.block_cache = rocksdb::NewLRUCache(
-      block_cache_size,
-      g_conf->rocksdb_cache_shard_bits);
-  } else if (g_conf->rocksdb_cache_type == "clock") {
-    bbt_opts.block_cache = rocksdb::NewClockCache(
-      block_cache_size,
-      g_conf->rocksdb_cache_shard_bits);
+
+  if (block_cache_size == 0) {
+    // disable block cache
+    dout(10) << __func__ << " block_cache_size " << block_cache_size
+             << ", setting no_block_cache " << dendl;
+    bbt_opts.no_block_cache = true;
   } else {
-    derr << "unrecognized rocksdb_cache_type '" << g_conf->rocksdb_cache_type
-	 << "'" << dendl;
-    return -EINVAL;
+    if (g_conf->rocksdb_cache_type == "lru") {
+      bbt_opts.block_cache = rocksdb::NewLRUCache(
+        block_cache_size,
+        g_conf->rocksdb_cache_shard_bits);
+    } else if (g_conf->rocksdb_cache_type == "clock") {
+      bbt_opts.block_cache = rocksdb::NewClockCache(
+        block_cache_size,
+        g_conf->rocksdb_cache_shard_bits);
+    } else {
+      derr << "unrecognized rocksdb_cache_type '" << g_conf->rocksdb_cache_type
+  	 << "'" << dendl;
+      return -EINVAL;
+    }
   }
   bbt_opts.block_size = g_conf->rocksdb_block_size;
 
-  opt.row_cache = rocksdb::NewLRUCache(row_cache_size,
+  if (row_cache_size > 0)
+    opt.row_cache = rocksdb::NewLRUCache(row_cache_size,
 				       g_conf->rocksdb_cache_shard_bits);
 
   if (g_conf->kstore_rocksdb_bloom_bits_per_key > 0) {

src/kv/RocksDBStore.h

@@ -76,6 +76,7 @@ class RocksDBStore : public KeyValueDB {
   string options_str;
 
   uint64_t cache_size = 0;
+  bool set_cache_flag = false;
 
   int do_open(ostream &out, bool create_if_missing);
 
@@ -439,6 +440,7 @@ class RocksDBStore : public KeyValueDB {
 
   int set_cache_size(uint64_t s) override {
     cache_size = s;
+    set_cache_flag = true;
     return 0;
   }
 

src/os/bluestore/BlueStore.cc

@@ -3691,17 +3691,34 @@ int BlueStore::_set_cache_sizes()
 {
   cache_meta_ratio = cct->_conf->bluestore_cache_meta_ratio;
   cache_kv_ratio = cct->_conf->bluestore_cache_kv_ratio;
+
+  double cache_size = cct->_conf->bluestore_cache_size;
+  double cache_kv_max = cct->_conf->bluestore_cache_kv_max;
+  double cache_kv_max_ratio = 0;
+
+  // if cache_kv_max is negative, disable it
+  if (cache_size > 0 && cache_kv_max >= 0) {
+    cache_kv_max_ratio = (double) cache_kv_max / (double) cache_size;
+    if (cache_kv_max_ratio < 1.0 && cache_kv_max_ratio < cache_kv_ratio) {
+      dout(1) << __func__ << " max " << cache_kv_max_ratio
+            << " < ratio " << cache_kv_ratio
+            << dendl;
+      cache_meta_ratio = cache_meta_ratio + cache_kv_ratio - cache_kv_max_ratio;
+      cache_kv_ratio = cache_kv_max_ratio;
+    }
+  }  
+
   cache_data_ratio =
     (double)1.0 - (double)cache_meta_ratio - (double)cache_kv_ratio;
 
-  if (cache_meta_ratio <= 0 || cache_meta_ratio > 1.0) {
+  if (cache_meta_ratio < 0 || cache_meta_ratio > 1.0) {
     derr << __func__ << "bluestore_cache_meta_ratio (" << cache_meta_ratio
-	 << ") must be in range (0,1.0]" << dendl;
+	 << ") must be in range [0,1.0]" << dendl;
     return -EINVAL;
   }
-  if (cache_kv_ratio <= 0 || cache_kv_ratio > 1.0) {
+  if (cache_kv_ratio < 0 || cache_kv_ratio > 1.0) {
     derr << __func__ << "bluestore_cache_kv_ratio (" << cache_kv_ratio
-	 << ") must be in range (0,1.0]" << dendl;
+	 << ") must be in range [0,1.0]" << dendl;
     return -EINVAL;
   }
   if (cache_meta_ratio + cache_kv_ratio > 1.0) {