rocksdb: add config for rate_limiter.refill_period (#8371)

Signed-off-by: tabokie <xy.tao@outlook.com>

### What is changed and how it works?

Expose rocksdb config `refill_period_us` for rate limiter to tikv.

[refill_period_us](https://github.com/facebook/rocksdb/blob/fdf882ded218344c136c97daf76dfb59e4bc155f/include/rocksdb/rate_limiter.h#L117) is used to control the largest IO quota that can be requested from IO rate limiter at a single time. Tuning down this parameter will flatten the IO burst with a penalty on CPU usage. Default value is 100 ms.

### Release note <!-- bugfixes or new feature need a release note -->

- Add config for rate limiter's refill-period

etc/config-template.toml

@@ -448,13 +448,16 @@
 ## 1. rate-bytes-per-sec is the only parameter you want to set most of the time. It controls the
 ##    total write rate of compaction and flush in bytes per second. Currently, RocksDB does not
 ##    enforce rate limit for anything other than flush and compaction, e.g. write to WAL.
-## 2. rate-limiter-mode indicates which types of operations count against the limit.
+## 2. rate-limiter-refill-period controls how often IO tokens are refilled. Smaller value will flatten
+##    IO bursts while introducing more CPU overhead.
+## 3. rate-limiter-mode indicates which types of operations count against the limit.
 ##    1 : ReadOnly
 ##    2 : WriteOnly
 ##    3 : AllIo
-## 3. auto_tuned enables dynamic adjustment of rate limit within the range
+## 4. auto_tuned enables dynamic adjustment of rate limit within the range
 ##    [rate_bytes_per_sec / 20, rate_bytes_per_sec], according to the recent demand for background I/O.
 # rate-bytes-per-sec = 0
+# rate-limiter-refill-period = "100ms"
 # rate-limiter-mode = 2
 # auto-tuned = false
 

src/config.rs

@@ -839,6 +839,8 @@ pub struct DbConfig {
     #[config(skip)]
     pub info_log_dir: String,
     pub rate_bytes_per_sec: ReadableSize,
+    #[config(skip)]
+    pub rate_limiter_refill_period: ReadableDuration,
     #[serde(with = "rocks_config::rate_limiter_mode_serde")]
     #[config(skip)]
     pub rate_limiter_mode: DBRateLimiterMode,
@@ -896,6 +898,7 @@ impl Default for DbConfig {
             info_log_dir: "".to_owned(),
             info_log_level: LogLevel::Info,
             rate_bytes_per_sec: ReadableSize::kb(0),
+            rate_limiter_refill_period: ReadableDuration::millis(100),
             rate_limiter_mode: DBRateLimiterMode::WriteOnly,
             auto_tuned: false,
             bytes_per_sync: ReadableSize::mb(1),
@@ -939,6 +942,7 @@ impl DbConfig {
         if self.rate_bytes_per_sec.0 > 0 {
             opts.set_ratelimiter_with_auto_tuned(
                 self.rate_bytes_per_sec.0 as i64,
+                (self.rate_limiter_refill_period.as_millis() * 1000) as i64,
                 self.rate_limiter_mode,
                 self.auto_tuned,
             );

tests/integrations/config/mod.rs

@@ -239,6 +239,7 @@ fn test_serde_custom_tikv_config() {
         info_log_dir: "/var".to_owned(),
         info_log_level: LogLevel::Info,
         rate_bytes_per_sec: ReadableSize::kb(1),
+        rate_limiter_refill_period: ReadableDuration::millis(10),
         rate_limiter_mode: DBRateLimiterMode::AllIo,
         auto_tuned: true,
         bytes_per_sync: ReadableSize::mb(1),

tests/integrations/config/test-custom.toml

@@ -183,6 +183,7 @@ info-log-roll-time = "12s"
 info-log-keep-log-file-num = 1000
 info-log-dir = "/var"
 rate-bytes-per-sec = "1KB"
+rate-limiter-refill-period = "10ms"
 rate-limiter-mode = 3
 auto-tuned = true
 bytes-per-sync = "1MB"