engine: support setting memory limit for raft engine (tikv#12458)

close tikv#12255

Support setting memory limit for raft engine

Signed-off-by: tabokie <xy.tao@outlook.com>
Signed-off-by: 3AceShowHand <jinl1037@hotmail.com>

components/raft_log_engine/Cargo.toml

@@ -14,7 +14,7 @@ num_cpus = "1"
 online_config = { path = "../online_config" }
 protobuf = "2"
 raft = { version = "0.7.0", default-features = false, features = ["protobuf-codec"] }
-raft-engine = { git = "https://github.com/tikv/raft-engine.git" }
+raft-engine = { git = "https://github.com/tikv/raft-engine.git", features = ["swap"] }
 serde = "1.0"
 serde_derive = "1.0"
 slog = { version = "2.3", features = ["max_level_trace", "release_max_level_debug"] }

components/raft_log_engine/src/engine.rs

@@ -19,7 +19,7 @@ use raft_engine::{
     env::{DefaultFileSystem, FileSystem, Handle, WriteExt},
     Command, Engine as RawRaftEngine, Error as RaftEngineError, LogBatch, MessageExt,
 };
-pub use raft_engine::{Config as RaftEngineConfig, RecoveryMode};
+pub use raft_engine::{Config as RaftEngineConfig, ReadableSize, RecoveryMode};
 use tikv_util::Either;
 
 #[derive(Clone)]

components/raft_log_engine/src/lib.rs

@@ -21,4 +21,4 @@
 extern crate tikv_util;
 
 mod engine;
-pub use engine::{RaftEngineConfig, RaftLogBatch, RaftLogEngine, RecoveryMode};
+pub use engine::{RaftEngineConfig, RaftLogBatch, RaftLogEngine, ReadableSize, RecoveryMode};

etc/config-template.toml

@@ -989,6 +989,10 @@
 ## Default: 4. Minimum: 1.
 # recovery-threads = 4
 
+## Memory usage limit for Raft Engine.
+## When it's not set, 15% of available system memory will be used.
+# memory-limit = "1GB"
+
 [security]
 ## The path for TLS certificates. Empty string means disabling secure connections.
 # ca-path = ""

metrics/grafana/tikv_details.json

@@ -3985,6 +3985,13 @@
               "intervalFactor": 1,
               "legendFormat": "{{instance}}-{{name}}",
               "refId": "A"
+            },
+            {
+              "expr": "raft_engine_memory_usage{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "{{instance}}-raft-engine",
+              "refId": "B"
             }
           ],
           "thresholds": [],
@@ -29713,10 +29720,16 @@
             {
               "exemplar": true,
               "expr": "avg(raft_engine_log_file_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (type)",
-              "interval": "",
+              "intervalFactor": 1,
               "legendFormat": "{{type}}",
-              "queryType": "randomWalk",
               "refId": "A"
+            },
+            {
+              "exemplar": true,
+              "expr": "avg(raft_engine_swap_file_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"})",
+              "intervalFactor": 1,
+              "legendFormat": "swap",
+              "refId": "B"
             }
           ],
           "thresholds": [],

src/config.rs

@@ -43,7 +43,9 @@ use keys::region_raft_prefix_len;
 use kvproto::kvrpcpb::ApiVersion;
 use online_config::{ConfigChange, ConfigManager, ConfigValue, OnlineConfig, Result as CfgResult};
 use pd_client::Config as PdConfig;
-use raft_log_engine::RaftEngineConfig as RawRaftEngineConfig;
+use raft_log_engine::{
+    RaftEngineConfig as RawRaftEngineConfig, ReadableSize as RaftEngineReadableSize,
+};
 use raftstore::{
     coprocessor::{Config as CopConfig, RegionInfoAccessor},
     store::{CompactionGuardGeneratorFactory, Config as RaftstoreConfig, SplitConfig},
@@ -77,6 +79,9 @@ pub const DEFAULT_ROCKSDB_SUB_DIR: &str = "db";
 pub const BLOCK_CACHE_RATE: f64 = 0.45;
 /// By default, TiKV will try to limit memory usage to 75% of system memory.
 pub const MEMORY_USAGE_LIMIT_RATE: f64 = 0.75;
+/// Maximum of 15% of system memory can be used by Raft Engine. Normally its
+/// memory usage is much smaller than that.
+const RAFT_ENGINE_MEMORY_LIMIT_RATE: f64 = 0.15;
 
 const LOCKCF_MIN_MEM: usize = 256 * MIB as usize;
 const LOCKCF_MAX_MEM: usize = GIB as usize;
@@ -1415,6 +1420,11 @@ impl Default for RaftEngineConfig {
 impl RaftEngineConfig {
     fn validate(&mut self) -> Result<(), Box<dyn Error>> {
         self.config.sanitize().map_err(Box::new)?;
+        if self.config.memory_limit.is_none() {
+            let total_mem = SysQuota::memory_limit_in_bytes() as f64;
+            let memory_limit = total_mem * RAFT_ENGINE_MEMORY_LIMIT_RATE;
+            self.config.memory_limit = Some(RaftEngineReadableSize(memory_limit as u64));
+        }
         Ok(())
     }
 
@@ -4834,6 +4844,7 @@ mod tests {
         cfg.pd.retry_max_count = default_cfg.pd.retry_max_count; // Both -1 and isize::MAX are the same.
         cfg.storage.block_cache.capacity = None; // Either `None` and a value is computed or `Some(_)` fixed value.
         cfg.memory_usage_limit = None;
+        cfg.raft_engine.mut_config().memory_limit = None;
         cfg.coprocessor_v2.coprocessor_plugin_directory = None; // Default is `None`, which is represented by not setting the key.
 
         assert_eq!(cfg, default_cfg);

tests/integrations/config/mod.rs

@@ -16,7 +16,7 @@ use engine_traits::PerfLevel;
 use file_system::{IOPriority, IORateLimitMode};
 use kvproto::encryptionpb::EncryptionMethod;
 use pd_client::Config as PdConfig;
-use raft_log_engine::RecoveryMode;
+use raft_log_engine::{ReadableSize as RaftEngineReadableSize, RecoveryMode};
 use raftstore::{
     coprocessor::{Config as CopConfig, ConsistencyCheckMethod},
     store::Config as RaftstoreConfig,
@@ -633,6 +633,7 @@ fn test_serde_custom_tikv_config() {
     raft_engine_config.recovery_mode = RecoveryMode::TolerateTailCorruption;
     raft_engine_config.recovery_read_block_size.0 = ReadableSize::kb(1).0;
     raft_engine_config.recovery_threads = 2;
+    raft_engine_config.memory_limit = Some(RaftEngineReadableSize::gb(1));
     value.storage = StorageConfig {
         data_dir: "/var".to_owned(),
         gc_ratio_threshold: 1.2,

tests/integrations/config/test-custom.toml

@@ -576,6 +576,7 @@ purge-threshold = "1GB"
 recovery-mode = "tolerate-tail-corruption"
 recovery-read-block-size = "1KB"
 recovery-threads = 2
+memory-limit = "1GB"
 
 [security]
 ca-path = "invalid path"