update default value for end_point_request_max_handle_duration (#15029)

close #15022

Update default end_point_request_max_handle_duration for big region size

Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com>

src/config/mod.rs

@@ -3433,6 +3433,8 @@ impl TikvConfig {
             .optimize_for(self.coprocessor.region_split_size());
         self.raft_store
             .optimize_for(self.storage.engine == EngineType::RaftKv2);
+        self.server
+            .optimize_for(self.coprocessor.region_split_size());
         if self.storage.engine == EngineType::RaftKv2 {
             self.raft_store.store_io_pool_size = cmp::max(self.raft_store.store_io_pool_size, 1);
         }
@@ -5430,6 +5432,35 @@ mod tests {
         check_cfg(&cfg);
     }
 
+    #[test]
+    fn test_endpoint_config() {
+        let mut default_cfg = TikvConfig::default();
+        default_cfg.storage.engine = EngineType::RaftKv;
+        default_cfg.validate().unwrap();
+        assert_eq!(
+            default_cfg.server.end_point_request_max_handle_duration(),
+            ReadableDuration::secs(60)
+        );
+
+        let mut default_cfg = TikvConfig::default();
+        default_cfg.storage.engine = EngineType::RaftKv2;
+        default_cfg.validate().unwrap();
+        assert_eq!(
+            default_cfg.server.end_point_request_max_handle_duration(),
+            ReadableDuration::secs(1800)
+        );
+
+        let mut default_cfg = TikvConfig::default();
+        default_cfg.storage.engine = EngineType::RaftKv2;
+        default_cfg.server.end_point_request_max_handle_duration =
+            Some(ReadableDuration::secs(900));
+        default_cfg.validate().unwrap();
+        assert_eq!(
+            default_cfg.server.end_point_request_max_handle_duration(),
+            ReadableDuration::secs(900)
+        );
+    }
+
     #[test]
     fn test_compatible_adjust_validate_equal() {
         // After calling many time of `compatible_adjust` and `validate` should has
@@ -5910,6 +5941,9 @@ mod tests {
         default_cfg
             .coprocessor
             .optimize_for(default_cfg.storage.engine == EngineType::RaftKv2);
+        default_cfg
+            .server
+            .optimize_for(default_cfg.coprocessor.region_split_size());
         default_cfg.security.redact_info_log = Some(false);
         default_cfg.coprocessor.region_max_size = Some(default_cfg.coprocessor.region_max_size());
         default_cfg.coprocessor.region_max_keys = Some(default_cfg.coprocessor.region_max_keys());

src/coprocessor/endpoint.rs

@@ -103,7 +103,7 @@ impl<E: Engine> Endpoint<E> {
             batch_row_limit: cfg.end_point_batch_row_limit,
             stream_batch_row_limit: cfg.end_point_stream_batch_row_limit,
             stream_channel_size: cfg.end_point_stream_channel_size,
-            max_handle_duration: cfg.end_point_request_max_handle_duration.0,
+            max_handle_duration: cfg.end_point_request_max_handle_duration().0,
             slow_log_threshold: cfg.end_point_slow_log_threshold.0,
             quota_limiter,
             _phantom: Default::default(),
@@ -1492,9 +1492,9 @@ mod tests {
         ));
 
         let config = Config {
-            end_point_request_max_handle_duration: ReadableDuration(
+            end_point_request_max_handle_duration: Some(ReadableDuration(
                 (PAYLOAD_SMALL + PAYLOAD_LARGE) * 2,
-            ),
+            )),
             ..Default::default()
         };
 

src/server/config.rs

@@ -139,7 +139,7 @@ pub struct Config {
     #[online_config(skip)]
     pub end_point_enable_batch_if_possible: bool,
     #[online_config(skip)]
-    pub end_point_request_max_handle_duration: ReadableDuration,
+    pub end_point_request_max_handle_duration: Option<ReadableDuration>,
     #[online_config(skip)]
     pub end_point_max_concurrency: usize,
     #[serde(with = "perf_level_serde")]
@@ -246,9 +246,7 @@ impl Default for Config {
             end_point_batch_row_limit: DEFAULT_ENDPOINT_BATCH_ROW_LIMIT,
             end_point_stream_batch_row_limit: DEFAULT_ENDPOINT_STREAM_BATCH_ROW_LIMIT,
             end_point_enable_batch_if_possible: true,
-            end_point_request_max_handle_duration: ReadableDuration::secs(
-                DEFAULT_ENDPOINT_REQUEST_MAX_HANDLE_SECS,
-            ),
+            end_point_request_max_handle_duration: None,
             end_point_max_concurrency: cmp::max(cpu_num as usize, MIN_ENDPOINT_MAX_CONCURRENCY),
             end_point_perf_level: PerfLevel::Uninitialized,
             snap_io_max_bytes_per_sec: ReadableSize(DEFAULT_SNAP_MAX_BYTES_PER_SEC),
@@ -351,7 +349,7 @@ impl Config {
             return Err(box_err!("server.end-point-recursion-limit is too small"));
         }
 
-        if self.end_point_request_max_handle_duration.as_secs()
+        if self.end_point_request_max_handle_duration().as_secs()
             < DEFAULT_ENDPOINT_REQUEST_MAX_HANDLE_SECS
         {
             return Err(box_err!(
@@ -405,6 +403,32 @@ impl Config {
             GrpcCompressionType::Gzip => CompressionAlgorithms::GRPC_COMPRESS_GZIP,
         }
     }
+
+    pub fn end_point_request_max_handle_duration(&self) -> ReadableDuration {
+        if let Some(end_point_request_max_handle_duration) =
+            self.end_point_request_max_handle_duration
+        {
+            return end_point_request_max_handle_duration;
+        }
+        ReadableDuration::secs(DEFAULT_ENDPOINT_REQUEST_MAX_HANDLE_SECS)
+    }
+
+    pub fn optimize_for(&mut self, region_size: ReadableSize) {
+        // It turns out for 256MB region size, 60s is typically enough.
+        const THRESHOLD_SIZE: ReadableSize = ReadableSize::mb(256);
+        if region_size.0 < THRESHOLD_SIZE.0 {
+            self.end_point_request_max_handle_duration
+                .get_or_insert(ReadableDuration::secs(
+                    DEFAULT_ENDPOINT_REQUEST_MAX_HANDLE_SECS,
+                ));
+        } else {
+            self.end_point_request_max_handle_duration
+                .get_or_insert(ReadableDuration::secs(cmp::min(
+                    1800,
+                    region_size.0 / THRESHOLD_SIZE.0 * DEFAULT_ENDPOINT_REQUEST_MAX_HANDLE_SECS,
+                )));
+        }
+    }
 }
 
 pub struct ServerConfigManager {
@@ -514,7 +538,7 @@ mod tests {
         invalid_cfg.validate().unwrap_err();
 
         let mut invalid_cfg = cfg.clone();
-        invalid_cfg.end_point_request_max_handle_duration = ReadableDuration::secs(0);
+        invalid_cfg.end_point_request_max_handle_duration = Some(ReadableDuration::secs(0));
         invalid_cfg.validate().unwrap_err();
 
         invalid_cfg = Config::default();

tests/failpoints/cases/test_coprocessor.rs

@@ -63,7 +63,9 @@ fn test_deadline_3() {
     let (_, endpoint, _) = {
         let engine = tikv::storage::TestEngineBuilder::new().build().unwrap();
         let cfg = tikv::server::Config {
-            end_point_request_max_handle_duration: tikv_util::config::ReadableDuration::secs(1),
+            end_point_request_max_handle_duration: Some(tikv_util::config::ReadableDuration::secs(
+                1,
+            )),
             ..Default::default()
         };
         init_data_with_details(Context::default(), engine, &product, &data, true, &cfg)

tests/integrations/config/mod.rs

@@ -111,7 +111,7 @@ fn test_serde_custom_tikv_config() {
         end_point_batch_row_limit: 64,
         end_point_stream_batch_row_limit: 4096,
         end_point_enable_batch_if_possible: true,
-        end_point_request_max_handle_duration: ReadableDuration::secs(12),
+        end_point_request_max_handle_duration: Some(ReadableDuration::secs(12)),
         end_point_max_concurrency: 10,
         end_point_perf_level: PerfLevel::EnableTime,
         snap_io_max_bytes_per_sec: ReadableSize::mb(10),