metrics: add asynchronous write io panels (#11408) (#11438)

close #10540, ref #11408 Signed-off-by: ti-srebot <ti-srebot@pingcap.com> Signed-off-by: gengliqi <gengliqiii@gmail.com> Co-authored-by: Liqi Geng <gengliqiii@gmail.com> Co-authored-by: gengliqi <gengliqiii@gmail.com> Co-authored-by: Ti Chi Robot <ti-community-prow-bot@tidb.io>
tikv · Jun 15, 2022 · 99efd95 · 99efd95
1 parent 1a51a4c
commit 99efd95
Show file tree

Hide file tree

Showing 3 changed files with 3,651 additions and 909 deletions.
diff --git a/components/raftstore/src/store/async_io/write.rs b/components/raftstore/src/store/async_io/write.rs
@@ -423,14 +423,7 @@ where
 
             self.write_to_db(true);
 
-            self.metrics.flush();
-
             self.clear_latency_inspect();
-            // update config
-            if let Some(incoming) = self.cfg_tracker.any_new() {
-                self.raft_write_size_limit = incoming.raft_write_size_limit.0 as usize;
-                self.metrics.waterfall_metrics = incoming.waterfall_metrics;
-            }
 
             STORE_WRITE_LOOP_DURATION_HISTOGRAM
                 .observe(duration_to_sec(handle_begin.saturating_elapsed()));
@@ -604,6 +597,8 @@ where
         }
 
         let total_cost = now.saturating_duration_since(timer);
+        STORE_WRITE_TO_DB_DURATION_HISTOGRAM.observe(duration_to_sec(total_cost));
+
         slow_log!(
             total_cost,
             "[store {}] async write too slow, write_kv: {}s, write_raft: {}s, send: {}s, callback: {}s thread: {}",
@@ -616,6 +611,13 @@ where
         );
 
         self.batch.clear();
+        self.metrics.flush();
+
+        // update config
+        if let Some(incoming) = self.cfg_tracker.any_new() {
+            self.raft_write_size_limit = incoming.raft_write_size_limit.0 as usize;
+            self.metrics.waterfall_metrics = incoming.waterfall_metrics;
+        }
     }
 
     pub fn is_empty(&self) -> bool {

diff --git a/components/raftstore/src/store/metrics.rs b/components/raftstore/src/store/metrics.rs
@@ -248,9 +248,9 @@ lazy_static! {
         ).unwrap();
     pub static ref STORE_WRITE_RAFTDB_DURATION_HISTOGRAM: Histogram =
         register_histogram!(
-            "tikv_raftstore_append_log_duration_seconds",
-            "Bucketed histogram of peer appending log duration.",
-            exponential_buckets(0.0005, 2.0, 20).unwrap()
+            "tikv_raftstore_store_write_raftdb_duration_seconds",
+            "Bucketed histogram of store write raft db duration.",
+            exponential_buckets(0.00001, 2.0, 26).unwrap()
         ).unwrap();
     pub static ref STORE_WRITE_SEND_DURATION_HISTOGRAM: Histogram =
         register_histogram!(
@@ -264,6 +264,12 @@ lazy_static! {
             "Bucketed histogram of sending callback to store thread duration.",
             exponential_buckets(0.00001, 2.0, 26).unwrap()
         ).unwrap();
+    pub static ref STORE_WRITE_TO_DB_DURATION_HISTOGRAM: Histogram =
+        register_histogram!(
+            "tikv_raftstore_append_log_duration_seconds",
+            "Bucketed histogram of peer appending log duration.",
+            exponential_buckets(0.00001, 2.0, 26).unwrap()
+        ).unwrap();
     pub static ref STORE_WRITE_LOOP_DURATION_HISTOGRAM: Histogram =
         register_histogram!(
             "tikv_raftstore_store_write_loop_duration_seconds",