From 6f5c6428b5519144f12ec51715cd74b0d7a32fca Mon Sep 17 00:00:00 2001 From: lucasliang Date: Thu, 9 May 2024 10:34:08 +0800 Subject: [PATCH] raft-engine: reduce the threshold of compression and add related metrics on it. (#16906) ref tikv/tikv#16907 This pr contains: - Reduce the default compression threshold in RaftEngine to `4kb` as default if users do not manually set it. - Add relevant metrics on the compression ratio of RaftEngine.write. Signed-off-by: lucasliang --- Cargo.lock | 8 +- metrics/grafana/tikv_details.dashboard.py | 6 + metrics/grafana/tikv_details.json | 871 +++++++++++++--------- metrics/grafana/tikv_details.json.sha256 | 2 +- src/config/mod.rs | 42 ++ tests/integrations/config/mod.rs | 74 +- 6 files changed, 662 insertions(+), 341 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ff034bf1729..a5a6ded518e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4159,8 +4159,8 @@ dependencies = [ [[package]] name = "raft-engine" -version = "0.4.1" -source = "git+https://github.com/tikv/raft-engine.git#e505d631c8c6d63f7fc63d83ea6e8fb88cf970a5" +version = "0.4.2" +source = "git+https://github.com/tikv/raft-engine.git#cd1533d79cfea02d005b458bd70fd9675846c9c6" dependencies = [ "byteorder", "crc32fast", @@ -4193,8 +4193,8 @@ dependencies = [ [[package]] name = "raft-engine-ctl" -version = "0.4.1" -source = "git+https://github.com/tikv/raft-engine.git#fa56f891fdf0b1cb5b7849b7bee3c5dadbb96103" +version = "0.4.2" +source = "git+https://github.com/tikv/raft-engine.git#cd1533d79cfea02d005b458bd70fd9675846c9c6" dependencies = [ "clap 3.1.6", "env_logger 0.10.0", diff --git a/metrics/grafana/tikv_details.dashboard.py b/metrics/grafana/tikv_details.dashboard.py index 3310d6289ae..67d55ac8262 100644 --- a/metrics/grafana/tikv_details.dashboard.py +++ b/metrics/grafana/tikv_details.dashboard.py @@ -6007,6 +6007,12 @@ def RaftEngine() -> RowPanel: ), ], ), + graph_panel_histogram_quantiles( + title="Write Compression Ratio", + description="The compression ratio per write", + yaxes=yaxes(left_format=UNITS.NONE_FORMAT), + metric="raft_engine_write_compression_ratio", + ), ] ) return layout.row_panel diff --git a/metrics/grafana/tikv_details.json b/metrics/grafana/tikv_details.json index d95053b78ea..e4d4a9ca479 100644 --- a/metrics/grafana/tikv_details.json +++ b/metrics/grafana/tikv_details.json @@ -25927,7 +25927,7 @@ }, "gridPos": { "h": 7, - "w": 24, + "w": 12, "x": 0, "y": 28 }, @@ -26033,6 +26033,207 @@ "align": false, "alignLevel": 0 } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_TEST-CLUSTER}", + "description": "The compression ratio per write", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [] + } + } + }, + "fill": 1, + "fillGradient": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 28 + }, + "height": null, + "hideTimeOverride": false, + "id": 188, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxDataPoints": null, + "maxPerRow": null, + "minSpan": null, + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": null, + "seriesOverrides": [ + { + "alias": "count", + "bars": false, + "dashLength": 1, + "dashes": true, + "fill": 2, + "fillBelowTo": null, + "lines": true, + "spaceLength": 1, + "transform": "negative-Y", + "yaxis": 2, + "zindex": -3 + }, + { + "alias": "avg", + "bars": false, + "fill": 7, + "fillBelowTo": null, + "lines": true, + "yaxis": 1, + "zindex": 0 + } + ], + "span": null, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "${DS_TEST-CLUSTER}", + "expr": "histogram_quantile(0.9999,(\n sum(rate(\n raft_engine_write_compression_ratio_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "99.99%", + "metric": "", + "query": "histogram_quantile(0.9999,(\n sum(rate(\n raft_engine_write_compression_ratio_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", + "refId": "", + "step": 10, + "target": "" + }, + { + "datasource": "${DS_TEST-CLUSTER}", + "expr": "histogram_quantile(0.99,(\n sum(rate(\n raft_engine_write_compression_ratio_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "99%", + "metric": "", + "query": "histogram_quantile(0.99,(\n sum(rate(\n raft_engine_write_compression_ratio_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", + "refId": "", + "step": 10, + "target": "" + }, + { + "datasource": "${DS_TEST-CLUSTER}", + "expr": "(sum(rate(\n raft_engine_write_compression_ratio_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) / sum(rate(\n raft_engine_write_compression_ratio_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) )", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "avg", + "metric": "", + "query": "(sum(rate(\n raft_engine_write_compression_ratio_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) / sum(rate(\n raft_engine_write_compression_ratio_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) )", + "refId": "", + "step": 10, + "target": "" + }, + { + "datasource": "${DS_TEST-CLUSTER}", + "expr": "sum(rate(\n raft_engine_write_compression_ratio_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) ", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "count", + "metric": "", + "query": "sum(rate(\n raft_engine_write_compression_ratio_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) ", + "refId": "", + "step": 10, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Write Compression Ratio", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "transformations": [], + "transparent": false, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": 0 + } } ], "repeat": null, @@ -26069,7 +26270,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 188, + "id": 189, "interval": null, "links": [], "maxDataPoints": 100, @@ -26108,7 +26309,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 189, + "id": 190, "interval": null, "isNew": true, "legend": { @@ -26301,7 +26502,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 190, + "id": 191, "interval": null, "isNew": true, "legend": { @@ -26479,7 +26680,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 191, + "id": 192, "interval": null, "isNew": true, "legend": { @@ -26687,7 +26888,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 192, + "id": 193, "interval": null, "isNew": true, "legend": { @@ -26865,7 +27066,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 193, + "id": 194, "interval": null, "isNew": true, "legend": { @@ -27028,7 +27229,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 194, + "id": 195, "interval": null, "isNew": true, "legend": { @@ -27206,7 +27407,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 195, + "id": 196, "interval": null, "isNew": true, "legend": { @@ -27339,7 +27540,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 196, + "id": 197, "interval": null, "isNew": true, "legend": { @@ -27517,7 +27718,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 197, + "id": 198, "interval": null, "isNew": true, "legend": { @@ -27650,7 +27851,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 198, + "id": 199, "interval": null, "isNew": true, "legend": { @@ -27828,7 +28029,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 199, + "id": 200, "interval": null, "isNew": true, "legend": { @@ -27961,7 +28162,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 200, + "id": 201, "interval": null, "isNew": true, "legend": { @@ -28139,7 +28340,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 201, + "id": 202, "interval": null, "isNew": true, "legend": { @@ -28317,7 +28518,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 202, + "id": 203, "interval": null, "isNew": true, "legend": { @@ -28495,7 +28696,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 203, + "id": 204, "interval": null, "isNew": true, "legend": { @@ -28628,7 +28829,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 204, + "id": 205, "interval": null, "isNew": true, "legend": { @@ -28761,7 +28962,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 205, + "id": 206, "interval": null, "isNew": true, "legend": { @@ -28894,7 +29095,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 206, + "id": 207, "interval": null, "isNew": true, "legend": { @@ -29117,7 +29318,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 207, + "id": 208, "interval": null, "isNew": true, "legend": { @@ -29310,7 +29511,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 208, + "id": 209, "interval": null, "isNew": true, "legend": { @@ -29473,7 +29674,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 209, + "id": 210, "interval": null, "isNew": true, "legend": { @@ -29666,7 +29867,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 210, + "id": 211, "interval": null, "isNew": true, "legend": { @@ -29814,7 +30015,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 211, + "id": 212, "interval": null, "isNew": true, "legend": { @@ -29947,7 +30148,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 212, + "id": 213, "interval": null, "isNew": true, "legend": { @@ -30095,7 +30296,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 213, + "id": 214, "interval": null, "isNew": true, "legend": { @@ -30273,7 +30474,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 214, + "id": 215, "interval": null, "isNew": true, "legend": { @@ -30436,7 +30637,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 215, + "id": 216, "interval": null, "isNew": true, "legend": { @@ -30614,7 +30815,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 216, + "id": 217, "interval": null, "isNew": true, "legend": { @@ -30747,7 +30948,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 217, + "id": 218, "interval": null, "isNew": true, "legend": { @@ -30880,7 +31081,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 218, + "id": 219, "interval": null, "isNew": true, "legend": { @@ -31013,7 +31214,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 219, + "id": 220, "interval": null, "isNew": true, "legend": { @@ -31146,7 +31347,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 220, + "id": 221, "interval": null, "isNew": true, "legend": { @@ -31279,7 +31480,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 221, + "id": 222, "interval": null, "isNew": true, "legend": { @@ -31412,7 +31613,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 222, + "id": 223, "interval": null, "isNew": true, "legend": { @@ -31545,7 +31746,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 223, + "id": 224, "interval": null, "isNew": true, "legend": { @@ -31746,7 +31947,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 224, + "id": 225, "interval": null, "isNew": true, "legend": { @@ -31879,7 +32080,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 225, + "id": 226, "interval": null, "isNew": true, "legend": { @@ -32064,7 +32265,7 @@ "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, - "id": 226, + "id": 227, "interval": null, "legend": { "show": false @@ -32161,7 +32362,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 227, + "id": 228, "interval": null, "isNew": true, "legend": { @@ -32297,7 +32498,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 228, + "id": 229, "interval": null, "links": [], "maxDataPoints": 100, @@ -32336,7 +32537,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 229, + "id": 230, "interval": null, "isNew": true, "legend": { @@ -32484,7 +32685,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 230, + "id": 231, "interval": null, "isNew": true, "legend": { @@ -32632,7 +32833,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 231, + "id": 232, "interval": null, "isNew": true, "legend": { @@ -32765,7 +32966,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 232, + "id": 233, "interval": null, "isNew": true, "legend": { @@ -32898,7 +33099,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 233, + "id": 234, "interval": null, "isNew": true, "legend": { @@ -33076,7 +33277,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 234, + "id": 235, "interval": null, "isNew": true, "legend": { @@ -33254,7 +33455,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 235, + "id": 236, "interval": null, "isNew": true, "legend": { @@ -33432,7 +33633,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 236, + "id": 237, "interval": null, "isNew": true, "legend": { @@ -33565,7 +33766,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 237, + "id": 238, "interval": null, "isNew": true, "legend": { @@ -33743,7 +33944,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 238, + "id": 239, "interval": null, "isNew": true, "legend": { @@ -33876,7 +34077,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 239, + "id": 240, "interval": null, "isNew": true, "legend": { @@ -34039,7 +34240,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 240, + "id": 241, "interval": null, "isNew": true, "legend": { @@ -34217,7 +34418,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 241, + "id": 242, "interval": null, "isNew": true, "legend": { @@ -34395,7 +34596,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 242, + "id": 243, "interval": null, "isNew": true, "legend": { @@ -34573,7 +34774,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 243, + "id": 244, "interval": null, "isNew": true, "legend": { @@ -34706,7 +34907,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 244, + "id": 245, "interval": null, "isNew": true, "legend": { @@ -34884,7 +35085,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 245, + "id": 246, "interval": null, "isNew": true, "legend": { @@ -35017,7 +35218,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 246, + "id": 247, "interval": null, "isNew": true, "legend": { @@ -35195,7 +35396,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 247, + "id": 248, "interval": null, "isNew": true, "legend": { @@ -35328,7 +35529,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 248, + "id": 249, "interval": null, "isNew": true, "legend": { @@ -35461,7 +35662,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 249, + "id": 250, "interval": null, "isNew": true, "legend": { @@ -35639,7 +35840,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 250, + "id": 251, "interval": null, "isNew": true, "legend": { @@ -35817,7 +36018,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 251, + "id": 252, "interval": null, "isNew": true, "legend": { @@ -35950,7 +36151,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 252, + "id": 253, "interval": null, "isNew": true, "legend": { @@ -36128,7 +36329,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 253, + "id": 254, "interval": null, "isNew": true, "legend": { @@ -36261,7 +36462,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 254, + "id": 255, "interval": null, "isNew": true, "legend": { @@ -36439,7 +36640,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 255, + "id": 256, "interval": null, "isNew": true, "legend": { @@ -36575,7 +36776,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 256, + "id": 257, "interval": null, "links": [], "maxDataPoints": 100, @@ -36614,7 +36815,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 257, + "id": 258, "interval": null, "isNew": true, "legend": { @@ -36747,7 +36948,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 258, + "id": 259, "interval": null, "isNew": true, "legend": { @@ -36880,7 +37081,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 259, + "id": 260, "interval": null, "isNew": true, "legend": { @@ -37013,7 +37214,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 260, + "id": 261, "interval": null, "isNew": true, "legend": { @@ -37153,7 +37354,7 @@ "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, - "id": 261, + "id": 262, "interval": null, "legend": { "show": false @@ -37257,7 +37458,7 @@ "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, - "id": 262, + "id": 263, "interval": null, "legend": { "show": false @@ -37361,7 +37562,7 @@ "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, - "id": 263, + "id": 264, "interval": null, "legend": { "show": false @@ -37458,7 +37659,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 264, + "id": 265, "interval": null, "isNew": true, "legend": { @@ -37662,7 +37863,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 265, + "id": 266, "interval": null, "links": [], "maxDataPoints": 100, @@ -37701,7 +37902,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 266, + "id": 267, "interval": null, "isNew": true, "legend": { @@ -37849,7 +38050,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 267, + "id": 268, "interval": null, "isNew": true, "legend": { @@ -37989,7 +38190,7 @@ "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, - "id": 268, + "id": 269, "interval": null, "legend": { "show": false @@ -38086,7 +38287,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 269, + "id": 270, "interval": null, "isNew": true, "legend": { @@ -38219,7 +38420,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 270, + "id": 271, "interval": null, "isNew": true, "legend": { @@ -38352,7 +38553,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 271, + "id": 272, "interval": null, "isNew": true, "legend": { @@ -38530,7 +38731,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 272, + "id": 273, "interval": null, "isNew": true, "legend": { @@ -38693,7 +38894,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 273, + "id": 274, "interval": null, "isNew": true, "legend": { @@ -38841,7 +39042,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 274, + "id": 275, "interval": null, "isNew": true, "legend": { @@ -38974,7 +39175,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 275, + "id": 276, "interval": null, "isNew": true, "legend": { @@ -39110,7 +39311,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 276, + "id": 277, "interval": null, "links": [], "maxDataPoints": 100, @@ -39149,7 +39350,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 277, + "id": 278, "interval": null, "isNew": true, "legend": { @@ -39297,7 +39498,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 278, + "id": 279, "interval": null, "isNew": true, "legend": { @@ -39430,7 +39631,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 279, + "id": 280, "interval": null, "isNew": true, "legend": { @@ -39563,7 +39764,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 280, + "id": 281, "interval": null, "isNew": true, "legend": { @@ -39696,7 +39897,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 281, + "id": 282, "interval": null, "isNew": true, "legend": { @@ -39829,7 +40030,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 282, + "id": 283, "interval": null, "isNew": true, "legend": { @@ -39984,7 +40185,7 @@ "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, - "id": 283, + "id": 284, "interval": null, "legend": { "show": false @@ -40084,7 +40285,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 284, + "id": 285, "interval": null, "links": [], "maxDataPoints": 100, @@ -40123,7 +40324,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 285, + "id": 286, "interval": null, "isNew": true, "legend": { @@ -40271,7 +40472,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 286, + "id": 287, "interval": null, "isNew": true, "legend": { @@ -40472,7 +40673,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 287, + "id": 288, "interval": null, "isNew": true, "legend": { @@ -40673,7 +40874,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 288, + "id": 289, "interval": null, "isNew": true, "legend": { @@ -40874,7 +41075,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 289, + "id": 290, "interval": null, "isNew": true, "legend": { @@ -41075,7 +41276,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 290, + "id": 291, "interval": null, "isNew": true, "legend": { @@ -41208,7 +41409,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 291, + "id": 292, "interval": null, "isNew": true, "legend": { @@ -41341,7 +41542,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 292, + "id": 293, "interval": null, "isNew": true, "legend": { @@ -41474,7 +41675,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 293, + "id": 294, "interval": null, "isNew": true, "legend": { @@ -41607,7 +41808,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 294, + "id": 295, "interval": null, "isNew": true, "legend": { @@ -41815,7 +42016,7 @@ "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, - "id": 295, + "id": 296, "interval": null, "legend": { "show": false @@ -41915,7 +42116,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 296, + "id": 297, "interval": null, "links": [], "maxDataPoints": 100, @@ -41961,7 +42162,7 @@ "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, - "id": 297, + "id": 298, "interval": null, "legend": { "show": false @@ -42058,7 +42259,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 298, + "id": 299, "interval": null, "isNew": true, "legend": { @@ -42259,7 +42460,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 299, + "id": 300, "interval": null, "isNew": true, "legend": { @@ -42392,7 +42593,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 300, + "id": 301, "interval": null, "isNew": true, "legend": { @@ -42525,7 +42726,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 301, + "id": 302, "interval": null, "isNew": true, "legend": { @@ -42658,7 +42859,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 302, + "id": 303, "interval": null, "isNew": true, "legend": { @@ -42859,7 +43060,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 303, + "id": 304, "interval": null, "isNew": true, "legend": { @@ -42992,7 +43193,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 304, + "id": 305, "interval": null, "isNew": true, "legend": { @@ -43128,7 +43329,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 305, + "id": 306, "interval": null, "links": [], "maxDataPoints": 100, @@ -43167,7 +43368,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 306, + "id": 307, "interval": null, "isNew": true, "legend": { @@ -43368,7 +43569,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 307, + "id": 308, "interval": null, "isNew": true, "legend": { @@ -43569,7 +43770,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 308, + "id": 309, "interval": null, "isNew": true, "legend": { @@ -43770,7 +43971,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 309, + "id": 310, "interval": null, "isNew": true, "legend": { @@ -43971,7 +44172,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 310, + "id": 311, "interval": null, "isNew": true, "legend": { @@ -44104,7 +44305,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 311, + "id": 312, "interval": null, "isNew": true, "legend": { @@ -44237,7 +44438,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 312, + "id": 313, "interval": null, "isNew": true, "legend": { @@ -44370,7 +44571,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 313, + "id": 314, "interval": null, "isNew": true, "legend": { @@ -44503,7 +44704,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 314, + "id": 315, "interval": null, "isNew": true, "legend": { @@ -44636,7 +44837,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 315, + "id": 316, "interval": null, "isNew": true, "legend": { @@ -44776,7 +44977,7 @@ "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, - "id": 316, + "id": 317, "interval": null, "legend": { "show": false @@ -44873,7 +45074,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 317, + "id": 318, "interval": null, "isNew": true, "legend": { @@ -45077,7 +45278,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 318, + "id": 319, "interval": null, "links": [], "maxDataPoints": 100, @@ -45116,7 +45317,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 319, + "id": 320, "interval": null, "isNew": true, "legend": { @@ -45249,7 +45450,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 320, + "id": 321, "interval": null, "isNew": true, "legend": { @@ -45382,7 +45583,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 321, + "id": 322, "interval": null, "isNew": true, "legend": { @@ -45522,7 +45723,7 @@ "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, - "id": 322, + "id": 323, "interval": null, "legend": { "show": false @@ -45619,7 +45820,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 323, + "id": 324, "interval": null, "isNew": true, "legend": { @@ -45820,7 +46021,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 324, + "id": 325, "interval": null, "isNew": true, "legend": { @@ -46021,7 +46222,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 325, + "id": 326, "interval": null, "isNew": true, "legend": { @@ -46225,7 +46426,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 326, + "id": 327, "interval": null, "links": [], "maxDataPoints": 100, @@ -46264,7 +46465,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 327, + "id": 328, "interval": null, "isNew": true, "legend": { @@ -46442,7 +46643,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 328, + "id": 329, "interval": null, "isNew": true, "legend": { @@ -46643,7 +46844,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 329, + "id": 330, "interval": null, "isNew": true, "legend": { @@ -46776,7 +46977,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 330, + "id": 331, "interval": null, "isNew": true, "legend": { @@ -46909,7 +47110,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 331, + "id": 332, "interval": null, "isNew": true, "legend": { @@ -47042,7 +47243,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 332, + "id": 333, "interval": null, "isNew": true, "legend": { @@ -47175,7 +47376,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 333, + "id": 334, "interval": null, "isNew": true, "legend": { @@ -47308,7 +47509,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 334, + "id": 335, "interval": null, "isNew": true, "legend": { @@ -47437,7 +47638,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 335, + "id": 336, "interval": null, "links": [], "maxDataPoints": 100, @@ -47512,7 +47713,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 336, + "id": 337, "interval": null, "links": [], "maxDataPoints": 100, @@ -47591,7 +47792,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 337, + "id": 338, "interval": null, "isNew": true, "legend": { @@ -47844,7 +48045,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 338, + "id": 339, "interval": null, "isNew": true, "legend": { @@ -47977,7 +48178,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 339, + "id": 340, "interval": null, "isNew": true, "legend": { @@ -48113,7 +48314,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 340, + "id": 341, "interval": null, "links": [], "maxDataPoints": 100, @@ -48152,7 +48353,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 341, + "id": 342, "interval": null, "isNew": true, "legend": { @@ -48300,7 +48501,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 342, + "id": 343, "interval": null, "isNew": true, "legend": { @@ -48433,7 +48634,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 343, + "id": 344, "interval": null, "isNew": true, "legend": { @@ -48634,7 +48835,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 344, + "id": 345, "interval": null, "isNew": true, "legend": { @@ -48782,7 +48983,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 345, + "id": 346, "interval": null, "isNew": true, "legend": { @@ -48983,7 +49184,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 346, + "id": 347, "interval": null, "isNew": true, "legend": { @@ -49116,7 +49317,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 347, + "id": 348, "interval": null, "isNew": true, "legend": { @@ -49249,7 +49450,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 348, + "id": 349, "interval": null, "isNew": true, "legend": { @@ -49382,7 +49583,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 349, + "id": 350, "interval": null, "isNew": true, "legend": { @@ -49515,7 +49716,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 350, + "id": 351, "interval": null, "isNew": true, "legend": { @@ -49655,7 +49856,7 @@ "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, - "id": 351, + "id": 352, "interval": null, "legend": { "show": false @@ -49752,7 +49953,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 352, + "id": 353, "interval": null, "isNew": true, "legend": { @@ -49956,7 +50157,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 353, + "id": 354, "interval": null, "links": [], "maxDataPoints": 100, @@ -49995,7 +50196,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 354, + "id": 355, "interval": null, "isNew": true, "legend": { @@ -50128,7 +50329,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 355, + "id": 356, "interval": null, "isNew": true, "legend": { @@ -50261,7 +50462,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 356, + "id": 357, "interval": null, "isNew": true, "legend": { @@ -50394,7 +50595,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 357, + "id": 358, "interval": null, "isNew": true, "legend": { @@ -50530,7 +50731,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 358, + "id": 359, "interval": null, "links": [], "maxDataPoints": 100, @@ -50569,7 +50770,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 359, + "id": 360, "interval": null, "isNew": true, "legend": { @@ -50702,7 +50903,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 360, + "id": 361, "interval": null, "isNew": true, "legend": { @@ -50835,7 +51036,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 361, + "id": 362, "interval": null, "isNew": true, "legend": { @@ -50983,7 +51184,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 362, + "id": 363, "interval": null, "isNew": true, "legend": { @@ -51116,7 +51317,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 363, + "id": 364, "interval": null, "isNew": true, "legend": { @@ -51249,7 +51450,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 364, + "id": 365, "interval": null, "isNew": true, "legend": { @@ -51382,7 +51583,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 365, + "id": 366, "interval": null, "isNew": true, "legend": { @@ -51518,7 +51719,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 366, + "id": 367, "interval": null, "links": [], "maxDataPoints": 100, @@ -51557,7 +51758,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 367, + "id": 368, "interval": null, "isNew": true, "legend": { @@ -51690,7 +51891,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 368, + "id": 369, "interval": null, "isNew": true, "legend": { @@ -51823,7 +52024,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 369, + "id": 370, "interval": null, "isNew": true, "legend": { @@ -51956,7 +52157,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 370, + "id": 371, "interval": null, "isNew": true, "legend": { @@ -52089,7 +52290,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 371, + "id": 372, "interval": null, "isNew": true, "legend": { @@ -52222,7 +52423,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 372, + "id": 373, "interval": null, "isNew": true, "legend": { @@ -52358,7 +52559,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 373, + "id": 374, "interval": null, "links": [], "maxDataPoints": 100, @@ -52397,7 +52598,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 374, + "id": 375, "interval": null, "isNew": true, "legend": { @@ -52530,7 +52731,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 375, + "id": 376, "interval": null, "isNew": true, "legend": { @@ -52663,7 +52864,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 376, + "id": 377, "interval": null, "isNew": true, "legend": { @@ -52796,7 +52997,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 377, + "id": 378, "interval": null, "isNew": true, "legend": { @@ -52959,7 +53160,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 378, + "id": 379, "interval": null, "isNew": true, "legend": { @@ -53092,7 +53293,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 379, + "id": 380, "interval": null, "isNew": true, "legend": { @@ -53225,7 +53426,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 380, + "id": 381, "interval": null, "isNew": true, "legend": { @@ -53373,7 +53574,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 381, + "id": 382, "interval": null, "isNew": true, "legend": { @@ -53524,7 +53725,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 382, + "id": 383, "interval": null, "links": [], "maxDataPoints": 100, @@ -53563,7 +53764,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 383, + "id": 384, "interval": null, "isNew": true, "legend": { @@ -53696,7 +53897,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 384, + "id": 385, "interval": null, "isNew": true, "legend": { @@ -53829,7 +54030,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 385, + "id": 386, "interval": null, "isNew": true, "legend": { @@ -53962,7 +54163,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 386, + "id": 387, "interval": null, "isNew": true, "legend": { @@ -54095,7 +54296,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 387, + "id": 388, "interval": null, "isNew": true, "legend": { @@ -54228,7 +54429,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 388, + "id": 389, "interval": null, "isNew": true, "legend": { @@ -54361,7 +54562,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 389, + "id": 390, "interval": null, "isNew": true, "legend": { @@ -54494,7 +54695,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 390, + "id": 391, "interval": null, "isNew": true, "legend": { @@ -54627,7 +54828,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 391, + "id": 392, "interval": null, "isNew": true, "legend": { @@ -54767,7 +54968,7 @@ "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, - "id": 392, + "id": 393, "interval": null, "legend": { "show": false @@ -54864,7 +55065,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 393, + "id": 394, "interval": null, "isNew": true, "legend": { @@ -54997,7 +55198,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 394, + "id": 395, "interval": null, "isNew": true, "legend": { @@ -55145,7 +55346,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 395, + "id": 396, "interval": null, "isNew": true, "legend": { @@ -55293,7 +55494,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 396, + "id": 397, "interval": null, "isNew": true, "legend": { @@ -55433,7 +55634,7 @@ "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, - "id": 397, + "id": 398, "interval": null, "legend": { "show": false @@ -55530,7 +55731,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 398, + "id": 399, "interval": null, "isNew": true, "legend": { @@ -55663,7 +55864,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 399, + "id": 400, "interval": null, "isNew": true, "legend": { @@ -55799,7 +56000,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 400, + "id": 401, "interval": null, "links": [], "maxDataPoints": 100, @@ -55838,7 +56039,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 401, + "id": 402, "interval": null, "isNew": true, "legend": { @@ -55971,7 +56172,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 402, + "id": 403, "interval": null, "isNew": true, "legend": { @@ -56134,7 +56335,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 403, + "id": 404, "interval": null, "isNew": true, "legend": { @@ -56282,7 +56483,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 404, + "id": 405, "interval": null, "isNew": true, "legend": { @@ -56415,7 +56616,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 405, + "id": 406, "interval": null, "isNew": true, "legend": { @@ -56555,7 +56756,7 @@ "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, - "id": 406, + "id": 407, "interval": null, "legend": { "show": false @@ -56659,7 +56860,7 @@ "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, - "id": 407, + "id": 408, "interval": null, "legend": { "show": false @@ -56763,7 +56964,7 @@ "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, - "id": 408, + "id": 409, "interval": null, "legend": { "show": false @@ -56860,7 +57061,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 409, + "id": 410, "interval": null, "isNew": true, "legend": { @@ -57000,7 +57201,7 @@ "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, - "id": 410, + "id": 411, "interval": null, "legend": { "show": false @@ -57104,7 +57305,7 @@ "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, - "id": 411, + "id": 412, "interval": null, "legend": { "show": false @@ -57208,7 +57409,7 @@ "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, - "id": 412, + "id": 413, "interval": null, "legend": { "show": false @@ -57305,7 +57506,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 413, + "id": 414, "interval": null, "isNew": true, "legend": { @@ -57438,7 +57639,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 414, + "id": 415, "interval": null, "isNew": true, "legend": { @@ -57571,7 +57772,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 415, + "id": 416, "interval": null, "isNew": true, "legend": { @@ -57711,7 +57912,7 @@ "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, - "id": 416, + "id": 417, "interval": null, "legend": { "show": false @@ -57808,7 +58009,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 417, + "id": 418, "interval": null, "isNew": true, "legend": { @@ -57944,7 +58145,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 418, + "id": 419, "interval": null, "links": [], "maxDataPoints": 100, @@ -57983,7 +58184,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 419, + "id": 420, "interval": null, "isNew": true, "legend": { @@ -58146,7 +58347,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 420, + "id": 421, "interval": null, "isNew": true, "legend": { @@ -58279,7 +58480,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 421, + "id": 422, "interval": null, "isNew": true, "legend": { @@ -58419,7 +58620,7 @@ "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, - "id": 422, + "id": 423, "interval": null, "legend": { "show": false @@ -58523,7 +58724,7 @@ "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, - "id": 423, + "id": 424, "interval": null, "legend": { "show": false @@ -58620,7 +58821,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 424, + "id": 425, "interval": null, "isNew": true, "legend": { @@ -58775,7 +58976,7 @@ "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, - "id": 425, + "id": 426, "interval": null, "legend": { "show": false @@ -58879,7 +59080,7 @@ "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, - "id": 426, + "id": 427, "interval": null, "legend": { "show": false @@ -58983,7 +59184,7 @@ "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, - "id": 427, + "id": 428, "interval": null, "legend": { "show": false @@ -59080,7 +59281,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 428, + "id": 429, "interval": null, "isNew": true, "legend": { @@ -59250,7 +59451,7 @@ "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, - "id": 429, + "id": 430, "interval": null, "legend": { "show": false @@ -59347,7 +59548,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 430, + "id": 431, "interval": null, "isNew": true, "legend": { @@ -59548,7 +59749,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 431, + "id": 432, "interval": null, "isNew": true, "legend": { @@ -59749,7 +59950,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 432, + "id": 433, "interval": null, "isNew": true, "legend": { @@ -59882,7 +60083,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 433, + "id": 434, "interval": null, "isNew": true, "legend": { @@ -60045,7 +60246,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 434, + "id": 435, "interval": null, "isNew": true, "legend": { @@ -60178,7 +60379,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 435, + "id": 436, "interval": null, "isNew": true, "legend": { @@ -60311,7 +60512,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 436, + "id": 437, "interval": null, "isNew": true, "legend": { @@ -60512,7 +60713,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 437, + "id": 438, "interval": null, "isNew": true, "legend": { @@ -60645,7 +60846,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 438, + "id": 439, "interval": null, "isNew": true, "legend": { @@ -60785,7 +60986,7 @@ "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, - "id": 439, + "id": 440, "interval": null, "legend": { "show": false @@ -60889,7 +61090,7 @@ "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, - "id": 440, + "id": 441, "interval": null, "legend": { "show": false @@ -60993,7 +61194,7 @@ "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, - "id": 441, + "id": 442, "interval": null, "legend": { "show": false @@ -61097,7 +61298,7 @@ "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, - "id": 442, + "id": 443, "interval": null, "legend": { "show": false @@ -61201,7 +61402,7 @@ "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, - "id": 443, + "id": 444, "interval": null, "legend": { "show": false @@ -61305,7 +61506,7 @@ "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, - "id": 444, + "id": 445, "interval": null, "legend": { "show": false @@ -61409,7 +61610,7 @@ "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, - "id": 445, + "id": 446, "interval": null, "legend": { "show": false @@ -61506,7 +61707,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 446, + "id": 447, "interval": null, "isNew": true, "legend": { @@ -61654,7 +61855,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 447, + "id": 448, "interval": null, "isNew": true, "legend": { @@ -61787,7 +61988,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 448, + "id": 449, "interval": null, "isNew": true, "legend": { @@ -61920,7 +62121,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 449, + "id": 450, "interval": null, "isNew": true, "legend": { @@ -62068,7 +62269,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 450, + "id": 451, "interval": null, "isNew": true, "legend": { @@ -62204,7 +62405,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 451, + "id": 452, "interval": null, "links": [], "maxDataPoints": 100, @@ -62255,7 +62456,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 452, + "id": 453, "interval": null, "links": [], "maxDataPoints": 100, @@ -62351,7 +62552,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 453, + "id": 454, "interval": null, "links": [], "maxDataPoints": 100, @@ -62426,7 +62627,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 454, + "id": 455, "interval": null, "links": [], "maxDataPoints": 100, @@ -62501,7 +62702,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 455, + "id": 456, "interval": null, "links": [], "maxDataPoints": 100, @@ -62576,7 +62777,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 456, + "id": 457, "interval": null, "links": [], "maxDataPoints": 100, @@ -62651,7 +62852,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 457, + "id": 458, "interval": null, "links": [], "maxDataPoints": 100, @@ -62726,7 +62927,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 458, + "id": 459, "interval": null, "links": [], "maxDataPoints": 100, @@ -62801,7 +63002,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 459, + "id": 460, "interval": null, "links": [], "maxDataPoints": 100, @@ -62880,7 +63081,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 460, + "id": 461, "interval": null, "isNew": true, "legend": { @@ -63013,7 +63214,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 461, + "id": 462, "interval": null, "isNew": true, "legend": { @@ -63146,7 +63347,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 462, + "id": 463, "interval": null, "isNew": true, "legend": { @@ -63279,7 +63480,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 463, + "id": 464, "interval": null, "isNew": true, "legend": { @@ -63412,7 +63613,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 464, + "id": 465, "interval": null, "isNew": true, "legend": { @@ -63545,7 +63746,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 465, + "id": 466, "interval": null, "isNew": true, "legend": { @@ -63693,7 +63894,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 466, + "id": 467, "interval": null, "isNew": true, "legend": { @@ -63826,7 +64027,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 467, + "id": 468, "interval": null, "isNew": true, "legend": { @@ -63959,7 +64160,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 468, + "id": 469, "interval": null, "isNew": true, "legend": { @@ -64125,7 +64326,7 @@ "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, - "id": 469, + "id": 470, "interval": null, "legend": { "show": false @@ -64229,7 +64430,7 @@ "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, - "id": 470, + "id": 471, "interval": null, "legend": { "show": false @@ -64333,7 +64534,7 @@ "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, - "id": 471, + "id": 472, "interval": null, "legend": { "show": false @@ -64437,7 +64638,7 @@ "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, - "id": 472, + "id": 473, "interval": null, "legend": { "show": false @@ -64541,7 +64742,7 @@ "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, - "id": 473, + "id": 474, "interval": null, "legend": { "show": false @@ -64645,7 +64846,7 @@ "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, - "id": 474, + "id": 475, "interval": null, "legend": { "show": false @@ -64749,7 +64950,7 @@ "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, - "id": 475, + "id": 476, "interval": null, "legend": { "show": false @@ -64853,7 +65054,7 @@ "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, - "id": 476, + "id": 477, "interval": null, "legend": { "show": false @@ -64950,7 +65151,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 477, + "id": 478, "interval": null, "isNew": true, "legend": { @@ -65083,7 +65284,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 478, + "id": 479, "interval": null, "isNew": true, "legend": { @@ -65216,7 +65417,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 479, + "id": 480, "interval": null, "isNew": true, "legend": { @@ -65349,7 +65550,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 480, + "id": 481, "interval": null, "isNew": true, "legend": { @@ -65482,7 +65683,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 481, + "id": 482, "interval": null, "isNew": true, "legend": { @@ -65615,7 +65816,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 482, + "id": 483, "interval": null, "isNew": true, "legend": { @@ -65748,7 +65949,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 483, + "id": 484, "interval": null, "isNew": true, "legend": { @@ -65881,7 +66082,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 484, + "id": 485, "interval": null, "isNew": true, "legend": { @@ -66021,7 +66222,7 @@ "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, - "id": 485, + "id": 486, "interval": null, "legend": { "show": false @@ -66125,7 +66326,7 @@ "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, - "id": 486, + "id": 487, "interval": null, "legend": { "show": false @@ -66222,7 +66423,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 487, + "id": 488, "interval": null, "isNew": true, "legend": { @@ -66355,7 +66556,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 488, + "id": 489, "interval": null, "isNew": true, "legend": { @@ -66488,7 +66689,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 489, + "id": 490, "interval": null, "isNew": true, "legend": { @@ -66621,7 +66822,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 490, + "id": 491, "interval": null, "isNew": true, "legend": { @@ -66754,7 +66955,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 491, + "id": 492, "interval": null, "isNew": true, "legend": { @@ -66887,7 +67088,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 492, + "id": 493, "interval": null, "isNew": true, "legend": { @@ -67023,7 +67224,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 493, + "id": 494, "interval": null, "links": [], "maxDataPoints": 100, @@ -67062,7 +67263,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 494, + "id": 495, "interval": null, "isNew": true, "legend": { @@ -67210,7 +67411,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 495, + "id": 496, "interval": null, "isNew": true, "legend": { @@ -67343,7 +67544,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 496, + "id": 497, "interval": null, "isNew": true, "legend": { @@ -67476,7 +67677,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 497, + "id": 498, "interval": null, "isNew": true, "legend": { @@ -67612,7 +67813,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 498, + "id": 499, "interval": null, "links": [], "maxDataPoints": 100, @@ -67651,7 +67852,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 499, + "id": 500, "interval": null, "isNew": true, "legend": { @@ -67784,7 +67985,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 500, + "id": 501, "interval": null, "isNew": true, "legend": { @@ -67917,7 +68118,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 501, + "id": 502, "interval": null, "isNew": true, "legend": { @@ -68050,7 +68251,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 502, + "id": 503, "interval": null, "isNew": true, "legend": { @@ -68183,7 +68384,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 503, + "id": 504, "interval": null, "isNew": true, "legend": { @@ -68316,7 +68517,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 504, + "id": 505, "interval": null, "isNew": true, "legend": { @@ -68452,7 +68653,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 505, + "id": 506, "interval": null, "links": [], "maxDataPoints": 100, @@ -68491,7 +68692,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 506, + "id": 507, "interval": null, "isNew": true, "legend": { @@ -68692,7 +68893,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 507, + "id": 508, "interval": null, "isNew": true, "legend": { @@ -68828,7 +69029,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 508, + "id": 509, "interval": null, "links": [], "maxDataPoints": 100, @@ -68867,7 +69068,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 509, + "id": 510, "interval": null, "isNew": true, "legend": { @@ -69000,7 +69201,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 510, + "id": 511, "interval": null, "isNew": true, "legend": { @@ -69133,7 +69334,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 511, + "id": 512, "interval": null, "isNew": true, "legend": { @@ -69266,7 +69467,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 512, + "id": 513, "interval": null, "isNew": true, "legend": { @@ -69399,7 +69600,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 513, + "id": 514, "interval": null, "isNew": true, "legend": { @@ -69547,7 +69748,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 514, + "id": 515, "interval": null, "isNew": true, "legend": { @@ -69751,7 +69952,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 515, + "id": 516, "interval": null, "links": [], "maxDataPoints": 100, @@ -69790,7 +69991,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 516, + "id": 517, "interval": null, "isNew": true, "legend": { @@ -69923,7 +70124,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 517, + "id": 518, "interval": null, "isNew": true, "legend": { @@ -70056,7 +70257,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 518, + "id": 519, "interval": null, "isNew": true, "legend": { @@ -70189,7 +70390,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 519, + "id": 520, "interval": null, "isNew": true, "legend": { @@ -70322,7 +70523,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 520, + "id": 521, "interval": null, "isNew": true, "legend": { @@ -70519,7 +70720,7 @@ }, "height": null, "hideTimeOverride": false, - "id": 521, + "id": 522, "interval": null, "links": [], "maxDataPoints": 100, diff --git a/metrics/grafana/tikv_details.json.sha256 b/metrics/grafana/tikv_details.json.sha256 index 08ce21a5e02..864d0852710 100644 --- a/metrics/grafana/tikv_details.json.sha256 +++ b/metrics/grafana/tikv_details.json.sha256 @@ -1 +1 @@ -3bf63c2b30a553769b0a54757c40e2b47573928b3b8bbee329da306b4b2ea0ca ./metrics/grafana/tikv_details.json +4ecd39c00ee6b32b6673b5be8cbfa252b5b05c61501886262c5f5024e421737f ./metrics/grafana/tikv_details.json diff --git a/src/config/mod.rs b/src/config/mod.rs index 4f00520a9e6..5e49ea5cddf 100644 --- a/src/config/mod.rs +++ b/src/config/mod.rs @@ -1986,6 +1986,34 @@ impl RaftEngineConfig { Ok(()) } + fn optimize_for(&mut self, raft_store: &RaftstoreConfig, raft_kv_v2: bool) { + if raft_kv_v2 { + return; + } + let default_config = RawRaftEngineConfig::default(); + let cur_batch_compression_thd = self.config().batch_compression_threshold; + // Currently, it only takes whether the configuration + // batch-compression-threshold of RaftEngine are set manually + // into consideration to determine whether the RaftEngine is customized. + let customized = cur_batch_compression_thd != default_config.batch_compression_threshold; + // As the async-io is enabled by default (raftstore.store_io_pool_size == 1), + // testing records shows that using 4kb as the default value can achieve + // better performance and reduce the IO overhead. + // Meanwhile, the batch_compression_threshold cannot be modified dynamically if + // the threads count of async-io are changed manually. + if !customized && raft_store.store_io_pool_size > 0 { + let adaptive_batch_comp_thd = RaftEngineReadableSize(std::cmp::max( + cur_batch_compression_thd.0 / (raft_store.store_io_pool_size + 1) as u64, + RaftEngineReadableSize::kb(4).0, + )); + self.mut_config().batch_compression_threshold = adaptive_batch_comp_thd; + warn!( + "raft-engine.batch-compression-threshold {} should be adpative to the size of async-io. Set it to {} instead.", + cur_batch_compression_thd, adaptive_batch_comp_thd, + ); + } + } + pub fn config(&self) -> RawRaftEngineConfig { self.config.clone() } @@ -3694,6 +3722,8 @@ impl TikvConfig { if self.storage.engine == EngineType::RaftKv2 { self.raft_store.store_io_pool_size = cmp::max(self.raft_store.store_io_pool_size, 1); } + self.raft_engine + .optimize_for(&self.raft_store, self.storage.engine == EngineType::RaftKv2); if self.storage.block_cache.capacity.is_none() { let total_mem = SysQuota::memory_limit_in_bytes(); let capacity = if self.storage.engine == EngineType::RaftKv2 { @@ -6932,6 +6962,14 @@ mod tests { default_cfg .raft_store .optimize_for(default_cfg.storage.engine == EngineType::RaftKv2); + default_cfg.raft_engine.optimize_for( + &default_cfg.raft_store, + default_cfg.storage.engine == EngineType::RaftKv2, + ); + assert_eq!( + default_cfg.raft_engine.config().batch_compression_threshold, + RaftEngineReadableSize::kb(4) + ); default_cfg.security.redact_info_log = Some(false); default_cfg.coprocessor.region_max_size = Some(default_cfg.coprocessor.region_max_size()); default_cfg.coprocessor.region_max_keys = Some(default_cfg.coprocessor.region_max_keys()); @@ -7024,6 +7062,10 @@ mod tests { cfg.coprocessor .optimize_for(default_cfg.storage.engine == EngineType::RaftKv2); + cfg.raft_engine.optimize_for( + &cfg.raft_store, + default_cfg.storage.engine == EngineType::RaftKv2, + ); assert_eq_debug(&cfg, &default_cfg); } diff --git a/tests/integrations/config/mod.rs b/tests/integrations/config/mod.rs index cc9dc32348a..2b054a40ed1 100644 --- a/tests/integrations/config/mod.rs +++ b/tests/integrations/config/mod.rs @@ -17,7 +17,9 @@ use engine_traits::PerfLevel; use file_system::{IoPriority, IoRateLimitMode}; use kvproto::encryptionpb::EncryptionMethod; use pd_client::Config as PdConfig; -use raft_log_engine::{ReadableSize as RaftEngineReadableSize, RecoveryMode}; +use raft_log_engine::{ + RaftEngineConfig as RawRaftEngineConfig, ReadableSize as RaftEngineReadableSize, RecoveryMode, +}; use raftstore::{ coprocessor::{Config as CopConfig, ConsistencyCheckMethod}, store::Config as RaftstoreConfig, @@ -978,3 +980,73 @@ engine = "partitioned-raft-kv" let new_cfg: TikvConfig = toml::from_str(new_content).unwrap(); assert_eq_debug(&old_cfg, &new_cfg); } + +#[test] +fn test_raft_engine_compression_thd() { + let default_batch_compression_threshold = + RawRaftEngineConfig::default().batch_compression_threshold; + let disable_async_io_content = r#" + [raftstore] + store-io-pool-size = 0 + + [raft-engine] + batch-compression-threshold = "64KB" + "#; + let mut cfg: TikvConfig = toml::from_str(disable_async_io_content).unwrap(); + assert_eq!( + cfg.raft_engine.config().batch_compression_threshold, + RaftEngineReadableSize::kb(64) + ); + cfg.validate().unwrap(); + assert_eq!( + cfg.raft_engine.config().batch_compression_threshold, + RaftEngineReadableSize::kb(64) + ); + + let async_io_content = r#" + [raftstore] + store-io-pool-size = 3 + + [raft-engine] + batch-compression-threshold = "64KB" + "#; + cfg = toml::from_str(async_io_content).unwrap(); + assert_eq!( + cfg.raft_engine.config().batch_compression_threshold, + RaftEngineReadableSize::kb(64) + ); + cfg.validate().unwrap(); + assert_eq!( + cfg.raft_engine.config().batch_compression_threshold, + RaftEngineReadableSize::kb(64) + ); + + let async_io_content = r#" + [raftstore] + store-io-pool-size = 5 + "#; + cfg = toml::from_str(async_io_content).unwrap(); + assert_eq!( + cfg.raft_engine.config().batch_compression_threshold, + default_batch_compression_threshold + ); + cfg.validate().unwrap(); + assert_eq!( + cfg.raft_engine.config().batch_compression_threshold, + RaftEngineReadableSize::kb(4) + ); + let disable_async_io_content = r#" + [raftstore] + store-io-pool-size = 0 + "#; + let mut cfg: TikvConfig = toml::from_str(disable_async_io_content).unwrap(); + assert_eq!( + cfg.raft_engine.config().batch_compression_threshold, + default_batch_compression_threshold + ); + cfg.validate().unwrap(); + assert_eq!( + cfg.raft_engine.config().batch_compression_threshold, + default_batch_compression_threshold + ); +}