From d26d796837a7868ed47ee577d601c02b1be460f6 Mon Sep 17 00:00:00 2001 From: ZhengQian Date: Wed, 25 Oct 2017 19:55:33 +0800 Subject: [PATCH] update Grafana json (#259) --- etc/DiskPerformance.json | 935 ++++++ etc/Syncer.json | 154 +- etc/pd.json | 353 ++- etc/tidb.json | 6394 +++++++++++++++++++------------------- etc/tikv.json | 5876 +++++++++++++++++++++++++---------- 5 files changed, 8785 insertions(+), 4927 deletions(-) create mode 100644 etc/DiskPerformance.json diff --git a/etc/DiskPerformance.json b/etc/DiskPerformance.json new file mode 100644 index 000000000000..95921b5f4fe0 --- /dev/null +++ b/etc/DiskPerformance.json @@ -0,0 +1,935 @@ +{ + "__inputs": [ + { + "name": "DS_USER-CREDITS", + "label": "user-credits", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "4.1.2" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "text", + "name": "Text", + "version": "" + } + ], + "annotations": { + "list": [] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 1, + "hideControls": true, + "id": null, + "links": [], + "refresh": false, + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "content": "You can click on an individual disk device on the legend to filter on it or multiple ones by holding Alt button.", + "datasource": "${DS_USER-CREDITS}", + "editable": true, + "error": false, + "height": "50px", + "id": 8, + "links": [], + "mode": "text", + "span": 12, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "${DS_USER-CREDITS}", + "decimals": 2, + "description": "Shows average latency for Reads and Writes IO Devices. Higher than typical latency for highly loaded storage indicates saturation (overload) and is frequent cause of performance problems. Higher than normal latency also can indicate internal storage problems.", + "editable": true, + "error": false, + "fill": 2, + "grid": {}, + "id": 11, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sort": null, + "sortDesc": null, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 1, + "points": true, + "renderer": "flot", + "seriesOverrides": [], + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "calculatedInterval": "2m", + "datasourceErrors": {}, + "errors": {}, + "expr": "(rate(node_disk_read_time_ms{device=~\"$device\", instance=\"$host\"}[$interval]) / rate(node_disk_reads_completed{device=~\"$device\", instance=\"$host\"}[$interval])) or (irate(node_disk_read_time_ms{device=~\"$device\", instance=\"$host\"}[5m]) / irate(node_disk_reads_completed{device=~\"$device\", instance=\"$host\"}[5m]))", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Read: {{ device }}", + "metric": "", + "refId": "A", + "step": 300, + "target": "" + }, + { + "calculatedInterval": "2m", + "datasourceErrors": {}, + "errors": {}, + "expr": "(rate(node_disk_write_time_ms{device=~\"$device\", instance=\"$host\"}[$interval]) / rate(node_disk_writes_completed{device=~\"$device\", instance=\"$host\"}[$interval])) or (irate(node_disk_write_time_ms{device=~\"$device\", instance=\"$host\"}[5m]) / irate(node_disk_writes_completed{device=~\"$device\", instance=\"$host\"}[5m]))", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Write: {{ device }}", + "metric": "", + "refId": "B", + "step": 300, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Disk Latency", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "label": "", + "logBase": 2, + "max": null, + "min": 0, + "show": true + }, + { + "format": "ms", + "label": "", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "${DS_USER-CREDITS}", + "decimals": 2, + "description": "Shows amount of physical IOs (reads and writes) different devices are serving. Spikes in number of IOs served often corresponds to performance problems due to IO subsystem overload.", + "editable": true, + "error": false, + "fill": 2, + "grid": {}, + "id": 15, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideEmpty": false, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sort": null, + "sortDesc": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 1, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "calculatedInterval": "2m", + "datasourceErrors": {}, + "errors": {}, + "expr": "rate(node_disk_reads_completed{device=~\"$device\", instance=\"$host\"}[$interval]) or irate(node_disk_reads_completed{device=~\"$device\", instance=\"$host\"}[5m])", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Read: {{ device }}", + "metric": "", + "refId": "A", + "step": 300, + "target": "" + }, + { + "calculatedInterval": "2m", + "datasourceErrors": {}, + "errors": {}, + "expr": "rate(node_disk_writes_completed{device=~\"$device\", instance=\"$host\"}[$interval]) or irate(node_disk_writes_completed{device=~\"$device\", instance=\"$host\"}[5m])", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Write: {{ device }}", + "metric": "", + "refId": "B", + "step": 300, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Disk Operations", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "iops", + "label": "", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "${DS_USER-CREDITS}", + "decimals": 2, + "description": "Shows volume of reads and writes the storage is handling. This can be better measure of IO capacity usage for network attached and SSD storage as it is often bandwidth limited. Amount of data being written to the disk can be used to estimate Flash storage life time.", + "editable": true, + "error": false, + "fill": 2, + "grid": {}, + "id": 16, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideEmpty": false, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sort": null, + "sortDesc": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 1, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "calculatedInterval": "2m", + "datasourceErrors": {}, + "errors": {}, + "expr": "rate(node_disk_bytes_read{device=~\"$device\", instance=\"$host\"}[$interval]) or irate(node_disk_bytes_read{device=~\"$device\", instance=\"$host\"}[5m])", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Read: {{ device }}", + "metric": "", + "refId": "A", + "step": 300, + "target": "" + }, + { + "calculatedInterval": "2m", + "datasourceErrors": {}, + "errors": {}, + "expr": "rate(node_disk_bytes_written{device=~\"$device\", instance=\"$host\"}[$interval]) or irate(node_disk_bytes_written{device=~\"$device\", instance=\"$host\"}[5m])", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Write: {{ device }}", + "metric": "", + "refId": "B", + "step": 300, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Disk Bandwidth", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": "", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "${DS_USER-CREDITS}", + "decimals": 2, + "description": "Shows how much disk was loaded for reads or writes as average number of outstanding requests at different period of time. High disk load is a good measure of actual storage utilization. Different storage types handle load differently - some will show latency increases on low loads others can handle higher load with no problems.", + "editable": true, + "error": false, + "fill": 2, + "grid": {}, + "id": 14, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideEmpty": false, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 1, + "points": true, + "renderer": "flot", + "seriesOverrides": [], + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "calculatedInterval": "2m", + "datasourceErrors": {}, + "errors": {}, + "expr": "rate(node_disk_read_time_ms{device=~\"$device\", instance=\"$host\"}[$interval])/1000 or irate(node_disk_read_time_ms{device=~\"$device\", instance=\"$host\"}[5m])/1000", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Read: {{ device }}", + "metric": "", + "refId": "A", + "step": 300, + "target": "" + }, + { + "calculatedInterval": "2m", + "datasourceErrors": {}, + "errors": {}, + "expr": "rate(node_disk_write_time_ms{device=~\"$device\", instance=\"$host\"}[$interval])/1000 or irate(node_disk_write_time_ms{device=~\"$device\", instance=\"$host\"}[5m])/1000", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Write: {{ device }}", + "metric": "", + "refId": "B", + "step": 300, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Disk Load", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "${DS_USER-CREDITS}", + "decimals": 2, + "description": "Shows disk Utilization as percent of the time when there was at least one IO request in flight. It is designed to match utilization available in iostat tool. It is not very good measure of true IO Capacity Utilization. Consider looking at IO latency and Disk Load Graphs instead.", + "editable": true, + "error": false, + "fill": 2, + "grid": {}, + "id": 17, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideEmpty": false, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 1, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "calculatedInterval": "2m", + "datasourceErrors": {}, + "errors": {}, + "expr": "rate(node_disk_io_time_ms{device=~\"$device\", instance=\"$host\"}[$interval])/1000 or irate(node_disk_io_time_ms{device=~\"$device\", instance=\"$host\"}[5m])/1000", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "{{ device }}", + "metric": "", + "refId": "A", + "step": 300, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Disk IO Utilization", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": "", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "${DS_USER-CREDITS}", + "decimals": 2, + "description": "Shows how effectively Operating System is able to merge logical IO requests into physical requests. This is a good measure of the IO locality which can be used for workload characterization.", + "editable": true, + "error": false, + "fill": 2, + "grid": {}, + "id": 18, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 1, + "points": true, + "renderer": "flot", + "seriesOverrides": [], + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "calculatedInterval": "2m", + "datasourceErrors": {}, + "errors": {}, + "expr": "(1 + rate(node_disk_reads_merged{device=~\"$device\", instance=\"$host\"}[$interval]) / rate(node_disk_reads_completed{device=~\"$device\", instance=\"$host\"}[$interval])) or (1 + irate(node_disk_reads_merged{device=~\"$device\", instance=\"$host\"}[5m]) / irate(node_disk_reads_completed{device=~\"$device\", instance=\"$host\"}[5m]))", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Read Ratio: {{ device }}", + "metric": "", + "refId": "A", + "step": 300, + "target": "" + }, + { + "calculatedInterval": "2m", + "datasourceErrors": {}, + "errors": {}, + "expr": "(1 + rate(node_disk_writes_merged{device=~\"$device\", instance=\"$host\"}[$interval]) / rate(node_disk_writes_completed{device=~\"$device\", instance=\"$host\"}[$interval])) or (1 + irate(node_disk_writes_merged{device=~\"$device\", instance=\"$host\"}[5m]) / irate(node_disk_writes_completed{device=~\"$device\", instance=\"$host\"}[5m]))", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Write Ratio: {{ device }}", + "metric": "", + "refId": "B", + "step": 300, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Disk Operations Merge Ratio", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + "Read IO size: sdb": "#2F575E", + "Read: sdb": "#3F6833" + }, + "bars": false, + "datasource": "${DS_USER-CREDITS}", + "decimals": 2, + "description": "Shows average size of a single disk operation.", + "editable": true, + "error": false, + "fill": 2, + "grid": {}, + "id": 20, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 1, + "points": true, + "renderer": "flot", + "seriesOverrides": [], + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "calculatedInterval": "2m", + "datasourceErrors": {}, + "errors": {}, + "expr": "rate(node_disk_sectors_read{instance=\"$host\", device=~\"$device\"}[$interval]) * 512 / rate(node_disk_reads_completed{instance=\"$host\", device=~\"$device\"}[$interval]) or irate(node_disk_sectors_read{instance=\"$host\", device=~\"$device\"}[5m]) * 512 / irate(node_disk_reads_completed{instance=\"$host\", device=~\"$device\"}[5m]) ", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Read size: {{ device }}", + "metric": "", + "refId": "A", + "step": 300, + "target": "" + }, + { + "calculatedInterval": "2m", + "datasourceErrors": {}, + "errors": {}, + "expr": "rate(node_disk_sectors_written{instance=\"$host\", device=~\"$device\"}[$interval]) * 512 / rate(node_disk_writes_completed{instance=\"$host\", device=~\"$device\"}[$interval]) or irate(node_disk_sectors_written{instance=\"$host\", device=~\"$device\"}[5m]) * 512 / irate(node_disk_writes_completed{instance=\"$host\", device=~\"$device\"}[5m]) ", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Write size: {{ device }}", + "metric": "", + "refId": "B", + "step": 300, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Disk IO Size", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Disk Stats", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allFormat": "glob", + "auto": true, + "auto_count": 200, + "auto_min": "1s", + "current": { + "text": "auto", + "value": "$__auto_interval" + }, + "datasource": "Prometheus", + "hide": 0, + "includeAll": false, + "label": "Interval", + "multi": false, + "multiFormat": "glob", + "name": "interval", + "options": [ + { + "selected": true, + "text": "auto", + "value": "$__auto_interval" + }, + { + "selected": false, + "text": "1s", + "value": "1s" + }, + { + "selected": false, + "text": "5s", + "value": "5s" + }, + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": false, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": false, + "text": "1d", + "value": "1d" + } + ], + "query": "1s,5s,1m,5m,1h,6h,1d", + "refresh": 2, + "type": "interval" + }, + { + "allFormat": "glob", + "allValue": null, + "current": {}, + "datasource": "${DS_USER-CREDITS}", + "hide": 0, + "includeAll": false, + "label": "Host", + "multi": false, + "multiFormat": "regex values", + "name": "host", + "options": [], + "query": "label_values(node_disk_reads_completed, instance)", + "refresh": 1, + "refresh_on_load": false, + "regex": "", + "sort": 1, + "tagValuesQuery": "instance", + "tags": [], + "tagsQuery": "up", + "type": "query", + "useTags": false + }, + { + "allFormat": "glob", + "allValue": null, + "current": {}, + "datasource": "${DS_USER-CREDITS}", + "hide": 0, + "includeAll": true, + "label": "Device", + "multi": true, + "multiFormat": "regex values", + "name": "device", + "options": [], + "query": "label_values(node_disk_reads_completed{instance=\"$host\", device!~\"dm-.+\"}, device)", + "refresh": 1, + "refresh_on_load": false, + "regex": "", + "sort": 1, + "tagValuesQuery": "instance", + "tags": [], + "tagsQuery": "up", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-12h", + "to": "now" + }, + "timepicker": { + "collapse": false, + "enable": true, + "notice": false, + "now": true, + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "status": "Stable", + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ], + "type": "timepicker" + }, + "timezone": "browser", + "title": "Disk Performance", + "version": 1 +} \ No newline at end of file diff --git a/etc/Syncer.json b/etc/Syncer.json index 207ac9fd201e..db8ef34108af 100644 --- a/etc/Syncer.json +++ b/etc/Syncer.json @@ -1,8 +1,8 @@ { "__inputs": [ { - "name": "DS_Syncer", - "label": "Syncer", + "name": "DS_BIGDATA-CLUSTER", + "label": "bigdata-cluster", "description": "", "type": "datasource", "pluginId": "prometheus", @@ -47,17 +47,21 @@ { "aliasColors": {}, "bars": false, - "datasource": "${DS_Syncer}", + "datasource": "${DS_BIGDATA-CLUSTER}", "fill": 1, "id": 1, "legend": { + "alignAsTable": true, "avg": false, - "current": false, - "max": false, + "current": true, + "max": true, "min": false, + "rightSide": true, "show": true, + "sort": "current", + "sortDesc": true, "total": false, - "values": false + "values": true }, "lines": true, "linewidth": 1, @@ -75,9 +79,10 @@ { "expr": "irate(syncer_binlog_events_total[1m])", "intervalFactor": 2, + "legendFormat": "{{job}} - {{type}}", "metric": "syncer_binlog_events_total", "refId": "A", - "step": 2 + "step": 20 } ], "thresholds": [], @@ -130,15 +135,19 @@ { "aliasColors": {}, "bars": false, - "datasource": "${DS_Syncer}", + "datasource": "${DS_BIGDATA-CLUSTER}", "fill": 1, "id": 2, "legend": { + "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, + "rightSide": false, "show": true, + "sort": "current", + "sortDesc": false, "total": false, "values": false }, @@ -156,11 +165,19 @@ "steppedLine": false, "targets": [ { - "expr": "syncer_binlog_pos", + "expr": "syncer_binlog_pos{node=\"syncer\"}", "intervalFactor": 2, - "metric": "syncer_binlog_meta_pos", + "legendFormat": "{{job}} {{node}}", + "metric": "", "refId": "A", - "step": 2 + "step": 30 + }, + { + "expr": "syncer_binlog_pos{node=\"master\"}", + "intervalFactor": 2, + "legendFormat": "{{job}} {{node}}", + "refId": "B", + "step": 30 } ], "thresholds": [], @@ -201,7 +218,7 @@ { "aliasColors": {}, "bars": false, - "datasource": "${DS_Syncer}", + "datasource": "${DS_BIGDATA-CLUSTER}", "fill": 1, "id": 4, "legend": { @@ -227,11 +244,18 @@ "steppedLine": false, "targets": [ { - "expr": "syncer_binlog_file", + "expr": "syncer_binlog_file{node=\"master\"}", "intervalFactor": 2, - "legendFormat": "", + "legendFormat": "{{job}} {{node}}", "refId": "A", - "step": 2 + "step": 30 + }, + { + "expr": "syncer_binlog_file{node=\"syncer\"}", + "intervalFactor": 2, + "legendFormat": "{{job}} {{node}}", + "refId": "B", + "step": 30 } ], "thresholds": [], @@ -284,7 +308,7 @@ { "aliasColors": {}, "bars": false, - "datasource": "${DS_Syncer}", + "datasource": "${DS_BIGDATA-CLUSTER}", "decimals": null, "fill": 1, "id": 5, @@ -319,7 +343,7 @@ "intervalFactor": 2, "legendFormat": "{{job}}", "refId": "A", - "step": 2 + "step": 20 } ], "thresholds": [], @@ -387,9 +411,9 @@ "model": { "expr": " syncer_binlog_file{node=\"master\"} - ON(instance, job) syncer_binlog_file{node=\"syncer\"} ", "intervalFactor": 10, - "legendFormat": "", + "legendFormat": "{{job}}", "refId": "A", - "step": 10 + "step": 50 }, "params": [ "A", @@ -409,21 +433,29 @@ "handler": 1, "name": "syncer_binlog_file alert", "noDataState": "no_data", - "notifications": [] + "notifications": [ + { + "id": 1 + } + ] }, "aliasColors": {}, "bars": false, - "datasource": "${DS_Syncer}", + "datasource": "${DS_BIGDATA-CLUSTER}", "fill": 1, "id": 6, "legend": { + "alignAsTable": true, "avg": false, - "current": false, - "max": false, + "current": true, + "max": true, "min": false, + "rightSide": true, "show": true, + "sort": "max", + "sortDesc": true, "total": false, - "values": false + "values": true }, "lines": true, "linewidth": 1, @@ -441,9 +473,9 @@ { "expr": " syncer_binlog_file{node=\"master\"} - ON(instance, job) syncer_binlog_file{node=\"syncer\"} ", "intervalFactor": 10, - "legendFormat": "", + "legendFormat": "{{job}}", "refId": "A", - "step": 10 + "step": 100 } ], "thresholds": [ @@ -504,17 +536,19 @@ { "aliasColors": {}, "bars": false, - "datasource": "${DS_Syncer}", + "datasource": "${DS_BIGDATA-CLUSTER}", "fill": 1, "id": 3, "legend": { + "alignAsTable": true, "avg": false, - "current": false, - "max": false, + "current": true, + "max": true, "min": false, + "rightSide": true, "show": true, "total": false, - "values": false + "values": true }, "lines": true, "linewidth": 1, @@ -532,9 +566,10 @@ { "expr": "irate(syncer_binlog_skipped_events_total[1m])", "intervalFactor": 2, + "legendFormat": "{{job}} {{type}}", "metric": "syncer_binlog_skipped_events_total", "refId": "A", - "step": 2 + "step": 20 } ], "thresholds": [], @@ -585,9 +620,46 @@ "height": 250, "panels": [ { + "alert": { + "conditions": [ + { + "evaluator": { + "params": [ + 20 + ], + "type": "gt" + }, + "operator": { + "type": "and" + }, + "query": { + "params": [ + "A", + "5m", + "now" + ] + }, + "reducer": { + "params": [], + "type": "avg" + }, + "type": "query" + } + ], + "executionErrorState": "alerting", + "frequency": "60s", + "handler": 1, + "name": "syncer_txn_costs_gauge_in_second alert", + "noDataState": "no_data", + "notifications": [ + { + "id": 1 + } + ] + }, "aliasColors": {}, "bars": false, - "datasource": "${DS_Syncer}", + "datasource": "${DS_BIGDATA-CLUSTER}", "fill": 1, "id": 7, "legend": { @@ -600,6 +672,8 @@ "min": false, "rightSide": true, "show": true, + "sort": "max", + "sortDesc": true, "total": false, "values": true }, @@ -621,10 +695,18 @@ "intervalFactor": 2, "legendFormat": "{{job}}", "refId": "A", - "step": 2 + "step": 20 + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 20 } ], - "thresholds": [], "timeFrom": null, "timeShift": null, "title": "syncer_txn_costs_gauge_in_second", @@ -675,7 +757,7 @@ "list": [] }, "time": { - "from": "now-5m", + "from": "now-3h", "to": "now" }, "timepicker": { @@ -705,5 +787,5 @@ }, "timezone": "browser", "title": "Syncer", - "version": 20 + "version": 24 } \ No newline at end of file diff --git a/etc/pd.json b/etc/pd.json index 2ad4e46ae55c..02da3d131766 100644 --- a/etc/pd.json +++ b/etc/pd.json @@ -9,6 +9,87 @@ "height": "300px", "repeatRowId": null, "panels": [ + { + "id": 55, + "title": "PD Role", + "span": 2, + "type": "singlestat", + "targets": [ + { + "refId": "A", + "expr": "delta(pd_server_tso{type=\"save\",instance=\"$instance\"}[15s])", + "intervalFactor": 2, + "metric": "pd_server_tso", + "step": 60, + "legendFormat": "" + } + ], + "links": [], + "datasource": "${DS_TIDB-CLUSTER}", + "maxDataPoints": 100, + "interval": null, + "cacheTimeout": null, + "format": "none", + "prefix": "", + "postfix": "", + "nullText": null, + "valueMaps": [ + { + "value": "null", + "op": "=", + "text": "N/A" + } + ], + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "rangeMaps": [ + { + "from": "1", + "to": "100000", + "text": "Leader" + }, + { + "from": "0", + "to": "1", + "text": "Follower" + } + ], + "mappingType": 2, + "nullPointMode": "connected", + "valueName": "current", + "prefixFontSize": "50%", + "valueFontSize": "50%", + "postfixFontSize": "50%", + "thresholds": "", + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "sparkline": { + "show": false, + "full": false, + "lineColor": "rgb(31, 120, 193)", + "fillColor": "rgba(31, 118, 189, 0.18)" + }, + "gauge": { + "show": false, + "minValue": 0, + "maxValue": 100, + "thresholdMarkers": true, + "thresholdLabels": false + } + }, { "mappingTypes": [ { @@ -60,7 +141,7 @@ "targets": [ { "intervalFactor": 2, - "expr": "pd_cluster_status{instance=\"$instance\",type=\"storage_capacity\"}", + "expr": "sum(pd_cluster_status{instance=\"$instance\",namespace=~\"$namespace\",type=\"storage_capacity\"})", "step": 60, "refId": "A" } @@ -137,7 +218,7 @@ "targets": [ { "intervalFactor": 2, - "expr": "pd_cluster_status{instance=\"$instance\",type=\"storage_size\"}", + "expr": "sum(pd_cluster_status{instance=\"$instance\",namespace=~\"$namespace\",type=\"storage_size\"})", "step": 60, "refId": "A" } @@ -215,7 +296,7 @@ "targets": [ { "intervalFactor": 2, - "expr": "pd_cluster_status{instance=\"$instance\",type=\"region_count\"}", + "expr": "sum(pd_cluster_status{instance=\"$instance\",namespace=~\"$namespace\",type=\"region_count\"})", "step": 60, "refId": "A" } @@ -279,7 +360,7 @@ "id": 37, "maxDataPoints": 100, "mappingType": 1, - "span": 2, + "span": 1, "colorBackground": false, "title": "Leader Balance Ratio", "sparkline": { @@ -291,7 +372,7 @@ "targets": [ { "intervalFactor": 2, - "expr": "holt_winters(pd_cluster_status{instance=\"$instance\",type=\"leader_balance_ratio\"}[5m], 0.8, 0.1)", + "expr": "1 - min(pd_scheduler_balance_score{instance=\"$instance\", namespace=~\"$namespace\", type=\"leader\"}) / max(pd_scheduler_balance_score{instance=\"$instance\", namespace=~\"$namespace\", type=\"leader\"})", "step": 60, "refId": "A" } @@ -356,7 +437,7 @@ "id": 36, "maxDataPoints": 100, "mappingType": 1, - "span": 2, + "span": 1, "colorBackground": false, "title": "Region Balance Ratio", "sparkline": { @@ -368,7 +449,7 @@ "targets": [ { "intervalFactor": 2, - "expr": "holt_winters(pd_cluster_status{instance=\"$instance\",type=\"region_balance_ratio\"}[5m], 0.2, 0.1)", + "expr": "1 - min(pd_scheduler_balance_score{instance=\"$instance\", namespace=~\"$namespace\", type=\"region\"}) / max(pd_scheduler_balance_score{instance=\"$instance\", namespace=~\"$namespace\", type=\"region\"})", "step": 60, "refId": "A", "legendFormat": "" @@ -435,7 +516,7 @@ "scroll": false, "targets": [ { - "expr": "pd_cluster_status{instance=\"$instance\",type=\"store_up_count\"}", + "expr": "sum(pd_cluster_status{instance=\"$instance\", namespace=~\"$namespace\", type=\"store_up_count\"})", "metric": "pd_cluster_status", "interval": "", "step": 20, @@ -445,29 +526,43 @@ }, { "intervalFactor": 2, - "expr": "pd_cluster_status{instance=\"$instance\",type=\"store_down_count\"}", + "expr": "sum(pd_cluster_status{instance=\"$instance\", namespace=~\"$namespace\",type=\"store_disconnected_count\"})", "step": 20, "refId": "B", - "legendFormat": "Down Stores" + "legendFormat": "Disconnect Stores" }, { "intervalFactor": 2, - "expr": "pd_cluster_status{instance=\"$instance\",type=\"store_offline_count\"}", + "expr": "sum(pd_cluster_status{instance=\"$instance\", namespace=~\"$namespace\",type=\"store_low_space_count\"})", "step": 20, "refId": "C", - "legendFormat": "Offline Stores" + "legendFormat": "LowSpace Stores" }, { "intervalFactor": 2, - "expr": "pd_cluster_status{instance=\"$instance\",type=\"store_tombstone_count\"}", + "expr": "sum(pd_cluster_status{instance=\"$instance\", namespace=~\"$namespace\",type=\"store_down_count\"})", "step": 20, "refId": "D", + "legendFormat": "Down Stores" + }, + { + "intervalFactor": 2, + "expr": "sum(pd_cluster_status{instance=\"$instance\", namespace=~\"$namespace\",type=\"store_offline_count\"})", + "step": 20, + "refId": "E", + "legendFormat": "Offline Stores" + }, + { + "intervalFactor": 2, + "expr": "sum(pd_cluster_status{instance=\"$instance\", namespace=~\"$namespace\",type=\"store_tombstone_count\"})", + "step": 20, + "refId": "F", "legendFormat": "Tombstone Stores" } ], "transparent": false, "hideTimeOverride": false, - "fontSize": "120%", + "fontSize": "100%", "datasource": "${DS_TIDB-CLUSTER}", "error": false, "type": "table", @@ -518,7 +613,7 @@ "targets": [ { "hide": false, - "expr": "pd_cluster_status{instance=\"$instance\",type=\"storage_size\"}", + "expr": "pd_cluster_status{instance=\"$instance\", namespace=~\"$namespace\", type=\"storage_size\"}", "step": 10, "legendFormat": "strage size", "intervalFactor": 2, @@ -526,7 +621,7 @@ }, { "hide": false, - "expr": "avg(pd_cluster_status{type=\"storage_size\"}) / avg(pd_cluster_status{type=\"storage_capacity\"})", + "expr": "avg(pd_cluster_status{type=\"storage_size\", namespace=~\"$namespace\"}) / avg(pd_cluster_status{type=\"storage_capacity\", namespace=~\"$namespace\"})", "step": 20, "legendFormat": "used ratio", "intervalFactor": 4, @@ -651,7 +746,7 @@ "targets": [ { "intervalFactor": 2, - "expr": "pd_cluster_status{instance=\"$instance\",type=\"region_count\"}", + "expr": "sum(pd_cluster_status{instance=\"$instance\", namespace=~\"$namespace\", type=\"region_count\"})", "step": 10, "refId": "A", "legendFormat": "count" @@ -709,7 +804,7 @@ "targets": [ { "intervalFactor": 2, - "expr": "sum(delta(pd_schedule_operators_count{instance=\"$instance\",state=\"finished\"}[1m])) by (type,state)", + "expr": "sum(delta(pd_schedule_operators_count{instance=\"$instance\"}[1m])) by (type)", "step": 10, "refId": "A", "legendFormat": "{{type}}" @@ -794,7 +889,7 @@ "id": 40, "fill": 1, "span": 6, - "title": "Leader Balance Ratio", + "title": "Store leader score", "tooltip": { "sort": 0, "shared": true, @@ -802,29 +897,23 @@ "msResolution": false }, "legend": { + "rightSide": true, + "alignAsTable": true, "total": false, "show": false, - "max": false, - "min": false, - "current": false, + "max": true, + "min": true, + "current": true, "values": false, "avg": false }, "targets": [ { "intervalFactor": 2, - "expr": "pd_cluster_status{instance=\"$instance\",type=\"leader_balance_ratio\"}", + "expr": "pd_scheduler_balance_score{instance=\"$instance\", namespace=~\"$namespace\", type=\"leader\"}", "step": 10, "refId": "A", - "legendFormat": "leader max diff ratio" - }, - { - "hide": true, - "expr": "min(pd_cluster_status{type=\"leader_balance_ratio\"})", - "step": 4, - "legendFormat": "ratio", - "intervalFactor": 2, - "refId": "B" + "legendFormat": "tikv-{{store}}" } ], "yaxes": [ @@ -856,49 +945,6 @@ "type": "graph", "error": false, "editable": true, - "alert": { - "noDataState": "no_data", - "name": "Leader Banlace Ratio alert", - "frequency": "60s", - "notifications": [], - "handler": 1, - "executionErrorState": "alerting", - "message": "Region leader balance ratio is high.", - "conditions": [ - { - "operator": { - "type": "and" - }, - "query": { - "params": [ - "B", - "1m", - "now" - ], - "model": { - "hide": true, - "expr": "min(pd_cluster_status{type=\"leader_balance_ratio\"})", - "step": 4, - "legendFormat": "ratio", - "intervalFactor": 2, - "refId": "B" - }, - "datasourceId": 1 - }, - "evaluator": { - "type": "gt", - "params": [ - 0.2 - ] - }, - "reducer": { - "type": "avg", - "params": [] - }, - "type": "query" - } - ] - }, "grid": {}, "stack": false, "timeShift": null, @@ -929,7 +975,7 @@ "id": 41, "fill": 1, "span": 6, - "title": "Region Balance Ratio", + "title": "Store region score", "tooltip": { "sort": 0, "shared": true, @@ -937,6 +983,8 @@ "msResolution": false }, "legend": { + "rightSide": true, + "alignAsTable": true, "total": false, "show": false, "max": false, @@ -948,19 +996,11 @@ "targets": [ { "hide": false, - "expr": "pd_cluster_status{instance=\"$instance\",type=\"region_balance_ratio\"}", + "expr": "pd_scheduler_balance_score{instance=\"$instance\", namespace=~\"$namespace\", type=\"region\"}", "step": 10, "legendFormat": "\bstore balance ratio", "intervalFactor": 2, "refId": "A" - }, - { - "hide": true, - "expr": "min(pd_cluster_status{type=\"region_balance_ratio\"})", - "step": 4, - "legendFormat": "ratio", - "intervalFactor": 2, - "refId": "B" } ], "yaxes": [ @@ -992,49 +1032,6 @@ "type": "graph", "error": false, "editable": true, - "alert": { - "noDataState": "no_data", - "name": "Region Balance Ratio alert", - "frequency": "60s", - "notifications": [], - "handler": 1, - "executionErrorState": "alerting", - "message": "Store balance ratio is high", - "conditions": [ - { - "operator": { - "type": "and" - }, - "query": { - "params": [ - "B", - "5m", - "now" - ], - "model": { - "hide": true, - "expr": "min(pd_cluster_status{type=\"region_balance_ratio\"})", - "step": 4, - "legendFormat": "ratio", - "intervalFactor": 2, - "refId": "B" - }, - "datasourceId": 1 - }, - "evaluator": { - "type": "gt", - "params": [ - 0.2 - ] - }, - "reducer": { - "type": "avg", - "params": [] - }, - "type": "query" - } - ] - }, "grid": {}, "stack": false, "timeShift": null, @@ -3129,6 +3126,86 @@ "points": false, "datasource": "${DS_TIDB-CLUSTER}", "pointradius": 5 + }, + { + "bars": false, + "timeFrom": null, + "links": [], + "thresholds": [], + "nullPointMode": "null as zero", + "renderer": "flot", + "id": 54, + "linewidth": 1, + "steppedLine": false, + "targets": [ + { + "hide": false, + "expr": "sum(rate(pd_scheduler_region_heartbeat{instance=\"$instance\"}[1m])) by (store, type, status)", + "step": 4, + "legendFormat": "store{{store}}-{{type}}-{{status}}", + "intervalFactor": 2, + "refId": "A" + } + ], + "fill": 0, + "span": 6, + "title": "Region heartbeat", + "tooltip": { + "sort": 0, + "shared": true, + "value_type": "individual", + "msResolution": false + }, + "legend": { + "rightSide": true, + "total": false, + "min": false, + "max": true, + "show": true, + "current": false, + "hideEmpty": true, + "values": true, + "alignAsTable": true, + "avg": false, + "hideZero": true + }, + "yaxes": [ + { + "logBase": 1, + "show": true, + "max": null, + "format": "ops", + "min": "0", + "label": null + }, + { + "logBase": 1, + "show": true, + "max": null, + "format": "s", + "min": null, + "label": null + } + ], + "xaxis": { + "show": true, + "values": [], + "mode": "time", + "name": null + }, + "seriesOverrides": [], + "percentage": false, + "type": "graph", + "error": false, + "editable": true, + "grid": {}, + "stack": false, + "timeShift": null, + "aliasColors": {}, + "lines": true, + "points": false, + "datasource": "${DS_TIDB-CLUSTER}", + "pointradius": 5 } ], "showTitle": true, @@ -3453,6 +3530,26 @@ "tagsQuery": null, "options": [], "includeAll": false + }, + { + "allValue": ".*", + "current": {}, + "datasource": "${DS_TIDB-CLUSTER}", + "hide": 0, + "includeAll": true, + "label": "Namespace", + "multi": false, + "name": "namespace", + "options": [], + "query": "label_values(pd_cluster_status{instance=\"$instance\"}, namespace)", + "refresh": 1, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false } ] } diff --git a/etc/tidb.json b/etc/tidb.json index 68be7fbcce84..7d722d5d49b6 100644 --- a/etc/tidb.json +++ b/etc/tidb.json @@ -1,3463 +1,3629 @@ { - "style": "dark", + "__inputs": [ + { + "name": "DS_TEST-CLUSTER", + "label": "test-cluster", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "4.1.2" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + } + ], + "annotations": { + "list": [] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + { + "icon": "external link", + "tags": [], + "type": "dashboards" + } + ], + "refresh": "30s", "rows": [ { - "repeat": null, - "titleSize": "h6", - "repeatIteration": null, - "title": "Query", - "height": "240", - "repeatRowId": null, + "collapse": false, + "height": "240", "panels": [ { - "bars": false, - "timeFrom": null, - "links": [], - "thresholds": [ - { - "colorMode": "critical", - "line": true, - "op": "gt", - "value": 1, - "fill": true - } - ], - "nullPointMode": "null as zero", - "renderer": "flot", - "linewidth": 1, - "steppedLine": false, - "id": 23, - "fill": 1, - "span": 4, - "title": "Query Duration 80th percentile", - "tooltip": { - "sort": 0, - "shared": true, - "value_type": "cumulative", - "msResolution": true - }, - "legend": { - "total": false, - "show": true, - "max": false, - "min": false, - "current": false, - "values": false, - "avg": false - }, - "targets": [ - { - "intervalFactor": 2, - "expr": "histogram_quantile(0.80, sum(rate(tidb_server_handle_query_duration_seconds_bucket[1m])) by (le))", - "step": 60, - "refId": "A", - "legendFormat": "average" - }, - { - "intervalFactor": 2, - "expr": "histogram_quantile(0.80, sum(rate(tidb_server_handle_query_duration_seconds_bucket[1m])) by (le, instance))", - "step": 60, - "refId": "B", - "legendFormat": "{{instance}}" - } - ], - "yaxes": [ - { - "logBase": 1, - "show": true, - "max": null, - "format": "s", - "min": 0, - "label": null - }, - { - "logBase": 1, - "show": false, - "max": null, - "format": "short", - "min": null, - "label": null - } - ], - "xaxis": { - "show": true, - "values": [], - "mode": "time", - "name": null - }, - "seriesOverrides": [], - "percentage": false, - "type": "graph", - "error": false, - "editable": true, "alert": { - "noDataState": "no_data", - "name": "Query Seconds 80 alert", - "frequency": "60s", - "notifications": [], - "handler": 1, - "executionErrorState": "alerting", - "message": "Query \u5904\u7406\u65f6\u95f4\u5f02\u5e38\uff01", "conditions": [ { + "evaluator": { + "params": [ + 1 + ], + "type": "gt" + }, "operator": { "type": "and" - }, + }, "query": { - "params": [ - "A", - "10s", - "now" - ], + "datasourceId": 1, "model": { - "intervalFactor": 2, - "expr": "histogram_quantile(0.80, sum(rate(tidb_server_handle_query_duration_seconds_bucket[1m])) by (le))", - "step": 60, - "refId": "A", - "legendFormat": "average" - }, - "datasourceId": 1 - }, - "evaluator": { - "type": "gt", + "expr": "histogram_quantile(0.80, sum(rate(tidb_server_handle_query_duration_seconds_bucket[1m])) by (le))", + "intervalFactor": 2, + "legendFormat": "average", + "refId": "A", + "step": 60 + }, "params": [ - 1 + "A", + "10s", + "now" ] - }, + }, "reducer": { - "type": "avg", - "params": [] - }, + "params": [], + "type": "avg" + }, "type": "query" } - ] - }, - "grid": {}, - "stack": false, - "timeShift": null, - "aliasColors": {}, - "lines": true, - "points": false, - "datasource": "${DS_TIDB-CLUSTER}", - "pointradius": 5 - }, - { - "bars": false, - "timeFrom": null, - "links": [], - "thresholds": [ - { - "colorMode": "critical", - "line": true, - "op": "gt", - "value": 1, - "fill": true - } - ], - "nullPointMode": "null as zero", - "renderer": "flot", - "linewidth": 1, - "steppedLine": false, - "id": 1, - "fill": 1, - "span": 4, - "title": "Query Duration 95th percentile", - "tooltip": { - "sort": 0, - "shared": false, - "value_type": "cumulative", - "msResolution": true - }, + ], + "executionErrorState": "alerting", + "frequency": "60s", + "handler": 1, + "message": "Query 处理时间异常！", + "name": "Query Seconds 80 alert", + "noDataState": "no_data", + "notifications": [] + }, + "aliasColors": {}, + "bars": false, + "datasource": "${DS_TEST-CLUSTER}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 23, "legend": { - "total": false, - "min": false, - "max": false, - "show": true, - "current": false, - "hideEmpty": true, - "values": false, - "avg": false, - "hideZero": true - }, + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, "targets": [ { - "intervalFactor": 2, - "expr": "histogram_quantile(0.95, sum(rate(tidb_server_handle_query_duration_seconds_bucket[1m])) by (le))", - "step": 60, - "refId": "A", - "legendFormat": "average" - }, + "expr": "histogram_quantile(0.80, sum(rate(tidb_server_handle_query_duration_seconds_bucket[1m])) by (le))", + "intervalFactor": 2, + "legendFormat": "average", + "refId": "A", + "step": 60 + }, { - "intervalFactor": 2, - "expr": "histogram_quantile(0.95, sum(rate(tidb_server_handle_query_duration_seconds_bucket[1m])) by (le, instance))", - "step": 60, - "refId": "B", - "legendFormat": "{{ instance }}" + "expr": "histogram_quantile(0.80, sum(rate(tidb_server_handle_query_duration_seconds_bucket[1m])) by (le, instance))", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "B", + "step": 60 } - ], - "yaxes": [ + ], + "thresholds": [ { - "logBase": 1, - "show": true, - "max": null, - "format": "s", - "min": "0", - "label": null - }, - { - "logBase": 1, - "show": false, - "max": null, - "format": "short", - "min": null, - "label": null - } - ], + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 1 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Query Duration 80th percentile", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", "xaxis": { - "show": true, - "values": [ - "max" - ], - "mode": "time", - "name": null - }, - "seriesOverrides": [], - "percentage": false, - "type": "graph", - "transparent": false, - "error": false, - "editable": true, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { "alert": { - "noDataState": "no_data", - "name": "Query Duration 95th percentile alert", - "frequency": "60s", - "notifications": [], - "handler": 1, - "executionErrorState": "alerting", - "message": "Query duration at 95th percentile is high.", "conditions": [ { + "evaluator": { + "params": [ + 1 + ], + "type": "gt" + }, "operator": { "type": "and" - }, + }, "query": { - "params": [ - "A", - "10s", - "now" - ], + "datasourceId": 1, "model": { - "intervalFactor": 2, - "expr": "histogram_quantile(0.95, sum(rate(tidb_server_handle_query_duration_seconds_bucket[1m])) by (le))", - "step": 60, - "refId": "A", - "legendFormat": "average" - }, - "datasourceId": 1 - }, - "evaluator": { - "type": "gt", + "expr": "histogram_quantile(0.95, sum(rate(tidb_server_handle_query_duration_seconds_bucket[1m])) by (le))", + "intervalFactor": 2, + "legendFormat": "average", + "refId": "A", + "step": 60 + }, "params": [ - 1 + "A", + "10s", + "now" ] - }, + }, "reducer": { - "type": "avg", - "params": [] - }, + "params": [], + "type": "avg" + }, "type": "query" } - ] - }, - "grid": {}, - "stack": false, - "timeShift": null, - "aliasColors": {}, - "lines": true, - "points": false, - "datasource": "${DS_TIDB-CLUSTER}", - "pointradius": 5 - }, - { - "bars": false, - "timeFrom": null, - "links": [], - "thresholds": [ - { - "colorMode": "critical", - "line": true, - "op": "gt", - "value": 10, - "fill": true - } - ], - "nullPointMode": "null as zero", - "renderer": "flot", - "linewidth": 1, - "steppedLine": false, - "id": 25, - "fill": 1, - "span": 4, - "title": "Query Duration 99th percentile", - "tooltip": { - "sort": 0, - "shared": true, - "value_type": "cumulative", - "msResolution": true - }, + ], + "executionErrorState": "alerting", + "frequency": "60s", + "handler": 1, + "message": "Query duration at 95th percentile is high.", + "name": "Query Duration 95th percentile alert", + "noDataState": "no_data", + "notifications": [] + }, + "aliasColors": {}, + "bars": false, + "datasource": "${DS_TEST-CLUSTER}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 1, "legend": { - "total": false, - "show": true, - "max": false, - "min": false, - "current": false, - "values": false, - "avg": false - }, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, "targets": [ { - "intervalFactor": 2, - "expr": "histogram_quantile(0.99, sum(rate(tidb_server_handle_query_duration_seconds_bucket[1m])) by (le))", - "step": 60, - "refId": "B", - "legendFormat": "average" - }, + "expr": "histogram_quantile(0.95, sum(rate(tidb_server_handle_query_duration_seconds_bucket[1m])) by (le))", + "intervalFactor": 2, + "legendFormat": "average", + "refId": "A", + "step": 60 + }, { - "intervalFactor": 2, - "expr": "histogram_quantile(0.99, sum(rate(tidb_server_handle_query_duration_seconds_bucket[1m])) by (le, instance))", - "step": 60, - "refId": "A", - "legendFormat": "{{instance}}" + "expr": "histogram_quantile(0.95, sum(rate(tidb_server_handle_query_duration_seconds_bucket[1m])) by (le, instance))", + "intervalFactor": 2, + "legendFormat": "{{ instance }}", + "refId": "B", + "step": 60 } - ], - "yaxes": [ + ], + "thresholds": [ { - "logBase": 1, - "show": true, - "max": null, - "format": "s", - "min": "0", - "label": null - }, - { - "logBase": 1, - "show": true, - "max": null, - "format": "short", - "min": null, - "label": null - } - ], + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 1 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Query Duration 95th percentile", + "tooltip": { + "msResolution": true, + "shared": false, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", "xaxis": { - "show": true, - "values": [], - "mode": "time", - "name": null - }, - "seriesOverrides": [], - "percentage": false, - "type": "graph", - "error": false, - "editable": true, + "mode": "time", + "name": null, + "show": true, + "values": [ + "max" + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { "alert": { - "noDataState": "no_data", - "name": "Query Duration 99th percentile alert", - "frequency": "60s", - "notifications": [], - "handler": 1, - "executionErrorState": "alerting", - "message": "Query duration for 99th percentile is high.", "conditions": [ { + "evaluator": { + "params": [ + 10 + ], + "type": "gt" + }, "operator": { "type": "and" - }, + }, "query": { - "params": [ - "A", - "1m", - "now" - ], + "datasourceId": 1, "model": { - "intervalFactor": 2, - "expr": "histogram_quantile(0.99, sum(rate(tidb_server_handle_query_duration_seconds_bucket[1m])) by (le, instance))", - "step": 60, - "refId": "A", - "legendFormat": "{{instance}}" - }, - "datasourceId": 1 - }, - "evaluator": { - "type": "gt", + "expr": "histogram_quantile(0.99, sum(rate(tidb_server_handle_query_duration_seconds_bucket[1m])) by (le, instance))", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A", + "step": 60 + }, "params": [ - 10 + "A", + "1m", + "now" ] - }, + }, "reducer": { - "type": "avg", - "params": [] - }, + "params": [], + "type": "avg" + }, "type": "query" } - ] - }, - "grid": {}, - "stack": false, - "timeShift": null, - "aliasColors": {}, - "lines": true, - "points": false, - "datasource": "${DS_TIDB-CLUSTER}", - "pointradius": 5 - }, - { - "bars": false, - "timeFrom": null, - "links": [], - "thresholds": [ - { - "colorMode": "critical", - "line": true, - "op": "gt", - "value": 1, - "fill": true - } - ], - "nullPointMode": "null as zero", - "renderer": "flot", - "linewidth": 1, - "steppedLine": false, - "id": 37, - "fill": 0, - "span": 4, - "title": "Average Query Duration", - "tooltip": { - "sort": 0, - "shared": true, - "value_type": "cumulative", - "msResolution": true - }, + ], + "executionErrorState": "alerting", + "frequency": "60s", + "handler": 1, + "message": "Query duration for 99th percentile is high.", + "name": "Query Duration 99th percentile alert", + "noDataState": "no_data", + "notifications": [] + }, + "aliasColors": {}, + "bars": false, + "datasource": "${DS_TEST-CLUSTER}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 25, "legend": { - "rightSide": true, - "total": false, - "min": false, - "max": true, - "show": false, - "current": false, - "hideEmpty": true, - "values": true, - "alignAsTable": false, - "avg": false, - "hideZero": true - }, + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, "targets": [ { - "intervalFactor": 2, - "expr": "irate(tidb_server_handle_query_duration_seconds_sum[30s]) / irate(tidb_server_handle_query_duration_seconds_count[30s])", - "step": 60, - "refId": "A", - "legendFormat": "{{instance}}" - }, + "expr": "histogram_quantile(0.99, sum(rate(tidb_server_handle_query_duration_seconds_bucket[1m])) by (le))", + "intervalFactor": 2, + "legendFormat": "average", + "refId": "B", + "step": 60 + }, { - "intervalFactor": 2, - "expr": "sum(irate(tidb_server_handle_query_duration_seconds_sum[30s])) / sum(irate(tidb_server_handle_query_duration_seconds_count[30s]))", - "step": 60, - "refId": "B", - "legendFormat": "average" + "expr": "histogram_quantile(0.99, sum(rate(tidb_server_handle_query_duration_seconds_bucket[1m])) by (le, instance))", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A", + "step": 60 } - ], - "yaxes": [ + ], + "thresholds": [ { - "logBase": 1, - "show": true, - "max": null, - "format": "s", - "min": "0", - "label": null - }, - { - "logBase": 1, - "show": true, - "max": null, - "format": "short", - "min": null, - "label": null - } - ], + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 10 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Query Duration 99th percentile", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", "xaxis": { - "show": true, - "values": [], - "mode": "time", - "name": null - }, - "seriesOverrides": [], - "percentage": false, - "type": "graph", - "error": false, - "editable": true, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { "alert": { - "noDataState": "no_data", - "name": "Average Query Duration alert", - "frequency": "60s", - "notifications": [], - "handler": 1, - "executionErrorState": "alerting", - "message": "Average query duration is high.", "conditions": [ { + "evaluator": { + "params": [ + 1 + ], + "type": "gt" + }, "operator": { "type": "and" - }, + }, "query": { - "params": [ - "A", - "5m", - "now" - ], + "datasourceId": 1, "model": { - "intervalFactor": 2, - "expr": "irate(tidb_server_handle_query_duration_seconds_sum[30s]) / irate(tidb_server_handle_query_duration_seconds_count[30s])", - "step": 60, - "refId": "A", - "legendFormat": "{{instance}}" - }, - "datasourceId": 1 - }, - "evaluator": { - "type": "gt", + "expr": "irate(tidb_server_handle_query_duration_seconds_sum[30s]) / irate(tidb_server_handle_query_duration_seconds_count[30s])", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A", + "step": 60 + }, "params": [ - 1 + "A", + "5m", + "now" ] - }, + }, "reducer": { - "type": "avg", - "params": [] - }, + "params": [], + "type": "avg" + }, "type": "query" } - ] - }, - "grid": {}, - "stack": false, - "timeShift": null, - "aliasColors": {}, - "lines": true, - "points": false, - "datasource": "${DS_TIDB-CLUSTER}", - "pointradius": 5 - }, - { - "bars": false, - "timeFrom": null, - "links": [], - "thresholds": [], - "nullPointMode": "null as zero", - "renderer": "flot", - "linewidth": 1, - "steppedLine": false, - "id": 2, - "fill": 0, - "span": 8, - "title": "QPS", - "tooltip": { - "sort": 2, - "shared": true, - "value_type": "individual", - "msResolution": true - }, + ], + "executionErrorState": "alerting", + "frequency": "60s", + "handler": 1, + "message": "Average query duration is high.", + "name": "Average Query Duration alert", + "noDataState": "no_data", + "notifications": [] + }, + "aliasColors": {}, + "bars": false, + "datasource": "${DS_TEST-CLUSTER}", + "editable": true, + "error": false, + "fill": 0, + "grid": {}, + "id": 37, "legend": { - "sort": "max", - "rightSide": true, - "total": false, - "sideWidth": 250, - "min": false, - "max": true, - "show": true, - "current": true, - "hideEmpty": true, - "values": true, - "alignAsTable": true, - "avg": false, - "sortDesc": true, - "hideZero": true - }, + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, "targets": [ { - "intervalFactor": 2, - "expr": "rate(tidb_server_query_total[1m])", - "step": 30, - "refId": "A", - "legendFormat": "{{instance}}-{{job}} {{type}} {{status}}" + "expr": "irate(tidb_server_handle_query_duration_seconds_sum[30s]) / irate(tidb_server_handle_query_duration_seconds_count[30s])", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A", + "step": 60 + }, + { + "expr": "sum(irate(tidb_server_handle_query_duration_seconds_sum[30s])) / sum(irate(tidb_server_handle_query_duration_seconds_count[30s]))", + "intervalFactor": 2, + "legendFormat": "average", + "refId": "B", + "step": 60 } - ], - "yaxes": [ + ], + "thresholds": [ { - "logBase": 1, - "show": true, - "max": null, - "format": "short", - "min": "0", - "label": null - }, - { - "logBase": 1, - "show": true, - "max": null, - "format": "short", - "min": null, - "label": null - } - ], + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 1 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Query Duration", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", "xaxis": { - "show": true, - "values": [], - "mode": "time", - "name": null - }, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "${DS_TEST-CLUSTER}", + "editable": true, + "error": false, + "fill": 0, + "grid": {}, + "id": 2, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 250, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 12, + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", "seriesOverrides": [ { - "alias": "total", + "alias": "total", "lines": false } - ], - "percentage": false, - "type": "graph", - "error": false, - "editable": true, - "grid": {}, - "stack": false, - "timeShift": null, - "aliasColors": {}, - "lines": true, - "points": false, - "datasource": "${DS_TIDB-CLUSTER}", - "pointradius": 5, - "minSpan": 12 - }, - { - "bars": false, - "timeFrom": null, - "links": [], - "thresholds": [], - "nullPointMode": "null as zero", - "renderer": "flot", - "linewidth": 1, - "steppedLine": false, - "id": 42, - "fill": 1, - "span": 4, - "title": "QPS Total", + ], + "span": 8, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(tidb_server_query_total[1m])", + "intervalFactor": 2, + "legendFormat": "{{instance}}-{{job}} {{type}} {{status}}", + "refId": "A", + "step": 30 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "QPS", "tooltip": { - "sort": 0, - "shared": true, - "value_type": "cumulative", - "msResolution": true - }, + "msResolution": true, + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "${DS_TEST-CLUSTER}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 42, "legend": { - "sort": "max", - "rightSide": false, - "total": false, - "sideWidth": 250, - "min": false, - "max": true, - "show": true, - "current": true, - "hideEmpty": true, - "values": true, - "alignAsTable": true, - "avg": false, - "sortDesc": false, - "hideZero": false - }, + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": false, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": 250, + "sort": "max", + "sortDesc": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 12, + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, "targets": [ { - "intervalFactor": 2, - "expr": "sum(rate(tidb_server_query_total[1m])) by (status)", - "step": 60, - "refId": "A", - "legendFormat": "query {{status}}" - }, - { - "intervalFactor": 3, - "expr": "sum(rate(tidb_server_query_total{status=\"OK\"}[1m] offset 1d))", - "step": 90, - "refId": "B", - "legendFormat": "yesterday" - } - ], + "expr": "sum(rate(tidb_server_query_total[1m])) by (status)", + "intervalFactor": 2, + "legendFormat": "query {{status}}", + "refId": "A", + "step": 60 + }, + { + "expr": "sum(rate(tidb_server_query_total{status=\"OK\"}[1m] offset 1d))", + "intervalFactor": 3, + "legendFormat": "yesterday", + "refId": "B", + "step": 90 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "QPS Total", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, "yaxes": [ { - "logBase": 1, - "show": true, - "max": null, - "format": "ops", - "min": "0", - "label": null - }, - { - "logBase": 1, - "show": true, - "max": null, - "format": "short", - "min": null, - "label": null - } - ], - "xaxis": { - "show": true, - "values": [], - "mode": "time", - "name": null - }, - "seriesOverrides": [], - "percentage": false, - "type": "graph", - "error": false, - "editable": true, - "grid": {}, - "stack": false, - "timeShift": null, - "aliasColors": {}, - "lines": true, - "points": false, - "datasource": "${DS_TIDB-CLUSTER}", - "pointradius": 5, - "minSpan": 12 - }, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, { - "bars": false, - "timeFrom": null, - "links": [], - "thresholds": [], - "nullPointMode": "null as zero", - "renderer": "flot", - "linewidth": 1, - "steppedLine": false, - "id": 21, - "fill": 1, - "span": 8, - "title": "Statement Count", - "tooltip": { - "sort": 0, - "shared": true, - "value_type": "cumulative", - "msResolution": false - }, + "aliasColors": {}, + "bars": false, + "datasource": "${DS_TEST-CLUSTER}", + "decimals": null, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 21, "legend": { - "sort": null, - "rightSide": true, - "total": false, - "min": false, - "max": true, - "show": true, - "current": true, - "hideEmpty": true, - "values": true, - "alignAsTable": true, - "avg": true, - "sortDesc": null, - "hideZero": true - }, + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": null, + "sortDesc": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 8, + "stack": false, + "steppedLine": false, "targets": [ { - "intervalFactor": 2, - "expr": "sum(irate(tidb_executor_statement_node_total[1m])) by (type)", - "step": 30, - "refId": "A", - "legendFormat": "{{type}}" - } - ], + "expr": "sum(irate(tidb_executor_statement_node_total[1m])) by (type)", + "intervalFactor": 2, + "legendFormat": "{{type}}", + "refId": "A", + "step": 30 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Statement Count", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, "yaxes": [ { - "logBase": 10, - "show": true, - "max": null, - "format": "short", - "min": "0", - "label": null - }, - { - "logBase": 1, - "show": true, - "max": null, - "format": "short", - "min": null, - "label": null - } - ], - "xaxis": { - "show": true, - "values": [], - "mode": "time", - "name": null - }, - "seriesOverrides": [], - "percentage": false, - "type": "graph", - "error": false, - "editable": true, - "grid": {}, - "stack": false, - "timeShift": null, - "aliasColors": {}, - "lines": true, - "points": false, - "datasource": "${DS_TIDB-CLUSTER}", - "pointradius": 5, - "decimals": null + "format": "short", + "label": null, + "logBase": 10, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] } - ], - "showTitle": true, - "collapse": false - }, + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Query", + "titleSize": "h6" + }, { - "repeat": null, - "titleSize": "h6", - "repeatIteration": null, - "title": "Query", - "height": "250px", - "repeatRowId": null, + "collapse": false, + "height": "250px", "panels": [ { - "bars": false, - "timeFrom": null, - "links": [], - "thresholds": [], - "nullPointMode": "null as zero", - "renderer": "flot", - "id": 8, - "linewidth": 1, - "steppedLine": true, - "targets": [ - { - "intervalFactor": 2, - "expr": "tidb_server_connections", - "step": 30, - "refId": "A", - "legendFormat": "{{instance}}-{{job}}" - }, - { - "intervalFactor": 2, - "expr": "sum(tidb_server_connections)", - "step": 30, - "refId": "B", - "legendFormat": "total" - } - ], - "fill": 1, - "span": 6, - "title": "Connection Count", - "tooltip": { - "sort": 0, - "shared": true, - "value_type": "individual", - "msResolution": false - }, + "aliasColors": {}, + "bars": false, + "datasource": "${DS_TEST-CLUSTER}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 8, "legend": { - "rightSide": true, - "total": false, - "min": false, - "max": false, - "show": true, - "current": true, - "values": true, - "alignAsTable": true, - "avg": false - }, - "yaxes": [ - { - "logBase": 1, - "show": true, - "max": null, - "format": "short", - "min": "0", - "label": null - }, - { - "logBase": 1, - "show": true, - "max": null, - "format": "short", - "min": null, - "label": null - } - ], - "xaxis": { - "show": true, - "values": [], - "mode": "time", - "name": null - }, + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", "seriesOverrides": [ { - "alias": "total", - "lines": false, - "fill": 0 - } - ], - "percentage": false, - "type": "graph", - "error": false, - "editable": true, - "grid": {}, - "stack": true, - "timeShift": null, - "aliasColors": {}, - "lines": true, - "points": false, - "datasource": "${DS_TIDB-CLUSTER}", - "pointradius": 5 - }, - { - "bars": false, - "timeFrom": null, - "links": [], - "thresholds": [ - { - "colorMode": "critical", - "line": true, - "op": "gt", - "value": 1000000000, - "fill": true - } - ], - "nullPointMode": "null as zero", - "renderer": "flot", - "linewidth": 1, - "steppedLine": false, - "id": 3, - "fill": 0, - "span": 6, - "title": "Heap Memory Usage", - "tooltip": { - "sort": 0, - "shared": true, - "value_type": "cumulative", - "msResolution": true - }, - "legend": { - "rightSide": true, - "total": false, - "sideWidth": null, - "min": false, - "max": true, - "show": true, - "current": true, - "hideEmpty": true, - "values": true, - "alignAsTable": true, - "avg": false, - "hideZero": false - }, + "alias": "total", + "fill": 0, + "lines": false + } + ], + "span": 6, + "stack": true, + "steppedLine": true, "targets": [ { - "hide": false, - "expr": "go_memstats_heap_inuse_bytes{job=~\"tidb.*\"}", - "metric": "go_memstats_heap_inuse_bytes", - "step": 30, - "legendFormat": "{{instance}}-{{job}}", - "intervalFactor": 2, - "refId": "B" - } - ], + "expr": "tidb_server_connections", + "intervalFactor": 2, + "legendFormat": "{{instance}}-{{job}}", + "refId": "A", + "step": 30 + }, + { + "expr": "sum(tidb_server_connections)", + "intervalFactor": 2, + "legendFormat": "total", + "refId": "B", + "step": 30 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Connection Count", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, "yaxes": [ { - "logBase": 1, - "show": true, - "max": null, - "format": "bytes", - "min": "0", - "label": "" - }, - { - "logBase": 1, - "show": false, - "max": null, - "format": "short", - "min": null, - "label": "" - } - ], - "xaxis": { - "show": true, - "values": [], - "mode": "time", - "name": null - }, - "seriesOverrides": [], - "percentage": false, - "type": "graph", - "error": false, - "editable": true, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { "alert": { - "noDataState": "no_data", - "name": "TiDB Heap Memory Usage alert", - "frequency": "60s", - "notifications": [], - "handler": 1, - "executionErrorState": "alerting", - "message": "TiDB mem heap is over 1GiB", "conditions": [ { + "evaluator": { + "params": [ + 1000000000 + ], + "type": "gt" + }, "operator": { "type": "and" - }, + }, "query": { - "params": [ - "B", - "10s", - "now" - ], + "datasourceId": 1, "model": { - "hide": false, - "expr": "go_memstats_heap_inuse_bytes{job=~\"tidb.*\"}", - "metric": "go_memstats_heap_inuse_bytes", - "step": 30, - "legendFormat": "{{instance}}-{{job}}", - "intervalFactor": 2, - "refId": "B" - }, - "datasourceId": 1 - }, - "evaluator": { - "type": "gt", + "expr": "go_memstats_heap_inuse_bytes{job=~\"tidb.*\"}", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{instance}}-{{job}}", + "metric": "go_memstats_heap_inuse_bytes", + "refId": "B", + "step": 30 + }, "params": [ - 1000000000 + "B", + "10s", + "now" ] - }, + }, "reducer": { - "type": "avg", - "params": [] - }, + "params": [], + "type": "avg" + }, "type": "query" } - ] - }, - "grid": {}, - "stack": false, - "timeShift": null, - "aliasColors": {}, - "lines": true, - "points": false, - "datasource": "${DS_TIDB-CLUSTER}", - "pointradius": 5 + ], + "executionErrorState": "alerting", + "frequency": "60s", + "handler": 1, + "message": "TiDB mem heap is over 1GiB", + "name": "TiDB Heap Memory Usage alert", + "noDataState": "no_data", + "notifications": [] + }, + "aliasColors": {}, + "bars": false, + "datasource": "${DS_TEST-CLUSTER}", + "editable": true, + "error": false, + "fill": 0, + "grid": {}, + "id": 3, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": false, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_memstats_heap_inuse_bytes{job=~\"tidb.*\"}", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{instance}}-{{job}}", + "metric": "go_memstats_heap_inuse_bytes", + "refId": "B", + "step": 30 + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 1000000000 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Heap Memory Usage", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] } - ], - "showTitle": true, - "collapse": false - }, + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Query", + "titleSize": "h6" + }, { - "repeat": null, - "titleSize": "h6", - "repeatIteration": null, - "title": "Distsql", - "height": "250px", - "repeatRowId": null, + "collapse": true, + "height": "250px", "panels": [ { - "bars": false, - "timeFrom": null, + "aliasColors": {}, + "bars": false, + "datasource": "${DS_TEST-CLUSTER}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 12, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, "links": [ { "type": "dashboard" } - ], - "thresholds": [], - "nullPointMode": "null as zero", - "renderer": "flot", - "id": 12, - "linewidth": 2, - "steppedLine": false, + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, "targets": [ { - "hide": false, - "expr": "histogram_quantile(0.99, sum(rate(tidb_distsql_handle_query_duration_seconds_bucket[1m])) by (le))", - "metric": "tidb_distsql_handle_query_duration_seconds_bucket", - "step": 4, - "legendFormat": "", - "intervalFactor": 2, - "refId": "A" - } - ], - "fill": 1, - "span": 6, - "title": "Distsql Seconds 99", + "expr": "histogram_quantile(0.99, sum(rate(tidb_distsql_handle_query_duration_seconds_bucket[1m])) by (le))", + "hide": false, + "intervalFactor": 2, + "legendFormat": "", + "metric": "tidb_distsql_handle_query_duration_seconds_bucket", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Distsql Seconds 99", "tooltip": { - "sort": 0, - "shared": true, - "value_type": "cumulative", - "msResolution": false - }, - "legend": { - "total": false, - "show": false, - "max": false, - "min": false, - "current": false, - "values": false, - "alignAsTable": false, - "avg": false - }, + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, "yaxes": [ { - "logBase": 1, - "show": true, - "max": null, - "format": "s", - "min": 0, - "label": null - }, - { - "logBase": 1, - "show": true, - "max": null, - "format": "short", - "min": null, - "label": null - } - ], - "xaxis": { - "show": true, - "values": [], - "mode": "time", - "name": null - }, - "seriesOverrides": [], - "percentage": false, - "type": "graph", - "error": false, - "editable": true, - "grid": {}, - "stack": false, - "timeShift": null, - "aliasColors": {}, - "lines": true, - "points": false, - "datasource": "${DS_TIDB-CLUSTER}", - "pointradius": 5 - }, + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, { - "bars": false, - "timeFrom": null, - "links": [], - "thresholds": [], - "nullPointMode": "null as zero", - "renderer": "flot", - "id": 14, - "linewidth": 2, - "steppedLine": false, + "aliasColors": {}, + "bars": false, + "datasource": "${DS_TEST-CLUSTER}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 14, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, "targets": [ { - "expr": "sum(rate(tidb_distsql_query_total [1m]))", - "metric": "tidb_distsql_query_total", - "step": 4, - "legendFormat": "", - "intervalFactor": 2, - "refId": "A" - } - ], - "fill": 1, - "span": 6, - "title": "Distsql QPS", + "expr": "sum(rate(tidb_distsql_query_total [1m]))", + "intervalFactor": 2, + "legendFormat": "", + "metric": "tidb_distsql_query_total", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Distsql QPS", "tooltip": { - "sort": 0, - "shared": true, - "value_type": "cumulative", - "msResolution": false - }, - "legend": { - "rightSide": false, - "total": false, - "show": false, - "max": false, - "min": false, - "current": false, - "values": false, - "avg": false - }, + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, "yaxes": [ { - "logBase": 1, - "show": true, - "max": null, - "format": "short", - "min": 0, - "label": null - }, - { - "logBase": 1, - "show": true, - "max": null, - "format": "short", - "min": null, - "label": null - } - ], - "xaxis": { - "show": true, - "values": [], - "mode": "time", - "name": null - }, - "seriesOverrides": [], - "percentage": false, - "type": "graph", - "error": false, - "editable": true, - "grid": {}, - "stack": false, - "timeShift": null, - "aliasColors": {}, - "lines": true, - "points": false, - "datasource": "${DS_TIDB-CLUSTER}", - "pointradius": 5 + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] } - ], - "showTitle": true, - "collapse": true - }, + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Distsql", + "titleSize": "h6" + }, { - "repeat": null, - "titleSize": "h6", - "repeatIteration": null, - "title": "Coprocessor", - "height": "250px", - "repeatRowId": null, + "collapse": true, + "height": "250px", "panels": [ { - "bars": false, - "timeFrom": null, - "links": [], - "thresholds": [], - "nullPointMode": "null as zero", - "renderer": "flot", - "id": 40, - "linewidth": 2, - "steppedLine": false, + "aliasColors": {}, + "bars": false, + "datasource": "${DS_TEST-CLUSTER}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 40, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "total", + "sortDesc": true, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, "targets": [ { - "intervalFactor": 2, - "expr": "sum(rate(tidb_tikvclient_cop_count[1m])) by (type)", - "step": 4, - "refId": "A", - "legendFormat": "{{type}}" - } - ], - "fill": 1, - "span": 6, - "title": "Coprocessor Count", + "expr": "sum(rate(tidb_tikvclient_cop_count[1m])) by (type)", + "intervalFactor": 2, + "legendFormat": "{{type}}", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Coprocessor Count", "tooltip": { - "sort": 0, - "shared": true, - "value_type": "cumulative", - "msResolution": false - }, - "legend": { - "sort": "total", - "rightSide": true, - "total": true, - "min": false, - "max": false, - "show": true, - "current": false, - "hideEmpty": true, - "values": true, - "alignAsTable": true, - "avg": false, - "sortDesc": true, - "hideZero": true - }, + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, "yaxes": [ { - "logBase": 1, - "show": true, - "max": null, - "format": "short", - "min": null, - "label": null - }, - { - "logBase": 1, - "show": true, - "max": null, - "format": "short", - "min": null, - "label": null - } - ], - "xaxis": { - "show": true, - "values": [], - "mode": "time", - "name": null - }, - "seriesOverrides": [], - "percentage": false, - "type": "graph", - "error": false, - "editable": true, - "grid": {}, - "stack": false, - "timeShift": null, - "aliasColors": {}, - "lines": true, - "points": false, - "datasource": "${DS_TIDB-CLUSTER}", - "pointradius": 5 - }, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, { - "bars": false, - "timeFrom": null, - "links": [], - "thresholds": [], - "nullPointMode": "null as zero", - "renderer": "flot", - "id": 41, - "linewidth": 2, - "steppedLine": false, + "aliasColors": {}, + "bars": false, + "datasource": "${DS_TEST-CLUSTER}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 41, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, "targets": [ { - "intervalFactor": 2, - "expr": "histogram_quantile(0.999, sum(rate(tidb_tikvclient_cop_seconds_bucket[1m])) by (le, instance))", - "step": 4, - "refId": "A", - "legendFormat": "{{instance}}" - } - ], - "fill": 1, - "span": 6, - "title": "Coprocessor Seconds 999", + "expr": "histogram_quantile(0.999, sum(rate(tidb_tikvclient_cop_seconds_bucket[1m])) by (le, instance))", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Coprocessor Seconds 999", "tooltip": { - "sort": 0, - "shared": true, - "value_type": "cumulative", - "msResolution": false - }, - "legend": { - "sort": "max", - "rightSide": true, - "total": false, - "min": false, - "max": true, - "show": true, - "current": false, - "values": true, - "alignAsTable": true, - "avg": false, - "sortDesc": true - }, + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, "yaxes": [ { - "logBase": 1, - "show": true, - "max": null, - "format": "s", - "min": "0", - "label": null - }, - { - "logBase": 1, - "show": true, - "max": null, - "format": "short", - "min": null, - "label": null - } - ], - "xaxis": { - "show": true, - "values": [], - "mode": "time", - "name": null - }, - "seriesOverrides": [], - "percentage": false, - "type": "graph", - "error": false, - "editable": true, - "grid": {}, - "stack": false, - "timeShift": null, - "aliasColors": {}, - "lines": true, - "points": false, - "datasource": "${DS_TIDB-CLUSTER}", - "pointradius": 5 + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] } - ], - "showTitle": true, - "collapse": true - }, + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Coprocessor", + "titleSize": "h6" + }, { - "repeat": null, - "titleSize": "h6", - "repeatIteration": null, - "title": "KV", - "height": "250px", - "repeatRowId": null, + "collapse": false, + "height": "250px", "panels": [ { - "bars": false, - "timeFrom": null, - "links": [], - "thresholds": [], - "nullPointMode": "null as zero", - "renderer": "flot", - "id": 5, - "linewidth": 2, - "steppedLine": false, + "aliasColors": {}, + "bars": false, + "datasource": "${DS_TEST-CLUSTER}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 5, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, "targets": [ { - "intervalFactor": 2, - "expr": "sum(rate(tidb_tikvclient_txn_cmd_total[1m])) by (type)", - "step": 30, - "refId": "A", - "legendFormat": "{{type}}" - } - ], - "fill": 1, - "span": 6, - "title": "KV Cmd Count", + "expr": "sum(rate(tidb_tikvclient_txn_cmd_total[1m])) by (type)", + "intervalFactor": 2, + "legendFormat": "{{type}}", + "refId": "A", + "step": 30 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "KV Cmd Count", "tooltip": { - "sort": 0, - "shared": true, - "value_type": "cumulative", - "msResolution": true - }, - "legend": { - "sort": "max", - "rightSide": true, - "total": false, - "min": false, - "max": true, - "show": true, - "current": true, - "hideEmpty": true, - "values": true, - "alignAsTable": true, - "avg": false, - "sortDesc": true, - "hideZero": true - }, + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, "yaxes": [ { - "logBase": 1, - "show": true, - "max": null, - "format": "short", - "min": null, - "label": null - }, - { - "logBase": 1, - "show": true, - "max": null, - "format": "short", - "min": null, - "label": null - } - ], - "xaxis": { - "show": true, - "values": [], - "mode": "time", - "name": null - }, - "seriesOverrides": [], - "percentage": false, - "type": "graph", - "error": false, - "editable": true, - "grid": {}, - "stack": false, - "timeShift": null, - "aliasColors": {}, - "lines": true, - "points": false, - "datasource": "${DS_TIDB-CLUSTER}", - "pointradius": 5 - }, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, { - "bars": false, - "timeFrom": null, - "links": [], - "thresholds": [], - "nullPointMode": "null as zero", - "renderer": "flot", - "id": 4, - "linewidth": 2, - "steppedLine": false, + "aliasColors": {}, + "bars": false, + "datasource": "${DS_TEST-CLUSTER}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 4, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, "targets": [ { - "intervalFactor": 2, - "expr": "sum(rate(tidb_tikvclient_txn_total[1m])) by (instance)", - "step": 30, - "refId": "A", - "legendFormat": "{{instance}}" - } - ], - "fill": 1, - "span": 6, - "title": "KV Txn Count", + "expr": "sum(rate(tidb_tikvclient_txn_total[1m])) by (instance)", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A", + "step": 30 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "KV Txn Count", "tooltip": { - "sort": 0, - "shared": true, - "value_type": "cumulative", - "msResolution": true - }, - "legend": { - "rightSide": true, - "total": false, - "min": false, - "max": true, - "show": true, - "current": true, - "values": true, - "alignAsTable": true, - "avg": false - }, + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, "yaxes": [ { - "logBase": 1, - "show": true, - "max": null, - "format": "short", - "min": "0", - "label": null - }, - { - "logBase": 1, - "show": true, - "max": null, - "format": "short", - "min": null, - "label": null - } - ], - "xaxis": { - "show": true, - "values": [], - "mode": "time", - "name": null - }, - "seriesOverrides": [], - "percentage": false, - "type": "graph", - "error": false, - "editable": true, - "grid": {}, - "stack": false, - "timeShift": null, - "aliasColors": {}, - "lines": true, - "points": false, - "datasource": "${DS_TIDB-CLUSTER}", - "pointradius": 5 - }, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, { - "bars": false, - "timeFrom": null, - "links": [], - "thresholds": [], - "nullPointMode": "null as zero", - "renderer": "flot", - "id": 6, - "linewidth": 2, - "steppedLine": false, + "aliasColors": {}, + "bars": false, + "datasource": "${DS_TEST-CLUSTER}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 6, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": null, + "sortDesc": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, "targets": [ { - "intervalFactor": 2, - "expr": "histogram_quantile(0.9999, sum(rate(tidb_tikvclient_backoff_seconds_bucket[1m])) by (instance, le))", - "step": 30, - "refId": "A", - "legendFormat": "{{instance}}" - } - ], - "fill": 1, - "span": 6, - "title": "KV Retry Seconds 9999", + "expr": "histogram_quantile(0.9999, sum(rate(tidb_tikvclient_backoff_seconds_bucket[1m])) by (instance, le))", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A", + "step": 30 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "KV Retry Seconds 9999", "tooltip": { - "sort": 0, - "shared": true, - "value_type": "cumulative", - "msResolution": false - }, - "legend": { - "sort": null, - "rightSide": true, - "total": false, - "min": false, - "max": true, - "show": true, - "current": false, - "values": true, - "alignAsTable": true, - "avg": false, - "sortDesc": null - }, + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, "yaxes": [ { - "logBase": 1, - "show": true, - "max": null, - "format": "s", - "min": null, - "label": null - }, - { - "logBase": 1, - "show": true, - "max": null, - "format": "short", - "min": null, - "label": null - } - ], - "xaxis": { - "show": true, - "values": [], - "mode": "time", - "name": null - }, - "seriesOverrides": [], - "percentage": false, - "type": "graph", - "error": false, - "editable": true, - "grid": {}, - "stack": false, - "timeShift": null, - "aliasColors": {}, - "lines": true, - "points": false, - "datasource": "${DS_TIDB-CLUSTER}", - "pointradius": 5 - }, + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, { - "bars": false, - "timeFrom": null, - "links": [], - "thresholds": [], - "nullPointMode": "null as zero", - "renderer": "flot", - "id": 30, - "linewidth": 1, - "steppedLine": false, + "aliasColors": {}, + "bars": false, + "datasource": "${DS_TEST-CLUSTER}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 30, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, "targets": [ { - "intervalFactor": 2, - "expr": "histogram_quantile(0.9999, sum(rate(tidb_tikvclient_request_seconds_bucket[1m])) by (le, instance, type))", - "step": 30, - "refId": "A", - "legendFormat": "{{instance}} {{type}}" - } - ], - "fill": 1, - "span": 6, - "title": "KV Request Seconds 9999", + "expr": "histogram_quantile(0.9999, sum(rate(tidb_tikvclient_request_seconds_bucket[1m])) by (le, instance, type))", + "intervalFactor": 2, + "legendFormat": "{{instance}} {{type}}", + "refId": "A", + "step": 30 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "KV Request Seconds 9999", "tooltip": { - "sort": 0, - "shared": true, - "value_type": "individual", - "msResolution": false - }, - "legend": { - "sort": "max", - "rightSide": true, - "total": false, - "min": false, - "max": true, - "show": true, - "current": false, - "values": true, - "alignAsTable": true, - "avg": false, - "sortDesc": true - }, + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, "yaxes": [ { - "logBase": 1, - "show": true, - "max": null, - "format": "s", - "min": null, - "label": null - }, - { - "logBase": 1, - "show": true, - "max": null, - "format": "short", - "min": null, - "label": null - } - ], - "xaxis": { - "show": true, - "values": [], - "mode": "time", - "name": null - }, - "seriesOverrides": [], - "percentage": false, - "type": "graph", - "error": false, - "editable": true, - "grid": {}, - "stack": false, - "timeShift": null, - "aliasColors": {}, - "lines": true, - "points": false, - "datasource": "${DS_TIDB-CLUSTER}", - "pointradius": 5 - }, + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, { - "bars": false, - "timeFrom": null, - "links": [], - "thresholds": [], - "nullPointMode": "null as zero", - "renderer": "flot", - "id": 18, - "linewidth": 1, - "steppedLine": false, + "aliasColors": {}, + "bars": false, + "datasource": "${DS_TEST-CLUSTER}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 18, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, "targets": [ { - "intervalFactor": 2, - "expr": "histogram_quantile(0.99, sum(rate(tidb_tikvclient_txn_cmd_seconds_bucket[1m])) by (le, type))", - "step": 30, - "refId": "A", - "legendFormat": "{{type}}" - } - ], - "fill": 1, - "span": 6, - "title": "KV Cmd Seconds 99", + "expr": "histogram_quantile(0.99, sum(rate(tidb_tikvclient_txn_cmd_seconds_bucket[1m])) by (le, type))", + "intervalFactor": 2, + "legendFormat": "{{type}}", + "refId": "A", + "step": 30 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "KV Cmd Seconds 99", "tooltip": { - "sort": 0, - "shared": true, - "value_type": "cumulative", - "msResolution": false - }, - "legend": { - "rightSide": true, - "total": false, - "min": false, - "max": true, - "show": true, - "current": true, - "hideEmpty": true, - "values": true, - "alignAsTable": true, - "avg": false, - "hideZero": true - }, + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, "yaxes": [ { - "logBase": 1, - "show": true, - "max": null, - "format": "s", - "min": null, - "label": null - }, - { - "logBase": 1, - "show": true, - "max": null, - "format": "short", - "min": null, - "label": null - } - ], - "xaxis": { - "show": true, - "values": [], - "mode": "time", - "name": null - }, - "seriesOverrides": [], - "percentage": false, - "type": "graph", - "error": false, - "editable": true, - "grid": {}, - "stack": false, - "timeShift": null, - "aliasColors": {}, - "lines": true, - "points": false, - "datasource": "${DS_TIDB-CLUSTER}", - "pointradius": 5 - }, + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, { - "bars": false, - "timeFrom": null, - "links": [], - "thresholds": [], - "nullPointMode": "null as zero", - "renderer": "flot", - "id": 22, - "linewidth": 2, - "steppedLine": false, + "aliasColors": {}, + "bars": false, + "datasource": "${DS_TEST-CLUSTER}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 22, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, "targets": [ { - "intervalFactor": 2, - "expr": "histogram_quantile(0.999, sum(rate(tidb_tikvclient_txn_cmd_seconds_bucket[1m])) by (le, type))", - "step": 30, - "refId": "A", - "legendFormat": "{{type}}" - } - ], - "fill": 1, - "span": 6, - "title": "KV Cmd Seconds 9999", + "expr": "histogram_quantile(0.999, sum(rate(tidb_tikvclient_txn_cmd_seconds_bucket[1m])) by (le, type))", + "intervalFactor": 2, + "legendFormat": "{{type}}", + "refId": "A", + "step": 30 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "KV Cmd Seconds 9999", "tooltip": { - "sort": 0, - "shared": true, - "value_type": "cumulative", - "msResolution": false - }, - "legend": { - "rightSide": true, - "total": false, - "min": false, - "max": false, - "show": true, - "current": false, - "values": false, - "alignAsTable": true, - "avg": false - }, + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, "yaxes": [ { - "logBase": 1, - "show": true, - "max": null, - "format": "s", - "min": null, - "label": null - }, - { - "logBase": 1, - "show": true, - "max": null, - "format": "short", - "min": null, - "label": null - } - ], - "xaxis": { - "show": true, - "values": [], - "mode": "time", - "name": null - }, - "seriesOverrides": [], - "percentage": false, - "type": "graph", - "error": false, - "editable": true, - "grid": {}, - "stack": false, - "timeShift": null, - "aliasColors": {}, - "lines": true, - "points": false, - "datasource": "${DS_TIDB-CLUSTER}", - "pointradius": 5 - }, + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, { - "bars": false, - "timeFrom": null, - "links": [], - "thresholds": [], - "nullPointMode": "null as zero", - "renderer": "flot", - "id": 44, - "linewidth": 2, - "steppedLine": false, + "aliasColors": {}, + "bars": false, + "datasource": "${DS_TEST-CLUSTER}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 44, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, "targets": [ { - "intervalFactor": 2, - "expr": "histogram_quantile(0.90, sum(rate(tidb_tikvclient_txn_regions_num_bucket[1m])) by (le, instance))", - "step": 60, - "refId": "A", - "legendFormat": "{{instance}}" - } - ], - "fill": 1, - "span": 6, - "title": "90 Txn regions count", + "expr": "histogram_quantile(0.90, sum(rate(tidb_tikvclient_txn_regions_num_bucket[1m])) by (le, instance))", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A", + "step": 30 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "90 Txn regions count", "tooltip": { - "sort": 0, - "shared": true, - "value_type": "cumulative", - "msResolution": true - }, - "legend": { - "rightSide": true, - "total": false, - "min": false, - "max": true, - "show": true, - "current": true, - "values": true, - "alignAsTable": true, - "avg": false - }, + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, "yaxes": [ { - "logBase": 1, - "show": true, - "max": null, - "format": "short", - "min": "0", - "label": null - }, - { - "logBase": 1, - "show": true, - "max": null, - "format": "short", - "min": null, - "label": null - } - ], - "xaxis": { - "show": true, - "values": [], - "mode": "time", - "name": null - }, - "seriesOverrides": [], - "percentage": false, - "type": "graph", - "error": false, - "editable": true, - "grid": {}, - "stack": false, - "timeShift": null, - "aliasColors": {}, - "lines": true, - "points": false, - "datasource": "${DS_TIDB-CLUSTER}", - "pointradius": 5 + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] } - ], - "showTitle": true, - "collapse": false - }, + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "KV", + "titleSize": "h6" + }, { - "repeat": null, - "titleSize": "h6", - "repeatIteration": null, - "title": "KV 2", - "height": 250, - "repeatRowId": null, + "collapse": false, + "height": 250, "panels": [ { - "bars": false, - "timeFrom": null, - "links": [], - "thresholds": [], - "nullPointMode": "null as zero", - "renderer": "flot", - "id": 33, - "linewidth": 2, - "steppedLine": false, - "targets": [ - { - "intervalFactor": 2, - "expr": "histogram_quantile(1, sum(rate(tidb_tikvclient_txn_write_kv_count_bucket[1m])) by (le, instance))", - "step": 30, - "refId": "B", - "legendFormat": "{{instance}}" - } - ], - "fill": 1, - "span": 6, - "title": "KV Count Per Txn", - "tooltip": { - "sort": 2, - "shared": true, - "value_type": "cumulative", - "msResolution": false - }, + "aliasColors": {}, + "bars": false, + "datasource": "${DS_TEST-CLUSTER}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 33, "legend": { - "sort": "avg", - "rightSide": true, - "total": false, - "min": false, - "max": true, - "show": true, - "current": false, - "values": true, - "alignAsTable": true, - "avg": true, - "sortDesc": true - }, - "yaxes": [ - { - "logBase": 1, - "show": true, - "max": null, - "format": "short", - "min": null, - "label": null - }, - { - "logBase": 1, - "show": true, - "max": null, - "format": "short", - "min": null, - "label": null - } - ], - "xaxis": { - "show": true, - "values": [], - "mode": "time", - "name": null - }, - "seriesOverrides": [], - "percentage": false, - "type": "graph", - "error": false, - "editable": true, - "grid": {}, - "stack": false, - "timeShift": null, - "aliasColors": {}, - "lines": true, - "points": false, - "datasource": "${DS_TIDB-CLUSTER}", - "pointradius": 5 - }, - { - "bars": false, - "timeFrom": null, - "links": [], - "thresholds": [], - "nullPointMode": "null as zero", - "renderer": "flot", - "id": 34, - "linewidth": 2, - "steppedLine": false, + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, "targets": [ { - "intervalFactor": 2, - "expr": "histogram_quantile(1, sum(rate(tidb_tikvclient_txn_write_size_bucket[1m])) by (le, instance))", - "step": 30, - "refId": "A", - "legendFormat": "{{instance}}" - } - ], - "fill": 1, - "span": 6, - "title": "Size Per Txn", + "expr": "histogram_quantile(1, sum(rate(tidb_tikvclient_txn_write_kv_count_bucket[1m])) by (le, instance))", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "B", + "step": 30 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "KV Count Per Txn", "tooltip": { - "sort": 2, - "shared": true, - "value_type": "cumulative", - "msResolution": false - }, - "legend": { - "sort": "avg", - "rightSide": true, - "total": false, - "min": false, - "max": true, - "show": true, - "current": false, - "values": true, - "alignAsTable": true, - "avg": false, - "sortDesc": true - }, + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, "yaxes": [ { - "logBase": 10, - "show": true, - "max": null, - "format": "short", - "min": null, - "label": null - }, - { - "logBase": 1, - "show": false, - "max": null, - "format": "short", - "min": null, - "label": null - } - ], - "xaxis": { - "show": true, - "values": [], - "mode": "time", - "name": null - }, - "seriesOverrides": [], - "percentage": false, - "type": "graph", - "error": false, - "editable": true, - "grid": {}, - "stack": false, - "timeShift": null, - "aliasColors": {}, - "lines": true, - "points": false, - "datasource": "${DS_TIDB-CLUSTER}", - "pointradius": 5 - }, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, { - "bars": false, - "timeFrom": null, - "links": [], - "thresholds": [ - { - "colorMode": "critical", - "line": true, - "op": "gt", - "value": 0, - "fill": true - } - ], - "nullPointMode": "null as zero", - "renderer": "flot", - "linewidth": 1, - "steppedLine": false, - "id": 11, - "fill": 0, - "span": 6, - "title": "TiClient Region Error", - "tooltip": { - "sort": 0, - "shared": true, - "value_type": "cumulative", - "msResolution": false - }, + "aliasColors": {}, + "bars": false, + "datasource": "${DS_TEST-CLUSTER}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 34, "legend": { - "rightSide": true, - "total": false, - "min": false, - "max": true, - "show": true, - "current": true, - "hideEmpty": true, - "values": true, - "alignAsTable": true, - "avg": false, - "hideZero": true - }, + "alignAsTable": true, + "avg": false, + "current": false, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, "targets": [ { - "expr": "sum(rate(tidb_tikvclient_region_err_total[1m])) by (type, instance)", - "metric": "tidb_server_session_execute_parse_duration_count", - "step": 30, - "legendFormat": "{{instance}} {{type}}", - "intervalFactor": 2, - "refId": "A" - } - ], + "expr": "histogram_quantile(1, sum(rate(tidb_tikvclient_txn_write_size_bucket[1m])) by (le, instance))", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A", + "step": 30 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Size Per Txn", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, "yaxes": [ { - "logBase": 1, - "show": true, - "max": null, - "format": "ops", - "min": "0", - "label": null - }, - { - "logBase": 1, - "show": true, - "max": null, - "format": "short", - "min": null, - "label": null - } - ], - "xaxis": { - "show": true, - "values": [], - "mode": "time", - "name": null - }, - "seriesOverrides": [], - "percentage": false, - "type": "graph", - "error": false, - "editable": true, + "format": "short", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { "alert": { - "noDataState": "no_data", - "name": "TiDB TiClient Region Error alert", - "frequency": "60s", - "notifications": [], - "handler": 1, - "executionErrorState": "alerting", - "message": "TiDB report 'server is busy'", "conditions": [ { + "evaluator": { + "params": [ + 0 + ], + "type": "gt" + }, "operator": { "type": "and" - }, + }, "query": { - "params": [ - "A", - "1m", - "now" - ], + "datasourceId": 1, "model": { - "expr": "sum(rate(tidb_tikvclient_region_err_total[1m])) by (type, instance)", - "metric": "tidb_server_session_execute_parse_duration_count", - "step": 30, - "legendFormat": "{{instance}} {{type}}", - "intervalFactor": 2, - "refId": "A" - }, - "datasourceId": 1 - }, - "evaluator": { - "type": "gt", + "expr": "sum(rate(tidb_tikvclient_region_err_total[1m])) by (type, instance)", + "intervalFactor": 2, + "legendFormat": "{{instance}} {{type}}", + "metric": "tidb_server_session_execute_parse_duration_count", + "refId": "A", + "step": 30 + }, "params": [ - 0 + "A", + "1m", + "now" ] - }, + }, "reducer": { - "type": "avg", - "params": [] - }, + "params": [], + "type": "avg" + }, "type": "query" } - ] - }, - "grid": {}, - "stack": false, - "timeShift": null, - "aliasColors": {}, - "lines": true, - "points": false, - "datasource": "${DS_TIDB-CLUSTER}", - "pointradius": 5, - "decimals": 2 - }, - { - "bars": false, - "timeFrom": null, - "links": [], - "thresholds": [], - "nullPointMode": "null as zero", - "renderer": "flot", - "id": 32, - "linewidth": 2, - "steppedLine": false, + ], + "executionErrorState": "alerting", + "frequency": "60s", + "handler": 1, + "message": "TiDB report 'server is busy'", + "name": "TiDB TiClient Region Error alert", + "noDataState": "no_data", + "notifications": [] + }, + "aliasColors": {}, + "bars": false, + "datasource": "${DS_TEST-CLUSTER}", + "decimals": 2, + "editable": true, + "error": false, + "fill": 0, + "grid": {}, + "id": 11, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, "targets": [ { - "expr": "sum(rate(tidb_tikvclient_lock_resolver_actions_total[1m])) by (type)", - "metric": "tidb_tikvclient_lock_resolver_actions_total", - "step": 30, - "legendFormat": "{{type}}", - "intervalFactor": 2, - "refId": "A" - } - ], - "fill": 1, - "span": 6, - "title": "LockResolve", + "expr": "sum(rate(tidb_tikvclient_region_err_total[1m])) by (type, instance)", + "intervalFactor": 2, + "legendFormat": "{{instance}} {{type}}", + "metric": "tidb_server_session_execute_parse_duration_count", + "refId": "A", + "step": 30 + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 0 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "TiClient Region Error", "tooltip": { - "sort": 0, - "shared": true, - "value_type": "cumulative", - "msResolution": false - }, - "legend": { - "rightSide": false, - "total": false, - "min": false, - "max": true, - "show": true, - "current": false, - "hideEmpty": true, - "values": true, - "alignAsTable": false, - "avg": false, - "hideZero": true - }, + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, "yaxes": [ { - "logBase": 1, - "show": true, - "max": null, - "format": "short", - "min": "0", - "label": null - }, - { - "logBase": 1, - "show": true, - "max": null, - "format": "short", - "min": null, - "label": null - } - ], + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "${DS_TEST-CLUSTER}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 32, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(tidb_tikvclient_lock_resolver_actions_total[1m])) by (type)", + "intervalFactor": 2, + "legendFormat": "{{type}}", + "metric": "tidb_tikvclient_lock_resolver_actions_total", + "refId": "A", + "step": 30 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "LockResolve", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", "xaxis": { - "show": true, - "values": [], - "mode": "time", - "name": null - }, - "seriesOverrides": [], - "percentage": false, - "type": "graph", - "error": false, - "editable": true, - "grid": {}, - "stack": false, - "timeShift": null, - "aliasColors": {}, - "lines": true, - "points": false, - "datasource": "${DS_TIDB-CLUSTER}", - "pointradius": 5 + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] } - ], - "showTitle": true, - "collapse": false - }, + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "KV 2", + "titleSize": "h6" + }, { - "repeat": null, - "titleSize": "h6", - "repeatIteration": null, - "title": "PD Client", - "height": "250px", - "repeatRowId": null, + "collapse": true, + "height": "250px", "panels": [ { - "bars": false, - "timeFrom": null, - "links": [], - "thresholds": [], - "nullPointMode": "null as zero", - "renderer": "flot", - "id": 20, - "linewidth": 2, - "steppedLine": false, + "aliasColors": {}, + "bars": false, + "datasource": "${DS_TEST-CLUSTER}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 20, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 3, + "stack": false, + "steppedLine": false, "targets": [ { - "intervalFactor": 2, "expr": "sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket[1m])) by (type)", - "step": 10, - "refId": "A", - "legendFormat": "{{type}}" - } - ], - "fill": 1, - "span": 3, - "title": "PD Client cmd count", + "intervalFactor": 2, + "legendFormat": "{{type}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "PD Client cmd count", "tooltip": { - "sort": 0, - "shared": true, - "value_type": "cumulative", - "msResolution": false - }, - "legend": { - "total": false, - "show": true, - "max": false, - "min": false, - "current": false, - "values": false, - "avg": false - }, + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, "yaxes": [ { - "logBase": 1, - "show": true, - "max": null, - "format": "short", - "min": null, - "label": null - }, - { - "logBase": 1, - "show": true, - "max": null, - "format": "short", - "min": null, - "label": null - } - ], - "xaxis": { - "show": true, - "values": [], - "mode": "time", - "name": null - }, - "seriesOverrides": [], - "percentage": false, - "type": "graph", - "error": false, - "editable": true, - "grid": {}, - "stack": false, - "timeShift": null, - "aliasColors": {}, - "lines": true, - "points": false, - "datasource": "${DS_TIDB-CLUSTER}", - "pointradius": 5 - }, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, { - "bars": false, - "timeFrom": null, - "links": [], - "thresholds": [], - "nullPointMode": "null as zero", - "renderer": "flot", - "id": 35, - "linewidth": 1, - "steppedLine": false, + "aliasColors": {}, + "bars": false, + "datasource": "${DS_TEST-CLUSTER}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 35, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 3, + "stack": false, + "steppedLine": false, "targets": [ { - "intervalFactor": 2, - "expr": "histogram_quantile(0.999, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket[1m])) by (le, type))", - "step": 10, - "refId": "A", - "legendFormat": "{{type}}" - } - ], - "fill": 1, - "span": 3, - "title": "PD Client cmd duration 999", + "expr": "histogram_quantile(0.999, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket[1m])) by (le, type))", + "intervalFactor": 2, + "legendFormat": "{{type}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "PD Client cmd duration 999", "tooltip": { - "sort": 0, - "shared": true, - "value_type": "cumulative", - "msResolution": false - }, - "legend": { - "total": false, - "show": true, - "max": false, - "min": false, - "current": false, - "values": false, - "avg": false - }, + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, "yaxes": [ { - "logBase": 1, - "show": true, - "max": null, - "format": "s", - "min": null, - "label": null - }, - { - "logBase": 1, - "show": true, - "max": null, - "format": "short", - "min": null, - "label": null - } - ], - "xaxis": { - "show": true, - "values": [], - "mode": "time", - "name": null - }, - "seriesOverrides": [], - "percentage": false, - "type": "graph", - "error": false, - "editable": true, - "grid": {}, - "stack": false, - "timeShift": null, - "aliasColors": {}, - "lines": true, - "points": false, - "datasource": "${DS_TIDB-CLUSTER}", - "pointradius": 5 - }, + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, { - "bars": false, - "timeFrom": null, - "links": [], - "thresholds": [], - "nullPointMode": "null as zero", - "renderer": "flot", - "id": 45, - "linewidth": 1, - "steppedLine": false, + "aliasColors": {}, + "bars": false, + "datasource": "${DS_TEST-CLUSTER}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 45, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 3, + "stack": false, + "steppedLine": false, "targets": [ { - "intervalFactor": 2, "expr": "histogram_quantile(0.999, sum(rate(pd_client_request_handle_requests_duration_seconds_bucket[1m])) by (le, type))", - "step": 10, - "refId": "A", - "legendFormat": "{{type}}" - } - ], - "fill": 1, - "span": 3, - "title": "PD Client request duration 999", + "intervalFactor": 2, + "legendFormat": "{{type}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "PD Client request duration 999", "tooltip": { - "sort": 0, - "shared": true, - "value_type": "cumulative", - "msResolution": false - }, - "legend": { - "total": false, - "show": true, - "max": false, - "min": false, - "current": false, - "values": false, - "avg": false - }, + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, "yaxes": [ { - "logBase": 1, - "show": true, - "max": null, - "format": "s", - "min": null, - "label": null - }, - { - "logBase": 1, - "show": true, - "max": null, - "format": "short", - "min": null, - "label": null - } - ], - "xaxis": { - "show": true, - "values": [], - "mode": "time", - "name": null - }, - "seriesOverrides": [], - "percentage": false, - "type": "graph", - "error": false, - "editable": true, - "grid": {}, - "stack": false, - "timeShift": null, - "aliasColors": {}, - "lines": true, - "points": false, - "datasource": "${DS_TIDB-CLUSTER}", - "pointradius": 5 - }, + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, { - "bars": false, - "timeFrom": null, - "links": [], - "thresholds": [], - "nullPointMode": "null as zero", - "renderer": "flot", - "id": 43, - "linewidth": 2, - "steppedLine": false, + "aliasColors": {}, + "bars": false, + "datasource": "${DS_TEST-CLUSTER}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 43, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 3, + "stack": false, + "steppedLine": false, "targets": [ { - "intervalFactor": 2, "expr": "sum(rate(pd_client_cmd_handle_failed_cmds_duration_seconds_count[1m])) by (type)", - "step": 10, - "refId": "A", - "legendFormat": "{{type}}" - } - ], - "fill": 1, - "span": 3, - "title": "PD Client cmd fail", + "intervalFactor": 2, + "legendFormat": "{{type}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "PD Client cmd fail", "tooltip": { - "sort": 0, - "shared": true, - "value_type": "cumulative", - "msResolution": false - }, - "legend": { - "total": false, - "show": true, - "max": false, - "min": false, - "current": false, - "values": false, - "avg": false - }, + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, "yaxes": [ { - "logBase": 1, - "show": true, - "max": null, - "format": "ops", - "min": "0", - "label": null - }, - { - "logBase": 1, - "show": true, - "max": null, - "format": "short", - "min": null, - "label": null - } - ], - "xaxis": { - "show": true, - "values": [], - "mode": "time", - "name": null - }, - "seriesOverrides": [], - "percentage": false, - "type": "graph", - "error": false, - "editable": true, - "grid": {}, - "stack": false, - "timeShift": null, - "aliasColors": {}, - "lines": true, - "points": false, - "datasource": "${DS_TIDB-CLUSTER}", - "pointradius": 5 + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] } - ], - "showTitle": true, - "collapse": true - }, + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "PD Client", + "titleSize": "h6" + }, { - "repeat": null, - "titleSize": "h6", - "repeatIteration": null, - "title": "Schema Load", - "height": 250, - "repeatRowId": null, + "collapse": true, + "height": 250, "panels": [ { - "bars": false, - "timeFrom": null, - "links": [], - "thresholds": [ - { - "colorMode": "critical", - "line": true, - "op": "gt", - "value": 5, - "fill": true - } - ], - "nullPointMode": "null as zero", - "renderer": "flot", - "linewidth": 1, - "steppedLine": false, - "id": 27, - "fill": 1, - "span": 4, - "title": "Load Schema Duration", - "tooltip": { - "sort": 0, - "shared": true, - "value_type": "individual", - "msResolution": false - }, - "legend": { - "rightSide": true, - "total": false, - "min": false, - "max": false, - "show": true, - "current": false, - "values": false, - "alignAsTable": true, - "avg": false - }, - "targets": [ - { - "expr": "tidb_domain_load_schema_duration_sum / tidb_domain_load_schema_duration_count", - "metric": "", - "step": 10, - "legendFormat": "{{instance}}", - "intervalFactor": 2, - "refId": "A" - } - ], - "yaxes": [ - { - "logBase": 1, - "show": true, - "max": null, - "format": "s", - "min": "0", - "label": null - }, - { - "logBase": 1, - "show": true, - "max": null, - "format": "short", - "min": null, - "label": null - } - ], - "xaxis": { - "show": true, - "values": [], - "mode": "time", - "name": null - }, - "seriesOverrides": [], - "percentage": false, - "type": "graph", - "error": false, - "editable": true, "alert": { - "noDataState": "no_data", - "name": "Load Schema Duration alert", - "frequency": "60s", - "notifications": [], - "handler": 1, - "executionErrorState": "alerting", - "message": "TiDB load schema latency is over 5s", "conditions": [ { + "evaluator": { + "params": [ + 5 + ], + "type": "gt" + }, "operator": { "type": "and" - }, + }, "query": { - "params": [ - "A", - "5m", - "now" - ], + "datasourceId": 1, "model": { - "expr": "tidb_domain_load_schema_duration_sum / tidb_domain_load_schema_duration_count", - "metric": "", - "step": 10, - "legendFormat": "{{instance}}", - "intervalFactor": 2, - "refId": "A" - }, - "datasourceId": 1 - }, - "evaluator": { - "type": "gt", + "expr": "tidb_domain_load_schema_duration_sum / tidb_domain_load_schema_duration_count", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "metric": "", + "refId": "A", + "step": 10 + }, "params": [ - 5 + "A", + "5m", + "now" ] - }, + }, "reducer": { - "type": "avg", - "params": [] - }, + "params": [], + "type": "avg" + }, "type": "query" } - ] - }, - "grid": {}, - "stack": false, - "timeShift": null, - "aliasColors": {}, - "lines": true, - "points": false, - "datasource": "${DS_TIDB-CLUSTER}", - "pointradius": 5 - }, - { - "bars": false, - "timeFrom": null, - "links": [], - "thresholds": [ - { - "colorMode": "critical", - "line": true, - "op": "gt", - "value": 0, - "fill": true - } - ], - "nullPointMode": "null as zero", - "renderer": "flot", - "linewidth": 1, - "steppedLine": false, - "id": 28, - "fill": 0, - "span": 4, - "title": "Load Schema QPS", - "tooltip": { - "sort": 0, - "shared": true, - "value_type": "individual", - "msResolution": false - }, + ], + "executionErrorState": "alerting", + "frequency": "60s", + "handler": 1, + "message": "TiDB load schema latency is over 5s", + "name": "Load Schema Duration alert", + "noDataState": "no_data", + "notifications": [] + }, + "aliasColors": {}, + "bars": false, + "datasource": "${DS_TEST-CLUSTER}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 27, "legend": { - "rightSide": true, - "total": false, - "min": false, - "max": false, - "show": true, - "current": false, - "values": false, - "alignAsTable": true, - "avg": false - }, + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, "targets": [ { - "expr": "rate(tidb_domain_load_schema_total{type='succ'}[1m])", - "metric": "tidb_domain_load_schema_duration_count", - "step": 10, - "legendFormat": "{{instance}} succ", - "intervalFactor": 2, - "refId": "A" - }, - { - "hide": false, - "expr": "rate(tidb_domain_load_schema_total{type='failed'}[1m])", - "step": 10, - "legendFormat": "{{instance}} failed", - "intervalFactor": 2, - "refId": "B" - } - ], - "yaxes": [ + "expr": "tidb_domain_load_schema_duration_sum / tidb_domain_load_schema_duration_count", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "metric": "", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ { - "logBase": 10, - "show": true, - "max": null, - "format": "ops", - "min": "0", - "label": null - }, - { - "logBase": 1, - "show": true, - "max": null, - "format": "short", - "min": null, - "label": null - } - ], + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 5 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Load Schema Duration", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", "xaxis": { - "show": true, - "values": [], - "mode": "time", - "name": null - }, - "seriesOverrides": [ + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ { - "alias": "/.*failed/", - "bars": true - } - ], - "percentage": false, - "type": "graph", - "error": false, - "editable": true, + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { "alert": { - "noDataState": "no_data", - "name": "Load schema alert", - "frequency": "60s", - "notifications": [], - "handler": 1, - "executionErrorState": "alerting", - "message": "TiDB load schema fails", "conditions": [ { + "evaluator": { + "params": [ + 0 + ], + "type": "gt" + }, "operator": { "type": "and" - }, + }, "query": { - "params": [ - "B", - "1m", - "now" - ], + "datasourceId": 1, "model": { - "hide": false, - "expr": "rate(tidb_domain_load_schema_total{type='failed'}[1m])", - "step": 10, - "legendFormat": "{{instance}} failed", - "intervalFactor": 2, - "refId": "B" - }, - "datasourceId": 1 - }, - "evaluator": { - "type": "gt", + "expr": "rate(tidb_domain_load_schema_total{type='failed'}[1m])", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{instance}} failed", + "refId": "B", + "step": 10 + }, "params": [ - 0 + "B", + "1m", + "now" ] - }, + }, "reducer": { - "type": "avg", - "params": [] - }, + "params": [], + "type": "avg" + }, "type": "query" } - ] - }, - "grid": {}, - "stack": false, - "timeShift": null, - "aliasColors": {}, - "lines": true, - "points": false, - "datasource": "${DS_TIDB-CLUSTER}", - "pointradius": 5 - }, - { - "bars": false, - "timeFrom": null, - "links": [], - "thresholds": [ - { - "colorMode": "critical", - "line": true, - "op": "gt", - "value": 1, - "fill": true - } - ], - "nullPointMode": "null as zero", - "renderer": "flot", - "linewidth": 1, - "steppedLine": false, - "id": 29, - "fill": 0, - "span": 4, - "title": "Schema Lease Error Rate", - "tooltip": { - "sort": 0, - "shared": true, - "value_type": "individual", - "msResolution": false - }, + ], + "executionErrorState": "alerting", + "frequency": "60s", + "handler": 1, + "message": "TiDB load schema fails", + "name": "Load schema alert", + "noDataState": "no_data", + "notifications": [] + }, + "aliasColors": {}, + "bars": false, + "datasource": "${DS_TEST-CLUSTER}", + "editable": true, + "error": false, + "fill": 0, + "grid": {}, + "id": 28, "legend": { - "rightSide": true, - "total": false, - "min": false, - "max": false, - "show": true, - "current": false, - "hideEmpty": true, - "values": false, - "alignAsTable": true, - "avg": false, - "hideZero": true - }, + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*failed/", + "bars": true + } + ], + "span": 4, + "stack": false, + "steppedLine": false, "targets": [ { - "expr": "rate(tidb_server_schema_lease_error_counter[1m])", - "metric": "tidb_server_", - "step": 10, - "legendFormat": "{{instance}} {{type}}", - "intervalFactor": 2, - "refId": "A" + "expr": "rate(tidb_domain_load_schema_total{type='succ'}[1m])", + "intervalFactor": 2, + "legendFormat": "{{instance}} succ", + "metric": "tidb_domain_load_schema_duration_count", + "refId": "A", + "step": 10 + }, + { + "expr": "rate(tidb_domain_load_schema_total{type='failed'}[1m])", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{instance}} failed", + "refId": "B", + "step": 10 + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 0 } - ], + ], + "timeFrom": null, + "timeShift": null, + "title": "Load Schema QPS", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, "yaxes": [ { - "logBase": 1, - "show": true, - "max": null, - "format": "ops", - "min": "0", - "label": null - }, - { - "logBase": 1, - "show": true, - "max": null, - "format": "short", - "min": null, - "label": null - } - ], - "xaxis": { - "show": true, - "values": [], - "mode": "time", - "name": null - }, - "seriesOverrides": [], - "percentage": false, - "type": "graph", - "error": false, - "editable": true, + "format": "ops", + "label": null, + "logBase": 10, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { "alert": { - "noDataState": "no_data", - "name": "Schema Lease Error alert", - "frequency": "60s", - "notifications": [], - "handler": 1, - "executionErrorState": "alerting", - "message": "Schema lease error.", "conditions": [ { + "evaluator": { + "params": [ + 1 + ], + "type": "gt" + }, "operator": { "type": "and" - }, + }, "query": { - "params": [ - "A", - "1m", - "now" - ], + "datasourceId": 1, "model": { - "expr": "rate(tidb_server_schema_lease_error_counter[1m])", - "metric": "tidb_server_", - "step": 10, - "legendFormat": "{{instance}} {{type}}", - "intervalFactor": 2, - "refId": "A" - }, - "datasourceId": 1 - }, - "evaluator": { - "type": "gt", + "expr": "rate(tidb_server_schema_lease_error_counter[1m])", + "intervalFactor": 2, + "legendFormat": "{{instance}} {{type}}", + "metric": "tidb_server_", + "refId": "A", + "step": 10 + }, "params": [ - 1 + "A", + "1m", + "now" ] - }, + }, "reducer": { - "type": "avg", - "params": [] - }, + "params": [], + "type": "avg" + }, "type": "query" } - ] - }, - "grid": {}, - "stack": false, - "timeShift": null, - "aliasColors": {}, - "lines": true, - "points": false, - "datasource": "${DS_TIDB-CLUSTER}", - "pointradius": 5, - "decimals": 2 + ], + "executionErrorState": "alerting", + "frequency": "60s", + "handler": 1, + "message": "Schema lease error.", + "name": "Schema Lease Error alert", + "noDataState": "no_data", + "notifications": [] + }, + "aliasColors": {}, + "bars": false, + "datasource": "${DS_TEST-CLUSTER}", + "decimals": 2, + "editable": true, + "error": false, + "fill": 0, + "grid": {}, + "id": 29, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(tidb_server_schema_lease_error_counter[1m])", + "intervalFactor": 2, + "legendFormat": "{{instance}} {{type}}", + "metric": "tidb_server_", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 1 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Schema Lease Error Rate", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] } - ], - "showTitle": true, - "collapse": true - }, + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Schema Load", + "titleSize": "h6" + }, { - "repeat": null, - "titleSize": "h6", - "repeatIteration": null, - "title": "DDL", - "height": "250px", - "repeatRowId": null, + "collapse": true, + "height": "250px", "panels": [ { - "bars": false, - "timeFrom": null, - "links": [], - "thresholds": [], - "nullPointMode": "null as zero", - "renderer": "flot", - "id": 9, - "linewidth": 2, - "steppedLine": false, + "aliasColors": {}, + "bars": false, + "datasource": "${DS_TEST-CLUSTER}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, "targets": [ { - "intervalFactor": 2, - "expr": "histogram_quantile(0.95, sum(rate(tidb_ddl_handle_job_duration_seconds_bucket[1m])) by (le))", - "step": 10, - "refId": "A", - "legendFormat": "ddl handle job duration" - } - ], - "fill": 1, - "span": 4, - "title": "DDL Seconds 95", + "expr": "histogram_quantile(0.95, sum(rate(tidb_ddl_handle_job_duration_seconds_bucket[1m])) by (le))", + "intervalFactor": 2, + "legendFormat": "ddl handle job duration", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "DDL Seconds 95", "tooltip": { - "sort": 0, - "shared": true, - "value_type": "cumulative", - "msResolution": false - }, - "legend": { - "total": false, - "show": true, - "max": false, - "min": false, - "current": false, - "values": false, - "avg": false - }, + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, "yaxes": [ { - "logBase": 1, - "show": true, - "max": null, - "format": "s", - "min": "0", - "label": null - }, - { - "logBase": 1, - "show": true, - "max": null, - "format": "short", - "min": null, - "label": null - } - ], - "xaxis": { - "show": true, - "values": [], - "mode": "time", - "name": null - }, - "seriesOverrides": [], - "percentage": false, - "type": "graph", - "error": false, - "editable": true, - "grid": {}, - "stack": false, - "timeShift": null, - "aliasColors": {}, - "lines": true, - "points": false, - "datasource": "${DS_TIDB-CLUSTER}", - "pointradius": 5 - }, + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, { - "bars": false, - "timeFrom": null, - "links": [], - "thresholds": [], - "nullPointMode": "null as zero", - "renderer": "flot", - "id": 7, - "linewidth": 2, - "steppedLine": false, + "aliasColors": {}, + "bars": false, + "datasource": "${DS_TEST-CLUSTER}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 7, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sortDesc": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.95, sum(rate(tidb_ddl_batch_add_or_del_data_succ_bucket[1m])) by (le))", - "metric": "tidb_ddl_ba", - "step": 10, - "legendFormat": "ddl batch", - "intervalFactor": 2, - "refId": "A" - } - ], - "fill": 1, - "span": 4, - "title": "DDL Batch Seconds 95", + "expr": "histogram_quantile(0.95, sum(rate(tidb_ddl_batch_add_or_del_data_succ_bucket[1m])) by (le))", + "intervalFactor": 2, + "legendFormat": "ddl batch", + "metric": "tidb_ddl_ba", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "DDL Batch Seconds 95", "tooltip": { - "sort": 0, - "shared": true, - "value_type": "cumulative", - "msResolution": false - }, - "legend": { - "rightSide": false, - "total": false, - "min": false, - "max": false, - "show": true, - "current": false, - "values": false, - "alignAsTable": true, - "avg": false, - "sortDesc": true - }, + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, "yaxes": [ { - "logBase": 1, - "show": true, - "max": null, - "format": "s", - "min": "0", - "label": null - }, - { - "logBase": 1, - "show": true, - "max": null, - "format": "short", - "min": null, - "label": null - } - ], - "xaxis": { - "show": true, - "values": [], - "mode": "time", - "name": null - }, - "seriesOverrides": [], - "percentage": false, - "type": "graph", - "error": false, - "editable": true, - "grid": {}, - "stack": false, - "timeShift": null, - "aliasColors": {}, - "lines": true, - "points": false, - "datasource": "${DS_TIDB-CLUSTER}", - "pointradius": 5 - }, + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, { - "bars": false, - "timeFrom": null, - "links": [], - "thresholds": [], - "nullPointMode": "null as zero", - "renderer": "flot", - "id": 36, - "linewidth": 1, - "steppedLine": false, + "aliasColors": {}, + "bars": false, + "datasource": "${DS_TEST-CLUSTER}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 36, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, "targets": [ { - "intervalFactor": 2, - "expr": "sum(rate(tidb_server_session_retry_count[1m]))", - "step": 10, - "refId": "A", - "legendFormat": "session retry" - } - ], - "fill": 1, - "span": 4, - "title": "Session Retry", + "expr": "sum(rate(tidb_server_session_retry_count[1m]))", + "intervalFactor": 2, + "legendFormat": "session retry", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Session Retry", "tooltip": { - "sort": 0, - "shared": true, - "value_type": "individual", - "msResolution": false - }, - "legend": { - "total": false, - "show": true, - "max": false, - "min": false, - "current": false, - "values": false, - "avg": false - }, + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, "yaxes": [ { - "logBase": 1, - "show": true, - "max": null, - "format": "ops", - "min": "0", - "label": null - }, - { - "logBase": 1, - "show": true, - "max": null, - "format": "short", - "min": null, - "label": null - } - ], - "xaxis": { - "show": true, - "values": [], - "mode": "time", - "name": null - }, - "seriesOverrides": [], - "percentage": false, - "type": "graph", - "error": false, - "editable": true, - "grid": {}, - "stack": false, - "timeShift": null, - "aliasColors": {}, - "lines": true, - "points": false, - "datasource": "${DS_TIDB-CLUSTER}", - "pointradius": 5 - }, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, { - "bars": false, - "timeFrom": null, - "links": [], - "thresholds": [], - "nullPointMode": "null as zero", - "renderer": "flot", - "id": 38, - "linewidth": 2, - "steppedLine": false, + "aliasColors": {}, + "bars": false, + "datasource": "${DS_TEST-CLUSTER}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 38, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": "max", + "sortDesc": true, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 12, + "stack": false, + "steppedLine": false, "targets": [ { - "intervalFactor": 2, - "expr": "sum(rate(tidb_tikvclient_backoff_count[1m])) by (type)", - "step": 2, - "refId": "A", - "legendFormat": "{{type}}" - } - ], - "fill": 1, - "span": 12, - "title": "KV Backoff Count", + "expr": "sum(rate(tidb_tikvclient_backoff_count[1m])) by (type)", + "intervalFactor": 2, + "legendFormat": "{{type}}", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "KV Backoff Count", "tooltip": { - "sort": 0, - "shared": true, - "value_type": "cumulative", - "msResolution": false - }, + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "DDL", + "titleSize": "h6" + }, + { + "collapse": true, + "height": 250, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": "${DS_TEST-CLUSTER}", + "fill": 1, + "id": 46, "legend": { - "sort": "max", - "rightSide": true, - "total": true, - "min": false, - "max": true, - "show": true, - "current": false, - "hideEmpty": true, - "values": true, - "alignAsTable": true, - "avg": false, - "sortDesc": true, - "hideZero": true - }, + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.95, sum(rate(tidb_statistics_auto_analyze_duration_bucket[1m])) by (le))", + "intervalFactor": 2, + "legendFormat": "auto analyze duration", + "refId": "A", + "step": 30 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Auto Analyze Seconds 95", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, "yaxes": [ { - "logBase": 1, - "show": true, - "max": null, - "format": "short", - "min": null, - "label": null - }, - { - "logBase": 1, - "show": true, - "max": null, - "format": "short", - "min": null, - "label": null - } - ], + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "${DS_TEST-CLUSTER}", + "fill": 1, + "id": 47, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(tidb_statistics_auto_analyze_total{type='succ'}[1m])", + "intervalFactor": 2, + "legendFormat": "{{instance}} succ", + "refId": "A", + "step": 30 + }, + { + "expr": "rate(tidb_statistics_auto_analyze_total{type='failed'}[1m])", + "intervalFactor": 2, + "legendFormat": "{{instance}} failed", + "refId": "B", + "step": 30 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Auto Analyze QPS", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", "xaxis": { - "show": true, - "values": [], - "mode": "time", - "name": null - }, - "seriesOverrides": [], - "percentage": false, - "type": "graph", - "error": false, - "editable": true, - "grid": {}, - "stack": false, - "timeShift": null, - "aliasColors": {}, - "lines": true, - "points": false, - "datasource": "${DS_TIDB-CLUSTER}", - "pointradius": 5 + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] } - ], - "showTitle": true, - "collapse": true + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Statistics", + "titleSize": "h6" } - ], - "editMode": false, - "links": [ - { - "tags": [], - "type": "dashboards", - "icon": "external link" - } - ], - "tags": [], - "graphTooltip": 0, - "hideControls": false, - "title": "TiDB Cluster - tidb", - "editable": true, - "refresh": "30s", - "id": null, - "gnetId": null, + ], + "schemaVersion": 14, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-6h", + "to": "now" + }, "timepicker": { - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ], "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" ] - }, - "__inputs": [ - { - "description": "", - "pluginName": "Prometheus", - "label": "tidb-cluster", - "pluginId": "prometheus", - "type": "datasource", - "name": "DS_TIDB-CLUSTER" - } - ], - "version": 11, - "time": { - "to": "now", - "from": "now-6h" - }, - "__requires": [ - { - "version": "4.0.1", - "type": "grafana", - "id": "grafana", - "name": "Grafana" - }, - { - "version": "1.0.0", - "type": "datasource", - "id": "prometheus", - "name": "Prometheus" - } - ], - "timezone": "browser", - "schemaVersion": 14, - "annotations": { - "list": [] - }, - "templating": { - "list": [] - } -} + }, + "timezone": "browser", + "title": "TiDB Cluster - tidb", + "version": 0 +} \ No newline at end of file diff --git a/etc/tikv.json b/etc/tikv.json index 185140c27175..09666f8d1c25 100644 --- a/etc/tikv.json +++ b/etc/tikv.json @@ -54,12 +54,13 @@ "rows": [ { "collapse": false, - "height": "250px", + "height": "300", "panels": [ { "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, "editable": true, "error": false, "fill": 0, @@ -73,6 +74,7 @@ "min": false, "rightSide": true, "show": true, + "sideWidth": 250, "total": false, "values": true }, @@ -95,20 +97,12 @@ "steppedLine": false, "targets": [ { - "expr": "sum(tikv_pd_heartbeat_tick_total{type=\"leader\"}) by (instance,job)", + "expr": "sum(tikv_pd_heartbeat_tick_total{type=\"leader\"}) by (job)", "intervalFactor": 2, - "legendFormat": "{{instance}}-{{job}}", + "legendFormat": "{{job}}", "metric": "tikv_pd_heartbeat_tick_total", "refId": "A", "step": 10 - }, - { - "expr": "sum(tikv_pd_heartbeat_tick_total{type=\"leader\"}) ", - "hide": true, - "intervalFactor": 2, - "legendFormat": "total", - "refId": "B", - "step": 10 } ], "thresholds": [], @@ -151,6 +145,7 @@ "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, "editable": true, "error": false, "fill": 0, @@ -164,6 +159,7 @@ "min": false, "rightSide": true, "show": true, + "sideWidth": 250, "total": false, "values": true }, @@ -181,9 +177,9 @@ "steppedLine": false, "targets": [ { - "expr": "sum(tikv_pd_heartbeat_tick_total{type=\"region\"}) by (job,instance)", + "expr": "sum(tikv_pd_heartbeat_tick_total{type=\"region\"}) by (job)", "intervalFactor": 2, - "legendFormat": "{{instance}}-{{job}}", + "legendFormat": "{{job}}", "refId": "A", "step": 10 } @@ -228,6 +224,7 @@ "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, "editable": true, "error": false, "fill": 3, @@ -242,6 +239,7 @@ "min": false, "rightSide": true, "show": true, + "sideWidth": 250, "total": false, "values": true }, @@ -306,6 +304,7 @@ "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, "editable": true, "error": false, "fill": 5, @@ -319,6 +318,7 @@ "min": false, "rightSide": true, "show": true, + "sideWidth": 250, "sort": "current", "sortDesc": true, "total": false, @@ -397,12 +397,12 @@ "query": { "datasourceId": 1, "model": { - "expr": "sum(rate(tikv_channel_full_total[1m])) by (type, job)", + "expr": "sum(rate(tikv_channel_full_total[1m])) by (job, type)", "intervalFactor": 2, - "legendFormat": "{{job}}-{{type}}", + "legendFormat": "{{job}} - {{type}}", "metric": "", "refId": "A", - "step": 30 + "step": 10 }, "params": [ "A", @@ -428,6 +428,7 @@ "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, "editable": true, "error": false, "fill": 3, @@ -443,6 +444,7 @@ "min": false, "rightSide": true, "show": true, + "sideWidth": 250, "total": false, "values": true }, @@ -460,9 +462,9 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_channel_full_total[1m])) by (type, job)", + "expr": "sum(rate(tikv_channel_full_total[1m])) by (job, type)", "intervalFactor": 2, - "legendFormat": "{{job}}-{{type}}", + "legendFormat": "{{job}} - {{type}}", "metric": "", "refId": "A", "step": 10 @@ -516,6 +518,7 @@ "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, "editable": true, "error": false, "fill": 0, @@ -531,6 +534,7 @@ "min": false, "rightSide": true, "show": true, + "sideWidth": 250, "sort": "max", "sortDesc": true, "total": false, @@ -552,7 +556,7 @@ { "expr": "sum(rate(tikv_server_report_failure_msg_total[1m])) by (type,instance,job,store_id)", "intervalFactor": 2, - "legendFormat": "{{job}}-{{type}}-to-{{store_id}}", + "legendFormat": "{{job}} - {{type}} - to - {{store_id}}", "metric": "tikv_server_raft_store_msg_total", "refId": "A", "step": 10 @@ -561,7 +565,7 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "server report failure msg", + "title": "server report failures", "tooltip": { "msResolution": false, "shared": true, @@ -598,6 +602,7 @@ "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, "editable": true, "error": false, "fill": 1, @@ -611,6 +616,7 @@ "min": false, "rightSide": true, "show": true, + "sideWidth": 250, "total": false, "values": true }, @@ -628,7 +634,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.8, sum(rate(tikv_region_written_keys_bucket[1m])) by (le, job))", + "expr": "sum(rate(tikv_region_written_keys_sum[1m])) by (job) / sum(rate(tikv_region_written_keys_count[1m])) by (job)", "intervalFactor": 2, "legendFormat": "{{job}}", "metric": "tikv_region_written_keys_bucket", @@ -639,7 +645,7 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "80% region written keys", + "title": "region average written keys", "tooltip": { "msResolution": false, "shared": true, @@ -676,6 +682,7 @@ "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, "editable": true, "error": false, "fill": 1, @@ -690,6 +697,7 @@ "min": false, "rightSide": true, "show": true, + "sideWidth": 250, "total": false, "values": true }, @@ -707,7 +715,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.80, sum(rate(tikv_region_written_bytes_bucket[1m])) by (le, job))", + "expr": "sum(rate(tikv_region_written_bytes_sum[1m])) by (job) / sum(rate(tikv_region_written_bytes_count[1m])) by (job)", "intervalFactor": 2, "legendFormat": "{{job}}", "metric": "tikv_regi", @@ -718,7 +726,7 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "80% region written bytes", + "title": "region average written bytes", "tooltip": { "msResolution": false, "shared": true, @@ -755,6 +763,7 @@ "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, "editable": true, "error": false, "fill": 0, @@ -769,6 +778,7 @@ "min": false, "rightSide": true, "show": true, + "sideWidth": 250, "total": false, "values": true }, @@ -797,7 +807,7 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "active leaders", + "title": "active written leaders", "tooltip": { "msResolution": false, "shared": true, @@ -829,87 +839,34 @@ "show": false } ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Server", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ + }, { - "alert": { - "conditions": [ - { - "evaluator": { - "params": [ - 1 - ], - "type": "gt" - }, - "operator": { - "type": "and" - }, - "query": { - "datasourceId": 1, - "model": { - "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_raft_process_duration_secs_bucket{type='ready'}[1m])) by (le))", - "intervalFactor": 2, - "legendFormat": "99%", - "refId": "A", - "step": 30 - }, - "params": [ - "A", - "1m", - "now" - ] - }, - "reducer": { - "params": [], - "type": "max" - }, - "type": "query" - } - ], - "executionErrorState": "alerting", - "frequency": "60s", - "handler": 1, - "message": "TiKV raft process ready duration 99th percentile is above 1s", - "name": "TiKV raft process ready duration 99th percentile alert", - "noDataState": "no_data", - "notifications": [] - }, "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, "editable": true, "error": false, "fill": 0, "grid": {}, - "id": 12, + "id": 1481, "legend": { "alignAsTable": true, "avg": false, "current": true, + "hideEmpty": true, "max": true, "min": false, "rightSide": true, "show": true, - "sort": "max", - "sortDesc": true, + "sideWidth": 250, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], - "nullPointMode": "null", + "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, @@ -920,44 +877,47 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_raft_process_duration_secs_bucket{type='ready'}[1m])) by (le))", + "expr": "histogram_quantile(1.0, sum(rate(tikv_raftstore_region_size_bucket[1m])) by (le))", "intervalFactor": 2, - "legendFormat": "99%", + "legendFormat": "max", + "metric": "", "refId": "A", "step": 10 }, { - "expr": "histogram_quantile(0.95, sum(rate(tikv_raftstore_raft_process_duration_secs_bucket{type='ready'}[1m])) by (le))", + "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_region_size_bucket[1m])) by (le))", + "intervalFactor": 2, + "legendFormat": "99%", + "metric": "", + "refId": "B", + "step": 10 + }, + { + "expr": "histogram_quantile(0.95, sum(rate(tikv_raftstore_region_size_bucket[1m])) by (le))", "intervalFactor": 2, "legendFormat": "95%", + "metric": "", "refId": "C", "step": 10 }, { - "expr": "sum(rate(tikv_raftstore_raft_process_duration_secs_sum{type='ready'}[1m])) / sum(rate(tikv_raftstore_raft_process_duration_secs_count{type='ready'}[1m]))", + "expr": "sum(rate(tikv_raftstore_region_size_sum[1m])) / sum(rate(tikv_raftstore_region_size_count[1m])) ", "intervalFactor": 2, "legendFormat": "avg", - "refId": "B", + "metric": "", + "refId": "D", "step": 10 } ], - "thresholds": [ - { - "colorMode": "critical", - "fill": true, - "line": true, - "op": "gt", - "value": 1 - } - ], + "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "raft process ready duration", + "title": "approximate region size", "tooltip": { "msResolution": false, "shared": true, - "sort": 0, - "value_type": "cumulative" + "sort": 2, + "value_type": "individual" }, "type": "graph", "xaxis": { @@ -968,7 +928,7 @@ }, "yaxes": [ { - "format": "s", + "format": "bytes", "label": null, "logBase": 1, "max": null, @@ -981,19 +941,32 @@ "logBase": 1, "max": null, "min": null, - "show": true + "show": false } ] - }, + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Server", + "titleSize": "h6" + }, + { + "collapse": true, + "height": "300", + "panels": [ { "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, "editable": true, "error": false, "fill": 0, "grid": {}, - "id": 118, + "id": 1164, "legend": { "alignAsTable": true, "avg": false, @@ -1002,6 +975,7 @@ "min": false, "rightSide": true, "show": true, + "sideWidth": 250, "sort": "max", "sortDesc": true, "total": false, @@ -1021,11 +995,25 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.95, sum(rate(tikv_raftstore_raft_process_duration_secs_bucket{type='ready'}[1m])) by (le, job))", + "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_raft_process_duration_secs_bucket{type='tick'}[1m])) by (le))", "intervalFactor": 2, - "legendFormat": "{{job}}", + "legendFormat": "99%", + "refId": "A", + "step": 4 + }, + { + "expr": "histogram_quantile(0.95, sum(rate(tikv_raftstore_raft_process_duration_secs_bucket{type='tick'}[1m])) by (le))", + "intervalFactor": 2, + "legendFormat": "95%", "refId": "C", - "step": 10 + "step": 4 + }, + { + "expr": "sum(rate(tikv_raftstore_raft_process_duration_secs_sum{type='tick'}[1m])) / sum(rate(tikv_raftstore_raft_process_duration_secs_count{type='tick'}[1m]))", + "intervalFactor": 2, + "legendFormat": "avg", + "refId": "B", + "step": 4 } ], "thresholds": [ @@ -1039,7 +1027,7 @@ ], "timeFrom": null, "timeShift": null, - "title": "95% raft process ready duration", + "title": "raft process tick duration", "tooltip": { "msResolution": false, "shared": true, @@ -1076,11 +1064,12 @@ "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, "editable": true, "error": false, - "fill": 1, + "fill": 0, "grid": {}, - "id": 5, + "id": 1165, "legend": { "alignAsTable": true, "avg": false, @@ -1089,11 +1078,14 @@ "min": false, "rightSide": true, "show": true, + "sideWidth": 300, + "sort": "max", + "sortDesc": true, "total": false, "values": true }, "lines": true, - "linewidth": 2, + "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, @@ -1106,18 +1098,25 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_raftstore_raft_ready_handled_total[1m])) by (type)", + "expr": "histogram_quantile(0.95, sum(rate(tikv_raftstore_raft_process_duration_secs_bucket{type='tick'}[1m])) by (le, job))", "intervalFactor": 2, - "legendFormat": "{{type}}", - "metric": "tikv_raftstore_raft_ready_handled_total", - "refId": "A", - "step": 10 + "legendFormat": "{{job}}", + "refId": "C", + "step": 4 + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 1 } ], - "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "raft ready handled", + "title": "95% raft process tick duration", "tooltip": { "msResolution": false, "shared": true, @@ -1133,11 +1132,11 @@ }, "yaxes": [ { - "format": "short", + "format": "s", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -1151,14 +1150,57 @@ ] }, { + "alert": { + "conditions": [ + { + "evaluator": { + "params": [ + 1 + ], + "type": "gt" + }, + "operator": { + "type": "and" + }, + "query": { + "datasourceId": 1, + "model": { + "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_raft_process_duration_secs_bucket{type='ready'}[1m])) by (le))", + "intervalFactor": 2, + "legendFormat": "99%", + "refId": "A", + "step": 4 + }, + "params": [ + "A", + "1m", + "now" + ] + }, + "reducer": { + "params": [], + "type": "max" + }, + "type": "query" + } + ], + "executionErrorState": "alerting", + "frequency": "60s", + "handler": 1, + "message": "TiKV raft process ready duration 99th percentile is above 1s", + "name": "TiKV raft process ready duration 99th percentile alert", + "noDataState": "no_data", + "notifications": [] + }, "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, "editable": true, "error": false, - "fill": 1, + "fill": 0, "grid": {}, - "id": 7, + "id": 12, "legend": { "alignAsTable": true, "avg": false, @@ -1167,11 +1209,14 @@ "min": false, "rightSide": true, "show": true, + "sideWidth": 250, + "sort": "max", + "sortDesc": true, "total": false, "values": true }, "lines": true, - "linewidth": 2, + "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, @@ -1184,18 +1229,39 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_raftstore_proposal_total{type=~\"local_read|normal|read_index\"}[1m])) by (type)", + "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_raft_process_duration_secs_bucket{type='ready'}[1m])) by (le))", "intervalFactor": 2, - "legendFormat": "{{type}}", - "metric": "tikv_raftstore_proposal_total", + "legendFormat": "99%", "refId": "A", - "step": 10 + "step": 4 + }, + { + "expr": "histogram_quantile(0.95, sum(rate(tikv_raftstore_raft_process_duration_secs_bucket{type='ready'}[1m])) by (le))", + "intervalFactor": 2, + "legendFormat": "95%", + "refId": "C", + "step": 4 + }, + { + "expr": "sum(rate(tikv_raftstore_raft_process_duration_secs_sum{type='ready'}[1m])) / sum(rate(tikv_raftstore_raft_process_duration_secs_count{type='ready'}[1m]))", + "intervalFactor": 2, + "legendFormat": "avg", + "refId": "B", + "step": 4 + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 1 } ], - "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "raft proposal read/write", + "title": "raft process ready duration", "tooltip": { "msResolution": false, "shared": true, @@ -1211,11 +1277,11 @@ }, "yaxes": [ { - "format": "short", + "format": "s", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -1232,11 +1298,12 @@ "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, "editable": true, "error": false, - "fill": 1, + "fill": 0, "grid": {}, - "id": 76, + "id": 118, "legend": { "alignAsTable": true, "avg": false, @@ -1245,11 +1312,14 @@ "min": false, "rightSide": true, "show": true, + "sideWidth": 300, + "sort": "max", + "sortDesc": true, "total": false, "values": true }, "lines": true, - "linewidth": 2, + "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, @@ -1262,18 +1332,25 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_raftstore_proposal_total{type=~\"conf_change|transfer_leader\"}[1m])) by (type)", + "expr": "histogram_quantile(0.95, sum(rate(tikv_raftstore_raft_process_duration_secs_bucket{type='ready'}[1m])) by (le, job))", "intervalFactor": 2, - "legendFormat": "{{type}}", - "metric": "tikv_raftstore_proposal_total", - "refId": "A", - "step": 10 + "legendFormat": "{{job}}", + "refId": "C", + "step": 4 + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 1 } ], - "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "raft proposal admin", + "title": "95% raft process ready duration", "tooltip": { "msResolution": false, "shared": true, @@ -1289,11 +1366,11 @@ }, "yaxes": [ { - "format": "short", + "format": "s", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -1310,11 +1387,12 @@ "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, "editable": true, "error": false, "fill": 1, "grid": {}, - "id": 11, + "id": 5, "legend": { "alignAsTable": true, "avg": false, @@ -1323,6 +1401,7 @@ "min": false, "rightSide": true, "show": true, + "sideWidth": 300, "total": false, "values": true }, @@ -1340,17 +1419,18 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_raftstore_raft_sent_message_total[1m])) by (type)", + "expr": "sum(rate(tikv_raftstore_raft_ready_handled_total[1m])) by (type)", "intervalFactor": 2, "legendFormat": "{{type}}", + "metric": "tikv_raftstore_raft_ready_handled_total", "refId": "A", - "step": 10 + "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "raft sent message", + "title": "raft ready handled", "tooltip": { "msResolution": false, "shared": true, @@ -1366,7 +1446,7 @@ }, "yaxes": [ { - "format": "short", + "format": "ops", "label": null, "logBase": 1, "max": null, @@ -1387,11 +1467,9 @@ "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", - "editable": true, - "error": false, + "decimals": 1, "fill": 1, - "grid": {}, - "id": 25, + "id": 108, "legend": { "alignAsTable": true, "avg": false, @@ -1400,11 +1478,12 @@ "min": false, "rightSide": true, "show": true, + "sideWidth": 300, "total": false, "values": true }, "lines": true, - "linewidth": 2, + "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, @@ -1417,22 +1496,22 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_raftstore_raft_sent_message_total{type=\"vote\"}[1m])) by (job)", + "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_apply_proposal_bucket[1m])) by (le, job))", "intervalFactor": 2, - "legendFormat": "{{job}}-vote", + "legendFormat": "{{job}}", + "metric": "", "refId": "A", - "step": 10 + "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "vote", + "title": "raft proposals per ready", "tooltip": { - "msResolution": false, "shared": true, "sort": 0, - "value_type": "cumulative" + "value_type": "individual" }, "type": "graph", "xaxis": { @@ -1447,7 +1526,7 @@ "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { @@ -1464,11 +1543,12 @@ "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, "editable": true, "error": false, "fill": 1, "grid": {}, - "id": 70, + "id": 76, "legend": { "alignAsTable": true, "avg": false, @@ -1477,6 +1557,7 @@ "min": false, "rightSide": true, "show": true, + "sideWidth": 300, "total": false, "values": true }, @@ -1494,18 +1575,18 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_raftstore_check_split_total{type!=\"ignore\"}[1m])) by (type)", + "expr": "sum(rate(tikv_raftstore_proposal_total{type=~\"conf_change|transfer_leader\"}[1m])) by (type)", "intervalFactor": 2, "legendFormat": "{{type}}", - "metric": "tikv_raftstore_check_split_total", + "metric": "tikv_raftstore_proposal_total", "refId": "A", - "step": 10 + "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "check split", + "title": "raft admin proposals", "tooltip": { "msResolution": false, "shared": true, @@ -1521,7 +1602,7 @@ }, "yaxes": [ { - "format": "short", + "format": "ops", "label": null, "logBase": 1, "max": null, @@ -1542,27 +1623,28 @@ "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, "editable": true, "error": false, "fill": 1, "grid": {}, - "id": 71, + "id": 7, "legend": { "alignAsTable": true, "avg": false, "current": true, - "hideEmpty": true, "max": true, "min": false, "rightSide": true, "show": true, + "sideWidth": 300, "total": false, "values": true }, "lines": true, "linewidth": 2, "links": [], - "nullPointMode": "null as zero", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, @@ -1573,18 +1655,18 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.9999, sum(rate(tikv_raftstore_check_split_duration_seconds_bucket[1m])) by (le))", + "expr": "sum(rate(tikv_raftstore_proposal_total{type=~\"local_read|normal|read_index\"}[1m])) by (type)", "intervalFactor": 2, - "legendFormat": "duration", - "metric": "tikv_raftstore_check_split_duration_seconds_bucket", + "legendFormat": "{{type}}", + "metric": "tikv_raftstore_proposal_total", "refId": "A", - "step": 10 + "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "check split duration", + "title": "raft read/write proposals", "tooltip": { "msResolution": false, "shared": true, @@ -1600,11 +1682,11 @@ }, "yaxes": [ { - "format": "s", + "format": "ops", "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { @@ -1621,11 +1703,12 @@ "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, "editable": true, "error": false, "fill": 1, "grid": {}, - "id": 72, + "id": 119, "legend": { "alignAsTable": true, "avg": false, @@ -1634,13 +1717,14 @@ "min": false, "rightSide": true, "show": true, + "sideWidth": 300, "total": false, "values": true }, "lines": true, "linewidth": 2, "links": [], - "nullPointMode": "null as zero", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, @@ -1651,18 +1735,18 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.9999, sum(rate(tikv_raftstore_log_lag_bucket[1m])) by (le))", + "expr": "sum(rate(tikv_raftstore_proposal_total{type=~\"local_read|read_index\"}[1m])) by (job)", "intervalFactor": 2, - "legendFormat": "lag", - "metric": "tikv_raftstore_log_lag_bucket", + "legendFormat": "{{job}}", + "metric": "", "refId": "A", - "step": 10 + "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "99.99% raft log lag", + "title": "raft read proposals per server", "tooltip": { "msResolution": false, "shared": true, @@ -1678,11 +1762,11 @@ }, "yaxes": [ { - "format": "short", + "format": "ops", "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { @@ -1699,11 +1783,12 @@ "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, "editable": true, "error": false, "fill": 1, "grid": {}, - "id": 73, + "id": 120, "legend": { "alignAsTable": true, "avg": false, @@ -1712,6 +1797,7 @@ "min": false, "rightSide": true, "show": true, + "sideWidth": 300, "total": false, "values": true }, @@ -1729,18 +1815,18 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.9999, sum(rate(tikv_raftstore_propose_log_size_bucket[1m])) by (le))", + "expr": "sum(rate(tikv_raftstore_proposal_total{type=\"normal\"}[1m])) by (job)", "intervalFactor": 2, - "legendFormat": "size", - "metric": "tikv_raftstore_propose_log_size_bucket", + "legendFormat": "{{job}}", + "metric": "tikv_raftstore_proposal_total", "refId": "A", - "step": 10 + "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "99.99% raft log size", + "title": "raft write proposals per server", "tooltip": { "msResolution": false, "shared": true, @@ -1756,7 +1842,7 @@ }, "yaxes": [ { - "format": "bytes", + "format": "ops", "label": null, "logBase": 1, "max": null, @@ -1777,11 +1863,12 @@ "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, "editable": true, "error": false, "fill": 1, "grid": {}, - "id": 21, + "id": 72, "legend": { "alignAsTable": true, "avg": false, @@ -1790,13 +1877,14 @@ "min": false, "rightSide": true, "show": true, + "sideWidth": 300, "total": false, "values": true }, "lines": true, "linewidth": 2, "links": [], - "nullPointMode": "null", + "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, @@ -1807,18 +1895,18 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_raftstore_admin_cmd_total{status=\"success\", type=\"compact\"}[1m])) by (type, job)", + "expr": "histogram_quantile(0.9999, sum(rate(tikv_raftstore_log_lag_bucket[1m])) by (le, job))", "intervalFactor": 2, "legendFormat": "{{job}}", - "metric": "tikv_raftstore_admin_cmd_total", + "metric": "tikv_raftstore_log_lag_bucket", "refId": "A", - "step": 10 + "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "raft compact command", + "title": "99.99% raft log lag", "tooltip": { "msResolution": false, "shared": true, @@ -1834,11 +1922,11 @@ }, "yaxes": [ { - "format": "ops", + "format": "short", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -1855,11 +1943,12 @@ "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, "editable": true, "error": false, "fill": 1, "grid": {}, - "id": 77, + "id": 73, "legend": { "alignAsTable": true, "avg": false, @@ -1868,6 +1957,7 @@ "min": false, "rightSide": true, "show": true, + "sideWidth": 300, "total": false, "values": true }, @@ -1885,18 +1975,18 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_raftstore_admin_cmd_total{status=\"success\", type!=\"compact\"}[1m])) by (type)", + "expr": "histogram_quantile(0.9999, sum(rate(tikv_raftstore_propose_log_size_bucket[1m])) by (le, job))", "intervalFactor": 2, - "legendFormat": "{{type}}", - "metric": "tikv_raftstore_admin_cmd_total", + "legendFormat": "{{job}}", + "metric": "tikv_raftstore_propose_log_size_bucket", "refId": "A", - "step": 10 + "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "raft admin command", + "title": "99.99% raft log size", "tooltip": { "msResolution": false, "shared": true, @@ -1912,11 +2002,11 @@ }, "yaxes": [ { - "format": "ops", + "format": "bytes", "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { @@ -1933,21 +2023,26 @@ "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, + "editable": true, + "error": false, "fill": 1, - "id": 106, + "grid": {}, + "id": 77, "legend": { "alignAsTable": true, "avg": false, - "current": false, - "max": false, + "current": true, + "max": true, "min": false, "rightSide": true, "show": true, + "sideWidth": 300, "total": false, - "values": false + "values": true }, "lines": true, - "linewidth": 1, + "linewidth": 2, "links": [], "nullPointMode": "null", "percentage": false, @@ -1960,21 +2055,23 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_server_raft_message_recv_total[1m])) by (job)", + "expr": "sum(rate(tikv_raftstore_admin_cmd_total{status=\"success\", type!=\"compact\"}[1m])) by (type)", "intervalFactor": 2, - "legendFormat": "{{job}}", + "legendFormat": "{{type}}", + "metric": "tikv_raftstore_admin_cmd_total", "refId": "A", - "step": 10 + "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "raft message recv total", + "title": "raft admin commands", "tooltip": { + "msResolution": false, "shared": true, "sort": 0, - "value_type": "individual" + "value_type": "cumulative" }, "type": "graph", "xaxis": { @@ -1985,11 +2082,11 @@ }, "yaxes": [ { - "format": "short", + "format": "ops", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -2006,11 +2103,12 @@ "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, "editable": true, "error": false, "fill": 1, "grid": {}, - "id": 119, + "id": 21, "legend": { "alignAsTable": true, "avg": false, @@ -2019,6 +2117,7 @@ "min": false, "rightSide": true, "show": true, + "sideWidth": 300, "total": false, "values": true }, @@ -2036,18 +2135,18 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_raftstore_proposal_total{type=~\"local_read|read_index\"}[1m])) by (type, job)", + "expr": "sum(rate(tikv_raftstore_admin_cmd_total{status=\"success\", type=\"compact\"}[1m])) by (job)", "intervalFactor": 2, - "legendFormat": "{{job}}-{{type}}", - "metric": "tikv_raftstore_proposal_total", + "legendFormat": "{{job}}", + "metric": "tikv_raftstore_admin_cmd_total", "refId": "A", - "step": 10 + "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "raft proposal read per server", + "title": "raft compact commands", "tooltip": { "msResolution": false, "shared": true, @@ -2063,7 +2162,7 @@ }, "yaxes": [ { - "format": "short", + "format": "ops", "label": null, "logBase": 1, "max": null, @@ -2084,11 +2183,12 @@ "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, "editable": true, "error": false, "fill": 1, "grid": {}, - "id": 120, + "id": 70, "legend": { "alignAsTable": true, "avg": false, @@ -2097,6 +2197,7 @@ "min": false, "rightSide": true, "show": true, + "sideWidth": 300, "total": false, "values": true }, @@ -2114,18 +2215,18 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_raftstore_proposal_total{type=\"normal\"}[1m])) by (job)", + "expr": "sum(rate(tikv_raftstore_check_split_total{type!=\"ignore\"}[1m])) by (type)", "intervalFactor": 2, - "legendFormat": "{{job}}", - "metric": "tikv_raftstore_proposal_total", + "legendFormat": "{{type}}", + "metric": "tikv_raftstore_check_split_total", "refId": "A", - "step": 10 + "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "raft proposal write per server", + "title": "check split", "tooltip": { "msResolution": false, "shared": true, @@ -2141,7 +2242,7 @@ }, "yaxes": [ { - "format": "short", + "format": "ops", "label": null, "logBase": 1, "max": null, @@ -2162,23 +2263,29 @@ "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, + "editable": true, + "error": false, "fill": 1, - "id": 108, + "grid": {}, + "id": 71, "legend": { "alignAsTable": true, "avg": false, - "current": false, - "max": false, + "current": true, + "hideEmpty": true, + "max": true, "min": false, "rightSide": true, "show": true, + "sideWidth": 300, "total": false, - "values": false + "values": true }, "lines": true, - "linewidth": 1, + "linewidth": 2, "links": [], - "nullPointMode": "null", + "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, @@ -2189,22 +2296,23 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_apply_proposal_bucket[1m])) by (le, job))", + "expr": "histogram_quantile(0.9999, sum(rate(tikv_raftstore_check_split_duration_seconds_bucket[1m])) by (le, job))", "intervalFactor": 2, "legendFormat": "{{job}}", - "metric": "", + "metric": "tikv_raftstore_check_split_duration_seconds_bucket", "refId": "A", - "step": 10 + "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "proposes per ready", + "title": "99.99% check split duration", "tooltip": { + "msResolution": false, "shared": true, "sort": 0, - "value_type": "individual" + "value_type": "cumulative" }, "type": "graph", "xaxis": { @@ -2215,11 +2323,11 @@ }, "yaxes": [ { - "format": "short", + "format": "s", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -2231,28 +2339,17 @@ "show": true } ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Raft", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ + }, { "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, "editable": true, "error": false, "fill": 1, "grid": {}, - "id": 31, + "id": 11, "legend": { "alignAsTable": true, "avg": false, @@ -2261,13 +2358,14 @@ "min": false, "rightSide": true, "show": true, + "sideWidth": 300, "total": false, "values": true }, "lines": true, - "linewidth": 1, + "linewidth": 2, "links": [], - "nullPointMode": "null as zero", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, @@ -2278,32 +2376,17 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_apply_log_duration_seconds_bucket[1m])) by (le))", + "expr": "sum(rate(tikv_raftstore_raft_sent_message_total[1m])) by (type)", "intervalFactor": 2, - "legendFormat": " 99%", - "metric": "", + "legendFormat": "{{type}}", "refId": "A", - "step": 10 - }, - { - "expr": "histogram_quantile(0.95, sum(rate(tikv_raftstore_apply_log_duration_seconds_bucket[1m])) by (le))", - "intervalFactor": 2, - "legendFormat": "95%", - "refId": "B", - "step": 10 - }, - { - "expr": "sum(rate(tikv_raftstore_apply_log_duration_seconds_sum[1m])) / sum(rate(tikv_raftstore_apply_log_duration_seconds_count[1m])) ", - "intervalFactor": 2, - "legendFormat": "avg", - "refId": "C", - "step": 10 + "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "apply log duration", + "title": "raft sent messages", "tooltip": { "msResolution": false, "shared": true, @@ -2319,7 +2402,7 @@ }, "yaxes": [ { - "format": "s", + "format": "ops", "label": null, "logBase": 1, "max": null, @@ -2340,11 +2423,9 @@ "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", - "editable": true, - "error": false, + "decimals": 1, "fill": 1, - "grid": {}, - "id": 32, + "id": 106, "legend": { "alignAsTable": true, "avg": false, @@ -2353,6 +2434,7 @@ "min": false, "rightSide": true, "show": true, + "sideWidth": 300, "total": false, "values": true }, @@ -2370,22 +2452,21 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.95, sum(rate(tikv_raftstore_apply_log_duration_seconds_bucket[1m])) by (le, job))", + "expr": "sum(rate(tikv_server_raft_message_recv_total[1m])) by (job)", "intervalFactor": 2, - "legendFormat": " {{job}}", + "legendFormat": "{{job}}", "refId": "A", - "step": 10 + "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "95% apply log duration", + "title": "raft recv messages per server", "tooltip": { - "msResolution": false, "shared": true, "sort": 0, - "value_type": "cumulative" + "value_type": "individual" }, "type": "graph", "xaxis": { @@ -2396,7 +2477,7 @@ }, "yaxes": [ { - "format": "s", + "format": "ops", "label": null, "logBase": 1, "max": null, @@ -2417,11 +2498,12 @@ "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, "editable": true, "error": false, "fill": 1, "grid": {}, - "id": 39, + "id": 25, "legend": { "alignAsTable": true, "avg": false, @@ -2430,11 +2512,12 @@ "min": false, "rightSide": true, "show": true, + "sideWidth": 300, "total": false, "values": true }, "lines": true, - "linewidth": 1, + "linewidth": 2, "links": [], "nullPointMode": "null", "percentage": false, @@ -2447,32 +2530,17 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_append_log_duration_seconds_bucket[1m])) by (le))", + "expr": "sum(rate(tikv_raftstore_raft_sent_message_total{type=\"vote\"}[1m])) by (job)", "intervalFactor": 2, - "legendFormat": " 99%", - "metric": "", + "legendFormat": "{{job}}", "refId": "A", - "step": 10 - }, - { - "expr": "histogram_quantile(0.95, sum(rate(tikv_raftstore_append_log_duration_seconds_bucket[1m])) by (le))", - "intervalFactor": 2, - "legendFormat": "95%", - "refId": "B", - "step": 10 - }, - { - "expr": "sum(rate(tikv_raftstore_append_log_duration_seconds_sum[1m])) / sum(rate(tikv_raftstore_append_log_duration_seconds_count[1m]))", - "intervalFactor": 2, - "legendFormat": "avg", - "refId": "C", - "step": 10 + "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "append log duration", + "title": "vote", "tooltip": { "msResolution": false, "shared": true, @@ -2488,11 +2556,11 @@ }, "yaxes": [ { - "format": "s", + "format": "ops", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -2509,11 +2577,12 @@ "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, "editable": true, "error": false, "fill": 1, "grid": {}, - "id": 40, + "id": 1309, "legend": { "alignAsTable": true, "avg": false, @@ -2522,11 +2591,12 @@ "min": false, "rightSide": true, "show": true, + "sideWidth": 300, "total": false, "values": true }, "lines": true, - "linewidth": 1, + "linewidth": 2, "links": [], "nullPointMode": "null", "percentage": false, @@ -2539,17 +2609,17 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.95, sum(rate(tikv_raftstore_append_log_duration_seconds_bucket[1m])) by (le, job))", + "expr": "sum(rate(tikv_raftstore_raft_dropped_message_total[1m])) by (type)", "intervalFactor": 2, - "legendFormat": "{{job}} ", + "legendFormat": "{{type}}", "refId": "A", - "step": 10 + "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "95% append log duration", + "title": "raft dropped messages", "tooltip": { "msResolution": false, "shared": true, @@ -2565,7 +2635,7 @@ }, "yaxes": [ { - "format": "s", + "format": "ops", "label": null, "logBase": 1, "max": null, @@ -2581,7 +2651,19 @@ "show": true } ] - }, + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Raft", + "titleSize": "h6" + }, + { + "collapse": true, + "height": "250px", + "panels": [ { "aliasColors": {}, "bars": false, @@ -2590,7 +2672,7 @@ "error": false, "fill": 1, "grid": {}, - "id": 41, + "id": 31, "legend": { "alignAsTable": true, "avg": false, @@ -2603,9 +2685,9 @@ "values": true }, "lines": true, - "linewidth": 2, + "linewidth": 1, "links": [], - "nullPointMode": "null", + "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, @@ -2616,32 +2698,32 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_request_wait_time_duration_secs_bucket[1m])) by (le))", + "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_apply_log_duration_seconds_bucket[1m])) by (le))", "intervalFactor": 2, - "legendFormat": "99%", - "metric": "tikv_raftstore_request_wait_time_duration_secs_bucket", + "legendFormat": " 99%", + "metric": "", "refId": "A", - "step": 10 + "step": 4 }, { - "expr": "histogram_quantile(0.95, sum(rate(tikv_raftstore_request_wait_time_duration_secs_bucket[1m])) by (le))", + "expr": "histogram_quantile(0.95, sum(rate(tikv_raftstore_apply_log_duration_seconds_bucket[1m])) by (le))", "intervalFactor": 2, "legendFormat": "95%", "refId": "B", - "step": 10 + "step": 4 }, { - "expr": "sum(rate(tikv_raftstore_request_wait_time_duration_secs_sum[1m])) / sum(rate(tikv_raftstore_request_wait_time_duration_secs_count[1m]))", + "expr": "sum(rate(tikv_raftstore_apply_log_duration_seconds_sum[1m])) / sum(rate(tikv_raftstore_apply_log_duration_seconds_count[1m])) ", "intervalFactor": 2, "legendFormat": "avg", "refId": "C", - "step": 10 + "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "99% request wait duration", + "title": "apply log duration", "tooltip": { "msResolution": false, "shared": true, @@ -2682,7 +2764,7 @@ "error": false, "fill": 1, "grid": {}, - "id": 42, + "id": 32, "legend": { "alignAsTable": true, "avg": false, @@ -2695,7 +2777,7 @@ "values": true }, "lines": true, - "linewidth": 2, + "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, @@ -2708,17 +2790,17 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.95, sum(rate(tikv_raftstore_request_wait_time_duration_secs_bucket[1m])) by (le, job))", + "expr": "histogram_quantile(0.95, sum(rate(tikv_raftstore_apply_log_duration_seconds_bucket[1m])) by (le, job))", "intervalFactor": 2, - "legendFormat": "{{job}}", + "legendFormat": " {{job}}", "refId": "A", - "step": 10 + "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "95% request wait duration", + "title": "95% apply log duration", "tooltip": { "msResolution": false, "shared": true, @@ -2750,19 +2832,7 @@ "show": true } ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Raft Ready", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ + }, { "aliasColors": {}, "bars": false, @@ -2771,13 +2841,11 @@ "error": false, "fill": 1, "grid": {}, - "id": 2, + "id": 39, "legend": { "alignAsTable": true, "avg": false, "current": true, - "hideEmpty": true, - "hideZero": true, "max": true, "min": false, "rightSide": true, @@ -2788,7 +2856,7 @@ "lines": true, "linewidth": 1, "links": [], - "nullPointMode": "null as zero", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, @@ -2799,17 +2867,32 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_storage_command_total[1m])) by (type)", + "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_append_log_duration_seconds_bucket[1m])) by (le))", "intervalFactor": 2, - "legendFormat": "{{type}}", + "legendFormat": " 99%", + "metric": "", "refId": "A", - "step": 10 + "step": 4 + }, + { + "expr": "histogram_quantile(0.95, sum(rate(tikv_raftstore_append_log_duration_seconds_bucket[1m])) by (le))", + "intervalFactor": 2, + "legendFormat": "95%", + "refId": "B", + "step": 4 + }, + { + "expr": "sum(rate(tikv_raftstore_append_log_duration_seconds_sum[1m])) / sum(rate(tikv_raftstore_append_log_duration_seconds_count[1m]))", + "intervalFactor": 2, + "legendFormat": "avg", + "refId": "C", + "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "storage command", + "title": "append log duration", "tooltip": { "msResolution": false, "shared": true, @@ -2825,11 +2908,11 @@ }, "yaxes": [ { - "format": "ops", + "format": "s", "label": null, - "logBase": 10, + "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { @@ -2850,25 +2933,22 @@ "error": false, "fill": 1, "grid": {}, - "id": 8, + "id": 40, "legend": { "alignAsTable": true, "avg": false, "current": true, - "hideEmpty": true, - "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, - "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], - "nullPointMode": "null as zero", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, @@ -2879,20 +2959,19 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_storage_engine_async_request_total{status!~\"all|success\"}[1m])) by (type, status)", + "expr": "histogram_quantile(0.95, sum(rate(tikv_raftstore_append_log_duration_seconds_bucket[1m])) by (le, job))", "intervalFactor": 2, - "legendFormat": "{{type}}-{{status}}", - "metric": "tikv_raftstore_raft_process_duration_secs_bucket", + "legendFormat": "{{job}} ", "refId": "A", - "step": 10 + "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "storage async request error", + "title": "95% append log duration", "tooltip": { - "msResolution": true, + "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" @@ -2906,7 +2985,7 @@ }, "yaxes": [ { - "format": "short", + "format": "s", "label": null, "logBase": 1, "max": null, @@ -2931,12 +3010,11 @@ "error": false, "fill": 1, "grid": {}, - "id": 15, + "id": 41, "legend": { "alignAsTable": true, "avg": false, "current": true, - "hideEmpty": true, "max": true, "min": false, "rightSide": true, @@ -2945,7 +3023,7 @@ "values": true }, "lines": true, - "linewidth": 1, + "linewidth": 2, "links": [], "nullPointMode": "null", "percentage": false, @@ -2958,31 +3036,32 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tikv_storage_engine_async_request_duration_seconds_bucket{type=\"snapshot\"}[1m])) by (le))", + "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_request_wait_time_duration_secs_bucket[1m])) by (le))", "intervalFactor": 2, "legendFormat": "99%", + "metric": "tikv_raftstore_request_wait_time_duration_secs_bucket", "refId": "A", - "step": 10 + "step": 4 }, { - "expr": "histogram_quantile(0.95, sum(rate(tikv_storage_engine_async_request_duration_seconds_bucket{type=\"snapshot\"}[1m])) by (le))", + "expr": "histogram_quantile(0.95, sum(rate(tikv_raftstore_request_wait_time_duration_secs_bucket[1m])) by (le))", "intervalFactor": 2, "legendFormat": "95%", "refId": "B", - "step": 10 + "step": 4 }, { - "expr": "sum(rate(tikv_storage_engine_async_request_duration_seconds_sum{type=\"snapshot\"}[1m])) / sum(rate(tikv_storage_engine_async_request_duration_seconds_count{type=\"snapshot\"}[1m]))", + "expr": "sum(rate(tikv_raftstore_request_wait_time_duration_secs_sum[1m])) / sum(rate(tikv_raftstore_request_wait_time_duration_secs_count[1m]))", "intervalFactor": 2, "legendFormat": "avg", "refId": "C", - "step": 10 + "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "storage async snapshot duration", + "title": "99% request wait duration", "tooltip": { "msResolution": false, "shared": true, @@ -3023,12 +3102,11 @@ "error": false, "fill": 1, "grid": {}, - "id": 109, + "id": 42, "legend": { "alignAsTable": true, "avg": false, "current": true, - "hideEmpty": true, "max": true, "min": false, "rightSide": true, @@ -3037,7 +3115,7 @@ "values": true }, "lines": true, - "linewidth": 1, + "linewidth": 2, "links": [], "nullPointMode": "null", "percentage": false, @@ -3050,31 +3128,17 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tikv_storage_engine_async_request_duration_seconds_bucket{type=\"write\"}[1m])) by (le))", + "expr": "histogram_quantile(0.95, sum(rate(tikv_raftstore_request_wait_time_duration_secs_bucket[1m])) by (le, job))", "intervalFactor": 2, - "legendFormat": "99%", + "legendFormat": "{{job}}", "refId": "A", - "step": 10 - }, - { - "expr": "histogram_quantile(0.95, sum(rate(tikv_storage_engine_async_request_duration_seconds_bucket{type=\"write\"}[1m])) by (le))", - "intervalFactor": 2, - "legendFormat": "95%", - "refId": "B", - "step": 10 - }, - { - "expr": "sum(rate(tikv_storage_engine_async_request_duration_seconds_sum{type=\"write\"}[1m])) / sum(rate(tikv_storage_engine_async_request_duration_seconds_count{type=\"write\"}[1m]))", - "intervalFactor": 2, - "legendFormat": "avg", - "refId": "C", - "step": 10 + "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "storage async write duration", + "title": "95% request wait duration", "tooltip": { "msResolution": false, "shared": true, @@ -3112,38 +3176,41 @@ "repeatIteration": null, "repeatRowId": null, "showTitle": true, - "title": "Storage", + "title": "Raft Ready", "titleSize": "h6" }, { - "collapse": false, - "height": "250px", + "collapse": true, + "height": "300", "panels": [ { "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, "editable": true, "error": false, "fill": 1, "grid": {}, - "height": "", - "id": 1, + "id": 2, "legend": { "alignAsTable": true, "avg": false, "current": true, + "hideEmpty": true, + "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, + "sideWidth": 250, "total": false, "values": true }, "lines": true, - "linewidth": 2, + "linewidth": 1, "links": [], - "nullPointMode": "null", + "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, @@ -3154,25 +3221,23 @@ "steppedLine": false, "targets": [ { - "expr": "sum(tikv_scheduler_contex_total) by (job)", + "expr": "sum(rate(tikv_storage_command_total[1m])) by (type)", "intervalFactor": 2, - "legendFormat": "{{job}}", - "metric": "", + "legendFormat": "{{type}}", "refId": "A", - "step": 10 + "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "scheduler context total", + "title": "storage command total", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, - "transparent": false, "type": "graph", "xaxis": { "mode": "time", @@ -3182,11 +3247,11 @@ }, "yaxes": [ { - "format": "short", + "format": "ops", "label": null, - "logBase": 1, + "logBase": 10, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -3203,26 +3268,31 @@ "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, "editable": true, "error": false, "fill": 1, "grid": {}, - "id": 17, + "id": 8, "legend": { "alignAsTable": true, "avg": false, "current": true, + "hideEmpty": true, + "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, + "sideWidth": 250, + "sortDesc": true, "total": false, "values": true }, "lines": true, - "linewidth": 2, + "linewidth": 1, "links": [], - "nullPointMode": "null", + "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, @@ -3233,19 +3303,20 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_scheduler_stage_total{stage=\"error\"}[1m])) by (type)", + "expr": "sum(rate(tikv_storage_engine_async_request_total{status!~\"all|success\"}[1m])) by (status)", "intervalFactor": 2, - "legendFormat": "{{type}}", + "legendFormat": "{{status}}", + "metric": "tikv_raftstore_raft_process_duration_secs_bucket", "refId": "A", - "step": 10 + "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "scheduler stage error", + "title": "storage async request error", "tooltip": { - "msResolution": false, + "msResolution": true, "shared": true, "sort": 0, "value_type": "cumulative" @@ -3280,26 +3351,27 @@ "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, "editable": true, "error": false, "fill": 1, "grid": {}, - "id": 3, + "id": 15, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, - "hideZero": false, "max": true, "min": false, "rightSide": true, "show": true, + "sideWidth": 250, "total": false, "values": true }, "lines": true, - "linewidth": 2, + "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, @@ -3312,18 +3384,31 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tikv_scheduler_command_duration_seconds_bucket[1m])) by (le,type))", + "expr": "histogram_quantile(0.99, sum(rate(tikv_storage_engine_async_request_duration_seconds_bucket{type=\"snapshot\"}[1m])) by (le))", "intervalFactor": 2, - "legendFormat": "{{type}}", - "metric": "tikv_scheduler_command_duration_seconds_bucket", + "legendFormat": "99%", "refId": "A", - "step": 10 + "step": 4 + }, + { + "expr": "histogram_quantile(0.95, sum(rate(tikv_storage_engine_async_request_duration_seconds_bucket{type=\"snapshot\"}[1m])) by (le))", + "intervalFactor": 2, + "legendFormat": "95%", + "refId": "B", + "step": 4 + }, + { + "expr": "sum(rate(tikv_storage_engine_async_request_duration_seconds_sum{type=\"snapshot\"}[1m])) / sum(rate(tikv_storage_engine_async_request_duration_seconds_count{type=\"snapshot\"}[1m]))", + "intervalFactor": 2, + "legendFormat": "avg", + "refId": "C", + "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "99% scheduler command duration", + "title": "storage async snapshot duration", "tooltip": { "msResolution": false, "shared": true, @@ -3360,11 +3445,12 @@ "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, "editable": true, "error": false, "fill": 1, "grid": {}, - "id": 14, + "id": 109, "legend": { "alignAsTable": true, "avg": false, @@ -3374,6 +3460,7 @@ "min": false, "rightSide": true, "show": true, + "sideWidth": 250, "total": false, "values": true }, @@ -3391,17 +3478,31 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.95, sum(rate(tikv_scheduler_command_duration_seconds_bucket[1m])) by (le,type))", + "expr": "histogram_quantile(0.99, sum(rate(tikv_storage_engine_async_request_duration_seconds_bucket{type=\"write\"}[1m])) by (le))", "intervalFactor": 2, - "legendFormat": "{{type}}", + "legendFormat": "99%", "refId": "A", - "step": 10 + "step": 4 + }, + { + "expr": "histogram_quantile(0.95, sum(rate(tikv_storage_engine_async_request_duration_seconds_bucket{type=\"write\"}[1m])) by (le))", + "intervalFactor": 2, + "legendFormat": "95%", + "refId": "B", + "step": 4 + }, + { + "expr": "sum(rate(tikv_storage_engine_async_request_duration_seconds_sum{type=\"write\"}[1m])) / sum(rate(tikv_storage_engine_async_request_duration_seconds_count{type=\"write\"}[1m]))", + "intervalFactor": 2, + "legendFormat": "avg", + "refId": "C", + "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "95% scheduler command duration", + "title": "storage async write duration", "tooltip": { "msResolution": false, "shared": true, @@ -3438,12 +3539,12 @@ "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", - "decimals": 2, + "decimals": 1, "editable": true, "error": false, "fill": 1, "grid": {}, - "id": 23, + "id": 1310, "legend": { "alignAsTable": true, "avg": false, @@ -3453,34 +3554,61 @@ "min": false, "rightSide": true, "show": true, + "sideWidth": 250, "total": false, "values": true }, "lines": true, - "linewidth": 2, + "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "raft-95%", + "yaxis": 2 + } + ], "span": 6, "stack": false, "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tikv_scheduler_latch_wait_duration_seconds_bucket[1m])) by (le,type))", + "expr": "histogram_quantile(0.99, sum(rate(tikv_storage_batch_commands_total_bucket[30s])) by (le))", "intervalFactor": 2, - "legendFormat": "{{type}}", + "legendFormat": "99%", "refId": "A", - "step": 10 + "step": 4 + }, + { + "expr": "histogram_quantile(0.95, sum(rate(tikv_storage_batch_commands_total_bucket[30s])) by (le))", + "intervalFactor": 2, + "legendFormat": "95%", + "refId": "B", + "step": 4 + }, + { + "expr": "sum(rate(tikv_storage_batch_commands_total_sum[30s])) / sum(rate(tikv_storage_batch_commands_total_count[30s]))", + "intervalFactor": 2, + "legendFormat": "avg", + "refId": "C", + "step": 4 + }, + { + "expr": "histogram_quantile(0.95, sum(rate(tikv_raftstore_batch_snapshot_commands_total_bucket[30s])) by (le))", + "intervalFactor": 2, + "legendFormat": "raft-95%", + "refId": "D", + "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "99% scheduler latch wait duration", + "title": "storage async batch snapshot", "tooltip": { "msResolution": false, "shared": true, @@ -3496,74 +3624,92 @@ }, "yaxes": [ { - "format": "s", - "label": "", + "format": "short", + "label": "Storage Batch Size", "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { "format": "short", - "label": null, + "label": "Raftstore Batch Size", "logBase": 1, "max": null, "min": null, - "show": false + "show": true } ] - }, + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage", + "titleSize": "h6" + }, + { + "collapse": true, + "height": "300", + "panels": [ { "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", - "editable": true, - "error": false, + "decimals": 1, "fill": 1, - "grid": {}, - "id": 24, + "height": "400", + "id": 167, "legend": { "alignAsTable": true, "avg": false, "current": true, - "hideEmpty": true, "max": true, "min": false, "rightSide": true, "show": true, + "sideWidth": 300, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], + "minSpan": 12, "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], - "span": 6, + "span": 12, "stack": false, "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.95, sum(rate(tikv_scheduler_latch_wait_duration_seconds_bucket[1m])) by (le,type))", + "expr": "sum(rate(tikv_scheduler_too_busy_total[1m])) by (stage)", "intervalFactor": 2, - "legendFormat": "{{type}}", + "legendFormat": "busy", "refId": "A", - "step": 10 + "step": 20 + }, + { + "expr": "sum(rate(tikv_scheduler_stage_total[1m])) by (stage)", + "intervalFactor": 2, + "legendFormat": "{{stage}}", + "refId": "B", + "step": 20 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "95% scheduler latch wait duration", + "title": "scheduler stage total", "tooltip": { - "msResolution": false, "shared": true, "sort": 0, - "value_type": "cumulative" + "value_type": "individual" }, "type": "graph", "xaxis": { @@ -3574,11 +3720,11 @@ }, "yaxes": [ { - "format": "s", + "format": "ops", "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { @@ -3595,28 +3741,30 @@ "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, "editable": true, "error": false, "fill": 1, "grid": {}, - "id": 46, + "height": "", + "id": 1, "legend": { "alignAsTable": true, "avg": false, "current": true, - "hideEmpty": true, - "hideZero": false, "max": true, "min": false, "rightSide": true, "show": true, + "sideWidth": 250, "total": false, "values": true }, "lines": true, - "linewidth": 1, + "linewidth": 2, "links": [], - "nullPointMode": "null as zero", + "minSpan": 6, + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, @@ -3627,24 +3775,25 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_scheduler_kv_command_key_read_bucket[1m])) by (type)", + "expr": "sum(rate(tikv_scheduler_commands_pri_total[1m])) by (priority)", "intervalFactor": 2, - "legendFormat": "{{type}}", - "metric": "tikv_scheduler_kv_command_key_read_bucket", + "legendFormat": "{{priority}}", + "metric": "", "refId": "A", - "step": 10 + "step": 40 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "scheduler key reads", + "title": "scheduler priority commands", "tooltip": { "msResolution": false, "shared": true, "sort": 0, - "value_type": "individual" + "value_type": "cumulative" }, + "transparent": false, "type": "graph", "xaxis": { "mode": "time", @@ -3654,11 +3803,11 @@ }, "yaxes": [ { - "format": "short", + "format": "ops", "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { @@ -3670,29 +3819,18 @@ "show": true } ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Scheduler", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ + }, { "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", - "decimals": 5, + "decimals": 1, "editable": true, "error": false, "fill": 1, "grid": {}, - "id": 16, + "height": "", + "id": 193, "legend": { "alignAsTable": true, "avg": false, @@ -3701,54 +3839,44 @@ "min": false, "rightSide": true, "show": true, + "sideWidth": 250, "total": false, "values": true }, "lines": true, - "linewidth": 1, + "linewidth": 2, "links": [], + "minSpan": 6, "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], - "span": 4, + "span": 6, "stack": false, "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tikv_coprocessor_request_duration_seconds_bucket{req=\"select\"}[1m])) by (le))", + "expr": "sum(tikv_scheduler_contex_total) by (job)", "intervalFactor": 2, - "legendFormat": "99%", + "legendFormat": "{{job}}", + "metric": "", "refId": "A", - "step": 20 - }, - { - "expr": "histogram_quantile(0.95, sum(rate(tikv_coprocessor_request_duration_seconds_bucket{req=\"select\"}[1m])) by (le))", - "intervalFactor": 2, - "legendFormat": "95%", - "refId": "B", - "step": 20 - }, - { - "expr": " sum(rate(tikv_coprocessor_request_duration_seconds_sum{req=\"select\"}[1m])) / sum(rate(tikv_coprocessor_request_duration_seconds_count{req=\"select\"}[1m]))", - "intervalFactor": 2, - "legendFormat": "avg", - "refId": "C", - "step": 20 + "step": 40 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "coprocessor table request duration", + "title": "scheduler pending commands", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, + "transparent": false, "type": "graph", "xaxis": { "mode": "time", @@ -3758,7 +3886,7 @@ }, "yaxes": [ { - "format": "s", + "format": "ops", "label": null, "logBase": 1, "max": null, @@ -3774,17 +3902,27 @@ "show": true } ] - }, + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Scheduler", + "titleSize": "h6" + }, + { + "collapse": true, + "height": "300", + "panels": [ { "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", - "decimals": 5, - "editable": true, - "error": false, + "decimals": 1, "fill": 1, - "grid": {}, - "id": 13, + "height": "400", + "id": 168, "legend": { "alignAsTable": true, "avg": false, @@ -3793,55 +3931,47 @@ "min": false, "rightSide": true, "show": true, - "sortDesc": true, + "sideWidth": 300, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], + "minSpan": 12, "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], - "span": 4, + "span": 12, "stack": false, "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tikv_coprocessor_request_duration_seconds_bucket{req=\"index\"}[1m])) by (le))", + "expr": "sum(rate(tikv_scheduler_too_busy_total{type=\"$command\"}[1m]))", "intervalFactor": 2, - "legendFormat": "99%", - "metric": "tikv_coprocessor_request_duration_seconds_bucket", + "legendFormat": "busy", "refId": "A", - "step": 20 + "step": 4 }, { - "expr": "histogram_quantile(0.95, sum(rate(tikv_coprocessor_request_duration_seconds_bucket{req=\"index\"}[1m])) by (le))", + "expr": "sum(rate(tikv_scheduler_stage_total{type=\"$command\"}[1m])) by (stage)", "intervalFactor": 2, - "legendFormat": "95%", + "legendFormat": "{{stage}}", "refId": "B", - "step": 20 - }, - { - "expr": "sum(rate(tikv_coprocessor_request_duration_seconds_sum{req=\"index\"}[1m])) / sum(rate(tikv_coprocessor_request_duration_seconds_count{req=\"index\"}[1m]))", - "intervalFactor": 2, - "legendFormat": "avg", - "refId": "C", - "step": 20 + "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "coprocessor index request duration", + "title": "scheduler stage total", "tooltip": { - "msResolution": false, "shared": true, "sort": 0, - "value_type": "cumulative" + "value_type": "individual" }, "type": "graph", "xaxis": { @@ -3852,7 +3982,7 @@ }, "yaxes": [ { - "format": "s", + "format": "ops", "label": null, "logBase": 1, "max": null, @@ -3873,50 +4003,69 @@ "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", - "decimals": 5, + "decimals": 1, "editable": true, "error": false, "fill": 1, "grid": {}, - "id": 115, + "id": 3, "legend": { "alignAsTable": true, "avg": false, "current": true, + "hideEmpty": true, + "hideZero": false, "max": true, "min": false, "rightSide": true, "show": true, - "sortDesc": true, + "sideWidth": 250, "total": false, "values": true }, "lines": true, - "linewidth": 1, + "linewidth": 2, "links": [], + "minSpan": null, "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], - "span": 4, + "span": 6, "stack": false, "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.95, sum(rate(tikv_coprocessor_request_duration_seconds_bucket[1m])) by (le, job,req))", + "expr": "histogram_quantile(0.99, sum(rate(tikv_scheduler_command_duration_seconds_bucket{type=\"$command\"}[1m])) by (le))", "intervalFactor": 2, - "legendFormat": "{{job}}-{{req}}", - "metric": "tikv_coprocessor_request_duration_seconds_bucket", + "legendFormat": "99%", + "metric": "", "refId": "A", - "step": 20 + "step": 10 + }, + { + "expr": "histogram_quantile(0.95, sum(rate(tikv_scheduler_command_duration_seconds_bucket{type=\"$command\"}[1m])) by (le))", + "intervalFactor": 2, + "legendFormat": "95%", + "metric": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(rate(tikv_scheduler_command_duration_seconds_sum{type=\"$command\"}[1m])) / sum(rate(tikv_scheduler_command_duration_seconds_count{type=\"$command\"}[1m])) ", + "intervalFactor": 2, + "legendFormat": "avg", + "metric": "", + "refId": "C", + "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "95% coprocessor request duration", + "title": "scheduler command duration", "tooltip": { "msResolution": false, "shared": true, @@ -3953,62 +4102,69 @@ "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", - "decimals": 5, + "decimals": 1, "editable": true, "error": false, "fill": 1, "grid": {}, - "id": 111, + "id": 194, "legend": { "alignAsTable": true, "avg": false, "current": true, + "hideEmpty": true, + "hideZero": false, "max": true, "min": false, "rightSide": true, "show": true, + "sideWidth": 250, "total": false, "values": true }, "lines": true, - "linewidth": 1, + "linewidth": 2, "links": [], + "minSpan": null, "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], - "span": 4, + "span": 6, "stack": false, "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tikv_coprocessor_request_wait_seconds_bucket{req=\"select\"}[1m])) by (le))", + "expr": "histogram_quantile(0.99, sum(rate(tikv_scheduler_latch_wait_duration_seconds_bucket{type=\"$command\"}[1m])) by (le))", "intervalFactor": 2, "legendFormat": "99%", + "metric": "", "refId": "A", - "step": 20 + "step": 10 }, { - "expr": "histogram_quantile(0.95, sum(rate(tikv_coprocessor_request_wait_seconds_bucket{req=\"select\"}[1m])) by (le))", + "expr": "histogram_quantile(0.95, sum(rate(tikv_scheduler_latch_wait_duration_seconds_bucket{type=\"$command\"}[1m])) by (le))", "intervalFactor": 2, "legendFormat": "95%", + "metric": "", "refId": "B", - "step": 20 + "step": 10 }, { - "expr": " sum(rate(tikv_coprocessor_request_wait_seconds_sum{req=\"select\"}[1m])) / sum(rate(tikv_coprocessor_request_wait_seconds_count{req=\"select\"}[1m]))", + "expr": "sum(rate(tikv_scheduler_latch_wait_duration_seconds_sum{type=\"$command\"}[1m])) / sum(rate(tikv_scheduler_latch_wait_duration_seconds_count{type=\"$command\"}[1m])) ", "intervalFactor": 2, "legendFormat": "avg", + "metric": "", "refId": "C", - "step": 20 + "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "coprocessor table wait duration", + "title": "scheduler latch wait duration", "tooltip": { "msResolution": false, "shared": true, @@ -4045,62 +4201,69 @@ "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", - "decimals": 5, + "decimals": 1, "editable": true, "error": false, "fill": 1, "grid": {}, - "id": 112, + "id": 195, "legend": { "alignAsTable": true, "avg": false, "current": true, + "hideEmpty": true, + "hideZero": false, "max": true, "min": false, "rightSide": true, "show": true, + "sideWidth": 250, "total": false, "values": true }, "lines": true, - "linewidth": 1, + "linewidth": 2, "links": [], + "minSpan": null, "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], - "span": 4, + "span": 6, "stack": false, "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tikv_coprocessor_request_wait_seconds_bucket{req=\"index\"}[1m])) by (le))", + "expr": "histogram_quantile(0.99, sum(rate(tikv_scheduler_kv_command_key_read_bucket{type=\"$command\"}[1m])) by (le))", "intervalFactor": 2, "legendFormat": "99%", + "metric": "kv_command_key", "refId": "A", - "step": 20 + "step": 10 }, { - "expr": "histogram_quantile(0.95, sum(rate(tikv_coprocessor_request_wait_seconds_bucket{req=\"index\"}[1m])) by (le))", + "expr": "histogram_quantile(0.95, sum(rate(tikv_scheduler_kv_command_key_read_bucket{type=\"$command\"}[1m])) by (le))", "intervalFactor": 2, "legendFormat": "95%", + "metric": "", "refId": "B", - "step": 20 + "step": 10 }, { - "expr": " sum(rate(tikv_coprocessor_request_wait_seconds_sum{req=\"index\"}[1m])) / sum(rate(tikv_coprocessor_request_wait_seconds_count{req=\"index\"}[1m]))", + "expr": "sum(rate(tikv_scheduler_kv_command_key_read_sum{type=\"$command\"}[1m])) / sum(rate(tikv_scheduler_kv_command_key_read_count{type=\"$command\"}[1m])) ", "intervalFactor": 2, "legendFormat": "avg", + "metric": "", "refId": "C", - "step": 20 + "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "coprocessor index wait duration", + "title": "scheduler keys read", "tooltip": { "msResolution": false, "shared": true, @@ -4116,7 +4279,7 @@ }, "yaxes": [ { - "format": "s", + "format": "short", "label": null, "logBase": 1, "max": null, @@ -4137,48 +4300,69 @@ "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", - "decimals": 5, + "decimals": 1, "editable": true, "error": false, "fill": 1, "grid": {}, - "id": 116, + "id": 373, "legend": { "alignAsTable": true, "avg": false, "current": true, + "hideEmpty": true, + "hideZero": false, "max": true, "min": false, "rightSide": true, "show": true, + "sideWidth": 250, "total": false, "values": true }, "lines": true, - "linewidth": 1, + "linewidth": 2, "links": [], + "minSpan": null, "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], - "span": 4, + "span": 6, "stack": false, "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.95, sum(rate(tikv_coprocessor_request_wait_seconds_bucket[1m])) by (le, job,req))", + "expr": "histogram_quantile(0.99, sum(rate(tikv_scheduler_kv_command_key_write_bucket{type=\"$command\"}[1m])) by (le))", "intervalFactor": 2, - "legendFormat": "{{job}}-{{req}}", + "legendFormat": "99%", + "metric": "kv_command_key", + "refId": "A", + "step": 10 + }, + { + "expr": "histogram_quantile(0.95, sum(rate(tikv_scheduler_kv_command_key_write_bucket{type=\"$command\"}[1m])) by (le))", + "intervalFactor": 2, + "legendFormat": "95%", + "metric": "", "refId": "B", - "step": 20 + "step": 10 + }, + { + "expr": "sum(rate(tikv_scheduler_kv_command_key_write_sum{type=\"$command\"}[1m])) / sum(rate(tikv_scheduler_kv_command_key_write_count{type=\"$command\"}[1m])) ", + "intervalFactor": 2, + "legendFormat": "avg", + "metric": "", + "refId": "C", + "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "95% coprocessor wait duration", + "title": "scheduler keys written", "tooltip": { "msResolution": false, "shared": true, @@ -4194,7 +4378,7 @@ }, "yaxes": [ { - "format": "s", + "format": "short", "label": null, "logBase": 1, "max": null, @@ -4215,62 +4399,53 @@ "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", - "decimals": 5, + "decimals": 1, "editable": true, "error": false, "fill": 1, "grid": {}, - "id": 113, + "id": 560, "legend": { "alignAsTable": true, "avg": false, "current": true, + "hideEmpty": true, + "hideZero": false, "max": true, "min": false, "rightSide": true, "show": true, + "sideWidth": 250, "total": false, "values": true }, "lines": true, - "linewidth": 1, + "linewidth": 2, "links": [], + "minSpan": null, "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], - "span": 4, + "span": 6, "stack": false, "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tikv_coprocessor_request_handle_seconds_bucket{req=\"select\"}[1m])) by (le))", + "expr": "sum(rate(tikv_scheduler_kv_scan_details{req=\"$command\"}[1m])) by (tag)", "intervalFactor": 2, - "legendFormat": "99%", + "legendFormat": "{{tag}}", + "metric": "", "refId": "A", - "step": 20 - }, - { - "expr": "histogram_quantile(0.95, sum(rate(tikv_coprocessor_request_handle_seconds_bucket{req=\"select\"}[1m])) by (le))", - "intervalFactor": 2, - "legendFormat": "95%", - "refId": "B", - "step": 20 - }, - { - "expr": " sum(rate(tikv_coprocessor_request_handle_seconds_sum{req=\"select\"}[1m])) / sum(rate(tikv_coprocessor_request_handle_seconds_count{req=\"select\"}[1m]))", - "intervalFactor": 2, - "legendFormat": "avg", - "refId": "C", - "step": 20 + "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "coprocessor table handle duration", + "title": "scheduler scan details", "tooltip": { "msResolution": false, "shared": true, @@ -4286,7 +4461,7 @@ }, "yaxes": [ { - "format": "s", + "format": "short", "label": null, "logBase": 1, "max": null, @@ -4307,62 +4482,53 @@ "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", - "decimals": 5, + "decimals": 1, "editable": true, "error": false, "fill": 1, "grid": {}, - "id": 114, + "id": 675, "legend": { "alignAsTable": true, "avg": false, "current": true, + "hideEmpty": true, + "hideZero": false, "max": true, "min": false, "rightSide": true, "show": true, + "sideWidth": 250, "total": false, "values": true }, "lines": true, - "linewidth": 1, + "linewidth": 2, "links": [], + "minSpan": null, "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], - "span": 4, + "span": 6, "stack": false, "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tikv_coprocessor_request_handle_seconds_bucket{req=\"index\"}[1m])) by (le))", + "expr": "sum(rate(tikv_scheduler_kv_scan_details{req=\"$command\", cf=\"lock\"}[1m])) by (tag)", "intervalFactor": 2, - "legendFormat": "99%", + "legendFormat": "{{tag}}", + "metric": "", "refId": "A", - "step": 20 - }, - { - "expr": "histogram_quantile(0.95, sum(rate(tikv_coprocessor_request_handle_seconds_bucket{req=\"index\"}[1m])) by (le))", - "intervalFactor": 2, - "legendFormat": "95%", - "refId": "B", - "step": 20 - }, - { - "expr": " sum(rate(tikv_coprocessor_request_handle_seconds_sum{req=\"index\"}[1m])) / sum(rate(tikv_coprocessor_request_handle_seconds_count{req=\"index\"}[1m]))", - "intervalFactor": 2, - "legendFormat": "avg", - "refId": "C", - "step": 20 + "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "coprocessor index handle duration", + "title": "scheduler scan details [lock]", "tooltip": { "msResolution": false, "shared": true, @@ -4378,7 +4544,7 @@ }, "yaxes": [ { - "format": "s", + "format": "short", "label": null, "logBase": 1, "max": null, @@ -4399,48 +4565,53 @@ "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", - "decimals": 5, + "decimals": 1, "editable": true, "error": false, "fill": 1, "grid": {}, - "id": 117, + "id": 829, "legend": { "alignAsTable": true, "avg": false, "current": true, + "hideEmpty": true, + "hideZero": false, "max": true, "min": false, "rightSide": true, "show": true, + "sideWidth": 250, "total": false, "values": true }, "lines": true, - "linewidth": 1, + "linewidth": 2, "links": [], + "minSpan": null, "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], - "span": 4, + "span": 6, "stack": false, "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.95, sum(rate(tikv_coprocessor_request_handle_seconds_bucket[1m])) by (le, job,req))", + "expr": "sum(rate(tikv_scheduler_kv_scan_details{req=\"$command\", cf=\"write\"}[1m])) by (tag)", "intervalFactor": 2, - "legendFormat": "{{job}}-{{req}}", - "refId": "B", - "step": 20 + "legendFormat": "{{tag}}", + "metric": "", + "refId": "A", + "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "95% coprocessor handle duration", + "title": "scheduler scan details [write]", "tooltip": { "msResolution": false, "shared": true, @@ -4456,7 +4627,7 @@ }, "yaxes": [ { - "format": "s", + "format": "short", "label": null, "logBase": 1, "max": null, @@ -4477,25 +4648,30 @@ "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, "editable": true, "error": false, "fill": 1, "grid": {}, - "id": 52, + "id": 830, "legend": { "alignAsTable": true, "avg": false, "current": true, + "hideEmpty": true, + "hideZero": false, "max": true, "min": false, "rightSide": true, "show": true, + "sideWidth": 250, "total": false, "values": true }, "lines": true, - "linewidth": 1, + "linewidth": 2, "links": [], + "minSpan": null, "nullPointMode": "null", "percentage": false, "pointradius": 5, @@ -4507,11 +4683,10 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_coprocessor_scan_keys_bucket[1m])) by (req)", - "interval": "", + "expr": "sum(rate(tikv_scheduler_kv_scan_details{req=\"$command\", cf=\"default\"}[1m])) by (tag)", "intervalFactor": 2, - "legendFormat": "{{req}}", - "metric": "tikv_coprocessor_scan_keys_bucket", + "legendFormat": "{{tag}}", + "metric": "", "refId": "A", "step": 10 } @@ -4519,12 +4694,12 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "coprocessor scan keys", + "title": "scheduler scan details [default]", "tooltip": { "msResolution": false, "shared": true, "sort": 0, - "value_type": "individual" + "value_type": "cumulative" }, "type": "graph", "xaxis": { @@ -4551,16 +4726,29 @@ "show": true } ] - }, + } + ], + "repeat": "command", + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Scheduler - $command", + "titleSize": "h6" + }, + { + "collapse": true, + "height": "250px", + "panels": [ { "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 5, "editable": true, "error": false, "fill": 1, "grid": {}, - "id": 122, + "id": 16, "legend": { "alignAsTable": true, "avg": false, @@ -4581,29 +4769,41 @@ "points": false, "renderer": "flot", "seriesOverrides": [], - "span": 6, + "span": 4, "stack": false, "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_coprocessor_scan_details_bucket[1m])) by (req,cf,tag)", - "interval": "", + "expr": "histogram_quantile(0.99, sum(rate(tikv_coprocessor_request_duration_seconds_bucket{req=\"select\"}[1m])) by (le))", "intervalFactor": 2, - "legendFormat": "{{req}}-{{cf}}-{{tag}}", - "metric": "tikv_coprocessor_scan_details_bucket", + "legendFormat": "99%", "refId": "A", - "step": 10 + "step": 4 + }, + { + "expr": "histogram_quantile(0.95, sum(rate(tikv_coprocessor_request_duration_seconds_bucket{req=\"select\"}[1m])) by (le))", + "intervalFactor": 2, + "legendFormat": "95%", + "refId": "B", + "step": 4 + }, + { + "expr": " sum(rate(tikv_coprocessor_request_duration_seconds_sum{req=\"select\"}[1m])) / sum(rate(tikv_coprocessor_request_duration_seconds_count{req=\"select\"}[1m]))", + "intervalFactor": 2, + "legendFormat": "avg", + "refId": "C", + "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "coprocessor scan details", + "title": "coprocessor table request duration", "tooltip": { "msResolution": false, "shared": true, "sort": 0, - "value_type": "individual" + "value_type": "cumulative" }, "type": "graph", "xaxis": { @@ -4614,7 +4814,7 @@ }, "yaxes": [ { - "format": "short", + "format": "s", "label": null, "logBase": 1, "max": null, @@ -4635,11 +4835,12 @@ "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 5, "editable": true, "error": false, "fill": 1, "grid": {}, - "id": 74, + "id": 13, "legend": { "alignAsTable": true, "avg": false, @@ -4648,6 +4849,7 @@ "min": false, "rightSide": true, "show": true, + "sortDesc": true, "total": false, "values": true }, @@ -4660,29 +4862,42 @@ "points": false, "renderer": "flot", "seriesOverrides": [], - "span": 6, + "span": 4, "stack": false, "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_coprocessor_request_error[1m])) by (reason)", - "interval": "", + "expr": "histogram_quantile(0.99, sum(rate(tikv_coprocessor_request_duration_seconds_bucket{req=\"index\"}[1m])) by (le))", "intervalFactor": 2, - "legendFormat": "{{reason}}", - "metric": "tikv_coprocessor_request_error", + "legendFormat": "99%", + "metric": "tikv_coprocessor_request_duration_seconds_bucket", "refId": "A", - "step": 10 + "step": 4 + }, + { + "expr": "histogram_quantile(0.95, sum(rate(tikv_coprocessor_request_duration_seconds_bucket{req=\"index\"}[1m])) by (le))", + "intervalFactor": 2, + "legendFormat": "95%", + "refId": "B", + "step": 4 + }, + { + "expr": "sum(rate(tikv_coprocessor_request_duration_seconds_sum{req=\"index\"}[1m])) / sum(rate(tikv_coprocessor_request_duration_seconds_count{req=\"index\"}[1m]))", + "intervalFactor": 2, + "legendFormat": "avg", + "refId": "C", + "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "coprocessor request error", + "title": "coprocessor index request duration", "tooltip": { "msResolution": false, "shared": true, "sort": 0, - "value_type": "individual" + "value_type": "cumulative" }, "type": "graph", "xaxis": { @@ -4693,7 +4908,7 @@ }, "yaxes": [ { - "format": "short", + "format": "s", "label": null, "logBase": 1, "max": null, @@ -4709,28 +4924,17 @@ "show": true } ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Coprocessor", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ + }, { "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 5, "editable": true, "error": false, "fill": 1, "grid": {}, - "id": 26, + "id": 115, "legend": { "alignAsTable": true, "avg": false, @@ -4739,39 +4943,2167 @@ "min": false, "rightSide": true, "show": true, + "sortDesc": true, "total": false, "values": true }, "lines": true, - "linewidth": 2, + "linewidth": 1, "links": [], - "nullPointMode": "null as zero", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], - "span": 6, + "span": 4, "stack": false, "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.9999, max(rate(tikv_storage_mvcc_versions_bucket[1m])) by (le))", + "expr": "histogram_quantile(0.95, sum(rate(tikv_coprocessor_request_duration_seconds_bucket[1m])) by (le, job,req))", + "intervalFactor": 2, + "legendFormat": "{{job}}-{{req}}", + "metric": "tikv_coprocessor_request_duration_seconds_bucket", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "95% coprocessor request duration", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "${DS_TEST-CLUSTER}", + "decimals": 5, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 111, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(tikv_coprocessor_request_wait_seconds_bucket{req=\"select\"}[1m])) by (le))", + "intervalFactor": 2, + "legendFormat": "99%", + "refId": "A", + "step": 4 + }, + { + "expr": "histogram_quantile(0.95, sum(rate(tikv_coprocessor_request_wait_seconds_bucket{req=\"select\"}[1m])) by (le))", + "intervalFactor": 2, + "legendFormat": "95%", + "refId": "B", + "step": 4 + }, + { + "expr": " sum(rate(tikv_coprocessor_request_wait_seconds_sum{req=\"select\"}[1m])) / sum(rate(tikv_coprocessor_request_wait_seconds_count{req=\"select\"}[1m]))", + "intervalFactor": 2, + "legendFormat": "avg", + "refId": "C", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "coprocessor table wait duration", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "${DS_TEST-CLUSTER}", + "decimals": 5, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 112, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(tikv_coprocessor_request_wait_seconds_bucket{req=\"index\"}[1m])) by (le))", + "intervalFactor": 2, + "legendFormat": "99%", + "refId": "A", + "step": 4 + }, + { + "expr": "histogram_quantile(0.95, sum(rate(tikv_coprocessor_request_wait_seconds_bucket{req=\"index\"}[1m])) by (le))", + "intervalFactor": 2, + "legendFormat": "95%", + "refId": "B", + "step": 4 + }, + { + "expr": " sum(rate(tikv_coprocessor_request_wait_seconds_sum{req=\"index\"}[1m])) / sum(rate(tikv_coprocessor_request_wait_seconds_count{req=\"index\"}[1m]))", + "intervalFactor": 2, + "legendFormat": "avg", + "refId": "C", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "coprocessor index wait duration", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "${DS_TEST-CLUSTER}", + "decimals": 5, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 116, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.95, sum(rate(tikv_coprocessor_request_wait_seconds_bucket[1m])) by (le, job,req))", + "intervalFactor": 2, + "legendFormat": "{{job}}-{{req}}", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "95% coprocessor wait duration", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "${DS_TEST-CLUSTER}", + "decimals": 5, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 113, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(tikv_coprocessor_request_handle_seconds_bucket{req=\"select\"}[1m])) by (le))", + "intervalFactor": 2, + "legendFormat": "99%", + "refId": "A", + "step": 4 + }, + { + "expr": "histogram_quantile(0.95, sum(rate(tikv_coprocessor_request_handle_seconds_bucket{req=\"select\"}[1m])) by (le))", + "intervalFactor": 2, + "legendFormat": "95%", + "refId": "B", + "step": 4 + }, + { + "expr": " sum(rate(tikv_coprocessor_request_handle_seconds_sum{req=\"select\"}[1m])) / sum(rate(tikv_coprocessor_request_handle_seconds_count{req=\"select\"}[1m]))", + "intervalFactor": 2, + "legendFormat": "avg", + "refId": "C", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "coprocessor table handle duration", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "${DS_TEST-CLUSTER}", + "decimals": 5, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 114, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(tikv_coprocessor_request_handle_seconds_bucket{req=\"index\"}[1m])) by (le))", + "intervalFactor": 2, + "legendFormat": "99%", + "refId": "A", + "step": 4 + }, + { + "expr": "histogram_quantile(0.95, sum(rate(tikv_coprocessor_request_handle_seconds_bucket{req=\"index\"}[1m])) by (le))", + "intervalFactor": 2, + "legendFormat": "95%", + "refId": "B", + "step": 4 + }, + { + "expr": " sum(rate(tikv_coprocessor_request_handle_seconds_sum{req=\"index\"}[1m])) / sum(rate(tikv_coprocessor_request_handle_seconds_count{req=\"index\"}[1m]))", + "intervalFactor": 2, + "legendFormat": "avg", + "refId": "C", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "coprocessor index handle duration", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "${DS_TEST-CLUSTER}", + "decimals": 5, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 117, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.95, sum(rate(tikv_coprocessor_request_handle_seconds_bucket[1m])) by (le, job,req))", + "intervalFactor": 2, + "legendFormat": "{{job}}-{{req}}", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "95% coprocessor handle duration", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 52, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 250, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(tikv_coprocessor_scan_keys_bucket[1m])) by (req)", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{req}}", + "metric": "tikv_coprocessor_scan_keys_bucket", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "coprocessor scan keys", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 551, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 250, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(tikv_coprocessor_executor_count[1m])) by (type)", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{type}}", + "metric": "", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "coprocessor executor count", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "${DS_TEST-CLUSTER}", + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 74, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 250, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(tikv_coprocessor_request_error[1m])) by (reason)", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{reason}}", + "metric": "tikv_coprocessor_request_error", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "coprocessor request errors", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "${DS_TEST-CLUSTER}", + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 550, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 250, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(tikv_coprocessor_pending_request[1m])) by (req, priority)", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{ req }} - {{priority}}", + "metric": "tikv_coprocessor_request_error", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "coprocessor pending requests", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 552, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 250, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(tikv_coprocessor_scan_details{req=\"select\"}[1m])) by (tag)", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{tag}}", + "metric": "scan_details", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "coprocessor table scan details", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 122, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 250, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(tikv_coprocessor_scan_details{cf=\"lock\", req=\"select\"}[1m])) by (tag)", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{tag}}", + "metric": "scan_details", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "coprocessor table scan details [lock]", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 555, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 250, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(tikv_coprocessor_scan_details{cf=\"write\", req=\"select\"}[1m])) by (tag)", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{tag}}", + "metric": "scan_details", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "coprocessor table scan details [write]", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 556, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 250, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(tikv_coprocessor_scan_details{cf=\"default\", req=\"select\"}[1m])) by (tag)", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{tag}}", + "metric": "scan_details", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "coprocessor table scan details [default]", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 553, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 250, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(tikv_coprocessor_scan_details{req=\"index\"}[1m])) by (tag)", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{tag}}", + "metric": "scan_details", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "coprocessor index scan details", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 554, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 250, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "cf", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(tikv_coprocessor_scan_details{cf=\"lock\", req=\"index\"}[1m])) by (tag)", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{tag}}", + "metric": "scan_details", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "coprocessor index scan details - [lock]", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 557, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 250, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(tikv_coprocessor_scan_details{cf=\"write\", req=\"index\"}[1m])) by (tag)", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{tag}}", + "metric": "scan_details", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "coprocessor index scan details - [write]", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 558, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 250, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(tikv_coprocessor_scan_details{cf=\"default\", req=\"index\"}[1m])) by (tag)", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{tag}}", + "metric": "scan_details", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "coprocessor index scan details - [default]", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Coprocessor", + "titleSize": "h6" + }, + { + "collapse": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 26, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 250, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(1.0, sum(rate(tikv_storage_mvcc_versions_bucket[1m])) by (le))", + "intervalFactor": 2, + "legendFormat": " max", + "metric": "", + "refId": "A", + "step": 4 + }, + { + "expr": "histogram_quantile(0.99, sum(rate(tikv_storage_mvcc_versions_bucket[1m])) by (le))", + "intervalFactor": 2, + "legendFormat": "99%", + "metric": "", + "refId": "B", + "step": 4 + }, + { + "expr": "histogram_quantile(0.95, sum(rate(tikv_storage_mvcc_versions_bucket[1m])) by (le))", + "intervalFactor": 2, + "legendFormat": " 95%", + "metric": "", + "refId": "C", + "step": 4 + }, + { + "expr": "sum(rate(tikv_storage_mvcc_versions_sum[1m])) / sum(rate(tikv_storage_mvcc_versions_count[1m])) ", + "intervalFactor": 2, + "legendFormat": "avg", + "metric": "", + "refId": "D", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "MVCC Versions", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 559, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 250, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(1.0, sum(rate(tikv_storage_mvcc_gc_delete_versions_bucket[1m])) by (le))", + "intervalFactor": 2, + "legendFormat": " max", + "metric": "", + "refId": "A", + "step": 4 + }, + { + "expr": "histogram_quantile(0.99, sum(rate(tikv_storage_mvcc_gc_delete_versions_bucket[1m])) by (le))", + "intervalFactor": 2, + "legendFormat": "99%", + "metric": "", + "refId": "B", + "step": 4 + }, + { + "expr": "histogram_quantile(0.95, sum(rate(tikv_storage_mvcc_gc_delete_versions_bucket[1m])) by (le))", + "intervalFactor": 2, + "legendFormat": " 95%", + "metric": "", + "refId": "C", + "step": 4 + }, + { + "expr": "sum(rate(tikv_storage_mvcc_gc_delete_versions_sum[1m])) / sum(rate(tikv_storage_mvcc_gc_delete_versions_count[1m])) ", + "intervalFactor": 2, + "legendFormat": "avg", + "metric": "", + "refId": "D", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "MVCC Delete Versions", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, + "fill": 1, + "id": 121, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 250, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(tikv_storage_command_total{type=\"gc\"}[1m]))", + "intervalFactor": 2, + "legendFormat": "total", + "metric": "tikv_storage_command_total", + "refId": "A", + "step": 4 + }, + { + "expr": "sum(rate(tikv_storage_gc_skipped_counter[1m]))", + "intervalFactor": 2, + "legendFormat": "skipped", + "metric": "tikv_storage_gc_skipped_counter", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "GC Commands", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "${DS_TEST-CLUSTER}", + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 966, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 250, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(tidb_tikvclient_gc_worker_actions_total[1m])) by (type)", + "intervalFactor": 2, + "legendFormat": "{{type}}", + "metric": "", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "GC Worker Actions", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "${DS_TEST-CLUSTER}", + "decimals": 0, + "editable": true, + "error": false, + "format": "s", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "id": 27, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "null", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "targets": [ + { + "expr": "max(tidb_tikvclient_gc_config{type=\"tikv_gc_life_time\"})", + "interval": "", + "intervalFactor": 2, + "refId": "A", + "step": 60 + } + ], + "thresholds": "", + "title": "GC LifeTime", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "${DS_TEST-CLUSTER}", + "decimals": 0, + "editable": true, + "error": false, + "format": "s", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "id": 28, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "null", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "targets": [ + { + "expr": "max(tidb_tikvclient_gc_config{type=\"tikv_gc_run_interval\"})", + "intervalFactor": 2, + "refId": "A", + "step": 60 + } + ], + "thresholds": "", + "title": "GC interval", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "GC", + "titleSize": "h6" + }, + { + "collapse": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": "${DS_TEST-CLUSTER}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 35, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": false, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(delta(tikv_raftstore_raft_sent_message_total{type=\"snapshot\"}[1m]))", "intervalFactor": 2, "legendFormat": " ", - "metric": "tikv_storage_mvcc_versions_bucket", "refId": "A", - "step": 10 + "step": 60 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "99.99% MVCC Versions", + "title": "rate snapshot message", "tooltip": { "msResolution": false, "shared": true, - "sort": 2, + "sort": 0, "value_type": "cumulative" }, "type": "graph", @@ -4783,11 +7115,11 @@ }, "yaxes": [ { - "format": "short", + "format": "opm", "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { @@ -4808,11 +7140,12 @@ "error": false, "fill": 1, "grid": {}, - "id": 45, + "id": 36, "legend": { "alignAsTable": true, "avg": false, "current": true, + "hideEmpty": true, "max": true, "min": false, "rightSide": true, @@ -4829,28 +7162,41 @@ "points": false, "renderer": "flot", "seriesOverrides": [], - "span": 6, + "span": 4, "stack": false, "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.9999, sum(rate(tikv_storage_mvcc_gc_delete_versions_bucket[1m])) by (le))", + "expr": "histogram_quantile(0.99, sum(rate(tikv_server_send_snapshot_duration_seconds_bucket[1m])) by (le))", "intervalFactor": 2, - "legendFormat": " ", - "metric": "tikv_storage_mvcc_gc_delete_versions_bucket", + "legendFormat": "send", "refId": "A", - "step": 10 + "step": 60 + }, + { + "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_snapshot_duration_seconds_bucket{type=\"apply\"}[1m])) by (le))", + "intervalFactor": 2, + "legendFormat": "apply", + "refId": "B", + "step": 60 + }, + { + "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_snapshot_duration_seconds_bucket{type=\"generate\"}[1m])) by (le))", + "intervalFactor": 2, + "legendFormat": "generate", + "refId": "C", + "step": 60 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "99.99% MVCC delete versions", + "title": "99% handle snapshot duration", "tooltip": { "msResolution": false, "shared": true, "sort": 0, - "value_type": "individual" + "value_type": "cumulative" }, "type": "graph", "xaxis": { @@ -4861,11 +7207,11 @@ }, "yaxes": [ { - "format": "short", + "format": "s", "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { @@ -4886,11 +7232,12 @@ "error": false, "fill": 1, "grid": {}, - "id": 29, + "id": 38, "legend": { "alignAsTable": true, "avg": false, "current": true, + "hideEmpty": false, "max": true, "min": false, "rightSide": true, @@ -4907,22 +7254,23 @@ "points": false, "renderer": "flot", "seriesOverrides": [], - "span": 6, + "span": 4, "stack": false, - "steppedLine": false, + "steppedLine": true, "targets": [ { - "expr": "sum(rate(tikv_scheduler_stage_total{stage=\"new\", type=\"gc\"}[1m]))", + "expr": "sum(tikv_raftstore_snapshot_traffic_total) by (type)", "intervalFactor": 2, - "legendFormat": " ", + "legendFormat": "{{type}}", + "metric": "", "refId": "A", - "step": 10 + "step": 60 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "scheduler GC stage", + "title": "snapshot state count", "tooltip": { "msResolution": false, "shared": true, @@ -4942,7 +7290,7 @@ "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { @@ -4959,12 +7307,16 @@ "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", + "editable": true, + "error": false, "fill": 1, - "id": 121, + "grid": {}, + "id": 44, "legend": { "alignAsTable": true, "avg": false, "current": true, + "hideEmpty": true, "max": true, "min": false, "rightSide": true, @@ -4975,7 +7327,7 @@ "lines": true, "linewidth": 1, "links": [], - "nullPointMode": "null", + "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, @@ -4986,27 +7338,20 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_storage_command_total{type=\"gc\"}[1m]))", + "expr": "histogram_quantile(0.9999, sum(rate(tikv_snapshot_size_bucket[1m])) by (le))", "intervalFactor": 2, - "legendFormat": "GC Commands", - "metric": "tikv_storage_command_total", + "legendFormat": "size", + "metric": "tikv_snapshot_size_bucket", "refId": "A", - "step": 10 - }, - { - "expr": "sum(rate(tikv_storage_gc_skipped_counter[1m]))", - "intervalFactor": 2, - "legendFormat": "GC Skipped", - "metric": "tikv_storage_gc_skipped_counter", - "refId": "B", - "step": 10 + "step": 40 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "scheduler GC skipped", + "title": "99.99% snapshot size", "tooltip": { + "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" @@ -5020,7 +7365,7 @@ }, "yaxes": [ { - "format": "short", + "format": "bytes", "label": null, "logBase": 1, "max": null, @@ -5037,161 +7382,6 @@ } ] }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "datasource": "${DS_TEST-CLUSTER}", - "decimals": 0, - "editable": true, - "error": false, - "format": "s", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "id": 27, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "null", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 3, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "targets": [ - { - "expr": "max(tidb_tikvclient_gc_config{type=\"tikv_gc_life_time\"})", - "interval": "", - "intervalFactor": 2, - "refId": "A", - "step": 60 - } - ], - "thresholds": "", - "title": "GC LifeTime", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "datasource": "${DS_TEST-CLUSTER}", - "decimals": 0, - "editable": true, - "error": false, - "format": "s", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "id": 28, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "null", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 3, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "targets": [ - { - "expr": "max(tidb_tikvclient_gc_config{type=\"tikv_gc_run_interval\"})", - "intervalFactor": 2, - "refId": "A", - "step": 60 - } - ], - "thresholds": "", - "title": "GC interval", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, { "aliasColors": {}, "bars": false, @@ -5200,13 +7390,12 @@ "error": false, "fill": 1, "grid": {}, - "id": 30, + "id": 43, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, - "hideZero": false, "max": true, "min": false, "rightSide": true, @@ -5228,23 +7417,23 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.9999, sum(rate(tikv_compact_range_cf_duration_seconds_bucket[1m])) by (le, cf, job))", + "expr": "histogram_quantile(0.9999, sum(rate(tikv_snapshot_kv_count_bucket[1m])) by (le))", "intervalFactor": 2, - "legendFormat": "{{job}}-{{cf}}", - "metric": "", + "legendFormat": "count", + "metric": "tikv_snapshot_kv_count_bucket", "refId": "A", - "step": 10 + "step": 40 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "manual compact duration", + "title": "99.99% snapshot kv count", "tooltip": { "msResolution": false, "shared": true, "sort": 0, - "value_type": "cumulative" + "value_type": "individual" }, "type": "graph", "xaxis": { @@ -5277,31 +7466,34 @@ "repeatIteration": null, "repeatRowId": null, "showTitle": true, - "title": "GC", + "title": "Snapshot", "titleSize": "h6" }, { - "collapse": false, - "height": "250px", + "collapse": true, + "height": "300", "panels": [ { "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, "editable": true, "error": false, "fill": 1, "grid": {}, - "id": 35, + "id": 59, "legend": { "alignAsTable": true, "avg": false, "current": true, - "hideEmpty": false, + "hideEmpty": true, + "hideZero": false, "max": true, "min": false, "rightSide": true, "show": true, + "sideWidth": 400, "total": false, "values": true }, @@ -5314,27 +7506,28 @@ "points": false, "renderer": "flot", "seriesOverrides": [], - "span": 4, + "span": 6, "stack": false, "steppedLine": false, "targets": [ { - "expr": "sum(delta(tikv_raftstore_raft_sent_message_total{type=\"snapshot\"}[1m]))", + "expr": "sum(rate(tikv_worker_handled_task_total[1m])) by (name)", "intervalFactor": 2, - "legendFormat": " ", + "legendFormat": "{{name}}", + "metric": "tikv_pd_heartbeat_tick_total", "refId": "A", - "step": 20 + "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "rate snapshot message", + "title": "Worker Handled Tasks", "tooltip": { "msResolution": false, "shared": true, "sort": 0, - "value_type": "cumulative" + "value_type": "individual" }, "type": "graph", "xaxis": { @@ -5345,11 +7538,11 @@ }, "yaxes": [ { - "format": "opm", + "format": "ops", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -5366,67 +7559,57 @@ "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, "editable": true, "error": false, "fill": 1, "grid": {}, - "id": 36, + "id": 1395, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, + "hideZero": false, "max": true, "min": false, "rightSide": true, "show": true, + "sideWidth": 400, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], - "nullPointMode": "null as zero", + "nullPointMode": "null", "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.99, sum(rate(tikv_server_send_snapshot_duration_seconds_bucket[1m])) by (le))", - "intervalFactor": 2, - "legendFormat": "send", - "refId": "A", - "step": 20 - }, - { - "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_snapshot_duration_seconds_bucket{type=\"apply\"}[1m])) by (le))", - "intervalFactor": 2, - "legendFormat": "apply", - "refId": "B", - "step": 20 - }, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_snapshot_duration_seconds_bucket{type=\"generate\"}[1m])) by (le))", + "expr": "sum(rate(tikv_worker_pending_task_total[1m])) by (name)", "intervalFactor": 2, - "legendFormat": "generate", - "refId": "C", - "step": 20 + "legendFormat": "{{name}}", + "metric": "tikv_pd_heartbeat_tick_total", + "refId": "A", + "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "99% handle snapshot duration", + "title": "Worker Pending Tasks", "tooltip": { "msResolution": false, "shared": true, "sort": 0, - "value_type": "cumulative" + "value_type": "individual" }, "type": "graph", "xaxis": { @@ -5437,11 +7620,11 @@ }, "yaxes": [ { - "format": "s", + "format": "short", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -5453,25 +7636,83 @@ "show": true } ] - }, + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Task", + "titleSize": "h6" + }, + { + "collapse": true, + "height": 250, + "panels": [ { + "alert": { + "conditions": [ + { + "evaluator": { + "params": [ + 0.8 + ], + "type": "gt" + }, + "operator": { + "type": "and" + }, + "query": { + "datasourceId": 1, + "model": { + "expr": "sum(rate(tikv_thread_cpu_seconds_total{name=~\"raftstore_.*\"}[1m])) by (job, name)", + "intervalFactor": 2, + "legendFormat": "{{job}}", + "metric": "tikv_thread_cpu_seconds_total", + "refId": "A", + "step": 20 + }, + "params": [ + "A", + "1m", + "now" + ] + }, + "reducer": { + "params": [], + "type": "max" + }, + "type": "query" + } + ], + "executionErrorState": "alerting", + "frequency": "60s", + "handler": 1, + "message": "TiKV raftstore thread CPU usage is high", + "name": "TiKV raft store CPU alert", + "noDataState": "no_data", + "notifications": [] + }, "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, "editable": true, "error": false, - "fill": 1, + "fill": 0, "grid": {}, - "id": 38, + "id": 61, "legend": { "alignAsTable": true, "avg": false, "current": true, - "hideEmpty": false, + "hideEmpty": true, "max": true, "min": false, "rightSide": true, "show": true, + "sideWidth": 250, + "sortDesc": true, "total": false, "values": true }, @@ -5484,28 +7725,36 @@ "points": false, "renderer": "flot", "seriesOverrides": [], - "span": 4, + "span": 6, "stack": false, - "steppedLine": true, + "steppedLine": false, "targets": [ { - "expr": "sum(tikv_raftstore_snapshot_traffic_total) by (type)", + "expr": "sum(rate(tikv_thread_cpu_seconds_total{name=~\"raftstore_.*\"}[1m])) by (job, name)", "intervalFactor": 2, - "legendFormat": "{{type}}", - "metric": "", + "legendFormat": "{{job}}", + "metric": "tikv_thread_cpu_seconds_total", "refId": "A", - "step": 20 + "step": 4 + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 0.8 } ], - "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "snapshot state count", + "title": "raft store CPU", "tooltip": { "msResolution": false, "shared": true, "sort": 0, - "value_type": "cumulative" + "value_type": "individual" }, "type": "graph", "xaxis": { @@ -5516,7 +7765,7 @@ }, "yaxes": [ { - "format": "short", + "format": "percentunit", "label": null, "logBase": 1, "max": null, @@ -5537,27 +7786,28 @@ "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, "editable": true, "error": false, "fill": 1, "grid": {}, - "id": 44, + "id": 79, "legend": { "alignAsTable": true, "avg": false, "current": true, - "hideEmpty": true, "max": true, "min": false, "rightSide": true, "show": true, + "sideWidth": 250, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], - "nullPointMode": "null as zero", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, @@ -5568,18 +7818,18 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.9999, sum(rate(tikv_snapshot_size_bucket[1m])) by (le))", + "expr": "sum(rate(tikv_thread_cpu_seconds_total{name=\"apply_worker\"}[1m])) by (job)", "intervalFactor": 2, - "legendFormat": "size", - "metric": "tikv_snapshot_size_bucket", + "legendFormat": "{{job}}", + "metric": "tikv_thread_cpu_seconds_total", "refId": "A", - "step": 10 + "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "99.99% snapshot size", + "title": "async apply CPU", "tooltip": { "msResolution": false, "shared": true, @@ -5595,11 +7845,11 @@ }, "yaxes": [ { - "format": "bytes", + "format": "percentunit", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -5618,14 +7868,13 @@ "datasource": "${DS_TEST-CLUSTER}", "editable": true, "error": false, - "fill": 1, + "fill": 0, "grid": {}, - "id": 43, + "id": 63, "legend": { "alignAsTable": true, "avg": false, "current": true, - "hideEmpty": true, "max": true, "min": false, "rightSide": true, @@ -5636,7 +7885,7 @@ "lines": true, "linewidth": 1, "links": [], - "nullPointMode": "null as zero", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, @@ -5647,18 +7896,18 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.9999, sum(rate(tikv_snapshot_kv_count_bucket[1m])) by (le))", + "expr": "sum(rate(tikv_thread_cpu_seconds_total{name=~\"storage_schedul.*\"}[1m])) by (job)", "intervalFactor": 2, - "legendFormat": "count", - "metric": "tikv_snapshot_kv_count_bucket", + "legendFormat": "{{job}}", + "metric": "tikv_thread_cpu_seconds_total", "refId": "A", - "step": 10 + "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "99.99% snapshot kv count", + "title": "scheduler CPU", "tooltip": { "msResolution": false, "shared": true, @@ -5674,7 +7923,7 @@ }, "yaxes": [ { - "format": "short", + "format": "percentunit", "label": null, "logBase": 1, "max": null, @@ -5690,38 +7939,28 @@ "show": true } ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Snapshot", - "titleSize": "h6" - }, - { - "collapse": false, - "height": 250, - "panels": [ + }, { "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, "editable": true, "error": false, - "fill": 1, + "fill": 0, "grid": {}, - "id": 59, + "id": 64, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, - "hideZero": false, + "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, + "sideWidth": 250, "total": false, "values": true }, @@ -5739,18 +7978,18 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_worker_pending_task_total[1m])) by (name)", + "expr": "sum(rate(tikv_thread_cpu_seconds_total{name=~\"sched_worker.*\"}[1m])) by (job)", "intervalFactor": 2, - "legendFormat": "{{name}}", - "metric": "tikv_pd_heartbeat_tick_total", + "legendFormat": "{{job}}", + "metric": "tikv_thread_cpu_seconds_total", "refId": "A", - "step": 10 + "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Pending Task Total", + "title": "scheduler worker CPU", "tooltip": { "msResolution": false, "shared": true, @@ -5766,11 +8005,11 @@ }, "yaxes": [ { - "format": "short", + "format": "percentunit", "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { @@ -5787,26 +8026,29 @@ "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, "editable": true, "error": false, - "fill": 1, + "fill": 0, "grid": {}, - "id": 60, + "id": 78, "legend": { "alignAsTable": true, "avg": false, "current": true, + "hideEmpty": true, "max": true, "min": false, "rightSide": true, "show": true, + "sideWidth": 250, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], - "nullPointMode": "null", + "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, @@ -5817,22 +8059,23 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_coprocessor_pending_request[1m]) )by (req)", + "expr": "sum(rate(tikv_thread_cpu_seconds_total{name=~\"endpoint.*\"}[1m])) by (job)", + "interval": "", "intervalFactor": 2, - "legendFormat": "{{req}}", + "legendFormat": "{{job}}", "refId": "A", - "step": 10 + "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "Coprocessor pending request", + "title": "coprocessor CPU", "tooltip": { "msResolution": false, "shared": true, "sort": 0, - "value_type": "individual" + "value_type": "cumulative" }, "type": "graph", "xaxis": { @@ -5843,7 +8086,7 @@ }, "yaxes": [ { - "format": "short", + "format": "percentunit", "label": null, "logBase": 1, "max": null, @@ -5859,88 +8102,35 @@ "show": true } ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Task", - "titleSize": "h6" - }, - { - "collapse": false, - "height": 250, - "panels": [ - { - "alert": { - "conditions": [ - { - "evaluator": { - "params": [ - 0.8 - ], - "type": "gt" - }, - "operator": { - "type": "and" - }, - "query": { - "datasourceId": 1, - "model": { - "expr": "sum(rate(tikv_thread_cpu_seconds_total{name=~\"raftstore_.*\"}[1m])) by (job, name)", - "intervalFactor": 2, - "legendFormat": "{{job}}", - "metric": "tikv_thread_cpu_seconds_total", - "refId": "A", - "step": 30 - }, - "params": [ - "A", - "1m", - "now" - ] - }, - "reducer": { - "params": [], - "type": "max" - }, - "type": "query" - } - ], - "executionErrorState": "alerting", - "frequency": "60s", - "handler": 1, - "message": "TiKV raftstore thread CPU usage is high", - "name": "TiKV raft store CPU alert", - "noDataState": "no_data", - "notifications": [] - }, + }, + { "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, "editable": true, "error": false, "fill": 0, "grid": {}, - "id": 61, + "id": 67, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, + "hideZero": false, "max": true, "min": false, "rightSide": true, "show": true, - "sortDesc": true, + "sideWidth": 250, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], - "nullPointMode": "null", + "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, @@ -5951,26 +8141,18 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_thread_cpu_seconds_total{name=~\"raftstore_.*\"}[1m])) by (job, name)", + "expr": "sum(rate(tikv_thread_cpu_seconds_total{name=~\"snapshot_worker.*\"}[1m])) by (job)", "intervalFactor": 2, "legendFormat": "{{job}}", "metric": "tikv_thread_cpu_seconds_total", "refId": "A", - "step": 10 - } - ], - "thresholds": [ - { - "colorMode": "critical", - "fill": true, - "line": true, - "op": "gt", - "value": 0.8 + "step": 4 } ], + "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "raft store CPU", + "title": "snapshot worker CPU", "tooltip": { "msResolution": false, "shared": true, @@ -6007,11 +8189,12 @@ "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, "editable": true, "error": false, "fill": 0, "grid": {}, - "id": 63, + "id": 68, "legend": { "alignAsTable": true, "avg": false, @@ -6020,6 +8203,7 @@ "min": false, "rightSide": true, "show": true, + "sideWidth": 250, "total": false, "values": true }, @@ -6037,18 +8221,18 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_thread_cpu_seconds_total{name=~\"storage_schedul.*\"}[1m])) by (job)", + "expr": "sum(rate(tikv_thread_cpu_seconds_total{name=~\"split_check.*\"}[1m])) by (job)", "intervalFactor": 2, "legendFormat": "{{job}}", "metric": "tikv_thread_cpu_seconds_total", "refId": "A", - "step": 10 + "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "scheduler CPU", + "title": "split check CPU", "tooltip": { "msResolution": false, "shared": true, @@ -6068,7 +8252,7 @@ "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -6085,21 +8269,23 @@ "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, "editable": true, "error": false, "fill": 0, "grid": {}, - "id": 64, + "id": 69, "legend": { "alignAsTable": true, "avg": false, "current": true, - "hideEmpty": true, - "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, + "sideWidth": 250, + "sort": null, + "sortDesc": null, "total": false, "values": true }, @@ -6117,18 +8303,33 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_thread_cpu_seconds_total{name=~\"sched_worker.*\"}[1m])) by (job)", + "expr": "max(rate(tikv_thread_cpu_seconds_total{name=~\"rocksdb.*\"}[1m])) by (job)", "intervalFactor": 2, "legendFormat": "{{job}}", "metric": "tikv_thread_cpu_seconds_total", "refId": "A", - "step": 10 + "step": 4 + } + ], + "thresholds": [ + { + "colorMode": "warning", + "fill": true, + "line": true, + "op": "gt", + "value": 1 + }, + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 4 } ], - "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "scheduler worker CPU", + "title": "rocksdb CPU", "tooltip": { "msResolution": false, "shared": true, @@ -6165,27 +8366,25 @@ "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", - "editable": true, - "error": false, - "fill": 0, - "grid": {}, - "id": 78, + "decimals": 1, + "fill": 1, + "id": 105, "legend": { "alignAsTable": true, "avg": false, "current": true, - "hideEmpty": true, "max": true, "min": false, "rightSide": true, "show": true, + "sideWidth": 250, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], - "nullPointMode": "null as zero", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, @@ -6196,23 +8395,21 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_thread_cpu_seconds_total{name=~\"endpoint.*\"}[1m])) by (job)", - "interval": "", + "expr": "sum(rate(tikv_thread_cpu_seconds_total{name=~\"grpc.*\"}[1m])) by (job)", "intervalFactor": 2, "legendFormat": "{{job}}", "refId": "A", - "step": 10 + "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "coprocessor CPU", + "title": "grpc poll CPU", "tooltip": { - "msResolution": false, "shared": true, "sort": 0, - "value_type": "cumulative" + "value_type": "individual" }, "type": "graph", "xaxis": { @@ -6239,33 +8436,42 @@ "show": true } ] - }, + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Thread CPU", + "titleSize": "h6" + }, + { + "collapse": true, + "height": "300", + "panels": [ { "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", - "editable": true, - "error": false, - "fill": 0, - "grid": {}, - "id": 67, + "decimals": 1, + "fill": 1, + "id": 138, "legend": { "alignAsTable": true, "avg": false, "current": true, - "hideEmpty": true, - "hideZero": false, "max": true, "min": false, "rightSide": true, "show": true, + "sideWidth": 300, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], - "nullPointMode": "null as zero", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, @@ -6276,20 +8482,48 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_thread_cpu_seconds_total{name=~\"snapshot_worker.*\"}[1m])) by (job)", + "expr": "sum(rate(tikv_engine_memtable_efficiency{db=\"$db\", type=\"memtable_hit\"}[1m]))", "intervalFactor": 2, - "legendFormat": "{{job}}", - "metric": "tikv_thread_cpu_seconds_total", + "legendFormat": "memtable", + "metric": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(rate(tikv_engine_cache_efficiency{db=\"$db\", type=~\"block_cache_data_hit|block_cache_filter_hit\"}[1m]))", + "intervalFactor": 2, + "legendFormat": "block_cache", + "metric": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(rate(tikv_engine_get_served{db=\"$db\", type=\"get_hit_l0\"}[1m]))", + "intervalFactor": 2, + "legendFormat": "l0", "refId": "A", "step": 10 + }, + { + "expr": "sum(rate(tikv_engine_get_served{db=\"$db\", type=\"get_hit_l1\"}[1m]))", + "intervalFactor": 2, + "legendFormat": "l1", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(rate(tikv_engine_get_served{db=\"$db\", type=\"get_hit_l2_and_up\"}[1m]))", + "intervalFactor": 2, + "legendFormat": "l2_and_up", + "refId": "F", + "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "snapshot worker CPU", + "title": "Get Operations", "tooltip": { - "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" @@ -6303,7 +8537,7 @@ }, "yaxes": [ { - "format": "percentunit", + "format": "ops", "label": null, "logBase": 1, "max": null, @@ -6324,11 +8558,9 @@ "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", - "editable": true, - "error": false, + "decimals": 1, "fill": 0, - "grid": {}, - "id": 68, + "id": 82, "legend": { "alignAsTable": true, "avg": false, @@ -6337,12 +8569,14 @@ "min": false, "rightSide": true, "show": true, + "sideWidth": 300, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], + "minSpan": null, "nullPointMode": "null", "percentage": false, "pointradius": 5, @@ -6354,20 +8588,39 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_thread_cpu_seconds_total{name=~\"split_check.*\"}[1m])) by (job)", + "expr": "avg(tikv_engine_get_micro_seconds{db=\"$db\",type=\"get_max\"})", "intervalFactor": 2, - "legendFormat": "{{job}}", - "metric": "tikv_thread_cpu_seconds_total", + "legendFormat": "max", "refId": "A", "step": 10 + }, + { + "expr": "avg(tikv_engine_get_micro_seconds{db=\"$db\",type=\"get_percentile99\"})", + "intervalFactor": 2, + "legendFormat": "99%", + "refId": "B", + "step": 10 + }, + { + "expr": "avg(tikv_engine_get_micro_seconds{db=\"$db\",type=\"get_percentile95\"})", + "intervalFactor": 2, + "legendFormat": "95%", + "refId": "C", + "step": 10 + }, + { + "expr": "avg(tikv_engine_get_micro_seconds{db=\"$db\",type=\"get_average\"})", + "intervalFactor": 2, + "legendFormat": "avg", + "refId": "D", + "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "split check CPU", + "title": "Get Duration", "tooltip": { - "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" @@ -6381,9 +8634,9 @@ }, "yaxes": [ { - "format": "percentunit", + "format": "µs", "label": null, - "logBase": 1, + "logBase": 10, "max": null, "min": "0", "show": true @@ -6402,11 +8655,9 @@ "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", - "editable": true, - "error": false, - "fill": 0, - "grid": {}, - "id": 69, + "decimals": 1, + "fill": 1, + "id": 129, "legend": { "alignAsTable": true, "avg": false, @@ -6415,8 +8666,7 @@ "min": false, "rightSide": true, "show": true, - "sort": null, - "sortDesc": null, + "sideWidth": 300, "total": false, "values": true }, @@ -6434,35 +8684,59 @@ "steppedLine": false, "targets": [ { - "expr": "max(rate(tikv_thread_cpu_seconds_total{name=~\"rocksdb.*\"}[1m])) by (job)", + "expr": "sum(rate(tikv_engine_locate{db=\"$db\", type=\"number_db_seek\"}[1m]))", "intervalFactor": 2, - "legendFormat": "{{job}}", - "metric": "tikv_thread_cpu_seconds_total", + "legendFormat": "seek", + "metric": "", "refId": "A", "step": 10 - } - ], - "thresholds": [ + }, { - "colorMode": "warning", - "fill": true, - "line": true, - "op": "gt", - "value": 1 + "expr": "sum(rate(tikv_engine_locate{db=\"$db\", type=\"number_db_seek_found\"}[1m]))", + "intervalFactor": 2, + "legendFormat": "seek_found", + "metric": "", + "refId": "B", + "step": 10 }, { - "colorMode": "critical", - "fill": true, - "line": true, - "op": "gt", - "value": 4 + "expr": "sum(rate(tikv_engine_locate{db=\"$db\", type=\"number_db_next\"}[1m]))", + "intervalFactor": 2, + "legendFormat": "next", + "metric": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(rate(tikv_engine_locate{db=\"$db\", type=\"number_db_next_found\"}[1m]))", + "intervalFactor": 2, + "legendFormat": "next_found", + "metric": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(rate(tikv_engine_locate{db=\"$db\", type=\"number_db_prev\"}[1m]))", + "intervalFactor": 2, + "legendFormat": "prev", + "metric": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(rate(tikv_engine_locate{db=\"$db\", type=\"number_db_prev_found\"}[1m]))", + "intervalFactor": 2, + "legendFormat": "prev_found", + "metric": "", + "refId": "F", + "step": 10 } ], + "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "rocksdb CPU", + "title": "Seek Operations", "tooltip": { - "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" @@ -6476,7 +8750,7 @@ }, "yaxes": [ { - "format": "percentunit", + "format": "ops", "label": null, "logBase": 1, "max": null, @@ -6497,11 +8771,9 @@ "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "id": 79, + "decimals": 1, + "fill": 0, + "id": 125, "legend": { "alignAsTable": true, "avg": false, @@ -6510,12 +8782,14 @@ "min": false, "rightSide": true, "show": true, + "sideWidth": 300, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], + "minSpan": null, "nullPointMode": "null", "percentage": false, "pointradius": 5, @@ -6527,20 +8801,39 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_thread_cpu_seconds_total{name=\"apply_worker\"}[1m])) by (job)", + "expr": "avg(tikv_engine_seek_micro_seconds{db=\"$db\",type=\"seek_max\"})", "intervalFactor": 2, - "legendFormat": "{{job}}", - "metric": "tikv_thread_cpu_seconds_total", + "legendFormat": "max", "refId": "A", "step": 10 + }, + { + "expr": "avg(tikv_engine_seek_micro_seconds{db=\"$db\",type=\"seek_percentile99\"})", + "intervalFactor": 2, + "legendFormat": "99%", + "refId": "B", + "step": 10 + }, + { + "expr": "avg(tikv_engine_seek_micro_seconds{db=\"$db\",type=\"seek_percentile95\"})", + "intervalFactor": 2, + "legendFormat": "95%", + "refId": "C", + "step": 10 + }, + { + "expr": "avg(tikv_engine_seek_micro_seconds{db=\"$db\",type=\"seek_average\"})", + "intervalFactor": 2, + "legendFormat": "avg", + "refId": "D", + "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "async apply CPU", + "title": "Seek Duration", "tooltip": { - "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" @@ -6554,9 +8847,9 @@ }, "yaxes": [ { - "format": "percentunit", + "format": "µs", "label": null, - "logBase": 1, + "logBase": 10, "max": null, "min": "0", "show": true @@ -6575,18 +8868,20 @@ "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, "fill": 1, - "id": 105, + "id": 139, "legend": { "alignAsTable": true, "avg": false, - "current": false, - "max": false, + "current": true, + "max": true, "min": false, "rightSide": true, "show": true, + "sideWidth": 300, "total": false, - "values": false + "values": true }, "lines": true, "linewidth": 1, @@ -6602,17 +8897,31 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_thread_cpu_seconds_total{name=~\"grpc.*\"}[1m])) by (job)", + "expr": "sum(rate(tikv_engine_write_served{db=\"$db\", type=~\"write_done_by_self|write_done_by_other\"}[1m]))", "intervalFactor": 2, - "legendFormat": "{{job}}", + "legendFormat": "done", "refId": "A", "step": 10 + }, + { + "expr": "sum(rate(tikv_engine_write_served{db=\"$db\", type=\"write_timeout\"}[1m]))", + "intervalFactor": 2, + "legendFormat": "timeout", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(rate(tikv_engine_write_served{db=\"$db\", type=\"write_with_wal\"}[1m]))", + "intervalFactor": 2, + "legendFormat": "with_wal", + "refId": "C", + "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "grpc poll CPU", + "title": "Write Operations", "tooltip": { "shared": true, "sort": 0, @@ -6627,7 +8936,7 @@ }, "yaxes": [ { - "format": "percentunit", + "format": "ops", "label": null, "logBase": 1, "max": null, @@ -6643,41 +8952,31 @@ "show": true } ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Thread CPU", - "titleSize": "h6" - }, - { - "collapse": false, - "height": 250, - "panels": [ + }, { "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, "fill": 0, - "id": 80, + "id": 126, "legend": { "alignAsTable": true, "avg": false, - "current": false, - "max": false, + "current": true, + "max": true, "min": false, "rightSide": true, "show": true, + "sideWidth": 300, "total": false, - "values": false + "values": true }, "lines": true, "linewidth": 1, "links": [], - "minSpan": 6, - "nullPointMode": "connected", + "minSpan": null, + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, @@ -6688,23 +8987,43 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_engine_cache_efficiency{db=\"kv\", type=\"block_cache_hit\"}[1m])) by (job) / (sum(rate(tikv_engine_cache_efficiency{db=\"kv\", type=\"block_cache_hit\"}[1m])) by (job) + sum(rate(tikv_engine_cache_efficiency{db=\"kv\", type=\"block_cache_miss\"}[1m])) by (job))", + "expr": "avg(tikv_engine_write_micro_seconds{db=\"$db\",type=\"write_max\"})", "intervalFactor": 2, - "legendFormat": "{{job}}", + "legendFormat": "max", "refId": "A", "step": 10 + }, + { + "expr": "avg(tikv_engine_write_micro_seconds{db=\"$db\",type=\"write_percentile99\"})", + "intervalFactor": 2, + "legendFormat": "99%", + "refId": "B", + "step": 10 + }, + { + "expr": "avg(tikv_engine_write_micro_seconds{db=\"$db\",type=\"write_percentile95\"})", + "intervalFactor": 2, + "legendFormat": "95%", + "refId": "C", + "step": 10 + }, + { + "expr": "avg(tikv_engine_write_micro_seconds{db=\"$db\",type=\"write_average\"})", + "intervalFactor": 2, + "legendFormat": "avg", + "refId": "D", + "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "block cache hit rate", + "title": "Write Duration", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, - "transparent": false, "type": "graph", "xaxis": { "mode": "time", @@ -6714,20 +9033,20 @@ }, "yaxes": [ { - "format": "percentunit", + "format": "µs", "label": null, - "logBase": 1, + "logBase": 10, "max": null, "min": "0", "show": true }, { - "format": "ops", + "format": "short", "label": null, "logBase": 1, "max": null, "min": null, - "show": false + "show": true } ] }, @@ -6735,24 +9054,25 @@ "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", - "fill": 0, - "id": 88, + "decimals": 1, + "fill": 1, + "id": 137, "legend": { "alignAsTable": true, "avg": false, - "current": false, - "max": false, + "current": true, + "max": true, "min": false, "rightSide": true, "show": true, + "sideWidth": 300, "total": false, - "values": false + "values": true }, "lines": true, "linewidth": 1, "links": [], - "minSpan": 6, - "nullPointMode": "connected", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, @@ -6763,9 +9083,10 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_engine_memtable_efficiency{db=\"kv\", type=\"memtable_hit\"}[1m])) by (job) / (sum(rate(tikv_engine_memtable_efficiency{db=\"kv\", type=\"memtable_hit\"}[1m])) by (job) + sum(rate(tikv_engine_memtable_efficiency{db=\"kv\", type=\"memtable_miss\"}[1m])) by (job))", + "expr": "sum(rate(tikv_engine_wal_file_synced{db=\"$db\"}[1m]))", "intervalFactor": 2, - "legendFormat": "{{job}}", + "legendFormat": "sync", + "metric": "", "refId": "A", "step": 10 } @@ -6773,13 +9094,12 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "memtable hit rate", + "title": "WAL Sync Operations", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, - "transparent": false, "type": "graph", "xaxis": { "mode": "time", @@ -6789,20 +9109,20 @@ }, "yaxes": [ { - "format": "percentunit", + "format": "ops", "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { - "format": "ops", + "format": "short", "label": null, "logBase": 1, "max": null, "min": null, - "show": false + "show": true } ] }, @@ -6810,16 +9130,18 @@ "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, "fill": 0, - "id": 82, + "id": 135, "legend": { "alignAsTable": true, "avg": false, "current": true, - "max": false, + "max": true, "min": false, "rightSide": true, "show": true, + "sideWidth": 300, "total": false, "values": true }, @@ -6838,31 +9160,38 @@ "steppedLine": false, "targets": [ { - "expr": "avg(tikv_engine_get_micro_seconds{db=\"kv\", type=\"get_average\"})", + "expr": "avg(tikv_engine_wal_file_sync_micro_seconds{db=\"$db\",type=\"wal_file_sync_max\"})", "intervalFactor": 2, - "legendFormat": "average", + "legendFormat": "max", "refId": "A", "step": 10 }, { - "expr": "avg(tikv_engine_get_micro_seconds{db=\"kv\", type=\"get_percentile99\"})", + "expr": "avg(tikv_engine_wal_file_sync_micro_seconds{db=\"$db\",type=\"wal_file_sync_percentile99\"})", "intervalFactor": 2, - "legendFormat": "percentile99", + "legendFormat": "99%", "refId": "B", "step": 10 }, { - "expr": "avg(tikv_engine_get_micro_seconds{db=\"kv\", type=\"get_percentile95\"})", + "expr": "avg(tikv_engine_wal_file_sync_micro_seconds{db=\"$db\",type=\"wal_file_sync_percentile95\"})", "intervalFactor": 2, - "legendFormat": "percentile95", + "legendFormat": "95%", "refId": "C", "step": 10 + }, + { + "expr": "avg(tikv_engine_wal_file_sync_micro_seconds{db=\"$db\",type=\"wal_file_sync_average\"})", + "intervalFactor": 2, + "legendFormat": "avg", + "refId": "D", + "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "get", + "title": "WAL Sync Duration", "tooltip": { "shared": true, "sort": 0, @@ -6879,7 +9208,7 @@ { "format": "µs", "label": null, - "logBase": 1, + "logBase": 10, "max": null, "min": "0", "show": true @@ -6898,23 +9227,24 @@ "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", - "fill": 0, - "id": 83, + "decimals": 1, + "fill": 1, + "id": 128, "legend": { "alignAsTable": true, "avg": false, "current": true, - "max": false, + "max": true, "min": false, "rightSide": true, "show": true, + "sideWidth": 300, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], - "minSpan": 6, "nullPointMode": "null", "percentage": false, "pointradius": 5, @@ -6926,31 +9256,18 @@ "steppedLine": false, "targets": [ { - "expr": "avg(tikv_engine_write_micro_seconds{db=\"kv\", type=\"write_average\"})", - "intervalFactor": 2, - "legendFormat": "average", - "refId": "A", - "step": 10 - }, - { - "expr": "avg(tikv_engine_write_micro_seconds{db=\"kv\", type=\"write_percentile99\"})", + "expr": "sum(rate(tikv_engine_event_total{db=\"$db\"}[1m])) by (type)", "intervalFactor": 2, - "legendFormat": "percentile99", + "legendFormat": "{{type}}", + "metric": "tikv_engine_event_total", "refId": "B", "step": 10 - }, - { - "expr": "avg(tikv_engine_write_micro_seconds{db=\"kv\", type=\"write_percentile95\"})", - "intervalFactor": 2, - "legendFormat": "percentile95", - "refId": "C", - "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "write", + "title": "Compaction Operations", "tooltip": { "shared": true, "sort": 0, @@ -6965,11 +9282,11 @@ }, "yaxes": [ { - "format": "µs", + "format": "ops", "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { @@ -6986,23 +9303,25 @@ "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, "fill": 0, - "id": 84, + "id": 136, "legend": { "alignAsTable": true, "avg": false, "current": true, - "max": false, + "max": true, "min": false, "rightSide": true, "show": true, + "sideWidth": 300, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], - "minSpan": 6, + "minSpan": null, "nullPointMode": "null", "percentage": false, "pointradius": 5, @@ -7014,31 +9333,39 @@ "steppedLine": false, "targets": [ { - "expr": "avg(tikv_engine_seek_micro_seconds{db=\"kv\", type=\"seek_average\"})", + "expr": "avg(tikv_engine_compaction_time{db=\"$db\",type=\"compaction_time_max\"})", "intervalFactor": 2, - "legendFormat": "average", + "legendFormat": "max", + "metric": "", "refId": "A", "step": 10 }, { - "expr": "avg(tikv_engine_seek_micro_seconds{db=\"kv\", type=\"seek_percentile99\"})", + "expr": "avg(tikv_engine_compaction_time{db=\"$db\",type=\"compaction_time_percentile99\"})", "intervalFactor": 2, - "legendFormat": "percentile99", + "legendFormat": "99%", "refId": "B", "step": 10 }, { - "expr": "avg(tikv_engine_seek_micro_seconds{db=\"kv\", type=\"seek_percentile95\"})", + "expr": "avg(tikv_engine_compaction_time{db=\"$db\",type=\"compaction_time_percentile95\"})", "intervalFactor": 2, - "legendFormat": "percentile95", + "legendFormat": "95%", "refId": "C", "step": 10 + }, + { + "expr": "avg(tikv_engine_compaction_time{db=\"$db\",type=\"compaction_time_average\"})", + "intervalFactor": 2, + "legendFormat": "avg", + "refId": "D", + "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "seek", + "title": "Compaction Duration", "tooltip": { "shared": true, "sort": 0, @@ -7055,7 +9382,7 @@ { "format": "µs", "label": null, - "logBase": 1, + "logBase": 10, "max": null, "min": "0", "show": true @@ -7075,14 +9402,17 @@ "bars": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, - "fill": 0, - "id": 85, + "fill": 1, + "id": 140, "legend": { + "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, + "rightSide": true, "show": true, + "sideWidth": 300, "total": false, "values": true }, @@ -7094,44 +9424,48 @@ "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [ - { - "alias": "keys_read", - "yaxis": 2 - } - ], + "seriesOverrides": [], "span": 6, "stack": false, "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_engine_flow_bytes{db=\"kv\", type=\"bytes_read\"}[1m]))", - "hide": false, + "expr": "avg(rate(tikv_engine_sst_read_micros{db=\"$db\", type=\"sst_read_micros_max\"}[1m]))", "intervalFactor": 2, - "legendFormat": "bytes_read", + "legendFormat": "max", + "metric": "", "refId": "A", "step": 10 }, { - "expr": "sum(rate(tikv_engine_flow_bytes{db=\"kv\", type=\"keys_read\"}[1m]))", - "hide": false, + "expr": "avg(rate(tikv_engine_sst_read_micros{db=\"$db\", type=\"sst_read_micros_percentile99\"}[1m]))", "intervalFactor": 2, - "legendFormat": "keys_read", + "legendFormat": "99%", + "metric": "", "refId": "B", "step": 10 }, { - "expr": "sum(rate(tikv_engine_flow_bytes{db=\"kv\", type=\"iter_bytes_read\"}[1m]))", + "expr": "avg(rate(tikv_engine_sst_read_micros{db=\"$db\", type=\"sst_read_micros_percentile95\"}[1m]))", "intervalFactor": 2, - "legendFormat": "iter_bytes_read", + "legendFormat": "95%", + "metric": "", "refId": "C", "step": 10 + }, + { + "expr": "avg(rate(tikv_engine_sst_read_micros{db=\"$db\", type=\"sst_read_micros_average\"}[1m]))", + "intervalFactor": 2, + "legendFormat": "avg", + "metric": "", + "refId": "D", + "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "read flow", + "title": "SST Read Duration", "tooltip": { "shared": true, "sort": 0, @@ -7146,19 +9480,19 @@ }, "yaxes": [ { - "format": "Bps", + "format": "ms", "label": null, - "logBase": 1, + "logBase": 10, "max": null, - "min": "0", + "min": null, "show": true }, { - "format": "rps", + "format": "short", "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true } ] @@ -7169,13 +9503,16 @@ "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "fill": 1, - "id": 86, + "id": 87, "legend": { + "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, + "rightSide": true, "show": true, + "sideWidth": 300, "total": false, "values": true }, @@ -7187,37 +9524,48 @@ "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [ - { - "alias": "keys_written", - "yaxis": 2 - } - ], + "seriesOverrides": [], "span": 6, "stack": false, "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_engine_flow_bytes{db=\"kv\", type=\"bytes_written\"}[1m])) by (type)", - "hide": false, + "expr": "avg(rate(tikv_engine_write_stall{db=\"$db\", type=\"write_stall_max\"}[1m]))", "intervalFactor": 2, - "legendFormat": "{{type}}", + "legendFormat": "max", + "metric": "", "refId": "A", "step": 10 }, { - "expr": "sum(rate(tikv_engine_flow_bytes{db=\"kv\", type=\"keys_written\"}[1m])) by (type)", - "hide": false, + "expr": "avg(rate(tikv_engine_write_stall{db=\"$db\", type=\"write_stall_percentile99\"}[1m]))", "intervalFactor": 2, - "legendFormat": "{{type}}", + "legendFormat": "99%", + "metric": "", "refId": "B", "step": 10 + }, + { + "expr": "avg(rate(tikv_engine_write_stall{db=\"$db\", type=\"write_stall_percentile95\"}[1m]))", + "intervalFactor": 2, + "legendFormat": "95%", + "metric": "", + "refId": "C", + "step": 10 + }, + { + "expr": "avg(rate(tikv_engine_write_stall{db=\"$db\", type=\"write_stall_average\"}[1m]))", + "intervalFactor": 2, + "legendFormat": "avg", + "metric": "", + "refId": "D", + "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "write flow", + "title": "Write Stall Duration", "tooltip": { "shared": true, "sort": 0, @@ -7232,19 +9580,19 @@ }, "yaxes": [ { - "format": "Bps", + "format": "ms", "label": null, - "logBase": 1, + "logBase": 10, "max": null, - "min": "0", + "min": null, "show": true }, { - "format": "wps", + "format": "short", "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true } ] @@ -7253,16 +9601,20 @@ "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, "fill": 1, - "id": 87, + "id": 103, "legend": { + "alignAsTable": true, "avg": false, - "current": false, - "max": false, + "current": true, + "max": true, "min": false, + "rightSide": true, "show": true, + "sideWidth": 300, "total": false, - "values": false + "values": true }, "lines": true, "linewidth": 1, @@ -7278,9 +9630,9 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_engine_stall_micro_seconds{db=\"kv\"}[30s])) by (job)", + "expr": "avg(tikv_engine_memory_bytes{db=\"$db\", type=\"mem-tables\"}) by (cf)", "intervalFactor": 2, - "legendFormat": "{{job}}", + "legendFormat": "{{cf}}", "refId": "A", "step": 10 } @@ -7288,7 +9640,7 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "stall", + "title": "Memtable Size", "tooltip": { "shared": true, "sort": 0, @@ -7303,7 +9655,7 @@ }, "yaxes": [ { - "format": "ms", + "format": "bytes", "label": null, "logBase": 1, "max": null, @@ -7325,8 +9677,8 @@ "bars": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, - "fill": 1, - "id": 89, + "fill": 0, + "id": 88, "legend": { "alignAsTable": true, "avg": false, @@ -7335,32 +9687,28 @@ "min": false, "rightSide": true, "show": true, + "sideWidth": 300, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], - "nullPointMode": "null", + "minSpan": null, + "nullPointMode": "connected", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [ - { - "alias": "keys_written", - "yaxis": 2 - } - ], + "seriesOverrides": [], "span": 6, "stack": false, "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_engine_bloom_efficiency{db=\"kv\"}[1m])) by (type)", - "hide": false, + "expr": "sum(rate(tikv_engine_memtable_efficiency{db=\"$db\", type=\"memtable_hit\"}[1m])) / (sum(rate(tikv_engine_memtable_efficiency{db=\"$db\", type=\"memtable_hit\"}[1m])) + sum(rate(tikv_engine_memtable_efficiency{db=\"$db\", type=\"memtable_miss\"}[1m])))", "intervalFactor": 2, - "legendFormat": "{{type}}", + "legendFormat": "hit", "refId": "A", "step": 10 } @@ -7368,12 +9716,13 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "pending compaction bytes", + "title": "Memtable Hit", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, + "transparent": false, "type": "graph", "xaxis": { "mode": "time", @@ -7383,7 +9732,7 @@ }, "yaxes": [ { - "format": "bytes", + "format": "percentunit", "label": null, "logBase": 1, "max": null, @@ -7391,12 +9740,12 @@ "show": true }, { - "format": "wps", + "format": "ops", "label": null, "logBase": 1, "max": null, - "min": "0", - "show": true + "min": null, + "show": false } ] }, @@ -7406,7 +9755,7 @@ "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "fill": 1, - "id": 90, + "id": 102, "legend": { "alignAsTable": true, "avg": false, @@ -7415,6 +9764,7 @@ "min": false, "rightSide": true, "show": true, + "sideWidth": 300, "total": false, "values": true }, @@ -7426,21 +9776,15 @@ "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [ - { - "alias": "keys_written", - "yaxis": 2 - } - ], + "seriesOverrides": [], "span": 6, "stack": false, "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_engine_compaction_flow_bytes{db=\"kv\"}[1m])) by (job)", - "hide": false, + "expr": "avg(tikv_engine_block_cache_size_bytes{db=\"$db\"}) by(cf)", "intervalFactor": 2, - "legendFormat": "{{job}}", + "legendFormat": "{{cf}}", "refId": "A", "step": 10 } @@ -7448,7 +9792,7 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "compaction flow", + "title": "Block Cache Size", "tooltip": { "shared": true, "sort": 0, @@ -7463,19 +9807,19 @@ }, "yaxes": [ { - "format": "Bps", + "format": "bytes", "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { - "format": "wps", + "format": "short", "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true } ] @@ -7484,8 +9828,9 @@ "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", - "fill": 1, - "id": 102, + "decimals": 1, + "fill": 0, + "id": 80, "legend": { "alignAsTable": true, "avg": false, @@ -7494,13 +9839,15 @@ "min": false, "rightSide": true, "show": true, + "sideWidth": 300, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], - "nullPointMode": "null", + "minSpan": 6, + "nullPointMode": "connected", "percentage": false, "pointradius": 5, "points": false, @@ -7511,22 +9858,56 @@ "steppedLine": false, "targets": [ { - "expr": "sum(tikv_engine_block_cache_size_bytes{db=\"kv\"}) by(job, cf)", + "expr": "sum(rate(tikv_engine_cache_efficiency{db=\"$db\", type=\"block_cache_hit\"}[1m])) / (sum(rate(tikv_engine_cache_efficiency{db=\"$db\", type=\"block_cache_hit\"}[1m])) + sum(rate(tikv_engine_cache_efficiency{db=\"$db\", type=\"block_cache_miss\"}[1m])))", "intervalFactor": 2, - "legendFormat": "{{job}}-{{cf}}", + "legendFormat": "all", + "metric": "", "refId": "A", "step": 10 + }, + { + "expr": "sum(rate(tikv_engine_cache_efficiency{db=\"$db\", type=\"block_cache_data_hit\"}[1m])) / (sum(rate(tikv_engine_cache_efficiency{db=\"$db\", type=\"block_cache_data_hit\"}[1m])) + sum(rate(tikv_engine_cache_efficiency{db=\"$db\", type=\"block_cache_data_miss\"}[1m])))", + "intervalFactor": 2, + "legendFormat": "data", + "metric": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(rate(tikv_engine_cache_efficiency{db=\"$db\", type=\"block_cache_filter_hit\"}[1m])) / (sum(rate(tikv_engine_cache_efficiency{db=\"$db\", type=\"block_cache_filter_hit\"}[1m])) + sum(rate(tikv_engine_cache_efficiency{db=\"$db\", type=\"block_cache_filter_miss\"}[1m])))", + "intervalFactor": 2, + "legendFormat": "filter", + "metric": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(rate(tikv_engine_cache_efficiency{db=\"$db\", type=\"block_cache_index_hit\"}[1m])) / (sum(rate(tikv_engine_cache_efficiency{db=\"$db\", type=\"block_cache_index_hit\"}[1m])) + sum(rate(tikv_engine_cache_efficiency{db=\"$db\", type=\"block_cache_index_miss\"}[1m])))", + "intervalFactor": 2, + "legendFormat": "index", + "metric": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(rate(tikv_engine_bloom_efficiency{db=\"$db\", type=\"bloom_prefix_useful\"}[1m])) / sum(rate(tikv_engine_bloom_efficiency{db=\"$db\", type=\"bloom_prefix_checked\"}[1m]))", + "intervalFactor": 2, + "legendFormat": "bloom prefix", + "metric": "", + "refId": "E", + "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "cache", + "title": "Block Cache Hit", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, + "transparent": false, "type": "graph", "xaxis": { "mode": "time", @@ -7536,20 +9917,20 @@ }, "yaxes": [ { - "format": "bytes", + "format": "percentunit", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { - "format": "short", + "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, - "show": true + "show": false } ] }, @@ -7557,18 +9938,21 @@ "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", - "fill": 1, - "id": 103, + "decimals": 1, + "fill": 0, + "height": "", + "id": 467, "legend": { "alignAsTable": true, "avg": false, - "current": false, - "max": false, + "current": true, + "max": true, "min": false, "rightSide": true, "show": true, + "sideWidth": 300, "total": false, - "values": false + "values": true }, "lines": true, "linewidth": 1, @@ -7584,17 +9968,78 @@ "steppedLine": false, "targets": [ { - "expr": "sum(tikv_engine_memory_bytes{db=\"kv\", type=\"mem-tables\"}) by (job, cf)", + "expr": "sum(rate(tikv_engine_flow_bytes{db=\"$db\", type=\"block_cache_byte_read\"}[1m]))", + "hide": false, + "interval": "", "intervalFactor": 2, - "legendFormat": "{{job}}-{{cf}}", + "legendFormat": "total_read", "refId": "A", "step": 10 + }, + { + "expr": "sum(rate(tikv_engine_flow_bytes{db=\"$db\", type=\"block_cache_byte_write\"}[1m]))", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "total_written", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(rate(tikv_engine_cache_efficiency{db=\"$db\", type=\"block_cache_data_bytes_insert\"}[1m]))", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "data_insert", + "metric": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(rate(tikv_engine_cache_efficiency{db=\"$db\", type=\"block_cache_filter_bytes_insert\"}[1m]))", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "filter_insert", + "metric": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(rate(tikv_engine_cache_efficiency{db=\"$db\", type=\"block_cache_filter_bytes_evict\"}[1m]))", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "filter_evict", + "metric": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(rate(tikv_engine_cache_efficiency{db=\"$db\", type=\"block_cache_index_bytes_insert\"}[1m]))", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "index_insert", + "metric": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sum(rate(tikv_engine_cache_efficiency{db=\"$db\", type=\"block_cache_index_bytes_evict\"}[1m]))", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "index_evict", + "metric": "", + "refId": "G", + "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "mem table", + "title": "Block Cache Flow", "tooltip": { "shared": true, "sort": 0, @@ -7609,19 +10054,19 @@ }, "yaxes": [ { - "format": "short", + "format": "Bps", "label": null, - "logBase": 1, + "logBase": 10, "max": null, - "min": null, + "min": "0", "show": true }, { - "format": "short", + "format": "none", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true } ] @@ -7630,18 +10075,20 @@ "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, "fill": 1, - "id": 104, + "id": 468, "legend": { "alignAsTable": true, "avg": false, - "current": false, - "max": false, + "current": true, + "max": true, "min": false, "rightSide": true, "show": true, + "sideWidth": 300, "total": false, - "values": false + "values": true }, "lines": true, "linewidth": 1, @@ -7657,17 +10104,50 @@ "steppedLine": false, "targets": [ { - "expr": "sum(tikv_engine_memory_bytes{db=\"kv\", type=\"readers-mem\"}) by (job, cf)", + "expr": "sum(rate(tikv_engine_cache_efficiency{db=\"$db\", type=\"block_cache_add\"}[1m]))", "intervalFactor": 2, - "legendFormat": "{{job}}-{{cf}}", + "legendFormat": "total_add", + "metric": "", "refId": "A", "step": 10 + }, + { + "expr": "sum(rate(tikv_engine_cache_efficiency{db=\"$db\", type=\"block_cache_data_add\"}[1m]))", + "intervalFactor": 2, + "legendFormat": "data_add", + "metric": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(rate(tikv_engine_cache_efficiency{db=\"$db\", type=\"block_cache_filter_add\"}[1m]))", + "intervalFactor": 2, + "legendFormat": "filter_add", + "metric": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(rate(tikv_engine_cache_efficiency{db=\"$db\", type=\"block_cache_index_add\"}[1m]))", + "intervalFactor": 2, + "legendFormat": "index_add", + "metric": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(rate(tikv_engine_cache_efficiency{db=\"$db\", type=\"block_cache_add_failures\"}[1m]))", + "intervalFactor": 2, + "legendFormat": "add_failures", + "metric": "", + "refId": "B", + "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "table reader mem", + "title": "Block Cache Operations", "tooltip": { "shared": true, "sort": 0, @@ -7682,7 +10162,7 @@ }, "yaxes": [ { - "format": "short", + "format": "ops", "label": null, "logBase": 1, "max": null, @@ -7703,24 +10183,26 @@ "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, "fill": 0, - "id": 123, + "height": "", + "id": 132, "legend": { "alignAsTable": true, "avg": false, - "current": false, - "max": false, + "current": true, + "max": true, "min": false, "rightSide": true, "show": true, + "sideWidth": 300, "total": false, - "values": false + "values": true }, "lines": true, "linewidth": 1, "links": [], - "minSpan": 6, - "nullPointMode": "connected", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, @@ -7731,9 +10213,29 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_engine_bloom_efficiency{db=\"kv\", type=\"bloom_prefix_useful\"}[1m])) by (job) / sum(rate(tikv_engine_bloom_efficiency{db=\"kv\", type=\"bloom_prefix_checked\"}[1m])) by (job)", + "expr": "sum(rate(tikv_engine_flow_bytes{db=\"$db\", type=\"keys_read\"}[1m]))", + "hide": false, + "interval": "", "intervalFactor": 2, - "legendFormat": "{{job}}", + "legendFormat": "read", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(rate(tikv_engine_flow_bytes{db=\"$db\", type=\"keys_written\"}[1m]))", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "written", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(rate(tikv_engine_compaction_num_corrupt_keys{db=\"$db\"}[1m]))", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "corrupt", "metric": "", "refId": "A", "step": 10 @@ -7742,13 +10244,12 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "bloom prefix hit rate", + "title": "Keys Flow", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, - "transparent": false, "type": "graph", "xaxis": { "mode": "time", @@ -7758,7 +10259,7 @@ }, "yaxes": [ { - "format": "percentunit", + "format": "ops", "label": null, "logBase": 1, "max": null, @@ -7766,49 +10267,38 @@ "show": true }, { - "format": "ops", + "format": "short", "label": null, "logBase": 1, "max": null, - "min": null, - "show": false + "min": "0", + "show": true } ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Rocksdb", - "titleSize": "h6" - }, - { - "collapse": false, - "height": 250, - "panels": [ + }, { "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", - "fill": 0, - "id": 124, + "decimals": 1, + "fill": 1, + "id": 131, "legend": { "alignAsTable": true, "avg": false, - "current": false, - "max": false, + "current": true, + "max": true, "min": false, "rightSide": true, "show": true, + "sideWidth": 300, "total": false, - "values": false + "values": true }, "lines": true, "linewidth": 1, "links": [], - "minSpan": 6, - "nullPointMode": "connected", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, @@ -7819,9 +10309,11 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_engine_cache_efficiency{db=\"raft\", type=\"block_cache_hit\"}[1m])) by (job) / (sum(rate(tikv_engine_cache_efficiency{db=\"raft\", type=\"block_cache_hit\"}[1m])) by (job) + sum(rate(tikv_engine_cache_efficiency{db=\"raft\", type=\"block_cache_miss\"}[1m])) by (job))", + "expr": "sum(tikv_engine_estimate_num_keys{db=\"$db\"}) by (cf)", + "hide": false, "intervalFactor": 2, - "legendFormat": "{{job}}", + "legendFormat": "{{cf}}", + "metric": "tikv_engine_estimate_num_keys", "refId": "A", "step": 10 } @@ -7829,13 +10321,12 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "block cache hit rate", + "title": "Total Keys", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, - "transparent": false, "type": "graph", "xaxis": { "mode": "time", @@ -7845,7 +10336,7 @@ }, "yaxes": [ { - "format": "percentunit", + "format": "short", "label": null, "logBase": 1, "max": null, @@ -7853,12 +10344,12 @@ "show": true }, { - "format": "ops", + "format": "short", "label": null, "logBase": 1, "max": null, - "min": null, - "show": false + "min": "0", + "show": true } ] }, @@ -7866,24 +10357,26 @@ "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, "fill": 0, - "id": 125, + "height": "", + "id": 85, "legend": { "alignAsTable": true, "avg": false, - "current": false, - "max": false, + "current": true, + "max": true, "min": false, "rightSide": true, "show": true, + "sideWidth": 300, "total": false, - "values": false + "values": true }, "lines": true, "linewidth": 1, "links": [], - "minSpan": 6, - "nullPointMode": "connected", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, @@ -7894,23 +10387,33 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_engine_memtable_efficiency{db=\"raft\", type=\"memtable_hit\"}[1m])) by (job) / (sum(rate(tikv_engine_memtable_efficiency{db=\"raft\", type=\"memtable_hit\"}[1m])) by (job) + sum(rate(tikv_engine_memtable_efficiency{db=\"raft\", type=\"memtable_miss\"}[1m])) by (job))", + "expr": "sum(rate(tikv_engine_flow_bytes{db=\"$db\", type=\"bytes_read\"}[1m]))", + "hide": false, + "interval": "", "intervalFactor": 2, - "legendFormat": "{{job}}", + "legendFormat": "get", "refId": "A", "step": 10 + }, + { + "expr": "sum(rate(tikv_engine_flow_bytes{db=\"$db\", type=\"iter_bytes_read\"}[1m]))", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "scan", + "refId": "C", + "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "memtable hit rate", + "title": "Read Flow", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, - "transparent": false, "type": "graph", "xaxis": { "mode": "time", @@ -7920,7 +10423,7 @@ }, "yaxes": [ { - "format": "percentunit", + "format": "Bps", "label": null, "logBase": 1, "max": null, @@ -7928,12 +10431,12 @@ "show": true }, { - "format": "ops", + "format": "short", "label": null, "logBase": 1, "max": null, - "min": null, - "show": false + "min": "0", + "show": true } ] }, @@ -7941,16 +10444,18 @@ "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, "fill": 0, - "id": 126, + "id": 133, "legend": { "alignAsTable": true, "avg": false, "current": true, - "max": false, + "max": true, "min": false, "rightSide": true, "show": true, + "sideWidth": 300, "total": false, "values": true }, @@ -7969,31 +10474,38 @@ "steppedLine": false, "targets": [ { - "expr": "avg(tikv_engine_get_micro_seconds{db=\"raft\", type=\"get_average\"})", + "expr": "avg(tikv_engine_bytes_per_read{db=\"$db\",type=\"bytes_per_read_max\"})", "intervalFactor": 2, - "legendFormat": "average", + "legendFormat": "max", "refId": "A", "step": 10 }, { - "expr": "avg(tikv_engine_get_micro_seconds{db=\"raft\", type=\"get_percentile99\"})", + "expr": "avg(tikv_engine_bytes_per_read{db=\"$db\",type=\"bytes_per_read_percentile99\"})", "intervalFactor": 2, - "legendFormat": "percentile99", + "legendFormat": "99%", "refId": "B", "step": 10 }, { - "expr": "avg(tikv_engine_get_micro_seconds{db=\"raft\", type=\"get_percentile95\"})", + "expr": "avg(tikv_engine_bytes_per_read{db=\"$db\",type=\"bytes_per_read_percentile95\"})", "intervalFactor": 2, - "legendFormat": "percentile95", + "legendFormat": "95%", "refId": "C", "step": 10 + }, + { + "expr": "avg(tikv_engine_bytes_per_read{db=\"$db\",type=\"bytes_per_read_average\"})", + "intervalFactor": 2, + "legendFormat": "avg", + "refId": "D", + "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "get", + "title": "Bytes / Read", "tooltip": { "shared": true, "sort": 0, @@ -8008,9 +10520,9 @@ }, "yaxes": [ { - "format": "µs", + "format": "decbytes", "label": null, - "logBase": 1, + "logBase": 10, "max": null, "min": "0", "show": true @@ -8029,23 +10541,25 @@ "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", - "fill": 0, - "id": 127, + "decimals": 1, + "fill": 1, + "height": "", + "id": 86, "legend": { "alignAsTable": true, "avg": false, "current": true, - "max": false, + "max": true, "min": false, "rightSide": true, "show": true, + "sideWidth": 300, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], - "minSpan": 6, "nullPointMode": "null", "percentage": false, "pointradius": 5, @@ -8057,31 +10571,26 @@ "steppedLine": false, "targets": [ { - "expr": "avg(tikv_engine_write_micro_seconds{db=\"raft\", type=\"write_average\"})", - "intervalFactor": 2, - "legendFormat": "average", - "refId": "A", - "step": 10 - }, - { - "expr": "avg(tikv_engine_write_micro_seconds{db=\"raft\", type=\"write_percentile99\"})", + "expr": "sum(rate(tikv_engine_flow_bytes{db=\"$db\", type=\"wal_file_bytes\"}[1m]))", + "hide": false, "intervalFactor": 2, - "legendFormat": "percentile99", - "refId": "B", + "legendFormat": "wal", + "refId": "C", "step": 10 }, { - "expr": "avg(tikv_engine_write_micro_seconds{db=\"raft\", type=\"write_percentile95\"})", + "expr": "sum(rate(tikv_engine_flow_bytes{db=\"$db\", type=\"bytes_written\"}[1m]))", + "hide": false, "intervalFactor": 2, - "legendFormat": "percentile95", - "refId": "C", + "legendFormat": "write", + "refId": "A", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "write", + "title": "Write Flow", "tooltip": { "shared": true, "sort": 0, @@ -8096,7 +10605,7 @@ }, "yaxes": [ { - "format": "µs", + "format": "Bps", "label": null, "logBase": 1, "max": null, @@ -8108,7 +10617,7 @@ "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true } ] @@ -8117,16 +10626,18 @@ "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, "fill": 0, - "id": 128, + "id": 134, "legend": { "alignAsTable": true, "avg": false, "current": true, - "max": false, + "max": true, "min": false, "rightSide": true, "show": true, + "sideWidth": 300, "total": false, "values": true }, @@ -8145,31 +10656,38 @@ "steppedLine": false, "targets": [ { - "expr": "avg(tikv_engine_seek_micro_seconds{db=\"raft\", type=\"seek_average\"})", + "expr": "avg(tikv_engine_bytes_per_write{db=\"$db\",type=\"bytes_per_write_max\"})", "intervalFactor": 2, - "legendFormat": "average", + "legendFormat": "max", "refId": "A", "step": 10 }, { - "expr": "avg(tikv_engine_seek_micro_seconds{db=\"raft\", type=\"seek_percentile99\"})", + "expr": "avg(tikv_engine_bytes_per_write{db=\"$db\",type=\"bytes_per_write_percentile99\"})", "intervalFactor": 2, - "legendFormat": "percentile99", + "legendFormat": "99%", "refId": "B", "step": 10 }, { - "expr": "avg(tikv_engine_seek_micro_seconds{db=\"raft\", type=\"seek_percentile95\"})", + "expr": "avg(tikv_engine_bytes_per_write{db=\"$db\",type=\"bytes_per_write_percentile95\"})", "intervalFactor": 2, - "legendFormat": "percentile95", + "legendFormat": "95%", "refId": "C", "step": 10 + }, + { + "expr": "avg(tikv_engine_bytes_per_write{db=\"$db\",type=\"bytes_per_write_average\"})", + "intervalFactor": 2, + "legendFormat": "avg", + "refId": "D", + "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "seek", + "title": "Bytes / Write", "tooltip": { "shared": true, "sort": 0, @@ -8184,9 +10702,9 @@ }, "yaxes": [ { - "format": "µs", + "format": "decbytes", "label": null, - "logBase": 1, + "logBase": 10, "max": null, "min": "0", "show": true @@ -8206,14 +10724,17 @@ "bars": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, - "fill": 0, - "id": 129, + "fill": 1, + "id": 90, "legend": { + "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, + "rightSide": true, "show": true, + "sideWidth": 300, "total": false, "values": true }, @@ -8225,44 +10746,40 @@ "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [ - { - "alias": "keys_read", - "yaxis": 2 - } - ], + "seriesOverrides": [], "span": 6, "stack": false, "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_engine_flow_bytes{db=\"raft\", type=\"bytes_read\"}[1m]))", + "expr": "sum(rate(tikv_engine_compaction_flow_bytes{db=\"$db\", type=\"bytes_read\"}[1m]))", "hide": false, "intervalFactor": 2, - "legendFormat": "bytes_read", + "legendFormat": "read", "refId": "A", "step": 10 }, { - "expr": "sum(rate(tikv_engine_flow_bytes{db=\"raft\", type=\"keys_read\"}[1m]))", + "expr": "sum(rate(tikv_engine_compaction_flow_bytes{db=\"$db\", type=\"bytes_written\"}[1m]))", "hide": false, "intervalFactor": 2, - "legendFormat": "keys_read", - "refId": "B", + "legendFormat": "written", + "refId": "C", "step": 10 }, { - "expr": "sum(rate(tikv_engine_flow_bytes{db=\"raft\", type=\"iter_bytes_read\"}[1m]))", + "expr": "sum(rate(tikv_engine_flow_bytes{db=\"$db\", type=\"flush_write_bytes\"}[1m]))", + "hide": false, "intervalFactor": 2, - "legendFormat": "iter_bytes_read", - "refId": "C", + "legendFormat": "flushed", + "refId": "B", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "read flow", + "title": "Compaction Flow", "tooltip": { "shared": true, "sort": 0, @@ -8285,7 +10802,7 @@ "show": true }, { - "format": "rps", + "format": "Bps", "label": null, "logBase": 1, "max": null, @@ -8300,13 +10817,16 @@ "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "fill": 1, - "id": 130, + "id": 127, "legend": { + "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, + "rightSide": true, "show": true, + "sideWidth": 300, "total": false, "values": true }, @@ -8318,37 +10838,25 @@ "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [ - { - "alias": "keys_written", - "yaxis": 2 - } - ], + "seriesOverrides": [], "span": 6, "stack": false, "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_engine_flow_bytes{db=\"raft\", type=\"bytes_written\"}[1m])) by (type)", + "expr": "sum(rate(tikv_engine_pending_compaction_bytes{db=\"$db\"}[1m])) by (cf)", "hide": false, "intervalFactor": 2, - "legendFormat": "{{type}}", + "legendFormat": "{{cf}}", + "metric": "tikv_engine_pending_compaction_bytes", "refId": "A", "step": 10 - }, - { - "expr": "sum(rate(tikv_engine_flow_bytes{db=\"raft\", type=\"keys_written\"}[1m])) by (type)", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{type}}", - "refId": "B", - "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "write flow", + "title": "Compaction Pending Bytes", "tooltip": { "shared": true, "sort": 0, @@ -8363,7 +10871,7 @@ }, "yaxes": [ { - "format": "Bps", + "format": "bytes", "label": null, "logBase": 1, "max": null, @@ -8371,7 +10879,7 @@ "show": true }, { - "format": "wps", + "format": "Bps", "label": null, "logBase": 1, "max": null, @@ -8384,16 +10892,20 @@ "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, "fill": 1, - "id": 131, + "id": 518, "legend": { + "alignAsTable": true, "avg": false, - "current": false, - "max": false, + "current": true, + "max": true, "min": false, + "rightSide": true, "show": true, + "sideWidth": 300, "total": false, - "values": false + "values": true }, "lines": true, "linewidth": 1, @@ -8409,9 +10921,11 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_engine_stall_micro_seconds{db=\"raft\"}[30s])) by (job)", + "expr": "sum(rate(tikv_engine_read_amp_flow_bytes{db=\"$db\", type=\"read_amp_total_read_bytes\"}[1m])) by (job) / sum(rate(tikv_engine_read_amp_flow_bytes{db=\"$db\", type=\"read_amp_estimate_useful_bytes\"}[1m])) by (job)", + "hide": false, "intervalFactor": 2, "legendFormat": "{{job}}", + "metric": "", "refId": "A", "step": 10 } @@ -8419,7 +10933,7 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "stall", + "title": "Read Amplication", "tooltip": { "shared": true, "sort": 0, @@ -8434,11 +10948,11 @@ }, "yaxes": [ { - "format": "ms", + "format": "short", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -8446,7 +10960,7 @@ "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true } ] @@ -8457,7 +10971,7 @@ "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "fill": 1, - "id": 132, + "id": 863, "legend": { "alignAsTable": true, "avg": false, @@ -8466,6 +10980,7 @@ "min": false, "rightSide": true, "show": true, + "sideWidth": 300, "total": false, "values": true }, @@ -8477,21 +10992,17 @@ "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [ - { - "alias": "keys_written", - "yaxis": 2 - } - ], + "seriesOverrides": [], "span": 6, "stack": false, "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_engine_bloom_efficiency{db=\"raft\"}[1m])) by (type)", + "expr": "avg(tikv_engine_compression_ratio{db=\"$db\"}) by (level)", "hide": false, "intervalFactor": 2, - "legendFormat": "{{type}}", + "legendFormat": "level - {{level}}", + "metric": "", "refId": "A", "step": 10 } @@ -8499,7 +11010,7 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "pending compaction bytes", + "title": "Compression Ratio", "tooltip": { "shared": true, "sort": 0, @@ -8514,7 +11025,7 @@ }, "yaxes": [ { - "format": "bytes", + "format": "short", "label": null, "logBase": 1, "max": null, @@ -8522,7 +11033,7 @@ "show": true }, { - "format": "wps", + "format": "short", "label": null, "logBase": 1, "max": null, @@ -8537,7 +11048,7 @@ "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "fill": 1, - "id": 133, + "id": 516, "legend": { "alignAsTable": true, "avg": false, @@ -8546,6 +11057,7 @@ "min": false, "rightSide": true, "show": true, + "sideWidth": 300, "total": false, "values": true }, @@ -8557,21 +11069,17 @@ "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [ - { - "alias": "keys_written", - "yaxis": 2 - } - ], + "seriesOverrides": [], "span": 6, "stack": false, "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_engine_compaction_flow_bytes{db=\"raft\"}[1m])) by (job)", + "expr": "tikv_engine_num_snapshots{db=\"$db\"}", "hide": false, "intervalFactor": 2, "legendFormat": "{{job}}", + "metric": "", "refId": "A", "step": 10 } @@ -8579,7 +11087,7 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "compaction flow", + "title": "Number of Snapshots", "tooltip": { "shared": true, "sort": 0, @@ -8594,7 +11102,7 @@ }, "yaxes": [ { - "format": "Bps", + "format": "short", "label": null, "logBase": 1, "max": null, @@ -8602,7 +11110,7 @@ "show": true }, { - "format": "wps", + "format": "short", "label": null, "logBase": 1, "max": null, @@ -8615,8 +11123,9 @@ "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, "fill": 1, - "id": 134, + "id": 517, "legend": { "alignAsTable": true, "avg": false, @@ -8625,6 +11134,7 @@ "min": false, "rightSide": true, "show": true, + "sideWidth": 300, "total": false, "values": true }, @@ -8642,9 +11152,11 @@ "steppedLine": false, "targets": [ { - "expr": "sum(tikv_engine_block_cache_size_bytes{db=\"raft\"}) by(job, cf)", + "expr": "tikv_engine_oldest_snapshot_duration{db=\"$db\"}", + "hide": false, "intervalFactor": 2, - "legendFormat": "{{job}}-{{cf}}", + "legendFormat": "{{job}}", + "metric": "tikv_engine_oldest_snapshot_duration", "refId": "A", "step": 10 } @@ -8652,7 +11164,7 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "cache", + "title": "Oldest Snapshots Duration", "tooltip": { "shared": true, "sort": 0, @@ -8667,11 +11179,11 @@ }, "yaxes": [ { - "format": "bytes", + "format": "s", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -8679,27 +11191,41 @@ "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true } ] - }, + } + ], + "repeat": "db", + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rocksdb - $db", + "titleSize": "h6" + }, + { + "collapse": true, + "height": "300", + "panels": [ { "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, "fill": 1, - "id": 135, + "id": 95, "legend": { "alignAsTable": true, "avg": false, - "current": false, - "max": false, + "current": true, + "max": true, "min": false, "rightSide": true, "show": true, + "sideWidth": 300, "total": false, - "values": false + "values": true }, "lines": true, "linewidth": 1, @@ -8715,9 +11241,10 @@ "steppedLine": false, "targets": [ { - "expr": "sum(tikv_engine_memory_bytes{db=\"raft\", type=\"mem-tables\"}) by (job, cf)", + "expr": "sum(rate(tikv_grpc_msg_duration_seconds_count[1m])) by (type)", "intervalFactor": 2, - "legendFormat": "{{job}}-{{cf}}", + "legendFormat": "{{type}}", + "metric": "tikv_grpc_msg_duration_seconds_bucket", "refId": "A", "step": 10 } @@ -8725,7 +11252,7 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "mem table", + "title": "grpc message count", "tooltip": { "shared": true, "sort": 0, @@ -8740,7 +11267,7 @@ }, "yaxes": [ { - "format": "short", + "format": "ops", "label": null, "logBase": 1, "max": null, @@ -8761,18 +11288,20 @@ "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, "fill": 1, - "id": 136, + "id": 107, "legend": { "alignAsTable": true, "avg": false, - "current": false, - "max": false, + "current": true, + "max": true, "min": false, "rightSide": true, "show": true, + "sideWidth": 300, "total": false, - "values": false + "values": true }, "lines": true, "linewidth": 1, @@ -8788,9 +11317,10 @@ "steppedLine": false, "targets": [ { - "expr": "sum(tikv_engine_memory_bytes{db=\"raft\", type=\"readers-mem\"}) by (job, cf)", + "expr": "sum(rate(tikv_grpc_msg_fail_total[1m])) by (type)", "intervalFactor": 2, - "legendFormat": "{{job}}-{{cf}}", + "legendFormat": "{{type}}", + "metric": "tikv_grpc_msg_fail_total", "refId": "A", "step": 10 } @@ -8798,7 +11328,7 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "table reader mem", + "title": "grpc message failed", "tooltip": { "shared": true, "sort": 0, @@ -8813,7 +11343,7 @@ }, "yaxes": [ { - "format": "short", + "format": "ops", "label": null, "logBase": 1, "max": null, @@ -8834,24 +11364,25 @@ "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", - "fill": 0, - "id": 137, + "decimals": 1, + "fill": 1, + "id": 97, "legend": { "alignAsTable": true, "avg": false, - "current": false, - "max": false, + "current": true, + "max": true, "min": false, "rightSide": true, "show": true, + "sideWidth": 300, "total": false, - "values": false + "values": true }, "lines": true, "linewidth": 1, "links": [], - "minSpan": 6, - "nullPointMode": "connected", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, @@ -8862,10 +11393,9 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_engine_bloom_efficiency{db=\"raft\", type=\"bloom_prefix_useful\"}[1m])) by (job) / sum(rate(tikv_engine_bloom_efficiency{db=\"raft\", type=\"bloom_prefix_checked\"}[1m])) by (job)", + "expr": "histogram_quantile(0.8, sum(rate(tikv_grpc_msg_duration_seconds_bucket[1m])) by (le, type))", "intervalFactor": 2, - "legendFormat": "{{job}}", - "metric": "", + "legendFormat": "{{type}}", "refId": "A", "step": 10 } @@ -8873,13 +11403,12 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "bloom prefix hit rate", + "title": "80% grpc messge duration", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, - "transparent": false, "type": "graph", "xaxis": { "mode": "time", @@ -8889,51 +11418,41 @@ }, "yaxes": [ { - "format": "percentunit", + "format": "s", "label": null, - "logBase": 1, + "logBase": 10, "max": null, - "min": "0", + "min": null, "show": true }, { - "format": "ops", + "format": "short", "label": null, "logBase": 1, "max": null, "min": null, - "show": false + "show": true } ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Raft Rocksdb", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "300", - "panels": [ + }, { "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, "fill": 1, - "id": 95, + "id": 98, "legend": { "alignAsTable": true, "avg": false, - "current": false, - "max": false, + "current": true, + "max": true, "min": false, "rightSide": true, "show": true, + "sideWidth": 300, "total": false, - "values": false + "values": true }, "lines": true, "linewidth": 1, @@ -8949,10 +11468,9 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_grpc_msg_duration_seconds_count[1m])) by (type)", + "expr": "histogram_quantile(0.99, sum(rate(tikv_grpc_msg_duration_seconds_bucket[1m])) by (le, type))", "intervalFactor": 2, "legendFormat": "{{type}}", - "metric": "tikv_grpc_msg_duration_seconds_bucket", "refId": "A", "step": 10 } @@ -8960,7 +11478,7 @@ "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "grpc message count by type", + "title": "99% grpc messge duration", "tooltip": { "shared": true, "sort": 0, @@ -8975,9 +11493,9 @@ }, "yaxes": [ { - "format": "ops", + "format": "s", "label": null, - "logBase": 1, + "logBase": 10, "max": null, "min": null, "show": true @@ -8991,23 +11509,37 @@ "show": true } ] - }, + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Grpc", + "titleSize": "h6" + }, + { + "collapse": true, + "height": "300", + "panels": [ { "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, "fill": 1, - "id": 107, + "id": 1069, "legend": { "alignAsTable": true, "avg": false, - "current": false, - "max": false, + "current": true, + "max": true, "min": false, "rightSide": true, "show": true, + "sideWidth": 350, "total": false, - "values": false + "values": true }, "lines": true, "linewidth": 1, @@ -9023,18 +11555,17 @@ "steppedLine": false, "targets": [ { - "expr": "sum(tikv_grpc_msg_fail_total)by(type)", + "expr": "sum(rate(tikv_pd_request_duration_seconds_count[1m])) by (type)", "intervalFactor": 2, - "legendFormat": "{{type}}", - "metric": "tikv_grpc_msg_fail_total", + "legendFormat": "{{ type }}", "refId": "A", - "step": 10 + "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "grpc message failed", + "title": "PD requests", "tooltip": { "shared": true, "sort": 0, @@ -9070,18 +11601,20 @@ "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, "fill": 1, - "id": 97, + "id": 1070, "legend": { "alignAsTable": true, "avg": false, - "current": false, - "max": false, + "current": true, + "max": true, "min": false, "rightSide": true, "show": true, + "sideWidth": 350, "total": false, - "values": false + "values": true }, "lines": true, "linewidth": 1, @@ -9097,17 +11630,17 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.8, sum(rate(tikv_grpc_msg_duration_seconds_bucket[1m])) by (le, type))", + "expr": "sum(rate(tikv_pd_request_duration_seconds_sum[1m])) by (type) / sum(rate(tikv_pd_request_duration_seconds_count[1m])) by (type)", "intervalFactor": 2, - "legendFormat": "{{type}}", + "legendFormat": "{{ type }}", "refId": "A", - "step": 10 + "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "80% tikv grpc messge duration by type", + "title": "PD request duration (average)", "tooltip": { "shared": true, "sort": 0, @@ -9143,18 +11676,20 @@ "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, "fill": 1, - "id": 98, + "id": 1215, "legend": { "alignAsTable": true, "avg": false, - "current": false, - "max": false, + "current": true, + "max": true, "min": false, "rightSide": true, "show": true, + "sideWidth": 350, "total": false, - "values": false + "values": true }, "lines": true, "linewidth": 1, @@ -9170,17 +11705,17 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(tikv_grpc_msg_duration_seconds_bucket[1m])) by (le, type))", + "expr": "sum(rate(tikv_pd_heartbeat_message_total[1m])) by (type)", "intervalFactor": 2, - "legendFormat": "{{type}}", + "legendFormat": "{{ type }}", "refId": "A", - "step": 10 + "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "99% tikv grpc messge duration by type", + "title": "PD heartbeats", "tooltip": { "shared": true, "sort": 0, @@ -9195,7 +11730,7 @@ }, "yaxes": [ { - "format": "s", + "format": "ops", "label": null, "logBase": 1, "max": null, @@ -9216,18 +11751,20 @@ "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", + "decimals": 1, "fill": 1, - "id": 110, + "id": 1396, "legend": { "alignAsTable": true, "avg": false, - "current": false, - "max": false, + "current": true, + "max": true, "min": false, "rightSide": true, "show": true, + "sideWidth": 350, "total": false, - "values": false + "values": true }, "lines": true, "linewidth": 1, @@ -9243,18 +11780,18 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tikv_grpc_msg_duration_seconds_sum[1m])) by (job)", + "expr": "sum(rate(tikv_pd_validate_peer_total[1m])) by (type)", "intervalFactor": 2, - "legendFormat": "{{job}}", - "metric": "tikv_grpc_msg_duration_seconds_sum", + "legendFormat": "{{ type }}", + "metric": "", "refId": "A", - "step": 10 + "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, - "title": "grpc message count", + "title": "PD validate peers", "tooltip": { "shared": true, "sort": 0, @@ -9269,7 +11806,7 @@ }, "yaxes": [ { - "format": "short", + "format": "ops", "label": null, "logBase": 1, "max": null, @@ -9291,7 +11828,7 @@ "repeatIteration": null, "repeatRowId": null, "showTitle": true, - "title": "Grpc", + "title": "PD", "titleSize": "h6" } ], @@ -9299,7 +11836,48 @@ "style": "dark", "tags": [], "templating": { - "list": [] + "list": [ + { + "allValue": null, + "current": {}, + "datasource": "${DS_TEST-CLUSTER}", + "hide": 0, + "includeAll": true, + "label": "db", + "multi": true, + "name": "db", + "options": [], + "query": "label_values(tikv_engine_block_cache_size_bytes, db)", + "refresh": 1, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": {}, + "datasource": "${DS_TEST-CLUSTER}", + "hide": 0, + "includeAll": true, + "label": "command", + "multi": true, + "name": "command", + "options": [], + "query": "label_values(tikv_storage_command_total, type)", + "refresh": 1, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] }, "time": { "from": "now-1h", @@ -9331,6 +11909,6 @@ ] }, "timezone": "browser", - "title": "TiDB Cluster - tikv", - "version": 3 + "title": "Test-Cluster-TiKV", + "version": 2 }