diff --git a/monitor-snapshot/master/ansible-monitor.tar.gz b/monitor-snapshot/master/ansible-monitor.tar.gz index 151feafb..a4f844f6 100644 Binary files a/monitor-snapshot/master/ansible-monitor.tar.gz and b/monitor-snapshot/master/ansible-monitor.tar.gz differ diff --git a/monitor-snapshot/master/operator/dashboards/ticdc_new_arch.json b/monitor-snapshot/master/operator/dashboards/ticdc_new_arch.json index 4cbc82bb..b8bea1ea 100644 --- a/monitor-snapshot/master/operator/dashboards/ticdc_new_arch.json +++ b/monitor-snapshot/master/operator/dashboards/ticdc_new_arch.json @@ -6784,19 +6784,23 @@ }, { "aliasColors": {}, + "bars": false, "dashLength": 10, + "dashes": false, "datasource": "tidb-cluster", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, + "fillGradient": 0, "gridPos": { "h": 6, "w": 12, "x": 12, "y": 37 }, + "hiddenSeries": false, "id": 10071, "legend": { "alignAsTable": true, @@ -6814,41 +6818,47 @@ "options": { "alertThreshold": true }, + "percentage": false, "pluginVersion": "7.5.17", "pointradius": 2, + "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, + "stack": false, + "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.999, sum(rate(ticdc_maintainer_handle_event_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (le,namespace,changefeed))", - "legendFormat": "99.9-{{namespace}}-{{changefeed}}", - "interval": "", "exemplar": true, + "expr": "histogram_quantile(0.999, sum(rate(ticdc_maintainer_handle_event_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (le,namespace,changefeed))", "hide": false, + "interval": "", + "legendFormat": "99.9-{{namespace}}-{{changefeed}}", "queryType": "randomWalk", "refId": "A" }, { + "exemplar": true, "expr": "sum(rate(ticdc_maintainer_handle_event_duration_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed) / \nsum(rate(ticdc_maintainer_handle_event_duration_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed)", - "legendFormat": "avg-{{namespace}}-{{changefeed}}", + "hide": false, "interval": "", - "exemplar": true, - "refId": "B", - "hide": false + "legendFormat": "avg-{{namespace}}-{{changefeed}}", + "refId": "B" } ], "thresholds": [ { + "$$hashKey": "object:411", "colorMode": "critical", - "op": "gt", "fill": true, "line": true, - "yaxis": "left", - "$$hashKey": "object:411" + "op": "gt", + "yaxis": "left" } ], + "timeFrom": null, "timeRegions": [], + "timeShift": null, "title": "Maintainer Handle Event Duration", "tooltip": { "shared": true, @@ -6865,39 +6875,29 @@ }, "yaxes": [ { + "$$hashKey": "object:283", "decimals": 0, "format": "s", "label": null, "logBase": 2, "max": null, "min": "0", - "show": true, - "$$hashKey": "object:283" + "show": true }, { + "$$hashKey": "object:284", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, - "show": false, - "$$hashKey": "object:284" + "show": false } ], "yaxis": { "align": false, "alignLevel": null - }, - "bars": false, - "dashes": false, - "fillGradient": 0, - "hiddenSeries": false, - "percentage": false, - "points": false, - "stack": false, - "steppedLine": false, - "timeFrom": null, - "timeShift": null + } } ], "title": "Maintainer", @@ -6933,7 +6933,7 @@ "h": 8, "w": 12, "x": 0, - "y": 8 + "y": 9 }, "hiddenSeries": false, "id": 12050, @@ -7042,7 +7042,7 @@ "h": 8, "w": 12, "x": 12, - "y": 8 + "y": 9 }, "hiddenSeries": false, "id": 22296, @@ -7132,7 +7132,7 @@ "h": 8, "w": 12, "x": 0, - "y": 16 + "y": 17 }, "hiddenSeries": false, "id": 22261, @@ -7241,7 +7241,7 @@ "h": 8, "w": 12, "x": 12, - "y": 16 + "y": 17 }, "hiddenSeries": false, "id": 22299, @@ -7331,7 +7331,7 @@ "h": 8, "w": 12, "x": 0, - "y": 24 + "y": 25 }, "hiddenSeries": false, "id": 22291, @@ -7428,7 +7428,7 @@ "h": 8, "w": 12, "x": 12, - "y": 24 + "y": 25 }, "heatmap": {}, "hideZeroBuckets": true, @@ -7501,79 +7501,6 @@ }, "id": 12054, "panels": [ - { - "cards": { - "cardPadding": null, - "cardRound": null - }, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateSpectral", - "exponent": 0.5, - "min": 0, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": "tidb-cluster", - "description": "", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 9 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 12052, - "legend": { - "show": true - }, - "pluginVersion": "7.5.17", - "reverseYBuckets": false, - "targets": [ - { - "exemplar": true, - "expr": "sum(increase(ticdc_event_store_dispatcher_resolved_ts_lag_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le)", - "format": "heatmap", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{le}}", - "refId": "A" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Dispatcher Resolved Ts Lag", - "tooltip": { - "show": true, - "showHistogram": true - }, - "type": "heatmap", - "xAxis": { - "show": true - }, - "xBucketNumber": null, - "xBucketSize": null, - "yAxis": { - "decimals": 1, - "format": "s", - "logBase": 1, - "max": null, - "min": null, - "show": true, - "splitFactor": null - }, - "yBucketBound": "upper", - "yBucketNumber": null, - "yBucketSize": null - }, { "aliasColors": {}, "bars": false, @@ -7589,8 +7516,8 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 9 + "x": 0, + "y": 41 }, "hiddenSeries": false, "id": 20022, @@ -7677,9 +7604,11 @@ "dashLength": 10, "dashes": false, "datasource": "tidb-cluster", - "description": "The number of events received by event store.", + "description": "The lag of startTs when registering a dispatcher.", "fieldConfig": { - "defaults": {}, + "defaults": { + "links": [] + }, "overrides": [] }, "fill": 1, @@ -7687,11 +7616,11 @@ "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 17 + "x": 12, + "y": 41 }, "hiddenSeries": false, - "id": 12044, + "id": 22304, "legend": { "alignAsTable": true, "avg": false, @@ -7705,10 +7634,12 @@ }, "lines": true, "linewidth": 1, + "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, + "paceLength": 10, "percentage": false, "pluginVersion": "7.5.17", "pointradius": 2, @@ -7721,18 +7652,32 @@ "targets": [ { "exemplar": true, - "expr": "sum(rate(ticdc_event_store_input_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (instance, type)", - "format": "time_series", + "expr": "histogram_quantile(1.0, sum(rate(ticdc_event_store_register_dispatcher_start_ts_lag_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le, instance))", "interval": "", - "legendFormat": "{{instance}}-{{type}}", + "legendFormat": "{{instance}}-max", "refId": "A" + }, + { + "exemplar": true, + "expr": "histogram_quantile(0.95, sum(rate(ticdc_event_store_register_dispatcher_start_ts_lag_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le, instance))", + "hide": false, + "legendFormat": "{{instance}}-p95", + "refId": "B" + }, + { + "exemplar": true, + "expr": "histogram_quantile(0.80, sum(rate(ticdc_event_store_register_dispatcher_start_ts_lag_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le, instance))", + "hide": true, + "interval": "", + "legendFormat": "{{instance}}-p80", + "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Event Store Input Event Count / s", + "title": "Register Dispatcher StartTs Lag", "tooltip": { "shared": true, "sort": 0, @@ -7748,25 +7693,19 @@ }, "yaxes": [ { - "format": "short", - "label": null, + "format": "s", "logBase": 1, - "max": null, "min": "0", "show": true }, { "format": "short", - "label": null, "logBase": 1, - "max": null, - "min": null, "show": false } ], "yaxis": { - "align": false, - "alignLevel": null + "align": false } }, { @@ -7775,6 +7714,7 @@ "dashLength": 10, "dashes": false, "datasource": "tidb-cluster", + "description": "The Resolved Ts lag of subscriptions for event store.", "fieldConfig": { "defaults": {}, "overrides": [] @@ -7784,11 +7724,11 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 17 + "x": 0, + "y": 49 }, "hiddenSeries": false, - "id": 12056, + "id": 22452, "legend": { "alignAsTable": true, "avg": false, @@ -7818,26 +7758,35 @@ "targets": [ { "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(ticdc_event_store_dispatcher_watermark_lag_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$ticdc_instance\"}[1m])) by (le, instance))", + "expr": "histogram_quantile(1, sum(rate(ticdc_event_store_subscription_resolved_ts_lag_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$ticdc_instance\"}[1m])) by (le, instance))", + "hide": false, "interval": "", - "legendFormat": "{{instance}}-99", + "legendFormat": "{{instance}}-max", "queryType": "randomWalk", "refId": "A" }, { "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(ticdc_event_store_dispatcher_watermark_lag_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$ticdc_instance\"}[1m])) by (le, instance))", + "expr": "histogram_quantile(0.95, sum(rate(ticdc_event_store_subscription_resolved_ts_lag_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$ticdc_instance\"}[1m])) by (le, instance))", "hide": false, "interval": "", - "legendFormat": "{{instance}}-80", + "legendFormat": "{{instance}}-p95", "refId": "B" + }, + { + "exemplar": true, + "expr": "histogram_quantile(0.8, sum(rate(ticdc_event_store_subscription_resolved_ts_lag_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$ticdc_instance\"}[1m])) by (le, instance))", + "hide": true, + "interval": "", + "legendFormat": "{{instance}}-p80", + "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Dispatcher Watermark Lag", + "title": "Subscription Resolved Ts Lag", "tooltip": { "shared": true, "sort": 0, @@ -7880,7 +7829,7 @@ "dashLength": 10, "dashes": false, "datasource": "tidb-cluster", - "description": "The amount of pending data stored on-disk for event store", + "description": "The data gc lag of subscriptions for event store.", "fieldConfig": { "defaults": {}, "overrides": [] @@ -7890,19 +7839,21 @@ "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 25 + "x": 12, + "y": 49 }, "hiddenSeries": false, - "id": 22234, + "id": 12056, "legend": { + "alignAsTable": true, "avg": false, - "current": false, - "max": false, + "current": true, + "max": true, "min": false, + "rightSide": false, "show": true, "total": false, - "values": false + "values": true }, "lines": true, "linewidth": 1, @@ -7922,18 +7873,35 @@ "targets": [ { "exemplar": true, - "expr": "sum(ticdc_event_store_on_disk_data_size{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}) by (instance)", + "expr": "histogram_quantile(1, sum(rate(ticdc_event_store_subscription_data_gc_lag_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$ticdc_instance\"}[1m])) by (le, instance))", + "hide": false, "interval": "", - "legendFormat": "{{instance}}", + "legendFormat": "{{instance}}-max", "queryType": "randomWalk", "refId": "A" + }, + { + "exemplar": true, + "expr": "histogram_quantile(0.95, sum(rate(ticdc_event_store_subscription_data_gc_lag_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$ticdc_instance\"}[1m])) by (le, instance))", + "hide": false, + "interval": "", + "legendFormat": "{{instance}}-p95", + "refId": "B" + }, + { + "exemplar": true, + "expr": "histogram_quantile(0.8, sum(rate(ticdc_event_store_subscription_data_gc_lag_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$ticdc_instance\"}[1m])) by (le, instance))", + "hide": true, + "interval": "", + "legendFormat": "{{instance}}-p80", + "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Data Size On Disk", + "title": "Subscription Data GC Lag", "tooltip": { "shared": true, "sort": 0, @@ -7949,20 +7917,20 @@ }, "yaxes": [ { - "format": "bytes", + "format": "s", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { - "format": "bytes", + "format": "short", "label": null, "logBase": 1, "max": null, "min": null, - "show": true + "show": false } ], "yaxis": { @@ -7976,7 +7944,7 @@ "dashLength": 10, "dashes": false, "datasource": "tidb-cluster", - "description": "The amount of pending data stored in-memory for event store", + "description": "The number of events received by event store.", "fieldConfig": { "defaults": {}, "overrides": [] @@ -7986,19 +7954,21 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 25 + "x": 0, + "y": 57 }, "hiddenSeries": false, - "id": 22235, + "id": 12044, "legend": { + "alignAsTable": true, "avg": false, - "current": false, - "max": false, + "current": true, + "max": true, "min": false, + "rightSide": false, "show": true, "total": false, - "values": false + "values": true }, "lines": true, "linewidth": 1, @@ -8018,10 +7988,10 @@ "targets": [ { "exemplar": true, - "expr": "sum(ticdc_event_store_in_memory_data_size{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}) by (instance)", + "expr": "sum(rate(ticdc_event_store_input_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (instance, type)", + "format": "time_series", "interval": "", - "legendFormat": "{{instance}}", - "queryType": "randomWalk", + "legendFormat": "{{instance}}-{{type}}", "refId": "A" } ], @@ -8029,7 +7999,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Data Size In Memory", + "title": "Input Event Count / s", "tooltip": { "shared": true, "sort": 0, @@ -8045,20 +8015,20 @@ }, "yaxes": [ { - "format": "bytes", + "format": "short", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { - "format": "bytes", + "format": "short", "label": null, "logBase": 1, "max": null, "min": null, - "show": true + "show": false } ], "yaxis": { @@ -8072,7 +8042,7 @@ "dashLength": 10, "dashes": false, "datasource": "tidb-cluster", - "description": "The number of subscriptions created by event store.", + "description": "The number of bytes written by event store.", "fieldConfig": { "defaults": {}, "overrides": [] @@ -8082,11 +8052,11 @@ "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 33 + "x": 12, + "y": 57 }, "hiddenSeries": false, - "id": 20024, + "id": 20002, "legend": { "alignAsTable": true, "avg": false, @@ -8116,9 +8086,8 @@ "targets": [ { "exemplar": true, - "expr": "ticdc_event_store_subscription_num{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}", + "expr": "sum(rate(ticdc_event_store_write_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (instance)", "interval": "", - "intervalFactor": 1, "legendFormat": "{{instance}}", "queryType": "randomWalk", "refId": "A" @@ -8128,7 +8097,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Subscription Num", + "title": "Input Bytes / s", "tooltip": { "shared": true, "sort": 0, @@ -8144,8 +8113,7 @@ }, "yaxes": [ { - "decimals": 0, - "format": "short", + "format": "binBps", "label": null, "logBase": 1, "max": null, @@ -8172,7 +8140,7 @@ "dashLength": 10, "dashes": false, "datasource": "tidb-cluster", - "description": "The number of scan requests received by event store", + "description": "The number of write requests received by event store", "fieldConfig": { "defaults": {}, "overrides": [] @@ -8182,11 +8150,11 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 33 + "x": 0, + "y": 65 }, "hiddenSeries": false, - "id": 20004, + "id": 20034, "legend": { "alignAsTable": true, "avg": false, @@ -8216,8 +8184,7 @@ "targets": [ { "exemplar": true, - "expr": "sum(rate(ticdc_event_store_scan_requests_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (instance)", - "instant": false, + "expr": "sum(rate(ticdc_event_store_write_requests_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (instance)", "interval": "", "legendFormat": "{{instance}}", "queryType": "randomWalk", @@ -8228,7 +8195,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Scan Requests / s", + "title": "Write Requests / s", "tooltip": { "shared": true, "sort": 0, @@ -8244,7 +8211,7 @@ }, "yaxes": [ { - "format": "ops", + "format": "short", "label": null, "logBase": 1, "max": null, @@ -8271,7 +8238,7 @@ "dashLength": 10, "dashes": false, "datasource": "tidb-cluster", - "description": "", + "description": "The time of sorter write", "fieldConfig": { "defaults": {}, "overrides": [] @@ -8281,11 +8248,11 @@ "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 41 + "x": 12, + "y": 65 }, "hiddenSeries": false, - "id": 22260, + "id": 22225, "legend": { "alignAsTable": true, "avg": false, @@ -8299,6 +8266,7 @@ }, "lines": true, "linewidth": 1, + "links": [], "nullPointMode": "null", "options": { "alertThreshold": true @@ -8308,6 +8276,7 @@ "pointradius": 2, "points": false, "renderer": "flot", + "repeatDirection": "h", "seriesOverrides": [], "spaceLength": 10, "stack": false, @@ -8315,19 +8284,36 @@ "targets": [ { "exemplar": true, - "expr": "sum(rate(ticdc_event_service_channel_size{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\", instance=~\"$ticdc_instance\"}[1m])) by (instance, type)", + "expr": "histogram_quantile(1, sum(rate(ticdc_event_store_write_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le, instance))", + "format": "time_series", + "instant": false, "interval": "", "intervalFactor": 1, - "legendFormat": "{{instance}}-{{type}}", - "queryType": "randomWalk", + "legendFormat": "{{instance}}-max", "refId": "A" + }, + { + "exemplar": true, + "expr": "histogram_quantile(0.99, sum(rate(ticdc_event_store_write_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le, instance))", + "hide": true, + "interval": "", + "legendFormat": "{{instance}}-p99", + "refId": "B" + }, + { + "exemplar": true, + "expr": "sum(rate(ticdc_event_store_write_duration_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (instance) /\nsum(rate(ticdc_event_store_write_duration_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (instance)", + "hide": false, + "interval": "", + "legendFormat": "{{instance}}-avg", + "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Event service channel size ", + "title": "Write duration", "tooltip": { "shared": true, "sort": 0, @@ -8343,12 +8329,11 @@ }, "yaxes": [ { - "decimals": 0, - "format": "short", + "format": "s", "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { @@ -8371,7 +8356,7 @@ "dashLength": 10, "dashes": false, "datasource": "tidb-cluster", - "description": "The number of bytes scanned by event store.", + "description": "The number of rows compressed by event store per second.", "fieldConfig": { "defaults": {}, "overrides": [] @@ -8381,11 +8366,11 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 41 + "x": 0, + "y": 73 }, "hiddenSeries": false, - "id": 20006, + "id": 20035, "legend": { "alignAsTable": true, "avg": false, @@ -8415,7 +8400,7 @@ "targets": [ { "exemplar": true, - "expr": "sum(rate(ticdc_event_store_scan_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (instance)", + "expr": "sum(rate(ticdc_event_store_compressed_rows_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (instance)", "interval": "", "legendFormat": "{{instance}}", "queryType": "randomWalk", @@ -8426,7 +8411,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Scan Bytes / s", + "title": "Compressed Rows / s", "tooltip": { "shared": true, "sort": 0, @@ -8442,7 +8427,7 @@ }, "yaxes": [ { - "format": "binBps", + "format": "short", "label": null, "logBase": 1, "max": null, @@ -8469,7 +8454,7 @@ "dashLength": 10, "dashes": false, "datasource": "tidb-cluster", - "description": "The number of bytes written by event store.", + "description": "The number of subscriptions created by event store.", "fieldConfig": { "defaults": {}, "overrides": [] @@ -8479,11 +8464,11 @@ "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 49 + "x": 12, + "y": 73 }, "hiddenSeries": false, - "id": 20002, + "id": 20024, "legend": { "alignAsTable": true, "avg": false, @@ -8513,8 +8498,9 @@ "targets": [ { "exemplar": true, - "expr": "sum(rate(ticdc_event_store_write_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (instance)", + "expr": "ticdc_event_store_subscription_num{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}", "interval": "", + "intervalFactor": 1, "legendFormat": "{{instance}}", "queryType": "randomWalk", "refId": "A" @@ -8524,7 +8510,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Write Bytes / s", + "title": "Subscription Num", "tooltip": { "shared": true, "sort": 0, @@ -8540,7 +8526,8 @@ }, "yaxes": [ { - "format": "binBps", + "decimals": 0, + "format": "short", "label": null, "logBase": 1, "max": null, @@ -8561,13 +8548,154 @@ "alignLevel": null } }, + { + "cards": { + "cardPadding": null, + "cardRound": null + }, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateSpectral", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": "tidb-cluster", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 81 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 20032, + "legend": { + "show": true + }, + "pluginVersion": "7.5.17", + "reverseYBuckets": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(increase(ticdc_event_store_write_batch_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le)", + "format": "heatmap", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{le}}", + "queryType": "randomWalk", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Write Batch Size", + "tooltip": { + "show": true, + "showHistogram": true + }, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": 0, + "format": "bytes", + "logBase": 1, + "max": null, + "min": null, + "show": true, + "splitFactor": null + }, + "yBucketBound": "upper", + "yBucketNumber": null, + "yBucketSize": null + }, + { + "cards": { + "cardPadding": null, + "cardRound": null + }, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateSpectral", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": "tidb-cluster", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 81 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 20030, + "legend": { + "show": true + }, + "pluginVersion": "7.5.17", + "reverseYBuckets": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(increase(ticdc_event_store_write_batch_events_count_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le)", + "format": "heatmap", + "interval": "", + "legendFormat": "{{le}}", + "queryType": "randomWalk", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Write Batch Event Count", + "tooltip": { + "show": true, + "showHistogram": true + }, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": 0, + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true, + "splitFactor": null + }, + "yBucketBound": "upper", + "yBucketNumber": null, + "yBucketSize": null + }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "tidb-cluster", - "description": "The number of entries scanned by event store.", + "description": "The amount of pending data stored on-disk for event store", "fieldConfig": { "defaults": {}, "overrides": [] @@ -8577,21 +8705,20 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 49 + "x": 0, + "y": 89 }, "hiddenSeries": false, - "id": 22259, + "id": 22234, "legend": { "alignAsTable": true, "avg": false, - "current": true, - "max": true, + "current": false, + "max": false, "min": false, - "rightSide": false, "show": true, "total": false, - "values": true + "values": false }, "lines": true, "linewidth": 1, @@ -8611,7 +8738,7 @@ "targets": [ { "exemplar": true, - "expr": "sum(rate(ticdc_event_service_scanned_count_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (instance)", + "expr": "sum(ticdc_event_store_on_disk_data_size{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}) by (instance)", "interval": "", "legendFormat": "{{instance}}", "queryType": "randomWalk", @@ -8622,7 +8749,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Scanned entry count / s", + "title": "Data Size On Disk", "tooltip": { "shared": true, "sort": 0, @@ -8638,20 +8765,22 @@ }, "yaxes": [ { - "format": "short", + "$$hashKey": "object:692", + "format": "bytes", "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true }, { - "format": "short", + "$$hashKey": "object:693", + "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, - "show": false + "show": true } ], "yaxis": { @@ -8665,7 +8794,7 @@ "dashLength": 10, "dashes": false, "datasource": "tidb-cluster", - "description": "The time of sorter write", + "description": "The amount of pending data stored in-memory for event store", "fieldConfig": { "defaults": {}, "overrides": [] @@ -8675,25 +8804,23 @@ "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 57 + "x": 12, + "y": 89 }, "hiddenSeries": false, - "id": 22225, + "id": 22235, "legend": { "alignAsTable": true, "avg": false, - "current": true, - "max": true, + "current": false, + "max": false, "min": false, - "rightSide": false, "show": true, "total": false, - "values": true + "values": false }, "lines": true, "linewidth": 1, - "links": [], "nullPointMode": "null", "options": { "alertThreshold": true @@ -8703,7 +8830,6 @@ "pointradius": 2, "points": false, "renderer": "flot", - "repeatDirection": "h", "seriesOverrides": [], "spaceLength": 10, "stack": false, @@ -8711,28 +8837,18 @@ "targets": [ { "exemplar": true, - "expr": "histogram_quantile(0.999, sum(rate(ticdc_event_store_write_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le, instance))", - "format": "heatmap", - "instant": false, + "expr": "sum(ticdc_event_store_in_memory_data_size{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}) by (instance)", "interval": "", - "intervalFactor": 2, - "legendFormat": "{{instance}}-p999", + "legendFormat": "{{instance}}", + "queryType": "randomWalk", "refId": "A" - }, - { - "exemplar": true, - "expr": "sum(rate(ticdc_event_store_write_duration_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (instance) /\nsum(rate(ticdc_event_store_write_duration_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (instance)", - "hide": false, - "interval": "", - "legendFormat": "{{instance}}-avg", - "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Write duration", + "title": "Data Size In Memory", "tooltip": { "shared": true, "sort": 0, @@ -8748,7 +8864,8 @@ }, "yaxes": [ { - "format": "s", + "$$hashKey": "object:783", + "format": "bytes", "label": null, "logBase": 1, "max": null, @@ -8756,12 +8873,13 @@ "show": true }, { - "format": "short", + "$$hashKey": "object:784", + "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, - "show": false + "show": true } ], "yaxis": { @@ -8770,158 +8888,103 @@ } }, { - "cards": { - "cardPadding": null, - "cardRound": null - }, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateSpectral", - "exponent": 0.5, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": "tidb-cluster", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 57 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 20030, - "legend": { - "show": true - }, - "pluginVersion": "7.5.17", - "reverseYBuckets": false, - "targets": [ - { - "exemplar": true, - "expr": "sum(increase(ticdc_event_store_write_batch_events_count_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le)", - "format": "heatmap", - "interval": "", - "legendFormat": "{{le}}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Batch Event Count", - "tooltip": { - "show": true, - "showHistogram": true - }, - "type": "heatmap", - "xAxis": { - "show": true - }, - "xBucketNumber": null, - "xBucketSize": null, - "yAxis": { - "decimals": 0, - "format": "short", - "logBase": 1, - "max": null, - "min": null, - "show": true, - "splitFactor": null - }, - "yBucketBound": "upper", - "yBucketNumber": null, - "yBucketSize": null - }, - { - "cards": { - "cardPadding": 0, - "cardRound": 0 - }, - "color": { - "cardColor": "#FF9830", - "colorScale": "linear", - "colorScheme": "interpolateSpectral", - "exponent": 0.5, - "min": 0, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, "datasource": "tidb-cluster", - "description": "The time of sorter write", + "description": "The number of scan requests received by event store", "fieldConfig": { "defaults": {}, "overrides": [] }, + "fill": 1, + "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 65 + "y": 97 }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 274, + "hiddenSeries": false, + "id": 20004, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, - "rightSide": true, + "rightSide": false, "show": true, - "sort": "current", - "sortDesc": true, "total": false, "values": true }, - "links": [], - "maxPerRow": 3, - "repeatDirection": "h", - "reverseYBuckets": false, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, "targets": [ { "exemplar": true, - "expr": "sum(increase(ticdc_event_store_write_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le)", - "format": "heatmap", + "expr": "sum(rate(ticdc_event_store_scan_requests_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (instance)", "instant": false, "interval": "", - "intervalFactor": 2, - "legendFormat": "{{le}}", + "legendFormat": "{{instance}}", + "queryType": "randomWalk", "refId": "A" } ], - "title": "Write duration", + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Scan Requests / s", "tooltip": { - "show": true, - "showHistogram": true - }, - "tooltipDecimals": 1, - "type": "heatmap", - "xAxis": { - "show": true + "shared": true, + "sort": 0, + "value_type": "individual" }, - "xBucketNumber": null, - "xBucketSize": null, - "yAxis": { - "decimals": 1, - "format": "s", - "logBase": 1, - "max": null, - "min": null, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, "show": true, - "splitFactor": null + "values": [] }, - "yBucketBound": "lower", - "yBucketNumber": null, - "yBucketSize": null + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { "aliasColors": {}, @@ -8929,7 +8992,7 @@ "dashLength": 10, "dashes": false, "datasource": "tidb-cluster", - "description": "The number of write requests received by event store", + "description": "The number of bytes scanned by event store.", "fieldConfig": { "defaults": {}, "overrides": [] @@ -8940,10 +9003,10 @@ "h": 8, "w": 12, "x": 12, - "y": 65 + "y": 97 }, "hiddenSeries": false, - "id": 20034, + "id": 20006, "legend": { "alignAsTable": true, "avg": false, @@ -8973,7 +9036,7 @@ "targets": [ { "exemplar": true, - "expr": "sum(rate(ticdc_event_store_write_requests_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (instance)", + "expr": "sum(rate(ticdc_event_store_scan_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (instance)", "interval": "", "legendFormat": "{{instance}}", "queryType": "randomWalk", @@ -8984,7 +9047,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Write Requests / s", + "title": "Scan Bytes / s", "tooltip": { "shared": true, "sort": 0, @@ -9000,7 +9063,7 @@ }, "yaxes": [ { - "format": "short", + "format": "binBps", "label": null, "logBase": 1, "max": null, @@ -9023,18 +9086,20 @@ }, { "cards": { - "cardPadding": null, - "cardRound": null + "cardPadding": 0, + "cardRound": 0 }, "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", + "cardColor": "#FF9830", + "colorScale": "linear", "colorScheme": "interpolateSpectral", "exponent": 0.5, + "min": 0, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "tidb-cluster", + "description": "", "fieldConfig": { "defaults": {}, "overrides": [] @@ -9043,85 +9108,12 @@ "h": 8, "w": 12, "x": 0, - "y": 73 + "y": 105 }, "heatmap": {}, "hideZeroBuckets": true, "highlightCards": true, - "id": 20032, - "legend": { - "show": true - }, - "pluginVersion": "7.5.17", - "reverseYBuckets": false, - "targets": [ - { - "exemplar": true, - "expr": "sum(increase(ticdc_event_store_write_batch_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le)", - "format": "heatmap", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{le}}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Write Batch Size", - "tooltip": { - "show": true, - "showHistogram": true - }, - "type": "heatmap", - "xAxis": { - "show": true - }, - "xBucketNumber": null, - "xBucketSize": null, - "yAxis": { - "decimals": 0, - "format": "bytes", - "logBase": 1, - "max": null, - "min": null, - "show": true, - "splitFactor": null - }, - "yBucketBound": "upper", - "yBucketNumber": null, - "yBucketSize": null - }, - { - "cards": { - "cardPadding": 0, - "cardRound": 0 - }, - "color": { - "cardColor": "#FF9830", - "colorScale": "linear", - "colorScheme": "interpolateSpectral", - "exponent": 0.5, - "min": 0, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": "tidb-cluster", - "description": "The time of sorter write", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 73 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 20192, + "id": 20191, "legend": { "alignAsTable": true, "avg": false, @@ -9137,12 +9129,13 @@ }, "links": [], "maxPerRow": 3, + "pluginVersion": "7.5.17", "repeatDirection": "h", "reverseYBuckets": false, "targets": [ { "exemplar": true, - "expr": "sum(rate(ticdc_event_store_read_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\", type=\"close\"}[1m])) by (le)", + "expr": "sum(rate(ticdc_event_store_read_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\", type=\"next\"}[1m])) by (le)", "format": "heatmap", "instant": false, "interval": "", @@ -9151,7 +9144,9 @@ "refId": "A" } ], - "title": "Read duration - Close", + "timeFrom": null, + "timeShift": null, + "title": "Read duration - Next", "tooltip": { "show": true, "showHistogram": true @@ -9191,7 +9186,7 @@ }, "dataFormat": "tsbuckets", "datasource": "tidb-cluster", - "description": "The time of event store iterator first read duration", + "description": "The time of sorter write", "fieldConfig": { "defaults": {}, "overrides": [] @@ -9199,13 +9194,13 @@ "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 81 + "x": 12, + "y": 105 }, "heatmap": {}, "hideZeroBuckets": true, "highlightCards": true, - "id": 20190, + "id": 20192, "legend": { "alignAsTable": true, "avg": false, @@ -9226,7 +9221,7 @@ "targets": [ { "exemplar": true, - "expr": "sum(rate(ticdc_event_store_read_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\", type=\"first\"}[1m])) by (le)", + "expr": "sum(rate(ticdc_event_store_read_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\", type=\"close\"}[1m])) by (le)", "format": "heatmap", "instant": false, "interval": "", @@ -9235,7 +9230,7 @@ "refId": "A" } ], - "title": "Read duration - First ", + "title": "Read duration - Close", "tooltip": { "show": true, "showHistogram": true @@ -9275,7 +9270,7 @@ }, "dataFormat": "tsbuckets", "datasource": "tidb-cluster", - "description": "", + "description": "The time of event store iterator read first duration", "fieldConfig": { "defaults": {}, "overrides": [] @@ -9283,13 +9278,13 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 81 + "x": 0, + "y": 113 }, "heatmap": {}, "hideZeroBuckets": true, "highlightCards": true, - "id": 20191, + "id": 20190, "legend": { "alignAsTable": true, "avg": false, @@ -9305,13 +9300,12 @@ }, "links": [], "maxPerRow": 3, - "pluginVersion": "7.5.17", "repeatDirection": "h", "reverseYBuckets": false, "targets": [ { "exemplar": true, - "expr": "sum(rate(ticdc_event_store_read_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\", type=\"next\"}[1m])) by (le)", + "expr": "sum(rate(ticdc_event_store_read_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\", type=\"first\"}[1m])) by (le)", "format": "heatmap", "instant": false, "interval": "", @@ -9320,9 +9314,7 @@ "refId": "A" } ], - "timeFrom": null, - "timeShift": null, - "title": "Read duration - Next", + "title": "Read duration - First", "tooltip": { "show": true, "showHistogram": true @@ -9346,16 +9338,31 @@ "yBucketBound": "lower", "yBucketNumber": null, "yBucketSize": null - }, + } + ], + "title": "Event Store", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 12058, + "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "tidb-cluster", - "description": "The number of rows compressed by event store per second.", "fieldConfig": { - "defaults": {} + "defaults": {}, + "overrides": [] }, "fill": 1, "fillGradient": 0, @@ -9363,10 +9370,10 @@ "h": 8, "w": 12, "x": 0, - "y": 89 + "y": 10 }, "hiddenSeries": false, - "id": 20035, + "id": 12060, "legend": { "alignAsTable": true, "avg": false, @@ -9396,9 +9403,10 @@ "targets": [ { "exemplar": true, - "expr": "sum(rate(ticdc_event_store_compressed_rows_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (instance)", + "expr": "ticdc_schema_store_resolved_ts_lag{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}", "interval": "", - "legendFormat": "{{instance}}", + "intervalFactor": 1, + "legendFormat": "{{instance}}-resolvedts", "queryType": "randomWalk", "refId": "A" } @@ -9407,7 +9415,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Compressed Rows / s", + "title": "Resolved Ts Lag", "tooltip": { "shared": true, "sort": 0, @@ -9423,7 +9431,7 @@ }, "yaxes": [ { - "format": "short", + "format": "s", "label": null, "logBase": 1, "max": null, @@ -9450,11 +9458,8 @@ "dashLength": 10, "dashes": false, "datasource": "tidb-cluster", - "description": "The lag of startTs when registering a dispatcher.", "fieldConfig": { - "defaults": { - "links": [] - }, + "defaults": {}, "overrides": [] }, "fill": 1, @@ -9463,10 +9468,10 @@ "h": 8, "w": 12, "x": 12, - "y": 89 + "y": 10 }, "hiddenSeries": false, - "id": 22304, + "id": 12062, "legend": { "alignAsTable": true, "avg": false, @@ -9480,12 +9485,10 @@ }, "lines": true, "linewidth": 1, - "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, - "paceLength": 10, "percentage": false, "pluginVersion": "7.5.17", "pointradius": 2, @@ -9498,28 +9501,18 @@ "targets": [ { "exemplar": true, - "expr": "histogram_quantile(0.80, sum(rate(ticdc_event_store_register_dispatcher_start_ts_lag_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le, instance))", - "legendFormat": "{{instance}}-p80", + "expr": "ticdc_schema_store_register_table_num{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}", + "interval": "", + "legendFormat": "{{instance}}", + "queryType": "randomWalk", "refId": "A" - }, - { - "exemplar": true, - "expr": "histogram_quantile(0.95, sum(rate(ticdc_event_store_register_dispatcher_start_ts_lag_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le, instance))", - "legendFormat": "{{instance}}-p95", - "refId": "B" - }, - { - "exemplar": true, - "expr": "histogram_quantile(1.0, sum(rate(ticdc_event_store_register_dispatcher_start_ts_lag_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le, instance))", - "legendFormat": "{{instance}}-max", - "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Register Dispatcher StartTs Lag", + "title": "Register Table Num", "tooltip": { "shared": true, "sort": 0, @@ -9535,36 +9528,27 @@ }, "yaxes": [ { - "format": "s", + "format": "short", + "label": null, "logBase": 1, + "max": null, "min": "0", "show": true }, { "format": "short", + "label": null, "logBase": 1, + "max": null, + "min": null, "show": false } ], "yaxis": { - "align": false + "align": false, + "alignLevel": null } - } - ], - "title": "Event Store", - "type": "row" - }, - { - "collapsed": true, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 10 - }, - "id": 12058, - "panels": [ + }, { "aliasColors": {}, "bars": false, @@ -9581,17 +9565,16 @@ "h": 8, "w": 12, "x": 0, - "y": 10 + "y": 18 }, "hiddenSeries": false, - "id": 12060, + "id": 12064, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, - "rightSide": false, "show": true, "total": false, "values": true @@ -9614,10 +9597,9 @@ "targets": [ { "exemplar": true, - "expr": "ticdc_schema_store_resolved_ts_lag{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}", + "expr": "sum(rate(ticdc_schema_store_get_table_info_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (instance)", "interval": "", - "intervalFactor": 1, - "legendFormat": "{{instance}}-resolvedts", + "legendFormat": "{{instance}}", "queryType": "randomWalk", "refId": "A" } @@ -9626,7 +9608,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Resolved Ts Lag", + "title": "Get Table Info Count / s", "tooltip": { "shared": true, "sort": 0, @@ -9642,7 +9624,7 @@ }, "yaxes": [ { - "format": "s", + "format": "short", "label": null, "logBase": 1, "max": null, @@ -9664,232 +9646,39 @@ } }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, + "cards": { + "cardPadding": null, + "cardRound": null + }, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateSpectral", + "exponent": 0.5, + "min": 0, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", "datasource": "tidb-cluster", "fieldConfig": { "defaults": {}, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 10 + "y": 18 }, - "hiddenSeries": false, - "id": 12062, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 12066, "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true + "show": true }, - "percentage": false, "pluginVersion": "7.5.17", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "exemplar": true, - "expr": "ticdc_schema_store_register_table_num{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}", - "interval": "", - "legendFormat": "{{instance}}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Register Table Num", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "tidb-cluster", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 18 - }, - "hiddenSeries": false, - "id": 12064, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.17", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "exemplar": true, - "expr": "sum(rate(ticdc_schema_store_get_table_info_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (instance)", - "interval": "", - "legendFormat": "{{instance}}", - "queryType": "randomWalk", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Get Table Info Count / s", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "cards": { - "cardPadding": null, - "cardRound": null - }, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateSpectral", - "exponent": 0.5, - "min": 0, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": "tidb-cluster", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 18 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 12066, - "legend": { - "show": true - }, - "pluginVersion": "7.5.17", - "reverseYBuckets": false, + "reverseYBuckets": false, "targets": [ { "exemplar": true, @@ -10172,7 +9961,7 @@ "h": 6, "w": 12, "x": 0, - "y": 11 + "y": 123 }, "heatmap": {}, "hideZeroBuckets": true, @@ -10249,7 +10038,7 @@ "h": 6, "w": 12, "x": 12, - "y": 11 + "y": 123 }, "hiddenSeries": false, "id": 22300, @@ -10356,7 +10145,7 @@ "h": 8, "w": 12, "x": 0, - "y": 17 + "y": 129 }, "hiddenSeries": false, "id": 22262, @@ -10447,7 +10236,7 @@ "h": 8, "w": 12, "x": 12, - "y": 17 + "y": 129 }, "hiddenSeries": false, "id": 22263, @@ -10538,7 +10327,7 @@ "h": 8, "w": 12, "x": 0, - "y": 25 + "y": 137 }, "hiddenSeries": false, "id": 22264, @@ -10637,7 +10426,7 @@ "h": 8, "w": 12, "x": 12, - "y": 25 + "y": 137 }, "hiddenSeries": false, "id": 22265, @@ -10728,7 +10517,7 @@ "h": 8, "w": 12, "x": 0, - "y": 33 + "y": 145 }, "hiddenSeries": false, "id": 20018, @@ -10822,7 +10611,7 @@ "h": 8, "w": 12, "x": 12, - "y": 33 + "y": 145 }, "hiddenSeries": false, "id": 20015, @@ -10917,7 +10706,7 @@ "h": 8, "w": 12, "x": 0, - "y": 41 + "y": 153 }, "hiddenSeries": false, "id": 20025, @@ -11013,7 +10802,7 @@ "h": 8, "w": 12, "x": 12, - "y": 41 + "y": 153 }, "hiddenSeries": false, "id": 22258, @@ -11107,7 +10896,7 @@ "h": 8, "w": 12, "x": 0, - "y": 49 + "y": 161 }, "hiddenSeries": false, "id": 22266, @@ -11179,28 +10968,14 @@ "yaxis": { "align": false } - } - ], - "title": "Event Service", - "type": "row" - }, - { - "collapsed": true, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 12 - }, - "id": 20059, - "panels": [ + }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "tidb-cluster", + "description": "", "fieldConfig": { "defaults": {}, "overrides": [] @@ -11210,17 +10985,18 @@ "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 12 + "x": 12, + "y": 161 }, "hiddenSeries": false, - "id": 10054, + "id": 22260, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, + "rightSide": false, "show": true, "total": false, "values": true @@ -11243,35 +11019,19 @@ "targets": [ { "exemplar": true, - "expr": "sum(rate(ticdc_messaging_error_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance, target, type)", - "hide": false, + "expr": "sum(rate(ticdc_event_service_channel_size{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\", instance=~\"$ticdc_instance\"}[1m])) by (instance, type)", "interval": "", - "legendFormat": "err-{{instance}}-{{target}}-{{type}}", + "intervalFactor": 1, + "legendFormat": "{{instance}}-{{type}}", "queryType": "randomWalk", "refId": "A" - }, - { - "exemplar": true, - "expr": "sum(rate(ticdc_messaging_drop_message_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance, target, type)", - "hide": false, - "interval": "", - "legendFormat": "drop-{{instance}}-{{target}}-{{type}}", - "refId": "B" - }, - { - "exemplar": true, - "expr": "sum(rate(ticdc_messaging_send_message_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance, target, type)", - "hide": false, - "interval": "", - "legendFormat": "send-{{instance}}-{{target}}-{{type}}", - "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Sent Message Count Per Second", + "title": "Event service channel size ", "tooltip": { "shared": true, "sort": 0, @@ -11287,6 +11047,7 @@ }, "yaxes": [ { + "decimals": 0, "format": "short", "label": null, "logBase": 1, @@ -11307,22 +11068,248 @@ "align": false, "alignLevel": null } - } - ], - "title": "Message Center", - "type": "row" - }, - { - "collapsed": true, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 13 - }, - "id": 20103, - "panels": [ + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "tidb-cluster", + "description": "The number of entries scanned by event store.", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 169 + }, + "hiddenSeries": false, + "id": 22259, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(ticdc_event_service_scanned_count_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (instance)", + "interval": "", + "legendFormat": "{{instance}}", + "queryType": "randomWalk", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Scanned entry count / s", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "title": "Event Service", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 12 + }, + "id": 20059, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "tidb-cluster", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 12 + }, + "hiddenSeries": false, + "id": 10054, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(ticdc_messaging_error_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance, target, type)", + "hide": false, + "interval": "", + "legendFormat": "err-{{instance}}-{{target}}-{{type}}", + "queryType": "randomWalk", + "refId": "A" + }, + { + "exemplar": true, + "expr": "sum(rate(ticdc_messaging_drop_message_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance, target, type)", + "hide": false, + "interval": "", + "legendFormat": "drop-{{instance}}-{{target}}-{{type}}", + "refId": "B" + }, + { + "exemplar": true, + "expr": "sum(rate(ticdc_messaging_send_message_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance, target, type)", + "hide": false, + "interval": "", + "legendFormat": "send-{{instance}}-{{target}}-{{type}}", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Sent Message Count Per Second", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "title": "Message Center", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 13 + }, + "id": 20103, + "panels": [ { "aliasColors": {}, "bars": false, @@ -20536,87 +20523,589 @@ "max": null, "min": null, "show": true, - "splitFactor": null + "splitFactor": null + }, + "yBucketBound": "upper", + "yBucketNumber": null, + "yBucketSize": null + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "tidb-cluster", + "description": "The total count of rows that are processed by redo writer", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 40 + }, + "hiddenSeries": false, + "id": 10031, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(ticdc_redo_total_rows_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (instance)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "queryType": "randomWalk", + "refId": "A" + }, + { + "exemplar": true, + "expr": "sum(rate(ticdc_redo_total_rows_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])) by (changefeed)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "total", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Redo Write rows / s ", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "tidb-cluster", + "description": "Total number of bytes redo log written", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 40 + }, + "hiddenSeries": false, + "id": 10033, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(ticdc_redo_write_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (instance)", + "interval": "", + "legendFormat": "{{instance}}", + "queryType": "randomWalk", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Redo Write bytes / s ", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "tidb-cluster", + "description": "Redo bgUpdateLog worker busy ratio", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 48 + }, + "hiddenSeries": false, + "id": 10035, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(ticdc_redo_worker_busy_ratio{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])/10) by (changefeed,instance)", + "interval": "", + "legendFormat": "{{changefeed}}-{{instance}}", + "queryType": "randomWalk", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Worker Busy Ratio", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "title": "Redo", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 22 + }, + "id": 155, + "panels": [ + { + "aliasColors": {}, + "bars": true, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "tidb-cluster", + "description": "TiCDC process rss memory usage. TiCDC heap memory size in use ", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 25 + }, + "hiddenSeries": false, + "id": 157, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": false, + "linewidth": 1, + "links": [], + "maxPerRow": 3, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "seriesOverrides": [ + { + "alias": "alloc-from-os", + "fill": 3, + "lines": true, + "stack": false + }, + { + "alias": "gc-threshold", + "bars": false, + "color": "#C4162A", + "lines": true, + "linewidth": 2, + "stack": false + }, + { + "alias": "gc", + "bars": false, + "color": "#C4162A", + "hideTooltip": true, + "legend": false, + "pointradius": 3, + "points": true, + "stack": false + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "alloc-from-os", + "refId": "A" + }, + { + "expr": "go_memstats_next_gc_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} / (1 + ticdc_server_go_gc{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} / 100)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "estimate-inuse", + "refId": "H" + }, + { + "expr": "go_memstats_heap_alloc_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} - go_memstats_next_gc_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} / (1 + ticdc_server_go_gc{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} / 100)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "estimate-garbage", + "refId": "C" + }, + { + "expr": "go_memstats_heap_idle_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} - go_memstats_heap_released_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} + go_memstats_heap_inuse_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} - go_memstats_heap_alloc_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "reserved-by-go", + "refId": "B" + }, + { + "expr": "go_memstats_stack_sys_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} + go_memstats_mspan_sys_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} + go_memstats_mcache_sys_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} + go_memstats_buck_hash_sys_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} + go_memstats_gc_sys_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} + go_memstats_other_sys_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "used-by-go", + "refId": "D" + }, + { + "expr": "go_memstats_next_gc_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "gc-threshold", + "refId": "E" + }, + { + "expr": "(clamp_max(idelta(go_memstats_last_gc_time_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"}[1m]), 1) * go_memstats_next_gc_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"}) > 0", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "gc", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] }, - "yBucketBound": "upper", - "yBucketNumber": null, - "yBucketSize": null + "yaxes": [ + { + "format": "bytes", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { "aliasColors": {}, "bars": false, + "cacheTimeout": null, "dashLength": 10, "dashes": false, "datasource": "tidb-cluster", - "description": "The total count of rows that are processed by redo writer", + "description": "Count of live objects.", + "editable": true, + "error": false, "fieldConfig": { "defaults": {}, "overrides": [] }, - "fill": 1, + "fill": 0, "fillGradient": 0, + "grid": {}, "gridPos": { - "h": 8, + "h": 7, "w": 12, - "x": 0, - "y": 40 + "x": 12, + "y": 25 }, "hiddenSeries": false, - "id": 10031, + "id": 158, "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, "min": false, - "rightSide": true, + "rightSide": false, "show": true, + "sideWidth": null, "total": false, - "values": true + "values": false }, "lines": true, "linewidth": 1, + "links": [], + "maxPerRow": 3, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.17", - "pointradius": 2, + "pointradius": 5, "points": false, "renderer": "flot", + "repeat": null, + "repeatDirection": "h", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "exemplar": true, - "expr": "sum(rate(ticdc_redo_total_rows_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (instance)", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{instance}}", - "queryType": "randomWalk", - "refId": "A" - }, - { - "exemplar": true, - "expr": "sum(rate(ticdc_redo_total_rows_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])) by (changefeed)", + "expr": "go_memstats_heap_objects{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"}", "format": "time_series", "hide": false, - "interval": "", "intervalFactor": 1, - "legendFormat": "total", - "refId": "B" + "legendFormat": "objects", + "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Redo Write rows / s ", + "title": "Estimated Live Objects", "tooltip": { + "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" @@ -20632,15 +21121,15 @@ "yaxes": [ { "format": "short", - "label": null, + "label": "", "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { "format": "short", - "label": null, + "label": "", "logBase": 1, "max": null, "min": null, @@ -20655,66 +21144,94 @@ { "aliasColors": {}, "bars": false, + "cacheTimeout": null, "dashLength": 10, "dashes": false, "datasource": "tidb-cluster", - "description": "Total number of bytes redo log written", + "description": "TiCDC process Go garbage collection STW pause duration", + "editable": true, + "error": false, "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, + "grid": {}, "gridPos": { - "h": 8, + "h": 7, "w": 12, - "x": 12, - "y": 40 + "x": 0, + "y": 32 }, "hiddenSeries": false, - "id": 10033, + "id": 160, "legend": { + "alignAsTable": false, "avg": false, - "current": false, + "current": true, "max": false, "min": false, + "rightSide": false, "show": true, "total": false, - "values": false + "values": true }, "lines": true, "linewidth": 1, - "nullPointMode": "null", + "links": [], + "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.17", - "pointradius": 2, + "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + {} + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "exemplar": true, - "expr": "sum(rate(ticdc_redo_write_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (instance)", - "interval": "", - "legendFormat": "{{instance}}", - "queryType": "randomWalk", - "refId": "A" + "expr": "go_gc_duration_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\", quantile=\"0\"}", + "format": "time_series", + "hide": false, + "instant": false, + "intervalFactor": 1, + "legendFormat": "min", + "refId": "A", + "step": 40 + }, + { + "expr": "go_gc_duration_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\", quantile!~\"0|1\"}", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{quantile}}", + "refId": "B" + }, + { + "expr": "go_gc_duration_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\", quantile=\"1\"}", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "max", + "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Redo Write bytes / s ", + "title": "GC STW Duration (last 256 GC cycles)", "tooltip": { + "msResolution": false, "shared": true, - "sort": 0, + "sort": 1, "value_type": "individual" }, "type": "graph", @@ -20727,11 +21244,11 @@ }, "yaxes": [ { - "format": "bytes", + "format": "s", "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -20740,7 +21257,7 @@ "logBase": 1, "max": null, "min": null, - "show": true + "show": false } ], "yaxis": { @@ -20754,7 +21271,7 @@ "dashLength": 10, "dashes": false, "datasource": "tidb-cluster", - "description": "Redo bgUpdateLog worker busy ratio", + "description": "The throughput of Go's memory allocator.", "fieldConfig": { "defaults": {}, "overrides": [] @@ -20762,18 +21279,18 @@ "fill": 1, "fillGradient": 0, "gridPos": { - "h": 8, + "h": 7, "w": 12, - "x": 0, - "y": 48 + "x": 12, + "y": 32 }, "hiddenSeries": false, - "id": 10035, + "id": 161, "legend": { - "alignAsTable": true, + "alignAsTable": false, "avg": false, "current": true, - "max": false, + "max": true, "min": false, "rightSide": false, "show": true, @@ -20782,34 +21299,69 @@ }, "lines": true, "linewidth": 1, + "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.17", - "pointradius": 2, + "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "sweep", + "transform": "negative-Y" + }, + { + "alias": "alloc-ops", + "yaxis": 2 + }, + { + "alias": "swepp-ops", + "transform": "negative-Y", + "yaxis": 2 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "exemplar": true, - "expr": "sum(rate(ticdc_redo_worker_busy_ratio{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])/10) by (changefeed,instance)", - "interval": "", - "legendFormat": "{{changefeed}}-{{instance}}", - "queryType": "randomWalk", + "expr": "irate(go_memstats_alloc_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"}[30s])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "alloc", "refId": "A" + }, + { + "expr": "irate((go_memstats_alloc_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} - go_memstats_heap_alloc_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"})[30s:])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "sweep", + "refId": "B" + }, + { + "expr": "irate(go_memstats_mallocs_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"}[30s])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "alloc-ops", + "refId": "C" + }, + { + "expr": "irate(go_memstats_frees_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"}[30s])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "swepp-ops", + "refId": "D" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Worker Busy Ratio", + "title": "Allocator Throughput", "tooltip": { "shared": true, "sort": 0, @@ -20825,7 +21377,7 @@ }, "yaxes": [ { - "format": "percent", + "format": "Bps", "label": null, "logBase": 1, "max": null, @@ -20833,21 +21385,22 @@ "show": true }, { - "format": "short", + "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, - "show": false + "show": true } ], "yaxis": { - "align": false, + "align": true, "alignLevel": null } } ], - "title": "Redo", + "repeat": "runtime_instance", + "title": "Runtime $runtime_instance", "type": "row" }, { @@ -20857,9 +21410,9 @@ "h": 1, "w": 24, "x": 0, - "y": 22 + "y": 23 }, - "id": 155, + "id": 22447, "panels": [ { "aliasColors": {}, @@ -20885,7 +21438,7 @@ "y": 25 }, "hiddenSeries": false, - "id": 157, + "id": 22448, "legend": { "alignAsTable": false, "avg": false, @@ -20915,13 +21468,9 @@ "renderer": "flot", "repeat": null, "repeatDirection": "h", - "scopedVars": { - "runtime_instance": { - "selected": false, - "text": "10.2.7.5:8300", - "value": "10.2.7.5:8300" - } - }, + "repeatIteration": 1760151079658, + "repeatPanelId": 157, + "repeatedByRow": true, "seriesOverrides": [ { "alias": "alloc-from-os", @@ -21075,7 +21624,7 @@ "y": 25 }, "hiddenSeries": false, - "id": 158, + "id": 22449, "legend": { "alignAsTable": false, "avg": false, @@ -21105,13 +21654,9 @@ "renderer": "flot", "repeat": null, "repeatDirection": "h", - "scopedVars": { - "runtime_instance": { - "selected": false, - "text": "10.2.7.5:8300", - "value": "10.2.7.5:8300" - } - }, + "repeatIteration": 1760151079658, + "repeatPanelId": 158, + "repeatedByRow": true, "seriesOverrides": [], "spaceLength": 10, "stack": false, @@ -21192,7 +21737,7 @@ "y": 32 }, "hiddenSeries": false, - "id": 160, + "id": 22450, "legend": { "alignAsTable": false, "avg": false, @@ -21216,13 +21761,9 @@ "pointradius": 5, "points": false, "renderer": "flot", - "scopedVars": { - "runtime_instance": { - "selected": false, - "text": "10.2.7.5:8300", - "value": "10.2.7.5:8300" - } - }, + "repeatIteration": 1760151079658, + "repeatPanelId": 160, + "repeatedByRow": true, "seriesOverrides": [ {} ], @@ -21319,7 +21860,7 @@ "y": 32 }, "hiddenSeries": false, - "id": 161, + "id": 22451, "legend": { "alignAsTable": false, "avg": false, @@ -21343,13 +21884,9 @@ "pointradius": 5, "points": false, "renderer": "flot", - "scopedVars": { - "runtime_instance": { - "selected": false, - "text": "10.2.7.5:8300", - "value": "10.2.7.5:8300" - } - }, + "repeatIteration": 1760151079658, + "repeatPanelId": 161, + "repeatedByRow": true, "seriesOverrides": [ { "alias": "sweep", @@ -21440,14 +21977,8 @@ } } ], - "repeat": "runtime_instance", - "scopedVars": { - "runtime_instance": { - "selected": false, - "text": "10.2.7.5:8300", - "value": "10.2.7.5:8300" - } - }, + "repeatIteration": 1760151079658, + "repeatPanelId": 155, "title": "Runtime $runtime_instance", "type": "row" }, @@ -21458,7 +21989,7 @@ "h": 1, "w": 24, "x": 0, - "y": 127 + "y": 24 }, "id": 10000, "panels": [