diff --git a/client/go.mod b/client/go.mod index 56380b0b51c..9501576e816 100644 --- a/client/go.mod +++ b/client/go.mod @@ -6,7 +6,7 @@ require ( github.com/opentracing/opentracing-go v1.2.0 github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 - github.com/pingcap/kvproto v0.0.0-20220510035547-0e2f26c0a46a + github.com/pingcap/kvproto v0.0.0-20220805093305-ab1ee4d521ab github.com/pingcap/log v0.0.0-20211215031037-e024ba4eb0ee github.com/prometheus/client_golang v1.11.0 github.com/stretchr/testify v1.7.0 diff --git a/client/go.sum b/client/go.sum index 90019b9b382..c7ad0ca3963 100644 --- a/client/go.sum +++ b/client/go.sum @@ -104,8 +104,8 @@ github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c h1:xpW9bvK+HuuTm github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg= github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 h1:C3N3itkduZXDZFh4N3vQ5HEtld3S+Y+StULhWVvumU0= github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00/go.mod h1:4qGtCB0QK0wBzKtFEGDhxXnSnbQApw1gc9siScUl8ew= -github.com/pingcap/kvproto v0.0.0-20220510035547-0e2f26c0a46a h1:TxdHGOFeNa1q1mVv6TgReayf26iI4F8PQUm6RnZ/V/E= -github.com/pingcap/kvproto v0.0.0-20220510035547-0e2f26c0a46a/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= +github.com/pingcap/kvproto v0.0.0-20220805093305-ab1ee4d521ab h1:rb720P/QawBTbC50fjwig1LIpu30ObIiThKokBt5mMk= +github.com/pingcap/kvproto v0.0.0-20220805093305-ab1ee4d521ab/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= github.com/pingcap/log v0.0.0-20211215031037-e024ba4eb0ee h1:VO2t6IBpfvW34TdtD/G10VvnGqjLic1jzOuHjUb5VqM= github.com/pingcap/log v0.0.0-20211215031037-e024ba4eb0ee/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= diff --git a/go.mod b/go.mod index 51d5eb7b87c..aa6ecd4c9b8 100644 --- a/go.mod +++ b/go.mod @@ -26,7 +26,7 @@ require ( github.com/pingcap/errcode v0.3.0 github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c github.com/pingcap/failpoint v0.0.0-20200702092429-9f69995143ce - github.com/pingcap/kvproto v0.0.0-20220510035547-0e2f26c0a46a + github.com/pingcap/kvproto v0.0.0-20220805093305-ab1ee4d521ab github.com/pingcap/log v0.0.0-20210906054005-afc726e70354 github.com/pingcap/sysutil v0.0.0-20211208032423-041a72e5860d github.com/pingcap/tidb-dashboard v0.0.0-20220728104842-3743e533b594 diff --git a/go.sum b/go.sum index d4384a8d172..06f7489f382 100644 --- a/go.sum +++ b/go.sum @@ -417,8 +417,8 @@ github.com/pingcap/failpoint v0.0.0-20200702092429-9f69995143ce h1:Y1kCxlCtlPTMt github.com/pingcap/failpoint v0.0.0-20200702092429-9f69995143ce/go.mod h1:w4PEZ5y16LeofeeGwdgZB4ddv9bLyDuIX+ljstgKZyk= github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w= github.com/pingcap/kvproto v0.0.0-20200411081810-b85805c9476c/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= -github.com/pingcap/kvproto v0.0.0-20220510035547-0e2f26c0a46a h1:TxdHGOFeNa1q1mVv6TgReayf26iI4F8PQUm6RnZ/V/E= -github.com/pingcap/kvproto v0.0.0-20220510035547-0e2f26c0a46a/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= +github.com/pingcap/kvproto v0.0.0-20220805093305-ab1ee4d521ab h1:rb720P/QawBTbC50fjwig1LIpu30ObIiThKokBt5mMk= +github.com/pingcap/kvproto v0.0.0-20220805093305-ab1ee4d521ab/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= github.com/pingcap/log v0.0.0-20200511115504-543df19646ad/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= github.com/pingcap/log v0.0.0-20210625125904-98ed8e2eb1c7/go.mod h1:8AanEdAHATuRurdGxZXBz0At+9avep+ub7U1AGYLIMM= diff --git a/metrics/grafana/pd.json b/metrics/grafana/pd.json index 30c7fe8f427..0f70af39c8f 100644 --- a/metrics/grafana/pd.json +++ b/metrics/grafana/pd.json @@ -63,8 +63,8 @@ "editable": true, "gnetId": null, "graphTooltip": 1, - "id": 25, - "iteration": 1618283470402, + "id": 26, + "iteration": 1659690075027, "links": [], "panels": [ { @@ -748,12 +748,14 @@ "decimals": null, "description": "It records the unusual Regions' count which may include pending peers, down peers, extra peers, offline peers, missing peers or learner peers", "fill": 1, + "fillGradient": 0, "gridPos": { "h": 7, "w": 11, "x": 5, "y": 6 }, + "hiddenSeries": false, "id": 72, "legend": { "alignAsTable": true, @@ -770,8 +772,12 @@ "linewidth": 1, "links": [], "nullPointMode": "null", + "options": { + "alertThreshold": true + }, "paceLength": 10, "percentage": false, + "pluginVersion": "7.5.11", "pointradius": 5, "points": false, "renderer": "flot", @@ -810,7 +816,7 @@ "line": true, "op": "gt", "value": 100, - "yaxis": "left" + "visible": true } ], "timeFrom": null, @@ -864,6 +870,7 @@ "editable": true, "error": false, "fill": 0, + "fillGradient": 0, "grid": {}, "gridPos": { "h": 7, @@ -871,6 +878,7 @@ "x": 16, "y": 6 }, + "hiddenSeries": false, "id": 18, "legend": { "avg": false, @@ -885,8 +893,12 @@ "linewidth": 3, "links": [], "nullPointMode": "null", + "options": { + "alertThreshold": true + }, "paceLength": 10, "percentage": false, + "pluginVersion": "7.5.11", "pointradius": 5, "points": false, "renderer": "flot", @@ -1063,11 +1075,44 @@ }, { "datasource": "${DS_TEST-CLUSTER}", + "fieldConfig": { + "defaults": { + "custom": { + "align": "left", + "filterable": false + }, + "mappings": [ + { + "from": "", + "id": 1, + "text": "N/A", + "to": "", + "type": 1, + "value": "1.7976931348623157e+308" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, "gridPos": { "h": 6, "w": 4, "x": 8, - "y": 14 + "y": 13 }, "id": 1111, "options": { @@ -1076,37 +1121,18 @@ "pluginVersion": "7.5.11", "targets": [ { + "exemplar": true, "expr": "pd_cluster_eta{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=\"$instance\"}", - "legendFormat": "{{action}}-{{store}}", + "instant": true, "interval": "", - "exemplar": true, + "legendFormat": "{{action}}-{{store}}", "queryType": "randomWalk", - "refId": "A", - "instant": true + "refId": "A" } ], + "timeFrom": null, + "timeShift": null, "title": "Left time", - "type": "table", - "fieldConfig": { - "defaults": { - "custom": { - "align": "left", - "filterable": false - }, - "mappings": [ - { - "id": 1, - "type": 1, - "from": "", - "to": "", - "text": "N/A", - "value": "1.7976931348623157e+308" - } - ], - "unit": "s" - }, - "overrides": [] - }, "transformations": [ { "id": "reduce", @@ -1117,34 +1143,10 @@ } } ], - "timeFrom": null, - "timeShift": null + "type": "table" }, { - "type": "table", - "title": "Current scaling speed", "datasource": "${DS_TEST-CLUSTER}", - "gridPos": { - "x": 12, - "y": 13, - "w": 4, - "h": 6 - }, - "id": 1112, - "targets": [ - { - "expr": "pd_cluster_speed{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=\"$instance\"}", - "legendFormat": "{{action}}-{{store}}", - "interval": "", - "exemplar": true, - "refId": "A", - "queryType": "randomWalk", - "instant": true - } - ], - "options": { - "showHeader": false - }, "fieldConfig": { "defaults": { "custom": { @@ -1152,10 +1154,48 @@ "filterable": false }, "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, "unit": "MBs" }, "overrides": [] }, + "gridPos": { + "h": 6, + "w": 4, + "x": 12, + "y": 13 + }, + "id": 1112, + "options": { + "showHeader": false + }, + "pluginVersion": "7.5.11", + "targets": [ + { + "exemplar": true, + "expr": "pd_cluster_speed{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=\"$instance\"}", + "instant": true, + "interval": "", + "legendFormat": "{{action}}-{{store}}", + "queryType": "randomWalk", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Current scaling speed", "transformations": [ { "id": "reduce", @@ -1166,8 +1206,7 @@ } } ], - "timeFrom": null, - "timeShift": null + "type": "table" }, { "collapsed": true, @@ -3290,7 +3329,8 @@ "align": false, "alignLevel": null } - },{ + }, + { "aliasColors": {}, "bars": false, "dashLength": 10, @@ -6527,12 +6567,14 @@ "datasource": "${DS_TEST-CLUSTER}", "description": "The current running schedulers", "fill": 0, + "fillGradient": 0, "gridPos": { "h": 8, "w": 24, "x": 0, - "y": 19 + "y": 25 }, + "hiddenSeries": false, "id": 46, "legend": { "alignAsTable": true, @@ -6553,6 +6595,7 @@ "nullPointMode": "null", "paceLength": 10, "percentage": false, + "pluginVersion": "7.5.11", "pointradius": 1, "points": false, "renderer": "flot", @@ -6621,12 +6664,14 @@ "decimals": null, "description": "The leader movement details among TiKV instances", "fill": 1, + "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 27 + "y": 33 }, + "hiddenSeries": false, "id": 87, "legend": { "alignAsTable": true, @@ -6649,6 +6694,7 @@ "nullPointMode": "null", "paceLength": 10, "percentage": false, + "pluginVersion": "7.5.11", "pointradius": 5, "points": false, "renderer": "flot", @@ -6722,12 +6768,14 @@ "decimals": null, "description": "The Region movement details among TiKV instances", "fill": 1, + "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 27 + "y": 33 }, + "hiddenSeries": false, "hideTimeOverride": false, "id": 86, "legend": { @@ -6751,6 +6799,7 @@ "nullPointMode": "null", "paceLength": 10, "percentage": false, + "pluginVersion": "7.5.11", "pointradius": 5, "points": false, "renderer": "flot", @@ -6815,189 +6864,6 @@ "alignLevel": null } }, - { - "aliasColors": {}, - "bars": true, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", - "description": "The count of balance leader events", - "fill": 1, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 35 - }, - "id": 89, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": true, - "values": true - }, - "lines": false, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "paceLength": 10, - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum(delta(pd_scheduler_balance_leader{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", store=~\"$store\", instance=\"$instance\", type!=\"move-leader\"}[30s])) by (type, address, store)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{type}}-store-{{store}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Balance leader event", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": true, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", - "description": "The count of balance Region events", - "fill": 1, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 35 - }, - "id": 88, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sortDesc": false, - "total": true, - "values": true - }, - "lines": false, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "paceLength": 10, - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum(delta(pd_scheduler_balance_region{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", store=~\"$store\", instance=\"$instance\"}[30s])) by (type, address, store)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{type}}-store-{{store}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Balance Region event", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, { "aliasColors": {}, "bars": false, @@ -7006,12 +6872,14 @@ "datasource": "${DS_TEST-CLUSTER}", "description": "The inner status of balance leader scheduler", "fill": 0, + "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 43 + "y": 41 }, + "hiddenSeries": false, "id": 52, "legend": { "alignAsTable": true, @@ -7034,6 +6902,7 @@ "nullPointMode": "null", "paceLength": 10, "percentage": false, + "pluginVersion": "7.5.11", "pointradius": 5, "points": false, "renderer": "flot", @@ -7101,12 +6970,14 @@ "datasource": "${DS_TEST-CLUSTER}", "description": "The inner status of balance Region scheduler", "fill": 0, + "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 43 + "y": 41 }, + "hiddenSeries": false, "id": 53, "legend": { "alignAsTable": true, @@ -7129,6 +7000,7 @@ "nullPointMode": "null", "paceLength": 10, "percentage": false, + "pluginVersion": "7.5.11", "pointradius": 5, "points": false, "renderer": "flot", @@ -7187,7 +7059,8 @@ "align": false, "alignLevel": null } - }, { + }, + { "aliasColors": {}, "bars": false, "dashLength": 10, @@ -7195,12 +7068,14 @@ "datasource": "${DS_TEST-CLUSTER}", "description": "The inner status of balance Hot Region scheduler", "fill": 0, + "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 51 + "y": 49 }, + "hiddenSeries": false, "id": 1458, "legend": { "alignAsTable": true, @@ -7223,6 +7098,7 @@ "nullPointMode": "null", "paceLength": 10, "percentage": false, + "pluginVersion": "7.5.11", "pointradius": 5, "points": false, "renderer": "flot", @@ -7281,7 +7157,8 @@ "align": false, "alignLevel": null } - },{ + }, + { "aliasColors": {}, "bars": false, "dashLength": 10, @@ -7289,12 +7166,14 @@ "datasource": "${DS_TEST-CLUSTER}", "description": "The inner status of split bucket scheduler", "fill": 0, + "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 51 + "y": 49 }, + "hiddenSeries": false, "id": 1459, "legend": { "alignAsTable": true, @@ -7317,6 +7196,7 @@ "nullPointMode": "null", "paceLength": 10, "percentage": false, + "pluginVersion": "7.5.11", "pointradius": 5, "points": false, "renderer": "flot", @@ -7383,12 +7263,14 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "fill": 0, + "fillGradient": 0, "gridPos": { "h": 8, "w": 24, "x": 0, - "y": 59 + "y": 57 }, + "hiddenSeries": false, "id": 108, "legend": { "alignAsTable": true, @@ -7411,6 +7293,7 @@ "nullPointMode": "null", "paceLength": 10, "percentage": false, + "pluginVersion": "7.5.11", "pointradius": 5, "points": false, "renderer": "flot", @@ -7492,7 +7375,7 @@ "h": 8, "w": 12, "x": 0, - "y": 67 + "y": 65 }, "id": 1424, "interval": null, @@ -7561,12 +7444,14 @@ "datasource": "${DS_TEST-CLUSTER}", "description": "The replica checker's status", "fill": 0, + "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 67 + "y": 65 }, + "hiddenSeries": false, "id": 141, "legend": { "alignAsTable": true, @@ -7587,6 +7472,7 @@ "nullPointMode": "null", "paceLength": 10, "percentage": false, + "pluginVersion": "7.5.11", "pointradius": 5, "points": false, "renderer": "flot", @@ -7653,12 +7539,14 @@ "datasource": "${DS_TEST-CLUSTER}", "description": "The replica checker's status", "fill": 0, + "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 75 + "y": 73 }, + "hiddenSeries": false, "id": 70, "legend": { "alignAsTable": true, @@ -7679,6 +7567,7 @@ "nullPointMode": "null", "paceLength": 10, "percentage": false, + "pluginVersion": "7.5.11", "pointradius": 5, "points": false, "renderer": "flot", @@ -7745,12 +7634,14 @@ "datasource": "${DS_TEST-CLUSTER}", "description": "The merge checker's status", "fill": 0, + "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 75 + "y": 73 }, + "hiddenSeries": false, "id": 71, "legend": { "alignAsTable": true, @@ -7771,6 +7662,7 @@ "nullPointMode": "null", "paceLength": 10, "percentage": false, + "pluginVersion": "7.5.11", "pointradius": 5, "points": false, "renderer": "flot", @@ -7836,12 +7728,14 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "fill": 0, + "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 83 + "y": 81 }, + "hiddenSeries": false, "id": 109, "legend": { "alignAsTable": true, @@ -7864,6 +7758,7 @@ "nullPointMode": "null", "paceLength": 10, "percentage": false, + "pluginVersion": "7.5.11", "pointradius": 5, "points": false, "renderer": "flot", @@ -7930,12 +7825,14 @@ "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "fill": 0, + "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 83 + "y": 81 }, + "hiddenSeries": false, "id": 110, "legend": { "alignAsTable": true, @@ -7958,6 +7855,7 @@ "nullPointMode": "null", "paceLength": 10, "percentage": false, + "pluginVersion": "7.5.11", "pointradius": 5, "points": false, "renderer": "flot", @@ -10272,7 +10170,6 @@ "renderer": "flot", "seriesOverrides": [ { - "$$hashKey": "object:363", "alias": "/pending.*/", "yaxis": 2 } @@ -11006,7 +10903,8 @@ "align": false, "alignLevel": null } - }, { + }, + { "aliasColors": {}, "bars": false, "dashLength": 10, @@ -11102,7 +11000,8 @@ "align": false, "alignLevel": null } - },{ + }, + { "aliasColors": {}, "bars": false, "dashLength": 10, @@ -11573,7 +11472,8 @@ "title": "Region Heartbeat Interval", "transparent": true, "type": "bargauge" - },{ + }, + { "datasource": "${DS_TEST-CLUSTER}", "fieldConfig": { "defaults": { @@ -11621,7 +11521,8 @@ "title": "Bucket Report Interval", "transparent": true, "type": "bargauge" - },{ + }, + { "datasource": "${DS_TEST-CLUSTER}", "fieldConfig": { "defaults": { @@ -11857,7 +11758,7 @@ } ], "refresh": "30s", - "schemaVersion": 18, + "schemaVersion": 27, "style": "dark", "tags": [], "templating": { @@ -11866,6 +11767,7 @@ "allValue": null, "current": { }, "datasource": "${DS_TEST-CLUSTER}", + "definition": "", "hide": 2, "includeAll": false, "label": "K8s-cluster", @@ -11875,9 +11777,10 @@ "query": "label_values(pd_cluster_status, k8s_cluster)", "refresh": 2, "regex": "", + "skipUrlSync": false, "sort": 1, "tagValuesQuery": "", - "tags": [ ], + "tags": [], "tagsQuery": "", "type": "query", "useTags": false diff --git a/server/api/unsafe_operation.go b/server/api/unsafe_operation.go index c45771619b0..0391a5d5e7f 100644 --- a/server/api/unsafe_operation.go +++ b/server/api/unsafe_operation.go @@ -48,24 +48,29 @@ func (h *unsafeOperationHandler) RemoveFailedStores(w http.ResponseWriter, r *ht rc := getCluster(r) var input map[string]interface{} if err := apiutil.ReadJSONRespondError(h.rd, w, r.Body, &input); err != nil { + h.rd.JSON(w, http.StatusBadRequest, err.Error()) return } - storeSlice, ok := typeutil.JSONToUint64Slice(input["stores"]) - if !ok { - h.rd.JSON(w, http.StatusBadRequest, "Store ids are invalid") - return - } + stores := make(map[uint64]struct{}) - for _, store := range storeSlice { - stores[store] = struct{}{} + autoDetect, exists := input["auto-detect"].(bool) + if !exists || !autoDetect { + storeSlice, ok := typeutil.JSONToUint64Slice(input["stores"]) + if !ok { + h.rd.JSON(w, http.StatusBadRequest, "Store ids are invalid") + return + } + for _, store := range storeSlice { + stores[store] = struct{}{} + } } + timeout := uint64(600) - rawTimeout, exists := input["timeout"].(float64) - if exists { + if rawTimeout, exists := input["timeout"].(float64); exists { timeout = uint64(rawTimeout) } - if err := rc.GetUnsafeRecoveryController().RemoveFailedStores(stores, timeout); err != nil { + if err := rc.GetUnsafeRecoveryController().RemoveFailedStores(stores, timeout, autoDetect); err != nil { h.rd.JSON(w, http.StatusInternalServerError, err.Error()) return } diff --git a/server/api/unsafe_operation_test.go b/server/api/unsafe_operation_test.go index 77c4149ec3b..3cf9e2965e9 100644 --- a/server/api/unsafe_operation_test.go +++ b/server/api/unsafe_operation_test.go @@ -37,7 +37,7 @@ func TestUnsafeOperationTestSuite(t *testing.T) { suite.Run(t, new(unsafeOperationTestSuite)) } -func (suite *unsafeOperationTestSuite) SetupSuite() { +func (suite *unsafeOperationTestSuite) SetupTest() { re := suite.Require() suite.svr, suite.cleanup = mustNewServer(re) server.MustWaitLeader(re, []*server.Server{suite.svr}) @@ -49,14 +49,15 @@ func (suite *unsafeOperationTestSuite) SetupSuite() { mustPutStore(re, suite.svr, 1, metapb.StoreState_Offline, metapb.NodeState_Removing, nil) } -func (suite *unsafeOperationTestSuite) TearDownSuite() { +func (suite *unsafeOperationTestSuite) TearDownTest() { suite.cleanup() } func (suite *unsafeOperationTestSuite) TestRemoveFailedStores() { + re := suite.Require() + input := map[string]interface{}{"stores": []uint64{}} data, _ := json.Marshal(input) - re := suite.Require() err := tu.CheckPostJSON(testDialClient, suite.urlPrefix+"/remove-failed-stores", data, tu.StatusNotOK(re), tu.StringEqual(re, "\"[PD:unsaferecovery:ErrUnsafeRecoveryInvalidInput]invalid input no store specified\"\n")) suite.NoError(err) @@ -83,3 +84,18 @@ func (suite *unsafeOperationTestSuite) TestRemoveFailedStores() { err = tu.ReadGetJSON(re, testDialClient, suite.urlPrefix+"/remove-failed-stores/show", &output) suite.NoError(err) } + +func (suite *unsafeOperationTestSuite) TestRemoveFailedStoresAutoDetect() { + re := suite.Require() + + input := map[string]interface{}{"auto-detect": false} + data, _ := json.Marshal(input) + err := tu.CheckPostJSON(testDialClient, suite.urlPrefix+"/remove-failed-stores", data, tu.StatusNotOK(re), + tu.StringEqual(re, "\"Store ids are invalid\"\n")) + suite.NoError(err) + + input = map[string]interface{}{"auto-detect": true} + data, _ = json.Marshal(input) + err = tu.CheckPostJSON(testDialClient, suite.urlPrefix+"/remove-failed-stores", data, tu.StatusOK(re)) + suite.NoError(err) +} diff --git a/server/cluster/unsafe_recovery_controller.go b/server/cluster/unsafe_recovery_controller.go index 9782d1a20d8..a0ecf777721 100644 --- a/server/cluster/unsafe_recovery_controller.go +++ b/server/cluster/unsafe_recovery_controller.go @@ -110,6 +110,7 @@ type unsafeRecoveryController struct { step uint64 failedStores map[uint64]struct{} timeout time.Time + autoDetect bool // collected reports from store, if not reported yet, it would be nil storeReports map[uint64]*pdpb.StoreReport @@ -164,30 +165,32 @@ func (u *unsafeRecoveryController) IsRunning() bool { } // RemoveFailedStores removes failed stores from the cluster. -func (u *unsafeRecoveryController) RemoveFailedStores(failedStores map[uint64]struct{}, timeout uint64) error { +func (u *unsafeRecoveryController) RemoveFailedStores(failedStores map[uint64]struct{}, timeout uint64, autoDetect bool) error { if u.IsRunning() { return errs.ErrUnsafeRecoveryIsRunning.FastGenByArgs() } u.Lock() defer u.Unlock() - if len(failedStores) == 0 { - return errs.ErrUnsafeRecoveryInvalidInput.FastGenByArgs("no store specified") - } + if !autoDetect { + if len(failedStores) == 0 { + return errs.ErrUnsafeRecoveryInvalidInput.FastGenByArgs("no store specified") + } - // validate the stores and mark the store as tombstone forcibly - for failedStore := range failedStores { - store := u.cluster.GetStore(failedStore) - if store == nil { - return errs.ErrUnsafeRecoveryInvalidInput.FastGenByArgs(fmt.Sprintf("store %v doesn't exist", failedStore)) - } else if (store.IsPreparing() || store.IsServing()) && !store.IsDisconnected() { - return errs.ErrUnsafeRecoveryInvalidInput.FastGenByArgs(fmt.Sprintf("store %v is up and connected", failedStore)) + // validate the stores and mark the store as tombstone forcibly + for failedStore := range failedStores { + store := u.cluster.GetStore(failedStore) + if store == nil { + return errs.ErrUnsafeRecoveryInvalidInput.FastGenByArgs(fmt.Sprintf("store %v doesn't exist", failedStore)) + } else if (store.IsPreparing() || store.IsServing()) && !store.IsDisconnected() { + return errs.ErrUnsafeRecoveryInvalidInput.FastGenByArgs(fmt.Sprintf("store %v is up and connected", failedStore)) + } } - } - for failedStore := range failedStores { - err := u.cluster.BuryStore(failedStore, true) - if err != nil && !errors.ErrorEqual(err, errs.ErrStoreNotFound.FastGenByArgs(failedStore)) { - return err + for failedStore := range failedStores { + err := u.cluster.BuryStore(failedStore, true) + if err != nil && !errors.ErrorEqual(err, errs.ErrStoreNotFound.FastGenByArgs(failedStore)) { + return err + } } } @@ -204,6 +207,7 @@ func (u *unsafeRecoveryController) RemoveFailedStores(failedStores map[uint64]st u.timeout = time.Now().Add(time.Duration(timeout) * time.Second) u.failedStores = failedStores + u.autoDetect = autoDetect u.changeStage(collectReport) return nil } @@ -384,7 +388,7 @@ func (u *unsafeRecoveryController) dispatchPlan(heartbeat *pdpb.StoreHeartbeatRe if reported, exist := u.storeReports[storeID]; reported != nil || !exist { // the plan has been executed, no need to dispatch again - // or no need to displan plan to this store(e.g. Tiflash) + // or no need to dispatch plan to this store(e.g. Tiflash) return } @@ -445,17 +449,23 @@ func (u *unsafeRecoveryController) changeStage(stage unsafeRecoveryStage) { switch u.stage { case idle: case collectReport: - stores := "" - count := 0 - for store := range u.failedStores { - count += 1 - stores += fmt.Sprintf("%d", store) - if count != len(u.failedStores) { - stores += ", " + // TODO: clean up existing operators + output.Info = "Unsafe recovery enters collect report stage" + if u.autoDetect { + output.Details = append(output.Details, "auto detect mode with no specified failed stores") + } else { + stores := "" + count := 0 + for store := range u.failedStores { + count += 1 + stores += fmt.Sprintf("%d", store) + if count != len(u.failedStores) { + stores += ", " + } } + output.Details = append(output.Details, fmt.Sprintf("failed stores %s", stores)) } - // TODO: clean up existing operators - output.Info = fmt.Sprintf("Unsafe recovery enters collect report stage: failed stores %s", stores) + case tombstoneTiFlashLearner: output.Info = "Unsafe recovery enters tombstone TiFlash learner stage" output.Actions = u.getTombstoneTiFlashLearnerDigest() @@ -615,13 +625,22 @@ func (u *unsafeRecoveryController) recordAffectedRegion(region *metapb.Region) { } } +func (u *unsafeRecoveryController) isFailed(peer *metapb.Peer) bool { + _, isFailed := u.failedStores[peer.StoreId] + _, isLive := u.storeReports[peer.StoreId] + if isFailed || (u.autoDetect && !isLive) { + return true + } + return false +} + func (u *unsafeRecoveryController) canElectLeader(region *metapb.Region, onlyIncoming bool) bool { hasQuorum := func(voters []*metapb.Peer) bool { numFailedVoters := 0 numLiveVoters := 0 for _, voter := range voters { - if _, ok := u.failedStores[voter.StoreId]; ok { + if u.isFailed(voter) { numFailedVoters += 1 } else { numLiveVoters += 1 @@ -657,7 +676,7 @@ func (u *unsafeRecoveryController) getFailedPeers(region *metapb.Region) []*meta if peer.Role == metapb.PeerRole_Learner || peer.Role == metapb.PeerRole_DemotingVoter { continue } - if _, ok := u.failedStores[peer.StoreId]; ok { + if u.isFailed(peer) { failedPeers = append(failedPeers, peer) } } diff --git a/server/cluster/unsafe_recovery_controller_test.go b/server/cluster/unsafe_recovery_controller_test.go index ebb2c917b63..2ee4a8aebca 100644 --- a/server/cluster/unsafe_recovery_controller_test.go +++ b/server/cluster/unsafe_recovery_controller_test.go @@ -159,7 +159,7 @@ func TestFinished(t *testing.T) { re.NoError(recoveryController.RemoveFailedStores(map[uint64]struct{}{ 2: {}, 3: {}, - }, 60)) + }, 60, false)) reports := map[uint64]*pdpb.StoreReport{ 1: {PeerReports: []*pdpb.PeerReport{ @@ -239,7 +239,7 @@ func TestFailed(t *testing.T) { re.NoError(recoveryController.RemoveFailedStores(map[uint64]struct{}{ 2: {}, 3: {}, - }, 60)) + }, 60, false)) reports := map[uint64]*pdpb.StoreReport{ 1: {PeerReports: []*pdpb.PeerReport{ @@ -333,7 +333,7 @@ func TestForceLeaderFail(t *testing.T) { re.NoError(recoveryController.RemoveFailedStores(map[uint64]struct{}{ 3: {}, 4: {}, - }, 60)) + }, 60, false)) reports := map[uint64]*pdpb.StoreReport{ 1: { @@ -413,7 +413,7 @@ func TestAffectedTableID(t *testing.T) { re.NoError(recoveryController.RemoveFailedStores(map[uint64]struct{}{ 2: {}, 3: {}, - }, 60)) + }, 60, false)) reports := map[uint64]*pdpb.StoreReport{ 1: { @@ -454,7 +454,7 @@ func TestForceLeaderForCommitMerge(t *testing.T) { re.NoError(recoveryController.RemoveFailedStores(map[uint64]struct{}{ 2: {}, 3: {}, - }, 60)) + }, 60, false)) reports := map[uint64]*pdpb.StoreReport{ 1: { @@ -514,6 +514,58 @@ func TestForceLeaderForCommitMerge(t *testing.T) { re.Equal(demoteFailedVoter, recoveryController.GetStage()) } +func TestAutoDetectMode(t *testing.T) { + re := require.New(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + _, opt, _ := newTestScheduleConfig() + cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) + cluster.coordinator = newCoordinator(ctx, cluster, hbstream.NewTestHeartbeatStreams(ctx, cluster.meta.GetId(), cluster, true)) + cluster.coordinator.run() + for _, store := range newTestStores(1, "6.0.0") { + re.NoError(cluster.PutStore(store.GetMeta())) + } + recoveryController := newUnsafeRecoveryController(cluster) + re.NoError(recoveryController.RemoveFailedStores(nil, 60, true)) + + reports := map[uint64]*pdpb.StoreReport{ + 1: {PeerReports: []*pdpb.PeerReport{ + { + RaftState: &raft_serverpb.RaftLocalState{LastIndex: 10, HardState: &eraftpb.HardState{Term: 1, Commit: 10}}, + RegionState: &raft_serverpb.RegionLocalState{ + Region: &metapb.Region{ + Id: 1001, + RegionEpoch: &metapb.RegionEpoch{ConfVer: 7, Version: 10}, + Peers: []*metapb.Peer{ + {Id: 11, StoreId: 1}, {Id: 12, StoreId: 2}, {Id: 13, StoreId: 3}}}}}, + }}, + } + + advanceUntilFinished(re, recoveryController, reports) + + expects := map[uint64]*pdpb.StoreReport{ + 1: {PeerReports: []*pdpb.PeerReport{ + { + RaftState: &raft_serverpb.RaftLocalState{LastIndex: 10, HardState: &eraftpb.HardState{Term: 1, Commit: 10}}, + RegionState: &raft_serverpb.RegionLocalState{ + Region: &metapb.Region{ + Id: 1001, + RegionEpoch: &metapb.RegionEpoch{ConfVer: 8, Version: 10}, + Peers: []*metapb.Peer{ + {Id: 11, StoreId: 1}, {Id: 12, StoreId: 2, Role: metapb.PeerRole_Learner}, {Id: 13, StoreId: 3, Role: metapb.PeerRole_Learner}}}}}, + }}, + } + + for storeID, report := range reports { + if result, ok := expects[storeID]; ok { + re.Equal(result.PeerReports, report.PeerReports) + } else { + re.Empty(len(report.PeerReports)) + } + } +} + func TestOneLearner(t *testing.T) { re := require.New(t) ctx, cancel := context.WithCancel(context.Background()) @@ -530,7 +582,7 @@ func TestOneLearner(t *testing.T) { re.NoError(recoveryController.RemoveFailedStores(map[uint64]struct{}{ 2: {}, 3: {}, - }, 60)) + }, 60, false)) reports := map[uint64]*pdpb.StoreReport{ 1: {PeerReports: []*pdpb.PeerReport{ @@ -588,7 +640,7 @@ func TestTiflashLearnerPeer(t *testing.T) { re.NoError(recoveryController.RemoveFailedStores(map[uint64]struct{}{ 4: {}, 5: {}, - }, 60)) + }, 60, false)) reports := map[uint64]*pdpb.StoreReport{ 1: {PeerReports: []*pdpb.PeerReport{ @@ -760,7 +812,7 @@ func TestUninitializedPeer(t *testing.T) { re.NoError(recoveryController.RemoveFailedStores(map[uint64]struct{}{ 2: {}, 3: {}, - }, 60)) + }, 60, false)) reports := map[uint64]*pdpb.StoreReport{ 1: {PeerReports: []*pdpb.PeerReport{ @@ -816,7 +868,7 @@ func TestJointState(t *testing.T) { re.NoError(recoveryController.RemoveFailedStores(map[uint64]struct{}{ 4: {}, 5: {}, - }, 60)) + }, 60, false)) reports := map[uint64]*pdpb.StoreReport{ 1: {PeerReports: []*pdpb.PeerReport{ @@ -1009,7 +1061,7 @@ func TestTimeout(t *testing.T) { re.NoError(recoveryController.RemoveFailedStores(map[uint64]struct{}{ 2: {}, 3: {}, - }, 1)) + }, 1, false)) time.Sleep(time.Second) req := newStoreHeartbeat(1, nil) @@ -1037,7 +1089,7 @@ func TestExitForceLeader(t *testing.T) { re.NoError(recoveryController.RemoveFailedStores(map[uint64]struct{}{ 2: {}, 3: {}, - }, 60)) + }, 60, false)) reports := map[uint64]*pdpb.StoreReport{ 1: { @@ -1115,7 +1167,7 @@ func TestStep(t *testing.T) { re.NoError(recoveryController.RemoveFailedStores(map[uint64]struct{}{ 2: {}, 3: {}, - }, 60)) + }, 60, false)) reports := map[uint64]*pdpb.StoreReport{ 1: { @@ -1170,7 +1222,7 @@ func TestOnHealthyRegions(t *testing.T) { re.NoError(recoveryController.RemoveFailedStores(map[uint64]struct{}{ 4: {}, 5: {}, - }, 60)) + }, 60, false)) reports := map[uint64]*pdpb.StoreReport{ 1: {PeerReports: []*pdpb.PeerReport{ @@ -1246,7 +1298,7 @@ func TestCreateEmptyRegion(t *testing.T) { re.NoError(recoveryController.RemoveFailedStores(map[uint64]struct{}{ 2: {}, 3: {}, - }, 60)) + }, 60, false)) reports := map[uint64]*pdpb.StoreReport{ 1: {PeerReports: []*pdpb.PeerReport{ @@ -1355,7 +1407,7 @@ func TestRangeOverlap1(t *testing.T) { re.NoError(recoveryController.RemoveFailedStores(map[uint64]struct{}{ 4: {}, 5: {}, - }, 60)) + }, 60, false)) reports := map[uint64]*pdpb.StoreReport{ 1: {PeerReports: []*pdpb.PeerReport{ @@ -1450,7 +1502,7 @@ func TestRangeOverlap2(t *testing.T) { re.NoError(recoveryController.RemoveFailedStores(map[uint64]struct{}{ 4: {}, 5: {}, - }, 60)) + }, 60, false)) reports := map[uint64]*pdpb.StoreReport{ 1: {PeerReports: []*pdpb.PeerReport{ @@ -1548,11 +1600,11 @@ func TestRemoveFailedStores(t *testing.T) { re.Error(recoveryController.RemoveFailedStores(map[uint64]struct{}{ 1: {}, 3: {}, - }, 60)) + }, 60, false)) re.NoError(recoveryController.RemoveFailedStores(map[uint64]struct{}{ 1: {}, - }, 60)) + }, 60, false)) re.True(cluster.GetStore(uint64(1)).IsRemoved()) for _, s := range cluster.GetSchedulers() { paused, err := cluster.IsSchedulerAllowed(s) @@ -1566,7 +1618,7 @@ func TestRemoveFailedStores(t *testing.T) { re.Error(recoveryController.RemoveFailedStores( map[uint64]struct{}{ 2: {}, - }, 60)) + }, 60, false)) } func TestSplitPaused(t *testing.T) { @@ -1590,7 +1642,7 @@ func TestSplitPaused(t *testing.T) { failedStores := map[uint64]struct{}{ 1: {}, } - re.NoError(recoveryController.RemoveFailedStores(failedStores, 60)) + re.NoError(recoveryController.RemoveFailedStores(failedStores, 60, false)) askSplitReq := &pdpb.AskSplitRequest{} _, err := cluster.HandleAskSplit(askSplitReq) re.Equal("[PD:unsaferecovery:ErrUnsafeRecoveryIsRunning]unsafe recovery is running", err.Error()) diff --git a/server/grpc_service.go b/server/grpc_service.go old mode 100755 new mode 100644 index 377a5e2f629..3d05175e3d9 --- a/server/grpc_service.go +++ b/server/grpc_service.go @@ -1500,6 +1500,9 @@ func (s *GrpcServer) validateRequest(header *pdpb.RequestHeader) error { } func (s *GrpcServer) header() *pdpb.ResponseHeader { + if s.clusterID == 0 { + return s.wrapErrorToHeader(pdpb.ErrorType_NOT_BOOTSTRAPPED, "cluster id is not ready") + } return &pdpb.ResponseHeader{ClusterId: s.clusterID} } diff --git a/server/schedule/filter/filters.go b/server/schedule/filter/filters.go index 5c13e1ca101..c3567fcd4f6 100644 --- a/server/schedule/filter/filters.go +++ b/server/schedule/filter/filters.go @@ -34,11 +34,14 @@ import ( func SelectSourceStores(stores []*core.StoreInfo, filters []Filter, opt *config.PersistOptions) []*core.StoreInfo { return filterStoresBy(stores, func(s *core.StoreInfo) bool { return slice.AllOf(filters, func(i int) bool { - if !filters[i].Source(opt, s).IsOK() { - sourceID := strconv.FormatUint(s.GetID(), 10) - targetID := "" - filterCounter.WithLabelValues("filter-source", s.GetAddress(), - sourceID, filters[i].Scope(), filters[i].Type(), sourceID, targetID).Inc() + status := filters[i].Source(opt, s) + if !status.IsOK() { + if status != statusStoreTombstone { + sourceID := strconv.FormatUint(s.GetID(), 10) + targetID := "" + filterCounter.WithLabelValues("filter-source", s.GetAddress(), + sourceID, filters[i].Scope(), filters[i].Type(), sourceID, targetID).Inc() + } return false } return true @@ -51,15 +54,18 @@ func SelectTargetStores(stores []*core.StoreInfo, filters []Filter, opt *config. return filterStoresBy(stores, func(s *core.StoreInfo) bool { return slice.AllOf(filters, func(i int) bool { filter := filters[i] - if !filter.Target(opt, s).IsOK() { - cfilter, ok := filter.(comparingFilter) - targetID := strconv.FormatUint(s.GetID(), 10) - sourceID := "" - if ok { - sourceID = strconv.FormatUint(cfilter.GetSourceStoreID(), 10) + status := filter.Target(opt, s) + if !status.IsOK() { + if status != statusStoreTombstone { + cfilter, ok := filter.(comparingFilter) + targetID := strconv.FormatUint(s.GetID(), 10) + sourceID := "" + if ok { + sourceID = strconv.FormatUint(cfilter.GetSourceStoreID(), 10) + } + filterCounter.WithLabelValues("filter-target", s.GetAddress(), + targetID, filters[i].Scope(), filters[i].Type(), sourceID, targetID).Inc() } - filterCounter.WithLabelValues("filter-target", s.GetAddress(), - targetID, filters[i].Scope(), filters[i].Type(), sourceID, targetID).Inc() return false } return true @@ -99,11 +105,14 @@ func Source(opt *config.PersistOptions, store *core.StoreInfo, filters []Filter) storeAddress := store.GetAddress() storeID := strconv.FormatUint(store.GetID(), 10) for _, filter := range filters { - if !filter.Source(opt, store).IsOK() { - sourceID := storeID - targetID := "" - filterCounter.WithLabelValues("filter-source", storeAddress, - sourceID, filter.Scope(), filter.Type(), sourceID, targetID).Inc() + status := filter.Source(opt, store) + if !status.IsOK() { + if status != statusStoreTombstone { + sourceID := storeID + targetID := "" + filterCounter.WithLabelValues("filter-source", storeAddress, + sourceID, filter.Scope(), filter.Type(), sourceID, targetID).Inc() + } return false } } @@ -115,15 +124,18 @@ func Target(opt *config.PersistOptions, store *core.StoreInfo, filters []Filter) storeAddress := store.GetAddress() storeID := strconv.FormatUint(store.GetID(), 10) for _, filter := range filters { - if !filter.Target(opt, store).IsOK() { - cfilter, ok := filter.(comparingFilter) - targetID := storeID - sourceID := "" - if ok { - sourceID = strconv.FormatUint(cfilter.GetSourceStoreID(), 10) + status := filter.Target(opt, store) + if !status.IsOK() { + if status != statusStoreTombstone { + cfilter, ok := filter.(comparingFilter) + targetID := storeID + sourceID := "" + if ok { + sourceID = strconv.FormatUint(cfilter.GetSourceStoreID(), 10) + } + filterCounter.WithLabelValues("filter-target", storeAddress, + targetID, filter.Scope(), filter.Type(), sourceID, targetID).Inc() } - filterCounter.WithLabelValues("filter-target", storeAddress, - targetID, filter.Scope(), filter.Type(), sourceID, targetID).Inc() return false } } diff --git a/server/schedulers/balance_leader.go b/server/schedulers/balance_leader.go index b3e018322a5..a8286428b17 100644 --- a/server/schedulers/balance_leader.go +++ b/server/schedulers/balance_leader.go @@ -399,11 +399,9 @@ func createTransferLeaderOperator(cs *candidateStores, dir string, l *balanceLea switch dir { case transferOut: plan.source, plan.target = store, nil - l.counter.WithLabelValues("high-score", plan.SourceMetricLabel()).Inc() creator = l.transferLeaderOut case transferIn: plan.source, plan.target = nil, store - l.counter.WithLabelValues("low-score", plan.TargetMetricLabel()).Inc() creator = l.transferLeaderIn } var op *operator.Operator @@ -418,7 +416,6 @@ func createTransferLeaderOperator(cs *candidateStores, dir string, l *balanceLea } if op != nil { l.retryQuota.ResetLimit(store) - op.Counters = append(op.Counters, l.counter.WithLabelValues(dir, plan.SourceMetricLabel())) } else { l.Attenuate(store) log.Debug("no operator created for selected stores", zap.String("scheduler", l.GetName()), zap.Uint64(dir, store.GetID())) diff --git a/server/schedulers/balance_test.go b/server/schedulers/balance_test.go index d6ade983493..82aa1dcdb1d 100644 --- a/server/schedulers/balance_test.go +++ b/server/schedulers/balance_test.go @@ -564,7 +564,7 @@ func (suite *balanceLeaderRangeSchedulerTestSuite) TestSingleRangeBalance() { ops, _ := lb.Schedule(suite.tc, false) suite.NotEmpty(ops) suite.Len(ops, 1) - suite.Len(ops[0].Counters, 2) + suite.Len(ops[0].Counters, 1) suite.Len(ops[0].FinishedCounters, 3) lb, err = schedule.CreateScheduler(BalanceLeaderType, suite.oc, storage.NewStorageWithMemoryBackend(), schedule.ConfigSliceDecoder(BalanceLeaderType, []string{"h", "n"})) suite.NoError(err) diff --git a/server/storage/endpoint/key_path.go b/server/storage/endpoint/key_path.go index db7032b654b..65b89b452f5 100644 --- a/server/storage/endpoint/key_path.go +++ b/server/storage/endpoint/key_path.go @@ -32,8 +32,11 @@ const ( customScheduleConfigPath = "scheduler_config" gcWorkerServiceSafePointID = "gc_worker" minResolvedTS = "min_resolved_ts" - keySpaceSafePointPrefix = "key_space/gc_safepoint" - keySpaceGCSafePointSuffix = "gc" + keyspaceSafePointPrefix = "keyspaces/gc_safepoint" + keyspaceGCSafePointSuffix = "gc" + keyspacePrefix = "keyspaces" + keyspaceMetaInfix = "meta" + keyspaceIDInfix = "id" ) // AppendToRootPath appends the given key to the rootPath. @@ -108,31 +111,58 @@ func MinResolvedTSPath() string { } // KeySpaceServiceSafePointPrefix returns the prefix of given service's service safe point. -// Prefix: /key_space/gc_safepoint/{space_id}/service/ +// Prefix: /keyspaces/gc_safepoint/{space_id}/service/ func KeySpaceServiceSafePointPrefix(spaceID string) string { - return path.Join(keySpaceSafePointPrefix, spaceID, "service") + "/" + return path.Join(keyspaceSafePointPrefix, spaceID, "service") + "/" } // KeySpaceGCSafePointPath returns the gc safe point's path of the given key-space. -// Path: /key_space/gc_safepoint/{space_id}/gc +// Path: /keyspaces/gc_safepoint/{space_id}/gc func KeySpaceGCSafePointPath(spaceID string) string { - return path.Join(keySpaceSafePointPrefix, spaceID, keySpaceGCSafePointSuffix) + return path.Join(keyspaceSafePointPrefix, spaceID, keyspaceGCSafePointSuffix) } // KeySpaceServiceSafePointPath returns the path of given service's service safe point. -// Path: /key_space/gc_safepoint/{space_id}/service/{service_id} +// Path: /keyspaces/gc_safepoint/{space_id}/service/{service_id} func KeySpaceServiceSafePointPath(spaceID, serviceID string) string { return path.Join(KeySpaceServiceSafePointPrefix(spaceID), serviceID) } // KeySpaceSafePointPrefix returns prefix for all key-spaces' safe points. -// Path: /key_space/gc_safepoint/ +// Path: /keyspaces/gc_safepoint/ func KeySpaceSafePointPrefix() string { - return keySpaceSafePointPrefix + "/" + return keyspaceSafePointPrefix + "/" } // KeySpaceGCSafePointSuffix returns the suffix for any gc safepoint. // Postfix: /gc func KeySpaceGCSafePointSuffix() string { - return "/" + keySpaceGCSafePointSuffix + return "/" + keyspaceGCSafePointSuffix +} + +// KeyspaceMetaPrefix returns the prefix of keyspaces' metadata. +// Prefix: keyspaces/meta/ +func KeyspaceMetaPrefix() string { + return path.Join(keyspacePrefix, keyspaceMetaInfix) + "/" +} + +// KeyspaceMetaPath returns the path to the given keyspace's metadata. +// Path: keyspaces/meta/{space_id} +func KeyspaceMetaPath(spaceID uint32) string { + idStr := encodeKeyspaceID(spaceID) + return path.Join(KeyspaceMetaPrefix(), idStr) +} + +// KeyspaceIDPath returns the path to keyspace id from the given name. +// Path: keyspaces/id/{name} +func KeyspaceIDPath(name string) string { + return path.Join(keyspacePrefix, keyspaceIDInfix, name) +} + +// encodeKeyspaceID from uint32 to string. +// It adds extra padding to make encoded ID ordered. +// Encoded ID can be decoded directly with strconv.ParseUint. +// Width of the padded keyspaceID is 8 (decimal representation of uint24max is 16777215). +func encodeKeyspaceID(spaceID uint32) string { + return fmt.Sprintf("%08d", spaceID) } diff --git a/server/storage/endpoint/keyspace.go b/server/storage/endpoint/keyspace.go new file mode 100644 index 00000000000..bc3c28c05b0 --- /dev/null +++ b/server/storage/endpoint/keyspace.go @@ -0,0 +1,116 @@ +// Copyright 2022 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package endpoint + +import ( + "strconv" + + "github.com/gogo/protobuf/proto" + "github.com/pingcap/kvproto/pkg/keyspacepb" + "go.etcd.io/etcd/clientv3" +) + +const ( + // spaceIDBase is base used to encode/decode spaceID. + // It's set to 10 for better readability. + spaceIDBase = 10 + // spaceIDBitSizeMax is the max bitSize of spaceID. + // It's currently set to 24 (3bytes). + spaceIDBitSizeMax = 24 +) + +// KeyspaceStorage defines storage operations on keyspace related data. +type KeyspaceStorage interface { + // SaveKeyspace saves the given keyspace to the storage. + SaveKeyspace(*keyspacepb.KeyspaceMeta) error + // LoadKeyspace loads keyspace specified by spaceID. + LoadKeyspace(spaceID uint32, keyspace *keyspacepb.KeyspaceMeta) (bool, error) + // RemoveKeyspace removes target keyspace specified by spaceID. + RemoveKeyspace(spaceID uint32) error + // LoadRangeKeyspace loads no more than limit keyspaces starting at startID. + LoadRangeKeyspace(startID uint32, limit int) ([]*keyspacepb.KeyspaceMeta, error) + // SaveKeyspaceIDByName saves keyspace name to ID lookup information. + // It saves the ID onto the path encoded with name. + SaveKeyspaceIDByName(spaceID uint32, name string) error + // LoadKeyspaceIDByName loads keyspace ID for the given keyspace specified by name. + // It first constructs path to spaceID with the given name, then attempt to retrieve + // target spaceID. If the target keyspace does not exist, result boolean is set to false. + LoadKeyspaceIDByName(name string) (bool, uint32, error) +} + +var _ KeyspaceStorage = (*StorageEndpoint)(nil) + +// SaveKeyspace saves the given keyspace to the storage. +func (se *StorageEndpoint) SaveKeyspace(keyspace *keyspacepb.KeyspaceMeta) error { + key := KeyspaceMetaPath(keyspace.GetId()) + return se.saveProto(key, keyspace) +} + +// LoadKeyspace loads keyspace specified by spaceID. +func (se *StorageEndpoint) LoadKeyspace(spaceID uint32, keyspace *keyspacepb.KeyspaceMeta) (bool, error) { + key := KeyspaceMetaPath(spaceID) + return se.loadProto(key, keyspace) +} + +// RemoveKeyspace removes target keyspace specified by spaceID. +func (se *StorageEndpoint) RemoveKeyspace(spaceID uint32) error { + key := KeyspaceMetaPath(spaceID) + return se.Remove(key) +} + +// LoadRangeKeyspace loads keyspaces starting at startID. +// limit specifies the limit of loaded keyspaces. +func (se *StorageEndpoint) LoadRangeKeyspace(startID uint32, limit int) ([]*keyspacepb.KeyspaceMeta, error) { + startKey := KeyspaceMetaPath(startID) + endKey := clientv3.GetPrefixRangeEnd(KeyspaceMetaPrefix()) + keys, values, err := se.LoadRange(startKey, endKey, limit) + if err != nil { + return nil, err + } + if len(keys) == 0 { + return []*keyspacepb.KeyspaceMeta{}, nil + } + keyspaces := make([]*keyspacepb.KeyspaceMeta, 0, len(keys)) + for _, value := range values { + keyspace := &keyspacepb.KeyspaceMeta{} + if err = proto.Unmarshal([]byte(value), keyspace); err != nil { + return nil, err + } + keyspaces = append(keyspaces, keyspace) + } + return keyspaces, nil +} + +// SaveKeyspaceIDByName saves keyspace name to ID lookup information to storage. +func (se *StorageEndpoint) SaveKeyspaceIDByName(spaceID uint32, name string) error { + key := KeyspaceIDPath(name) + idStr := strconv.FormatUint(uint64(spaceID), spaceIDBase) + return se.Save(key, idStr) +} + +// LoadKeyspaceIDByName loads keyspace ID for the given keyspace name +func (se *StorageEndpoint) LoadKeyspaceIDByName(name string) (bool, uint32, error) { + key := KeyspaceIDPath(name) + idStr, err := se.Load(key) + // Failed to load the keyspaceID if loading operation errored, or if keyspace does not exist. + if err != nil || idStr == "" { + return false, 0, err + } + id64, err := strconv.ParseUint(idStr, spaceIDBase, spaceIDBitSizeMax) + if err != nil { + return false, 0, err + } + return true, uint32(id64), nil +} diff --git a/server/storage/keyspace_test.go b/server/storage/keyspace_test.go new file mode 100644 index 00000000000..967a87c28e3 --- /dev/null +++ b/server/storage/keyspace_test.go @@ -0,0 +1,148 @@ +// Copyright 2022 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "testing" + "time" + + "github.com/pingcap/kvproto/pkg/keyspacepb" + "github.com/stretchr/testify/require" + "github.com/tikv/pd/server/storage/endpoint" +) + +func TestSaveLoadKeyspace(t *testing.T) { + re := require.New(t) + storage := NewStorageWithMemoryBackend() + + keyspaces := makeTestKeyspaces() + for _, keyspace := range keyspaces { + re.NoError(storage.SaveKeyspace(keyspace)) + } + + for _, keyspace := range keyspaces { + spaceID := keyspace.GetId() + loadedKeyspace := &keyspacepb.KeyspaceMeta{} + // Test load keyspace. + success, err := storage.LoadKeyspace(spaceID, loadedKeyspace) + re.True(success) + re.NoError(err) + re.Equal(keyspace, loadedKeyspace) + // Test remove keyspace. + re.NoError(storage.RemoveKeyspace(spaceID)) + success, err = storage.LoadKeyspace(spaceID, loadedKeyspace) + // Loading a non-existing keyspace should be unsuccessful. + re.False(success) + // Loading a non-existing keyspace should not return error. + re.NoError(err) + } +} + +func TestLoadRangeKeyspaces(t *testing.T) { + re := require.New(t) + storage := NewStorageWithMemoryBackend() + + keyspaces := makeTestKeyspaces() + for _, keyspace := range keyspaces { + re.NoError(storage.SaveKeyspace(keyspace)) + } + + // Load all keyspaces. + loadedKeyspaces, err := storage.LoadRangeKeyspace(keyspaces[0].GetId(), 0) + re.NoError(err) + re.ElementsMatch(keyspaces, loadedKeyspaces) + + // Load keyspaces with id >= second test keyspace's id. + loadedKeyspaces2, err := storage.LoadRangeKeyspace(keyspaces[1].GetId(), 0) + re.NoError(err) + re.ElementsMatch(keyspaces[1:], loadedKeyspaces2) + + // Load keyspace with the smallest id. + loadedKeyspace3, err := storage.LoadRangeKeyspace(1, 1) + re.NoError(err) + re.ElementsMatch(keyspaces[:1], loadedKeyspace3) +} + +func TestSaveLoadKeyspaceID(t *testing.T) { + re := require.New(t) + storage := NewStorageWithMemoryBackend() + + ids := []uint32{100, 200, 300} + names := []string{"keyspace1", "keyspace2", "keyspace3"} + for i := range ids { + re.NoError(storage.SaveKeyspaceIDByName(ids[i], names[i])) + } + + for i := range names { + success, id, err := storage.LoadKeyspaceIDByName(names[i]) + re.NoError(err) + re.True(success) + re.Equal(ids[i], id) + } + // Loading non-existing id should return false, 0, nil. + success, id, err := storage.LoadKeyspaceIDByName("non-existing") + re.NoError(err) + re.False(success) + re.Equal(uint32(0), id) +} + +func makeTestKeyspaces() []*keyspacepb.KeyspaceMeta { + now := time.Now().Unix() + return []*keyspacepb.KeyspaceMeta{ + { + Id: 10, + Name: "keyspace1", + State: keyspacepb.KeyspaceState_ENABLED, + CreatedAt: now, + StateChangedAt: now, + Config: map[string]string{ + "gc_life_time": "6000", + "gc_interval": "3000", + }, + }, + { + Id: 11, + Name: "keyspace2", + State: keyspacepb.KeyspaceState_ARCHIVED, + CreatedAt: now + 300, + StateChangedAt: now + 300, + Config: map[string]string{ + "gc_life_time": "1000", + "gc_interval": "5000", + }, + }, + { + Id: 100, + Name: "keyspace3", + State: keyspacepb.KeyspaceState_DISABLED, + CreatedAt: now + 500, + StateChangedAt: now + 500, + Config: map[string]string{ + "gc_life_time": "4000", + "gc_interval": "2000", + }, + }, + } +} + +// TestEncodeSpaceID test spaceID encoding. +func TestEncodeSpaceID(t *testing.T) { + re := require.New(t) + re.Equal("keyspaces/meta/00000000", endpoint.KeyspaceMetaPath(0)) + re.Equal("keyspaces/meta/16777215", endpoint.KeyspaceMetaPath(1<<24-1)) + re.Equal("keyspaces/meta/00000100", endpoint.KeyspaceMetaPath(100)) + re.Equal("keyspaces/meta/00000011", endpoint.KeyspaceMetaPath(11)) + re.Equal("keyspaces/meta/00000010", endpoint.KeyspaceMetaPath(10)) +} diff --git a/server/storage/storage.go b/server/storage/storage.go index 97e1f8dc06e..78b8bcd3283 100644 --- a/server/storage/storage.go +++ b/server/storage/storage.go @@ -40,6 +40,7 @@ type Storage interface { endpoint.GCSafePointStorage endpoint.MinResolvedTSStorage endpoint.KeySpaceGCSafePointStorage + endpoint.KeyspaceStorage } // NewStorageWithMemoryBackend creates a new storage with memory backend. diff --git a/server/tso/global_allocator.go b/server/tso/global_allocator.go old mode 100755 new mode 100644 diff --git a/tests/client/go.mod b/tests/client/go.mod index 80e2d3d3b93..86743eb8af2 100644 --- a/tests/client/go.mod +++ b/tests/client/go.mod @@ -5,7 +5,7 @@ go 1.18 require ( github.com/gogo/protobuf v1.3.2 github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 - github.com/pingcap/kvproto v0.0.0-20220510035547-0e2f26c0a46a + github.com/pingcap/kvproto v0.0.0-20220805093305-ab1ee4d521ab github.com/stretchr/testify v1.7.0 github.com/tikv/pd v0.0.0-00010101000000-000000000000 github.com/tikv/pd/client v0.0.0-00010101000000-000000000000 diff --git a/tests/client/go.sum b/tests/client/go.sum index f42e33f8063..5cb44a3310b 100644 --- a/tests/client/go.sum +++ b/tests/client/go.sum @@ -390,8 +390,8 @@ github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 h1:C3N3itkduZXDZ github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00/go.mod h1:4qGtCB0QK0wBzKtFEGDhxXnSnbQApw1gc9siScUl8ew= github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w= github.com/pingcap/kvproto v0.0.0-20200411081810-b85805c9476c/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= -github.com/pingcap/kvproto v0.0.0-20220510035547-0e2f26c0a46a h1:TxdHGOFeNa1q1mVv6TgReayf26iI4F8PQUm6RnZ/V/E= -github.com/pingcap/kvproto v0.0.0-20220510035547-0e2f26c0a46a/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= +github.com/pingcap/kvproto v0.0.0-20220805093305-ab1ee4d521ab h1:rb720P/QawBTbC50fjwig1LIpu30ObIiThKokBt5mMk= +github.com/pingcap/kvproto v0.0.0-20220805093305-ab1ee4d521ab/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= github.com/pingcap/log v0.0.0-20200511115504-543df19646ad/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= github.com/pingcap/log v0.0.0-20210625125904-98ed8e2eb1c7/go.mod h1:8AanEdAHATuRurdGxZXBz0At+9avep+ub7U1AGYLIMM= diff --git a/tools/pd-ctl/pdctl/command/unsafe_command.go b/tools/pd-ctl/pdctl/command/unsafe_command.go index 513be6d6171..8494eca4b5d 100644 --- a/tools/pd-ctl/pdctl/command/unsafe_command.go +++ b/tools/pd-ctl/pdctl/command/unsafe_command.go @@ -38,11 +38,14 @@ func NewUnsafeCommand() *cobra.Command { // NewRemoveFailedStoresCommand returns the unsafe remove failed stores command. func NewRemoveFailedStoresCommand() *cobra.Command { cmd := &cobra.Command{ - Use: "remove-failed-stores [,,...]", + Use: "remove-failed-stores [,,...]", Short: "Remove failed stores unsafely", Run: removeFailedStoresCommandFunc, } cmd.PersistentFlags().Float64("timeout", 300, "timeout in seconds") + cmd.PersistentFlags().Bool("auto-detect", false, `detect failed stores automatically without needing to pass failed store ids, and all stores not in PD stores list are regarded as failed; +Note: DO NOT RECOMMEND to use this flag for general use, it's used only for case that PD doesn't have the store information of failed stores after pd-recover; +Note: Do it with caution to make sure all live stores's heartbeats has been reported PD already, otherwise it may regarded some stores as failed mistakenly.`) cmd.AddCommand(NewRemoveFailedStoresShowCommand()) return cmd } @@ -58,23 +61,40 @@ func NewRemoveFailedStoresShowCommand() *cobra.Command { func removeFailedStoresCommandFunc(cmd *cobra.Command, args []string) { prefix := fmt.Sprintf("%s/remove-failed-stores", unsafePrefix) - if len(args) < 1 { - cmd.Usage() + postInput := make(map[string]interface{}, 3) + + autoDetect, err := cmd.Flags().GetBool("auto-detect") + if err != nil { + cmd.Println(err) return } - strStores := strings.Split(args[0], ",") - var stores []uint64 - for _, strStore := range strStores { - store, err := strconv.ParseUint(strStore, 10, 64) - if err != nil { - cmd.Println(err) + + if autoDetect { + if len(args) > 0 { + cmd.Println("The flag `auto-detect` is set, no need to specify failed store ids") return } - stores = append(stores, store) - } - postInput := map[string]interface{}{ - "stores": stores, + postInput["auto-detect"] = autoDetect + } else { + if len(args) < 1 { + cmd.Println("Failed store ids are not specified") + cmd.Usage() + return + } + + strStores := strings.Split(args[0], ",") + var stores []uint64 + for _, strStore := range strStores { + store, err := strconv.ParseUint(strStore, 10, 64) + if err != nil { + cmd.Println(err) + return + } + stores = append(stores, store) + } + postInput["stores"] = stores } + timeout, err := cmd.Flags().GetFloat64("timeout") if err != nil { cmd.Println(err) @@ -82,6 +102,7 @@ func removeFailedStoresCommandFunc(cmd *cobra.Command, args []string) { } else if timeout != 300 { postInput["timeout"] = timeout } + postJSON(cmd, prefix, postInput) } diff --git a/tools/pd-tso-bench/go.sum b/tools/pd-tso-bench/go.sum index da6ab07bbab..940c35fe9cf 100644 --- a/tools/pd-tso-bench/go.sum +++ b/tools/pd-tso-bench/go.sum @@ -104,8 +104,8 @@ github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c h1:xpW9bvK+HuuTm github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg= github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 h1:C3N3itkduZXDZFh4N3vQ5HEtld3S+Y+StULhWVvumU0= github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00/go.mod h1:4qGtCB0QK0wBzKtFEGDhxXnSnbQApw1gc9siScUl8ew= -github.com/pingcap/kvproto v0.0.0-20220510035547-0e2f26c0a46a h1:TxdHGOFeNa1q1mVv6TgReayf26iI4F8PQUm6RnZ/V/E= -github.com/pingcap/kvproto v0.0.0-20220510035547-0e2f26c0a46a/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= +github.com/pingcap/kvproto v0.0.0-20220805093305-ab1ee4d521ab h1:rb720P/QawBTbC50fjwig1LIpu30ObIiThKokBt5mMk= +github.com/pingcap/kvproto v0.0.0-20220805093305-ab1ee4d521ab/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= github.com/pingcap/log v0.0.0-20211215031037-e024ba4eb0ee h1:VO2t6IBpfvW34TdtD/G10VvnGqjLic1jzOuHjUb5VqM= github.com/pingcap/log v0.0.0-20211215031037-e024ba4eb0ee/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=