diff --git a/docs/assets/images/dashboard_view_1.png b/docs/assets/images/dashboard_view_1.png new file mode 100644 index 0000000000..030ab270dc Binary files /dev/null and b/docs/assets/images/dashboard_view_1.png differ diff --git a/docs/assets/images/dashboard_view_2.png b/docs/assets/images/dashboard_view_2.png new file mode 100644 index 0000000000..96f9e46c9d Binary files /dev/null and b/docs/assets/images/dashboard_view_2.png differ diff --git a/docs/assets/images/dashboard_view_3.png b/docs/assets/images/dashboard_view_3.png new file mode 100644 index 0000000000..48a5ad3a86 Binary files /dev/null and b/docs/assets/images/dashboard_view_3.png differ diff --git a/docs/assets/images/datasource_1.png b/docs/assets/images/datasource_1.png new file mode 100644 index 0000000000..3bb7eb1ec4 Binary files /dev/null and b/docs/assets/images/datasource_1.png differ diff --git a/docs/assets/images/datasource_2.png b/docs/assets/images/datasource_2.png new file mode 100644 index 0000000000..ef6958c982 Binary files /dev/null and b/docs/assets/images/datasource_2.png differ diff --git a/docs/assets/images/grafana.png b/docs/assets/images/grafana.png new file mode 100644 index 0000000000..f3ef338db8 Binary files /dev/null and b/docs/assets/images/grafana.png differ diff --git a/docs/assets/images/img.png b/docs/assets/images/img.png new file mode 100644 index 0000000000..de412e9b86 Binary files /dev/null and b/docs/assets/images/img.png differ diff --git a/docs/assets/images/import_via_panel_json.png b/docs/assets/images/import_via_panel_json.png new file mode 100644 index 0000000000..16ff5627ea Binary files /dev/null and b/docs/assets/images/import_via_panel_json.png differ diff --git a/docs/assets/images/metrics_data.png b/docs/assets/images/metrics_data.png new file mode 100644 index 0000000000..65a3b9876c Binary files /dev/null and b/docs/assets/images/metrics_data.png differ diff --git a/docs/assets/other/json/apisix-ingress-controller-grafana.json b/docs/assets/other/json/apisix-ingress-controller-grafana.json new file mode 100644 index 0000000000..77c27d2fdf --- /dev/null +++ b/docs/assets/other/json/apisix-ingress-controller-grafana.json @@ -0,0 +1,848 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "Prometheus", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "limit": 100, + "name": "Annotations & Alerts", + "showIn": 0, + "type": "dashboard" + } + ] + }, + "description": "Ingress Controller of Apache APISIX", + "editable": true, + "gnetId": 11719, + "graphTooltip": 0, + "id": 9, + "iteration": 1632975622801, + "links": [], + "panels": [ + { + "collapsed": false, + "datasource": "Prometheus", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 10, + "panels": [], + "title": "APISIX Ingress controller", + "type": "row" + }, + { + "datasource": null, + "description": "Whether the role of controller instance is leader", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "from": "", + "id": 1, + "text": "Candidate", + "to": "", + "type": 1, + "value": "0" + }, + { + "from": "", + "id": 2, + "text": "Leader", + "to": "", + "type": 1, + "value": "1" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 1 + }, + "id": 33, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "7.5.6", + "targets": [ + { + "exemplar": true, + "expr": "apisix_ingress_controller_is_leader", + "instant": false, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Is Leader", + "type": "stat" + }, + { + "cacheTimeout": null, + "datasource": "Prometheus", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 1 + }, + "id": 16, + "interval": null, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "7.5.6", + "targets": [ + { + "exemplar": true, + "expr": "irate(apisix_ingress_controller_cache_sync_total[5m])", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{result}}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Cache Sync Operations", + "type": "stat" + }, + { + "cacheTimeout": null, + "datasource": "Prometheus", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 1 + }, + "id": 11, + "interval": null, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "7.5.6", + "targets": [ + { + "exemplar": true, + "expr": "irate(apisix_ingress_controller_sync_operation_total[5m])", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{resource}}-{{result}}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Sync Operations", + "type": "stat" + }, + { + "datasource": null, + "description": "Number of events handled by the controller", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 6 + }, + "id": 35, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "7.5.6", + "targets": [ + { + "exemplar": true, + "expr": "irate(apisix_ingress_controller_events_total[5m])", + "interval": "", + "legendFormat": "{{resource}}-{{operation}}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Total Events", + "type": "stat" + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 6 + }, + "id": 29, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "7.5.6", + "targets": [ + { + "exemplar": true, + "expr": "irate(apisix_ingress_controller_check_cluster_health_total[5m])", + "interval": "", + "legendFormat": "{{result}}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Cluster Health Check", + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 6 + }, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "7.5.6", + "targets": [ + { + "exemplar": true, + "expr": "sum(apisix_ingress_controller_apisix_bad_status_codes{status_code =~ \"[4-5].*\"}) / sum(apisix_ingress_controller_apisix_requests)", + "hide": false, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Response Error Rate", + "type": "stat" + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 13 + }, + "id": 31, + "panels": [], + "title": "Apisix Requests", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "description": "", + "fieldConfig": { + "defaults": { + "unit": "none" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 14 + }, + "hiddenSeries": false, + "id": 8, + "interval": null, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxDataPoints": 100, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.6", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "apisix_ingress_controller_apisix_requests", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{resource}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Total Requests", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1442", + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:1443", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 20 + }, + "hiddenSeries": false, + "id": 17, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.6", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "apisix_ingress_controller_apisix_bad_status_codes", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{resource}}-{{status_code}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Status Codes", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:567", + "decimals": null, + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:568", + "decimals": null, + "format": "Misc", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "description": "Request latencies with APISIX", + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 27 + }, + "hiddenSeries": false, + "id": 2, + "interval": "", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.6", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum (\n rate(\n apisix_ingress_controller_apisix_request_latencies_sum{}[1m]\n)) / sum (\n rate(\n apisix_ingress_controller_apisix_request_latencies_count{}[1m]\n )\n)", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "average", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Requests Latencies", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:439", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:440", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "5m", + "schemaVersion": 27, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allValue": ".*", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": "Prometheus", + "definition": "label_values(resource)", + "description": null, + "error": null, + "hide": 0, + "includeAll": true, + "label": null, + "multi": true, + "name": "resource", + "options": [], + "query": { + "query": "label_values(resource)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": ".*", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": "Prometheus", + "definition": "label_values(operation)", + "description": null, + "error": null, + "hide": 0, + "includeAll": true, + "label": null, + "multi": true, + "name": "operation", + "options": [], + "query": { + "query": "label_values(operation)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Apache APISIX Ingress Controller", + "uid": "cJp7SjHnk", + "version": 1 +} diff --git a/docs/en/latest/plugins/prometheus.md b/docs/en/latest/plugins/prometheus.md new file mode 100644 index 0000000000..693698efc7 --- /dev/null +++ b/docs/en/latest/plugins/prometheus.md @@ -0,0 +1,124 @@ +--- +title: prometheus +--- + + + +This guide shows how to monitor Apache APISIX Ingress Controller using Prometheus and Grafana. + +## Enable Prometheus + +Use CRD file to enable Prometheus in global configurations. The definition file for custom resources is `ApisixClusterConfig`, so the configuration should be: + +```yaml +apiVersion: apisix.apache.org/v2beta3 +kind: ApisixClusterConfig +metadata: + name: default +spec: + monitoring: + prometheus: + enable: true +``` + +## Configure Prometheus Server + +The Prometheus server address should be `127.0.0.1:9090` by default. You can set the target url for `apisix-ingress-controller` manually in `prometheus.yml`. +For example: + +```yaml +... +scrape_configs: + - job_name: "apisix-ingress-controller" + static_configs: + - targets: ["localhost:9092"] +``` + +According to the above example, the metrics are exposed in `http://localhost:9092/metrics` now. +Visit `http://localhost:9090` in your browser, and select `apisix-ingress-controller` in `targets` +or visit `http://localhost:9092/metrics` to see the data. +![metrics_data.png](../../../assets/images/metrics_data.png) + +## Grafana Dashboard + +Grafana dashboard shows the metrics exposed in Prometheus graphically. +[Installing Grafana](https://grafana.com/docs/grafana/latest/#installing-grafana) +Visit `http://localhost:3000/` in your browser to access Grafana. The default username and password are `admin`. +Then create a new dashboard for `apisix-ingress-controller`. +![grafana.png](../../../assets/images/grafana.png) +Follow the steps to apply the configuration of Grafana Dashboard for `apisix-ingress-controller`. + +- Add an empty panel, and import via panel json in `apisix-ingress-controller/docs/assets/other/json/apisix-ingress-controller-grafana.json`. +![img.png](../../../assets/images/img.png) +![import_via_panel_json.png](../../../assets/images/import_via_panel_json.png) +- Select `Prometheus database` as the datasource. Set the URL according to your Prometheus server configuration. +![datasource_1.png](../../../assets/images/datasource_1.png) +![datasource_2.png](../../../assets/images/datasource_2.png) + +### Preview + +![dashboard_view_1.png](../../../assets/images/dashboard_view_1.png) +![dashboard_view_2.png](../../../assets/images/dashboard_view_2.png) +![dashboard_view_3.png](../../../assets/images/dashboard_view_3.png) + +## Available metrics + +- `Is leader` A gauge type metric with value 0 or 1, indicates whether the role of controller instance is leader, for leader is 1 and candidate is 0. +Labels: + - controller_pod + - controller_namespace +- `Status codes` status codes of requests to APISIX. +Labels: + - controller_pod + - controller_namespace + - status_code: the HTTP status code returned by APISIX. + - resource +- `Latency` Request latencies with APISIX. +Labels: + - controller_pod + - controller_namespace +- `Requests` Number of requests to APISIX. +Labels: + - controller_pod + - controller_namespace + - resource +- `Check cluster health` Number of cluster health check operations. +Labels: + - controller_pod + - controller_namespace + - name: cluster name. +- `Sync operation` Number of sync operations. +Labels: + - controller_pod + - controller_namespace + - resource + - result: sync success or failure. +- `Cache sync` Number of cache sync operations. +Labels: + - controller_pod + - controller_namespace + - result: sync success or failure. +- `Controller events` Number of events handled by the controller. +Labels: + - controller_pod + - controller_namespace + - resource + - operation: includes `add`, `update`, `delete`. + \ No newline at end of file