From e79e8388906cbb1b57418df989b749c082d543fd Mon Sep 17 00:00:00 2001 From: Hardikl <83282894+Hardikl@users.noreply.github.com> Date: Thu, 27 Apr 2023 18:38:40 +0530 Subject: [PATCH] feat: collect external_service_op perf object (#1941) * feat: collect external_service_op perf object * feat: collect external_service_op perf object * feat: dashboard changes * feat: CI changes * feat: CI changes * feat: added new panels in svm dashboard * feat: moved new panels to new dashboard * feat: merge main * feat: adding operation var, aggregation at servicename and operation * fix: added unique key for topk * fix: minor changes * fix: few name correction * fix: topk default to 5 --- .../externalserviceoperation.go | 30 + cmd/collectors/zapiperf/zapiperf.go | 3 + cmd/tools/grafana/dashboard_test.go | 8 + .../9.8.0/external_service_operation.yaml | 39 + conf/zapiperf/default.yaml | 95 +- .../dashboards/cmode/external_service_op.json | 1197 +++++++++++++++++ integration/test/dashboard_json_test.go | 2 + 7 files changed, 1327 insertions(+), 47 deletions(-) create mode 100644 cmd/collectors/zapiperf/plugins/externalserviceoperation/externalserviceoperation.go create mode 100644 conf/zapiperf/cdot/9.8.0/external_service_operation.yaml create mode 100644 grafana/dashboards/cmode/external_service_op.json diff --git a/cmd/collectors/zapiperf/plugins/externalserviceoperation/externalserviceoperation.go b/cmd/collectors/zapiperf/plugins/externalserviceoperation/externalserviceoperation.go new file mode 100644 index 000000000..85a60c354 --- /dev/null +++ b/cmd/collectors/zapiperf/plugins/externalserviceoperation/externalserviceoperation.go @@ -0,0 +1,30 @@ +/* + * Copyright NetApp Inc, 2023 All rights reserved + */ +package externalserviceoperation + +import ( + "github.com/netapp/harvest/v2/cmd/poller/plugin" + "github.com/netapp/harvest/v2/pkg/matrix" +) + +const Hyphen = "-" + +type ExternalServiceOperation struct { + *plugin.AbstractPlugin +} + +func New(p *plugin.AbstractPlugin) plugin.Plugin { + return &ExternalServiceOperation{AbstractPlugin: p} +} + +func (e *ExternalServiceOperation) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) { + data := dataMap[e.Object] + datacenterClusterKey := data.GetGlobalLabels().Get("datacenter") + Hyphen + data.GetGlobalLabels().Get("cluster") + Hyphen + for _, instance := range data.GetInstances() { + // generate unique key by appending datacenter, cluster, svm, service_name and operation to support topk in grafana dashboard + key := datacenterClusterKey + instance.GetLabel("svm") + Hyphen + instance.GetLabel("service_name") + Hyphen + instance.GetLabel("operation") + instance.SetLabel("key", key) + } + return nil, nil +} diff --git a/cmd/collectors/zapiperf/zapiperf.go b/cmd/collectors/zapiperf/zapiperf.go index e10483296..58f47869a 100644 --- a/cmd/collectors/zapiperf/zapiperf.go +++ b/cmd/collectors/zapiperf/zapiperf.go @@ -27,6 +27,7 @@ package zapiperf import ( "errors" "github.com/netapp/harvest/v2/cmd/collectors/zapiperf/plugins/disk" + "github.com/netapp/harvest/v2/cmd/collectors/zapiperf/plugins/externalserviceoperation" "github.com/netapp/harvest/v2/cmd/collectors/zapiperf/plugins/fcp" "github.com/netapp/harvest/v2/cmd/collectors/zapiperf/plugins/headroom" "github.com/netapp/harvest/v2/cmd/collectors/zapiperf/plugins/nic" @@ -125,6 +126,8 @@ func (z *ZapiPerf) LoadPlugin(kind string, abc *plugin.AbstractPlugin) plugin.Pl return vscan.New(abc) case "Disk": return disk.New(abc) + case "ExternalServiceOperation": + return externalserviceoperation.New(abc) default: z.Logger.Info().Msgf("no zapiPerf plugin found for %s", kind) } diff --git a/cmd/tools/grafana/dashboard_test.go b/cmd/tools/grafana/dashboard_test.go index 1aded9122..c8e7a160c 100644 --- a/cmd/tools/grafana/dashboard_test.go +++ b/cmd/tools/grafana/dashboard_test.go @@ -484,6 +484,7 @@ func TestIDIsBlank(t *testing.T) { []string{"../../../grafana/dashboards/cmode", "../../../grafana/dashboards/storagegrid"}, func(path string, data []byte) { checkUIDIsBlank(t, path, data) + checkIDIsNull(t, path, data) }) } @@ -508,6 +509,13 @@ func checkUIDIsBlank(t *testing.T, path string, data []byte) { } } +func checkIDIsNull(t *testing.T, path string, data []byte) { + id := gjson.GetBytes(data, "id").String() + if id != "" { + t.Errorf(`dashboard=%s id should be null but is %s`, shortPath(path), id) + } +} + func TestUniquePanelIDs(t *testing.T) { visitDashboards( []string{"../../../grafana/dashboards/cmode", "../../../grafana/dashboards/storagegrid"}, diff --git a/conf/zapiperf/cdot/9.8.0/external_service_operation.yaml b/conf/zapiperf/cdot/9.8.0/external_service_operation.yaml new file mode 100644 index 000000000..bbb8e85a0 --- /dev/null +++ b/conf/zapiperf/cdot/9.8.0/external_service_operation.yaml @@ -0,0 +1,39 @@ + +name: ExternalServiceOperation +query: external_service_op +object: external_service_op + +instance_key: uuid + +counters: + - instance_name + - instance_uuid + - node_name => node + - num_not_found_responses + - num_request_failures + - num_requests_sent + - num_responses_received + - num_successful_responses + - num_timeouts + - operation + - process_name + - request_latency + - request_latency_hist + - server_ip_address + - server_name + - service_name + - vserver_name => svm + +plugins: + - ExternalServiceOperation + +export_options: + instance_keys: + - key + - node + - operation + - process_name + - server_ip_address + - server_name + - service_name + - svm diff --git a/conf/zapiperf/default.yaml b/conf/zapiperf/default.yaml index 4b78ce131..fc47ac2a8 100644 --- a/conf/zapiperf/default.yaml +++ b/conf/zapiperf/default.yaml @@ -9,54 +9,55 @@ schedule: objects: # Node-level metrics - CIFSNode: cifs_node.yaml - Disk: disk.yaml - ExtCacheObj: ext_cache_obj.yaml - FCVI: fcvi.yaml - FcpPort: fcp.yaml - HeadroomAggr: resource_headroom_aggr.yaml - HeadroomCPU: resource_headroom_cpu.yaml - HostAdapter: hostadapter.yaml -# Netstat: netstat.yaml - NFSv3Node: nfsv3_node.yaml - NFSv41Node: nfsv4_1_node.yaml - NFSv42Node: nfsv4_2_node.yaml - NFSv4Node: nfsv4_node.yaml - NVMfLif: nvmf_lif.yaml - Namespace: namespace.yaml - NicCommon: nic_common.yaml - ObjectStoreClient: object_store_client_op.yaml - Path: path.yaml - Qtree: qtree.yaml - SystemNode: system_node.yaml - TokenManager: token_manager.yaml - VolumeNode: volume_node.yaml - WAFL: wafl.yaml - WAFLAggr: wafl_hya_per_aggr.yaml - WAFLSizer: wafl_hya_sizer.yaml -# NFSv4Pool: nfsv4_pool.yaml + CIFSNode: cifs_node.yaml + Disk: disk.yaml + ExtCacheObj: ext_cache_obj.yaml + FCVI: fcvi.yaml + FcpPort: fcp.yaml + HeadroomAggr: resource_headroom_aggr.yaml + HeadroomCPU: resource_headroom_cpu.yaml + HostAdapter: hostadapter.yaml +# Netstat: netstat.yaml + NFSv3Node: nfsv3_node.yaml + NFSv41Node: nfsv4_1_node.yaml + NFSv42Node: nfsv4_2_node.yaml + NFSv4Node: nfsv4_node.yaml + NVMfLif: nvmf_lif.yaml + Namespace: namespace.yaml + NicCommon: nic_common.yaml + ObjectStoreClient: object_store_client_op.yaml + Path: path.yaml + Qtree: qtree.yaml + SystemNode: system_node.yaml + TokenManager: token_manager.yaml + VolumeNode: volume_node.yaml + WAFL: wafl.yaml + WAFLAggr: wafl_hya_per_aggr.yaml + WAFLSizer: wafl_hya_sizer.yaml +# NFSv4Pool: nfsv4_pool.yaml # SVM-level metrics - CIFSvserver: cifs_vserver.yaml - CopyManager: copy_manager.yaml - FcpLif: fcp_lif.yaml - ISCSI: iscsi_lif.yaml - LIF: lif.yaml - Lun: lun.yaml - NFSv3: nfsv3.yaml - NFSv41: nfsv4_1.yaml - NFSv42: nfsv4_2.yaml - NFSv4: nfsv4.yaml - OntapS3SVM: ontap_s3_svm.yaml - SMB2: smb2.yaml - Volume: volume.yaml - VolumeSvm: volume_svm.yaml - WAFLCompBin: wafl_comp_aggr_vol_bin.yaml - Vscan: vscan.yaml - VscanSVM: vscan_svm.yaml + CIFSvserver: cifs_vserver.yaml + CopyManager: copy_manager.yaml +# ExternalServiceOperation: external_service_operation.yaml + FcpLif: fcp_lif.yaml + ISCSI: iscsi_lif.yaml + LIF: lif.yaml + Lun: lun.yaml + NFSv3: nfsv3.yaml + NFSv41: nfsv4_1.yaml + NFSv42: nfsv4_2.yaml + NFSv4: nfsv4.yaml + OntapS3SVM: ontap_s3_svm.yaml + SMB2: smb2.yaml + Volume: volume.yaml + VolumeSvm: volume_svm.yaml + WAFLCompBin: wafl_comp_aggr_vol_bin.yaml + Vscan: vscan.yaml + VscanSVM: vscan_svm.yaml # Uncomment to collect workload/QOS counters -# Workload: workload.yaml -# WorkloadDetail: workload_detail.yaml -# WorkloadVolume: workload_volume.yaml -# WorkloadDetailVolume: workload_detail_volume.yaml \ No newline at end of file +# Workload: workload.yaml +# WorkloadDetail: workload_detail.yaml +# WorkloadVolume: workload_volume.yaml +# WorkloadDetailVolume: workload_detail_volume.yaml \ No newline at end of file diff --git a/grafana/dashboards/cmode/external_service_op.json b/grafana/dashboards/cmode/external_service_op.json new file mode 100644 index 000000000..c3a460466 --- /dev/null +++ b/grafana/dashboards/cmode/external_service_op.json @@ -0,0 +1,1197 @@ +{ + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "Prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "8.1.8" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, + { + "type": "panel", + "id": "table", + "name": "Table", + "version": "" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "", + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": null, + "graphTooltip": 1, + "id": null, + "iteration": 1681729577481, + "links": [ + { + "asDropdown": true, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [ + "cdot" + ], + "targetBlank": false, + "title": "Related Dashboards", + "tooltip": "", + "type": "dashboards", + "url": "" + } + ], + "liveNow": false, + "panels": [ + { + "datasource": "${DS_PROMETHEUS}", + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 29, + "options": { + "content": "You must uncomment the external_service_operation.yaml section in your $HARVEST/conf/zapiperf/default.yaml file.", + "mode": "markdown" + }, + "pluginVersion": "8.1.8", + "title": "Important Information about External Service Operation dashboard", + "type": "text" + }, + { + "collapsed": false, + "datasource": "${DS_PROMETHEUS}", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 2 + }, + "id": 68, + "panels": [], + "title": "Highlights", + "type": "row" + }, + { + "description": "This panel includes Average request latency for operations on specified server.", + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 3 + }, + "id": 76, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "exemplar": false, + "expr": "topk($TopResources, avg by (operation, service_name, svm, cluster) (external_service_op_request_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",service_name=~\"$ServiceName\",operation=~\"$Operation\",key=~\"$TopRequestLatency\"}))", + "interval": "", + "legendFormat": "{{cluster}} - {{svm}} - {{operation}}", + "refId": "A" + } + ], + "title": "Top $TopResources Request Latency to Server", + "type": "timeseries" + }, + { + "cards": {}, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateRdYlGn", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "description": "This panel tracks histograms for request latency values of this operation to the specified server", + "datasource": "${DS_PROMETHEUS}", + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 3 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 81, + "interval": "6m", + "legend": { + "show": false + }, + "maxDataPoints": 25, + "reverseYBuckets": false, + "targets": [ + { + "exemplar": false, + "expr": "sum(increase(external_service_op_request_latency_hist_bucket{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",service_name=~\"$ServiceName\",operation=~\"$Operation\"}[$__interval])) by (le)", + "format": "heatmap", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": " {{ le }}", + "refId": "A" + } + ], + "title": "Request Latency Heatmap to Server", + "tooltip": { + "show": true, + "showHistogram": false + }, + "type": "heatmap", + "xAxis": { + "show": true + }, + "yAxis": { + "format": "µs", + "logBase": 1, + "show": true + }, + "yBucketBound": "auto" + }, + { + "description": "This panel shows Number of 'Not Found' responses for the operations.", + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 12 + }, + "id": 83, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "exemplar": false, + "expr": "topk($TopResources, sum by (operation, service_name, svm, cluster) (external_service_op_num_not_found_responses{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",service_name=~\"$ServiceName\",operation=~\"$Operation\",key=~\"$TopNotFoundResponse\"}))", + "interval": "", + "legendFormat": "{{cluster}} - {{svm}} - {{operation}}", + "refId": "A" + } + ], + "title": "Top $TopResources Number of 'Not Found' Responses per Operation", + "type": "timeseries" + }, + { + "description": "This panel shows cumulative count of all request failures.", + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 12 + }, + "id": 85, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "exemplar": false, + "expr": "topk($TopResources, sum by (operation, service_name, svm, cluster) (external_service_op_num_request_failures{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",service_name=~\"$ServiceName\",operation=~\"$Operation\",key=~\"$TopRequestFailed\"}))", + "interval": "", + "legendFormat": "{{cluster}} - {{svm}} - {{operation}}", + "refId": "A" + } + ], + "title": "Top $TopResources Number of Request Failures", + "type": "timeseries" + }, + { + "description": "This panel shows Number of requests sent to the service.", + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 21 + }, + "id": 87, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "exemplar": false, + "expr": "topk($TopResources, sum by (operation, service_name, svm, cluster) (external_service_op_num_requests_sent{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",service_name=~\"$ServiceName\",operation=~\"$Operation\",key=~\"$TopRequestSent\"}))", + "interval": "", + "legendFormat": "{{cluster}} - {{svm}} - {{operation}}", + "refId": "A" + } + ], + "title": "Top $TopResources Number of Request Sent", + "type": "timeseries" + }, + { + "description": "This panel shows Number of responses received from the server (does not include timeouts).", + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 21 + }, + "id": 89, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "exemplar": false, + "expr": "topk($TopResources, sum by (operation, service_name, svm, cluster) (external_service_op_num_responses_received{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",service_name=~\"$ServiceName\",operation=~\"$Operation\",key=~\"$TopRequestReceived\"}))", + "interval": "", + "legendFormat": "{{cluster}} - {{svm}} - {{operation}}", + "refId": "A" + } + ], + "title": "Top $TopResources Number of Responses Received", + "type": "timeseries" + }, + { + "description": "This panel shows Number of successful responses to the operation.", + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 30 + }, + "id": 91, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "exemplar": false, + "expr": "topk($TopResources, sum by (operation, service_name, svm, cluster) (external_service_op_num_successful_responses{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",service_name=~\"$ServiceName\",operation=~\"$Operation\",key=~\"$TopSuccessResponse\"}))", + "interval": "", + "legendFormat": "{{cluster}} - {{svm}} - {{operation}}", + "refId": "A" + } + ], + "title": "Top $TopResources Number of Successful Responses", + "type": "timeseries" + }, + { + "description": "This panel shows Number of times requests to the server for this operation timed out.", + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 30 + }, + "id": 93, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "exemplar": false, + "expr": "topk($TopResources, sum by (operation, service_name, svm, cluster) (external_service_op_num_timeouts{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",service_name=~\"$ServiceName\",operation=~\"$Operation\",key=~\"$TopTimeout\"}))", + "interval": "", + "legendFormat": "{{cluster}} - {{svm}} - {{operation}}", + "refId": "A" + } + ], + "title": "Top $TopResources Number of Timeouts", + "type": "timeseries" + } + ], + "refresh": "", + "schemaVersion": 35, + "style": "dark", + "tags": [ + "harvest", + "ontap", + "cdot" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 2, + "includeAll": false, + "label": "Data Source", + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": null, + "current": {}, + "definition": "label_values(external_service_op_request_latency{system_type!=\"7mode\"}, datacenter)", + "hide": 0, + "includeAll": false, + "multi": true, + "name": "Datacenter", + "options": [], + "query": { + "query": "label_values(external_service_op_request_latency{system_type!=\"7mode\"}, datacenter)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": {}, + "definition": "label_values(external_service_op_request_latency{system_type!=\"7mode\",datacenter=~\"$Datacenter\"}, cluster)", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "Cluster", + "options": [], + "query": { + "query": "label_values(external_service_op_request_latency{system_type!=\"7mode\",datacenter=~\"$Datacenter\"}, cluster)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": {}, + "definition": "label_values(external_service_op_request_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"}, svm)", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "SVM", + "options": [], + "query": { + "query": "label_values(external_service_op_request_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"}, svm)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": {}, + "definition": "label_values(external_service_op_request_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"}, service_name)", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "ServiceName", + "options": [], + "query": { + "query": "label_values(external_service_op_request_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"}, service_name)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": {}, + "definition": "label_values(external_service_op_request_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",service_name=~\"$ServiceName\"}, operation)", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "Operation", + "options": [], + "query": { + "query": "label_values(external_service_op_request_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",service_name=~\"$ServiceName\"}, operation)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "selected": true, + "text": "5", + "value": "5" + }, + "description": null, + "error": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "TopResources", + "options": [ + { + "selected": false, + "text": "1", + "value": "1" + }, + { + "selected": false, + "text": "2", + "value": "2" + }, + { + "selected": false, + "text": "3", + "value": "3" + }, + { + "selected": false, + "text": "4", + "value": "4" + }, + { + "selected": true, + "text": "5", + "value": "5" + }, + { + "selected": false, + "text": "6", + "value": "6" + }, + { + "selected": false, + "text": "8", + "value": "8" + }, + { + "selected": false, + "text": "10", + "value": "10" + }, + { + "selected": false, + "text": "15", + "value": "15" + }, + { + "selected": false, + "text": "25", + "value": "25" + }, + { + "selected": false, + "text": "50", + "value": "50" + }, + { + "selected": false, + "text": "100", + "value": "100" + }, + { + "selected": false, + "text": "250", + "value": "250" + }, + { + "selected": false, + "text": "500", + "value": "500" + } + ], + "query": "1,2,3,4,5,6,8,10,15,25,50,100,250,500", + "queryValue": "", + "skipUrlSync": false, + "type": "custom" + }, + { + "allValue": null, + "current": {}, + "datasource": null, + "definition": "query_result(topk($TopResources, avg by (key) (avg_over_time(external_service_op_request_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",service_name=~\"$ServiceName\",operation=~\"$Operation\"}[${__range}]))))", + "description": null, + "error": null, + "hide": 2, + "includeAll": true, + "label": null, + "multi": true, + "name": "TopRequestLatency", + "options": [], + "query": { + "query": "query_result(topk($TopResources, avg by (key) (avg_over_time(external_service_op_request_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",service_name=~\"$ServiceName\",operation=~\"$Operation\"}[${__range}]))))", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": ".*key=\\\"(.*?)\\\".*", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": null, + "current": {}, + "datasource": null, + "definition": "query_result(topk($TopResources, sum by (key) (avg_over_time(external_service_op_num_not_found_responses{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",service_name=~\"$ServiceName\",operation=~\"$Operation\"}[${__range}]))))", + "description": null, + "error": null, + "hide": 2, + "includeAll": true, + "label": null, + "multi": true, + "name": "TopNotFoundResponse", + "options": [], + "query": { + "query": "query_result(topk($TopResources, sum by (key) (avg_over_time(external_service_op_num_not_found_responses{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",service_name=~\"$ServiceName\",operation=~\"$Operation\"}[${__range}]))))", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": ".*key=\\\"(.*?)\\\".*", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": null, + "current": {}, + "datasource": null, + "definition": "query_result(topk($TopResources, sum by (key) (avg_over_time(external_service_op_num_request_failures{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",service_name=~\"$ServiceName\",operation=~\"$Operation\"}[${__range}]))))", + "description": null, + "error": null, + "hide": 2, + "includeAll": true, + "label": null, + "multi": true, + "name": "TopRequestFailed", + "options": [], + "query": { + "query": "query_result(topk($TopResources, sum by (key) (avg_over_time(external_service_op_num_request_failures{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",service_name=~\"$ServiceName\",operation=~\"$Operation\"}[${__range}]))))", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": ".*key=\\\"(.*?)\\\".*", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": null, + "current": {}, + "datasource": null, + "definition": "query_result(topk($TopResources, sum by (key) (avg_over_time(external_service_op_num_requests_sent{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",service_name=~\"$ServiceName\",operation=~\"$Operation\"}[${__range}]))))", + "description": null, + "error": null, + "hide": 2, + "includeAll": true, + "label": null, + "multi": true, + "name": "TopRequestSent", + "options": [], + "query": { + "query": "query_result(topk($TopResources, sum by (key) (avg_over_time(external_service_op_num_requests_sent{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",service_name=~\"$ServiceName\",operation=~\"$Operation\"}[${__range}]))))", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": ".*key=\\\"(.*?)\\\".*", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": null, + "current": {}, + "datasource": null, + "definition": "query_result(topk($TopResources, sum by (key) (avg_over_time(external_service_op_num_responses_received{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",service_name=~\"$ServiceName\",operation=~\"$Operation\"}[${__range}]))))", + "description": null, + "error": null, + "hide": 2, + "includeAll": true, + "label": null, + "multi": true, + "name": "TopRequestReceived", + "options": [], + "query": { + "query": "query_result(topk($TopResources, sum by (key) (avg_over_time(external_service_op_num_responses_received{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",service_name=~\"$ServiceName\",operation=~\"$Operation\"}[${__range}]))))", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": ".*key=\\\"(.*?)\\\".*", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": null, + "current": {}, + "datasource": null, + "definition": "query_result(topk($TopResources, sum by (key) (avg_over_time(external_service_op_num_successful_responses{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",service_name=~\"$ServiceName\",operation=~\"$Operation\"}[${__range}]))))", + "description": null, + "error": null, + "hide": 2, + "includeAll": true, + "label": null, + "multi": true, + "name": "TopSuccessResponse", + "options": [], + "query": { + "query": "query_result(topk($TopResources, sum by (key) (avg_over_time(external_service_op_num_successful_responses{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",service_name=~\"$ServiceName\",operation=~\"$Operation\"}[${__range}]))))", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": ".*key=\\\"(.*?)\\\".*", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": null, + "current": {}, + "datasource": null, + "definition": "query_result(topk($TopResources, sum by (key) (avg_over_time(external_service_op_num_timeouts{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",service_name=~\"$ServiceName\",operation=~\"$Operation\"}[${__range}]))))", + "description": null, + "error": null, + "hide": 2, + "includeAll": true, + "label": null, + "multi": true, + "name": "TopTimeout", + "options": [], + "query": { + "query": "query_result(topk($TopResources, sum by (key) (avg_over_time(external_service_op_num_timeouts{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",service_name=~\"$ServiceName\",operation=~\"$Operation\"}[${__range}]))))", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": ".*key=\\\"(.*?)\\\".*", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "ONTAP: External Service Operation", + "uid": "", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/integration/test/dashboard_json_test.go b/integration/test/dashboard_json_test.go index bc2bb722b..66c5bdfa1 100644 --- a/integration/test/dashboard_json_test.go +++ b/integration/test/dashboard_json_test.go @@ -52,6 +52,8 @@ var excludeCounters = []string{ "cluster_peer", "efficiency_savings", "ems_events", + "external_service_op_num_", + "external_service_op_request_", "fabricpool_cloud_bin_op_latency_average", "fabricpool_cloud_bin_operation", "fcp",