From ecbdaeb84ecbb37f4a5e27b828e644a4ff65e558 Mon Sep 17 00:00:00 2001 From: Hardikl <83282894+Hardikl@users.noreply.github.com> Date: Wed, 25 Jan 2023 13:32:30 +0530 Subject: [PATCH] fix: exclude node/svm vols and include data svms (#1658) * fix: exclude node/svm vols and include data svms * fix: exclude node/svm vols and include data svms in rest * fix: add ShowRoot var in dashboard * fix: CI changes for var used in other var * fix: moved root_volume/svm to label from key * fix: handle perf metrics * fix: rename var to includeRoot --- cmd/tools/grafana/dashboard_test.go | 12 +++++- conf/rest/9.12.0/svm.yaml | 8 ++++ conf/rest/9.12.0/volume.yaml | 11 ++++-- conf/zapi/cdot/9.8.0/aggr.yaml | 4 +- conf/zapi/cdot/9.8.0/svm.yaml | 9 +++++ conf/zapi/cdot/9.8.0/volume.yaml | 13 +++++- grafana/dashboards/cmode/cdot.json | 61 +++++++++++++++++++++-------- 7 files changed, 93 insertions(+), 25 deletions(-) diff --git a/cmd/tools/grafana/dashboard_test.go b/cmd/tools/grafana/dashboard_test.go index 91339ff22..6b713d2ae 100644 --- a/cmd/tools/grafana/dashboard_test.go +++ b/cmd/tools/grafana/dashboard_test.go @@ -332,13 +332,17 @@ func TestNoUnusedVariables(t *testing.T) { } func checkUnusedVariables(t *testing.T, path string, data []byte) { - // collect are variable names, except data source + // collect are variable names, expressions except data source vars := make([]string, 0) + varExpression := make([]string, 0) gjson.GetBytes(data, "templating.list").ForEach(func(key, value gjson.Result) bool { if value.Get("type").String() == "datasource" { return true } + // name of variable vars = append(vars, value.Get("name").String()) + // query expression of variable + varExpression = append(varExpression, value.Get("definition").String()) return true }) @@ -366,6 +370,12 @@ varLoop: continue varLoop } } + for _, varExpr := range varExpression { + if strings.Contains(varExpr, variable) { + continue varLoop + } + } + t.Errorf("dashboard=%s has unused variable [%s]", shortPath(path), variable) } } diff --git a/conf/rest/9.12.0/svm.yaml b/conf/rest/9.12.0/svm.yaml index 587ebf6d6..8177d02e5 100644 --- a/conf/rest/9.12.0/svm.yaml +++ b/conf/rest/9.12.0/svm.yaml @@ -8,6 +8,7 @@ counters: - ^^vserver => svm - ^anti_ransomware_default_volume_state => anti_ransomware_state - ^operational_state => state + - ^type => type endpoints: - query: api/svm/svms @@ -65,6 +66,11 @@ endpoints: plugins: - SVM - LabelAgent: + replace: + - type root_svm `data` `No` + - type root_svm `node` `Yes` + - type root_svm `admin` `Yes` + - type root_svm `system` `Yes` value_to_num: - new_status state online online `0` - ldap_signed ldap_session_security sign sign `0` @@ -93,4 +99,6 @@ export_options: - nis_authentication_enabled - ns_source - ns_db + - type + - root_svm diff --git a/conf/rest/9.12.0/volume.yaml b/conf/rest/9.12.0/volume.yaml index 1b5c26b4d..d58f99253 100644 --- a/conf/rest/9.12.0/volume.yaml +++ b/conf/rest/9.12.0/volume.yaml @@ -1,3 +1,5 @@ +# This api would provide all volumes(but node). + name: Volume query: api/storage/volumes object: volume @@ -70,12 +72,14 @@ plugins: compute_metric: - inode_used_percent PERCENT inode_files_used inode_files_total - LabelAgent: - value_to_num: - - new_status state online online `0` exclude_equals: - style `flexgroup_constituent` + value_to_num: + - new_status state online online `0` replace: - style style `flexgroup_constituent` `flexgroup` + - svm_root root_volume `false` `No` + - svm_root root_volume `true` `Yes` # To prevent visibility of transient volumes, uncomment the following lines # exclude_regex: # # Exclude SnapProtect/CommVault Intellisnap, Clone volumes have a “_CVclone” suffix @@ -110,5 +114,4 @@ export_options: - snaplock_type - antiRansomwareState - isHardwareEncrypted - - + - root_volume diff --git a/conf/zapi/cdot/9.8.0/aggr.yaml b/conf/zapi/cdot/9.8.0/aggr.yaml index 214aecbe1..49e1b8c14 100644 --- a/conf/zapi/cdot/9.8.0/aggr.yaml +++ b/conf/zapi/cdot/9.8.0/aggr.yaml @@ -61,11 +61,11 @@ counters: plugins: LabelAgent: + exclude_equals: + - root_aggr `true` # metric label zapi_value rest_value `default_value` value_to_num: - new_status state online online `0` - exclude_equals: - - root_aggr `true` export_options: instance_keys: diff --git a/conf/zapi/cdot/9.8.0/svm.yaml b/conf/zapi/cdot/9.8.0/svm.yaml index 925223fa7..71425856d 100644 --- a/conf/zapi/cdot/9.8.0/svm.yaml +++ b/conf/zapi/cdot/9.8.0/svm.yaml @@ -8,6 +8,7 @@ counters: - ^^uuid => svm_uuid - ^vserver-name => svm - ^state => state + - ^vserver-type => type - ^anti-ransomware-default-volume-state => anti_ransomware_state - name-server-switch: - ^nsswitch => nameservice_switch @@ -19,6 +20,11 @@ plugins: schedule: - data: 900s # should be multiple of data poll duration - LabelAgent: + replace: + - type root_svm `data` `No` + - type root_svm `node` `Yes` + - type root_svm `admin` `Yes` + - type root_svm `system` `Yes` value_to_num: - new_status state online online `0` - ldap_signed ldap_session_security sign sign `0` @@ -47,3 +53,6 @@ export_options: - nfs_kerberos_protocol_enabled - smb_encryption_required - smb_signing_required + - type + - root_svm + diff --git a/conf/zapi/cdot/9.8.0/volume.yaml b/conf/zapi/cdot/9.8.0/volume.yaml index 8ee6b6d66..ca35a6f63 100644 --- a/conf/zapi/cdot/9.8.0/volume.yaml +++ b/conf/zapi/cdot/9.8.0/volume.yaml @@ -59,6 +59,8 @@ counters: - volume-state-attributes: - ^state - ^status + - ^is-node-root => node_root + - ^is-vserver-root => svm_root - volume-snapshot-attributes: - ^auto-snapshots-enabled => auto_snapshots_enabled @@ -74,11 +76,11 @@ plugins: compute_metric: - inode_used_percent PERCENT inode_files_used inode_files_total - LabelAgent: + exclude_equals: + - style `flexgroup_constituent` # metric label zapi_value rest_value `default_value` value_to_num: - new_status state online online `0` - exclude_equals: - - style `flexgroup_constituent` # To prevent visibility of transient volumes, uncomment the following lines # exclude_regex: # # Exclude SnapProtect/CommVault Intellisnap, Clone volumes have a “_CVclone” suffix @@ -95,6 +97,10 @@ plugins: # - volume `MDV_aud_.+` replace: - style style `flexgroup_constituent` `flexgroup` + - node_root root_volume `false` `No` + - svm_root root_volume `false` `No` + - node_root root_volume `true` `Yes` + - svm_root root_volume `true` `Yes` - Aggregator: - volumevolume node,svm,aggr,style @@ -112,4 +118,7 @@ export_options: - type - isEncrypted - isHardwareEncrypted + - svm_root + - node_root + - root_volume diff --git a/grafana/dashboards/cmode/cdot.json b/grafana/dashboards/cmode/cdot.json index 8edc2ec30..d00260bd9 100644 --- a/grafana/dashboards/cmode/cdot.json +++ b/grafana/dashboards/cmode/cdot.json @@ -2027,6 +2027,35 @@ "skipUrlSync": false, "type": "custom" }, + { + "current": { + "selected": true, + "text": "No", + "value": "No" + }, + "description": "", + "hide": 0, + "includeAll": false, + "label": "", + "multi": false, + "name": "IncludeRoot", + "options": [ + { + "selected": true, + "text": "No", + "value": "No" + }, + { + "selected": false, + "text": "Yes", + "value": "Yes" + } + ], + "query": "No,Yes", + "queryValue": "", + "skipUrlSync": false, + "type": "custom" + }, { "current": {}, "definition": "query_result(topk($TopResources, sum by (cluster) (avg_over_time(node_cifs_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"}[${__range}]))))", @@ -2086,14 +2115,14 @@ }, { "current": {}, - "definition": "query_result(topk($TopResources, avg_over_time(volume_avg_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\"}[${__range}])))", + "definition": "query_result(topk($TopResources, avg_over_time(volume_avg_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\"}[${__range}]) * on(cluster,svm,volume) (volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",root_volume=~\"No|$IncludeRoot\"})))", "hide": 2, "includeAll": true, "multi": true, "name": "TopVolumeAvgLatency", "options": [], "query": { - "query": "query_result(topk($TopResources, avg_over_time(volume_avg_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\"}[${__range}])))", + "query": "query_result(topk($TopResources, avg_over_time(volume_avg_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\"}[${__range}]) * on(cluster,svm,volume) (volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",root_volume=~\"No|$IncludeRoot\"})))", "refId": "StandardVariableQuery" }, "refresh": 1, @@ -2104,14 +2133,14 @@ }, { "current": {}, - "definition": "query_result(topk($TopResources, avg_over_time(volume_read_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\"}[${__range}])+avg_over_time(volume_write_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\"}[${__range}])))", + "definition": "query_result(topk($TopResources, (avg_over_time(volume_read_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\"}[${__range}])+avg_over_time(volume_write_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\"}[${__range}])) * on(cluster,svm,volume) (volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",root_volume=~\"No|$IncludeRoot\"})))", "hide": 2, "includeAll": true, "multi": true, "name": "TopVolumeAvgThroughput", "options": [], "query": { - "query": "query_result(topk($TopResources, avg_over_time(volume_read_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\"}[${__range}])+avg_over_time(volume_write_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\"}[${__range}])))", + "query": "query_result(topk($TopResources, (avg_over_time(volume_read_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\"}[${__range}])+avg_over_time(volume_write_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\"}[${__range}])) * on(cluster,svm,volume) (volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",root_volume=~\"No|$IncludeRoot\"})))", "refId": "StandardVariableQuery" }, "refresh": 1, @@ -2122,14 +2151,14 @@ }, { "current": {}, - "definition": "query_result(topk($TopResources, avg_over_time(volume_total_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\"}[${__range}])))", + "definition": "query_result(topk($TopResources, avg_over_time(volume_total_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\"}[${__range}]) * on(cluster,svm,volume) (volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",root_volume=~\"No|$IncludeRoot\"})))", "hide": 2, "includeAll": true, "multi": true, "name": "TopVolumeTotalIops", "options": [], "query": { - "query": "query_result(topk($TopResources, avg_over_time(volume_total_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\"}[${__range}])))", + "query": "query_result(topk($TopResources, avg_over_time(volume_total_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\"}[${__range}]) * on(cluster,svm,volume) (volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",root_volume=~\"No|$IncludeRoot\"})))", "refId": "StandardVariableQuery" }, "refresh": 1, @@ -2140,14 +2169,14 @@ }, { "current": {}, - "definition": "query_result(topk($TopResources, avg_over_time(svm_vol_avg_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"}[${__range}])))", + "definition": "query_result(topk($TopResources, avg_over_time(svm_vol_avg_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"}[${__range}]) * on(cluster,svm) (svm_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",root_svm=~\"No|$IncludeRoot\"})))", "hide": 2, "includeAll": true, "multi": true, "name": "TopSVMAvgLatency", "options": [], "query": { - "query": "query_result(topk($TopResources, avg_over_time(svm_vol_avg_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"}[${__range}])))", + "query": "query_result(topk($TopResources, avg_over_time(svm_vol_avg_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"}[${__range}]) * on(cluster,svm) (svm_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",root_svm=~\"No|$IncludeRoot\"})))", "refId": "StandardVariableQuery" }, "refresh": 1, @@ -2158,14 +2187,14 @@ }, { "current": {}, - "definition": "query_result(topk($TopResources, avg_over_time(svm_vol_read_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"}[${__range}])+avg_over_time(svm_vol_write_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"}[${__range}])))", + "definition": "query_result(topk($TopResources, (avg_over_time(svm_vol_read_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"}[${__range}])+avg_over_time(svm_vol_write_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"}[${__range}])) * on(cluster,svm) (svm_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",root_svm=~\"No|$IncludeRoot\"})))", "hide": 2, "includeAll": true, "multi": true, "name": "TopSVMTotalThroughput", "options": [], "query": { - "query": "query_result(topk($TopResources, avg_over_time(svm_vol_read_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"}[${__range}])+avg_over_time(svm_vol_write_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"}[${__range}])))", + "query": "query_result(topk($TopResources, (avg_over_time(svm_vol_read_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"}[${__range}])+avg_over_time(svm_vol_write_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"}[${__range}])) * on(cluster,svm) (svm_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",root_svm=~\"No|$IncludeRoot\"})))", "refId": "StandardVariableQuery" }, "refresh": 1, @@ -2176,14 +2205,14 @@ }, { "current": {}, - "definition": "query_result(topk($TopResources, avg_over_time(svm_vol_total_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"}[${__range}])))", + "definition": "query_result(topk($TopResources, avg_over_time(svm_vol_total_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"}[${__range}]) * on(cluster,svm) (svm_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",root_svm=~\"No|$IncludeRoot\"})))", "hide": 2, "includeAll": true, "multi": true, "name": "TopSVMTotalIops", "options": [], "query": { - "query": "query_result(topk($TopResources, avg_over_time(svm_vol_total_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"}[${__range}])))", + "query": "query_result(topk($TopResources, avg_over_time(svm_vol_total_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"}[${__range}]) * on(cluster,svm) (svm_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",root_svm=~\"No|$IncludeRoot\"})))", "refId": "StandardVariableQuery" }, "refresh": 1, @@ -2194,14 +2223,14 @@ }, { "current": {}, - "definition": "query_result(topk($TopResources,(100 * sum by (cluster, svm) (volume_size_used{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"}) / sum by (cluster, svm) (volume_size_total{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"}))))", + "definition": "query_result(topk($TopResources,((100 * sum by (cluster, svm) (volume_size_used{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"}) / sum by (cluster, svm) (volume_size_total{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"})) * on(cluster,svm) (svm_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",root_svm=~\"No|$IncludeRoot\"}))))", "hide": 2, "includeAll": true, "multi": true, "name": "TopSVMUsed", "options": [], "query": { - "query": "query_result(topk($TopResources,(100 * sum by (cluster, svm) (volume_size_used{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"}) / sum by (cluster, svm) (volume_size_total{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"}))))", + "query": "query_result(topk($TopResources,((100 * sum by (cluster, svm) (volume_size_used{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"}) / sum by (cluster, svm) (volume_size_total{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"})) * on(cluster,svm) (svm_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",root_svm=~\"No|$IncludeRoot\"}))))", "refId": "StandardVariableQuery" }, "refresh": 1, @@ -2212,14 +2241,14 @@ }, { "current": {}, - "definition": "query_result(topk($TopResources, (100 * volume_size_used{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\"} / volume_size_total{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\"})))", + "definition": "query_result(topk($TopResources, ((100 * volume_size_used{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"} / volume_size_total{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\"}) * on(cluster,svm,volume) (volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",root_volume=~\"No|$IncludeRoot\"}))))", "hide": 2, "includeAll": true, "multi": true, "name": "TopVolumeUsed", "options": [], "query": { - "query": "query_result(topk($TopResources, (100 * volume_size_used{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\"} / volume_size_total{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\"})))", + "query": "query_result(topk($TopResources, ((100 * volume_size_used{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"} / volume_size_total{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\"}) * on(cluster,svm,volume) (volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",root_volume=~\"No|$IncludeRoot\"}))))", "refId": "StandardVariableQuery" }, "refresh": 1,