fix: exclude node/svm vols and include data svms (#1658)

* fix: exclude node/svm vols and include data svms * fix: exclude node/svm vols and include data svms in rest * fix: add ShowRoot var in dashboard * fix: CI changes for var used in other var * fix: moved root_volume/svm to label from key * fix: handle perf metrics * fix: rename var to includeRoot
NetApp · Jan 25, 2023 · ecbdaeb · ecbdaeb
1 parent e3edf9d
commit ecbdaeb
Show file tree

Hide file tree

Showing 7 changed files with 93 additions and 25 deletions.
diff --git a/cmd/tools/grafana/dashboard_test.go b/cmd/tools/grafana/dashboard_test.go
@@ -332,13 +332,17 @@ func TestNoUnusedVariables(t *testing.T) {
 }
 
 func checkUnusedVariables(t *testing.T, path string, data []byte) {
-	// collect are variable names, except data source
+	// collect are variable names, expressions except data source
 	vars := make([]string, 0)
+	varExpression := make([]string, 0)
 	gjson.GetBytes(data, "templating.list").ForEach(func(key, value gjson.Result) bool {
 		if value.Get("type").String() == "datasource" {
 			return true
 		}
+		// name of variable
 		vars = append(vars, value.Get("name").String())
+		// query expression of variable
+		varExpression = append(varExpression, value.Get("definition").String())
 		return true
 	})
 
@@ -366,6 +370,12 @@ varLoop:
 				continue varLoop
 			}
 		}
+		for _, varExpr := range varExpression {
+			if strings.Contains(varExpr, variable) {
+				continue varLoop
+			}
+		}
+
 		t.Errorf("dashboard=%s has unused variable [%s]", shortPath(path), variable)
 	}
 }

diff --git a/conf/rest/9.12.0/svm.yaml b/conf/rest/9.12.0/svm.yaml
@@ -8,6 +8,7 @@ counters:
   - ^^vserver                               => svm
   - ^anti_ransomware_default_volume_state   => anti_ransomware_state
   - ^operational_state                      => state
+  - ^type                                   => type
 
 endpoints:
   - query: api/svm/svms
@@ -65,6 +66,11 @@ endpoints:
 plugins:
   - SVM
   - LabelAgent:
+      replace:
+        - type root_svm `data` `No`
+        - type root_svm `node` `Yes`
+        - type root_svm `admin` `Yes`
+        - type root_svm `system` `Yes`
       value_to_num:
         - new_status state online online `0`
         - ldap_signed ldap_session_security sign sign `0`
@@ -93,4 +99,6 @@ export_options:
     - nis_authentication_enabled
     - ns_source
     - ns_db
+    - type
+    - root_svm
 
diff --git a/conf/rest/9.12.0/volume.yaml b/conf/rest/9.12.0/volume.yaml
@@ -1,3 +1,5 @@
+# This api would provide all volumes(but node).
+
 name:                     Volume
 query:                    api/storage/volumes
 object:                   volume
@@ -70,12 +72,14 @@ plugins:
       compute_metric:
         - inode_used_percent PERCENT inode_files_used inode_files_total
   - LabelAgent:
-      value_to_num:
-        - new_status state online online `0`
       exclude_equals:
         - style `flexgroup_constituent`
+      value_to_num:
+        - new_status state online online `0`
       replace:
         - style style `flexgroup_constituent` `flexgroup`
+        - svm_root root_volume `false` `No`
+        - svm_root root_volume `true` `Yes`
       # To prevent visibility of transient volumes, uncomment the following lines
 #      exclude_regex:
 #        # Exclude SnapProtect/CommVault Intellisnap, Clone volumes have a “_CVclone” suffix
@@ -110,5 +114,4 @@ export_options:
     - snaplock_type
     - antiRansomwareState
     - isHardwareEncrypted
-
-
+    - root_volume
diff --git a/conf/zapi/cdot/9.8.0/aggr.yaml b/conf/zapi/cdot/9.8.0/aggr.yaml
@@ -61,11 +61,11 @@ counters:
 
 plugins:
   LabelAgent:
+    exclude_equals:
+      - root_aggr `true`
     # metric label zapi_value rest_value `default_value`
     value_to_num:
       - new_status state online online `0`
-    exclude_equals:
-      - root_aggr `true`
 
 export_options:
   instance_keys:

diff --git a/conf/zapi/cdot/9.8.0/svm.yaml b/conf/zapi/cdot/9.8.0/svm.yaml
@@ -8,6 +8,7 @@ counters:
     - ^^uuid                                  => svm_uuid
     - ^vserver-name                           => svm
     - ^state                                  => state
+    - ^vserver-type                           => type
     - ^anti-ransomware-default-volume-state   => anti_ransomware_state
     - name-server-switch:
         - ^nsswitch                           => nameservice_switch
@@ -19,6 +20,11 @@ plugins:
       schedule:
         - data: 900s  # should be multiple of data poll duration
   - LabelAgent:
+      replace:
+        - type root_svm `data` `No`
+        - type root_svm `node` `Yes`
+        - type root_svm `admin` `Yes`
+        - type root_svm `system` `Yes`
       value_to_num:
         - new_status state online online `0`
         - ldap_signed ldap_session_security sign sign `0`
@@ -47,3 +53,6 @@ export_options:
     - nfs_kerberos_protocol_enabled
     - smb_encryption_required
     - smb_signing_required
+    - type
+    - root_svm
+
diff --git a/conf/zapi/cdot/9.8.0/volume.yaml b/conf/zapi/cdot/9.8.0/volume.yaml
@@ -59,6 +59,8 @@ counters:
     - volume-state-attributes:
       - ^state
       - ^status
+      - ^is-node-root                       => node_root
+      - ^is-vserver-root                    => svm_root
 
     - volume-snapshot-attributes:
       - ^auto-snapshots-enabled             => auto_snapshots_enabled
@@ -74,11 +76,11 @@ plugins:
       compute_metric:
         - inode_used_percent PERCENT inode_files_used inode_files_total
   - LabelAgent:
+    exclude_equals:
+      - style `flexgroup_constituent`
     # metric label zapi_value rest_value `default_value`
     value_to_num:
       - new_status state online online `0`
-    exclude_equals:
-      - style `flexgroup_constituent`
     # To prevent visibility of transient volumes, uncomment the following lines
 #    exclude_regex:
 #      # Exclude SnapProtect/CommVault Intellisnap, Clone volumes have a “_CVclone” suffix
@@ -95,6 +97,10 @@ plugins:
 #      - volume `MDV_aud_.+`
     replace:
       - style style `flexgroup_constituent` `flexgroup`
+      - node_root root_volume `false` `No`
+      - svm_root root_volume `false` `No`
+      - node_root root_volume `true` `Yes`
+      - svm_root root_volume `true` `Yes`
   - Aggregator:
     - volume<style=flexgroup>volume node,svm,aggr,style
 
@@ -112,4 +118,7 @@ export_options:
     - type
     - isEncrypted
     - isHardwareEncrypted
+    - svm_root
+    - node_root
+    - root_volume
 
diff --git a/grafana/dashboards/cmode/cdot.json b/grafana/dashboards/cmode/cdot.json
@@ -2027,6 +2027,35 @@
         "skipUrlSync": false,
         "type": "custom"
       },
+      {
+        "current": {
+          "selected": true,
+          "text": "No",
+          "value": "No"
+        },
+        "description": "",
+        "hide": 0,
+        "includeAll": false,
+        "label": "",
+        "multi": false,
+        "name": "IncludeRoot",
+        "options": [
+          {
+            "selected": true,
+            "text": "No",
+            "value": "No"
+          },
+          {
+            "selected": false,
+            "text": "Yes",
+            "value": "Yes"
+          }
+        ],
+        "query": "No,Yes",
+        "queryValue": "",
+        "skipUrlSync": false,
+        "type": "custom"
+      },
       {
         "current": {},
         "definition": "query_result(topk($TopResources, sum by (cluster) (avg_over_time(node_cifs_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"}[${__range}]))))",
@@ -2086,14 +2115,14 @@
       },
       {
         "current": {},
-        "definition": "query_result(topk($TopResources, avg_over_time(volume_avg_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\"}[${__range}])))",
+        "definition": "query_result(topk($TopResources, avg_over_time(volume_avg_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\"}[${__range}]) * on(cluster,svm,volume) (volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",root_volume=~\"No|$IncludeRoot\"})))",
         "hide": 2,
         "includeAll": true,
         "multi": true,
         "name": "TopVolumeAvgLatency",
         "options": [],
         "query": {
-          "query": "query_result(topk($TopResources, avg_over_time(volume_avg_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\"}[${__range}])))",
+          "query": "query_result(topk($TopResources, avg_over_time(volume_avg_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\"}[${__range}]) * on(cluster,svm,volume) (volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",root_volume=~\"No|$IncludeRoot\"})))",
           "refId": "StandardVariableQuery"
         },
         "refresh": 1,
@@ -2104,14 +2133,14 @@
       },
       {
         "current": {},
-        "definition": "query_result(topk($TopResources, avg_over_time(volume_read_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\"}[${__range}])+avg_over_time(volume_write_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\"}[${__range}])))",
+        "definition": "query_result(topk($TopResources, (avg_over_time(volume_read_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\"}[${__range}])+avg_over_time(volume_write_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\"}[${__range}])) * on(cluster,svm,volume) (volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",root_volume=~\"No|$IncludeRoot\"})))",
         "hide": 2,
         "includeAll": true,
         "multi": true,
         "name": "TopVolumeAvgThroughput",
         "options": [],
         "query": {
-          "query": "query_result(topk($TopResources, avg_over_time(volume_read_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\"}[${__range}])+avg_over_time(volume_write_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\"}[${__range}])))",
+          "query": "query_result(topk($TopResources, (avg_over_time(volume_read_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\"}[${__range}])+avg_over_time(volume_write_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\"}[${__range}])) * on(cluster,svm,volume) (volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",root_volume=~\"No|$IncludeRoot\"})))",
           "refId": "StandardVariableQuery"
         },
         "refresh": 1,
@@ -2122,14 +2151,14 @@
       },
       {
         "current": {},
-        "definition": "query_result(topk($TopResources, avg_over_time(volume_total_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\"}[${__range}])))",
+        "definition": "query_result(topk($TopResources, avg_over_time(volume_total_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\"}[${__range}]) * on(cluster,svm,volume) (volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",root_volume=~\"No|$IncludeRoot\"})))",
         "hide": 2,
         "includeAll": true,
         "multi": true,
         "name": "TopVolumeTotalIops",
         "options": [],
         "query": {
-          "query": "query_result(topk($TopResources, avg_over_time(volume_total_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\"}[${__range}])))",
+          "query": "query_result(topk($TopResources, avg_over_time(volume_total_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\"}[${__range}]) * on(cluster,svm,volume) (volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",root_volume=~\"No|$IncludeRoot\"})))",
           "refId": "StandardVariableQuery"
         },
         "refresh": 1,
@@ -2140,14 +2169,14 @@
       },
       {
         "current": {},
-        "definition": "query_result(topk($TopResources, avg_over_time(svm_vol_avg_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"}[${__range}])))",
+        "definition": "query_result(topk($TopResources, avg_over_time(svm_vol_avg_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"}[${__range}]) * on(cluster,svm) (svm_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",root_svm=~\"No|$IncludeRoot\"})))",
         "hide": 2,
         "includeAll": true,
         "multi": true,
         "name": "TopSVMAvgLatency",
         "options": [],
         "query": {
-          "query": "query_result(topk($TopResources, avg_over_time(svm_vol_avg_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"}[${__range}])))",
+          "query": "query_result(topk($TopResources, avg_over_time(svm_vol_avg_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"}[${__range}]) * on(cluster,svm) (svm_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",root_svm=~\"No|$IncludeRoot\"})))",
           "refId": "StandardVariableQuery"
         },
         "refresh": 1,
@@ -2158,14 +2187,14 @@
       },
       {
         "current": {},
-        "definition": "query_result(topk($TopResources, avg_over_time(svm_vol_read_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"}[${__range}])+avg_over_time(svm_vol_write_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"}[${__range}])))",
+        "definition": "query_result(topk($TopResources, (avg_over_time(svm_vol_read_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"}[${__range}])+avg_over_time(svm_vol_write_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"}[${__range}])) * on(cluster,svm) (svm_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",root_svm=~\"No|$IncludeRoot\"})))",
         "hide": 2,
         "includeAll": true,
         "multi": true,
         "name": "TopSVMTotalThroughput",
         "options": [],
         "query": {
-          "query": "query_result(topk($TopResources, avg_over_time(svm_vol_read_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"}[${__range}])+avg_over_time(svm_vol_write_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"}[${__range}])))",
+          "query": "query_result(topk($TopResources, (avg_over_time(svm_vol_read_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"}[${__range}])+avg_over_time(svm_vol_write_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"}[${__range}])) * on(cluster,svm) (svm_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",root_svm=~\"No|$IncludeRoot\"})))",
           "refId": "StandardVariableQuery"
         },
         "refresh": 1,
@@ -2176,14 +2205,14 @@
       },
       {
         "current": {},
-        "definition": "query_result(topk($TopResources, avg_over_time(svm_vol_total_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"}[${__range}])))",
+        "definition": "query_result(topk($TopResources, avg_over_time(svm_vol_total_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"}[${__range}]) * on(cluster,svm) (svm_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",root_svm=~\"No|$IncludeRoot\"})))",
         "hide": 2,
         "includeAll": true,
         "multi": true,
         "name": "TopSVMTotalIops",
         "options": [],
         "query": {
-          "query": "query_result(topk($TopResources, avg_over_time(svm_vol_total_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"}[${__range}])))",
+          "query": "query_result(topk($TopResources, avg_over_time(svm_vol_total_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"}[${__range}]) * on(cluster,svm) (svm_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",root_svm=~\"No|$IncludeRoot\"})))",
           "refId": "StandardVariableQuery"
         },
         "refresh": 1,
@@ -2194,14 +2223,14 @@
       },
       {
         "current": {},
-        "definition": "query_result(topk($TopResources,(100 * sum by (cluster, svm) (volume_size_used{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"}) /   sum by (cluster, svm) (volume_size_total{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"}))))",
+        "definition": "query_result(topk($TopResources,((100 * sum by (cluster, svm) (volume_size_used{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"}) /   sum by (cluster, svm) (volume_size_total{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"})) * on(cluster,svm) (svm_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",root_svm=~\"No|$IncludeRoot\"}))))",
         "hide": 2,
         "includeAll": true,
         "multi": true,
         "name": "TopSVMUsed",
         "options": [],
         "query": {
-          "query": "query_result(topk($TopResources,(100 * sum by (cluster, svm) (volume_size_used{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"}) /   sum by (cluster, svm) (volume_size_total{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"}))))",
+          "query": "query_result(topk($TopResources,((100 * sum by (cluster, svm) (volume_size_used{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"}) /   sum by (cluster, svm) (volume_size_total{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"})) * on(cluster,svm) (svm_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",root_svm=~\"No|$IncludeRoot\"}))))",
           "refId": "StandardVariableQuery"
         },
         "refresh": 1,
@@ -2212,14 +2241,14 @@
       },
       {
         "current": {},
-        "definition": "query_result(topk($TopResources, (100 * volume_size_used{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\"} / volume_size_total{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\"})))",
+        "definition": "query_result(topk($TopResources, ((100 * volume_size_used{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"} / volume_size_total{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\"}) * on(cluster,svm,volume) (volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",root_volume=~\"No|$IncludeRoot\"}))))",
         "hide": 2,
         "includeAll": true,
         "multi": true,
         "name": "TopVolumeUsed",
         "options": [],
         "query": {
-          "query": "query_result(topk($TopResources, (100 * volume_size_used{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\"} / volume_size_total{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\"})))",
+          "query": "query_result(topk($TopResources, ((100 * volume_size_used{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"} / volume_size_total{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\"}) * on(cluster,svm,volume) (volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",root_volume=~\"No|$IncludeRoot\"}))))",
           "refId": "StandardVariableQuery"
         },
         "refresh": 1,