Skip to content

Commit

Permalink
feat: collect external_service_op perf object (#1941)
Browse files Browse the repository at this point in the history
* feat: collect external_service_op perf object

* feat: collect external_service_op perf object

* feat: dashboard changes

* feat: CI changes

* feat: CI changes

* feat: added new panels in svm dashboard

* feat: moved new panels to new dashboard

* feat: merge main

* feat: adding operation var, aggregation at servicename and operation

* fix: added unique key for topk

* fix: minor changes

* fix: few name correction

* fix: topk default to 5
  • Loading branch information
Hardikl committed Apr 27, 2023
1 parent cacfd7a commit e79e838
Show file tree
Hide file tree
Showing 7 changed files with 1,327 additions and 47 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
/*
* Copyright NetApp Inc, 2023 All rights reserved
*/
package externalserviceoperation

import (
"github.com/netapp/harvest/v2/cmd/poller/plugin"
"github.com/netapp/harvest/v2/pkg/matrix"
)

const Hyphen = "-"

type ExternalServiceOperation struct {
*plugin.AbstractPlugin
}

func New(p *plugin.AbstractPlugin) plugin.Plugin {
return &ExternalServiceOperation{AbstractPlugin: p}
}

func (e *ExternalServiceOperation) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) {
data := dataMap[e.Object]
datacenterClusterKey := data.GetGlobalLabels().Get("datacenter") + Hyphen + data.GetGlobalLabels().Get("cluster") + Hyphen
for _, instance := range data.GetInstances() {
// generate unique key by appending datacenter, cluster, svm, service_name and operation to support topk in grafana dashboard
key := datacenterClusterKey + instance.GetLabel("svm") + Hyphen + instance.GetLabel("service_name") + Hyphen + instance.GetLabel("operation")
instance.SetLabel("key", key)
}
return nil, nil
}
3 changes: 3 additions & 0 deletions cmd/collectors/zapiperf/zapiperf.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ package zapiperf
import (
"errors"
"github.com/netapp/harvest/v2/cmd/collectors/zapiperf/plugins/disk"
"github.com/netapp/harvest/v2/cmd/collectors/zapiperf/plugins/externalserviceoperation"
"github.com/netapp/harvest/v2/cmd/collectors/zapiperf/plugins/fcp"
"github.com/netapp/harvest/v2/cmd/collectors/zapiperf/plugins/headroom"
"github.com/netapp/harvest/v2/cmd/collectors/zapiperf/plugins/nic"
Expand Down Expand Up @@ -125,6 +126,8 @@ func (z *ZapiPerf) LoadPlugin(kind string, abc *plugin.AbstractPlugin) plugin.Pl
return vscan.New(abc)
case "Disk":
return disk.New(abc)
case "ExternalServiceOperation":
return externalserviceoperation.New(abc)
default:
z.Logger.Info().Msgf("no zapiPerf plugin found for %s", kind)
}
Expand Down
8 changes: 8 additions & 0 deletions cmd/tools/grafana/dashboard_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -484,6 +484,7 @@ func TestIDIsBlank(t *testing.T) {
[]string{"../../../grafana/dashboards/cmode", "../../../grafana/dashboards/storagegrid"},
func(path string, data []byte) {
checkUIDIsBlank(t, path, data)
checkIDIsNull(t, path, data)
})
}

Expand All @@ -508,6 +509,13 @@ func checkUIDIsBlank(t *testing.T, path string, data []byte) {
}
}

func checkIDIsNull(t *testing.T, path string, data []byte) {
id := gjson.GetBytes(data, "id").String()
if id != "" {
t.Errorf(`dashboard=%s id should be null but is %s`, shortPath(path), id)
}
}

func TestUniquePanelIDs(t *testing.T) {
visitDashboards(
[]string{"../../../grafana/dashboards/cmode", "../../../grafana/dashboards/storagegrid"},
Expand Down
39 changes: 39 additions & 0 deletions conf/zapiperf/cdot/9.8.0/external_service_operation.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@

name: ExternalServiceOperation
query: external_service_op
object: external_service_op

instance_key: uuid

counters:
- instance_name
- instance_uuid
- node_name => node
- num_not_found_responses
- num_request_failures
- num_requests_sent
- num_responses_received
- num_successful_responses
- num_timeouts
- operation
- process_name
- request_latency
- request_latency_hist
- server_ip_address
- server_name
- service_name
- vserver_name => svm

plugins:
- ExternalServiceOperation

export_options:
instance_keys:
- key
- node
- operation
- process_name
- server_ip_address
- server_name
- service_name
- svm
95 changes: 48 additions & 47 deletions conf/zapiperf/default.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -9,54 +9,55 @@ schedule:

objects:
# Node-level metrics
CIFSNode: cifs_node.yaml
Disk: disk.yaml
ExtCacheObj: ext_cache_obj.yaml
FCVI: fcvi.yaml
FcpPort: fcp.yaml
HeadroomAggr: resource_headroom_aggr.yaml
HeadroomCPU: resource_headroom_cpu.yaml
HostAdapter: hostadapter.yaml
# Netstat: netstat.yaml
NFSv3Node: nfsv3_node.yaml
NFSv41Node: nfsv4_1_node.yaml
NFSv42Node: nfsv4_2_node.yaml
NFSv4Node: nfsv4_node.yaml
NVMfLif: nvmf_lif.yaml
Namespace: namespace.yaml
NicCommon: nic_common.yaml
ObjectStoreClient: object_store_client_op.yaml
Path: path.yaml
Qtree: qtree.yaml
SystemNode: system_node.yaml
TokenManager: token_manager.yaml
VolumeNode: volume_node.yaml
WAFL: wafl.yaml
WAFLAggr: wafl_hya_per_aggr.yaml
WAFLSizer: wafl_hya_sizer.yaml
# NFSv4Pool: nfsv4_pool.yaml
CIFSNode: cifs_node.yaml
Disk: disk.yaml
ExtCacheObj: ext_cache_obj.yaml
FCVI: fcvi.yaml
FcpPort: fcp.yaml
HeadroomAggr: resource_headroom_aggr.yaml
HeadroomCPU: resource_headroom_cpu.yaml
HostAdapter: hostadapter.yaml
# Netstat: netstat.yaml
NFSv3Node: nfsv3_node.yaml
NFSv41Node: nfsv4_1_node.yaml
NFSv42Node: nfsv4_2_node.yaml
NFSv4Node: nfsv4_node.yaml
NVMfLif: nvmf_lif.yaml
Namespace: namespace.yaml
NicCommon: nic_common.yaml
ObjectStoreClient: object_store_client_op.yaml
Path: path.yaml
Qtree: qtree.yaml
SystemNode: system_node.yaml
TokenManager: token_manager.yaml
VolumeNode: volume_node.yaml
WAFL: wafl.yaml
WAFLAggr: wafl_hya_per_aggr.yaml
WAFLSizer: wafl_hya_sizer.yaml
# NFSv4Pool: nfsv4_pool.yaml

# SVM-level metrics
CIFSvserver: cifs_vserver.yaml
CopyManager: copy_manager.yaml
FcpLif: fcp_lif.yaml
ISCSI: iscsi_lif.yaml
LIF: lif.yaml
Lun: lun.yaml
NFSv3: nfsv3.yaml
NFSv41: nfsv4_1.yaml
NFSv42: nfsv4_2.yaml
NFSv4: nfsv4.yaml
OntapS3SVM: ontap_s3_svm.yaml
SMB2: smb2.yaml
Volume: volume.yaml
VolumeSvm: volume_svm.yaml
WAFLCompBin: wafl_comp_aggr_vol_bin.yaml
Vscan: vscan.yaml
VscanSVM: vscan_svm.yaml
CIFSvserver: cifs_vserver.yaml
CopyManager: copy_manager.yaml
# ExternalServiceOperation: external_service_operation.yaml
FcpLif: fcp_lif.yaml
ISCSI: iscsi_lif.yaml
LIF: lif.yaml
Lun: lun.yaml
NFSv3: nfsv3.yaml
NFSv41: nfsv4_1.yaml
NFSv42: nfsv4_2.yaml
NFSv4: nfsv4.yaml
OntapS3SVM: ontap_s3_svm.yaml
SMB2: smb2.yaml
Volume: volume.yaml
VolumeSvm: volume_svm.yaml
WAFLCompBin: wafl_comp_aggr_vol_bin.yaml
Vscan: vscan.yaml
VscanSVM: vscan_svm.yaml

# Uncomment to collect workload/QOS counters
# Workload: workload.yaml
# WorkloadDetail: workload_detail.yaml
# WorkloadVolume: workload_volume.yaml
# WorkloadDetailVolume: workload_detail_volume.yaml
# Workload: workload.yaml
# WorkloadDetail: workload_detail.yaml
# WorkloadVolume: workload_volume.yaml
# WorkloadDetailVolume: workload_detail_volume.yaml
Loading

0 comments on commit e79e838

Please sign in to comment.