Skip to content

Commit

Permalink
feat: support flexgroup constituents in template (#2410)
Browse files Browse the repository at this point in the history 
* feat: handled arw and constituents fields for cluster lower than 9.10
* feat: handled disk api resposne in zapi/rest
* feat: default include constituents to false
  • Loading branch information
@Hardikl
Hardikl committed Oct 17, 2023
1 parent 344f8d4 commit bcd09c6
Show file tree
Hide file tree
Showing 13 changed files with 3,479 additions and 407 deletions.
10 changes: 10 additions & 0 deletions cmd/collectors/commonutils.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"github.com/netapp/harvest/v2/pkg/matrix"
"github.com/netapp/harvest/v2/pkg/tree/node"
"github.com/tidwall/gjson"
"strconv"
"strings"
"time"
)
Expand Down Expand Up @@ -234,3 +235,12 @@ func UpdateLagTime(instance *matrix.Instance, lastTransferSize *matrix.Metric, l
func IsValidUnit(unit string) bool {
return validUnits[unit]
}

func ReadPluginKey(param *node.Node, key string) bool {
if val := param.GetChildContentS(key); val != "" {
if boolValue, err := strconv.ParseBool(val); err == nil {
return boolValue
}
}
return false
}
207 changes: 151 additions & 56 deletions cmd/collectors/rest/plugins/volume/volume.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
package volume

import (
"github.com/hashicorp/go-version"
"github.com/netapp/harvest/v2/cmd/collectors"
"github.com/netapp/harvest/v2/cmd/poller/plugin"
"github.com/netapp/harvest/v2/cmd/tools/rest"
Expand All @@ -18,113 +19,154 @@ import (
)

const HoursInMonth = 24 * 30
const ARWSupportedVersion = "9.10.0"

type Volume struct {
*plugin.AbstractPlugin
currentVal int
client *rest.Client
aggrsMap map[string]string // aggregate-uuid -> aggregate-name map
arw *matrix.Matrix
currentVal int
client *rest.Client
aggrsMap map[string]bool // aggregate-name -> exist map
arw *matrix.Matrix
includeConstituents bool
isArwSupportedVersion bool
}

type volumeInfo struct {
arwStartTime string
arwState string
snapshotAutodelete string
cloneSnapshotName string
cloneSplitEstimateMetric float64
}

func New(p *plugin.AbstractPlugin) plugin.Plugin {
return &Volume{AbstractPlugin: p}
}

func (my *Volume) Init() error {
func (v *Volume) Init() error {

var err error

if err = my.InitAbc(); err != nil {
if err = v.InitAbc(); err != nil {
return err
}

my.aggrsMap = make(map[string]string)
v.aggrsMap = make(map[string]bool)

// Assigned the value to currentVal so that plugin would be invoked first time to populate cache.
my.currentVal = my.SetPluginInterval()
v.currentVal = v.SetPluginInterval()

if my.Options.IsTest {
if v.Options.IsTest {
return nil
}

timeout, _ := time.ParseDuration(rest.DefaultTimeout)
if my.client, err = rest.New(conf.ZapiPoller(my.ParentParams), timeout, my.Auth); err != nil {
my.Logger.Error().Stack().Err(err).Msg("connecting")
if v.client, err = rest.New(conf.ZapiPoller(v.ParentParams), timeout, v.Auth); err != nil {
v.Logger.Error().Stack().Err(err).Msg("connecting")
return err
}

if err = my.client.Init(5); err != nil {
if err = v.client.Init(5); err != nil {
return err
}

my.arw = matrix.New(my.Parent+".Volume", "volume_arw", "volume_arw")
v.arw = matrix.New(v.Parent+".Volume", "volume_arw", "volume_arw")
exportOptions := node.NewS("export_options")
instanceKeys := exportOptions.NewChildS("instance_keys", "")
instanceKeys.NewChildS("", "ArwStatus")
my.arw.SetExportOptions(exportOptions)
_, err = my.arw.NewMetricFloat64("status", "status")
v.arw.SetExportOptions(exportOptions)
_, err = v.arw.NewMetricFloat64("status", "status")
if err != nil {
my.Logger.Error().Stack().Err(err).Msg("add metric")
v.Logger.Error().Stack().Err(err).Msg("add metric")
return err
}

// Read template to decide inclusion of flexgroup constituents
v.includeConstituents = collectors.ReadPluginKey(v.Params, "include_constituents")
// ARW feature is supported from 9.10 onwards, If we ask this field in Rest call in plugin, then it will be failed.
v.isArwSupportedVersion = v.versionHigherThan(ARWSupportedVersion)
return nil
}

func (my *Volume) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) {
data := dataMap[my.Object]
if my.currentVal >= my.PluginInvocationRate {
my.currentVal = 0
func (v *Volume) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) {
data := dataMap[v.Object]
if v.currentVal >= v.PluginInvocationRate {
v.currentVal = 0

// invoke disk rest and populate info in aggrsMap
if disks, err := my.getEncryptedDisks(); err != nil {
if disks, err := v.getEncryptedDisks(); err != nil {
if errs.IsRestErr(err, errs.APINotFound) {
my.Logger.Debug().Err(err).Msg("Failed to collect disk data")
v.Logger.Debug().Err(err).Msg("Failed to collect disk data")
} else {
my.Logger.Error().Err(err).Msg("Failed to collect disk data")
v.Logger.Error().Err(err).Msg("Failed to collect disk data")
}
} else {
// update aggrsMap based on disk data
my.updateAggrMap(disks)
v.updateAggrMap(disks)
}
}

volumeMap, err := v.getVolumeInfo()
if err != nil {
v.Logger.Error().Err(err).Msg("Failed to collect volume info data")
}

// update volume instance labels
my.updateVolumeLabels(data)
v.updateVolumeLabels(data, volumeMap)

// parse anti_ransomware_start_time, antiRansomwareState for all volumes and export at cluster level
my.handleARWProtection(data)
v.handleARWProtection(data)

my.currentVal++
return []*matrix.Matrix{my.arw}, nil
v.currentVal++
return []*matrix.Matrix{v.arw}, nil
}

func (my *Volume) updateVolumeLabels(data *matrix.Matrix) {
func (v *Volume) updateVolumeLabels(data *matrix.Matrix, volumeMap map[string]volumeInfo) {
var err error
cloneSplitEstimateMetric := data.GetMetric("clone_split_estimate")
if cloneSplitEstimateMetric == nil {
if cloneSplitEstimateMetric, err = data.NewMetricFloat64("clone_split_estimate"); err != nil {
v.Logger.Error().Stack().Msg("error while creating clone split estimate metric")
}
}
for _, volume := range data.GetInstances() {
// For flexgroup, aggrUuid in Rest should be empty for parity with Zapi response
if volumeStyle := volume.GetLabel("style"); volumeStyle == "flexgroup" {
volume.SetLabel("aggrUuid", "")
if !volume.IsExportable() {
continue
}

if volume.GetLabel("style") == "flexgroup_constituent" {
volume.SetExportable(v.includeConstituents)
}
aggrUUID := volume.GetLabel("aggrUuid")

_, exist := my.aggrsMap[aggrUUID]
volume.SetLabel("isHardwareEncrypted", strconv.FormatBool(exist))
volume.SetLabel("isHardwareEncrypted", strconv.FormatBool(v.aggrsMap[volume.GetLabel("aggr")]))

if vInfo, ok := volumeMap[volume.GetLabel("volume")+volume.GetLabel("svm")]; ok {
volume.SetLabel("anti_ransomware_start_time", vInfo.arwStartTime)
volume.SetLabel("antiRansomwareState", vInfo.arwState)
volume.SetLabel("snapshot_autodelete", vInfo.snapshotAutodelete)
if volume.GetLabel("is_flexclone") == "true" {
volume.SetLabel("clone_parent_snapshot", vInfo.cloneSnapshotName)
if err = cloneSplitEstimateMetric.SetValueFloat64(volume, vInfo.cloneSplitEstimateMetric); err != nil {
v.Logger.Error().Err(err).Str("metric", "cloneSplitEstimateMetric").Msg("Unable to set value on metric")
}
}
}
}
}

func (my *Volume) handleARWProtection(data *matrix.Matrix) {
func (v *Volume) handleARWProtection(data *matrix.Matrix) {
var (
arwInstance *matrix.Instance
arwStartTimeValue time.Time
err error
)

// Purge and reset data
my.arw.PurgeInstances()
my.arw.Reset()
v.arw.PurgeInstances()
v.arw.Reset()

// Set all global labels
my.arw.SetGlobalLabels(data.GetGlobalLabels())
v.arw.SetGlobalLabels(data.GetGlobalLabels())
arwStatusValue := "Active Mode"
// Case where cluster don't have any volumes, arwStatus show as 'Not Monitoring'
if len(data.GetInstances()) == 0 {
Expand Down Expand Up @@ -153,7 +195,7 @@ func (my *Volume) handleARWProtection(data *matrix.Matrix) {
}
// If ARW startTime is more than 30 days old, which indicates that learning mode has been finished.
if arwStartTimeValue, err = time.Parse(time.RFC3339, arwStartTime); err != nil {
my.Logger.Error().Err(err).Msg("Failed to parse arw start time")
v.Logger.Error().Err(err).Msg("Failed to parse arw start time")
arwStartTimeValue = time.Now()
}
if time.Since(arwStartTimeValue).Hours() > HoursInMonth {
Expand All @@ -165,53 +207,106 @@ func (my *Volume) handleARWProtection(data *matrix.Matrix) {
}

arwInstanceKey := data.GetGlobalLabels()["cluster"] + data.GetGlobalLabels()["datacenter"]
if arwInstance, err = my.arw.NewInstance(arwInstanceKey); err != nil {
my.Logger.Error().Err(err).Str("arwInstanceKey", arwInstanceKey).Msg("Failed to create arw instance")
if arwInstance, err = v.arw.NewInstance(arwInstanceKey); err != nil {
v.Logger.Error().Err(err).Str("arwInstanceKey", arwInstanceKey).Msg("Failed to create arw instance")
return
}

arwInstance.SetLabel("ArwStatus", arwStatusValue)
m := my.arw.GetMetric("status")
m := v.arw.GetMetric("status")
// populate numeric data
value := 1.0
if err = m.SetValueFloat64(arwInstance, value); err != nil {
my.Logger.Error().Stack().Err(err).Float64("value", value).Msg("Failed to parse value")
v.Logger.Error().Stack().Err(err).Float64("value", value).Msg("Failed to parse value")
} else {
my.Logger.Debug().Float64("value", value).Msg("added value")
v.Logger.Debug().Float64("value", value).Msg("added value")
}
}

func (my *Volume) getEncryptedDisks() ([]gjson.Result, error) {
func (v *Volume) getEncryptedDisks() ([]gjson.Result, error) {
var (
result []gjson.Result
err error
)

fields := []string{"aggregates.name", "aggregates.uuid"}
fields := []string{"aggregates.name"}
query := "api/storage/disks"
href := rest.NewHrefBuilder().
APIPath(query).
Fields(fields).
Filter([]string{"protection_mode=!data|full"}).
Build()

if result, err = collectors.InvokeRestCall(my.client, href, my.Logger); err != nil {
if result, err = collectors.InvokeRestCall(v.client, href, v.Logger); err != nil {
return nil, err
}
return result, nil
}

func (my *Volume) updateAggrMap(disks []gjson.Result) {
func (v *Volume) getVolumeInfo() (map[string]volumeInfo, error) {
volumeMap := make(map[string]volumeInfo)
fields := []string{"name", "svm.name", "space.snapshot.autodelete_enabled", "clone.parent_snapshot.name", "clone.split_estimate"}
if !v.isArwSupportedVersion {
return v.getVolume("", fields, volumeMap)
}

// Only ask this field when ARW would be supported, is_constituent is supported from 9.10 onwards in public api same as ARW
fields = append(fields, "anti_ransomware.dry_run_start_time", "anti_ransomware.state")
if _, err := v.getVolume("is_constituent=false", fields, volumeMap); err != nil {
return nil, err
}
return v.getVolume("is_constituent=true", fields, volumeMap)
}

func (v *Volume) getVolume(field string, fields []string, volumeMap map[string]volumeInfo) (map[string]volumeInfo, error) {
var (
result []gjson.Result
err error
)
query := "api/storage/volumes"
href := rest.NewHrefBuilder().
APIPath(query).
Fields(fields).
Filter([]string{field}).
Build()

if result, err = collectors.InvokeRestCall(v.client, href, v.Logger); err != nil {
return nil, err
}

for _, volume := range result {
volName := volume.Get("name").String()
svmName := volume.Get("svm.name").String()
arwStartTime := volume.Get("anti_ransomware.dry_run_start_time").String()
arwState := volume.Get("anti_ransomware.state").String()
snapshotAutodelete := volume.Get("space.snapshot.autodelete_enabled").String()
cloneSnapshotName := volume.Get("clone.parent_snapshot.name").String()
cloneSplitEstimate := volume.Get("clone.split_estimate").Float()
volumeMap[volName+svmName] = volumeInfo{arwStartTime: arwStartTime, arwState: arwState, snapshotAutodelete: snapshotAutodelete, cloneSnapshotName: cloneSnapshotName, cloneSplitEstimateMetric: cloneSplitEstimate}
}
return volumeMap, nil
}

func (v *Volume) updateAggrMap(disks []gjson.Result) {
if disks != nil {
// Clean aggrsMap map
my.aggrsMap = make(map[string]string)

clear(v.aggrsMap)
for _, disk := range disks {
aggrName := disk.Get("aggregates.name").String()
aggrUUID := disk.Get("aggregates.uuid").String()
if aggrUUID != "" {
my.aggrsMap[aggrUUID] = aggrName
aggrName := disk.Get("aggregates.#.name").Array()
for _, aggr := range aggrName {
v.aggrsMap[aggr.String()] = true
}
}
}
}

func (v *Volume) versionHigherThan(minVersion string) bool {
currentVersion, err := version.NewVersion(v.client.Cluster().GetVersion())
if err != nil {
return false
}
minSupportedVersion, err := version.NewVersion(minVersion)
if err != nil {
return false
}
return currentVersion.GreaterThanOrEqual(minSupportedVersion)
}
9 changes: 7 additions & 2 deletions cmd/collectors/restperf/plugins/volume/volume.go
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package volume

import (
"github.com/netapp/harvest/v2/cmd/collectors"
"github.com/netapp/harvest/v2/cmd/poller/plugin"
"github.com/netapp/harvest/v2/pkg/matrix"
"github.com/netapp/harvest/v2/pkg/set"
Expand All @@ -12,7 +13,8 @@ import (

type Volume struct {
*plugin.AbstractPlugin
styleType string
styleType string
includeConstituents bool
}

func New(p *plugin.AbstractPlugin) plugin.Plugin {
Expand All @@ -31,6 +33,9 @@ func (v *Volume) Init() error {
if v.Params.HasChildS("historicalLabels") {
v.styleType = "type"
}

// Read template to decide inclusion of flexgroup constituents
v.includeConstituents = collectors.ReadPluginKey(v.Params, "include_constituents")
return nil
}

Expand Down Expand Up @@ -92,7 +97,7 @@ func (v *Volume) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error
fgAggrMap[key].Add(i.GetLabel("aggr"))
flexgroupAggrsMap[key].Add(i.GetLabel("aggr"))
i.SetLabel(style, "flexgroup_constituent")
i.SetExportable(false)
i.SetExportable(v.includeConstituents)
} else {
i.SetLabel(style, "flexvol")
key := i.GetLabel("svm") + "." + i.GetLabel("volume")
Expand Down
Loading

0 comments on commit bcd09c6

Please sign in to comment.