From 98ccc9f845b4739c35764e0a4e8b7f8afc00bd2a Mon Sep 17 00:00:00 2001 From: hardikl Date: Wed, 1 Mar 2023 18:06:55 +0530 Subject: [PATCH 1/3] fix: backward compatibility for qtree metrics in rest --- cmd/collectors/rest/plugins/qtree/qtree.go | 194 ++++++++++++++++++--- conf/rest/9.12.0/qtree.yaml | 13 +- 2 files changed, 177 insertions(+), 30 deletions(-) diff --git a/cmd/collectors/rest/plugins/qtree/qtree.go b/cmd/collectors/rest/plugins/qtree/qtree.go index 35b6ab623..2609fc9c2 100644 --- a/cmd/collectors/rest/plugins/qtree/qtree.go +++ b/cmd/collectors/rest/plugins/qtree/qtree.go @@ -12,6 +12,7 @@ import ( "github.com/netapp/harvest/v2/pkg/dict" "github.com/netapp/harvest/v2/pkg/errs" "github.com/netapp/harvest/v2/pkg/matrix" + "github.com/netapp/harvest/v2/pkg/tree/node" "github.com/netapp/harvest/v2/pkg/util" "github.com/tidwall/gjson" "strconv" @@ -21,12 +22,13 @@ import ( type Qtree struct { *plugin.AbstractPlugin - data *matrix.Matrix - instanceKeys map[string]string - instanceLabels map[string]*dict.Dict - client *rest.Client - query string - quotaType []string + data *matrix.Matrix + instanceKeys map[string]string + instanceLabels map[string]*dict.Dict + client *rest.Client + query string + quotaType []string + historicalLabels bool // supports labels, metrics for 22.05 } func New(p *plugin.AbstractPlugin) plugin.Plugin { @@ -77,6 +79,35 @@ func (my *Qtree) Init() error { my.data = matrix.New(my.Parent+".Qtree", "quota", "quota") my.instanceKeys = make(map[string]string) my.instanceLabels = make(map[string]*dict.Dict) + my.historicalLabels = false + + if my.Params.HasChildS("historicalLabels") { + exportOptions := node.NewS("export_options") + instanceKeys := exportOptions.NewChildS("instance_keys", "") + + // apply all instance keys, instance labels from parent (qtree.yaml) to all quota metrics + if exportOption := my.ParentParams.GetChildS("export_options"); exportOption != nil { + //parent instancekeys would be added in plugin metrics + if parentKeys := exportOption.GetChildS("instance_keys"); parentKeys != nil { + for _, parentKey := range parentKeys.GetAllChildContentS() { + instanceKeys.NewChildS("", parentKey) + } + } + // parent instacelabels would be added in plugin metrics + if parentLabels := exportOption.GetChildS("instance_labels"); parentLabels != nil { + for _, parentLabel := range parentLabels.GetAllChildContentS() { + instanceKeys.NewChildS("", parentLabel) + } + } + } + + instanceKeys.NewChildS("", "type") + instanceKeys.NewChildS("", "index") + instanceKeys.NewChildS("", "unit") + + my.data.SetExportOptions(exportOptions) + my.historicalLabels = true + } quotaType := my.Params.GetChildS("quotaType") if quotaType != nil { @@ -119,6 +150,15 @@ func (my *Qtree) Run(data *matrix.Matrix) ([]*matrix.Matrix, error) { filter := []string{"return_unmatched_nested_array_objects=true", "show_default_records=false", "type=" + strings.Join(my.quotaType[:], "|")} + // In 22.05, all qtrees were exported + if my.historicalLabels { + for _, qtreeInstance := range data.GetInstances() { + qtreeInstance.SetExportable(true) + } + // In 22.05, we would need default records as well. + filter = []string{"return_unmatched_nested_array_objects=true", "show_default_records=true", "type=" + strings.Join(my.quotaType[:], "|")} + } + href := rest.BuildHref("", "*", filter, "", "", "", "", my.query) if result, err = collectors.InvokeRestCall(my.client, href, my.Logger); err != nil { @@ -128,12 +168,132 @@ func (my *Qtree) Run(data *matrix.Matrix) ([]*matrix.Matrix, error) { quotaCount := 0 cluster, _ := data.GetGlobalLabels().GetHas("cluster") + if my.historicalLabels { + // In 22.05, populate metrics with qtree prefix and old labels + err = my.handlingHistoricalMetrics(result, data, cluster, "aCount, &numMetrics) + } else { + // Populate metrics with quota prefix and current labels + err = my.handlingQuotaMetrics(result, cluster, "aCount, &numMetrics) + } + + if err != nil { + return nil, err + } + + my.Logger.Info(). + Int("numQuotas", quotaCount). + Int("metrics", numMetrics). + Msg("Collected") + + // metrics with qtree prefix and quota prefix are available to support backward compatibility + qtreePluginData := my.data.Clone(true, true, true) + qtreePluginData.UUID = my.Parent + ".Qtree" + qtreePluginData.Object = "qtree" + qtreePluginData.Identifier = "qtree" + return []*matrix.Matrix{qtreePluginData, my.data}, nil +} + +func (my Qtree) handlingHistoricalMetrics(result []gjson.Result, data *matrix.Matrix, cluster string, quotaIndex *int, numMetrics *int) error { + for _, quota := range result { + var tree string + var quotaInstance *matrix.Instance + var err error + + if !quota.IsObject() { + my.Logger.Error().Str("type", quota.Type.String()).Msg("Quota is not an object, skipping") + return errs.New(errs.ErrNoInstance, "quota is not an object") + } + + if quota.Get("qtree.name").Exists() { + tree = quota.Get("qtree.name").String() + } + volume := quota.Get("volume.name").String() + vserver := quota.Get("svm.name").String() + qIndex := quota.Get("index").String() + quotaType := quota.Get("type").String() + + // If quota-type is not a tree, then skip + if quotaType != "tree" { + my.Logger.Trace().Str("quotaType", quotaType).Msg("Quota is not tree type, skipping") + continue + } + + // Ex. InstanceKey: vserver1vol1qtree15989279 + quotaInstanceKey := vserver + volume + tree + qIndex + + if quotaInstance = my.data.GetInstance(quotaInstanceKey); quotaInstance == nil { + if quotaInstance, err = my.data.NewInstance(quotaInstanceKey); err != nil { + my.Logger.Error().Stack().Err(err).Str("quotaInstanceKey", quotaInstanceKey).Msg("Failed to create quota instance") + return err + } + my.Logger.Debug().Msgf("add (%s) quota instance: %s.%s.%s.%s", quotaInstanceKey, vserver, volume, tree, qIndex) + } + + qtreeInstance := data.GetInstance(vserver + volume + tree) + if qtreeInstance == nil { + my.Logger.Warn(). + Str("tree", tree). + Str("volume", volume). + Str("vserver", vserver). + Msg("No instance matching tree.volume.vserver") + continue + } + + if !qtreeInstance.IsExportable() { + continue + } + + for _, label := range my.data.GetExportOptions().GetChildS("instance_keys").GetAllChildContentS() { + if value := qtreeInstance.GetLabel(label); value != "" { + quotaInstance.SetLabel(label, value) + } + } + + //set labels + quotaInstance.SetLabel("type", quotaType) + quotaInstance.SetLabel("qtree", tree) + quotaInstance.SetLabel("volume", volume) + quotaInstance.SetLabel("svm", vserver) + quotaInstance.SetLabel("index", cluster+"_"+strconv.Itoa(*quotaIndex)) + + // If the Qtree is the volume itself, than qtree label is empty, so copy the volume name to qtree. + if tree == "" { + quotaInstance.SetLabel("qtree", volume) + } + + *quotaIndex++ + for attribute, m := range my.data.GetMetrics() { + value := 0.0 + + if attrValue := quota.Get(attribute); attrValue.Exists() { + // space limits are in bytes, converted to kilobytes + if attribute == "space.hard_limit" || attribute == "space.soft_limit" { + value = attrValue.Float() / 1024 + quotaInstance.SetLabel("unit", "Kbyte") + } else { + value = attrValue.Float() + } + } + + // populate numeric data + if err = m.SetValueFloat64(quotaInstance, value); err != nil { + my.Logger.Error().Stack().Err(err).Str("attribute", attribute).Float64("value", value).Msg("Failed to parse value") + } else { + *numMetrics++ + my.Logger.Debug().Str("attribute", attribute).Float64("value", value).Msg("added value") + } + } + } + return nil +} + +func (my Qtree) handlingQuotaMetrics(result []gjson.Result, cluster string, quotaCount *int, numMetrics *int) error { for quotaIndex, quota := range result { var tree string if !quota.IsObject() { my.Logger.Error().Str("type", quota.Type.String()).Msg("Quota is not an object, skipping") - return nil, errs.New(errs.ErrNoInstance, "quota is not an object") + return errs.New(errs.ErrNoInstance, "quota is not an object") } if quota.Get("qtree.name").Exists() { @@ -145,7 +305,7 @@ func (my *Qtree) Run(data *matrix.Matrix) ([]*matrix.Matrix, error) { uName := quota.Get("users.0.name").String() uid := quota.Get("users.0.id").String() group := quota.Get("group.name").String() - quotaCount++ + *quotaCount++ for attribute, m := range my.data.GetMetrics() { // set -1 for unlimited @@ -156,7 +316,7 @@ func (my *Qtree) Run(data *matrix.Matrix) ([]*matrix.Matrix, error) { quotaInstance, err := my.data.NewInstance(quotaInstanceKey) if err != nil { my.Logger.Debug().Msgf("add (%s) instance: %v", attribute, err) - return nil, err + return err } //set labels quotaInstance.SetLabel("type", quotaType) @@ -192,22 +352,10 @@ func (my *Qtree) Run(data *matrix.Matrix) ([]*matrix.Matrix, error) { if err = m.SetValueFloat64(quotaInstance, value); err != nil { my.Logger.Error().Stack().Err(err).Str("attribute", attribute).Float64("value", value).Msg("Failed to parse value") } else { - numMetrics++ + *numMetrics++ my.Logger.Trace().Str("attribute", attribute).Float64("value", value).Msg("added value") } } - } - - my.Logger.Info(). - Int("numQuotas", quotaCount). - Int("metrics", numMetrics). - Msg("Collected") - - // This would generate quota metrics prefix with `qtree_`. These are deprecated now and will be removed later. - qtreePluginData := my.data.Clone(true, true, true) - qtreePluginData.UUID = my.Parent + ".Qtree" - qtreePluginData.Object = "qtree" - qtreePluginData.Identifier = "qtree" - return []*matrix.Matrix{qtreePluginData, my.data}, nil + return nil } diff --git a/conf/rest/9.12.0/qtree.yaml b/conf/rest/9.12.0/qtree.yaml index a95386021..3d2c504b7 100644 --- a/conf/rest/9.12.0/qtree.yaml +++ b/conf/rest/9.12.0/qtree.yaml @@ -11,8 +11,6 @@ counters: - ^export_policy.name => export_policy - ^security_style => security_style - id => id - - filter: - - name=!"" endpoints: @@ -23,17 +21,18 @@ endpoints: - ^^qtree => qtree - ^status => status - ^oplock_mode => oplocks - - filter: - - qtree=!"" + plugins: + - LabelAgent: + exclude_equals: + - qtree `` + replace: + - oplocks oplocks `enable` `enabled` - Qtree: quotaType: - tree # - user # - group - - LabelAgent: - replace: - - oplocks oplocks `enable` `enabled` export_options: instance_keys: From 49b45b51361fb5e7753a5e4944e00bde96db0583 Mon Sep 17 00:00:00 2001 From: hardikl Date: Thu, 2 Mar 2023 18:51:52 +0530 Subject: [PATCH 2/3] fix: revert qtree template change as rest dont support merge --- conf/rest/9.12.0/qtree.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/conf/rest/9.12.0/qtree.yaml b/conf/rest/9.12.0/qtree.yaml index 3d2c504b7..246c5a4fe 100644 --- a/conf/rest/9.12.0/qtree.yaml +++ b/conf/rest/9.12.0/qtree.yaml @@ -11,6 +11,8 @@ counters: - ^export_policy.name => export_policy - ^security_style => security_style - id => id + - filter: + - name=!"" endpoints: @@ -21,11 +23,10 @@ endpoints: - ^^qtree => qtree - ^status => status - ^oplock_mode => oplocks - + - filter: + - qtree=!"" plugins: - LabelAgent: - exclude_equals: - - qtree `` replace: - oplocks oplocks `enable` `enabled` - Qtree: From 824839021dda5a72f6506fba7dd3a73e031dddfa Mon Sep 17 00:00:00 2001 From: hardikl Date: Thu, 2 Mar 2023 18:54:42 +0530 Subject: [PATCH 3/3] fix: minor order change --- conf/rest/9.12.0/qtree.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/conf/rest/9.12.0/qtree.yaml b/conf/rest/9.12.0/qtree.yaml index 246c5a4fe..a95386021 100644 --- a/conf/rest/9.12.0/qtree.yaml +++ b/conf/rest/9.12.0/qtree.yaml @@ -26,14 +26,14 @@ endpoints: - filter: - qtree=!"" plugins: - - LabelAgent: - replace: - - oplocks oplocks `enable` `enabled` - Qtree: quotaType: - tree # - user # - group + - LabelAgent: + replace: + - oplocks oplocks `enable` `enabled` export_options: instance_keys: