Merge pull request #2683 from NetApp/release/24.02.0

ci: merge 24.02 into main

.github/workflows/codeql-analysis.yml

@@ -12,7 +12,7 @@
 name: "CodeQL"
 
 env:
-  GO_VERSION: "1.21.7"
+  GO_VERSION: "1.22.0"
 
 on:
   push:

.github/workflows/go.yml

@@ -1,7 +1,7 @@
 name: Build, Test, Lint License
 
 env:
-  GO_VERSION: "1.21.7"
+  GO_VERSION: "1.22.0"
 
 on:
   push:
@@ -55,9 +55,7 @@ jobs:
           go-version: ${{ env.GO_VERSION }}
 
       - name: Install wwhrd
-        env:
-          GO111MODULE: 'off'
-        run: go get -u github.com/frapposelli/wwhrd
+        run: go install github.com/frapposelli/wwhrd@latest
       - name: go mod vendor
         env:
           GO111MODULE: 'on'

.github/workflows/golangci-lint.yml

@@ -19,7 +19,7 @@ jobs:
     steps:
       - uses: actions/setup-go@v5
         with:
-          go-version: '1.21.7'
+          go-version: '1.22.0'
       - uses: actions/checkout@v4
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v4

Makefile

@@ -4,7 +4,7 @@
 .PHONY: help deps clean build test fmt vet package asup dev fetch-asup
 
 SHELL := /bin/bash
-REQUIRED_GO_VERSION := 1.21
+REQUIRED_GO_VERSION := 1.22
 ifneq (, $(shell which go))
 FOUND_GO_VERSION := $(shell go version | cut -d" " -f3 | cut -d"o" -f 2)
 CORRECT_GO_VERSION := $(shell expr `go version | cut -d" " -f3 | cut -d"o" -f 2` \>= ${REQUIRED_GO_VERSION})

cmd/collectors/rest/rest.go

@@ -314,12 +314,16 @@ func (r *Rest) PollData() (map[string]*matrix.Matrix, error) {
 		return nil, errs.New(errs.ErrNoInstance, "no "+r.Object+" instances on cluster")
 	}
 
-	return r.pollData(startTime, records, func(e *endPoint) ([]gjson.Result, error) {
+	return r.pollData(startTime, records, func(e *endPoint) ([]gjson.Result, time.Duration, error) {
 		return r.processEndPoint(e)
 	})
 }
 
-func (r *Rest) pollData(startTime time.Time, records []gjson.Result, endpointFunc func(e *endPoint) ([]gjson.Result, error)) (map[string]*matrix.Matrix, error) {
+func (r *Rest) pollData(
+	startTime time.Time,
+	records []gjson.Result,
+	endpointFunc func(e *endPoint) ([]gjson.Result, time.Duration, error),
+) (map[string]*matrix.Matrix, error) {
 
 	var (
 		count        uint64
@@ -332,13 +336,13 @@ func (r *Rest) pollData(startTime time.Time, records []gjson.Result, endpointFun
 	count = r.HandleResults(records, r.Prop, false)
 
 	// process endpoints
-	eCount := r.processEndPoints(endpointFunc)
+	eCount, endpointAPID := r.processEndPoints(endpointFunc)
 	count += eCount
 	parseD = time.Since(startTime)
 
 	numRecords := len(r.Matrix[r.Object].GetInstances())
 
-	_ = r.Metadata.LazySetValueInt64("api_time", "data", apiD.Microseconds())
+	_ = r.Metadata.LazySetValueInt64("api_time", "data", (apiD + endpointAPID).Microseconds())
 	_ = r.Metadata.LazySetValueInt64("parse_time", "data", parseD.Microseconds())
 	_ = r.Metadata.LazySetValueUint64("metrics", "data", count)
 	_ = r.Metadata.LazySetValueUint64("instances", "data", uint64(numRecords))
@@ -347,29 +351,37 @@ func (r *Rest) pollData(startTime time.Time, records []gjson.Result, endpointFun
 	return r.Matrix, nil
 }
 
-func (r *Rest) processEndPoint(e *endPoint) ([]gjson.Result, error) {
+func (r *Rest) processEndPoint(e *endPoint) ([]gjson.Result, time.Duration, error) {
 	href := rest.NewHrefBuilder().
 		APIPath(r.query(e)).
 		Fields(r.fields(e)).
 		Filter(r.filter(e)).
 		ReturnTimeout(r.Prop.ReturnTimeOut).
 		Build()
 
-	return r.GetRestData(href)
+	now := time.Now()
+	data, err := r.GetRestData(href)
+	if err != nil {
+		return nil, 0, err
+	}
+	return data, time.Since(now), nil
 }
 
-func (r *Rest) processEndPoints(endpointFunc func(e *endPoint) ([]gjson.Result, error)) uint64 {
+func (r *Rest) processEndPoints(endpointFunc func(e *endPoint) ([]gjson.Result, time.Duration, error)) (uint64, time.Duration) {
 	var (
-		err   error
-		count uint64
+		err       error
+		count     uint64
+		totalAPID time.Duration
 	)
 
 	for _, endpoint := range r.endpoints {
 		var (
 			records []gjson.Result
+			apiD    time.Duration
 		)
 
-		records, err = endpointFunc(endpoint)
+		records, apiD, err = endpointFunc(endpoint)
+		totalAPID += apiD
 
 		if err != nil {
 			r.Logger.Error().Err(err).Str("api", endpoint.prop.Query).Send()
@@ -383,7 +395,7 @@ func (r *Rest) processEndPoints(endpointFunc func(e *endPoint) ([]gjson.Result,
 		count = r.HandleResults(records, endpoint.prop, true)
 	}
 
-	return count
+	return count, totalAPID
 }
 
 // returns private if api endpoint has private keyword in it else public

cmd/collectors/rest/rest_test.go

@@ -144,10 +144,10 @@ func Test_pollDataVolume(t *testing.T) {
 	}
 }
 
-func volumeEndpoints(e *endPoint) ([]gjson.Result, error) {
+func volumeEndpoints(e *endPoint) ([]gjson.Result, time.Duration, error) {
 	path := "testdata/" + strings.ReplaceAll(e.prop.Query, "/", "-") + ".json.gz"
 	gson := collectors.JSONToGson(path, true)
-	return gson, nil
+	return gson, 0, nil
 }
 
 func newRest(object string, path string) *Rest {

cmd/harvest/harvest.go

@@ -89,7 +89,7 @@ func doManageCmd(cmd *cobra.Command, args []string) {
 		err                                              error
 	)
 	opts.command = cmd.Name()
-	HarvestHomePath = conf.Path()
+	HarvestHomePath = conf.Path("")
 	HarvestConfigPath = conf.Path(conf.HarvestYML)
 
 	if opts.verbose {

cmd/poller/collector/helpers.go

@@ -31,7 +31,7 @@ import (
 // ImportTemplate looks for a collector's template by searching confPaths for the first template that exists in
 // confPath/collectorName/templateName
 func ImportTemplate(confPaths []string, templateName, collectorName string) (*node.Node, error) {
-	homePath := conf.Path()
+	homePath := conf.Path("")
 	for _, confPath := range confPaths {
 		fp := filepath.Join(homePath, confPath, strings.ToLower(collectorName), templateName)
 		_, err := os.Stat(fp)
@@ -70,7 +70,7 @@ func (c *AbstractCollector) ImportSubTemplate(model, filename string, ver [3]int
 	if err != nil {
 		return nil, "", fmt.Errorf("no best-fit template found due to err=%w", err)
 	}
-	homePath := conf.Path()
+	homePath := conf.Path("")
 
 nextFile:
 	for _, f := range filenames {

cmd/poller/options/options.go

@@ -82,7 +82,7 @@ func (o *Options) SetDefaults() *Options {
 		o.Hostname = hostname
 	}
 
-	o.HomePath = conf.Path()
+	o.HomePath = conf.Path("")
 	o.LogPath = conf.GetHarvestLogPath()
 	o.SetConfPath(o.ConfPath)
 

cmd/tools/grafana/grafana.go

@@ -72,7 +72,7 @@ type Folder struct {
 
 func adjustOptions() {
 	opts.config = conf.ConfigPath(opts.config)
-	homePath = conf.Path()
+	homePath = conf.Path("")
 	opts.dirGrafanaFolderMap = make(map[string]*Folder)
 
 	// When opt.addr starts with https don't change it

cmd/tools/zapi/export.go

@@ -89,7 +89,7 @@ func exportCounters(item *node.Node, c *client.Client, args *Args) error {
 	*/
 	var fp []string
 
-	harvestHomePath = conf.Path()
+	harvestHomePath = conf.Path("")
 	fp = append(fp, harvestHomePath)
 	fp = append(fp, "conf/")
 	fp = append(fp, "zapiperf/")

container/onePollerPerContainer/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.21.7 as builder
+FROM golang:1.22.0 as builder
 
 SHELL ["/bin/bash", "-c"]
 

docs/help/config-collection.md

@@ -0,0 +1,40 @@
+# Harvest Config Collection Guide
+
+This guide is designed to help you validate your Harvest configuration (`harvest.yml`) on various platforms.
+The commands in this guide will generate redacted output that personally identifiable information (PII) removed.
+This makes it safe for you to share the output.
+Follow the instructions specific to your platform.
+If you wish to share it with the Harvest team,
+please email them at [ng-harvest-files@netapp.com](mailto:ng-harvest-files@netapp.com).
+
+## RPM, DEB, and Native Installations
+
+To print a redacted version of your Harvest configuration to the console, use the following command:
+
+```bash
+cd /opt/harvest
+export CONFIG_FILE_NAME=harvest.yml
+bin/harvest doctor --print --config $CONFIG_FILE_NAME
+```
+
+## Docker Container
+
+For Docker containers, use the following command to print a redacted version of your Harvest configuration to the console:
+
+```bash
+cd to/where/your/harvest.yml/is
+export CONFIG_FILE_NAME=harvest.yml
+docker run --rm --entrypoint "bin/harvest" --volume "$(pwd)/$CONFIG_FILE_NAME:/opt/harvest/harvest.yml" ghcr.io/netapp/harvest doctor --print
+```
+
+## NABox
+
+If you're using NABox, you'll need to [ssh](https://nabox.org/documentation/configuration/) into your NABox instance.
+Then, use the following command to print a redacted version of your Harvest configuration to the console:
+
+```bash
+dc exec -w /conf nabox-harvest2 /netapp-harvest/bin/harvest doctor --print
+```
+
+If your configuration file name is different from the default `harvest.yml`,
+remember to change the `CONFIG_FILE_NAME` environment variable to match your file name.

go.mod

@@ -1,6 +1,6 @@
 module github.com/netapp/harvest/v2
 
-go 1.21
+go 1.22
 
 require (
 	dario.cat/mergo v1.0.0

integration/Jenkinsfile

@@ -18,7 +18,7 @@ pipeline {
     environment {
         BUILD_ID="dontKillMe"
         JENKINS_NODE_COOKIE="dontKillMe"
-        GO_VERSION = "1.21.7"
+        GO_VERSION = "1.22.0"
     }
 
     stages {

integration/go.mod

@@ -1,15 +1,17 @@
 module github.com/Netapp/harvest-automation
 
-go 1.21
+go 1.22
+
+toolchain go1.22.0
 
 replace github.com/netapp/harvest/v2 => ../
 
 require (
 	github.com/carlmjohnson/requests v0.23.5
 	github.com/hashicorp/go-version v1.6.0
-	github.com/netapp/harvest/v2 v2.0.0-20240208073132-cf13d5bd569d
+	github.com/netapp/harvest/v2 v2.0.0-20240215072426-503b4600075d
 	github.com/rs/zerolog v1.32.0
-	github.com/tidwall/gjson v1.17.0
+	github.com/tidwall/gjson v1.17.1
 	golang.org/x/text v0.14.0
 )
 

integration/go.sum

@@ -84,8 +84,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO
 github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
-github.com/tidwall/gjson v1.17.0 h1:/Jocvlh98kcTfpN2+JzGQWQcqrPQwDrVEMApx/M5ZwM=
-github.com/tidwall/gjson v1.17.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/tidwall/gjson v1.17.1 h1:wlYEnwqAHgzmhNUFfw7Xalt2JzQvsMx2Se4PcoFCT/U=
+github.com/tidwall/gjson v1.17.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
 github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
 github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
 github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=

jenkins/artifacts/jenkinsfile

@@ -37,7 +37,7 @@ pipeline {
         jfrogImagePrefix = "netappdownloads.jfrog.io/oss-docker-harvest-production/harvest"
         jfrogRepo = "netappdownloads.jfrog.io"
         COMMIT_ID = sh(returnStdout: true, script: 'git rev-parse HEAD')
-        GO_VERSION = "1.21.7"
+        GO_VERSION = "1.22.0"
     }
 
     stages {

mkdocs.yml

@@ -43,6 +43,7 @@ nav:
       - 'Troubleshooting': 'help/troubleshooting.md'
       - 'FAQ': 'help/faq.md'
       - 'Log Collection': 'help/log-collection.md'
+      - 'Config Collection': 'help/config-collection.md'
   - Reference:
       - 'Matrix': 'resources/matrix.md'
       - 'ONTAP Metrics': 'ontap-metrics.md'

pkg/conf/conf.go

@@ -18,6 +18,7 @@ import (
 	"regexp"
 	"sort"
 	"strconv"
+	"strings"
 )
 
 var Config = HarvestConfig{}
@@ -247,13 +248,22 @@ func PollerNamed(name string) (*Poller, error) {
 	return poller, nil
 }
 
-// Path joins a set of path elems into a single path.
-// The final path will be relative to the HARVEST_CONF environment variable
-// or ./ when the environment variable is not set
-func Path(elem ...string) string {
-	home := os.Getenv(HomeEnvVar)
-	paths := append([]string{home}, elem...)
-	return filepath.Join(paths...)
+// Path returns a path based on aPath and the HARVEST_CONF environment variable.
+// If aPath is absolute, it is returned unchanged.
+// When the HARVEST_CONF environment variable is set, a new path is returned relative to HARVEST_CONF.
+// Otherwise, a new path is returned relative to the current working directory.
+func Path(aPath string) string {
+	confDir := os.Getenv(HomeEnvVar)
+	if aPath == "" {
+		return confDir
+	}
+	if filepath.IsAbs(aPath) {
+		return aPath
+	}
+	if strings.HasPrefix(aPath, confDir) {
+		return aPath
+	}
+	return filepath.Join(confDir, aPath)
 }
 
 func GetHarvestLogPath() string {