feat: bin/rest should be able to query all clusters (#1866)

* feat: bin/rest should be able to query all clusters feat: bin/rest should print the equivalent curl commands feat: bin/rest should collect cluster name, version, and poll time feat: bin/rest should have a cli option to set timeout docs: reformat bin/rest examples and include jq example * feat: bin/rest should be able to query all clusters feat: bin/rest should print the equivalent curl commands feat: bin/rest should collect cluster name, version, and poll time feat: bin/rest should have a cli option to set timeout docs: reformat bin/rest examples and include jq example
NetApp · Mar 27, 2023 · 03a2e21 · 03a2e21
1 parent bc7aa03
commit 03a2e21
Show file tree

Hide file tree

Showing 38 changed files with 260 additions and 170 deletions.
diff --git a/cmd/collectors/rest/plugins/certificate/certificate.go b/cmd/collectors/rest/plugins/certificate/certificate.go
@@ -36,7 +36,7 @@ func (my *Certificate) Init() error {
 	}
 
 	timeout, _ := time.ParseDuration(rest.DefaultTimeout)
-	if my.client, err = rest.New(conf.ZapiPoller(my.ParentParams), timeout); err != nil {
+	if my.client, err = rest.New(conf.ZapiPoller(my.ParentParams), timeout, my.Auth); err != nil {
 		my.Logger.Error().Stack().Err(err).Msg("connecting")
 		return err
 	}

diff --git a/cmd/collectors/rest/plugins/qtree/qtree.go b/cmd/collectors/rest/plugins/qtree/qtree.go
@@ -65,7 +65,7 @@ func (my *Qtree) Init() error {
 	} else {
 		my.Logger.Info().Str("timeout", timeout.String()).Msg("Using default timeout")
 	}
-	if my.client, err = rest.New(conf.ZapiPoller(my.ParentParams), timeout); err != nil {
+	if my.client, err = rest.New(conf.ZapiPoller(my.ParentParams), timeout, my.Auth); err != nil {
 		my.Logger.Error().Stack().Err(err).Msg("connecting")
 		return err
 	}

diff --git a/cmd/collectors/rest/plugins/securityaccount/securityaccount.go b/cmd/collectors/rest/plugins/securityaccount/securityaccount.go
@@ -40,7 +40,7 @@ func (s *SecurityAccount) Init() error {
 	} else {
 		s.Logger.Info().Str("timeout", timeout.String()).Msg("Using default timeout")
 	}
-	if s.client, err = rest.New(conf.ZapiPoller(s.ParentParams), timeout); err != nil {
+	if s.client, err = rest.New(conf.ZapiPoller(s.ParentParams), timeout, s.Auth); err != nil {
 		s.Logger.Error().Stack().Err(err).Msg("connecting")
 		return err
 	}

diff --git a/cmd/collectors/rest/plugins/snapmirror/snapmirror.go b/cmd/collectors/rest/plugins/snapmirror/snapmirror.go
@@ -1,6 +1,7 @@
 /*
  * Copyright NetApp Inc, 2022 All rights reserved
  */
+
 package snapmirror
 
 import (
@@ -42,7 +43,7 @@ func (my *SnapMirror) Init() error {
 	}
 
 	timeout, _ := time.ParseDuration(rest.DefaultTimeout)
-	if my.client, err = rest.New(conf.ZapiPoller(my.ParentParams), timeout); err != nil {
+	if my.client, err = rest.New(conf.ZapiPoller(my.ParentParams), timeout, my.Auth); err != nil {
 		my.Logger.Error().Stack().Err(err).Msg("connecting")
 		return err
 	}

diff --git a/cmd/collectors/rest/plugins/volume/volume.go b/cmd/collectors/rest/plugins/volume/volume.go
@@ -37,7 +37,7 @@ func (my *Volume) Init() error {
 	}
 
 	timeout, _ := time.ParseDuration(rest.DefaultTimeout)
-	if my.client, err = rest.New(conf.ZapiPoller(my.ParentParams), timeout); err != nil {
+	if my.client, err = rest.New(conf.ZapiPoller(my.ParentParams), timeout, my.Auth); err != nil {
 		my.Logger.Error().Stack().Err(err).Msg("connecting")
 		return err
 	}

diff --git a/cmd/collectors/rest/plugins/volumeanalytics/volumeanalytics.go b/cmd/collectors/rest/plugins/volumeanalytics/volumeanalytics.go
@@ -54,7 +54,7 @@ func (v *VolumeAnalytics) Init() error {
 	}
 
 	timeout, _ := time.ParseDuration(rest.DefaultTimeout)
-	if v.client, err = rest.New(conf.ZapiPoller(v.ParentParams), timeout); err != nil {
+	if v.client, err = rest.New(conf.ZapiPoller(v.ParentParams), timeout, v.Auth); err != nil {
 		v.Logger.Error().Stack().Err(err).Msg("connecting")
 		return err
 	}

diff --git a/cmd/collectors/rest/rest.go b/cmd/collectors/rest/rest.go
@@ -17,6 +17,7 @@ import (
 	"github.com/netapp/harvest/v2/cmd/poller/collector"
 	"github.com/netapp/harvest/v2/cmd/poller/plugin"
 	"github.com/netapp/harvest/v2/cmd/tools/rest"
+	"github.com/netapp/harvest/v2/pkg/auth"
 	"github.com/netapp/harvest/v2/pkg/conf"
 	"github.com/netapp/harvest/v2/pkg/errs"
 	"github.com/netapp/harvest/v2/pkg/matrix"
@@ -148,7 +149,7 @@ func (r *Rest) InitClient() error {
 
 	var err error
 	a := r.AbstractCollector
-	if r.Client, err = r.getClient(a); err != nil {
+	if r.Client, err = r.getClient(a, r.Auth); err != nil {
 		return err
 	}
 
@@ -176,7 +177,7 @@ func (r *Rest) InitMatrix() error {
 	return nil
 }
 
-func (r *Rest) getClient(a *collector.AbstractCollector) (*rest.Client, error) {
+func (r *Rest) getClient(a *collector.AbstractCollector, c *auth.Credentials) (*rest.Client, error) {
 	var (
 		poller *conf.Poller
 		err    error
@@ -193,7 +194,7 @@ func (r *Rest) getClient(a *collector.AbstractCollector) (*rest.Client, error) {
 		return nil, errs.New(errs.ErrMissingParam, "addr")
 	}
 	timeout, _ := time.ParseDuration(rest.DefaultTimeout)
-	if client, err = rest.New(poller, timeout); err != nil {
+	if client, err = rest.New(poller, timeout, c); err != nil {
 		r.Logger.Error().Err(err).Str("poller", opt.Poller).Msg("error creating new client")
 		os.Exit(1)
 	}

diff --git a/cmd/collectors/restperf/plugins/disk/disk.go b/cmd/collectors/restperf/plugins/disk/disk.go
@@ -155,7 +155,7 @@ func (d *Disk) Init() error {
 	}
 
 	timeout, _ := time.ParseDuration(rest.DefaultTimeout)
-	if d.client, err = rest.New(conf.ZapiPoller(d.ParentParams), timeout); err != nil {
+	if d.client, err = rest.New(conf.ZapiPoller(d.ParentParams), timeout, d.Auth); err != nil {
 		d.Logger.Error().Stack().Err(err).Msg("connecting")
 		return err
 	}

diff --git a/cmd/collectors/restperf/plugins/volumetag/volumetag.go b/cmd/collectors/restperf/plugins/volumetag/volumetag.go
@@ -24,7 +24,7 @@ func (v *VolumeTag) Init() error {
 	}
 
 	timeout, _ := time.ParseDuration(rest.DefaultTimeout)
-	if v.client, err = rest.New(conf.ZapiPoller(v.ParentParams), timeout); err != nil {
+	if v.client, err = rest.New(conf.ZapiPoller(v.ParentParams), timeout, v.Auth); err != nil {
 		v.Logger.Error().Stack().Err(err).Msg("connecting")
 		return err
 	}

diff --git a/cmd/collectors/storagegrid/plugins/bucket/bucket.go b/cmd/collectors/storagegrid/plugins/bucket/bucket.go
@@ -26,7 +26,7 @@ func (b *Bucket) Init() error {
 	}
 
 	clientTimeout := b.ParentParams.GetChildContentS("client_timeout")
-	if b.client, err = rest.NewClient(b.Options.Poller, clientTimeout); err != nil {
+	if b.client, err = rest.NewClient(b.Options.Poller, clientTimeout, b.Auth); err != nil {
 		b.Logger.Error().Stack().Err(err).Msg("connecting")
 		return err
 	}

diff --git a/cmd/collectors/storagegrid/plugins/joinrest/joinrest.go b/cmd/collectors/storagegrid/plugins/joinrest/joinrest.go
@@ -155,7 +155,7 @@ func (t *JoinRest) updateCache(model join, bytes *[]byte) {
 func (t *JoinRest) initClient() error {
 	var err error
 
-	if t.client, err = rest.NewClient(t.Options.Poller, t.Params.GetChildContentS("client_timeout")); err != nil {
+	if t.client, err = rest.NewClient(t.Options.Poller, t.Params.GetChildContentS("client_timeout"), t.Auth); err != nil {
 		return err
 	}
 

diff --git a/cmd/collectors/storagegrid/rest/client.go b/cmd/collectors/storagegrid/rest/client.go
@@ -37,6 +37,7 @@ type Client struct {
 	Timeout  time.Duration
 	logRest  bool // used to log Rest request/response
 	APIPath  string
+	auth     *auth.Credentials
 }
 
 type Cluster struct {
@@ -46,7 +47,7 @@ type Cluster struct {
 	Version [3]int
 }
 
-func NewClient(pollerName string, clientTimeout string) (*Client, error) {
+func NewClient(pollerName string, clientTimeout string, c *auth.Credentials) (*Client, error) {
 	var (
 		poller  *conf.Poller
 		err     error
@@ -65,14 +66,14 @@ func NewClient(pollerName string, clientTimeout string) (*Client, error) {
 	if err != nil {
 		timeout, _ = time.ParseDuration(DefaultTimeout)
 	}
-	if client, err = New(poller, timeout); err != nil {
+	if client, err = New(poller, timeout, c); err != nil {
 		return nil, fmt.Errorf("uanble to create poller [%s]. err: %w", pollerName, err)
 	}
 
 	return client, err
 }
 
-func New(poller *conf.Poller, timeout time.Duration) (*Client, error) {
+func New(poller *conf.Poller, timeout time.Duration, c *auth.Credentials) (*Client, error) {
 	var (
 		client         Client
 		httpclient     *http.Client
@@ -84,7 +85,9 @@ func New(poller *conf.Poller, timeout time.Duration) (*Client, error) {
 		err            error
 	)
 
-	client = Client{}
+	client = Client{
+		auth: c,
+	}
 	client.Logger = logging.Get().SubLogger("StorageGrid", "Client")
 
 	if addr = poller.Addr; addr == "" {
@@ -135,7 +138,7 @@ func New(poller *conf.Poller, timeout time.Duration) (*Client, error) {
 		}
 	} else {
 		username := poller.Username
-		password := auth.Get().Password()
+		password := c.Password()
 		client.username = username
 		if username == "" {
 			return nil, errs.New(errs.ErrMissingParam, "username")
@@ -386,7 +389,7 @@ func (c *Client) fetchToken() error {
 	}
 	authB := authBody{
 		Username: c.username,
-		Password: auth.Get().Password(),
+		Password: c.auth.Password(),
 	}
 	postBody, err := json.Marshal(authB)
 	if err != nil {

diff --git a/cmd/collectors/storagegrid/storagegrid.go b/cmd/collectors/storagegrid/storagegrid.go
@@ -398,7 +398,7 @@ func (s *StorageGrid) handleResults(result []gjson.Result) uint64 {
 func (s *StorageGrid) initClient() error {
 	var err error
 
-	if s.client, err = srest.NewClient(s.Options.Poller, s.Params.GetChildContentS("client_timeout")); err != nil {
+	if s.client, err = srest.NewClient(s.Options.Poller, s.Params.GetChildContentS("client_timeout"), s.Auth); err != nil {
 		return err
 	}
 

diff --git a/cmd/collectors/zapi/collector/zapi.go b/cmd/collectors/zapi/collector/zapi.go
@@ -87,7 +87,7 @@ func (z *Zapi) InitVars() error {
 
 	var err error
 
-	if z.Client, err = client.New(conf.ZapiPoller(z.Params)); err != nil { // convert to connection error, so poller aborts
+	if z.Client, err = client.New(conf.ZapiPoller(z.Params), z.Auth); err != nil { // convert to connection error, so poller aborts
 		return errs.New(errs.ErrConnection, err.Error())
 	}
 	z.Client.TraceLogSet(z.Name, z.Params)

diff --git a/cmd/collectors/zapi/plugins/certificate/certificate.go b/cmd/collectors/zapi/plugins/certificate/certificate.go
@@ -39,7 +39,7 @@ func (my *Certificate) Init() error {
 		return err
 	}
 
-	if my.client, err = zapi.New(conf.ZapiPoller(my.ParentParams)); err != nil {
+	if my.client, err = zapi.New(conf.ZapiPoller(my.ParentParams), my.Auth); err != nil {
 		my.Logger.Error().Stack().Err(err).Msg("connecting")
 		return err
 	}

diff --git a/cmd/collectors/zapi/plugins/qtree/qtree.go b/cmd/collectors/zapi/plugins/qtree/qtree.go
@@ -43,7 +43,7 @@ func (my *Qtree) Init() error {
 		return err
 	}
 
-	if my.client, err = zapi.New(conf.ZapiPoller(my.ParentParams)); err != nil {
+	if my.client, err = zapi.New(conf.ZapiPoller(my.ParentParams), my.Auth); err != nil {
 		my.Logger.Error().Stack().Err(err).Msg("connecting")
 		return err
 	}

diff --git a/cmd/collectors/zapi/plugins/security/security.go b/cmd/collectors/zapi/plugins/security/security.go
@@ -34,7 +34,7 @@ func (my *Security) Init() error {
 		return err
 	}
 
-	if my.client, err = zapi.New(conf.ZapiPoller(my.ParentParams)); err != nil {
+	if my.client, err = zapi.New(conf.ZapiPoller(my.ParentParams), my.Auth); err != nil {
 		my.Logger.Error().Stack().Err(err).Msg("connecting")
 		return err
 	}

diff --git a/cmd/collectors/zapi/plugins/sensor/sensor_test.go b/cmd/collectors/zapi/plugins/sensor/sensor_test.go
@@ -14,7 +14,7 @@ import (
 
 var testxml = "../../../../../cmd/collectors/zapi/plugins/sensor/testdata/sensor.xml"
 var mat *matrix.Matrix
-var sensor = &Sensor{AbstractPlugin: plugin.New("sensor", nil, nil, nil, "sensor")}
+var sensor = &Sensor{AbstractPlugin: plugin.New("sensor", nil, nil, nil, "sensor", nil)}
 
 func init() {
 	//setup matrix data

diff --git a/cmd/collectors/zapi/plugins/shelf/shelf.go b/cmd/collectors/zapi/plugins/shelf/shelf.go
@@ -37,7 +37,7 @@ func (my *Shelf) Init() error {
 		return err
 	}
 
-	if my.client, err = zapi.New(conf.ZapiPoller(my.ParentParams)); err != nil {
+	if my.client, err = zapi.New(conf.ZapiPoller(my.ParentParams), my.Auth); err != nil {
 		my.Logger.Error().Stack().Err(err).Msg("connecting")
 		return err
 	}

diff --git a/cmd/collectors/zapi/plugins/snapmirror/snapmirror.go b/cmd/collectors/zapi/plugins/snapmirror/snapmirror.go
@@ -41,7 +41,7 @@ func (my *SnapMirror) Init() error {
 	if err = my.InitAbc(); err != nil {
 		return err
 	}
-	if my.client, err = zapi.New(conf.ZapiPoller(my.ParentParams)); err != nil {
+	if my.client, err = zapi.New(conf.ZapiPoller(my.ParentParams), my.Auth); err != nil {
 		my.Logger.Error().Stack().Err(err).Msg("connecting")
 		return err
 	}

diff --git a/cmd/collectors/zapi/plugins/svm/svm.go b/cmd/collectors/zapi/plugins/svm/svm.go
@@ -67,7 +67,7 @@ func (my *SVM) Init() error {
 		return err
 	}
 
-	if my.client, err = zapi.New(conf.ZapiPoller(my.ParentParams)); err != nil {
+	if my.client, err = zapi.New(conf.ZapiPoller(my.ParentParams), my.Auth); err != nil {
 		my.Logger.Error().Stack().Err(err).Msg("connecting")
 		return err
 	}

diff --git a/cmd/collectors/zapi/plugins/volume/volume.go b/cmd/collectors/zapi/plugins/volume/volume.go
@@ -36,7 +36,7 @@ func (my *Volume) Init() error {
 		return err
 	}
 
-	if my.client, err = zapi.New(conf.ZapiPoller(my.ParentParams)); err != nil {
+	if my.client, err = zapi.New(conf.ZapiPoller(my.ParentParams), my.Auth); err != nil {
 		my.Logger.Error().Stack().Err(err).Msg("connecting")
 		return err
 	}

diff --git a/cmd/collectors/zapiperf/plugins/disk/disk.go b/cmd/collectors/zapiperf/plugins/disk/disk.go
@@ -114,7 +114,7 @@ func (d *Disk) Init() error {
 		return err
 	}
 
-	if d.client, err = zapi.New(conf.ZapiPoller(d.ParentParams)); err != nil {
+	if d.client, err = zapi.New(conf.ZapiPoller(d.ParentParams), d.Auth); err != nil {
 		d.Logger.Error().Stack().Err(err).Msg("connecting")
 		return err
 	}

diff --git a/cmd/collectors/zapiperf/plugins/volumetag/volumetag.go b/cmd/collectors/zapiperf/plugins/volumetag/volumetag.go
@@ -25,7 +25,7 @@ func (v *VolumeTag) Init() error {
 		return err
 	}
 
-	if v.client, err = zapi.New(conf.ZapiPoller(v.ParentParams)); err != nil {
+	if v.client, err = zapi.New(conf.ZapiPoller(v.ParentParams), v.Auth); err != nil {
 		v.Logger.Error().Stack().Err(err).Msg("connecting")
 		return err
 	}

diff --git a/cmd/poller/collector/collector.go b/cmd/poller/collector/collector.go
@@ -18,6 +18,7 @@ package collector
 
 import (
 	"errors"
+	"github.com/netapp/harvest/v2/pkg/auth"
 	"github.com/netapp/harvest/v2/pkg/conf"
 	"github.com/netapp/harvest/v2/pkg/logging"
 	"golang.org/x/text/cases"
@@ -85,7 +86,7 @@ type AbstractCollector struct {
 	Object  string           // object of the collector, describes what that collector is collecting
 	Logger  *logging.Logger  // logger used for logging
 	Status  uint8            // current state of th
-	Message string           // reason if collector is in failed state
+	Message string           // reason if a collector is in failed state
 	Options *options.Options // poller options
 	Params  *node.Node       // collector parameters
 	// note that this is a merge of poller parameters, collector conf and object conf ("subtemplate")
@@ -97,27 +98,23 @@ type AbstractCollector struct {
 	collectCount uint64                     // count of collected data points
 	// this is different from what the collector will have in its metadata, since this variable
 	// holds count independent of the poll interval of the collector, used to give stats to Poller
-	countMux    *sync.Mutex // used for atomic access to collectCount
+	countMux    *sync.Mutex       // used for atomic access to collectCount
+	Auth        *auth.Credentials // used for authing the collector
 	HostVersion string
 	HostModel   string
 	HostUUID    string
 }
 
-// New creates an AbstractCollector with the given arguments:
-// @name	- name of the collector
-// @object	- object of the collector (something that best describes the data)
-// @options	- poller options
-// @params	- collector parameters
-func New(name, object string, options *options.Options, params *node.Node) *AbstractCollector {
-	c := AbstractCollector{
+func New(name, object string, options *options.Options, params *node.Node, credentials *auth.Credentials) *AbstractCollector {
+	return &AbstractCollector{
 		Name:     name,
 		Object:   object,
 		Options:  options,
 		Logger:   logging.Get().SubLogger("collector", name+":"+object),
 		Params:   params,
 		countMux: &sync.Mutex{},
+		Auth:     credentials,
 	}
-	return &c
 }
 
 // Init initializes a collector and does the trick of "inheritance",
@@ -583,7 +580,7 @@ func (c *AbstractCollector) LoadPlugins(params *node.Node, collector Collector,
 			x.SetNameS(name)
 		}
 
-		abc = plugin.New(c.Name, c.Options, x, c.Params, c.Object)
+		abc = plugin.New(c.Name, c.Options, x, c.Params, c.Object, c.Auth)
 
 		// case 1: available as built-in plugin
 		if p = GetBuiltinPlugin(name, abc); p != nil {

diff --git a/cmd/poller/plugin/aggregator/aggregator_test.go b/cmd/poller/plugin/aggregator/aggregator_test.go
@@ -17,7 +17,7 @@ func TestMain(m *testing.M) {
 	params := node.NewS("Aggregator")
 	params.NewChildS("", "node")
 
-	abc := plugin.New("Test", nil, params, nil, "")
+	abc := plugin.New("Test", nil, params, nil, "", nil)
 	p = &Aggregator{AbstractPlugin: abc}
 
 	if err := p.Init(); err != nil {

diff --git a/cmd/poller/plugin/labelagent/label_agent_test.go b/cmd/poller/plugin/labelagent/label_agent_test.go
@@ -70,7 +70,7 @@ func TestInitPlugin(t *testing.T) {
 	// exclude instance if label "volstatus" has value which starts with "stopped_"
 	params.NewChildS("exclude_contains", "").NewChildS("", "volstatus `stop`")
 
-	abc := plugin.New("Test", nil, params, nil, "")
+	abc := plugin.New("Test", nil, params, nil, "", nil)
 	p = &LabelAgent{AbstractPlugin: abc}
 
 	if err := p.Init(); err != nil {

diff --git a/cmd/poller/plugin/metricagent/metric_agent_test.go b/cmd/poller/plugin/metricagent/metric_agent_test.go
@@ -32,7 +32,7 @@ func TestInitPlugin(t *testing.T) {
 	// create metric "transmission_rate", which is division of the metric value of transfer.bytes_transferred by transfer.total_duration
 	params.NewChildS("compute_metric", "").NewChildS("", "transmission_rate DIVIDE transfer.bytes_transferred transfer.total_duration")
 
-	abc := plugin.New("Test", nil, params, nil, "")
+	abc := plugin.New("Test", nil, params, nil, "", nil)
 	p = &MetricAgent{AbstractPlugin: abc}
 
 	if err := p.Init(); err != nil {