feat: Harvest should define and document auth precedence

Fixes: #1867

cmd/collectors/storagegrid/rest/client.go

@@ -106,20 +106,11 @@ func New(poller *conf.Poller, timeout time.Duration, c *auth.Credentials) (*Clie
 		useInsecureTLS = false
 	}
 
-	// check if a credentials file is being used and if so, parse and use the values from it
-	if poller.CredentialsFile != "" {
-		err := conf.ReadCredentialsFile(poller.CredentialsFile, poller)
-		if err != nil {
-			client.Logger.Error().
-				Err(err).
-				Str("credPath", poller.CredentialsFile).
-				Str("poller", poller.Name).
-				Msg("Unable to read credentials file")
-			return nil, err
-		}
+	pollerAuth, err := c.GetPollerAuth()
+	if err != nil {
+		return nil, err
 	}
-	// set authentication method
-	if poller.AuthStyle == "certificate_auth" {
+	if pollerAuth.IsCert {
 		certPath := poller.SslCert
 		keyPath := poller.SslKey
 		if certPath == "" {
@@ -138,7 +129,7 @@ func New(poller *conf.Poller, timeout time.Duration, c *auth.Credentials) (*Clie
 		}
 	} else {
 		username := poller.Username
-		password := c.Password()
+		password := pollerAuth.Password
 		client.username = username
 		if username == "" {
 			return nil, errs.New(errs.ErrMissingParam, "username")

cmd/poller/poller.go

@@ -249,9 +249,17 @@ func (p *Poller) Init() error {
 	} else {
 		p.target = p.params.Addr
 	}
+
+	// create a shared auth service that all collectors will use
+	p.auth = auth.NewCredentials(p.params, logger)
+	pollerAuth, err := p.auth.GetPollerAuth()
+	if err != nil {
+		return err
+	}
+
 	// check optional parameter auth_style
 	// if certificates are missing use default paths
-	if p.params.AuthStyle == "certificate_auth" {
+	if pollerAuth.IsCert {
 		if p.params.SslCert == "" {
 			fp := path.Join(p.options.HomePath, "cert/", p.options.Hostname+".pem")
 			p.params.SslCert = fp
@@ -272,9 +280,6 @@ func (p *Poller) Init() error {
 		}
 	}
 
-	// create a shared auth service that all collectors will use
-	p.auth = auth.NewCredentials(p.params, logger)
-
 	// initialize our metadata, the metadata will host status of our
 	// collectors and exporters, as well as ping stats to target host
 	p.loadMetadata()

cmd/tools/rest/client.go

@@ -86,20 +86,12 @@ func New(poller *conf.Poller, timeout time.Duration, auth *auth.Credentials) (*C
 		useInsecureTLS = false
 	}
 
-	// check if a credentials file is being used and if so, parse and use the values from it
-	if poller.CredentialsFile != "" {
-		err := conf.ReadCredentialsFile(poller.CredentialsFile, poller)
-		if err != nil {
-			client.Logger.Error().
-				Err(err).
-				Str("credPath", poller.CredentialsFile).
-				Str("poller", poller.Name).
-				Msg("Unable to read credentials file")
-			return nil, err
-		}
+	pollerAuth, err := auth.GetPollerAuth()
+	if err != nil {
+		return nil, err
 	}
-	// set authentication method
-	if poller.AuthStyle == "certificate_auth" {
+
+	if pollerAuth.IsCert {
 		sslCertPath := poller.SslCert
 		keyPath := poller.SslKey
 		caCertPath := poller.CaCertPath
@@ -137,20 +129,19 @@ func New(poller *conf.Poller, timeout time.Duration, auth *auth.Credentials) (*C
 				InsecureSkipVerify: useInsecureTLS}, //nolint:gosec
 		}
 	} else {
-		username := poller.Username
-		password := auth.Password()
-		client.username = username
-		if username == "" {
+		if pollerAuth.Username == "" {
 			return nil, errs.New(errs.ErrMissingParam, "username")
-		} else if password == "" {
+		} else if pollerAuth.Password == "" {
 			return nil, errs.New(errs.ErrMissingParam, "password")
 		}
+		client.username = pollerAuth.Username
 
 		transport = &http.Transport{
 			Proxy:           http.ProxyFromEnvironment,
 			TLSClientConfig: &tls.Config{InsecureSkipVerify: useInsecureTLS}, //nolint:gosec
 		}
 	}
+
 	transport.DialContext = (&net.Dialer{Timeout: DefaultDialerTimeout}).DialContext
 	httpclient = &http.Client{Transport: transport, Timeout: timeout}
 	client.client = httpclient

docs/configure-harvest-basic.md

@@ -11,11 +11,11 @@ All pollers are defined in `harvest.yml`, the main configuration file of Harvest
 | `addr`                 | required by some collectors                    | IPv4 or FQDN of the target system                                                                                                                                                                                                                                                                                                                                         |                    |
 | `collectors`           | **required**                                   | List of collectors to run for this poller                                                                                                                                                                                                                                                                                                                                 |                    |
 | `exporters`            | **required**                                   | List of exporter names from the `Exporters` section. Note: this should be the name of the exporter (e.g. `prometheus1`), not the value of the `exporter` key (e.g. `Prometheus`)                                                                                                                                                                                          |                    |
-| `auth_style`           | required by Zapi* collectors                   | Either `basic_auth` or `certificate_auth`                                                                                                                                                                                                                                                                                                                                 | `basic_auth`       |
+| `auth_style`           | required by Zapi* collectors                   | Either `basic_auth` or `certificate_auth` See [authentication](#authentication) for details                                                                                                                                                                                                                                                                               | `basic_auth`       |
 | `username`, `password` | required if `auth_style` is `basic_auth`       |                                                                                                                                                                                                                                                                                                                                                                           |                    |
 | `ssl_cert`, `ssl_key`  | optional if `auth_style` is `certificate_auth` | Absolute paths to SSL (client) certificate and key used to authenticate with the target system.<br /><br />If not provided, the poller will look for `<hostname>.key` and `<hostname>.pem` in `$HARVEST_HOME/cert/`.<br/><br/>To create certificates for ONTAP systems, see [using certificate authentication](prepare-cdot-clusters.md#using-certificate-authentication) |                    |
 | `use_insecure_tls`     | optional, bool                                 | If true, disable TLS verification when connecting to ONTAP cluster                                                                                                                                                                                                                                                                                                        | false              |
-| `credentials_file`     | optional, string                               | Path to a yaml file that contains cluster credentials. The file should have the same shape as `harvest.yml`. See [here](configure-harvest-basic.md#credentials-file) for examples. Path can be relative to `harvest.yml` or absolute                                                                                                                                      |                    |          
+| `credentials_file`     | optional, string                               | Path to a yaml file that contains cluster credentials. The file should have the same shape as `harvest.yml`. See [here](configure-harvest-basic.md#credentials-file) for examples. Path can be relative to `harvest.yml` or absolute.                                                                                                                                     |                    |          
 | `credentials_script`   | optional, section                              | Section that defines how Harvest should fetch credentials via external script. See [here](configure-harvest-basic.md#credentials-script) for details.                                                                                                                                                                                                                     |                    |          
 | `tls_min_version`      | optional, string                               | Minimum TLS version to use when connecting to ONTAP cluster: One of tls10, tls11, tls12 or tls13                                                                                                                                                                                                                                                                          | Platform decides   | 
 | `labels`               | optional, list of key-value pairs              | Each of the key-value pairs will be added to a poller's metrics. Details [below](configure-harvest-basic.md#labels)                                                                                                                                                                                                                                                       |                    |
@@ -131,6 +131,40 @@ node_vol_cifs_write_data{org="meg",ns="rtp",datacenter="DC-01",cluster="cluster-
 Keep in mind that each unique combination of key-value pairs increases the amount of stored data. Use them sparingly.
 See [PrometheusNaming](https://prometheus.io/docs/practices/naming/#labels) for details.
 
+# Authentication
+
+When authenticating with ONTAP and StorageGRID clusters,
+Harvest supports both client certificates and basic authentication.
+
+These methods of authentication are defined in the `Pollers` or `Defaults` section of your `harvest.yml` using one or more
+of the following parameters.
+
+| parameter            | description                                                                | default      | Link                        |
+|----------------------|----------------------------------------------------------------------------|--------------|-----------------------------|
+| `auth_sytle`         | One of `basic_auth` or `certificate_auth`                                  | `basic_auth` | [link](#Pollers)            |
+| `username`           | Username used for authenticating to the remote system                      |              | [link](#Pollers)            |
+| `password`           | Password used for authenticating to the remote system                      |              | [link](#Pollers)            |
+| `credentials_file`   | Relative or absolute path to a yaml file that contains cluster credentials |              | [link](#credentials-file)   |
+| `credentials_script` | External script Harvest executes to retrieve credentials                   |              | [link](#credentials-script) |
+
+When multiple authentication parameters are defined at the same time,
+Harvest tries each method listed below, in the following order, to resolve authentication requests. 
+The first method that returns a non-empty password stops the search. 
+
+When these parameters exist in both the `Pollers` and `Defaults` section,
+the `Pollers` section will be consulted before the `Defaults`.
+
+| section    | parameter                                           |
+|------------|-----------------------------------------------------|
+| `Pollers`  | auth_style: `certificate_auth`                      |
+| `Pollers`  | auth_style: `basic_auth` with username and password |
+| `Pollers`  | `credentials_script`                                |
+| `Pollers`  | `credentials_script`                                |
+| `Defaults` | auth_style: `certificate_auth`                      |
+| `Defaults` | auth_style: `basic_auth` with username and password |
+| `Defaults` | `credentials_script`                                |
+| `Defaults` | `credentials_script`                                |
+
 ## Credentials File
 
 If you would rather not list cluster credentials in your `harvest.yml`, you can use the `credentials_file` section

pkg/api/ontapi/zapi/client.go

@@ -102,20 +102,11 @@ func New(poller *conf.Poller, c *auth.Credentials) (*Client, error) {
 		useInsecureTLS = *poller.UseInsecureTLS
 	}
 
-	if poller.CredentialsFile != "" {
-		err := conf.ReadCredentialsFile(poller.CredentialsFile, poller)
-		if err != nil {
-			client.Logger.Error().
-				Err(err).
-				Str("credPath", poller.CredentialsFile).
-				Str("poller", poller.Name).
-				Msg("Unable to read credentials file")
-			return nil, err
-		}
+	pollerAuth, err := c.GetPollerAuth()
+	if err != nil {
+		return nil, err
 	}
-	// set authentication method
-	if poller.AuthStyle == "certificate_auth" {
-
+	if pollerAuth.IsCert {
 		sslCertPath := poller.SslCert
 		keyPath := poller.SslKey
 		caCertPath := poller.CaCertPath
@@ -154,7 +145,7 @@ func New(poller *conf.Poller, c *auth.Credentials) (*Client, error) {
 			},
 		}
 	} else {
-		password := c.Password()
+		password := pollerAuth.Password
 		if poller.Username == "" {
 			return nil, errs.New(errs.ErrMissingParam, "username")
 		} else if password == "" {

pkg/api/ontapi/zapi/client_test.go

@@ -20,27 +20,27 @@ func TestNew(t *testing.T) {
 	certificatePollerFail := node.NewS("test")
 	certificatePollerFail.NewChildS("datacenter", "cluster-01")
 	certificatePollerFail.NewChildS("addr", "localhost")
-	certificatePollerFail.NewChildS("auth_style", "certificate_auth")
+	certificatePollerFail.NewChildS("auth_style", conf.CertificateAuth)
 	certificatePollerFail.NewChildS("use_insecure_tls", "false")
 
 	certificatePollerPass := node.NewS("test")
 	certificatePollerPass.NewChildS("datacenter", "cluster-01")
 	certificatePollerPass.NewChildS("addr", "localhost")
-	certificatePollerPass.NewChildS("auth_style", "certificate_auth")
+	certificatePollerPass.NewChildS("auth_style", conf.CertificateAuth)
 	certificatePollerPass.NewChildS("use_insecure_tls", "false")
 	certificatePollerPass.NewChildS("ssl_cert", "testdata/ubuntu.pem")
 	certificatePollerPass.NewChildS("ssl_key", "testdata/ubuntu.key")
 
 	basicAuthPollerFail := node.NewS("test")
 	basicAuthPollerFail.NewChildS("datacenter", "cluster-01")
 	basicAuthPollerFail.NewChildS("addr", "localhost")
-	basicAuthPollerFail.NewChildS("auth_style", "basic_auth")
+	basicAuthPollerFail.NewChildS("auth_style", conf.BasicAuth)
 	basicAuthPollerFail.NewChildS("use_insecure_tls", "false")
 
 	basicAuthPollerPass := node.NewS("test")
 	basicAuthPollerPass.NewChildS("datacenter", "cluster-01")
 	basicAuthPollerPass.NewChildS("addr", "localhost")
-	basicAuthPollerPass.NewChildS("auth_style", "basic_auth")
+	basicAuthPollerPass.NewChildS("auth_style", conf.BasicAuth)
 	basicAuthPollerPass.NewChildS("use_insecure_tls", "false")
 	basicAuthPollerPass.NewChildS("username", "username")
 	basicAuthPollerPass.NewChildS("password", "password")

pkg/auth/auth.go

@@ -33,25 +33,29 @@ type Credentials struct {
 }
 
 func (c *Credentials) Password() string {
-	if c.poller.CredentialsScript.Path == "" {
-		return c.poller.Password
+	return c.password(c.poller)
+}
+
+func (c *Credentials) password(poller *conf.Poller) string {
+	if poller.CredentialsScript.Path == "" {
+		return poller.Password
 	}
 	c.authMu.Lock()
 	defer c.authMu.Unlock()
 	if time.Now().After(c.nextUpdate) {
-		c.poller.Password = c.fetchPassword()
+		poller.Password = c.fetchPassword(poller)
 		c.setNextUpdate()
 	}
-	return c.poller.Password
+	return poller.Password
 }
 
-func (c *Credentials) fetchPassword() string {
-	path, err := exec.LookPath(c.poller.CredentialsScript.Path)
+func (c *Credentials) fetchPassword(p *conf.Poller) string {
+	path, err := exec.LookPath(p.CredentialsScript.Path)
 	if err != nil {
-		c.logger.Error().Err(err).Str("path", c.poller.CredentialsScript.Path).Msg("Credentials script lookup failed")
+		c.logger.Error().Err(err).Str("path", p.CredentialsScript.Path).Msg("Credentials script lookup failed")
 		return ""
 	}
-	timeout := c.poller.CredentialsScript.Timeout
+	timeout := p.CredentialsScript.Timeout
 	if timeout == "" {
 		timeout = defaultTimeout
 	}
@@ -65,7 +69,7 @@ func (c *Credentials) fetchPassword() string {
 	}
 	ctx, cancelFunc := context.WithTimeout(context.Background(), duration)
 	defer cancelFunc()
-	cmd := exec.CommandContext(ctx, path, c.poller.Addr, c.poller.Username)
+	cmd := exec.CommandContext(ctx, path, p.Addr, p.Username)
 
 	// Create process group - so we can kill any forked processes
 	cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
@@ -120,3 +124,58 @@ func (c *Credentials) setNextUpdate() {
 	}
 	c.nextUpdate = time.Now().Add(duration)
 }
+
+type PollerAuth struct {
+	Username string
+	Password string
+	IsCert   bool
+}
+
+func (c *Credentials) GetPollerAuth() (PollerAuth, error) {
+	auth, err := getPollerAuth(c, c.poller)
+	if err != nil {
+		return PollerAuth{}, err
+	}
+	if auth.Password != "" {
+		c.poller.Username = auth.Username
+		c.poller.Password = auth.Password
+		return auth, nil
+	}
+
+	if conf.Config.Defaults == nil {
+		return auth, nil
+	}
+
+	copyDefault := *conf.Config.Defaults
+	copyDefault.Name = c.poller.Name
+	defaultAuth, err := getPollerAuth(c, &copyDefault)
+	if err != nil {
+		return PollerAuth{}, err
+	}
+	if auth.Username != "" {
+		defaultAuth.Username = auth.Username
+	}
+	c.poller.Username = defaultAuth.Username
+	c.poller.Password = defaultAuth.Password
+	return defaultAuth, nil
+}
+
+func getPollerAuth(c *Credentials, poller *conf.Poller) (PollerAuth, error) {
+	if poller.AuthStyle == conf.CertificateAuth {
+		return PollerAuth{IsCert: true}, nil
+	}
+	if poller.Password != "" {
+		return PollerAuth{Username: poller.Username, Password: poller.Password}, nil
+	}
+	if poller.CredentialsScript.Path != "" {
+		return PollerAuth{Username: poller.Username, Password: c.password(poller)}, nil
+	}
+	if poller.CredentialsFile != "" {
+		err := conf.ReadCredentialFile(poller.CredentialsFile, poller)
+		if err != nil {
+			return PollerAuth{}, err
+		}
+		return PollerAuth{Username: poller.Username, Password: poller.Password}, nil
+	}
+	return PollerAuth{Username: poller.Username}, nil
+}