feat: add harvest.yml environment variable expansion

cmd/tools/doctor/doctor.go

@@ -25,6 +25,7 @@ type options struct {
 	MergeTemplate      string
 	zapiDataCenterName string
 	restDataCenterName string
+	expandVar          bool
 }
 
 var opts = &options{
@@ -105,6 +106,15 @@ func doDoctor(path string) string {
 		fmt.Printf("error reading config file. err=%+v\n", err)
 		return ""
 	}
+
+	if opts.expandVar {
+		contents, err = conf.ExpandVars(contents)
+		if err != nil {
+			fmt.Printf("error reading config file. err=%+v\n", err)
+			return ""
+		}
+	}
+
 	parentRoot, err := printRedactedConfig(path, contents)
 	if err != nil {
 		fmt.Printf("error processing parent config file=[%s] %+v\n", path, err)
@@ -592,8 +602,9 @@ func init() {
 		"print",
 		"p",
 		false,
-		"print config to console with sensitive info redacted",
+		"Print config to console with sensitive info redacted",
 	)
 
 	Cmd.Flags().StringVar(&opts.Color, "color", "auto", "When to use colors. One of: auto | always | never. Auto will guess based on tty.")
+	Cmd.Flags().BoolVar(&opts.expandVar, "expand-var", false, "Expand environment variables in config (default: false)")
 }

docs/configure-harvest-advanced.md

@@ -1,3 +1,41 @@
-This chapter describes additional advanced configuration possibilities of NetApp Harvest. For a typical installation
+This chapter describes additional advanced configuration possibilities of NetApp Harvest. For a typical installation, 
 this level of detail is likely not needed.
 
+## Variable Expansion
+
+The `harvest.yml` configuration file supports variable expansion.
+This allows you to use environment variables in the configuration file.
+Harvest will expand strings with the format `$__env{VAR}` or `${VAR}`,
+replacing the variable `VAR` with the value of the environment variable.
+If the environment variable is not set, the variable will be replaced with an empty string.
+
+Here's an example snippet from `harvest.yml`:
+
+```yaml
+Pollers:
+  netapp_frankfurt:
+    addr: 10.0.1.2
+    username: $__env{NETAPP_FRANKFURT_RO_USER}
+  netapp_london:
+    addr: uk-cluster
+    username: ${NETAPP_LONDON_RO_USER}
+  netapp_rtp:
+    addr: 10.0.1.4
+    username: $__env{NETAPP_RTP_RO_USER}
+```
+
+If you set the environment variable `NETAPP_FRANKFURT_RO_USER` to `harvest1` and `NETAPP_LONDON_RO_USER` to `harvest2`,
+the configuration will be expanded to:
+
+```yaml
+Pollers:
+  netapp_frankfurt:
+    addr: 10.0.1.2
+    username: harvest1
+  netapp_london:
+    addr: uk-cluster
+    username: harvest2
+  netapp_rtp:
+    addr: 10.0.1.4
+    username: 
+```

pkg/conf/conf.go

@@ -146,6 +146,11 @@ func unmarshalConfig(contents []byte) (*HarvestConfig, error) {
 		err           error
 	)
 
+	contents, err = ExpandVars(contents)
+	if err != nil {
+		return nil, fmt.Errorf("error expanding vars: %w", err)
+	}
+
 	err = yaml.Unmarshal(contents, &cfg)
 	if err != nil {
 		return nil, fmt.Errorf("error unmarshalling config: %w", err)

pkg/conf/expanders.go

@@ -0,0 +1,48 @@
+package conf
+
+import (
+	"bytes"
+	"fmt"
+	"os"
+	"regexp"
+)
+
+var patternRe = regexp.MustCompile(`\$(|__\w+){([^}]+)}`)
+
+func ExpandVars(in []byte) ([]byte, error) {
+	empty := []byte("")
+	matches := patternRe.FindAllSubmatch(in, -1)
+
+	for _, match := range matches {
+		if len(match) < 3 {
+			return empty, fmt.Errorf("regex error, got %d results back for match, expected 3", len(match))
+		}
+
+		if bytes.Equal(match[1], []byte("__env")) || bytes.Equal(match[1], empty) {
+			updated, err := expandEnv(match[2])
+			if err != nil {
+				return empty, err
+			}
+
+			in = bytes.Replace(in, match[0], updated, 1)
+		}
+	}
+
+	return in, nil
+}
+
+func expandEnv(bytes []byte) ([]byte, error) {
+	s := string(bytes)
+	envValue := os.Getenv(s)
+
+	// if env variable is hostname, and the var is empty, use os.Hostname
+	if s == "HOSTNAME" && envValue == "" {
+		hostname, err := os.Hostname()
+		if err != nil {
+			return nil, err
+		}
+		return []byte(hostname), nil
+	}
+
+	return []byte(envValue), nil
+}

pkg/conf/expanders_test.go

@@ -0,0 +1,42 @@
+package conf
+
+import (
+	"testing"
+)
+
+func Test_expandVars(t *testing.T) {
+	tests := []struct {
+		name    string
+		in      string
+		want    string
+		setEnvs map[string]string
+	}{
+		{name: "empty", in: "", want: ""},
+		{name: "no env var", in: "hi", want: "hi"},
+		{name: "missing an underscore", in: "$_env{NETAPP_U2}", want: "$_env{NETAPP_U2}"},
+
+		{name: "empty var", in: "$__env{ENV}", want: ""},
+		{name: "empty var2", in: "${ENV}", want: ""},
+
+		{name: "env1", in: "$__env{VAR1} is cool", want: "Harvest is cool", setEnvs: map[string]string{"VAR1": "Harvest"}},
+		{name: "dup", in: "$__env{VAR1} and $__env{VAR1}", want: "Harvest and Harvest", setEnvs: map[string]string{"VAR1": "Harvest"}},
+		{name: "both", in: "a $__env{VAR1} $__env{VAR2}", want: "a b c", setEnvs: map[string]string{"VAR1": "b", "VAR2": "c"}},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			for k, v := range tt.setEnvs {
+				t.Setenv(k, v)
+			}
+			g, err := ExpandVars([]byte(tt.in))
+			got := string(g)
+			if err != nil {
+				t.Errorf("ExpandVars() error %v", err)
+				return
+			}
+			if got != tt.want {
+				t.Errorf("ExpandVars() got %v, want %v", got, tt.want)
+			}
+		})
+	}
+}