Allow environment variables in config

config/config.go

@@ -20,6 +20,7 @@ package config
 import (
 	"errors"
 	"fmt"
+	"os"
 	"strings"
 
 	"github.com/spf13/viper"
@@ -244,12 +245,19 @@ func loadExtensions(v *viper.Viper, factories map[string]extension.Factory) (con
 		extensionCfg.SetType(typeStr)
 		extensionCfg.SetName(fullName)
 
-		// Now that the default config struct is created we can Unmarshal into it
-		// and it will apply user-defined config on top of the default.
-		if err := subViper.UnmarshalKey(key, extensionCfg); err != nil {
-			return nil, &configError{
-				code: errUnmarshalError,
-				msg:  fmt.Sprintf("error reading settings for extension type %q: %v", typeStr, err),
+		// Unmarsh only the subconfig for this extension.
+		sv := subViper.Sub(key)
+		if sv != nil {
+			// Before unmarshilng first expand all environment variables.
+			sv = expandEnvConfig(sv)
+
+			// Now that the default config struct is created we can Unmarshal into it
+			// and it will apply user-defined config on top of the default.
+			if err := sv.Unmarshal(extensionCfg); err != nil {
+				return nil, &configError{
+					code: errUnmarshalError,
+					msg:  fmt.Sprintf("error reading settings for extension type %q: %v", typeStr, err),
+				}
 			}
 		}
 
@@ -322,18 +330,25 @@ func loadReceivers(v *viper.Viper, factories map[string]receiver.Factory) (confi
 		receiverCfg.SetType(typeStr)
 		receiverCfg.SetName(fullName)
 
-		// Now that the default config struct is created we can Unmarshal into it
-		// and it will apply user-defined config on top of the default.
-		customUnmarshaler := factory.CustomUnmarshaler()
-		if customUnmarshaler != nil {
-			// This configuration requires a custom unmarshaler, use it.
-			err = customUnmarshaler(subViper, key, receiverCfg)
-		} else {
-			// Standard viper unmarshaler is fine.
-			// TODO(ccaraman): UnmarshallExact should be used to catch erroneous config entries.
-			// 	This leads to quickly identifying config values that are not supported and reduce confusion for
-			// 	users.
-			err = subViper.UnmarshalKey(key, receiverCfg)
+		// Unmarsh only the subconfig for this receiver.
+		sv := subViper.Sub(key)
+		if sv != nil {
+			// Before unmarshilng first expand all environment variables.
+			sv = expandEnvConfig(sv)
+
+			// Now that the default config struct is created we can Unmarshal into it
+			// and it will apply user-defined config on top of the default.
+			customUnmarshaler := factory.CustomUnmarshaler()
+			if customUnmarshaler != nil {
+				// This configuration requires a custom unmarshaler, use it.
+				err = customUnmarshaler(subViper, key, receiverCfg)
+			} else {
+				// Standard viper unmarshaler is fine.
+				// TODO(ccaraman): UnmarshallExact should be used to catch erroneous config entries.
+				// 	This leads to quickly identifying config values that are not supported and reduce confusion for
+				// 	users.
+				err = sv.Unmarshal(receiverCfg)
+			}
 		}
 
 		if err != nil {
@@ -350,7 +365,6 @@ func loadReceivers(v *viper.Viper, factories map[string]receiver.Factory) (confi
 			}
 		}
 		receivers[fullName] = receiverCfg
-
 	}
 
 	return receivers, nil
@@ -399,12 +413,19 @@ func loadExporters(v *viper.Viper, factories map[string]exporter.Factory) (confi
 		exporterCfg.SetType(typeStr)
 		exporterCfg.SetName(fullName)
 
-		// Now that the default config struct is created we can Unmarshal into it
-		// and it will apply user-defined config on top of the default.
-		if err := subViper.UnmarshalKey(key, exporterCfg); err != nil {
-			return nil, &configError{
-				code: errUnmarshalError,
-				msg:  fmt.Sprintf("error reading settings for exporter type %q: %v", typeStr, err),
+		// Unmarsh only the subconfig for this exporter.
+		sv := subViper.Sub(key)
+		if sv != nil {
+			// Before unmarshilng first expand all environment variables.
+			sv = expandEnvConfig(sv)
+
+			// Now that the default config struct is created we can Unmarshal into it
+			// and it will apply user-defined config on top of the default.
+			if err := sv.Unmarshal(exporterCfg); err != nil {
+				return nil, &configError{
+					code: errUnmarshalError,
+					msg:  fmt.Sprintf("error reading settings for exporter type %q: %v", typeStr, err),
+				}
 			}
 		}
 
@@ -456,12 +477,19 @@ func loadProcessors(v *viper.Viper, factories map[string]processor.Factory) (con
 		processorCfg.SetType(typeStr)
 		processorCfg.SetName(fullName)
 
-		// Now that the default config struct is created we can Unmarshal into it
-		// and it will apply user-defined config on top of the default.
-		if err := subViper.UnmarshalKey(key, processorCfg); err != nil {
-			return nil, &configError{
-				code: errUnmarshalError,
-				msg:  fmt.Sprintf("error reading settings for processor type %q: %v", typeStr, err),
+		// Unmarsh only the subconfig for this processor.
+		sv := subViper.Sub(key)
+		if sv != nil {
+			// Before unmarshilng first expand all environment variables.
+			sv = expandEnvConfig(sv)
+
+			// Now that the default config struct is created we can Unmarshal into it
+			// and it will apply user-defined config on top of the default.
+			if err := sv.Unmarshal(processorCfg); err != nil {
+				return nil, &configError{
+					code: errUnmarshalError,
+					msg:  fmt.Sprintf("error reading settings for processor type %q: %v", typeStr, err),
+				}
 			}
 		}
 
@@ -813,3 +841,40 @@ func validateProcessors(cfg *configmodels.Config) {
 		}
 	}
 }
+
+// expandEnvConfig creates a new viper config with expanded values for all the values (simple, list or map value).
+// It does not expand the keys.
+// Need to copy everything because of a bug in Viper: Set a value "map[string]interface{}" where a key has a ".",
+// then AllSettings will return the previous value not the newly set one.
+func expandEnvConfig(vip *viper.Viper) *viper.Viper {
+	newCfg := make(map[string]interface{})
+	for k, v := range vip.AllSettings() {
+		newCfg[k] = expandStringValues(v)
+	}
+	newVip := viper.New()
+	newVip.MergeConfigMap(newCfg)
+	return newVip
+}
+
+func expandStringValues(value interface{}) interface{} {
+	switch v := value.(type) {
+	default:
+		return v
+	case string:
+		return os.ExpandEnv(v)
+	case []interface{}:
+		// Viper treats all the slices as []interface{} (at least in what the otelsvc tests).
+		nslice := make([]interface{}, 0, len(v))
+		for _, vint := range v {
+			nslice = append(nslice, expandStringValues(vint))
+		}
+		return nslice
+	case map[string]interface{}:
+		nmap := make(map[string]interface{}, len(v))
+		// Viper treats all the maps as [string]interface{} (at least in what the otelsvc tests).
+		for k, vint := range v {
+			nmap[k] = expandStringValues(vint)
+		}
+		return nmap
+	}
+}

config/config_test.go

@@ -15,6 +15,7 @@
 package config
 
 import (
+	"os"
 	"path"
 	"testing"
 
@@ -124,6 +125,171 @@ func TestDecodeConfig(t *testing.T) {
 		"Did not load pipeline config correctly")
 }
 
+func TestSimpleConfig(t *testing.T) {
+	var testCases = []struct {
+		name string // test case name (also file name containing config yaml)
+	}{
+		{name: "simple-config-with-no-env"},
+		{name: "simple-config-with-partial-env"},
+		{name: "simple-config-with-all-env"},
+	}
+
+	const extensionExtra = "some extension string"
+	const extensionExtraMapValue = "some extension map value"
+	const extensionExtraListElement = "some extension list value"
+	os.Setenv("EXTENSIONS_EXAMPLEEXTENSION_EXTRA", extensionExtra)
+	os.Setenv("EXTENSIONS_EXAMPLEEXTENSION_EXTRA_MAP_EXT_VALUE_1", extensionExtraMapValue+"_1")
+	os.Setenv("EXTENSIONS_EXAMPLEEXTENSION_EXTRA_MAP_EXT_VALUE_2", extensionExtraMapValue+"_2")
+	os.Setenv("EXTENSIONS_EXAMPLEEXTENSION_EXTRA_LIST_VALUE_1", extensionExtraListElement+"_1")
+	os.Setenv("EXTENSIONS_EXAMPLEEXTENSION_EXTRA_LIST_VALUE_2", extensionExtraListElement+"_2")
+
+	const receiverExtra = "some receiver string"
+	const receiverExtraMapValue = "some receiver map value"
+	const receiverExtraListElement = "some receiver list value"
+	os.Setenv("RECEIVERS_EXAMPLERECEIVER_EXTRA", receiverExtra)
+	os.Setenv("RECEIVERS_EXAMPLERECEIVER_EXTRA_MAP_RECV_VALUE_1", receiverExtraMapValue+"_1")
+	os.Setenv("RECEIVERS_EXAMPLERECEIVER_EXTRA_MAP_RECV_VALUE_2", receiverExtraMapValue+"_2")
+	os.Setenv("RECEIVERS_EXAMPLERECEIVER_EXTRA_LIST_VALUE_1", receiverExtraListElement+"_1")
+	os.Setenv("RECEIVERS_EXAMPLERECEIVER_EXTRA_LIST_VALUE_2", receiverExtraListElement+"_2")
+
+	const processorExtra = "some processor string"
+	const processorExtraMapValue = "some processor map value"
+	const processorExtraListElement = "some processor list value"
+	os.Setenv("PROCESSORS_EXAMPLEPROCESSOR_EXTRA", processorExtra)
+	os.Setenv("PROCESSORS_EXAMPLEPROCESSOR_EXTRA_MAP_PROC_VALUE_1", processorExtraMapValue+"_1")
+	os.Setenv("PROCESSORS_EXAMPLEPROCESSOR_EXTRA_MAP_PROC_VALUE_2", processorExtraMapValue+"_2")
+	os.Setenv("PROCESSORS_EXAMPLEPROCESSOR_EXTRA_LIST_VALUE_1", processorExtraListElement+"_1")
+	os.Setenv("PROCESSORS_EXAMPLEPROCESSOR_EXTRA_LIST_VALUE_2", processorExtraListElement+"_2")
+
+	const exporterExtra = "some exporter string"
+	const exporterExtraMapValue = "some exporter map value"
+	const exporterExtraListElement = "some exporter list value"
+	os.Setenv("EXPORTERS_EXAMPLEEXPORTER_EXTRA_INT", "65")
+	os.Setenv("EXPORTERS_EXAMPLEEXPORTER_EXTRA", exporterExtra)
+	os.Setenv("EXPORTERS_EXAMPLEEXPORTER_EXTRA_MAP_EXP_VALUE_1", exporterExtraMapValue+"_1")
+	os.Setenv("EXPORTERS_EXAMPLEEXPORTER_EXTRA_MAP_EXP_VALUE_2", exporterExtraMapValue+"_2")
+	os.Setenv("EXPORTERS_EXAMPLEEXPORTER_EXTRA_LIST_VALUE_1", exporterExtraListElement+"_1")
+	os.Setenv("EXPORTERS_EXAMPLEEXPORTER_EXTRA_LIST_VALUE_2", exporterExtraListElement+"_2")
+
+	defer func() {
+		os.Unsetenv("EXTENSIONS_EXAMPLEEXTENSION_EXTRA")
+		os.Unsetenv("EXTENSIONS_EXAMPLEEXTENSION_EXTRA_MAP_EXT_VALUE")
+		os.Unsetenv("EXTENSIONS_EXAMPLEEXTENSION_EXTRA_LIST_VALUE_1")
+
+		os.Unsetenv("RECEIVERS_EXAMPLERECEIVER_EXTRA")
+		os.Unsetenv("RECEIVERS_EXAMPLERECEIVER_EXTRA_MAP_RECV_VALUE")
+		os.Unsetenv("RECEIVERS_EXAMPLERECEIVER_EXTRA_LIST_VALUE_1")
+
+		os.Unsetenv("PROCESSORS_EXAMPLEPROCESSOR_EXTRA")
+		os.Unsetenv("PROCESSORS_EXAMPLEPROCESSOR_EXTRA_MAP_PROC_VALUE")
+		os.Unsetenv("PROCESSORS_EXAMPLEPROCESSOR_EXTRA_LIST_VALUE_1")
+
+		os.Unsetenv("EXPORTERS_EXAMPLEEXPORTER_EXTRA_INT")
+		os.Unsetenv("EXPORTERS_EXAMPLEEXPORTER_EXTRA")
+		os.Unsetenv("EXPORTERS_EXAMPLEEXPORTER_EXTRA_MAP_EXP_VALUE")
+		os.Unsetenv("EXPORTERS_EXAMPLEEXPORTER_EXTRA_LIST_VALUE_1")
+	}()
+
+	for _, test := range testCases {
+		t.Logf("TEST[%s]", test.name)
+		factories, err := ExampleComponents()
+		assert.Nil(t, err)
+
+		// Load the config
+		config, err := LoadConfigFile(t, path.Join(".", "testdata", test.name+".yaml"), factories)
+		if err != nil {
+			t.Fatalf("TEST[%s] unable to load config, %v", test.name, err)
+		}
+
+		// Verify extensions.
+		assert.Equalf(t, 1, len(config.Extensions), "TEST[%s]", test.name)
+		assert.Truef(t, config.Extensions["exampleextension"].IsEnabled(), "TEST[%s]", test.name)
+		assert.Equalf(t,
+			&ExampleExtension{
+				ExtensionSettings: configmodels.ExtensionSettings{
+					TypeVal: "exampleextension",
+					NameVal: "exampleextension",
+				},
+				ExtraSetting:     extensionExtra,
+				ExtraMapSetting:  map[string]string{"ext-1": extensionExtraMapValue + "_1", "ext-2": extensionExtraMapValue + "_2"},
+				ExtraListSetting: []string{extensionExtraListElement + "_1", extensionExtraListElement + "_2"},
+			},
+			config.Extensions["exampleextension"],
+			"TEST[%s] Did not load extension config correctly", test.name)
+
+		// Verify service.
+		assert.Equalf(t, 1, len(config.Service.Extensions), "TEST[%s]", test.name)
+		assert.Equalf(t, "exampleextension", config.Service.Extensions[0], "TEST[%s]", test.name)
+
+		// Verify receivers
+		assert.Equalf(t, 1, len(config.Receivers), "TEST[%s]", test.name)
+		assert.Truef(t, config.Receivers["examplereceiver"].IsEnabled(), "TEST[%s]", test.name)
+
+		assert.Equalf(t,
+			&ExampleReceiver{
+				ReceiverSettings: configmodels.ReceiverSettings{
+					TypeVal:  "examplereceiver",
+					NameVal:  "examplereceiver",
+					Endpoint: "127.0.0.1:1234",
+				},
+				ExtraSetting:     receiverExtra,
+				ExtraMapSetting:  map[string]string{"recv.1": receiverExtraMapValue + "_1", "recv.2": receiverExtraMapValue + "_2"},
+				ExtraListSetting: []string{receiverExtraListElement + "_1", receiverExtraListElement + "_2"},
+			},
+			config.Receivers["examplereceiver"],
+			"TEST[%s] Did not load receiver config correctly", test.name)
+
+		// Verify exporters
+		assert.Equalf(t, 1, len(config.Exporters), "TEST[%s]", test.name)
+		assert.Truef(t, config.Exporters["exampleexporter"].IsEnabled(), "TEST[%s]", test.name)
+
+		assert.Equalf(t,
+			&ExampleExporter{
+				ExporterSettings: configmodels.ExporterSettings{
+					NameVal: "exampleexporter",
+					TypeVal: "exampleexporter",
+				},
+				ExtraInt:         65,
+				ExtraSetting:     exporterExtra,
+				ExtraMapSetting:  map[string]string{"exp_1": exporterExtraMapValue + "_1", "exp_2": exporterExtraMapValue + "_2"},
+				ExtraListSetting: []string{exporterExtraListElement + "_1", exporterExtraListElement + "_2"},
+			},
+			config.Exporters["exampleexporter"],
+			"TEST[%s] Did not load exporter config correctly", test.name)
+
+		// Verify Processors
+		assert.Equalf(t, 1, len(config.Processors), "TEST[%s]", test.name)
+		assert.Truef(t, config.Processors["exampleprocessor"].IsEnabled(), "TEST[%s]", test.name)
+
+		assert.Equalf(t,
+			&ExampleProcessor{
+				ProcessorSettings: configmodels.ProcessorSettings{
+					TypeVal: "exampleprocessor",
+					NameVal: "exampleprocessor",
+				},
+				ExtraSetting:     processorExtra,
+				ExtraMapSetting:  map[string]string{"proc_1": processorExtraMapValue + "_1", "proc_2": processorExtraMapValue + "_2"},
+				ExtraListSetting: []string{processorExtraListElement + "_1", processorExtraListElement + "_2"},
+			},
+			config.Processors["exampleprocessor"],
+			"TEST[%s] Did not load processor config correctly", test.name)
+
+		// Verify Pipelines
+		assert.Equalf(t, 1, len(config.Pipelines), "TEST[%s]", test.name)
+
+		assert.Equalf(t,
+			&configmodels.Pipeline{
+				Name:       "traces",
+				InputType:  configmodels.TracesDataType,
+				Receivers:  []string{"examplereceiver"},
+				Processors: []string{"exampleprocessor"},
+				Exporters:  []string{"exampleexporter"},
+			},
+			config.Pipelines["traces"],
+			"TEST[%s] Did not load pipeline config correctly", test.name)
+	}
+}
+
 func TestDecodeConfig_MultiProto(t *testing.T) {
 	factories, err := ExampleComponents()
 	assert.Nil(t, err)