Allow environment variables in config

config/config.go

@@ -20,6 +20,7 @@ package config
 import (
 	"errors"
 	"fmt"
+	"os"
 	"strings"
 
 	"github.com/spf13/viper"
@@ -244,9 +245,12 @@ func loadExtensions(v *viper.Viper, factories map[string]extension.Factory) (con
 		extensionCfg.SetType(typeStr)
 		extensionCfg.SetName(fullName)
 
+		// Unmarshal only the subconfig for this exporter.
+		sv := getConfigSection(subViper, key)
+
 		// Now that the default config struct is created we can Unmarshal into it
 		// and it will apply user-defined config on top of the default.
-		if err := subViper.UnmarshalKey(key, extensionCfg); err != nil {
+		if err := sv.Unmarshal(extensionCfg); err != nil {
 			return nil, &configError{
 				code: errUnmarshalError,
 				msg:  fmt.Sprintf("error reading settings for extension type %q: %v", typeStr, err),
@@ -322,6 +326,9 @@ func loadReceivers(v *viper.Viper, factories map[string]receiver.Factory) (confi
 		receiverCfg.SetType(typeStr)
 		receiverCfg.SetName(fullName)
 
+		// Unmarshal only the subconfig for this exporter.
+		sv := getConfigSection(subViper, key)
+
 		// Now that the default config struct is created we can Unmarshal into it
 		// and it will apply user-defined config on top of the default.
 		customUnmarshaler := factory.CustomUnmarshaler()
@@ -333,7 +340,7 @@ func loadReceivers(v *viper.Viper, factories map[string]receiver.Factory) (confi
 			// TODO(ccaraman): UnmarshallExact should be used to catch erroneous config entries.
 			// 	This leads to quickly identifying config values that are not supported and reduce confusion for
 			// 	users.
-			err = subViper.UnmarshalKey(key, receiverCfg)
+			err = sv.Unmarshal(receiverCfg)
 		}
 
 		if err != nil {
@@ -350,7 +357,6 @@ func loadReceivers(v *viper.Viper, factories map[string]receiver.Factory) (confi
 			}
 		}
 		receivers[fullName] = receiverCfg
-
 	}
 
 	return receivers, nil
@@ -399,9 +405,12 @@ func loadExporters(v *viper.Viper, factories map[string]exporter.Factory) (confi
 		exporterCfg.SetType(typeStr)
 		exporterCfg.SetName(fullName)
 
+		// Unmarshal only the subconfig for this exporter.
+		sv := getConfigSection(subViper, key)
+
 		// Now that the default config struct is created we can Unmarshal into it
 		// and it will apply user-defined config on top of the default.
-		if err := subViper.UnmarshalKey(key, exporterCfg); err != nil {
+		if err := sv.Unmarshal(exporterCfg); err != nil {
 			return nil, &configError{
 				code: errUnmarshalError,
 				msg:  fmt.Sprintf("error reading settings for exporter type %q: %v", typeStr, err),
@@ -456,9 +465,12 @@ func loadProcessors(v *viper.Viper, factories map[string]processor.Factory) (con
 		processorCfg.SetType(typeStr)
 		processorCfg.SetName(fullName)
 
+		// Unmarshal only the subconfig for this exporter.
+		sv := getConfigSection(subViper, key)
+
 		// Now that the default config struct is created we can Unmarshal into it
 		// and it will apply user-defined config on top of the default.
-		if err := subViper.UnmarshalKey(key, processorCfg); err != nil {
+		if err := sv.Unmarshal(processorCfg); err != nil {
 			return nil, &configError{
 				code: errUnmarshalError,
 				msg:  fmt.Sprintf("error reading settings for processor type %q: %v", typeStr, err),
@@ -813,3 +825,54 @@ func validateProcessors(cfg *configmodels.Config) {
 		}
 	}
 }
+
+// getConfigSection returns a sub-config from the viper config that has the corresponding given key.
+// It also expands all the string values.
+func getConfigSection(v *viper.Viper, key string) *viper.Viper {
+	// Unmarsh only the subconfig for this processor.
+	sv := v.Sub(key)
+	if sv == nil {
+		// When the config for this key is empty Sub returns nil. In order to avoid nil checks
+		// just return an empty config.
+		return viper.New()
+	}
+	// Before unmarshaling first expand all environment variables.
+	return expandEnvConfig(sv)
+}
+
+// expandEnvConfig creates a new viper config with expanded values for all the values (simple, list or map value).
+// It does not expand the keys.
+// Need to copy everything because of a bug in Viper: Set a value "map[string]interface{}" where a key has a ".",
+// then AllSettings will return the previous value not the newly set one.
+func expandEnvConfig(v *viper.Viper) *viper.Viper {
+	newCfg := make(map[string]interface{})
+	for k, val := range v.AllSettings() {
+		newCfg[k] = expandStringValues(val)
+	}
+	newVip := viper.New()
+	newVip.MergeConfigMap(newCfg)
+	return newVip
+}
+
+func expandStringValues(value interface{}) interface{} {
+	switch v := value.(type) {
+	default:
+		return v
+	case string:
+		return os.ExpandEnv(v)
+	case []interface{}:
+		// Viper treats all the slices as []interface{} (at least in what the otelsvc tests).
+		nslice := make([]interface{}, 0, len(v))
+		for _, vint := range v {
+			nslice = append(nslice, expandStringValues(vint))
+		}
+		return nslice
+	case map[string]interface{}:
+		nmap := make(map[string]interface{}, len(v))
+		// Viper treats all the maps as [string]interface{} (at least in what the otelsvc tests).
+		for k, vint := range v {
+			nmap[k] = expandStringValues(vint)
+		}
+		return nmap
+	}
+}

config/config_test.go

@@ -15,6 +15,7 @@
 package config
 
 import (
+	"os"
 	"path"
 	"testing"
 
@@ -124,6 +125,171 @@ func TestDecodeConfig(t *testing.T) {
 		"Did not load pipeline config correctly")
 }
 
+func TestSimpleConfig(t *testing.T) {
+	var testCases = []struct {
+		name string // test case name (also file name containing config yaml)
+	}{
+		{name: "simple-config-with-no-env"},
+		{name: "simple-config-with-partial-env"},
+		{name: "simple-config-with-all-env"},
+	}
+
+	const extensionExtra = "some extension string"
+	const extensionExtraMapValue = "some extension map value"
+	const extensionExtraListElement = "some extension list value"
+	os.Setenv("EXTENSIONS_EXAMPLEEXTENSION_EXTRA", extensionExtra)
+	os.Setenv("EXTENSIONS_EXAMPLEEXTENSION_EXTRA_MAP_EXT_VALUE_1", extensionExtraMapValue+"_1")
+	os.Setenv("EXTENSIONS_EXAMPLEEXTENSION_EXTRA_MAP_EXT_VALUE_2", extensionExtraMapValue+"_2")
+	os.Setenv("EXTENSIONS_EXAMPLEEXTENSION_EXTRA_LIST_VALUE_1", extensionExtraListElement+"_1")
+	os.Setenv("EXTENSIONS_EXAMPLEEXTENSION_EXTRA_LIST_VALUE_2", extensionExtraListElement+"_2")
+
+	const receiverExtra = "some receiver string"
+	const receiverExtraMapValue = "some receiver map value"
+	const receiverExtraListElement = "some receiver list value"
+	os.Setenv("RECEIVERS_EXAMPLERECEIVER_EXTRA", receiverExtra)
+	os.Setenv("RECEIVERS_EXAMPLERECEIVER_EXTRA_MAP_RECV_VALUE_1", receiverExtraMapValue+"_1")
+	os.Setenv("RECEIVERS_EXAMPLERECEIVER_EXTRA_MAP_RECV_VALUE_2", receiverExtraMapValue+"_2")
+	os.Setenv("RECEIVERS_EXAMPLERECEIVER_EXTRA_LIST_VALUE_1", receiverExtraListElement+"_1")
+	os.Setenv("RECEIVERS_EXAMPLERECEIVER_EXTRA_LIST_VALUE_2", receiverExtraListElement+"_2")
+
+	const processorExtra = "some processor string"
+	const processorExtraMapValue = "some processor map value"
+	const processorExtraListElement = "some processor list value"
+	os.Setenv("PROCESSORS_EXAMPLEPROCESSOR_EXTRA", processorExtra)
+	os.Setenv("PROCESSORS_EXAMPLEPROCESSOR_EXTRA_MAP_PROC_VALUE_1", processorExtraMapValue+"_1")
+	os.Setenv("PROCESSORS_EXAMPLEPROCESSOR_EXTRA_MAP_PROC_VALUE_2", processorExtraMapValue+"_2")
+	os.Setenv("PROCESSORS_EXAMPLEPROCESSOR_EXTRA_LIST_VALUE_1", processorExtraListElement+"_1")
+	os.Setenv("PROCESSORS_EXAMPLEPROCESSOR_EXTRA_LIST_VALUE_2", processorExtraListElement+"_2")
+
+	const exporterExtra = "some exporter string"
+	const exporterExtraMapValue = "some exporter map value"
+	const exporterExtraListElement = "some exporter list value"
+	os.Setenv("EXPORTERS_EXAMPLEEXPORTER_EXTRA_INT", "65")
+	os.Setenv("EXPORTERS_EXAMPLEEXPORTER_EXTRA", exporterExtra)
+	os.Setenv("EXPORTERS_EXAMPLEEXPORTER_EXTRA_MAP_EXP_VALUE_1", exporterExtraMapValue+"_1")
+	os.Setenv("EXPORTERS_EXAMPLEEXPORTER_EXTRA_MAP_EXP_VALUE_2", exporterExtraMapValue+"_2")
+	os.Setenv("EXPORTERS_EXAMPLEEXPORTER_EXTRA_LIST_VALUE_1", exporterExtraListElement+"_1")
+	os.Setenv("EXPORTERS_EXAMPLEEXPORTER_EXTRA_LIST_VALUE_2", exporterExtraListElement+"_2")
+
+	defer func() {
+		os.Unsetenv("EXTENSIONS_EXAMPLEEXTENSION_EXTRA")
+		os.Unsetenv("EXTENSIONS_EXAMPLEEXTENSION_EXTRA_MAP_EXT_VALUE")
+		os.Unsetenv("EXTENSIONS_EXAMPLEEXTENSION_EXTRA_LIST_VALUE_1")
+
+		os.Unsetenv("RECEIVERS_EXAMPLERECEIVER_EXTRA")
+		os.Unsetenv("RECEIVERS_EXAMPLERECEIVER_EXTRA_MAP_RECV_VALUE")
+		os.Unsetenv("RECEIVERS_EXAMPLERECEIVER_EXTRA_LIST_VALUE_1")
+
+		os.Unsetenv("PROCESSORS_EXAMPLEPROCESSOR_EXTRA")
+		os.Unsetenv("PROCESSORS_EXAMPLEPROCESSOR_EXTRA_MAP_PROC_VALUE")
+		os.Unsetenv("PROCESSORS_EXAMPLEPROCESSOR_EXTRA_LIST_VALUE_1")
+
+		os.Unsetenv("EXPORTERS_EXAMPLEEXPORTER_EXTRA_INT")
+		os.Unsetenv("EXPORTERS_EXAMPLEEXPORTER_EXTRA")
+		os.Unsetenv("EXPORTERS_EXAMPLEEXPORTER_EXTRA_MAP_EXP_VALUE")
+		os.Unsetenv("EXPORTERS_EXAMPLEEXPORTER_EXTRA_LIST_VALUE_1")
+	}()
+
+	for _, test := range testCases {
+		t.Logf("TEST[%s]", test.name)
+		factories, err := ExampleComponents()
+		assert.Nil(t, err)
+
+		// Load the config
+		config, err := LoadConfigFile(t, path.Join(".", "testdata", test.name+".yaml"), factories)
+		if err != nil {
+			t.Fatalf("TEST[%s] unable to load config, %v", test.name, err)
+		}
+
+		// Verify extensions.
+		assert.Equalf(t, 1, len(config.Extensions), "TEST[%s]", test.name)
+		assert.Truef(t, config.Extensions["exampleextension"].IsEnabled(), "TEST[%s]", test.name)
+		assert.Equalf(t,
+			&ExampleExtension{
+				ExtensionSettings: configmodels.ExtensionSettings{
+					TypeVal: "exampleextension",
+					NameVal: "exampleextension",
+				},
+				ExtraSetting:     extensionExtra,
+				ExtraMapSetting:  map[string]string{"ext-1": extensionExtraMapValue + "_1", "ext-2": extensionExtraMapValue + "_2"},
+				ExtraListSetting: []string{extensionExtraListElement + "_1", extensionExtraListElement + "_2"},
+			},
+			config.Extensions["exampleextension"],
+			"TEST[%s] Did not load extension config correctly", test.name)
+
+		// Verify service.
+		assert.Equalf(t, 1, len(config.Service.Extensions), "TEST[%s]", test.name)
+		assert.Equalf(t, "exampleextension", config.Service.Extensions[0], "TEST[%s]", test.name)
+
+		// Verify receivers
+		assert.Equalf(t, 1, len(config.Receivers), "TEST[%s]", test.name)
+		assert.Truef(t, config.Receivers["examplereceiver"].IsEnabled(), "TEST[%s]", test.name)
+
+		assert.Equalf(t,
+			&ExampleReceiver{
+				ReceiverSettings: configmodels.ReceiverSettings{
+					TypeVal:  "examplereceiver",
+					NameVal:  "examplereceiver",
+					Endpoint: "127.0.0.1:1234",
+				},
+				ExtraSetting:     receiverExtra,
+				ExtraMapSetting:  map[string]string{"recv.1": receiverExtraMapValue + "_1", "recv.2": receiverExtraMapValue + "_2"},
+				ExtraListSetting: []string{receiverExtraListElement + "_1", receiverExtraListElement + "_2"},
+			},
+			config.Receivers["examplereceiver"],
+			"TEST[%s] Did not load receiver config correctly", test.name)
+
+		// Verify exporters
+		assert.Equalf(t, 1, len(config.Exporters), "TEST[%s]", test.name)
+		assert.Truef(t, config.Exporters["exampleexporter"].IsEnabled(), "TEST[%s]", test.name)
+
+		assert.Equalf(t,
+			&ExampleExporter{
+				ExporterSettings: configmodels.ExporterSettings{
+					NameVal: "exampleexporter",
+					TypeVal: "exampleexporter",
+				},
+				ExtraInt:         65,
+				ExtraSetting:     exporterExtra,
+				ExtraMapSetting:  map[string]string{"exp_1": exporterExtraMapValue + "_1", "exp_2": exporterExtraMapValue + "_2"},
+				ExtraListSetting: []string{exporterExtraListElement + "_1", exporterExtraListElement + "_2"},
+			},
+			config.Exporters["exampleexporter"],
+			"TEST[%s] Did not load exporter config correctly", test.name)
+
+		// Verify Processors
+		assert.Equalf(t, 1, len(config.Processors), "TEST[%s]", test.name)
+		assert.Truef(t, config.Processors["exampleprocessor"].IsEnabled(), "TEST[%s]", test.name)
+
+		assert.Equalf(t,
+			&ExampleProcessor{
+				ProcessorSettings: configmodels.ProcessorSettings{
+					TypeVal: "exampleprocessor",
+					NameVal: "exampleprocessor",
+				},
+				ExtraSetting:     processorExtra,
+				ExtraMapSetting:  map[string]string{"proc_1": processorExtraMapValue + "_1", "proc_2": processorExtraMapValue + "_2"},
+				ExtraListSetting: []string{processorExtraListElement + "_1", processorExtraListElement + "_2"},
+			},
+			config.Processors["exampleprocessor"],
+			"TEST[%s] Did not load processor config correctly", test.name)
+
+		// Verify Pipelines
+		assert.Equalf(t, 1, len(config.Pipelines), "TEST[%s]", test.name)
+
+		assert.Equalf(t,
+			&configmodels.Pipeline{
+				Name:       "traces",
+				InputType:  configmodels.TracesDataType,
+				Receivers:  []string{"examplereceiver"},
+				Processors: []string{"exampleprocessor"},
+				Exporters:  []string{"exampleexporter"},
+			},
+			config.Pipelines["traces"],
+			"TEST[%s] Did not load pipeline config correctly", test.name)
+	}
+}
+
 func TestDecodeConfig_MultiProto(t *testing.T) {
 	factories, err := ExampleComponents()
 	assert.Nil(t, err)

config/example_factories.go

@@ -35,6 +35,8 @@ import (
 type ExampleReceiver struct {
 	configmodels.ReceiverSettings `mapstructure:",squash"` // squash ensures fields are correctly decoded in embedded struct
 	ExtraSetting                  string                   `mapstructure:"extra"`
+	ExtraMapSetting               map[string]string        `mapstructure:"extra_map"`
+	ExtraListSetting              []string                 `mapstructure:"extra_list"`
 
 	// FailTraceCreation causes CreateTraceReceiver to fail. Useful for testing.
 	FailTraceCreation bool `mapstructure:"-"`
@@ -64,7 +66,9 @@ func (f *ExampleReceiverFactory) CreateDefaultConfig() configmodels.Receiver {
 			TypeVal:  "examplereceiver",
 			Endpoint: "localhost:1000",
 		},
-		ExtraSetting: "some string",
+		ExtraSetting:     "some string",
+		ExtraMapSetting:  nil,
+		ExtraListSetting: nil,
 	}
 }
 
@@ -242,7 +246,10 @@ func (f *MultiProtoReceiverFactory) CreateMetricsReceiver(
 // for "exampleexporter" exporter type.
 type ExampleExporter struct {
 	configmodels.ExporterSettings `mapstructure:",squash"` // squash ensures fields are correctly decoded in embedded struct
+	ExtraInt                      int32                    `mapstructure:"extra_int"`
 	ExtraSetting                  string                   `mapstructure:"extra"`
+	ExtraMapSetting               map[string]string        `mapstructure:"extra_map"`
+	ExtraListSetting              []string                 `mapstructure:"extra_list"`
 	ExporterShutdown              bool
 }
 
@@ -260,6 +267,8 @@ func (f *ExampleExporterFactory) CreateDefaultConfig() configmodels.Exporter {
 	return &ExampleExporter{
 		ExporterSettings: configmodels.ExporterSettings{},
 		ExtraSetting:     "some export string",
+		ExtraMapSetting:  nil,
+		ExtraListSetting: nil,
 	}
 }
 
@@ -308,6 +317,8 @@ func (exp *ExampleExporterConsumer) Shutdown() error {
 type ExampleProcessor struct {
 	configmodels.ProcessorSettings `mapstructure:",squash"` // squash ensures fields are correctly decoded in embedded struct
 	ExtraSetting                   string                   `mapstructure:"extra"`
+	ExtraMapSetting                map[string]string        `mapstructure:"extra_map"`
+	ExtraListSetting               []string                 `mapstructure:"extra_list"`
 }
 
 // ExampleProcessorFactory is factory for ExampleProcessor.
@@ -324,6 +335,8 @@ func (f *ExampleProcessorFactory) CreateDefaultConfig() configmodels.Processor {
 	return &ExampleProcessor{
 		ProcessorSettings: configmodels.ProcessorSettings{},
 		ExtraSetting:      "some export string",
+		ExtraMapSetting:   nil,
+		ExtraListSetting:  nil,
 	}
 }
 
@@ -350,6 +363,8 @@ func (f *ExampleProcessorFactory) CreateMetricsProcessor(
 type ExampleExtension struct {
 	configmodels.ExtensionSettings `mapstructure:",squash"` // squash ensures fields are correctly decoded in embedded struct
 	ExtraSetting                   string                   `mapstructure:"extra"`
+	ExtraMapSetting                map[string]string        `mapstructure:"extra_map"`
+	ExtraListSetting               []string                 `mapstructure:"extra_list"`
 }
 
 // ExampleExtensionFactory is factory for ExampleExtension.
@@ -366,6 +381,8 @@ func (f *ExampleExtensionFactory) CreateDefaultConfig() configmodels.Extension {
 	return &ExampleExtension{
 		ExtensionSettings: configmodels.ExtensionSettings{},
 		ExtraSetting:      "extra string setting",
+		ExtraMapSetting:   nil,
+		ExtraListSetting:  nil,
 	}
 }