Merge 4484644 into 9f7fb70

CHANGELOG.md

@@ -21,6 +21,9 @@
   explicitly where required.
 - **[Breaking]** Module providers take in a `tally.Scope` so they can be
   propagated through module creation.
+- **[Breaking]** Remove lookup function from the YAML provider constructor and add
+  explicit constructors `WithExpand` to use interpolation.
+- Add a SetStaticConfigFiles method to load yaml files that should not be interpolated.
 
 ## v1.0.0-beta3 (28 Mar 2017)
 

config/config.go

@@ -61,9 +61,12 @@ type Loader struct {
 	staticProviderFuncs  []ProviderFunc
 	dynamicProviderFuncs []DynamicProviderFunc
 
-	// Files to load.
+	// Files to load, they will be replaced with values from environment variables.
 	configFiles []string
 
+	// Files to load without interpolation.
+	staticFiles []string
+
 	// Dirs to load from.
 	dirs []string
 
@@ -130,7 +133,8 @@ func (l *Loader) ResolvePath(relative string) (string, error) {
 }
 
 func (l *Loader) baseFiles() []string {
-	return []string{_baseFile, l.Environment() + ".yaml", _secretsFile}
+	// Order is important: last files override values in the first files.
+	return []string{_baseFile, l.Environment() + ".yaml"}
 }
 
 func (l *Loader) getResolver() FileResolver {
@@ -140,7 +144,11 @@ func (l *Loader) getResolver() FileResolver {
 // YamlProvider returns function to create Yaml based configuration provider
 func (l *Loader) YamlProvider() ProviderFunc {
 	return func() (Provider, error) {
-		return NewYAMLProviderFromFiles(false, l.getResolver(), l.getFiles()...), nil
+		static := NewYAMLProviderFromFiles(false, l.getResolver(), l.getStaticFiles()...)
+		expanded := NewYAMLProviderWithExpand(false, l.getResolver(), os.LookupEnv, l.getFiles()...)
+
+		// Static files will have higher priority than expanded.
+		return NewProviderGroup("yaml", expanded, static), nil
 	}
 }
 
@@ -163,19 +171,28 @@ func (l *Loader) Paths() []string {
 }
 
 // SetConfigFiles overrides the set of available config files for the service.
+// Configs will be interpolated with values from environment variables.
 func (l *Loader) SetConfigFiles(files ...string) {
 	l.lock.Lock()
 	defer l.lock.Unlock()
 
 	l.configFiles = files
 }
 
+// SetStaticConfigFiles overrides the set of available config files for the service.
+// Config values will not be interpolated.
+func (l *Loader) SetStaticConfigFiles(files ...string) {
+	l.lock.Lock()
+	defer l.lock.Unlock()
+
+	l.staticFiles = files
+}
+
 func (l *Loader) getFiles() []string {
 	l.lock.RLock()
 	defer l.lock.RUnlock()
 
 	files := l.configFiles
-
 	// Check if files where explicitly set.
 	if len(files) == 0 {
 		files = l.baseFiles()
@@ -186,6 +203,21 @@ func (l *Loader) getFiles() []string {
 	return res
 }
 
+func (l *Loader) getStaticFiles() []string {
+	l.lock.RLock()
+	defer l.lock.RUnlock()
+
+	files := l.staticFiles
+	// Check if files where explicitly set.
+	if len(files) == 0 {
+		return []string{_secretsFile}
+	}
+
+	res := make([]string, len(files))
+	copy(res, files)
+	return res
+}
+
 // SetDirs overrides the set of dirs to load config files from.
 func (l *Loader) SetDirs(dirs ...string) {
 	l.lock.Lock()

config/config_test.go

@@ -486,14 +486,11 @@ rpc:
         port: ${COMPANY_TCHANNEL_PORT:321}
 `
 	lookup := func(key string) (string, bool) {
-		if key == "COMPANY_TCHANNEL_PORT" {
-			return "4324", true
-		}
-
-		return "", false
+		require.Equal(t, "COMPANY_TCHANNEL_PORT", key)
+		return "4324", true
 	}
 
-	p := newYAMLProviderCore(lookup, ioutil.NopCloser(bytes.NewBufferString(rpc)))
+	p := NewYAMLProviderFromReaderWithExpand(lookup, ioutil.NopCloser(bytes.NewBufferString(rpc)))
 
 	cfg := &YARPCConfig{}
 	v := p.Get("rpc")
@@ -649,3 +646,46 @@ func TestLoader_LoadFromTestEnvironment(t *testing.T) {
 
 	withBase(t, f, "value: base")
 }
+
+func TestLoader_DefaultLoadOfStaticConfigFiles(t *testing.T) {
+	t.Parallel()
+	f := func(dir string) {
+		l := NewLoader()
+		l.SetDirs(dir)
+		s, err := os.Create(path.Join(dir, _secretsFile))
+		require.NoError(t, err)
+		defer func() { require.NoError(t, os.Remove(s.Name())) }()
+
+		fmt.Fprint(s, "password: ${don't interpolate me}")
+		require.NoError(t, s.Close())
+
+		p := l.Load()
+		assert.Equal(t, "base", p.Get("value").AsString())
+		assert.Equal(t, "${don't interpolate me}", p.Get("password").AsString())
+	}
+
+	withBase(t, f, "value: base")
+}
+
+func TestLoader_OverrideStaticConfigFiles(t *testing.T) {
+	t.Parallel()
+	f := func(dir string) {
+		l := NewLoader()
+		l.SetDirs(dir)
+		l.SetStaticConfigFiles("static.yaml")
+
+		s, err := os.Create(path.Join(dir, "static.yaml"))
+		require.NoError(t, err)
+		defer func() { require.NoError(t, os.Remove(s.Name())) }()
+
+		// Order is important, we want to override base.yaml
+		fmt.Fprint(s, "static: ${null}\nvalue: override")
+		require.NoError(t, s.Close())
+
+		p := l.Load()
+		assert.Equal(t, "override", p.Get("value").AsString())
+		assert.Equal(t, "${null}", p.Get("static").AsString())
+	}
+
+	withBase(t, f, "value: base")
+}

config/static_provider.go

@@ -20,7 +20,12 @@
 
 package config
 
-import "gopkg.in/yaml.v2"
+import (
+	"bytes"
+	"gopkg.in/yaml.v2"
+	"io"
+	"io/ioutil"
+)
 
 type staticProvider struct {
 	Provider
@@ -29,12 +34,17 @@ type staticProvider struct {
 // NewStaticProvider should only be used in tests to isolate config from your environment
 // It is not race free, because underlying objects can be accessed with Value().
 func NewStaticProvider(data interface{}) Provider {
-	b, err := yaml.Marshal(data)
-	if err != nil {
-		panic(err)
+	return staticProvider{
+		Provider: NewYAMLProviderFromReader(toReadCloser(data)),
 	}
+}
 
-	return staticProvider{Provider: NewYAMLProviderFromBytes(b)}
+// NewStaticProviderWithExpand returns a static provider with values replaced by a mapping function.
+func NewStaticProviderWithExpand(data interface{}, mapping func(string) (string, bool)) Provider {
+
+	return staticProvider{
+		Provider: NewYAMLProviderFromReaderWithExpand(mapping, toReadCloser(data)),
+	}
 }
 
 // StaticProvider returns function to create StaticProvider during configuration initialization
@@ -47,3 +57,12 @@ func StaticProvider(data interface{}) ProviderFunc {
 func (staticProvider) Name() string {
 	return "static"
 }
+
+func toReadCloser(data interface{}) io.ReadCloser {
+	b, err := yaml.Marshal(data)
+	if err != nil {
+		panic(err)
+	}
+
+	return ioutil.NopCloser(bytes.NewBuffer(b))
+}

config/static_provider_test.go

@@ -219,6 +219,37 @@ func TestPopulateNonPointerType(t *testing.T) {
 	assert.Contains(t, err.Error(), "can't populate non pointer type")
 }
 
+func TestStaticProviderWithExpand(t *testing.T) {
+	t.Parallel()
+
+	p := NewStaticProviderWithExpand(map[string]interface{}{
+		"slice": []interface{}{"one", "${iTwo:2}"},
+		"value": `${iValue:""}`,
+		"map": map[string]interface{}{
+			"drink?": "${iMap:tea?}",
+			"tea?":   "with cream",
+		},
+	}, func(key string) (string, bool) {
+		switch key {
+		case "iValue":
+			return "null", true
+		case "iTwo":
+			return "3", true
+		case "iMap":
+			return "rum please!", true
+		}
+
+		return "", false
+	})
+
+	assert.Equal(t, "one", p.Get("slice.0").AsString())
+	assert.Equal(t, "3", p.Get("slice.1").AsString())
+	assert.Equal(t, "null", p.Get("value").Value())
+
+	assert.Equal(t, "rum please!", p.Get("map.drink?").AsString())
+	assert.Equal(t, "with cream", p.Get("map.tea?").AsString())
+}
+
 func TestPopulateForMapOfDifferentKeyTypes(t *testing.T) {
 	t.Parallel()