Merge pull request #4 from iver-wharf/feature/config

Config

CHANGELOG.md

@@ -0,0 +1,17 @@
+# Wharf core library changelog
+
+This project tries to follow [SemVer 2.0.0](https://semver.org/).
+
+<!--
+	When composing new changes to this list, try to follow convention.
+	The WIP release shall be updated just before adding the Git tag.
+	From (WIP) to (YYYY-MM-DD), ex: (2021-02-09) for 9th of Febuary, 2021
+	A good source on conventions can be found here:
+	https://changelog.md/
+-->
+
+## v1.0.0 (WIP)
+
+- Added `pkg/config` with newly added `config.Config` interface and
+  implementation to let you load configs from environment variables or YAML
+  files. This is done via [spf13/viper](https://github.com/spf13/viper). (#4)

go.mod

@@ -0,0 +1,9 @@
+module github.com/iver-wharf/wharf-core
+
+go 1.16
+
+require (
+	github.com/spf13/viper v1.7.1
+	github.com/stretchr/testify v1.7.0
+	gopkg.in/yaml.v2 v2.4.0
+)

pkg/config/config.go

@@ -0,0 +1,200 @@
+package config
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"io"
+	"io/fs"
+	"strings"
+
+	"github.com/spf13/viper"
+	"gopkg.in/yaml.v2"
+)
+
+const configTypeYAML = "yaml"
+
+// Builder type has methods for registering configuration sources, and
+// then using those sources you can unmarshal into a struct to read the
+// configuration.
+//
+// Later added config sources will merge on top of the previous on a
+// per config field basis. Later added sources will override earlier added
+// sources.
+type Builder interface {
+	// AddConfigYAMLFile appends the path of a YAML file to the list of sources
+	// for this configuration.
+	//
+	// Later added config sources will merge on top of the previous on a
+	// per config field basis. Later added sources will override earlier added
+	// sources.
+	AddConfigYAMLFile(path string)
+
+	// AddConfigYAML appends a byte reader for UTF-8 and YAML formatted content.
+	// Useful for reading from embedded files, database stored configs, and from
+	// HTTP response bodies.
+	//
+	// Later added config sources will merge on top of the previous on a
+	// per config field basis. Later added sources will override earlier added
+	// sources.
+	AddConfigYAML(reader io.Reader)
+
+	// AddEnvironmentVariables appends an environment variable source.
+	//
+	// Later added config sources will merge on top of the previous on a
+	// per config field basis. Later added sources will override earlier added
+	// sources.
+	//
+	// However, multiple environment variable sources cannot be added, due to
+	// technical limitations with the implementation. Even if they use different
+	// prefixes.
+	//
+	// Environment variables must be in all uppercase letters, and nested
+	// structs use a single underscore "_" as delimiter. Example:
+	//
+	//  c := config.NewBuilder(myConfigDefaults)
+	//  c.AddEnvironmentVariables("FOO")
+	//
+	//  type MyConfig struct {
+	//  	Bar        string // set via "FOO_BAR"
+	//  	LoremIpsum string // set via "FOO_LOREMIPSUM"
+	//  	Hello      struct {
+	//  		World string // set via "FOO_HELLO_WORLD"
+	//  	}
+	//  }
+	//
+	// The prefix shall be without a trailing underscore "_" as this package
+	// adds that in by itself. To not use a prefix, pass in an empty string as
+	// prefix instead.
+	AddEnvironmentVariables(prefix string)
+
+	// Unmarshal applies the configuration, based on the numerous added sources,
+	// on to an existing struct.
+	//
+	// For any field of type pointer and is set to nil, this function will
+	// create a new instance and assign that before populating that branch.
+	//
+	// If none of the Builder.Add...() functions has been called before
+	// this function, then this function will effectively only apply the default
+	// configuration onto this new object.
+	//
+	// The error that is returned is caused by any of the added config sources,
+	// such as from invalid YAML syntax in an added YAML file.
+	Unmarshal(config interface{}) error
+}
+
+// NewBuilder creates a new Builder based on a default configuration.
+//
+// Due to technical limitations, it's vital that this default configuration is
+// of the same type that the config that you wish to unmarshal later, or at
+// least that it contains fields with the same names.
+func NewBuilder(defaultConfig interface{}) Builder {
+	return &builder{
+		defaultConfig: defaultConfig,
+	}
+}
+
+type builder struct {
+	defaultConfig interface{}
+	sources       []configSource
+}
+
+type configSource interface {
+	name() string
+	apply(v *viper.Viper) error
+}
+
+func (b *builder) AddConfigYAMLFile(path string) {
+	b.sources = append(b.sources, yamlFileSource{path})
+}
+
+func (b *builder) AddConfigYAML(reader io.Reader) {
+	b.sources = append(b.sources, yamlSource{reader})
+}
+
+func (b *builder) AddEnvironmentVariables(prefix string) {
+	b.sources = append(b.sources, envVarsSource{prefix})
+}
+
+func (b *builder) Unmarshal(config interface{}) error {
+	v := viper.New()
+	initDefaults(v, b.defaultConfig)
+	for _, s := range b.sources {
+		if err := s.apply(v); err != nil {
+			return fmt.Errorf("applying config source: %s: %T: %w", s.name(), err, err)
+		}
+	}
+	return v.Unmarshal(config)
+}
+
+func initDefaults(v *viper.Viper, defaultConfig interface{}) error {
+	// Uses a workaround to force viper to read environment variables
+	// by making it aware of all fields that exists so it can later map
+	// environment variables correctly.
+	// https://github.com/spf13/viper/issues/188#issuecomment-413368673
+	b, err := yaml.Marshal(defaultConfig)
+	if err != nil {
+		return fmt.Errorf("setting config defaults: %w", err)
+	}
+	defaultCfg := bytes.NewReader(b)
+	v.SetConfigType(configTypeYAML)
+	if err := v.MergeConfig(defaultCfg); err != nil {
+		return fmt.Errorf("setting config defaults: %w", err)
+	}
+	return nil
+}
+
+type yamlFileSource struct {
+	path string
+}
+
+func (s yamlFileSource) name() string {
+	return s.path
+}
+
+func (s yamlFileSource) apply(v *viper.Viper) error {
+	if s.path == "" {
+		// viper does not set config file if its empty, so viper.MergeInConfig()
+		// would then use the previously set config path value
+		return nil
+	}
+	v.SetConfigType(configTypeYAML)
+	v.SetConfigFile(s.path)
+	err := v.MergeInConfig()
+	// ignore not-found errors
+	if errors.Is(err, fs.ErrNotExist) {
+		return nil
+	}
+	if _, ok := err.(viper.ConfigFileNotFoundError); ok {
+		return nil
+	}
+	return err
+}
+
+type yamlSource struct {
+	reader io.Reader
+}
+
+func (s yamlSource) name() string {
+	return "YAML io.Reader"
+}
+
+func (s yamlSource) apply(v *viper.Viper) error {
+	v.SetConfigType(configTypeYAML)
+	return v.MergeConfig(s.reader)
+}
+
+type envVarsSource struct {
+	prefix string
+}
+
+func (s envVarsSource) name() string {
+	return "environment variables"
+}
+
+func (s envVarsSource) apply(v *viper.Viper) error {
+	v.SetEnvPrefix(s.prefix)
+	v.AutomaticEnv()
+	v.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))
+	return nil
+}

pkg/config/config_example_test.go

@@ -0,0 +1,72 @@
+package config_test
+
+import (
+	"bytes"
+	_ "embed"
+	"fmt"
+	"os"
+
+	"github.com/iver-wharf/wharf-core/pkg/config"
+)
+
+type Logging struct {
+	LogLevel string
+}
+
+type DBConfig struct {
+	Host string
+	Port int
+}
+
+type Config struct {
+	// Both mapstructure:",squash" and yaml:",inline" needs to be set in
+	// embedded structs, even if you're only reading environment variables
+	Logging  `mapstructure:",squash" yaml:",inline"`
+	Username string
+	Password string
+	DB       DBConfig
+}
+
+var defaultConfig = Config{
+	Logging: Logging{
+		LogLevel: "Warning",
+	},
+	Username: "postgres",
+	DB: DBConfig{
+		Host: "localhost",
+		Port: 5432,
+	},
+}
+
+//go:embed testdata/embedded-config.yml
+var embeddedConfig []byte
+
+func ExampleConfig() {
+	cfgBuilder := config.NewBuilder(defaultConfig)
+	cfgBuilder.AddConfigYAML(bytes.NewReader(embeddedConfig))
+	cfgBuilder.AddConfigYAMLFile("/etc/my-app/config.yml")
+	cfgBuilder.AddConfigYAMLFile("$HOME/.config/my-app/config.yml")
+	cfgBuilder.AddConfigYAMLFile("my-app-config.yml") // from working directory
+	cfgBuilder.AddEnvironmentVariables("MYAPP")
+
+	os.Setenv("MYAPP_PASSWORD", "Sommar2020")
+
+	var cfg Config
+	if err := cfgBuilder.Unmarshal(&cfg); err != nil {
+		fmt.Println("Failed to read config:", err)
+		return
+	}
+
+	fmt.Println("Log level:", cfg.LogLevel) // set from embeddedConfig
+	fmt.Println("Username: ", cfg.Username) // uses defaultConfig.Username
+	fmt.Println("Password: ", cfg.Password) // set from environment variable
+	fmt.Println("DB host:  ", cfg.DB.Host)  // uses defaultConfig.DB.Host
+	fmt.Println("DB port:  ", cfg.DB.Port)  // set from embeddedConfig
+
+	// Output:
+	// Log level: Info
+	// Username:  postgres
+	// Password:  Sommar2020
+	// DB host:   localhost
+	// DB port:   8080
+}

pkg/config/config_test.go

@@ -0,0 +1,96 @@
+package config
+
+import (
+	"fmt"
+	"os"
+	"strconv"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+type TestLogging struct {
+	LogLevel string
+}
+
+type TestDBConfig struct {
+	Host string
+	Port int
+}
+
+type TestConfig struct {
+	// Both mapstructure:",squash" and yaml:",inline" needs to be set in
+	// embedded structs, even if you're only reading environment variables
+	TestLogging `mapstructure:",squash" yaml:",inline"`
+	Username    string
+	Password    string
+	DB          TestDBConfig
+}
+
+const (
+	defaultLogLevel = "default log level"
+	defaultUsername = "default username"
+	defaultPassword = "default password"
+	defaultDBHost   = "default db host"
+	defaultDBPort   = 12345
+	updatedLogLevel = "updated log level"
+	updatedPassword = "updated password"
+	updatedPort     = 8080
+)
+
+var defaultConfig = TestConfig{
+	TestLogging: TestLogging{
+		LogLevel: defaultLogLevel,
+	},
+	Username: defaultUsername,
+	Password: defaultPassword,
+	DB: TestDBConfig{
+		Host: defaultDBHost,
+		Port: defaultDBPort,
+	},
+}
+
+func assertUnmarshaledConfig(t *testing.T, c Builder) {
+	var cfg TestConfig
+	require.Nil(t, c.Unmarshal(&cfg), "failed to read config")
+	assert.Equal(t, updatedLogLevel, cfg.LogLevel)
+	assert.Equal(t, defaultUsername, cfg.Username)
+	assert.Equal(t, updatedPassword, cfg.Password)
+	assert.Equal(t, defaultDBHost, cfg.DB.Host)
+	assert.Equal(t, updatedPort, cfg.DB.Port)
+}
+
+func TestConfig_AddEnvironmentVariables(t *testing.T) {
+	cb := NewBuilder(defaultConfig)
+	cb.AddEnvironmentVariables("")
+
+	os.Clearenv()
+	os.Setenv("DB_PORT", strconv.FormatInt(updatedPort, 10))
+	os.Setenv("PASSWORD", updatedPassword)
+	os.Setenv("LOGLEVEL", updatedLogLevel)
+	// Environment variables must be all uppercase
+	os.Setenv("Username", "not used")
+
+	assertUnmarshaledConfig(t, cb)
+}
+
+func TestConfig_AddConfigYAML(t *testing.T) {
+	yamlContent := fmt.Sprintf(`
+logLevel: %s
+# YAML key names are case-insensitive
+pAssWOrD: %s
+db:
+  port: %d
+`, updatedLogLevel, updatedPassword, updatedPort)
+	cb := NewBuilder(defaultConfig)
+	cb.AddConfigYAML(strings.NewReader(yamlContent))
+	assertUnmarshaledConfig(t, cb)
+}
+
+func TestConfig_AddConfigYAMLFile(t *testing.T) {
+	cb := NewBuilder(defaultConfig)
+	cb.AddConfigYAMLFile("testdata/add-config-yaml-file.yml")
+	assertUnmarshaledConfig(t, cb)
+}