From a6665dd25f51462ffbdb5439db506535394fab7d Mon Sep 17 00:00:00 2001 From: "Katlinsky, Ilya" Date: Mon, 5 Jun 2023 12:22:31 +0200 Subject: [PATCH 1/2] feat: allow to use env variables in config --- api/internal/conf/conf.go | 37 +++++++++++++++++---- api/internal/conf/conf_test.go | 60 ++++++++++++++++++++++++++++++++++ 2 files changed, 90 insertions(+), 7 deletions(-) diff --git a/api/internal/conf/conf.go b/api/internal/conf/conf.go index 3879eafe80..56d0f32922 100644 --- a/api/internal/conf/conf.go +++ b/api/internal/conf/conf.go @@ -22,10 +22,12 @@ import ( "io/ioutil" "os" "path/filepath" + "reflect" "runtime" "strings" "github.com/gorilla/sessions" + "github.com/mitchellh/mapstructure" "github.com/spf13/viper" "github.com/tidwall/gjson" "golang.org/x/oauth2" @@ -184,6 +186,16 @@ func InitConf() { initSchema() } +func unmarshalConfig() *Config { + config := Config{} + err := viper.Unmarshal(&config, viper.DecodeHook(substituteEnvironmentVariables())) + if err != nil { + panic(fmt.Sprintf("fail to unmarshal configuration: %s, err: %s", ConfigFile, err.Error())) + } + + return &config +} + func setupConfig() { // setup config file path if ConfigFile == "" { @@ -204,11 +216,7 @@ func setupConfig() { } // unmarshal config - config := Config{} - err := viper.Unmarshal(&config) - if err != nil { - panic(fmt.Sprintf("fail to unmarshal configuration: %s, err: %s", ConfigFile, err.Error())) - } + config := unmarshalConfig() // listen if config.Conf.Listen.Port != 0 { @@ -251,7 +259,7 @@ func setupConfig() { if strings.HasPrefix(ErrorLogPath, "winfile") { return } - ErrorLogPath, err = filepath.Abs(filepath.Join(WorkDir, ErrorLogPath)) + ErrorLogPath, err := filepath.Abs(filepath.Join(WorkDir, ErrorLogPath)) if err != nil { panic(err) } @@ -263,7 +271,7 @@ func setupConfig() { if strings.HasPrefix(AccessLogPath, "winfile") { return } - AccessLogPath, err = filepath.Abs(filepath.Join(WorkDir, AccessLogPath)) + AccessLogPath, err := filepath.Abs(filepath.Join(WorkDir, AccessLogPath)) if err != nil { panic(err) } @@ -432,3 +440,18 @@ func initSecurity(conf Security) { ContentSecurityPolicy: DefaultCSP, } } + +// substituteEnvironmentVariables substitutes environment variables in the configuration with real values +func substituteEnvironmentVariables() mapstructure.DecodeHookFuncKind { + return func( + f reflect.Kind, + t reflect.Kind, + data interface{}, + ) (interface{}, error) { + if f != reflect.String { + return data, nil + } + + return os.ExpandEnv(data.(string)), nil + } +} diff --git a/api/internal/conf/conf_test.go b/api/internal/conf/conf_test.go index 1a1a1c3ef3..28713abfad 100644 --- a/api/internal/conf/conf_test.go +++ b/api/internal/conf/conf_test.go @@ -1,9 +1,12 @@ package conf import ( + "bytes" "encoding/json" + "os" "testing" + "github.com/spf13/viper" "github.com/stretchr/testify/assert" ) @@ -61,3 +64,60 @@ func Test_mergeSchema(t *testing.T) { }) } } + +func Test_unmarshalConfig(t *testing.T) { + tests := []struct { + name string + init func() + config []byte + assert func(*Config, *testing.T) + }{ + { + name: "should correctly parse config without environment variables", + init: func() {}, + config: []byte("conf:\n listen:\n port: \"9000\""), + assert: func(config *Config, t *testing.T) { + assert.Equal(t, 9000, config.Conf.Listen.Port) + }, + }, + { + name: "should correctly substitute int port from environment variables", + init: func() { + os.Setenv("PORT", "8080") + }, + config: []byte("conf:\n listen:\n port: \"$PORT\""), + assert: func(config *Config, t *testing.T) { + assert.Equal(t, 8080, config.Conf.Listen.Port) + }, + }, + { + name: "should correctly substitute string etcd endpoint from environment variables", + init: func() { + os.Setenv("ETCD_ENDPOINT", "127.0.0.1:2379") + }, + config: []byte("conf:\n etcd:\n endpoints:\n - $ETCD_ENDPOINT"), + assert: func(config *Config, t *testing.T) { + assert.Equal(t, "127.0.0.1:2379", config.Conf.Etcd.Endpoints[0]) + }, + }, + } + + for _, tt := range tests { + + t.Run(tt.name, func(t *testing.T) { + viper.SetConfigType("yaml") + viper.AutomaticEnv() + + tt.init() + + err := viper.ReadConfig(bytes.NewBuffer(tt.config)) + if err != nil { + t.Errorf("unable to read config: %v", err) + return + } + config := unmarshalConfig() + + tt.assert(config, t) + }) + } +} From 06577e2af5719b825d3776c6a085893cd032d95e Mon Sep 17 00:00:00 2001 From: "Katlinsky, Ilya" Date: Mon, 5 Jun 2023 14:43:49 +0200 Subject: [PATCH 2/2] fix(common): correct logic to fix logger settings --- api/internal/conf/conf.go | 15 +++++++++------ api/internal/conf/conf_test.go | 6 +++++- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/api/internal/conf/conf.go b/api/internal/conf/conf.go index 56d0f32922..ab859ba39d 100644 --- a/api/internal/conf/conf.go +++ b/api/internal/conf/conf.go @@ -186,14 +186,14 @@ func InitConf() { initSchema() } -func unmarshalConfig() *Config { +func unmarshalConfig() (*Config, error) { config := Config{} err := viper.Unmarshal(&config, viper.DecodeHook(substituteEnvironmentVariables())) if err != nil { - panic(fmt.Sprintf("fail to unmarshal configuration: %s, err: %s", ConfigFile, err.Error())) + return nil, err } - return &config + return &config, nil } func setupConfig() { @@ -216,7 +216,10 @@ func setupConfig() { } // unmarshal config - config := unmarshalConfig() + config, err := unmarshalConfig() + if err != nil { + panic(fmt.Sprintf("fail to unmarshal configuration: %s, err: %s", ConfigFile, err.Error())) + } // listen if config.Conf.Listen.Port != 0 { @@ -259,7 +262,7 @@ func setupConfig() { if strings.HasPrefix(ErrorLogPath, "winfile") { return } - ErrorLogPath, err := filepath.Abs(filepath.Join(WorkDir, ErrorLogPath)) + ErrorLogPath, err = filepath.Abs(filepath.Join(WorkDir, ErrorLogPath)) if err != nil { panic(err) } @@ -271,7 +274,7 @@ func setupConfig() { if strings.HasPrefix(AccessLogPath, "winfile") { return } - AccessLogPath, err := filepath.Abs(filepath.Join(WorkDir, AccessLogPath)) + AccessLogPath, err = filepath.Abs(filepath.Join(WorkDir, AccessLogPath)) if err != nil { panic(err) } diff --git a/api/internal/conf/conf_test.go b/api/internal/conf/conf_test.go index 28713abfad..2f05e90f27 100644 --- a/api/internal/conf/conf_test.go +++ b/api/internal/conf/conf_test.go @@ -115,7 +115,11 @@ func Test_unmarshalConfig(t *testing.T) { t.Errorf("unable to read config: %v", err) return } - config := unmarshalConfig() + config, err := unmarshalConfig() + if err != nil { + t.Errorf("unable to unmarshall config: %v", err) + return + } tt.assert(config, t) })