Skip to content

Commit

Permalink
Introduce global and cascading configuration options
Browse files Browse the repository at this point in the history 
This introduces the concept of global configurations and cascades struct-level configuration options onto all child fields. For example, marking a struct as required will mark all child fields as required. Similarly, setting a custom delimiter on a struct tag propagates that delimiter to all child fields of the struct.
  • Loading branch information
@sethvargo
sethvargo committed Dec 20, 2023
1 parent d42f4a2 commit ba15a27
Show file tree
Hide file tree
Showing 4 changed files with 269 additions and 62 deletions.
49 changes: 23 additions & 26 deletions README.md
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# Envconfig

[![GoDoc](https://img.shields.io/badge/go-documentation-blue.svg?style=flat-square)](https://pkg.go.dev/mod/github.com/sethvargo/go-envconfig)
[![GoDoc](https://img.shields.io/badge/go-documentation-blue.svg?style=flat-square)][godoc]

Envconfig populates struct field values based on environment variables or
arbitrary lookup functions. It supports pre-setting mutations, which is useful
Expand Down Expand Up @@ -289,8 +289,9 @@ type MyStruct struct {
```go
// Process variables, but look for the "APP_" prefix.
l := envconfig.PrefixLookuper("APP_", envconfig.OsLookuper())
if err := envconfig.ProcessWith(ctx, &c, l); err != nil {
if err := envconfig.ProcessWith(ctx, &c, &envconfig.Config{
Lookuper: envconfig.PrefixLookuper("APP_", envconfig.OsLookuper()),
}); err != nil {
panic(err)
}
```
Expand Down Expand Up @@ -372,7 +373,10 @@ func resolveSecretFunc(ctx context.Context, originalKey, resolvedKey, originalVa
}

var config Config
envconfig.ProcessWith(ctx, &config, envconfig.OsLookuper(), resolveSecretFunc)
envconfig.ProcessWith(ctx, &config, &envconfig.Config{
Lookuper: envconfig.OsLookuper(),
Mutators: []envconfig.Mutator{resolveSecretFunc},
})
```
Mutators are like middleware, and they have access to the initial and current
Expand Down Expand Up @@ -438,11 +442,13 @@ type Config struct {
}

var config Config
lookuper := envconfig.PrefixLookuper("REDIS_", envconfig.MapLookuper(map[string]string{
"PASSWORD": "original",
}))
mutators := []envconfig.Mutators{mutatorFunc1, mutatorFunc2, mutatorFunc3}
envconfig.ProcessWith(ctx, &config, lookuper, mutators...)
envconfig.ProcessWith(ctx, &config, &envconfig.Config{
Lookuper: envconfig.PrefixLookuper("REDIS_", envconfig.MapLookuper(map[string]string{
"PASSWORD": "original",
})),
Mutators: mutators,
})

func mutatorFunc1(ctx context.Context, originalKey, resolvedKey, originalValue, currentValue string) (string, bool, error) {
// originalKey is "PASSWORD"
Expand All @@ -467,6 +473,11 @@ func mutatorFunc3(ctx context.Context, originalKey, resolvedKey, originalValue,
```
## Advanced Processing
See the [godoc][] for examples.
## Testing
Relying on the environment in tests can be troublesome because environment
Expand All @@ -480,7 +491,9 @@ lookuper := envconfig.MapLookuper(map[string]string{
})

var config Config
envconfig.ProcessWith(ctx, &config, lookuper)
envconfig.ProcessWith(ctx, &config, &envconfig.Config{
Lookuper: lookuper,
})
```
Now you can parallelize all your tests by providing a map for the lookup
Expand All @@ -491,20 +504,4 @@ You can also combine multiple lookupers with `MultiLookuper`. See the GoDoc for
more information and examples.
## Inspiration
This library is conceptually similar to [kelseyhightower/envconfig](https://github.com/kelseyhightower/envconfig), with the following
major behavioral differences:
- Adds support for specifying a custom lookup function (such as a map), which
is useful for testing.
- Only populates fields if they contain zero or nil values if `overwrite` is
unset. This means you can pre-initialize a struct and any pre-populated
fields will not be overwritten during processing.
- Support for interpolation. The default value for a field can be the value of
another field.
- Support for arbitrary mutators that change/resolve data before type
conversion.
[godoc]: https://pkg.go.dev/mod/github.com/sethvargo/go-envconfig
161 changes: 126 additions & 35 deletions envconfig.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,9 +89,6 @@ const (
optPrefix = "prefix="
optRequired = "required"
optSeparator = "separator="

defaultDelimiter = ","
defaultSeparator = ":"
)

// Error is a custom error type for errors returned by envconfig.
Expand Down Expand Up @@ -230,21 +227,76 @@ type options struct {
Required bool
}

// Config represent inputs to the envconfig decoding.
type Config struct {
// Target is the destination structure to decode. This value is required.
Target any

// Lookuper is the lookuper implementation to use. If not provided, it
// defaults to the OS Lookuper.
Lookuper Lookuper

// DefaultDelimiter is the default value to use for the delimiter in maps and
// slices. This can be overridden on a per-field basis, which takes
// precedence. The default value is ",".
DefaultDelimiter string

// DefaultSeparator is the default value to use for the separator in maps.
// This can be overridden on a per-field basis, which takes precedence. The
// default value is ":".
DefaultSeparator string

// DefaultNoInit is the default value for skipping initialization of
// unprovided fields. The default value is false (deeply initialize all
// fields and nested structs).
DefaultNoInit bool

// DefaultOverwrite is the default value for overwriting an existing value set
// on the struct before processing. The default value is false.
DefaultOverwrite bool

// DefaultRequired is the default value for marking a field as required. The
// default value is false.
DefaultRequired bool

// Mutators is an optiona list of mutators to apply to lookups.
Mutators []Mutator
}

// Process processes the struct using the environment. See [ProcessWith] for a
// more customizable version.
func Process(ctx context.Context, i any, mus ...Mutator) error {
return ProcessWith(ctx, i, OsLookuper(), mus...)
return ProcessWith(ctx, &Config{
Target: i,
Mutators: mus,
})
}

// ProcessWith processes the given interface with the given lookuper. See the
// package-level documentation for specific examples and behaviors.
func ProcessWith(ctx context.Context, i any, l Lookuper, mus ...Mutator) error {
return processWith(ctx, i, l, false, mus...)
func ProcessWith(ctx context.Context, c *Config) error {
if c == nil {
c = new(Config)
}

// Deep copy the slice and remove any nil functions.
var mus []Mutator
for _, m := range c.Mutators {
if m != nil {
mus = append(mus, m)
}
}
c.Mutators = mus

return processWith(ctx, c)
}

// processWith is a helper that captures whether the parent wanted
// initialization.
func processWith(ctx context.Context, i any, l Lookuper, parentNoInit bool, mus ...Mutator) error {
func processWith(ctx context.Context, c *Config) error {
i := c.Target

l := c.Lookuper
if l == nil {
return ErrLookuperNil
}
Expand All @@ -261,6 +313,23 @@ func processWith(ctx context.Context, i any, l Lookuper, parentNoInit bool, mus

t := e.Type()

structDelimiter := c.DefaultDelimiter
if structDelimiter == "" {
structDelimiter = ","
}

structNoInit := c.DefaultNoInit

structSeparator := c.DefaultSeparator
if structSeparator == "" {
structSeparator = ":"
}

structOverwrite := c.DefaultOverwrite
structRequired := c.DefaultRequired

mutators := c.Mutators

for i := 0; i < t.NumField(); i++ {
ef := e.Field(i)
tf := t.Field(i)
Expand Down Expand Up @@ -291,7 +360,20 @@ func processWith(ctx context.Context, i any, l Lookuper, parentNoInit bool, mus
ef.Kind() != reflect.UnsafePointer {
return fmt.Errorf("%s: %w", tf.Name, ErrNoInitNotPtr)
}
shouldNotInit := opts.NoInit || parentNoInit

// Compute defaults from local tags.
delimiter := structDelimiter
if v := opts.Delimiter; v != "" {
delimiter = v
}
separator := structSeparator
if v := opts.Separator; v != "" {
separator = v
}

noInit := structNoInit || opts.NoInit
overwrite := structOverwrite || opts.Overwrite
required := structRequired || opts.Required

isNilStructPtr := false
setNilStruct := func(v reflect.Value) {
Expand All @@ -302,7 +384,7 @@ func processWith(ctx context.Context, i any, l Lookuper, parentNoInit bool, mus
// If a struct (after traversal) equals to the empty value, it means
// nothing was changed in any sub-fields. With the noinit opt, we skip
// setting the empty value to the original struct pointer (keep it nil).
if !reflect.DeepEqual(v.Interface(), empty) || !shouldNotInit {
if !reflect.DeepEqual(v.Interface(), empty) || !noInit {
origin.Set(v)
}
}
Expand Down Expand Up @@ -337,7 +419,7 @@ func processWith(ctx context.Context, i any, l Lookuper, parentNoInit bool, mus
// Lookup the value, ignoring an error if the key isn't defined. This is
// required for nested structs that don't declare their own `env` keys,
// but have internal fields with an `env` defined.
val, _, _, err := lookup(key, opts, l)
val, _, _, err := lookup(key, required, opts.Default, l)
if err != nil && !errors.Is(err, ErrMissingKey) {
return fmt.Errorf("%s: %w", tf.Name, err)
}
Expand All @@ -356,7 +438,16 @@ func processWith(ctx context.Context, i any, l Lookuper, parentNoInit bool, mus
plu = PrefixLookuper(opts.Prefix, l)
}

if err := processWith(ctx, ef.Interface(), plu, shouldNotInit, mus...); err != nil {
if err := processWith(ctx, &Config{
Target: ef.Interface(),
Lookuper: plu,
DefaultDelimiter: delimiter,
DefaultSeparator: separator,
DefaultNoInit: noInit,
DefaultOverwrite: overwrite,
DefaultRequired: required,
Mutators: mutators,
}); err != nil {
return fmt.Errorf("%s: %w", tf.Name, err)
}

Expand All @@ -377,11 +468,11 @@ func processWith(ctx context.Context, i any, l Lookuper, parentNoInit bool, mus

// The field already has a non-zero value and overwrite is false, do not
// overwrite.
if (pointerWasSet || !ef.IsZero()) && !opts.Overwrite {
if (pointerWasSet || !ef.IsZero()) && !overwrite {
continue
}

val, found, usedDefault, err := lookup(key, opts, l)
val, found, usedDefault, err := lookup(key, required, opts.Default, l)
if err != nil {
return fmt.Errorf("%s: %w", tf.Name, err)
}
Expand All @@ -405,11 +496,7 @@ func processWith(ctx context.Context, i any, l Lookuper, parentNoInit bool, mus
originalValue := val
stop := false

for _, mu := range mus {
if mu == nil {
continue
}

for _, mu := range mutators {
val, stop, err = mu.EnvMutate(ctx, originalKey, resolvedKey, originalValue, val)
if err != nil {
return fmt.Errorf("%s: %w", tf.Name, err)
Expand All @@ -420,29 +507,33 @@ func processWith(ctx context.Context, i any, l Lookuper, parentNoInit bool, mus
}
}

// If Delimiter is not defined set it to ","
if opts.Delimiter == "" {
opts.Delimiter = defaultDelimiter
}

// If Separator is not defined set it to ":"
if opts.Separator == "" {
opts.Separator = defaultSeparator
}

// Set value.
if err := processField(val, ef, opts.Delimiter, opts.Separator, opts.NoInit); err != nil {
if err := processField(val, ef, delimiter, separator, noInit); err != nil {
return fmt.Errorf("%s(%q): %w", tf.Name, val, err)
}
}

return nil
}

// SplitString splits the given string on the provided rune, unless the rune is
// escaped by the escape character.
func splitString(s string, on string, esc string) []string {
a := strings.Split(s, on)

for i := len(a) - 2; i >= 0; i-- {
if strings.HasSuffix(a[i], esc) {
a[i] = a[i][:len(a[i])-len(esc)] + on + a[i+1]
a = append(a[:i+1], a[i+2:]...)
}
}
return a
}

// keyAndOpts parses the given tag value (e.g. env:"foo,required") and
// returns the key name and options as a list.
func keyAndOpts(tag string) (string, *options, error) {
parts := strings.Split(tag, ",")
parts := splitString(tag, ",", "\\")
key, tagOpts := strings.TrimSpace(parts[0]), parts[1:]

if key != "" && !validateEnvName(key) {
Expand Down Expand Up @@ -485,34 +576,34 @@ LOOP:
// first boolean parameter indicates whether the value was found in the
// lookuper. The second boolean parameter indicates whether the default value
// was used.
func lookup(key string, opts *options, l Lookuper) (string, bool, bool, error) {
func lookup(key string, required bool, defaultValue string, l Lookuper) (string, bool, bool, error) {
if key == "" {
// The struct has something like `env:",required"`, which is likely a
// mistake. We could try to infer the envvar from the field name, but that
// feels too magical.
return "", false, false, ErrMissingKey
}

if opts.Required && opts.Default != "" {
if required && defaultValue != "" {
// Having a default value on a required value doesn't make sense.
return "", false, false, ErrRequiredAndDefault
}

// Lookup value.
val, found := l.Lookup(key)
if !found {
if opts.Required {
if required {
if keyer, ok := l.(KeyedLookuper); ok {
key = keyer.Key(key)
}

return "", false, false, fmt.Errorf("%w: %s", ErrMissingRequired, key)
}

if opts.Default != "" {
if defaultValue != "" {
// Expand the default value. This allows for a default value that maps to
// a different variable.
val = os.Expand(opts.Default, func(i string) string {
val = os.Expand(defaultValue, func(i string) string {
s, ok := l.Lookup(i)
if ok {
return s
Expand Down

0 comments on commit ba15a27

Please sign in to comment.