diff --git a/README.md b/README.md index 9eff708..d77de95 100644 --- a/README.md +++ b/README.md @@ -104,7 +104,7 @@ func main() { port = fs.Int("port", 8080, "listen port for server (also via PORT)") debug = fs.Bool("debug", false, "log debug information (also via DEBUG)") ) - ff.Parse(fs, os.Args[1:], ff.WithEnvVarNoPrefix()) + ff.Parse(fs, os.Args[1:], ff.WithEnvVars()) fmt.Printf("port %d, debug %v\n", *port, *debug) } diff --git a/env_parser.go b/env_parser.go new file mode 100644 index 0000000..b2cea78 --- /dev/null +++ b/env_parser.go @@ -0,0 +1,58 @@ +package ff + +import ( + "bufio" + "fmt" + "io" + "strconv" + "strings" +) + +// EnvParser is a parser for .env files. Each line is tokenized on the first `=` +// character. The first token is interpreted as the flag name, and the second +// token is interpreted as the value. Both tokens are trimmed of leading and +// trailing whitespace. If the value is "double quoted", control characters like +// `\n` are expanded. Lines beginning with `#` are interpreted as comments. +// +// EnvParser respects WithEnvVarPrefix, e.g. an .env file containing `A_B=c` +// will set a flag named "b" if Parse is called with WithEnvVarPrefix("A"). +func EnvParser(r io.Reader, set func(name, value string) error) error { + s := bufio.NewScanner(r) + for s.Scan() { + line := strings.TrimSpace(s.Text()) + if line == "" { + continue // skip empties + } + + if line[0] == '#' { + continue // skip comments + } + + index := strings.IndexRune(line, '=') + if index < 0 { + return fmt.Errorf("invalid line: %s", line) + } + + var ( + name = strings.TrimSpace(line[:index]) + value = strings.TrimSpace(line[index+1:]) + ) + + if len(name) <= 0 { + return fmt.Errorf("invalid line: %s", line) + } + + if len(value) <= 0 { + return fmt.Errorf("invalid line: %s", line) + } + + if unquoted, err := strconv.Unquote(value); err == nil { + value = unquoted + } + + if err := set(name, value); err != nil { + return err + } + } + return nil +} diff --git a/env_parser_test.go b/env_parser_test.go new file mode 100644 index 0000000..9f34e54 --- /dev/null +++ b/env_parser_test.go @@ -0,0 +1,66 @@ +package ff_test + +import ( + "path/filepath" + "testing" + "time" + + "github.com/peterbourgon/ff/v3" + "github.com/peterbourgon/ff/v3/fftest" +) + +func TestEnvFileParser(t *testing.T) { + t.Parallel() + + for _, testcase := range []struct { + file string + opts []ff.Option + want fftest.Vars + }{ + { + file: "testdata/empty.env", + want: fftest.Vars{}, + }, + { + file: "testdata/basic.env", + want: fftest.Vars{S: "bar", I: 99, B: true, D: time.Hour}, + }, + { + file: "testdata/prefix.env", + opts: []ff.Option{ff.WithEnvVarPrefix("MYPROG")}, + want: fftest.Vars{S: "bingo", I: 123}, + }, + { + file: "testdata/prefix-undef.env", + opts: []ff.Option{ff.WithEnvVarPrefix("MYPROG"), ff.WithIgnoreUndefined(true)}, + want: fftest.Vars{S: "bango", I: 9}, + }, + { + file: "testdata/quotes.env", + want: fftest.Vars{S: "", I: 32, X: []string{"1", "2 2", "3 3 3"}}, + }, + { + file: "testdata/no-value.env", + want: fftest.Vars{WantParseErrorString: "invalid line: D="}, + }, + { + file: "testdata/spaces.env", + want: fftest.Vars{X: []string{"1", "2", "3", "4", "5", " 6", " 7 ", " 8 "}}, + }, + { + file: "testdata/newlines.env", + want: fftest.Vars{S: "one\ntwo\nthree\n\n", X: []string{`A\nB\n\n`}}, + }, + { + file: "testdata/capitalization.env", + want: fftest.Vars{S: "hello", I: 12345}, + }, + } { + t.Run(filepath.Base(testcase.file), func(t *testing.T) { + testcase.opts = append(testcase.opts, ff.WithConfigFile(testcase.file), ff.WithConfigFileParser(ff.EnvParser)) + fs, vars := fftest.Pair() + vars.ParseError = ff.Parse(fs, []string{}, testcase.opts...) + fftest.Compare(t, &testcase.want, vars) + }) + } +} diff --git a/ffcli/command_test.go b/ffcli/command_test.go index e1988cb..45b1100 100644 --- a/ffcli/command_test.go +++ b/ffcli/command_test.go @@ -355,7 +355,7 @@ func TestIssue57(t *testing.T) { }, { args: []string{"bar", "-undefined"}, - parseErrStr: "error parsing commandline args: flag provided but not defined: -undefined", + parseErrStr: "error parsing commandline arguments: flag provided but not defined: -undefined", runErrIs: ffcli.ErrUnparsed, }, { @@ -368,7 +368,7 @@ func TestIssue57(t *testing.T) { }, { args: []string{"bar", "baz", "-also.undefined"}, - parseErrStr: "error parsing commandline args: flag provided but not defined: -also.undefined", + parseErrStr: "error parsing commandline arguments: flag provided but not defined: -also.undefined", runErrIs: ffcli.ErrUnparsed, }, } { diff --git a/options.go b/options.go new file mode 100644 index 0000000..de09749 --- /dev/null +++ b/options.go @@ -0,0 +1 @@ +package ff diff --git a/parse.go b/parse.go index a0f7f2c..0c53daf 100644 --- a/parse.go +++ b/parse.go @@ -1,7 +1,6 @@ package ff import ( - "bufio" "flag" "fmt" "io" @@ -9,18 +8,36 @@ import ( "strings" ) +// ConfigFileParser interprets the config file represented by the reader +// and calls the set function for each parsed flag pair. +type ConfigFileParser func(r io.Reader, set func(name, value string) error) error + +type lookupFunc func(fs *flag.FlagSet, name string) *flag.Flag + // Parse the flags in the flag set from the provided (presumably commandline) -// args. Additional options may be provided to parse from a config file and/or -// environment variables in that priority order. +// args. Additional options may be provided to have Parse also read from a +// config file, and/or environment variables, in that priority order. func Parse(fs *flag.FlagSet, args []string, options ...Option) error { var c Context for _, option := range options { option(&c) } + flag2env := map[*flag.Flag]string{} + env2flag := map[string]*flag.Flag{} + fs.VisitAll(func(f *flag.Flag) { + var key string + key = strings.ToUpper(f.Name) + key = flagNameToEnvVar.Replace(key) + key = maybePrefix(c.envVarPrefix != "", key, c.envVarPrefix) + env2flag[key] = f + flag2env[f] = key + }) + // First priority: commandline flags (explicit user preference). + if err := fs.Parse(args); err != nil { - return fmt.Errorf("error parsing commandline args: %w", err) + return fmt.Errorf("error parsing commandline arguments: %w", err) } provided := map[string]bool{} @@ -29,7 +46,8 @@ func Parse(fs *flag.FlagSet, args []string, options ...Option) error { }) // Second priority: environment variables (session). - if parseEnv := c.envVarPrefix != "" || c.envVarNoPrefix; parseEnv { + + if c.readEnvVars { var visitErr error fs.VisitAll(func(f *flag.Flag) { if visitErr != nil { @@ -40,10 +58,10 @@ func Parse(fs *flag.FlagSet, args []string, options ...Option) error { return } - var key string - key = strings.ToUpper(f.Name) - key = envVarReplacer.Replace(key) - key = maybePrefix(key, c.envVarNoPrefix, c.envVarPrefix) + key, ok := flag2env[f] + if !ok { + panic(fmt.Errorf("%s: invalid flag/env mapping", f.Name)) + } value := os.Getenv(key) if value == "" { @@ -52,13 +70,13 @@ func Parse(fs *flag.FlagSet, args []string, options ...Option) error { for _, v := range maybeSplit(value, c.envVarSplit) { if err := fs.Set(f.Name, v); err != nil { - visitErr = fmt.Errorf("error setting flag %q from env var %q: %w", f.Name, key, err) + visitErr = fmt.Errorf("error setting flag %q from environment variable %q: %w", f.Name, key, err) return } } }) if visitErr != nil { - return fmt.Errorf("error parsing env vars: %w", visitErr) + return fmt.Errorf("error parsing environment variables: %w", visitErr) } } @@ -66,19 +84,31 @@ func Parse(fs *flag.FlagSet, args []string, options ...Option) error { provided[f.Name] = true }) + // Third priority: config file (host). + var configFile string if c.configFileVia != nil { configFile = *c.configFileVia } - // Third priority: config file (host). if configFile == "" && c.configFileFlagName != "" { if f := fs.Lookup(c.configFileFlagName); f != nil { configFile = f.Value.String() } } - if parseConfig := configFile != "" && c.configFileParser != nil; parseConfig { + if c.configFileLookup == nil { + c.configFileLookup = func(fs *flag.FlagSet, name string) *flag.Flag { + return fs.Lookup(name) + } + } + + var ( + haveConfigFile = configFile != "" + haveParser = c.configFileParser != nil + parseConfigFile = haveConfigFile && haveParser + ) + if parseConfigFile { f, err := os.Open(configFile) switch { case err == nil: @@ -88,15 +118,31 @@ func Parse(fs *flag.FlagSet, args []string, options ...Option) error { return nil } - defined := fs.Lookup(name) != nil + var ( + f1 = fs.Lookup(name) + f2 = env2flag[name] + f *flag.Flag + ) switch { - case !defined && c.ignoreUndefined: + case f1 == nil && f2 == nil && c.ignoreUndefined: return nil - case !defined && !c.ignoreUndefined: + case f1 == nil && f2 == nil && !c.ignoreUndefined: return fmt.Errorf("config file flag %q not defined in flag set", name) + case f1 != nil && f2 == nil: + f = f1 + case f1 == nil && f2 != nil: + f = f2 + case f1 != nil && f2 != nil && f1 == f2: + f = f1 + case f1 != nil && f2 != nil && f1 != f2: + return fmt.Errorf("config file flag %q ambiguous: matches %s and %s", name, f1.Name, f2.Name) } - if err := fs.Set(name, value); err != nil { + if provided[f.Name] { + return nil + } + + if err := fs.Set(f.Name, value); err != nil { return fmt.Errorf("error setting flag %q from config file: %w", name, err) } @@ -125,9 +171,10 @@ type Context struct { configFileVia *string configFileFlagName string configFileParser ConfigFileParser + configFileLookup lookupFunc allowMissingConfigFile bool + readEnvVars bool envVarPrefix string - envVarNoPrefix bool envVarSplit string ignoreUndefined bool } @@ -136,17 +183,17 @@ type Context struct { type Option func(*Context) // WithConfigFile tells Parse to read the provided filename as a config file. -// Requires WithConfigFileParser, and overrides WithConfigFileFlag. -// Because config files should generally be user-specifiable, this option -// should be rarely used. Prefer WithConfigFileFlag. +// Requires WithConfigFileParser, and overrides WithConfigFileFlag. Because +// config files should generally be user-specifiable, this option should rarely +// be used; prefer WithConfigFileFlag. func WithConfigFile(filename string) Option { return WithConfigFileVia(&filename) } // WithConfigFileVia tells Parse to read the provided filename as a config file. -// Requires WithConfigFileParser, and overrides WithConfigFileFlag. -// This is useful for sharing a single root level flag for config files among -// multiple ffcli subcommands. +// Requires WithConfigFileParser, and overrides WithConfigFileFlag. This is +// useful for sharing a single root level flag for config files among multiple +// ffcli subcommands. func WithConfigFileVia(filename *string) Option { return func(c *Context) { c.configFileVia = filename @@ -158,8 +205,7 @@ func WithConfigFileVia(filename *string) Option { // WithConfigFile. // // To specify a default config file, provide it as the default value of the -// corresponding flag -- and consider also using the WithAllowMissingConfigFile -// option. +// corresponding flag. See also: WithAllowMissingConfigFile. func WithConfigFileFlag(flagname string) Option { return func(c *Context) { c.configFileFlagName = flagname @@ -175,33 +221,41 @@ func WithConfigFileParser(p ConfigFileParser) Option { } // WithAllowMissingConfigFile tells Parse to permit the case where a config file -// is specified but doesn't exist. By default, missing config files result in an -// error. +// is specified but doesn't exist. +// +// By default, missing config files cause Parse to fail. func WithAllowMissingConfigFile(allow bool) Option { return func(c *Context) { c.allowMissingConfigFile = allow } } -// WithEnvVarPrefix tells Parse to try to set flags from environment variables -// with the given prefix. Flag names are matched to environment variables with -// the given prefix, followed by an underscore, followed by the capitalized flag -// names, with separator characters like periods or hyphens replaced with -// underscores. By default, flags are not set from environment variables at all. -func WithEnvVarPrefix(prefix string) Option { +// WithEnvVarNoPrefix is an alias for WithEnvVars. +// +// DEPRECATED: prefer WithEnvVars. +var WithEnvVarNoPrefix = WithEnvVars + +// WithEnvVars tells Parse to set flags from environment variables. Flag +// names are matched to environment variables by capitalizing the flag name, and +// replacing separator characters like periods or hyphens with underscores. +// +// By default, flags are not set from environment variables at all. +func WithEnvVars() Option { return func(c *Context) { - c.envVarPrefix = prefix + c.readEnvVars = true } } -// WithEnvVarNoPrefix tells Parse to try to set flags from environment variables -// without any specific prefix. Flag names are matched to environment variables -// by capitalizing the flag name, and replacing separator characters like -// periods or hyphens with underscores. By default, flags are not set from -// environment variables at all. -func WithEnvVarNoPrefix() Option { +// WithEnvVarPrefix is like WithEnvVars, but only considers environment +// variables beginning with the given prefix followed by an underscore. That +// prefix (and underscore) are removed before matching to flag names. This +// option is also respected by the EnvParser config file parser. +// +// By default, flags are not set from environment variables at all. +func WithEnvVarPrefix(prefix string) Option { return func(c *Context) { - c.envVarNoPrefix = true + c.readEnvVars = true + c.envVarPrefix = prefix } } @@ -224,60 +278,17 @@ func WithIgnoreUndefined(ignore bool) Option { } } -// ConfigFileParser interprets the config file represented by the reader -// and calls the set function for each parsed flag pair. -type ConfigFileParser func(r io.Reader, set func(name, value string) error) error - -// PlainParser is a parser for config files in an extremely simple format. Each -// line is tokenized as a single key/value pair. The first whitespace-delimited -// token in the line is interpreted as the flag name, and all remaining tokens -// are interpreted as the value. Any leading hyphens on the flag name are -// ignored. -func PlainParser(r io.Reader, set func(name, value string) error) error { - s := bufio.NewScanner(r) - for s.Scan() { - line := strings.TrimSpace(s.Text()) - if line == "" { - continue // skip empties - } - - if line[0] == '#' { - continue // skip comments - } - - var ( - name string - value string - index = strings.IndexRune(line, ' ') - ) - if index < 0 { - name, value = line, "true" // boolean option - } else { - name, value = line[:index], strings.TrimSpace(line[index:]) - } - - if i := strings.Index(value, " #"); i >= 0 { - value = strings.TrimSpace(value[:i]) - } - - if err := set(name, value); err != nil { - return err - } - } - return nil -} - -var envVarReplacer = strings.NewReplacer( +var flagNameToEnvVar = strings.NewReplacer( "-", "_", ".", "_", "/", "_", ) -func maybePrefix(key string, noPrefix bool, prefix string) string { - if noPrefix { - return key +func maybePrefix(doPrefix bool, key string, prefix string) string { + if doPrefix { + key = strings.ToUpper(prefix) + "_" + key } - return strings.ToUpper(prefix) + "_" + key + return key } func maybeSplit(value, split string) []string { diff --git a/parse_test.go b/parse_test.go index 8524c5b..e20d7b4 100644 --- a/parse_test.go +++ b/parse_test.go @@ -108,9 +108,9 @@ func TestParseBasics(t *testing.T) { want: fftest.Vars{S: "three", X: []string{"one", "two", "three"}}, }, { - name: "WithEnvVarNoPrefix", + name: "WithEnvVars", env: map[string]string{"TEST_PARSE_S": "foo", "S": "bar"}, - opts: []ff.Option{ff.WithEnvVarNoPrefix()}, + opts: []ff.Option{ff.WithEnvVars()}, want: fftest.Vars{S: "bar"}, }, { @@ -305,5 +305,4 @@ func TestParseConfigFileVia(t *testing.T) { if want, have := 99, *i; want != have { t.Errorf("i: want %d, have %d", want, have) } - } diff --git a/plain_parser.go b/plain_parser.go new file mode 100644 index 0000000..682fcfe --- /dev/null +++ b/plain_parser.go @@ -0,0 +1,46 @@ +package ff + +import ( + "bufio" + "io" + "strings" +) + +// PlainParser is a parser for config files in an extremely simple format. Each +// line is tokenized as a single key/value pair. The first whitespace-delimited +// token in the line is interpreted as the flag name, and all remaining tokens +// are interpreted as the value. Any leading hyphens on the flag name are +// ignored. +func PlainParser(r io.Reader, set func(name, value string) error) error { + s := bufio.NewScanner(r) + for s.Scan() { + line := strings.TrimSpace(s.Text()) + if line == "" { + continue // skip empties + } + + if line[0] == '#' { + continue // skip comments + } + + var ( + name string + value string + index = strings.IndexRune(line, ' ') + ) + if index < 0 { + name, value = line, "true" // boolean option + } else { + name, value = line[:index], strings.TrimSpace(line[index:]) + } + + if i := strings.Index(value, " #"); i >= 0 { + value = strings.TrimSpace(value[:i]) + } + + if err := set(name, value); err != nil { + return err + } + } + return nil +} diff --git a/testdata/basic.env b/testdata/basic.env new file mode 100644 index 0000000..98a34bf --- /dev/null +++ b/testdata/basic.env @@ -0,0 +1,4 @@ +S=bar +I=99 +B=true +D=1h diff --git a/testdata/capitalization.env b/testdata/capitalization.env new file mode 100644 index 0000000..218bdea --- /dev/null +++ b/testdata/capitalization.env @@ -0,0 +1,2 @@ +s=hello +i=12345 diff --git a/testdata/empty.env b/testdata/empty.env new file mode 100644 index 0000000..e69de29 diff --git a/testdata/newlines.env b/testdata/newlines.env new file mode 100644 index 0000000..fb603a0 --- /dev/null +++ b/testdata/newlines.env @@ -0,0 +1,2 @@ +S="one\ntwo\nthree\n\n" +X=A\nB\n\n diff --git a/testdata/no-value.env b/testdata/no-value.env new file mode 100644 index 0000000..2cc4f9b --- /dev/null +++ b/testdata/no-value.env @@ -0,0 +1,3 @@ +I=32 +D= +S=this is fine diff --git a/testdata/prefix-undef.env b/testdata/prefix-undef.env new file mode 100644 index 0000000..bbb8753 --- /dev/null +++ b/testdata/prefix-undef.env @@ -0,0 +1,6 @@ + + +MYPROG_I=9 +OTHERPREFIX_B=true +MYPROG_S=bango +D=32m diff --git a/testdata/prefix.env b/testdata/prefix.env new file mode 100644 index 0000000..b8a2574 --- /dev/null +++ b/testdata/prefix.env @@ -0,0 +1,4 @@ + +MYPROG_S=bingo + +MYPROG_I=123 diff --git a/testdata/quotes.env b/testdata/quotes.env new file mode 100644 index 0000000..b1625e7 --- /dev/null +++ b/testdata/quotes.env @@ -0,0 +1,5 @@ +S="" +X=1 +X=2 2 +X="3 3 3" +I="32" diff --git a/testdata/spaces.env b/testdata/spaces.env new file mode 100644 index 0000000..e9ef4fe --- /dev/null +++ b/testdata/spaces.env @@ -0,0 +1,8 @@ +X = 1 +X= 2 +X =3 +X= 4 +X = 5 +X=" 6" +X= " 7 " +X = " 8 "