From 15b4dd39bb8dc130123ffae2efcded75961093a7 Mon Sep 17 00:00:00 2001 From: x1unix Date: Thu, 10 Sep 2020 07:27:10 +0300 Subject: [PATCH 01/11] refactor dotenv parser in order to support multi-line variable values declaration Signed-off-by: x1unix --- godotenv.go | 63 ++++++---------- parser.go | 206 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 230 insertions(+), 39 deletions(-) create mode 100644 parser.go diff --git a/godotenv.go b/godotenv.go index 466f2eb..9b4bd3e 100644 --- a/godotenv.go +++ b/godotenv.go @@ -14,10 +14,10 @@ package godotenv import ( - "bufio" "errors" "fmt" "io" + "io/ioutil" "os" "os/exec" "regexp" @@ -28,6 +28,16 @@ import ( const doubleQuoteSpecialChars = "\\\n\r\"!$`" +// Parse reads an env file from io.Reader, returning a map of keys and values. +func Parse(r io.Reader) (map[string]string, error) { + data, err := ioutil.ReadAll(r) + if err != nil { + return nil, err + } + + return UnmarshalBytes(data) +} + // Load will read your env file(s) and load them into ENV for this process. // // Call this function as close as possible to the start of your program (ideally in main) @@ -96,37 +106,16 @@ func Read(filenames ...string) (envMap map[string]string, err error) { return } -// Parse reads an env file from io.Reader, returning a map of keys and values. -func Parse(r io.Reader) (envMap map[string]string, err error) { - envMap = make(map[string]string) - - var lines []string - scanner := bufio.NewScanner(r) - for scanner.Scan() { - lines = append(lines, scanner.Text()) - } - - if err = scanner.Err(); err != nil { - return - } - - for _, fullLine := range lines { - if !isIgnoredLine(fullLine) { - var key, value string - key, value, err = parseLine(fullLine, envMap) - - if err != nil { - return - } - envMap[key] = value - } - } - return +// Unmarshal reads an env file from a string, returning a map of keys and values. +func Unmarshal(str string) (envMap map[string]string, err error) { + return UnmarshalBytes([]byte(str)) } -//Unmarshal reads an env file from a string, returning a map of keys and values. -func Unmarshal(str string) (envMap map[string]string, err error) { - return Parse(strings.NewReader(str)) +// UnmarshalBytes parses env file from byte slice of chars, returning a map of keys and values. +func UnmarshalBytes(src []byte) (map[string]string, error) { + out := make(map[string]string) + err := parseBytes(src, out) + return out, err } // Exec loads env vars from the specified filenames (empty map falls back to default) @@ -137,7 +126,9 @@ func Unmarshal(str string) (envMap map[string]string, err error) { // If you want more fine grained control over your command it's recommended // that you use `Load()` or `Read()` and the `os/exec` package yourself. func Exec(filenames []string, cmd string, cmdArgs []string) error { - Load(filenames...) + if err := Load(filenames...); err != nil { + return err + } command := exec.Command(cmd, cmdArgs...) command.Stdin = os.Stdin @@ -161,8 +152,7 @@ func Write(envMap map[string]string, filename string) error { if err != nil { return err } - file.Sync() - return err + return file.Sync() } // Marshal outputs the given environment as a dotenv-formatted environment file. @@ -202,7 +192,7 @@ func loadFile(filename string, overload bool) error { for key, value := range envMap { if !currentEnv[key] || overload { - os.Setenv(key, value) + _ = os.Setenv(key, value) } } @@ -343,11 +333,6 @@ func expandVariables(v string, m map[string]string) string { }) } -func isIgnoredLine(line string) bool { - trimmedLine := strings.TrimSpace(line) - return len(trimmedLine) == 0 || strings.HasPrefix(trimmedLine, "#") -} - func doubleQuoteEscape(line string) string { for _, c := range doubleQuoteSpecialChars { toReplace := "\\" + string(c) diff --git a/parser.go b/parser.go new file mode 100644 index 0000000..e44d902 --- /dev/null +++ b/parser.go @@ -0,0 +1,206 @@ +package godotenv + +import ( + "bytes" + "errors" + "fmt" + "strings" + "unicode" +) + +const ( + charComment = '#' + prefixSingleQuote = '\'' + prefixDoubleQuote = '"' + + exportPrefix = "export" +) + +func parseBytes(src []byte, out map[string]string) error { + cutset := src + for { + cutset = getStatementStart(cutset) + if cutset == nil { + // reached end of file + break + } + + key, left, err := locateKeyName(cutset) + if err != nil { + return err + } + + value, left, err := extractVarValue(left, out) + if err != nil { + return err + } + + out[key] = value + cutset = left + } + + return nil +} + +// getStatementPosition returns position of statement begin. +// +// It skips any comment line or non-whitespace character. +func getStatementStart(src []byte) []byte { + pos := indexOfNonSpaceChar(src) + if pos == -1 { + return nil + } + + src = src[pos:] + if src[0] != charComment { + return src + } + + // skip comment section + pos = bytes.IndexFunc(src, isCharFunc('\n')) + if pos == -1 { + return nil + } + + return getStatementStart(src[pos:]) +} + +// locateKeyName locates and parses key name and returns rest of slice +func locateKeyName(src []byte) (key string, cutset []byte, err error) { + // trim "export" and space at beginning + src = bytes.TrimLeftFunc(bytes.TrimPrefix(src, []byte(exportPrefix)), isSpace) + + // locate key name end and validate it in single loop + offset := 0 +loop: + for i, char := range src { + rchar := rune(char) + if isSpace(rchar) { + continue + } + + switch char { + case '=', ':': + // library also supports yaml-style value declaration + key = string(src[0:i]) + offset = i + 1 + break loop + case '_': + default: + // variable name should match [A-Za-z0-9_] + if unicode.IsLetter(rchar) || unicode.IsNumber(rchar) { + continue + } + + return "", nil, fmt.Errorf( + `unexpected character %q in variable name near %q`, + string(char), string(src)) + } + } + + if len(src) == 0 { + return "", nil, errors.New("zero length string") + } + + // trim whitespace + key = strings.TrimRightFunc(key, unicode.IsSpace) + cutset = bytes.TrimLeftFunc(src[offset:], isSpace) + return key, cutset, nil +} + +// extractVarValue extracts variable value and returns rest of slice +func extractVarValue(src []byte, vars map[string]string) (value string, rest []byte, err error) { + quote, hasPrefix := hasQuotePrefix(src) + if !hasPrefix { + // unquoted value - read until whitespace + end := bytes.IndexFunc(src, unicode.IsSpace) + if end == -1 { + return expandVariables(string(src), vars), nil, nil + } + + return expandVariables(string(src[0:end]), vars), src[end:], nil + } + + // lookup quoted string terminator + for i := 1; i < len(src); i++ { + if char := src[i]; char != quote { + continue + } + + // skip escaped quote symbol (\" or \', depends on quote) + if prevChar := src[i-1]; prevChar == '\\' { + continue + } + + // trim quotes + trimFunc := isCharFunc(rune(quote)) + value = string(bytes.TrimLeftFunc(bytes.TrimRightFunc(src[0:i], trimFunc), trimFunc)) + if quote == prefixDoubleQuote { + // unescape newlines for double quote (this is compat feature) + // and expand environment variables + value = expandVariables(expandEscapes(value), vars) + } + + return value, src[i+1:], nil + } + + // return formatted error if quoted string is not terminated + valEndIndex := bytes.IndexFunc(src, isCharFunc('\n')) + if valEndIndex == -1 { + valEndIndex = len(src) + } + + return "", nil, fmt.Errorf("unterminated quoted value %s", src[:valEndIndex]) +} + +func expandEscapes(str string) string { + out := escapeRegex.ReplaceAllStringFunc(str, func(match string) string { + c := strings.TrimPrefix(match, `\`) + switch c { + case "n": + return "\n" + case "r": + return "\r" + default: + return match + } + }) + return unescapeCharsRegex.ReplaceAllString(out, "$1") +} + +func indexOfNonSpaceChar(src []byte) int { + return bytes.IndexFunc(src, func(r rune) bool { + return !unicode.IsSpace(r) + }) +} + +// hasQuotePrefix reports whether charset starts with single or double quote and returns quote character +func hasQuotePrefix(src []byte) (prefix byte, isQuored bool) { + if len(src) == 0 { + return 0, false + } + + switch prefix := src[0]; prefix { + case prefixDoubleQuote, prefixSingleQuote: + return prefix, true + default: + return 0, false + } +} + +func isCharFunc(char rune) func(rune) bool { + return func(v rune) bool { + return v == char + } +} + +// isSpace reports whether the rune is a space character but not line break character +// +// this differs from unicode.IsSpace, which also applies line break as space +func isSpace(r rune) bool { + switch r { + case '\t', '\v', '\f', '\r', ' ', 0x85, 0xA0: + return true + } + return false +} From dd57617d18fa9e38f94b09e42758bc41f356f534 Mon Sep 17 00:00:00 2001 From: x1unix Date: Thu, 10 Sep 2020 07:27:38 +0300 Subject: [PATCH 02/11] Add multi-line var values test case and update comment test Signed-off-by: x1unix --- godotenv_test.go | 83 +++++++++++++++++++++++++++++++++--------------- 1 file changed, 58 insertions(+), 25 deletions(-) diff --git a/godotenv_test.go b/godotenv_test.go index 7274c14..0045a8c 100644 --- a/godotenv_test.go +++ b/godotenv_test.go @@ -271,6 +271,34 @@ func TestExpanding(t *testing.T) { } +func TestVariableStringValueSeparator(t *testing.T) { + input := "TEST_URLS=\"stratum+tcp://stratum.antpool.com:3333\nstratum+tcp://stratum.antpool.com:443\"" + want := map[string]string{ + "TEST_URLS": "stratum+tcp://stratum.antpool.com:3333\nstratum+tcp://stratum.antpool.com:443", + } + got, err := Parse(strings.NewReader(input)) + if err != nil { + t.Error(err) + } + + if len(got) != len(want) { + t.Fatalf( + "unexpected value:\nwant:\n\t%#v\n\ngot:\n\t%#v", want, got) + } + + for k, wantVal := range want { + gotVal, ok := got[k] + if !ok { + t.Fatalf("key %q doesn't present in result", k) + } + if wantVal != gotVal { + t.Fatalf( + "mismatch in %q value:\nwant:\n\t%s\n\ngot:\n\t%s", k, + wantVal, gotVal) + } + } +} + func TestActualEnvVarsAreLeftAlone(t *testing.T) { os.Clearenv() os.Setenv("OPTION_A", "actualenv") @@ -377,33 +405,38 @@ func TestParsing(t *testing.T) { } func TestLinesToIgnore(t *testing.T) { - // it 'ignores empty lines' do - // expect(env("\n \t \nfoo=bar\n \nfizz=buzz")).to eql('foo' => 'bar', 'fizz' => 'buzz') - if !isIgnoredLine("\n") { - t.Error("Line with nothing but line break wasn't ignored") - } - - if !isIgnoredLine("\r\n") { - t.Error("Line with nothing but windows-style line break wasn't ignored") - } - - if !isIgnoredLine("\t\t ") { - t.Error("Line full of whitespace wasn't ignored") - } - - // it 'ignores comment lines' do - // expect(env("\n\n\n # HERE GOES FOO \nfoo=bar")).to eql('foo' => 'bar') - if !isIgnoredLine("# comment") { - t.Error("Comment wasn't ignored") - } - - if !isIgnoredLine("\t#comment") { - t.Error("Indented comment wasn't ignored") + cases := map[string]struct { + input string + want string + }{ + "Line with nothing but line break": { + input: "\n", + }, + "Line with nothing but windows-style line break": { + input: "\r\n", + }, + "Line full of whitespace": { + input: "\t\t ", + }, + "Comment": { + input: "# Comment", + }, + "Indented comment": { + input: "\t # comment", + }, + "non-ignored value": { + input: `export OPTION_B='\n'`, + want: `export OPTION_B='\n'`, + }, } - // make sure we're not getting false positives - if isIgnoredLine(`export OPTION_B='\n'`) { - t.Error("ignoring a perfectly valid line to parse") + for n, c := range cases { + t.Run(n, func(t *testing.T) { + got := string(getStatementStart([]byte(c.input))) + if got != c.want { + t.Errorf("Expected:\t %q\nGot:\t %q", c.want, got) + } + }) } } From 77e8b7d7d658fee2a9ddb99b7b6c91e90e37b24e Mon Sep 17 00:00:00 2001 From: John Barton Date: Fri, 24 Sep 2021 20:33:25 +1000 Subject: [PATCH 03/11] Expand fixture tests to include multiline strings --- fixtures/quoted.env | 10 ++++++++++ godotenv_test.go | 4 ++++ 2 files changed, 14 insertions(+) diff --git a/fixtures/quoted.env b/fixtures/quoted.env index cc6376a..7958933 100644 --- a/fixtures/quoted.env +++ b/fixtures/quoted.env @@ -7,3 +7,13 @@ OPTION_F="2" OPTION_G="" OPTION_H="\n" OPTION_I = "echo 'asd'" +OPTION_J='line 1 +line 2' +OPTION_K='line one +this is \'quoted\' +one more line' +OPTION_L="line 1 +line 2" +OPTION_M="line one +this is \"quoted\" +one more line" diff --git a/godotenv_test.go b/godotenv_test.go index 0045a8c..f5820b6 100644 --- a/godotenv_test.go +++ b/godotenv_test.go @@ -189,6 +189,10 @@ func TestLoadQuotedEnv(t *testing.T) { "OPTION_G": "", "OPTION_H": "\n", "OPTION_I": "echo 'asd'", + "OPTION_J": "line 1\nline 2", + "OPTION_K": "line one\nthis is \\'quoted\\'\none more line", + "OPTION_L": "line 1\nline 2", + "OPTION_M": "line one\nthis is \"quoted\"\none more line", } loadEnvAndCompareValues(t, Load, envFileName, expectedValues, noopPresets) From 2f66a86dc91b0218524d7bdf76a021751ae98773 Mon Sep 17 00:00:00 2001 From: John Barton Date: Fri, 24 Sep 2021 20:38:24 +1000 Subject: [PATCH 04/11] Update go versions to test against --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6f91de3..26cef60 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,7 +8,7 @@ jobs: strategy: fail-fast: false matrix: - go: [ '1.15', '1.14' ] + go: [ '1.17', '1.16', '1.15', '1.14' ] os: [ ubuntu-latest, macOS-latest, windows-latest ] name: ${{ matrix.os }} Go ${{ matrix.go }} Tests steps: From 22f9782344a65bfc1ad82232a46440f344a4f143 Mon Sep 17 00:00:00 2001 From: John Barton Date: Fri, 24 Sep 2021 20:42:44 +1000 Subject: [PATCH 05/11] Switch to GOINSECURE for power8 CI task --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 26cef60..c10a96b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,6 +41,7 @@ jobs: env: | # YAML pipe GOARCH: ${{ matrix.arch.architecture }} CGO_ENABLED: 0 + GOINSECURE: "**/*" run: | apt-get update apt-get install -q -y curl wget git @@ -56,7 +57,7 @@ jobs: go env printf "\n\nSystem environment:\n\n" env - go get -insecure -v -t -d ./... + go get -v -t -d ./... go test ./... cd ./cmd/godotenv go build -trimpath -ldflags="-w -s" -v \ No newline at end of file From a4061f306ab491654e44b9f36cd4d94b4570fe4f Mon Sep 17 00:00:00 2001 From: John Barton Date: Fri, 24 Sep 2021 20:58:07 +1000 Subject: [PATCH 06/11] When tests fail, show source version of string (inc special chars) --- godotenv_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/godotenv_test.go b/godotenv_test.go index f5820b6..28192c0 100644 --- a/godotenv_test.go +++ b/godotenv_test.go @@ -35,7 +35,7 @@ func loadEnvAndCompareValues(t *testing.T, loader func(files ...string) error, e envValue := os.Getenv(k) v := expectedValues[k] if envValue != v { - t.Errorf("Mismatch for key '%v': expected '%v' got '%v'", k, v, envValue) + t.Errorf("Mismatch for key '%v': expected '%#v' got '%#v'", k, v, envValue) } } } From 687a37d9c9933b92446155ed0bca30442868b3f6 Mon Sep 17 00:00:00 2001 From: John Barton Date: Fri, 27 Jan 2023 12:00:11 +1100 Subject: [PATCH 07/11] Update parser.go Co-authored-by: Austin Sasko --- parser.go | 1 + 1 file changed, 1 insertion(+) diff --git a/parser.go b/parser.go index e44d902..196418c 100644 --- a/parser.go +++ b/parser.go @@ -17,6 +17,7 @@ const ( ) func parseBytes(src []byte, out map[string]string) error { + src = bytes.Replace(src, []byte("\r\n"), []byte("\n"), -1) cutset := src for { cutset = getStatementStart(cutset) From 08d75deb239ccfe93269f236ac22992ca6695d66 Mon Sep 17 00:00:00 2001 From: John Barton Date: Fri, 27 Jan 2023 12:55:00 +1100 Subject: [PATCH 08/11] Fix up bad merge --- godotenv.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/godotenv.go b/godotenv.go index a281d3a..95124e3 100644 --- a/godotenv.go +++ b/godotenv.go @@ -108,7 +108,15 @@ func Read(filenames ...string) (envMap map[string]string, err error) { // Unmarshal reads an env file from a string, returning a map of keys and values. func Unmarshal(str string) (envMap map[string]string, err error) { - return Parse(strings.NewReader(str)) + return UnmarshalBytes([]byte(str)) +} + +// UnmarshalBytes parses env file from byte slice of chars, returning a map of keys and values. +func UnmarshalBytes(src []byte) (map[string]string, error) { + out := make(map[string]string) + err := parseBytes(src, out) + + return out, err } // Exec loads env vars from the specified filenames (empty map falls back to default) From 02a6516148dd6f7bba61583ba4421a5870419729 Mon Sep 17 00:00:00 2001 From: John Barton Date: Fri, 27 Jan 2023 13:00:16 +1100 Subject: [PATCH 09/11] Add a full fixture for comments for extra piece of mind --- fixtures/comments.env | 4 ++++ godotenv_test.go | 11 +++++++++++ 2 files changed, 15 insertions(+) create mode 100644 fixtures/comments.env diff --git a/fixtures/comments.env b/fixtures/comments.env new file mode 100644 index 0000000..af9781f --- /dev/null +++ b/fixtures/comments.env @@ -0,0 +1,4 @@ +# Full line comment +foo=bar # baz +bar=foo#baz +baz="foo"#bar diff --git a/godotenv_test.go b/godotenv_test.go index 05aaca8..017d6b6 100644 --- a/godotenv_test.go +++ b/godotenv_test.go @@ -461,6 +461,17 @@ func TestErrorParsing(t *testing.T) { } } +func TestComments(t *testing.T) { + envFileName := "fixtures/comments.env" + expectedValues := map[string]string{ + "foo": "bar", + "bar": "foo#baz", + "baz": "foo", + } + + loadEnvAndCompareValues(t, Load, envFileName, expectedValues, noopPresets) +} + func TestWrite(t *testing.T) { writeAndCompare := func(env string, expected string) { envMap, _ := Unmarshal(env) From d4cc462af709ac39f05af4930620645b745e0e24 Mon Sep 17 00:00:00 2001 From: John Barton Date: Fri, 27 Jan 2023 13:06:52 +1100 Subject: [PATCH 10/11] Fix up some lint/staticcheck recommendations --- godotenv.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/godotenv.go b/godotenv.go index 95124e3..368e804 100644 --- a/godotenv.go +++ b/godotenv.go @@ -250,15 +250,15 @@ func parseLine(line string, envMap map[string]string) (key string, value string, } if len(splitString) != 2 { - err = errors.New("Can't separate key from value") + err = errors.New("can't separate key from value") return } // Parse the key key = splitString[0] - if strings.HasPrefix(key, "export") { - key = strings.TrimPrefix(key, "export") - } + + key = strings.TrimPrefix(key, "export") + key = strings.TrimSpace(key) key = exportRegex.ReplaceAllString(splitString[0], "$1") From 1e1ed3220773273780175d10e4eb139c8d63a0a3 Mon Sep 17 00:00:00 2001 From: John Barton Date: Fri, 27 Jan 2023 13:07:07 +1100 Subject: [PATCH 11/11] Test against go 1.19 too --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c740ad0..1aef374 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,7 +8,7 @@ jobs: strategy: fail-fast: false matrix: - go: [ '1.18', '1.17', '1.16', '1.15' ] + go: [ '1.19', '1.18', '1.17', '1.16', '1.15' ] os: [ ubuntu-latest, macOS-latest, windows-latest ] name: ${{ matrix.os }} Go ${{ matrix.go }} Tests steps: