From e28dea23a6f2ecf17141406891db68d98675eb99 Mon Sep 17 00:00:00 2001 From: Stefan Prodan Date: Mon, 8 Apr 2024 00:15:43 +0300 Subject: [PATCH 1/4] Add envsubst package Forked from https://github.com/drone/envsubst Signed-off-by: Stefan Prodan --- envsubst/README.md | 39 +++ envsubst/eval.go | 39 +++ envsubst/eval_test.go | 248 ++++++++++++++++ envsubst/funcs.go | 266 +++++++++++++++++ envsubst/funcs_test.go | 124 ++++++++ envsubst/go.mod | 5 + envsubst/go.sum | 2 + envsubst/parse/node.go | 106 +++++++ envsubst/parse/parse.go | 422 +++++++++++++++++++++++++++ envsubst/parse/parse_test.go | 551 +++++++++++++++++++++++++++++++++++ envsubst/parse/scan.go | 314 ++++++++++++++++++++ envsubst/path/match.go | 206 +++++++++++++ envsubst/template.go | 177 +++++++++++ 13 files changed, 2499 insertions(+) create mode 100644 envsubst/README.md create mode 100644 envsubst/eval.go create mode 100644 envsubst/eval_test.go create mode 100644 envsubst/funcs.go create mode 100644 envsubst/funcs_test.go create mode 100644 envsubst/go.mod create mode 100644 envsubst/go.sum create mode 100644 envsubst/parse/node.go create mode 100644 envsubst/parse/parse.go create mode 100644 envsubst/parse/parse_test.go create mode 100644 envsubst/parse/scan.go create mode 100644 envsubst/path/match.go create mode 100644 envsubst/template.go diff --git a/envsubst/README.md b/envsubst/README.md new file mode 100644 index 00000000..f8841de7 --- /dev/null +++ b/envsubst/README.md @@ -0,0 +1,39 @@ +# envsubst + +`github.com/fluxcd/pkg/envsubst` is a Go package for expanding variables in a string using `${var}` syntax. +Includes support for bash string replacement functions. + +This package is a fork of [drone/envsubst](https://github.com/drone/envsubst). + +## Supported Functions + +| __Expression__ | __Meaning__ | +|-------------------------------|---------------------------------------------------------------------| +| `${var}` | Value of `$var` | +| `${#var}` | String length of `$var` | +| `${var^}` | Uppercase first character of `$var` | +| `${var^^}` | Uppercase all characters in `$var` | +| `${var,}` | Lowercase first character of `$var` | +| `${var,,}` | Lowercase all characters in `$var` | +| `${var:n}` | Offset `$var` `n` characters from start | +| `${var:n:len}` | Offset `$var` `n` characters with max length of `len` | +| `${var#pattern}` | Strip shortest `pattern` match from start | +| `${var##pattern}` | Strip longest `pattern` match from start | +| `${var%pattern}` | Strip shortest `pattern` match from end | +| `${var%%pattern}` | Strip longest `pattern` match from end | +| `${var-default}` | If `$var` is not set, evaluate expression as `$default` | +| `${var:-default}` | If `$var` is not set or is empty, evaluate expression as `$default` | +| `${var=default}` | If `$var` is not set, evaluate expression as `$default` | +| `${var:=default}` | If `$var` is not set or is empty, evaluate expression as `$default` | +| `${var/pattern/replacement}` | Replace as few `pattern` matches as possible with `replacement` | +| `${var//pattern/replacement}` | Replace as many `pattern` matches as possible with `replacement` | +| `${var/#pattern/replacement}` | Replace `pattern` match with `replacement` from `$var` start | +| `${var/%pattern/replacement}` | Replace `pattern` match with `replacement` from `$var` end | + +For a deeper reference, see [bash-hackers](https://wiki.bash-hackers.org/syntax/pe#case_modification) or [gnu pattern matching](https://www.gnu.org/software/bash/manual/html_node/Pattern-Matching.html). + +## Unsupported Functions + +* `${var+default}` +* `${var:?default}` +* `${var:+default}` diff --git a/envsubst/eval.go b/envsubst/eval.go new file mode 100644 index 00000000..5255e697 --- /dev/null +++ b/envsubst/eval.go @@ -0,0 +1,39 @@ +/* +Copyright 2024 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Forked from: https://github.com/drone/envsubst +// MIT License +// Copyright (c) 2017 drone.io. + +package envsubst + +import "os" + +// Eval replaces ${var} in the string based on the mapping function. +func Eval(s string, mapping func(string) string) (string, error) { + t, err := Parse(s) + if err != nil { + return s, err + } + return t.Execute(mapping) +} + +// EvalEnv replaces ${var} in the string according to the values of the +// current environment variables. References to undefined variables are +// replaced by the empty string. +func EvalEnv(s string) (string, error) { + return Eval(s, os.Getenv) +} diff --git a/envsubst/eval_test.go b/envsubst/eval_test.go new file mode 100644 index 00000000..d3c1b19b --- /dev/null +++ b/envsubst/eval_test.go @@ -0,0 +1,248 @@ +/* +Copyright 2024 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Forked from: https://github.com/drone/envsubst +// MIT License +// Copyright (c) 2017 drone.io. + +package envsubst + +import "testing" + +// test cases sourced from tldp.org +// http://www.tldp.org/LDP/abs/html/parameter-substitution.html + +func TestExpand(t *testing.T) { + var expressions = []struct { + params map[string]string + input string + output string + }{ + // text-only + { + params: map[string]string{}, + input: "abcdEFGH28ij", + output: "abcdEFGH28ij", + }, + // length + { + params: map[string]string{"var01": "abcdEFGH28ij"}, + input: "${#var01}", + output: "12", + }, + // uppercase first + { + params: map[string]string{"var01": "abcdEFGH28ij"}, + input: "${var01^}", + output: "AbcdEFGH28ij", + }, + // uppercase + { + params: map[string]string{"var01": "abcdEFGH28ij"}, + input: "${var01^^}", + output: "ABCDEFGH28IJ", + }, + // lowercase first + { + params: map[string]string{"var01": "ABCDEFGH28IJ"}, + input: "${var01,}", + output: "aBCDEFGH28IJ", + }, + // lowercase + { + params: map[string]string{"var01": "ABCDEFGH28IJ"}, + input: "${var01,,}", + output: "abcdefgh28ij", + }, + // substring with position + { + params: map[string]string{"path_name": "/home/bozo/ideas/thoughts.for.today"}, + input: "${path_name:11}", + output: "ideas/thoughts.for.today", + }, + // substring with position and length + { + params: map[string]string{"path_name": "/home/bozo/ideas/thoughts.for.today"}, + input: "${path_name:11:5}", + output: "ideas", + }, + // default not used + { + params: map[string]string{"var": "abc"}, + input: "${var=xyz}", + output: "abc", + }, + // default used + { + params: map[string]string{}, + input: "${var=xyz}", + output: "xyz", + }, + { + params: map[string]string{"default_var": "foo"}, + input: "something ${var=${default_var}}", + output: "something foo", + }, + { + params: map[string]string{"default_var": "foo1"}, + input: `foo: ${var=${default_var}-suffix}`, + output: "foo: foo1-suffix", + }, + { + params: map[string]string{"default_var": "foo1"}, + input: `foo: ${var=prefix${default_var}-suffix}`, + output: "foo: prefixfoo1-suffix", + }, + { + params: map[string]string{}, + input: "${var:=xyz}", + output: "xyz", + }, + // replace suffix + { + params: map[string]string{"stringZ": "abcABC123ABCabc"}, + input: "${stringZ/%abc/XYZ}", + output: "abcABC123ABCXYZ", + }, + // replace prefix + { + params: map[string]string{"stringZ": "abcABC123ABCabc"}, + input: "${stringZ/#abc/XYZ}", + output: "XYZABC123ABCabc", + }, + // replace all + { + params: map[string]string{"stringZ": "abcABC123ABCabc"}, + input: "${stringZ//abc/xyz}", + output: "xyzABC123ABCxyz", + }, + // replace first + { + params: map[string]string{"stringZ": "abcABC123ABCabc"}, + input: "${stringZ/abc/xyz}", + output: "xyzABC123ABCabc", + }, + // delete shortest match prefix + { + params: map[string]string{"filename": "bash.string.txt"}, + input: "${filename#*.}", + output: "string.txt", + }, + { + params: map[string]string{"filename": "path/to/file"}, + input: "${filename#*/}", + output: "to/file", + }, + { + params: map[string]string{"filename": "/path/to/file"}, + input: "${filename#*/}", + output: "path/to/file", + }, + // delete longest match prefix + { + params: map[string]string{"filename": "bash.string.txt"}, + input: "${filename##*.}", + output: "txt", + }, + { + params: map[string]string{"filename": "path/to/file"}, + input: "${filename##*/}", + output: "file", + }, + { + params: map[string]string{"filename": "/path/to/file"}, + input: "${filename##*/}", + output: "file", + }, + // delete shortest match suffix + { + params: map[string]string{"filename": "bash.string.txt"}, + input: "${filename%.*}", + output: "bash.string", + }, + // delete longest match suffix + { + params: map[string]string{"filename": "bash.string.txt"}, + input: "${filename%%.*}", + output: "bash", + }, + + // nested parameters + { + params: map[string]string{"var01": "abcdEFGH28ij"}, + input: "${var=${var01^^}}", + output: "ABCDEFGH28IJ", + }, + // escaped + { + params: map[string]string{"var01": "abcdEFGH28ij"}, + input: "$${var01}", + output: "${var01}", + }, + { + params: map[string]string{"var01": "abcdEFGH28ij"}, + input: "some text ${var01}$${var$${var01}$var01${var01}", + output: "some text abcdEFGH28ij${var${var01}$var01abcdEFGH28ij", + }, + { + params: map[string]string{"default_var": "foo"}, + input: "something $${var=${default_var}}", + output: "something ${var=foo}", + }, + // some common escaping use cases + { + params: map[string]string{"stringZ": "foo/bar"}, + input: `${stringZ/\//-}`, + output: "foo-bar", + }, + { + params: map[string]string{"stringZ": "foo/bar/baz"}, + input: `${stringZ//\//-}`, + output: "foo-bar-baz", + }, + // escape outside of expansion shouldn't be processed + { + params: map[string]string{"default_var": "foo"}, + input: "\\\\something ${var=${default_var}}", + output: "\\\\something foo", + }, + // substitute with a blank string + { + params: map[string]string{"stringZ": "foo.bar"}, + input: `${stringZ/./}`, + output: "foobar", + }, + } + + for _, expr := range expressions { + t.Run(expr.input, func(t *testing.T) { + t.Logf(expr.input) + output, err := Eval(expr.input, func(s string) string { + return expr.params[s] + }) + if err != nil { + t.Errorf("Want %q expanded but got error %q", expr.input, err) + } + + if output != expr.output { + t.Errorf("Want %q expanded to %q, got %q", + expr.input, + expr.output, + output) + } + }) + } +} diff --git a/envsubst/funcs.go b/envsubst/funcs.go new file mode 100644 index 00000000..a9b6f329 --- /dev/null +++ b/envsubst/funcs.go @@ -0,0 +1,266 @@ +/* +Copyright 2024 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Forked from: https://github.com/drone/envsubst +// MIT License +// Copyright (c) 2017 drone.io. + +package envsubst + +import ( + "strconv" + "strings" + "unicode" + "unicode/utf8" + + "github.com/fluxcd/pkg/envsubst/path" +) + +// defines a parameter substitution function. +type substituteFunc func(string, ...string) string + +// toLen returns the length of string s. +func toLen(s string, args ...string) string { + return strconv.Itoa(len(s)) +} + +// toLower returns a copy of the string s with all characters +// mapped to their lower case. +func toLower(s string, args ...string) string { + return strings.ToLower(s) +} + +// toUpper returns a copy of the string s with all characters +// mapped to their upper case. +func toUpper(s string, args ...string) string { + return strings.ToUpper(s) +} + +// toLowerFirst returns a copy of the string s with the first +// character mapped to its lower case. +func toLowerFirst(s string, args ...string) string { + if s == "" { + return s + } + r, n := utf8.DecodeRuneInString(s) + return string(unicode.ToLower(r)) + s[n:] +} + +// toUpperFirst returns a copy of the string s with the first +// character mapped to its upper case. +func toUpperFirst(s string, args ...string) string { + if s == "" { + return s + } + r, n := utf8.DecodeRuneInString(s) + return string(unicode.ToUpper(r)) + s[n:] +} + +// toDefault returns a copy of the string s if not empty, else +// returns a concatenation of the args without a separator. +func toDefault(s string, args ...string) string { + if len(s) == 0 && len(args) > 0 { + // don't use any separator + s = strings.Join(args, "") + } + return s +} + +// toSubstr returns a slice of the string s at the specified +// length and position. +func toSubstr(s string, args ...string) string { + if len(args) == 0 { + return s // should never happen + } + + pos, err := strconv.Atoi(args[0]) + if err != nil { + // bash returns the string if the position + // cannot be parsed. + return s + } + + if pos < 0 { + // if pos is negative (counts from the end) add it + // to length to get first character offset + pos = len(s) + pos + + // if negative offset exceeds the length of the string + // start from 0 + if pos < 0 { + pos = 0 + } + } + + if len(args) == 1 { + if pos < len(s) { + return s[pos:] + } + // if the position exceeds the length of the + // string an empty string is returned + return "" + } + + length, err := strconv.Atoi(args[1]) + if err != nil { + // bash returns the string if the length + // cannot be parsed. + return s + } + + if pos+length >= len(s) { + if pos < len(s) { + // if the position exceeds the length of the + // string just return the rest of it like bash + return s[pos:] + } + // if the position exceeds the length of the + // string an empty string is returned + return "" + } + + return s[pos : pos+length] +} + +// replaceAll returns a copy of the string s with all instances +// of the substring replaced with the replacement string. +func replaceAll(s string, args ...string) string { + switch len(args) { + case 0: + return s + case 1: + return strings.Replace(s, args[0], "", -1) + default: + return strings.Replace(s, args[0], args[1], -1) + } +} + +// replaceFirst returns a copy of the string s with the first +// instance of the substring replaced with the replacement string. +func replaceFirst(s string, args ...string) string { + switch len(args) { + case 0: + return s + case 1: + return strings.Replace(s, args[0], "", 1) + default: + return strings.Replace(s, args[0], args[1], 1) + } +} + +// replacePrefix returns a copy of the string s with the matching +// prefix replaced with the replacement string. +func replacePrefix(s string, args ...string) string { + if len(args) != 2 { + return s + } + if strings.HasPrefix(s, args[0]) { + return strings.Replace(s, args[0], args[1], 1) + } + return s +} + +// replaceSuffix returns a copy of the string s with the matching +// suffix replaced with the replacement string. +func replaceSuffix(s string, args ...string) string { + if len(args) != 2 { + return s + } + if strings.HasSuffix(s, args[0]) { + s = strings.TrimSuffix(s, args[0]) + s = s + args[1] + } + return s +} + +// TODO + +func trimShortestPrefix(s string, args ...string) string { + if len(args) != 0 { + s = trimShortest(s, args[0]) + } + return s +} + +func trimShortestSuffix(s string, args ...string) string { + if len(args) != 0 { + r := reverse(s) + rarg := reverse(args[0]) + s = reverse(trimShortest(r, rarg)) + } + return s +} + +func trimLongestPrefix(s string, args ...string) string { + if len(args) != 0 { + s = trimLongest(s, args[0]) + } + return s +} + +func trimLongestSuffix(s string, args ...string) string { + if len(args) != 0 { + r := reverse(s) + rarg := reverse(args[0]) + s = reverse(trimLongest(r, rarg)) + } + return s +} + +func trimShortest(s, arg string) string { + var shortestMatch string + for i := 0; i < len(s); i++ { + match, err := path.Match(arg, s[0:len(s)-i]) + + if err != nil { + return s + } + + if match { + shortestMatch = s[0 : len(s)-i] + } + } + + if shortestMatch != "" { + return strings.TrimPrefix(s, shortestMatch) + } + + return s +} + +func trimLongest(s, arg string) string { + for i := 0; i < len(s); i++ { + match, err := path.Match(arg, s[0:len(s)-i]) + + if err != nil { + return s + } + + if match { + return strings.TrimPrefix(s, s[0:len(s)-i]) + } + } + + return s +} + +func reverse(s string) string { + r := []rune(s) + for i, j := 0, len(r)-1; i < len(r)/2; i, j = i+1, j-1 { + r[i], r[j] = r[j], r[i] + } + return string(r) +} diff --git a/envsubst/funcs_test.go b/envsubst/funcs_test.go new file mode 100644 index 00000000..66b05856 --- /dev/null +++ b/envsubst/funcs_test.go @@ -0,0 +1,124 @@ +/* +Copyright 2024 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Forked from: https://github.com/drone/envsubst +// MIT License +// Copyright (c) 2017 drone.io. + +package envsubst + +import "testing" + +func Test_len(t *testing.T) { + got, want := toLen("Hello World"), "11" + if got != want { + t.Errorf("Expect len function to return %s, got %s", want, got) + } +} + +func Test_lower(t *testing.T) { + got, want := toLower("Hello World"), "hello world" + if got != want { + t.Errorf("Expect lower function to return %s, got %s", want, got) + } +} + +func Test_lowerFirst(t *testing.T) { + got, want := toLowerFirst("HELLO WORLD"), "hELLO WORLD" + if got != want { + t.Errorf("Expect lowerFirst function to return %s, got %s", want, got) + } + defer func() { + if recover() != nil { + t.Errorf("Expect empty string does not panic lowerFirst") + } + }() + toLowerFirst("") +} + +func Test_upper(t *testing.T) { + got, want := toUpper("Hello World"), "HELLO WORLD" + if got != want { + t.Errorf("Expect upper function to return %s, got %s", want, got) + } +} + +func Test_upperFirst(t *testing.T) { + got, want := toUpperFirst("hello world"), "Hello world" + if got != want { + t.Errorf("Expect upperFirst function to return %s, got %s", want, got) + } + defer func() { + if recover() != nil { + t.Errorf("Expect empty string does not panic upperFirst") + } + }() + toUpperFirst("") +} + +func Test_default(t *testing.T) { + got, want := toDefault("Hello World", "Hola Mundo"), "Hello World" + if got != want { + t.Errorf("Expect default function uses variable value") + } + + got, want = toDefault("", "Hola Mundo"), "Hola Mundo" + if got != want { + t.Errorf("Expect default function uses default value, when variable empty. Got %s, Want %s", got, want) + } + + got, want = toDefault("", "Hola Mundo", "-Bonjour le monde", "-Halló heimur"), "Hola Mundo-Bonjour le monde-Halló heimur" + if got != want { + t.Errorf("Expect default function to use concatenated args when variable empty. Got %s, Want %s", got, want) + } +} + +func Test_substr(t *testing.T) { + got, want := toSubstr("123456789123456789", "0", "8"), "12345678" + if got != want { + t.Errorf("Expect substr function to cut from beginning to length") + } + + got, want = toSubstr("123456789123456789", "1", "8"), "23456789" + if got != want { + t.Errorf("Expect substr function to cut from offset to length") + } + + got, want = toSubstr("123456789123456789", "9"), "123456789" + if got != want { + t.Errorf("Expect substr function to cut beginnging with offset") + } + + got, want = toSubstr("123456789123456789", "9", "50"), "123456789" + if got != want { + t.Errorf("Expect substr function to ignore length if out of bound") + } + + got, want = toSubstr("123456789123456789", "-3", "2"), "78" + if got != want { + t.Errorf("Expect substr function to count negative offsets from the end") + } + + got, want = toSubstr("123456789123456789", "-300", "3"), "123" + if got != want { + t.Errorf("Expect substr function to cut from the beginning to length for negative offsets exceeding string length") + } + + got, want = toSubstr("12345678", "9", "1"), "" + if got != want { + t.Errorf("Expect substr function to cut entire string if pos is itself out of bound") + } +} diff --git a/envsubst/go.mod b/envsubst/go.mod new file mode 100644 index 00000000..99f1b323 --- /dev/null +++ b/envsubst/go.mod @@ -0,0 +1,5 @@ +module github.com/fluxcd/pkg/envsubst + +go 1.22 + +require github.com/google/go-cmp v0.6.0 diff --git a/envsubst/go.sum b/envsubst/go.sum new file mode 100644 index 00000000..5a8d551d --- /dev/null +++ b/envsubst/go.sum @@ -0,0 +1,2 @@ +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= diff --git a/envsubst/parse/node.go b/envsubst/parse/node.go new file mode 100644 index 00000000..12198345 --- /dev/null +++ b/envsubst/parse/node.go @@ -0,0 +1,106 @@ +/* +Copyright 2024 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Forked from: https://github.com/drone/envsubst +// MIT License +// Copyright (c) 2017 drone.io. + +package parse + +// Node is an element in the parse tree. +type Node interface { + node() +} + +// empty string node +var empty = new(TextNode) + +// a template is represented by a tree consisting of one +// or more of the following nodes. +type ( + // TextNode represents a string of text. + TextNode struct { + Value string + } + + // FuncNode represents a string function. + FuncNode struct { + Param string + Name string + Args []Node + } + + // ListNode represents a list of nodes. + ListNode struct { + Nodes []Node + } + + // ParamNode struct{ + // Name string + // } + // + // CaseNode struct { + // Name string + // First bool + // } + // + // LowerNode struct { + // Name string + // First bool + // } + // + // SubstrNode struct { + // Name string + // Pos Node + // Len Node + // } + // + // ReplaceNode struct { + // Name string + // Substring Node + // Replacement Node + // } + // + // TrimNode struct{ + // + // } + // + // DefaultNode struct { + // Name string + // Default Node + // } +) + +// newTextNode returns a new TextNode. +func newTextNode(text string) *TextNode { + return &TextNode{Value: text} +} + +// newListNode returns a new ListNode. +func newListNode(nodes ...Node) *ListNode { + return &ListNode{Nodes: nodes} +} + +// newFuncNode returns a new FuncNode. +func newFuncNode(name string) *FuncNode { + return &FuncNode{Param: name} +} + +// node() defines the node in a parse tree + +func (*TextNode) node() {} +func (*ListNode) node() {} +func (*FuncNode) node() {} diff --git a/envsubst/parse/parse.go b/envsubst/parse/parse.go new file mode 100644 index 00000000..39c72009 --- /dev/null +++ b/envsubst/parse/parse.go @@ -0,0 +1,422 @@ +/* +Copyright 2024 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Forked from: https://github.com/drone/envsubst +// MIT License +// Copyright (c) 2017 drone.io. + +package parse + +import ( + "errors" +) + +var ( + // ErrBadSubstitution represents a substitution parsing error. + ErrBadSubstitution = errors.New("bad substitution") + + // ErrMissingClosingBrace represents a missing closing brace "}" error. + ErrMissingClosingBrace = errors.New("missing closing brace") + + // ErrParseVariableName represents the error when unable to parse a + // variable name within a substitution. + ErrParseVariableName = errors.New("unable to parse variable name") + + // ErrParseFuncSubstitution represents the error when unable to parse the + // substitution within a function parameter. + ErrParseFuncSubstitution = errors.New("unable to parse substitution within function") + + // ErrParseDefaultFunction represent the error when unable to parse a + // default function. + ErrParseDefaultFunction = errors.New("unable to parse default function") +) + +// Tree is the representation of a single parsed SQL statement. +type Tree struct { + Root Node + + // Parsing only; cleared after parse. + scanner *scanner +} + +// Parse parses the string and returns a Tree. +func Parse(buf string) (*Tree, error) { + t := new(Tree) + t.scanner = new(scanner) + return t.Parse(buf) +} + +// Parse parses the string buffer to construct an ast +// representation for expansion. +func (t *Tree) Parse(buf string) (tree *Tree, err error) { + t.scanner.init(buf) + t.Root, err = t.parseAny() + return t, err +} + +func (t *Tree) parseAny() (Node, error) { + t.scanner.accept = acceptRune + t.scanner.mode = scanIdent | scanLbrack | scanEscape + t.scanner.escapeChars = dollar + + switch t.scanner.scan() { + case tokenIdent: + left := newTextNode( + t.scanner.string(), + ) + right, err := t.parseAny() + switch { + case err != nil: + return nil, err + case right == empty: + return left, nil + } + return newListNode(left, right), nil + case tokenEOF: + return empty, nil + case tokenLbrack: + left, err := t.parseFunc() + if err != nil { + return nil, err + } + + right, err := t.parseAny() + switch { + case err != nil: + return nil, err + case right == empty: + return left, nil + } + return newListNode(left, right), nil + } + + return nil, ErrBadSubstitution +} + +func (t *Tree) parseFunc() (Node, error) { + // Turn on all escape characters + t.scanner.escapeChars = escapeAll + switch t.scanner.peek() { + case '#': + return t.parseLenFunc() + } + + var name string + t.scanner.accept = acceptIdent + t.scanner.mode = scanIdent + + switch t.scanner.scan() { + case tokenIdent: + name = t.scanner.string() + default: + return nil, ErrParseVariableName + } + + switch t.scanner.peek() { + case ':': + return t.parseDefaultOrSubstr(name) + case '=': + return t.parseDefaultFunc(name) + case ',', '^': + return t.parseCasingFunc(name) + case '/': + return t.parseReplaceFunc(name) + case '#': + return t.parseRemoveFunc(name, acceptHashFunc) + case '%': + return t.parseRemoveFunc(name, acceptPercentFunc) + } + + t.scanner.accept = acceptIdent + t.scanner.mode = scanRbrack + switch t.scanner.scan() { + case tokenRbrack: + return newFuncNode(name), nil + default: + return nil, ErrMissingClosingBrace + } +} + +// parse a substitution function parameter. +func (t *Tree) parseParam(accept acceptFunc, mode byte) (Node, error) { + t.scanner.accept = accept + t.scanner.mode = mode | scanLbrack + switch t.scanner.scan() { + case tokenLbrack: + return t.parseFunc() + case tokenIdent: + return newTextNode( + t.scanner.string(), + ), nil + case tokenRbrack: + return newTextNode( + t.scanner.string(), + ), nil + default: + return nil, ErrParseFuncSubstitution + } +} + +// parse either a default or substring substitution function. +func (t *Tree) parseDefaultOrSubstr(name string) (Node, error) { + t.scanner.read() + r := t.scanner.peek() + t.scanner.unread() + switch r { + case '=', '-', '?', '+': + return t.parseDefaultFunc(name) + default: + return t.parseSubstrFunc(name) + } +} + +// parses the ${param:offset} string function +// parses the ${param:offset:length} string function +func (t *Tree) parseSubstrFunc(name string) (Node, error) { + node := new(FuncNode) + node.Param = name + + t.scanner.accept = acceptOneColon + t.scanner.mode = scanIdent + switch t.scanner.scan() { + case tokenIdent: + node.Name = t.scanner.string() + default: + return nil, ErrBadSubstitution + } + + // scan arg[1] + { + param, err := t.parseParam(rejectColonClose, scanIdent) + if err != nil { + return nil, err + } + + // param.Value = t.scanner.string() + node.Args = append(node.Args, param) + } + + // expect delimiter or close + t.scanner.accept = acceptColon + t.scanner.mode = scanIdent | scanRbrack + switch t.scanner.scan() { + case tokenRbrack: + return node, nil + case tokenIdent: + // no-op + default: + return nil, ErrBadSubstitution + } + + // scan arg[2] + { + param, err := t.parseParam(acceptNotClosing, scanIdent) + if err != nil { + return nil, err + } + node.Args = append(node.Args, param) + } + + return node, t.consumeRbrack() +} + +// parses the ${param%word} string function +// parses the ${param%%word} string function +// parses the ${param#word} string function +// parses the ${param##word} string function +func (t *Tree) parseRemoveFunc(name string, accept acceptFunc) (Node, error) { + node := new(FuncNode) + node.Param = name + + t.scanner.accept = accept + t.scanner.mode = scanIdent + switch t.scanner.scan() { + case tokenIdent: + node.Name = t.scanner.string() + default: + return nil, ErrBadSubstitution + } + + // scan arg[1] + { + param, err := t.parseParam(acceptNotClosing, scanIdent) + if err != nil { + return nil, err + } + + // param.Value = t.scanner.string() + node.Args = append(node.Args, param) + } + + return node, t.consumeRbrack() +} + +// parses the ${param/pattern/string} string function +// parses the ${param//pattern/string} string function +// parses the ${param/#pattern/string} string function +// parses the ${param/%pattern/string} string function +func (t *Tree) parseReplaceFunc(name string) (Node, error) { + node := new(FuncNode) + node.Param = name + + t.scanner.accept = acceptReplaceFunc + t.scanner.mode = scanIdent + switch t.scanner.scan() { + case tokenIdent: + node.Name = t.scanner.string() + default: + return nil, ErrBadSubstitution + } + + // scan arg[1] + { + param, err := t.parseParam(acceptNotSlash, scanIdent|scanEscape) + if err != nil { + return nil, err + } + node.Args = append(node.Args, param) + } + + // expect delimiter + t.scanner.accept = acceptSlash + t.scanner.mode = scanIdent + switch t.scanner.scan() { + case tokenIdent: + // no-op + default: + return nil, ErrBadSubstitution + } + + // check for blank string + switch t.scanner.peek() { + case '}': + return node, t.consumeRbrack() + } + + // scan arg[2] + { + param, err := t.parseParam(acceptNotClosing, scanIdent|scanEscape) + if err != nil { + return nil, err + } + node.Args = append(node.Args, param) + } + + return node, t.consumeRbrack() +} + +// parses the ${parameter=word} string function +// parses the ${parameter:=word} string function +// parses the ${parameter:-word} string function +// parses the ${parameter:?word} string function +// parses the ${parameter:+word} string function +func (t *Tree) parseDefaultFunc(name string) (Node, error) { + node := new(FuncNode) + node.Param = name + + t.scanner.accept = acceptDefaultFunc + if t.scanner.peek() == '=' { + t.scanner.accept = acceptOneEqual + } + t.scanner.mode = scanIdent + switch t.scanner.scan() { + case tokenIdent: + node.Name = t.scanner.string() + default: + return nil, ErrParseDefaultFunction + } + + // loop through all possible runes in default param + for { + // this acts as the break condition. Peek to see if we reached the end + switch t.scanner.peek() { + case '}': + return node, t.consumeRbrack() + } + param, err := t.parseParam(acceptNotClosing, scanIdent) + if err != nil { + return nil, err + } + + node.Args = append(node.Args, param) + } +} + +// parses the ${param,} string function +// parses the ${param,,} string function +// parses the ${param^} string function +// parses the ${param^^} string function +func (t *Tree) parseCasingFunc(name string) (Node, error) { + node := new(FuncNode) + node.Param = name + + t.scanner.accept = acceptCasingFunc + t.scanner.mode = scanIdent + switch t.scanner.scan() { + case tokenIdent: + node.Name = t.scanner.string() + default: + return nil, ErrBadSubstitution + } + + return node, t.consumeRbrack() +} + +// parses the ${#param} string function +func (t *Tree) parseLenFunc() (Node, error) { + node := new(FuncNode) + + t.scanner.accept = acceptOneHash + t.scanner.mode = scanIdent + switch t.scanner.scan() { + case tokenIdent: + node.Name = t.scanner.string() + default: + return nil, ErrBadSubstitution + } + + t.scanner.accept = acceptIdent + t.scanner.mode = scanIdent + switch t.scanner.scan() { + case tokenIdent: + node.Param = t.scanner.string() + default: + return nil, ErrBadSubstitution + } + + return node, t.consumeRbrack() +} + +// consumeRbrack consumes a right closing bracket. If a closing +// bracket token is not consumed an ErrBadSubstitution is returned. +func (t *Tree) consumeRbrack() error { + t.scanner.mode = scanRbrack + if t.scanner.scan() != tokenRbrack { + return ErrBadSubstitution + } + return nil +} + +// consumeDelimiter consumes a function argument delimiter. If a +// delimiter is not consumed an ErrBadSubstitution is returned. +// func (t *Tree) consumeDelimiter(accept acceptFunc, mode uint) error { +// t.scanner.accept = accept +// t.scanner.mode = mode +// if t.scanner.scan() != tokenRbrack { +// return ErrBadSubstitution +// } +// return nil +// } diff --git a/envsubst/parse/parse_test.go b/envsubst/parse/parse_test.go new file mode 100644 index 00000000..cd243796 --- /dev/null +++ b/envsubst/parse/parse_test.go @@ -0,0 +1,551 @@ +/* +Copyright 2024 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Forked from: https://github.com/drone/envsubst +// MIT License +// Copyright (c) 2017 drone.io. + +package parse + +import ( + "testing" + + "github.com/google/go-cmp/cmp" +) + +var tests = []struct { + Text string + Node Node +}{ + + // + // text only + // + { + Text: "text", + Node: &TextNode{Value: "text"}, + }, + { + Text: "}text", + Node: &TextNode{Value: "}text"}, + }, + { + Text: "http://github.com", + Node: &TextNode{Value: "http://github.com"}, // should not escape double slash + }, + { + Text: "$${string}", + Node: &TextNode{Value: "${string}"}, // should not escape double dollar + }, + { + Text: "$$string", + Node: &TextNode{Value: "$string"}, // should not escape double dollar + }, + { + Text: `\\.\pipe\pipename`, + Node: &TextNode{Value: `\\.\pipe\pipename`}, + }, + + // + // variable only + // + { + Text: "${string}", + Node: &FuncNode{Param: "string"}, + }, + + // + // text transform functions + // + { + Text: "${string,}", + Node: &FuncNode{ + Param: "string", + Name: ",", + Args: nil, + }, + }, + { + Text: "${string,,}", + Node: &FuncNode{ + Param: "string", + Name: ",,", + Args: nil, + }, + }, + { + Text: "${string^}", + Node: &FuncNode{ + Param: "string", + Name: "^", + Args: nil, + }, + }, + { + Text: "${string^^}", + Node: &FuncNode{ + Param: "string", + Name: "^^", + Args: nil, + }, + }, + + // + // substring functions + // + { + Text: "${string:position}", + Node: &FuncNode{ + Param: "string", + Name: ":", + Args: []Node{ + &TextNode{Value: "position"}, + }, + }, + }, + { + Text: "${string:position:length}", + Node: &FuncNode{ + Param: "string", + Name: ":", + Args: []Node{ + &TextNode{Value: "position"}, + &TextNode{Value: "length"}, + }, + }, + }, + + // + // string removal functions + // + { + Text: "${string#substring}", + Node: &FuncNode{ + Param: "string", + Name: "#", + Args: []Node{ + &TextNode{Value: "substring"}, + }, + }, + }, + { + Text: "${string##substring}", + Node: &FuncNode{ + Param: "string", + Name: "##", + Args: []Node{ + &TextNode{Value: "substring"}, + }, + }, + }, + { + Text: "${string%substring}", + Node: &FuncNode{ + Param: "string", + Name: "%", + Args: []Node{ + &TextNode{Value: "substring"}, + }, + }, + }, + { + Text: "${string%%substring}", + Node: &FuncNode{ + Param: "string", + Name: "%%", + Args: []Node{ + &TextNode{Value: "substring"}, + }, + }, + }, + + // + // string replace functions + // + { + Text: "${string/substring/replacement}", + Node: &FuncNode{ + Param: "string", + Name: "/", + Args: []Node{ + &TextNode{Value: "substring"}, + &TextNode{Value: "replacement"}, + }, + }, + }, + { + Text: "${string//substring/replacement}", + Node: &FuncNode{ + Param: "string", + Name: "//", + Args: []Node{ + &TextNode{Value: "substring"}, + &TextNode{Value: "replacement"}, + }, + }, + }, + { + Text: "${string/#substring/replacement}", + Node: &FuncNode{ + Param: "string", + Name: "/#", + Args: []Node{ + &TextNode{Value: "substring"}, + &TextNode{Value: "replacement"}, + }, + }, + }, + { + Text: "${string/%substring/replacement}", + Node: &FuncNode{ + Param: "string", + Name: "/%", + Args: []Node{ + &TextNode{Value: "substring"}, + &TextNode{Value: "replacement"}, + }, + }, + }, + + // + // default value functions + // + { + Text: "${string=default}", + Node: &FuncNode{ + Param: "string", + Name: "=", + Args: []Node{ + &TextNode{Value: "default"}, + }, + }, + }, + { + Text: "${string:=default}", + Node: &FuncNode{ + Param: "string", + Name: ":=", + Args: []Node{ + &TextNode{Value: "default"}, + }, + }, + }, + { + Text: "${string:-default}", + Node: &FuncNode{ + Param: "string", + Name: ":-", + Args: []Node{ + &TextNode{Value: "default"}, + }, + }, + }, + { + Text: "${string:?default}", + Node: &FuncNode{ + Param: "string", + Name: ":?", + Args: []Node{ + &TextNode{Value: "default"}, + }, + }, + }, + { + Text: "${string:+default}", + Node: &FuncNode{ + Param: "string", + Name: ":+", + Args: []Node{ + &TextNode{Value: "default"}, + }, + }, + }, + + // + // length function + // + { + Text: "${#string}", + Node: &FuncNode{ + Param: "string", + Name: "#", + }, + }, + + // + // special characters in argument + // + { + Text: "${string#$%:*{}", + Node: &FuncNode{ + Param: "string", + Name: "#", + Args: []Node{ + &TextNode{Value: "$%:*{"}, + }, + }, + }, + + // text before and after function + { + Text: "hello ${#string} world", + Node: &ListNode{ + Nodes: []Node{ + &TextNode{ + Value: "hello ", + }, + &ListNode{ + Nodes: []Node{ + &FuncNode{ + Param: "string", + Name: "#", + }, + &TextNode{ + Value: " world", + }, + }, + }, + }, + }, + }, + // text before and after function with \\ outside of function + { + Text: `\\ hello ${#string} world \\`, + Node: &ListNode{ + Nodes: []Node{ + &TextNode{ + Value: `\\ hello `, + }, + &ListNode{ + Nodes: []Node{ + &FuncNode{ + Param: "string", + Name: "#", + }, + &TextNode{ + Value: ` world \\`, + }, + }, + }, + }, + }, + }, + + // escaped function arguments + { + Text: `${string/\/position/length}`, + Node: &FuncNode{ + Param: "string", + Name: "/", + Args: []Node{ + &TextNode{ + Value: "/position", + }, + &TextNode{ + Value: "length", + }, + }, + }, + }, + { + Text: `${string/\/position\\/length}`, + Node: &FuncNode{ + Param: "string", + Name: "/", + Args: []Node{ + &TextNode{ + Value: "/position\\", + }, + &TextNode{ + Value: "length", + }, + }, + }, + }, + { + Text: `${string/position/\/length}`, + Node: &FuncNode{ + Param: "string", + Name: "/", + Args: []Node{ + &TextNode{ + Value: "position", + }, + &TextNode{ + Value: "/length", + }, + }, + }, + }, + { + Text: `${string/position/\/length\\}`, + Node: &FuncNode{ + Param: "string", + Name: "/", + Args: []Node{ + &TextNode{ + Value: "position", + }, + &TextNode{ + Value: "/length\\", + }, + }, + }, + }, + { + Text: `${string/position/\/leng\\th}`, + Node: &FuncNode{ + Param: "string", + Name: "/", + Args: []Node{ + &TextNode{ + Value: "position", + }, + &TextNode{ + Value: "/leng\\th", + }, + }, + }, + }, + + // functions in functions + { + Text: "${string:${position}}", + Node: &FuncNode{ + Param: "string", + Name: ":", + Args: []Node{ + &FuncNode{ + Param: "position", + }, + }, + }, + }, + { + Text: "${string:${stringy:position:length}:${stringz,,}}", + Node: &FuncNode{ + Param: "string", + Name: ":", + Args: []Node{ + &FuncNode{ + Param: "stringy", + Name: ":", + Args: []Node{ + &TextNode{Value: "position"}, + &TextNode{Value: "length"}, + }, + }, + &FuncNode{ + Param: "stringz", + Name: ",,", + }, + }, + }, + }, + { + Text: "${string#${stringz}}", + Node: &FuncNode{ + Param: "string", + Name: "#", + Args: []Node{ + &FuncNode{Param: "stringz"}, + }, + }, + }, + { + Text: "${string=${stringz}}", + Node: &FuncNode{ + Param: "string", + Name: "=", + Args: []Node{ + &FuncNode{Param: "stringz"}, + }, + }, + }, + { + Text: "${string=prefix-${var}}", + Node: &FuncNode{ + Param: "string", + Name: "=", + Args: []Node{ + &TextNode{Value: "prefix-"}, + &FuncNode{Param: "var"}, + }, + }, + }, + { + Text: "${string=${var}-suffix}", + Node: &FuncNode{ + Param: "string", + Name: "=", + Args: []Node{ + &FuncNode{Param: "var"}, + &TextNode{Value: "-suffix"}, + }, + }, + }, + { + Text: "${string=prefix-${var}-suffix}", + Node: &FuncNode{ + Param: "string", + Name: "=", + Args: []Node{ + &TextNode{Value: "prefix-"}, + &FuncNode{Param: "var"}, + &TextNode{Value: "-suffix"}, + }, + }, + }, + { + Text: "${string=prefix${var} suffix}", + Node: &FuncNode{ + Param: "string", + Name: "=", + Args: []Node{ + &TextNode{Value: "prefix"}, + &FuncNode{Param: "var"}, + &TextNode{Value: " suffix"}, + }, + }, + }, + { + Text: "${string//${stringy}/${stringz}}", + Node: &FuncNode{ + Param: "string", + Name: "//", + Args: []Node{ + &FuncNode{Param: "stringy"}, + &FuncNode{Param: "stringz"}, + }, + }, + }, +} + +func TestParse(t *testing.T) { + for _, test := range tests { + t.Log(test.Text) + t.Run(test.Text, func(t *testing.T) { + got, err := Parse(test.Text) + if err != nil { + t.Error(err) + } + + if diff := cmp.Diff(test.Node, got.Root); diff != "" { + t.Errorf(diff) + } + }) + } +} diff --git a/envsubst/parse/scan.go b/envsubst/parse/scan.go new file mode 100644 index 00000000..4f047428 --- /dev/null +++ b/envsubst/parse/scan.go @@ -0,0 +1,314 @@ +/* +Copyright 2024 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Forked from: https://github.com/drone/envsubst +// MIT License +// Copyright (c) 2017 drone.io. + +package parse + +import ( + "unicode" + "unicode/utf8" +) + +// eof rune sent when end of file is reached +var eof = rune(0) + +// token is a lexical token. +type token uint + +// list of lexical tokens. +const ( + // special tokens + tokenIllegal token = iota + tokenEOF + + // identifiers and literals + tokenIdent + + // operators and delimiters + tokenLbrack + tokenRbrack + tokenQuote +) + +// predefined mode bits to control recognition of tokens. +const ( + scanIdent byte = 1 << iota + scanLbrack + scanRbrack + scanEscape +) + +// predefined mode bits to control escape tokens. +const ( + dollar byte = 1 << iota + backslash + escapeAll = dollar | backslash +) + +// returns true if rune is accepted. +type acceptFunc func(r rune, i int) bool + +// scanner implements a lexical scanner that reads unicode +// characters and tokens from a string buffer. +type scanner struct { + buf string + pos int + start int + width int + mode byte + escapeChars byte + + accept acceptFunc +} + +// init initializes a scanner with a new buffer. +func (s *scanner) init(buf string) { + s.buf = buf + s.pos = 0 + s.start = 0 + s.width = 0 + s.accept = nil +} + +// read returns the next unicode character. It returns eof at +// the end of the string buffer. +func (s *scanner) read() rune { + if s.pos >= len(s.buf) { + s.width = 0 + return eof + } + r, w := utf8.DecodeRuneInString(s.buf[s.pos:]) + s.width = w + s.pos += s.width + return r +} + +func (s *scanner) unread() { + s.pos -= s.width +} + +// skip skips over the curring unicode character in the buffer +// by slicing and removing from the buffer. +func (s *scanner) skip() { + l := s.buf[:s.pos-1] + r := s.buf[s.pos:] + s.buf = l + r +} + +// peek returns the next unicode character in the buffer without +// advancing the scanner. It returns eof if the scanner's position +// is at the last character of the source. +func (s *scanner) peek() rune { + r := s.read() + s.unread() + return r +} + +// string returns the string corresponding to the most recently +// scanned token. Valid after calling scan(). +func (s *scanner) string() string { + return s.buf[s.start:s.pos] +} + +// tests if the bit exists for a given character bit +func (s *scanner) shouldEscape(character byte) bool { + return s.escapeChars&character != 0 +} + +// scan reads the next token or Unicode character from source and +// returns it. It returns EOF at the end of the source. +func (s *scanner) scan() token { + s.start = s.pos + r := s.read() + switch { + case r == eof: + return tokenEOF + case s.scanLbrack(r): + return tokenLbrack + case s.scanRbrack(r): + return tokenRbrack + case s.scanIdent(r): + return tokenIdent + } + return tokenIllegal +} + +// scanIdent reads the next token or Unicode character from source +// and returns true if the Ident character is accepted. +func (s *scanner) scanIdent(r rune) bool { + if s.mode&scanIdent == 0 { + return false + } + if s.scanEscaped(r) { + s.skip() + } else if !s.accept(r, s.pos-s.start) { + return false + } +loop: + for { + r := s.read() + switch { + case r == eof: + s.unread() + break loop + case s.scanLbrack(r): + s.unread() + s.unread() + break loop + } + if s.scanEscaped(r) { + s.skip() + continue + } + if !s.accept(r, s.pos-s.start) { + s.unread() + break loop + } + } + return true +} + +// scanLbrack reads the next token or Unicode character from source +// and returns true if the open bracket is encountered. +func (s *scanner) scanLbrack(r rune) bool { + if s.mode&scanLbrack == 0 { + return false + } + if r == '$' { + if s.read() == '{' { + return true + } + s.unread() + } + return false +} + +// scanRbrack reads the next token or Unicode character from source +// and returns true if the closing bracket is encountered. +func (s *scanner) scanRbrack(r rune) bool { + if s.mode&scanRbrack == 0 { + return false + } + return r == '}' +} + +// scanEscaped reads the next token or Unicode character from source +// and returns true if it being escaped and should be skipped. +func (s *scanner) scanEscaped(r rune) bool { + if s.mode&scanEscape == 0 { + return false + } + if r == '$' && s.shouldEscape(dollar) { + if s.peek() == '$' { + return true + } + } + if r == '\\' && s.shouldEscape(backslash) { + switch s.peek() { + case '/', '\\': + return true + default: + return false + } + } + + return false +} + +// +// scanner functions accept or reject runes. +// + +func acceptRune(r rune, i int) bool { + return true +} + +func acceptIdent(r rune, i int) bool { + return unicode.IsLetter(r) || unicode.IsDigit(r) || r == '_' +} + +func acceptColon(r rune, i int) bool { + return r == ':' +} + +func acceptOneHash(r rune, i int) bool { + return r == '#' && i == 1 +} + +func acceptNone(r rune, i int) bool { + return false +} + +func acceptNotClosing(r rune, i int) bool { + return r != '}' +} + +func acceptHashFunc(r rune, i int) bool { + return r == '#' && i < 3 +} + +func acceptPercentFunc(r rune, i int) bool { + return r == '%' && i < 3 +} + +func acceptDefaultFunc(r rune, i int) bool { + switch { + case i == 1 && r == ':': + return true + case i == 2 && (r == '=' || r == '-' || r == '?' || r == '+'): + return true + default: + return false + } +} + +func acceptReplaceFunc(r rune, i int) bool { + switch { + case i == 1 && r == '/': + return true + case i == 2 && (r == '/' || r == '#' || r == '%'): + return true + default: + return false + } +} + +func acceptOneEqual(r rune, i int) bool { + return i == 1 && r == '=' +} + +func acceptOneColon(r rune, i int) bool { + return i == 1 && r == ':' +} + +func rejectColonClose(r rune, i int) bool { + return r != ':' && r != '}' +} + +func acceptSlash(r rune, i int) bool { + return r == '/' +} + +func acceptNotSlash(r rune, i int) bool { + return r != '/' +} + +func acceptCasingFunc(r rune, i int) bool { + return (r == ',' || r == '^') && i < 3 +} diff --git a/envsubst/path/match.go b/envsubst/path/match.go new file mode 100644 index 00000000..2b3dfd68 --- /dev/null +++ b/envsubst/path/match.go @@ -0,0 +1,206 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package path + +import ( + "errors" + "unicode/utf8" +) + +// ErrBadPattern indicates a globbing pattern was malformed. +var ErrBadPattern = errors.New("syntax error in pattern") + +// Match reports whether name matches the shell file name pattern. +// The pattern syntax is: +// +// pattern: +// { term } +// term: +// '*' matches any sequence of non-/ characters +// '?' matches any single non-/ character +// '[' [ '^' ] { character-range } ']' +// character class (must be non-empty) +// c matches character c (c != '*', '?', '\\', '[') +// '\\' c matches character c +// +// character-range: +// c matches character c (c != '\\', '-', ']') +// '\\' c matches character c +// lo '-' hi matches character c for lo <= c <= hi +// +// Match requires pattern to match all of name, not just a substring. +// The only possible returned error is ErrBadPattern, when pattern +// is malformed. +func Match(pattern, name string) (matched bool, err error) { +Pattern: + for len(pattern) > 0 { + var star bool + var chunk string + star, chunk, pattern = scanChunk(pattern) + if star && chunk == "" { + // Trailing * matches rest of string unless it has a /. + // return !strings.Contains(name, "/"), nil + + // Return rest of string + return true, nil + } + // Look for match at current position. + t, ok, err := matchChunk(chunk, name) + // if we're the last chunk, make sure we've exhausted the name + // otherwise we'll give a false result even if we could still match + // using the star + if ok && (len(t) == 0 || len(pattern) > 0) { + name = t + continue + } + if err != nil { + return false, err + } + if star { + // Look for match skipping i+1 bytes. + for i := 0; i < len(name); i++ { + t, ok, err := matchChunk(chunk, name[i+1:]) + if ok { + // if we're the last chunk, make sure we exhausted the name + if len(pattern) == 0 && len(t) > 0 { + continue + } + name = t + continue Pattern + } + if err != nil { + return false, err + } + } + } + return false, nil + } + return len(name) == 0, nil +} + +// scanChunk gets the next segment of pattern, which is a non-star string +// possibly preceded by a star. +func scanChunk(pattern string) (star bool, chunk, rest string) { + for len(pattern) > 0 && pattern[0] == '*' { + pattern = pattern[1:] + star = true + } + inrange := false + var i int +Scan: + for i = 0; i < len(pattern); i++ { + switch pattern[i] { + case '\\': + // error check handled in matchChunk: bad pattern. + if i+1 < len(pattern) { + i++ + } + case '[': + inrange = true + case ']': + inrange = false + case '*': + if !inrange { + break Scan + } + } + } + return star, pattern[0:i], pattern[i:] +} + +// matchChunk checks whether chunk matches the beginning of s. +// If so, it returns the remainder of s (after the match). +// Chunk is all single-character operators: literals, char classes, and ?. +func matchChunk(chunk, s string) (rest string, ok bool, err error) { + for len(chunk) > 0 { + if len(s) == 0 { + return + } + switch chunk[0] { + case '[': + // character class + r, n := utf8.DecodeRuneInString(s) + s = s[n:] + chunk = chunk[1:] + // possibly negated + notNegated := true + if len(chunk) > 0 && chunk[0] == '^' { + notNegated = false + chunk = chunk[1:] + } + // parse all ranges + match := false + nrange := 0 + for { + if len(chunk) > 0 && chunk[0] == ']' && nrange > 0 { + chunk = chunk[1:] + break + } + var lo, hi rune + if lo, chunk, err = getEsc(chunk); err != nil { + return + } + hi = lo + if chunk[0] == '-' { + if hi, chunk, err = getEsc(chunk[1:]); err != nil { + return + } + } + if lo <= r && r <= hi { + match = true + } + nrange++ + } + if match != notNegated { + return + } + + case '?': + _, n := utf8.DecodeRuneInString(s) + s = s[n:] + chunk = chunk[1:] + + case '\\': + chunk = chunk[1:] + if len(chunk) == 0 { + err = ErrBadPattern + return + } + fallthrough + + default: + if chunk[0] != s[0] { + return + } + s = s[1:] + chunk = chunk[1:] + } + } + return s, true, nil +} + +// getEsc gets a possibly-escaped character from chunk, for a character class. +func getEsc(chunk string) (r rune, nchunk string, err error) { + if len(chunk) == 0 || chunk[0] == '-' || chunk[0] == ']' { + err = ErrBadPattern + return + } + if chunk[0] == '\\' { + chunk = chunk[1:] + if len(chunk) == 0 { + err = ErrBadPattern + return + } + } + r, n := utf8.DecodeRuneInString(chunk) + if r == utf8.RuneError && n == 1 { + err = ErrBadPattern + } + nchunk = chunk[n:] + if len(nchunk) == 0 { + err = ErrBadPattern + } + return +} diff --git a/envsubst/template.go b/envsubst/template.go new file mode 100644 index 00000000..db8e7225 --- /dev/null +++ b/envsubst/template.go @@ -0,0 +1,177 @@ +/* +Copyright 2024 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Forked from: https://github.com/drone/envsubst +// MIT License +// Copyright (c) 2017 drone.io. + +package envsubst + +import ( + "bytes" + "io" + "io/ioutil" + + "github.com/fluxcd/pkg/envsubst/parse" +) + +// state represents the state of template execution. It is not part of the +// template so that multiple executions can run in parallel. +type state struct { + template *Template + writer io.Writer + node parse.Node // current node + + // maps variable names to values + mapper func(string) string +} + +// Template is the representation of a parsed shell format string. +type Template struct { + tree *parse.Tree +} + +// Parse creates a new shell format template and parses the template +// definition from string s. +func Parse(s string) (t *Template, err error) { + t = new(Template) + t.tree, err = parse.Parse(s) + if err != nil { + return nil, err + } + return t, nil +} + +// ParseFile creates a new shell format template and parses the template +// definition from the named file. +func ParseFile(path string) (*Template, error) { + b, err := ioutil.ReadFile(path) + if err != nil { + return nil, err + } + return Parse(string(b)) +} + +// Execute applies a parsed template to the specified data mapping. +func (t *Template) Execute(mapping func(string) string) (str string, err error) { + b := new(bytes.Buffer) + s := new(state) + s.node = t.tree.Root + s.mapper = mapping + s.writer = b + err = t.eval(s) + if err != nil { + return + } + return b.String(), nil +} + +func (t *Template) eval(s *state) (err error) { + switch node := s.node.(type) { + case *parse.TextNode: + err = t.evalText(s, node) + case *parse.FuncNode: + err = t.evalFunc(s, node) + case *parse.ListNode: + err = t.evalList(s, node) + } + return err +} + +func (t *Template) evalText(s *state, node *parse.TextNode) error { + _, err := io.WriteString(s.writer, node.Value) + return err +} + +func (t *Template) evalList(s *state, node *parse.ListNode) (err error) { + for _, n := range node.Nodes { + s.node = n + err = t.eval(s) + if err != nil { + return err + } + } + return nil +} + +func (t *Template) evalFunc(s *state, node *parse.FuncNode) error { + var w = s.writer + var buf bytes.Buffer + var args []string + for _, n := range node.Args { + buf.Reset() + s.writer = &buf + s.node = n + err := t.eval(s) + if err != nil { + return err + } + args = append(args, buf.String()) + } + + // restore the origin writer + s.writer = w + s.node = node + + v := s.mapper(node.Param) + + fn := lookupFunc(node.Name, len(args)) + + _, err := io.WriteString(s.writer, fn(v, args...)) + return err +} + +// lookupFunc returns the parameters substitution function by name. If the +// named function does not exists, a default function is returned. +func lookupFunc(name string, args int) substituteFunc { + switch name { + case ",": + return toLowerFirst + case ",,": + return toLower + case "^": + return toUpperFirst + case "^^": + return toUpper + case "#": + if args == 0 { + return toLen + } + return trimShortestPrefix + case "##": + return trimLongestPrefix + case "%": + return trimShortestSuffix + case "%%": + return trimLongestSuffix + case ":": + return toSubstr + case "/#": + return replacePrefix + case "/%": + return replaceSuffix + case "/": + return replaceFirst + case "//": + return replaceAll + case "=", ":=", ":-": + return toDefault + case ":?", ":+", "-", "+": + return toDefault + default: + return toDefault + } +} From c0b6634f1437395dd9e006eaf9d3f74d0d7e5359 Mon Sep 17 00:00:00 2001 From: Stefan Prodan Date: Mon, 8 Apr 2024 00:31:09 +0300 Subject: [PATCH 2/4] envsubst: Add strict mode Incorporate https://github.com/drone/envsubst/pull/34 Signed-off-by: Stefan Prodan --- envsubst/eval.go | 14 +++++++--- envsubst/eval_test.go | 62 ++++++++++++++++++++++++++++++++++++++++--- envsubst/template.go | 16 +++++++---- 3 files changed, 81 insertions(+), 11 deletions(-) diff --git a/envsubst/eval.go b/envsubst/eval.go index 5255e697..e31165b4 100644 --- a/envsubst/eval.go +++ b/envsubst/eval.go @@ -23,7 +23,7 @@ package envsubst import "os" // Eval replaces ${var} in the string based on the mapping function. -func Eval(s string, mapping func(string) string) (string, error) { +func Eval(s string, mapping func(string) (string, bool)) (string, error) { t, err := Parse(s) if err != nil { return s, err @@ -34,6 +34,14 @@ func Eval(s string, mapping func(string) string) (string, error) { // EvalEnv replaces ${var} in the string according to the values of the // current environment variables. References to undefined variables are // replaced by the empty string. -func EvalEnv(s string) (string, error) { - return Eval(s, os.Getenv) +func EvalEnv(s string, strict bool) (string, error) { + mapping := Getenv + if strict { + mapping = os.LookupEnv + } + return Eval(s, mapping) +} + +func Getenv(s string) (string, bool) { + return os.Getenv(s), true } diff --git a/envsubst/eval_test.go b/envsubst/eval_test.go index d3c1b19b..430ac8c1 100644 --- a/envsubst/eval_test.go +++ b/envsubst/eval_test.go @@ -20,7 +20,10 @@ limitations under the License. package envsubst -import "testing" +import ( + "errors" + "testing" +) // test cases sourced from tldp.org // http://www.tldp.org/LDP/abs/html/parameter-substitution.html @@ -230,8 +233,8 @@ func TestExpand(t *testing.T) { for _, expr := range expressions { t.Run(expr.input, func(t *testing.T) { t.Logf(expr.input) - output, err := Eval(expr.input, func(s string) string { - return expr.params[s] + output, err := Eval(expr.input, func(s string) (string, bool) { + return expr.params[s], true }) if err != nil { t.Errorf("Want %q expanded but got error %q", expr.input, err) @@ -246,3 +249,56 @@ func TestExpand(t *testing.T) { }) } } + +func TestExpandStrict(t *testing.T) { + var expressions = []struct { + params map[string]string + input string + output string + wantErr error + }{ + // text-only + { + params: map[string]string{}, + input: "abcdEFGH28ij", + output: "abcdEFGH28ij", + wantErr: nil, + }, + // existing + { + params: map[string]string{"foo": "bar"}, + input: "${foo}", + output: "bar", + wantErr: nil, + }, + // missing + { + params: map[string]string{}, + input: "${missing}", + output: "", + wantErr: errVarNotSet, + }, + } + + for _, expr := range expressions { + t.Run(expr.input, func(t *testing.T) { + t.Logf(expr.input) + output, err := Eval(expr.input, func(s string) (string, bool) { + v, exists := expr.params[s] + return v, exists + }) + if expr.wantErr == nil && err != nil { + t.Errorf("Want %q expanded but got error %q", expr.input, err) + } + if expr.wantErr != nil && !errors.Is(err, expr.wantErr) { + t.Errorf("Want error %q but got error %q", expr.wantErr, err) + } + if output != expr.output { + t.Errorf("Want %q expanded to %q, got %q", + expr.input, + expr.output, + output) + } + }) + } +} diff --git a/envsubst/template.go b/envsubst/template.go index db8e7225..38c44219 100644 --- a/envsubst/template.go +++ b/envsubst/template.go @@ -22,8 +22,9 @@ package envsubst import ( "bytes" + "fmt" "io" - "io/ioutil" + "os" "github.com/fluxcd/pkg/envsubst/parse" ) @@ -36,7 +37,7 @@ type state struct { node parse.Node // current node // maps variable names to values - mapper func(string) string + mapper func(string) (value string, exists bool) } // Template is the representation of a parsed shell format string. @@ -58,7 +59,7 @@ func Parse(s string) (t *Template, err error) { // ParseFile creates a new shell format template and parses the template // definition from the named file. func ParseFile(path string) (*Template, error) { - b, err := ioutil.ReadFile(path) + b, err := os.ReadFile(path) if err != nil { return nil, err } @@ -66,7 +67,7 @@ func ParseFile(path string) (*Template, error) { } // Execute applies a parsed template to the specified data mapping. -func (t *Template) Execute(mapping func(string) string) (str string, err error) { +func (t *Template) Execute(mapping func(string) (string, bool)) (str string, err error) { b := new(bytes.Buffer) s := new(state) s.node = t.tree.Root @@ -107,6 +108,8 @@ func (t *Template) evalList(s *state, node *parse.ListNode) (err error) { return nil } +var errVarNotSet = fmt.Errorf("variable not set (strict mode)") + func (t *Template) evalFunc(s *state, node *parse.FuncNode) error { var w = s.writer var buf bytes.Buffer @@ -126,8 +129,11 @@ func (t *Template) evalFunc(s *state, node *parse.FuncNode) error { s.writer = w s.node = node - v := s.mapper(node.Param) + v, exists := s.mapper(node.Param) + if node.Name == "" && !exists { + return fmt.Errorf("%w: %q", errVarNotSet, node.Param) + } fn := lookupFunc(node.Name, len(args)) _, err := io.WriteString(s.writer, fn(v, args...)) From b3eaa12fbabd954e9c57eea758b72dc9ad97cf22 Mon Sep 17 00:00:00 2001 From: Stefan Prodan Date: Mon, 8 Apr 2024 11:01:57 +0300 Subject: [PATCH 3/4] envsubst: Add package doc Signed-off-by: Stefan Prodan --- envsubst/doc.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 envsubst/doc.go diff --git a/envsubst/doc.go b/envsubst/doc.go new file mode 100644 index 00000000..400c94dc --- /dev/null +++ b/envsubst/doc.go @@ -0,0 +1,19 @@ +/* +Copyright 2024 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package envsubst is a Go package for expanding variables in a string using `${var}` syntax. +// Includes support for bash string replacement functions. +package envsubst From c0615824c702baeab2e6f9fb97a7909a2a22c62b Mon Sep 17 00:00:00 2001 From: Stefan Prodan Date: Mon, 8 Apr 2024 12:04:14 +0300 Subject: [PATCH 4/4] envsubst: Improve strict mode tests Signed-off-by: Stefan Prodan --- envsubst/eval_test.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/envsubst/eval_test.go b/envsubst/eval_test.go index 430ac8c1..1bcead93 100644 --- a/envsubst/eval_test.go +++ b/envsubst/eval_test.go @@ -271,6 +271,13 @@ func TestExpandStrict(t *testing.T) { output: "bar", wantErr: nil, }, + // existing string empty + { + params: map[string]string{"foo": ""}, + input: "${foo}", + output: "", + wantErr: nil, + }, // missing { params: map[string]string{}, @@ -278,6 +285,13 @@ func TestExpandStrict(t *testing.T) { output: "", wantErr: errVarNotSet, }, + // missing but has default + { + params: map[string]string{"foo": "bar"}, + input: "${missing:=default}", + output: "default", + wantErr: nil, + }, } for _, expr := range expressions {