From 78d836339b3eebcd44b8523361bb93519a88d81b Mon Sep 17 00:00:00 2001 From: Herman Schaaf Date: Wed, 1 Feb 2023 13:28:31 +0000 Subject: [PATCH 1/3] Fix: handle numbers in env variables --- specs/spec_reader.go | 26 +++++++++++++++----------- specs/spec_reader_test.go | 12 ++++++++++++ specs/testdata/number.txt | 1 + specs/testdata/numbers.yml | 21 +++++++++++++++++++++ 4 files changed, 49 insertions(+), 11 deletions(-) create mode 100644 specs/testdata/number.txt create mode 100644 specs/testdata/numbers.yml diff --git a/specs/spec_reader.go b/specs/spec_reader.go index 01125412a0..87eb2e1b5c 100644 --- a/specs/spec_reader.go +++ b/specs/spec_reader.go @@ -3,6 +3,7 @@ package specs import ( "bytes" "fmt" + "math/rand" "os" "path/filepath" "regexp" @@ -185,20 +186,22 @@ func NewSpecReader(paths []string) (*SpecReader, error) { // strip yaml comments from the given yaml document by converting to JSON and back :) func stripYamlComments(b []byte) ([]byte, error) { - const openPlaceholder = "$$$OPEN$$$" - const closePlaceholder = "$$$CLOSE$$$" - - // return an error if the yaml already contains our temporary placeholder for env variables by some unlucky coincidence - if bytes.Contains(b, []byte(openPlaceholder)) || bytes.Contains(b, []byte(closePlaceholder)) { - return nil, fmt.Errorf("%s and %s are reserved words in CloudQuery config", openPlaceholder, closePlaceholder) - } - // replace placeholder variables with valid yaml, otherwise it cannot be parsed // in some cases. Short of writing our own yaml parser to remove comments, // this seems like the best we can do. + // We replace placeholder variables with random numbers, because numbers in quotes + // will then remain quoted in the final yaml. If we replace with strings, they will + // be unquoted in the final yaml. + r := rand.New(rand.NewSource(1)) + placeholders := map[string]string{} b = envRegex.ReplaceAllFunc(b, func(match []byte) []byte { content := envRegex.FindSubmatch(match)[1] - return []byte(openPlaceholder + string(content) + closePlaceholder) + k := fmt.Sprintf("%d", r.Int()) + for bytes.Contains(content, []byte(k)) { + k = fmt.Sprintf("%d", r.Int()) + } + placeholders[k] = string(content) + return []byte(k) }) j, err := yaml.YAMLToJSON(b) if err != nil { @@ -209,7 +212,8 @@ func stripYamlComments(b []byte) ([]byte, error) { return nil, err } // place back placeholder variables - b = bytes.ReplaceAll(b, []byte(openPlaceholder), []byte("${")) - b = bytes.ReplaceAll(b, []byte(closePlaceholder), []byte("}")) + for k, v := range placeholders { + b = bytes.ReplaceAll(b, []byte(k), []byte(fmt.Sprintf("${%s}", v))) + } return b, nil } diff --git a/specs/spec_reader_test.go b/specs/spec_reader_test.go index e2b98027d2..e596e3a501 100644 --- a/specs/spec_reader_test.go +++ b/specs/spec_reader_test.go @@ -129,6 +129,18 @@ var specLoaderTestCases = []specLoaderTestCase{ destinations: 1, envVariables: map[string]string{}, }, + { + name: "number in name field", + path: []string{getPath("numbers.yml")}, + err: func() string { + return "" + }, + sources: 2, + destinations: 1, + envVariables: map[string]string{ + "ACCOUNT_ID": "123456789", + }, + }, } func TestLoadSpecs(t *testing.T) { diff --git a/specs/testdata/number.txt b/specs/testdata/number.txt new file mode 100644 index 0000000000..bd41cba781 --- /dev/null +++ b/specs/testdata/number.txt @@ -0,0 +1 @@ +12345 \ No newline at end of file diff --git a/specs/testdata/numbers.yml b/specs/testdata/numbers.yml new file mode 100644 index 0000000000..cb0fe3485c --- /dev/null +++ b/specs/testdata/numbers.yml @@ -0,0 +1,21 @@ +kind: source +spec: + name: "${ACCOUNT_ID}" + version: v1 + destinations: ["0987654321"] + path: cloudquery/aws +--- +kind: source +spec: + name: "${file:./testdata/number.txt}" + version: v1 + destinations: ["0987654321"] + path: cloudquery/aws +--- +kind: destination +spec: + name: "0987654321" + path: cloudquery/postgresql + version: v1 + spec: + connection_string: postgresql://localhost:5432/cloudquery?sslmode=disable From c69cc89de273e53667cf848b862e6c9ecd03728d Mon Sep 17 00:00:00 2001 From: Herman Schaaf Date: Wed, 1 Feb 2023 14:03:21 +0000 Subject: [PATCH 2/3] Use numbers that start with 0 --- specs/spec_reader_test.go | 2 +- specs/testdata/number.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/spec_reader_test.go b/specs/spec_reader_test.go index e596e3a501..6ade0f2c1b 100644 --- a/specs/spec_reader_test.go +++ b/specs/spec_reader_test.go @@ -138,7 +138,7 @@ var specLoaderTestCases = []specLoaderTestCase{ sources: 2, destinations: 1, envVariables: map[string]string{ - "ACCOUNT_ID": "123456789", + "ACCOUNT_ID": "0123456789", }, }, } diff --git a/specs/testdata/number.txt b/specs/testdata/number.txt index bd41cba781..f54df5815f 100644 --- a/specs/testdata/number.txt +++ b/specs/testdata/number.txt @@ -1 +1 @@ -12345 \ No newline at end of file +012345 \ No newline at end of file From 63ca6c158a93d56c3563c31f0d2cfe6c439de1f2 Mon Sep 17 00:00:00 2001 From: Herman Schaaf Date: Wed, 1 Feb 2023 14:16:40 +0000 Subject: [PATCH 3/3] Add test for account numbers starting with 0 --- specs/spec_reader_test.go | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/specs/spec_reader_test.go b/specs/spec_reader_test.go index 6ade0f2c1b..910b4023bc 100644 --- a/specs/spec_reader_test.go +++ b/specs/spec_reader_test.go @@ -170,6 +170,32 @@ func TestLoadSpecs(t *testing.T) { } } +func TestLoadSpecWithAccountNumbers(t *testing.T) { + t.Setenv("ACCOUNT_ID", "0123456789") + specReader, err := NewSpecReader([]string{getPath("numbers.yml")}) + if err != nil { + t.Fatal(err) + } + if len(specReader.Sources) != 2 { + t.Fatalf("got: %d expected: %d", len(specReader.Sources), 2) + } + if len(specReader.Destinations) != 1 { + t.Fatalf("got: %d expected: %d", len(specReader.Destinations), 1) + } + if _, ok := specReader.Sources["0123456789"]; !ok { + t.Fatalf("expected source with account id 0123456789") + } + if specReader.Sources["0123456789"].Name != "0123456789" { + t.Fatalf("got: %s expected: %s", specReader.Sources["0123456789"].Name, "0123456789") + } + if _, ok := specReader.Destinations["0987654321"]; !ok { + t.Fatalf("expected destination with account id 0987654321") + } + if specReader.Destinations["0987654321"].Name != "0987654321" { + t.Fatalf("got: %s expected: %s", specReader.Destinations["0987654321"].Name, "0987654321") + } +} + func TestExpandFile(t *testing.T) { cfg := []byte(` kind: source