fix: Handle numbers in env variables

specs/spec_reader.go

@@ -3,6 +3,7 @@ package specs
 import (
 	"bytes"
 	"fmt"
+	"math/rand"
 	"os"
 	"path/filepath"
 	"regexp"
@@ -185,20 +186,22 @@ func NewSpecReader(paths []string) (*SpecReader, error) {
 
 // strip yaml comments from the given yaml document by converting to JSON and back :)
 func stripYamlComments(b []byte) ([]byte, error) {
-	const openPlaceholder = "$$$OPEN$$$"
-	const closePlaceholder = "$$$CLOSE$$$"
-
-	// return an error if the yaml already contains our temporary placeholder for env variables by some unlucky coincidence
-	if bytes.Contains(b, []byte(openPlaceholder)) || bytes.Contains(b, []byte(closePlaceholder)) {
-		return nil, fmt.Errorf("%s and %s are reserved words in CloudQuery config", openPlaceholder, closePlaceholder)
-	}
-
 	// replace placeholder variables with valid yaml, otherwise it cannot be parsed
 	// in some cases. Short of writing our own yaml parser to remove comments,
 	// this seems like the best we can do.
+	// We replace placeholder variables with random numbers, because numbers in quotes
+	// will then remain quoted in the final yaml. If we replace with strings, they will
+	// be unquoted in the final yaml.
+	r := rand.New(rand.NewSource(1))
+	placeholders := map[string]string{}
 	b = envRegex.ReplaceAllFunc(b, func(match []byte) []byte {
 		content := envRegex.FindSubmatch(match)[1]
-		return []byte(openPlaceholder + string(content) + closePlaceholder)
+		k := fmt.Sprintf("%d", r.Int())
+		for bytes.Contains(content, []byte(k)) {
+			k = fmt.Sprintf("%d", r.Int())
+		}
+		placeholders[k] = string(content)
+		return []byte(k)
 	})
 	j, err := yaml.YAMLToJSON(b)
 	if err != nil {
@@ -209,7 +212,8 @@ func stripYamlComments(b []byte) ([]byte, error) {
 		return nil, err
 	}
 	// place back placeholder variables
-	b = bytes.ReplaceAll(b, []byte(openPlaceholder), []byte("${"))
-	b = bytes.ReplaceAll(b, []byte(closePlaceholder), []byte("}"))
+	for k, v := range placeholders {
+		b = bytes.ReplaceAll(b, []byte(k), []byte(fmt.Sprintf("${%s}", v)))
+	}
 	return b, nil
 }

specs/spec_reader_test.go

@@ -129,6 +129,18 @@ var specLoaderTestCases = []specLoaderTestCase{
 		destinations: 1,
 		envVariables: map[string]string{},
 	},
+	{
+		name: "number in name field",
+		path: []string{getPath("numbers.yml")},
+		err: func() string {
+			return ""
+		},
+		sources:      2,
+		destinations: 1,
+		envVariables: map[string]string{
+			"ACCOUNT_ID": "0123456789",
+		},
+	},
 }
 
 func TestLoadSpecs(t *testing.T) {
@@ -158,6 +170,32 @@ func TestLoadSpecs(t *testing.T) {
 	}
 }
 
+func TestLoadSpecWithAccountNumbers(t *testing.T) {
+	t.Setenv("ACCOUNT_ID", "0123456789")
+	specReader, err := NewSpecReader([]string{getPath("numbers.yml")})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(specReader.Sources) != 2 {
+		t.Fatalf("got: %d expected: %d", len(specReader.Sources), 2)
+	}
+	if len(specReader.Destinations) != 1 {
+		t.Fatalf("got: %d expected: %d", len(specReader.Destinations), 1)
+	}
+	if _, ok := specReader.Sources["0123456789"]; !ok {
+		t.Fatalf("expected source with account id 0123456789")
+	}
+	if specReader.Sources["0123456789"].Name != "0123456789" {
+		t.Fatalf("got: %s expected: %s", specReader.Sources["0123456789"].Name, "0123456789")
+	}
+	if _, ok := specReader.Destinations["0987654321"]; !ok {
+		t.Fatalf("expected destination with account id 0987654321")
+	}
+	if specReader.Destinations["0987654321"].Name != "0987654321" {
+		t.Fatalf("got: %s expected: %s", specReader.Destinations["0987654321"].Name, "0987654321")
+	}
+}
+
 func TestExpandFile(t *testing.T) {
 	cfg := []byte(`
 kind: source

specs/testdata/number.txt

@@ -0,0 +1 @@
+012345

specs/testdata/numbers.yml

@@ -0,0 +1,21 @@
+kind: source
+spec:
+  name: "${ACCOUNT_ID}"
+  version: v1
+  destinations: ["0987654321"]
+  path: cloudquery/aws
+---
+kind: source
+spec:
+  name: "${file:./testdata/number.txt}"
+  version: v1
+  destinations: ["0987654321"]
+  path: cloudquery/aws
+---
+kind: destination
+spec:
+  name: "0987654321"
+  path: cloudquery/postgresql
+  version: v1
+  spec:
+    connection_string: postgresql://localhost:5432/cloudquery?sslmode=disable