fix: SpecReader should escape external JSON content from files and environment variables

specs/spec_reader.go

@@ -2,10 +2,12 @@ package specs
 
 import (
 	"bytes"
+	"encoding/json"
 	"fmt"
 	"math/rand"
 	"os"
 	"path/filepath"
+	"reflect"
 	"regexp"
 	"strings"
 
@@ -24,6 +26,29 @@ type SpecReader struct {
 var fileRegex = regexp.MustCompile(`\$\{file:([^}]+)\}`)
 var envRegex = regexp.MustCompile(`\$\{([^}]+)\}`)
 
+// readEncodeIfJSON reads the content of the file and if it is a JSON object or array, it encodes it as a string to satisfy YAML requirements
+// It will suppress any JSON unmarshalling errors and return the original content.
+func readEncodeIfJSON(content []byte) ([]byte, error) {
+	var isJSON any
+	if err := json.Unmarshal(content, &isJSON); err != nil {
+		return content, nil
+	}
+
+	k := reflect.TypeOf(isJSON).Kind()
+	if k == reflect.Map || k == reflect.Slice {
+		buffer := &bytes.Buffer{}
+		encoder := json.NewEncoder(buffer)
+		encoder.SetEscapeHTML(false)
+		if err := encoder.Encode(string(content)); err != nil {
+			return content, err
+		}
+
+		return bytes.TrimSuffix(buffer.Bytes(), []byte{'\n'}), nil
+	}
+
+	return content, nil
+}
+
 func expandFileConfig(cfg []byte) ([]byte, error) {
 	var expandErr error
 	cfg = fileRegex.ReplaceAllFunc(cfg, func(match []byte) []byte {
@@ -33,6 +58,10 @@ func expandFileConfig(cfg []byte) ([]byte, error) {
 			expandErr = err
 			return nil
 		}
+		content, err = readEncodeIfJSON(content)
+		if expandErr == nil {
+			expandErr = err
+		}
 		return content
 	})
 	return cfg, expandErr
@@ -48,7 +77,11 @@ func expandEnv(cfg []byte) ([]byte, error) {
 			expandErr = fmt.Errorf("env variable %s not found", envVar)
 			return nil
 		}
-		return []byte(content)
+		newcontent, err := readEncodeIfJSON([]byte(content))
+		if expandErr == nil {
+			expandErr = err
+		}
+		return newcontent
 	})
 
 	return cfg, expandErr

specs/spec_reader_test.go

@@ -7,6 +7,7 @@ import (
 	"runtime"
 	"testing"
 
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
@@ -287,6 +288,39 @@ spec:
 	}
 }
 
+func TestExpandFileJSON(t *testing.T) {
+	cfg := []byte(`
+kind: source
+spec:
+  name: gcp
+  path: cloudquery/gcp
+  version: v1.0.0
+  table_concurrency: 10
+  registry: local
+  destinations: [postgresql]
+  service_account_key_json: ${file:./testdata/creds2.json}
+	`)
+	expectedCfg := []byte(`
+kind: source
+spec:
+  name: gcp
+  path: cloudquery/gcp
+  version: v1.0.0
+  table_concurrency: 10
+  registry: local
+  destinations: [postgresql]
+  service_account_key_json: "{\n  \"key\": \"foo\",\n  \"secret\": \"bar<baz>\"\n}\n"
+	`)
+	expandedCfg, err := expandFileConfig(cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if runtime.GOOS == "windows" {
+		expectedCfg = bytes.ReplaceAll(expectedCfg, []byte(`\n`), []byte(`\r\n`))
+	}
+	assert.Equal(t, string(expectedCfg), string(expandedCfg))
+}
+
 func TestExpandEnv(t *testing.T) {
 	os.Setenv("TEST_ENV_CREDS", "mytestcreds")
 	os.Setenv("TEST_ENV_CREDS2", "anothercredtest")
@@ -332,3 +366,44 @@ spec:
 		t.Fatal("expected error, got nil")
 	}
 }
+
+func TestExpandEnvJSONNewlines(t *testing.T) {
+	expectedCreds := `{
+   "type": "service_account",
+  "private_key": "-----BEGIN PRIVATE KEY-----\nMIItest\n\n-----END PRIVATE KEY-----\n",
+  "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/test%40test.iam.gserviceaccount.com"
+}
+`
+	os.Setenv("TEST_ENV_CREDS3", expectedCreds)
+	cfg := []byte(`
+kind: source
+spec:
+  name: test
+  registry: local
+  path: /path/to/source
+  version: v1.0.0
+  tables: [ "some_table" ]
+  destinations: [ "test2" ]
+  spec:
+    credentials: ${TEST_ENV_CREDS3}
+    otherstuff: 2
+---
+kind: destination
+spec:
+  name: test2
+  registry: local
+  path: /path/to/dest
+`)
+
+	f, err := os.CreateTemp("", "testcase*.yaml")
+	assert.NoError(t, err)
+	defer os.Remove(f.Name())
+	assert.NoError(t, os.WriteFile(f.Name(), cfg, 0644))
+
+	s, err := NewSpecReader([]string{f.Name()})
+	assert.NoError(t, err)
+
+	assert.Equal(t, 1, len(s.Sources))
+	sp := s.Sources[0].Spec.(map[string]any)
+	assert.Equal(t, expectedCreds, sp["credentials"])
+}

specs/testdata/creds2.json

@@ -0,0 +1,4 @@
+{
+  "key": "foo",
+  "secret": "bar<baz>"
+}