fix: Embedded content handling with newlines

specs/spec_reader.go

@@ -9,6 +9,7 @@ import (
 	"path/filepath"
 	"reflect"
 	"regexp"
+	"strconv"
 	"strings"
 
 	"github.com/ghodss/yaml"
@@ -26,16 +27,20 @@ type SpecReader struct {
 var fileRegex = regexp.MustCompile(`\$\{file:([^}]+)\}`)
 var envRegex = regexp.MustCompile(`\$\{([^}]+)\}`)
 
-// readEncodeIfJSON reads the content of the file and if it is a JSON object or array, it encodes it as a string to satisfy YAML requirements
-// It will suppress any JSON unmarshalling errors and return the original content.
-func readEncodeIfJSON(content []byte) ([]byte, error) {
+// escapeExternalContent escapes the given content if it contains newlines or is a JSON object or array, to satisfy YAML requirements
+// It will suppress any JSON unmarshalling errors and may return the original content.
+func escapeExternalContent(content []byte) ([]byte, error) {
 	var isJSON any
 	if err := json.Unmarshal(content, &isJSON); err != nil {
+		if bytes.ContainsAny(content, "\n\r") {
+			return []byte(strconv.Quote(string(content))), nil
+		}
 		return content, nil
 	}
 
 	k := reflect.TypeOf(isJSON).Kind()
-	if k == reflect.Map || k == reflect.Slice {
+	switch k {
+	case reflect.Map, reflect.Slice:
 		buffer := &bytes.Buffer{}
 		encoder := json.NewEncoder(buffer)
 		encoder.SetEscapeHTML(false)
@@ -58,7 +63,7 @@ func expandFileConfig(cfg []byte) ([]byte, error) {
 			expandErr = err
 			return nil
 		}
-		content, err = readEncodeIfJSON(content)
+		content, err = escapeExternalContent(content)
 		if expandErr == nil {
 			expandErr = err
 		}
@@ -77,7 +82,7 @@ func expandEnv(cfg []byte) ([]byte, error) {
 			expandErr = fmt.Errorf("env variable %s not found", envVar)
 			return nil
 		}
-		newcontent, err := readEncodeIfJSON([]byte(content))
+		newcontent, err := escapeExternalContent([]byte(content))
 		if expandErr == nil {
 			expandErr = err
 		}

specs/spec_reader_test.go

@@ -402,6 +402,54 @@ spec:
 
 	s, err := NewSpecReader([]string{f.Name()})
 	assert.NoError(t, err)
+	if t.Failed() {
+		return
+	}
+
+	assert.Equal(t, 1, len(s.Sources))
+	sp := s.Sources[0].Spec.(map[string]any)
+	assert.Equal(t, expectedCreds, sp["credentials"])
+}
+
+func TestExpandEnvNewlines(t *testing.T) {
+	expectedCreds := `-----BEGIN PRIVATE KEY-----
+MIItest
+	tabbledline
+\backslashes\here
+-----END PRIVATE KEY-----
+`
+
+	os.Setenv("TEST_ENV_CREDS3", expectedCreds)
+	cfg := []byte(`
+kind: source
+spec:
+  name: test
+  registry: local
+  path: /path/to/source
+  version: v1.0.0
+  tables: [ "some_table" ]
+  destinations: [ "test2" ]
+  spec:
+    credentials: ${TEST_ENV_CREDS3}
+    otherstuff: 2
+---
+kind: destination
+spec:
+  name: test2
+  registry: local
+  path: /path/to/dest
+`)
+
+	f, err := os.CreateTemp("", "testcase*.yaml")
+	assert.NoError(t, err)
+	defer os.Remove(f.Name())
+	assert.NoError(t, os.WriteFile(f.Name(), cfg, 0644))
+
+	s, err := NewSpecReader([]string{f.Name()})
+	assert.NoError(t, err)
+	if t.Failed() {
+		return
+	}
 
 	assert.Equal(t, 1, len(s.Sources))
 	sp := s.Sources[0].Spec.(map[string]any)