Avoid encoding html entities in JSON documents

internal/docs/readme_test.go

@@ -11,6 +11,8 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+
+	"github.com/elastic/elastic-package/internal/packages"
 )
 
 func TestGenerateReadme(t *testing.T) {
@@ -152,6 +154,9 @@ An example event for ` + "`example`" + ` looks as following:
 			err = createSampleEventFile(c.packageRoot, c.dataStreamName, c.sampleEventJsonContents)
 			require.NoError(t, err)
 
+			err = createManifestFile(c.packageRoot)
+			require.NoError(t, err)
+
 			rendered, err := renderReadme(filename, c.packageRoot, templatePath, linksMap)
 			require.NoError(t, err)
 
@@ -293,6 +298,13 @@ func createSampleEventFile(packageRoot, dataStreamName, contents string) error {
 	return nil
 }
 
+func createManifestFile(packageRoot string) error {
+	// Minimal content needed to render readme.
+	manifest := `format_version: 2.10.0`
+	manifestFile := filepath.Join(packageRoot, packages.PackageManifestFile)
+	return os.WriteFile(manifestFile, []byte(manifest), 0644)
+}
+
 func createDataStreamFolder(packageRoot, dataStreamName string) (string, error) {
 	if dataStreamName == "" {
 		return "", nil

internal/docs/sample_event.go

@@ -10,7 +10,10 @@ import (
 	"path/filepath"
 	"strings"
 
+	"github.com/Masterminds/semver/v3"
+
 	"github.com/elastic/elastic-package/internal/formatter"
+	"github.com/elastic/elastic-package/internal/packages"
 )
 
 const sampleEventFile = "sample_event.json"
@@ -23,7 +26,17 @@ func renderSampleEvent(packageRoot, dataStreamName string) (string, error) {
 		return "", fmt.Errorf("reading sample event file failed (path: %s): %w", eventPath, err)
 	}
 
-	formatted, _, err := formatter.JSONFormatter(body)
+	manifest, err := packages.ReadPackageManifestFromPackageRoot(packageRoot)
+	if err != nil {
+		return "", fmt.Errorf("reading package manifest failed: %w", err)
+	}
+	specVersion, err := semver.NewVersion(manifest.SpecVersion)
+	if err != nil {
+		return "", fmt.Errorf("parsing format version %q failed: %w", manifest.SpecVersion, err)
+	}
+
+	jsonFormatter := formatter.JSONFormatterBuilder(*specVersion)
+	formatted, _, err := jsonFormatter.Format(body)
 	if err != nil {
 		return "", fmt.Errorf("formatting sample event file failed (path: %s): %w", eventPath, err)
 	}

internal/formatter/formatter.go

@@ -8,19 +8,36 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+
+	"github.com/Masterminds/semver/v3"
+
+	"github.com/elastic/elastic-package/internal/packages"
 )
 
 type formatter func(content []byte) ([]byte, bool, error)
 
-var formatters = map[string]formatter{
-	".json": JSONFormatter,
-	".yaml": YAMLFormatter,
-	".yml":  YAMLFormatter,
+func newFormatter(specVersion semver.Version, ext string) formatter {
+	switch ext {
+	case ".json":
+		return JSONFormatterBuilder(specVersion).Format
+	case ".yaml", ".yml":
+		return YAMLFormatter
+	default:
+		return nil
+	}
 }
 
 // Format method formats files inside of the integration directory.
 func Format(packageRoot string, failFast bool) error {
-	err := filepath.Walk(packageRoot, func(path string, info os.FileInfo, err error) error {
+	manifest, err := packages.ReadPackageManifestFromPackageRoot(packageRoot)
+	if err != nil {
+		return fmt.Errorf("failed to read package manifest: %w", err)
+	}
+	specVersion, err := semver.NewVersion(manifest.SpecVersion)
+	if err != nil {
+		return fmt.Errorf("failed to parse package format version %q: %w", manifest.SpecVersion, err)
+	}
+	err = filepath.Walk(packageRoot, func(path string, info os.FileInfo, err error) error {
 		if err != nil {
 			return err
 		}
@@ -31,7 +48,7 @@ func Format(packageRoot string, failFast bool) error {
 		if info.IsDir() {
 			return nil
 		}
-		err = formatFile(path, failFast)
+		err = formatFile(path, failFast, *specVersion)
 		if err != nil {
 			return fmt.Errorf("formatting file failed (path: %s): %w", path, err)
 		}
@@ -44,17 +61,15 @@ func Format(packageRoot string, failFast bool) error {
 	return nil
 }
 
-func formatFile(path string, failFast bool) error {
-	file := filepath.Base(path)
-	ext := filepath.Ext(file)
-
+func formatFile(path string, failFast bool, specVersion semver.Version) error {
 	content, err := os.ReadFile(path)
 	if err != nil {
 		return fmt.Errorf("reading file content failed: %w", err)
 	}
 
-	format, defined := formatters[ext]
-	if !defined {
+	ext := filepath.Ext(filepath.Base(path))
+	format := newFormatter(specVersion, ext)
+	if format == nil {
 		return nil // no errors returned as we have few files that will be never formatted (png, svg, log, etc.)
 	}
 

internal/formatter/json_formatter.go

@@ -5,13 +5,31 @@
 package formatter
 
 import (
+	"bytes"
 	"encoding/json"
 	"fmt"
+
+	"github.com/Masterminds/semver/v3"
 )
 
-// JSONFormatter function is responsible for formatting the given JSON input.
-// The function is exposed, so it can be used by other internal packages, e.g. to format sample events in docs.
-func JSONFormatter(content []byte) ([]byte, bool, error) {
+type JSONFormatter interface {
+	Format([]byte) ([]byte, bool, error)
+	Encode(doc any) ([]byte, error)
+}
+
+func JSONFormatterBuilder(specVersion semver.Version) JSONFormatter {
+	if specVersion.LessThan(semver.MustParse("2.12.0")) {
+		return &jsonFormatterWithHTMLEncoding{}
+	}
+
+	return &jsonFormatter{}
+}
+
+// jsonFormatterWithHTMLEncoding function is responsible for formatting the given JSON input.
+// It encodes special HTML characters.
+type jsonFormatterWithHTMLEncoding struct{}
+
+func (jsonFormatterWithHTMLEncoding) Format(content []byte) ([]byte, bool, error) {
 	var rawMessage json.RawMessage
 	err := json.Unmarshal(content, &rawMessage)
 	if err != nil {
@@ -24,3 +42,35 @@ func JSONFormatter(content []byte) ([]byte, bool, error) {
 	}
 	return formatted, string(content) == string(formatted), nil
 }
+
+func (jsonFormatterWithHTMLEncoding) Encode(doc any) ([]byte, error) {
+	return json.MarshalIndent(doc, "", "    ")
+}
+
+// jsonFormatter function is responsible for formatting the given JSON input.
+type jsonFormatter struct{}
+
+func (jsonFormatter) Format(content []byte) ([]byte, bool, error) {
+	var formatted bytes.Buffer
+	err := json.Indent(&formatted, content, "", "    ")
+	if err != nil {
+		return nil, false, fmt.Errorf("formatting JSON document failed: %w", err)
+	}
+
+	return formatted.Bytes(), bytes.Equal(content, formatted.Bytes()), nil
+}
+
+func (jsonFormatter) Encode(doc any) ([]byte, error) {
+	var formatted bytes.Buffer
+	enc := json.NewEncoder(&formatted)
+	enc.SetEscapeHTML(false)
+	enc.SetIndent("", "    ")
+
+	err := enc.Encode(doc)
+	if err != nil {
+		return nil, err
+	}
+
+	// Trimming to be consistent with MarshalIndent, that seems to trim the result.
+	return bytes.TrimSpace(formatted.Bytes()), nil
+}

internal/formatter/json_formatter_test.go

@@ -0,0 +1,106 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package formatter_test
+
+import (
+	"testing"
+
+	"github.com/Masterminds/semver/v3"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/elastic/elastic-package/internal/formatter"
+)
+
+func TestJSONFormatterFormat(t *testing.T) {
+	cases := []struct {
+		title    string
+		version  *semver.Version
+		content  string
+		expected string
+		valid    bool
+	}{
+		{
+			title:   "invalid json 2.0",
+			version: semver.MustParse("2.0.0"),
+			content: `{"foo":}`,
+			valid:   false,
+		},
+		{
+			title:   "invalid json 3.0",
+			version: semver.MustParse("3.0.0"),
+			content: `{"foo":}`,
+			valid:   false,
+		},
+		{
+			title:   "encode html in old versions",
+			version: semver.MustParse("2.0.0"),
+			content: `{"a": "<script></script>"}`,
+			expected: `{
+    "a": "\u003cscript\u003e\u003c/script\u003e"
+}`,
+			valid: true,
+		},
+		{
+			title:   "don't encode html since 2.12.0",
+			version: semver.MustParse("2.12.0"),
+			content: `{"a": "<script></script>"}`,
+			expected: `{
+    "a": "<script></script>"
+}`,
+			valid: true,
+		},
+	}
+
+	for _, c := range cases {
+		t.Run(c.title, func(t *testing.T) {
+			jsonFormatter := formatter.JSONFormatterBuilder(*c.version)
+			formatted, equal, err := jsonFormatter.Format([]byte(c.content))
+			if !c.valid {
+				assert.Error(t, err)
+				return
+			}
+			require.NoError(t, err)
+
+			assert.Equal(t, c.expected, string(formatted))
+			assert.Equal(t, c.content == c.expected, equal)
+		})
+	}
+}
+
+func TestJSONFormatterEncode(t *testing.T) {
+	cases := []struct {
+		title    string
+		version  *semver.Version
+		object   any
+		expected string
+	}{
+		{
+			title:   "encode html in old versions",
+			version: semver.MustParse("2.0.0"),
+			object:  map[string]any{"a": "<script></script>"},
+			expected: `{
+    "a": "\u003cscript\u003e\u003c/script\u003e"
+}`,
+		},
+		{
+			title:   "don't encode html since 2.12.0",
+			version: semver.MustParse("2.12.0"),
+			object:  map[string]any{"a": "<script></script>"},
+			expected: `{
+    "a": "<script></script>"
+}`,
+		},
+	}
+
+	for _, c := range cases {
+		t.Run(c.title, func(t *testing.T) {
+			jsonFormatter := formatter.JSONFormatterBuilder(*c.version)
+			formatted, err := jsonFormatter.Encode(c.object)
+			require.NoError(t, err)
+			assert.Equal(t, c.expected, string(formatted))
+		})
+	}
+}

internal/testrunner/runners/pipeline/runner.go

@@ -14,6 +14,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/Masterminds/semver/v3"
 	"gopkg.in/yaml.v3"
 
 	"github.com/elastic/elastic-package/internal/common"
@@ -283,10 +284,19 @@ func (r *runner) loadTestCaseFile(testCaseFile string) (*testCase, error) {
 func (r *runner) verifyResults(testCaseFile string, config *testConfig, result *testResult, fieldsValidator *fields.Validator) error {
 	testCasePath := filepath.Join(r.options.TestFolder.Path, testCaseFile)
 
+	manifest, err := packages.ReadPackageManifestFromPackageRoot(r.options.PackageRootPath)
+	if err != nil {
+		return fmt.Errorf("failed to read package manifest: %w", err)
+	}
+	specVersion, err := semver.NewVersion(manifest.SpecVersion)
+	if err != nil {
+		return fmt.Errorf("failed to parse package format version %q: %w", manifest.SpecVersion, err)
+	}
+
 	if r.options.GenerateTestResult {
 		// TODO: Add tests to cover regressive use of json.Unmarshal in writeTestResult.
 		// See https://github.com/elastic/elastic-package/pull/717.
-		err := writeTestResult(testCasePath, result)
+		err := writeTestResult(testCasePath, result, *specVersion)
 		if err != nil {
 			return fmt.Errorf("writing test result failed: %w", err)
 		}
@@ -301,7 +311,7 @@ func (r *runner) verifyResults(testCaseFile string, config *testConfig, result *
 	if stackConfig.Provider == stack.ProviderServerless {
 		skipGeoIP = true
 	}
-	err = compareResults(testCasePath, config, result, skipGeoIP)
+	err = compareResults(testCasePath, config, result, skipGeoIP, *specVersion)
 	if _, ok := err.(testrunner.ErrTestCaseFailed); ok {
 		return err
 	}