diff --git a/internal/docs/readme_test.go b/internal/docs/readme_test.go
index ab0afeb8bf..f34eeb7b34 100644
--- a/internal/docs/readme_test.go
+++ b/internal/docs/readme_test.go
@@ -11,6 +11,8 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
+
+ "github.com/elastic/elastic-package/internal/packages"
)
func TestGenerateReadme(t *testing.T) {
@@ -152,6 +154,9 @@ An example event for ` + "`example`" + ` looks as following:
err = createSampleEventFile(c.packageRoot, c.dataStreamName, c.sampleEventJsonContents)
require.NoError(t, err)
+ err = createManifestFile(c.packageRoot)
+ require.NoError(t, err)
+
rendered, err := renderReadme(filename, c.packageRoot, templatePath, linksMap)
require.NoError(t, err)
@@ -293,6 +298,13 @@ func createSampleEventFile(packageRoot, dataStreamName, contents string) error {
return nil
}
+func createManifestFile(packageRoot string) error {
+ // Minimal content needed to render readme.
+ manifest := `format_version: 2.10.0`
+ manifestFile := filepath.Join(packageRoot, packages.PackageManifestFile)
+ return os.WriteFile(manifestFile, []byte(manifest), 0644)
+}
+
func createDataStreamFolder(packageRoot, dataStreamName string) (string, error) {
if dataStreamName == "" {
return "", nil
diff --git a/internal/docs/sample_event.go b/internal/docs/sample_event.go
index 60305e532a..c677731795 100644
--- a/internal/docs/sample_event.go
+++ b/internal/docs/sample_event.go
@@ -10,7 +10,10 @@ import (
"path/filepath"
"strings"
+ "github.com/Masterminds/semver/v3"
+
"github.com/elastic/elastic-package/internal/formatter"
+ "github.com/elastic/elastic-package/internal/packages"
)
const sampleEventFile = "sample_event.json"
@@ -23,7 +26,17 @@ func renderSampleEvent(packageRoot, dataStreamName string) (string, error) {
return "", fmt.Errorf("reading sample event file failed (path: %s): %w", eventPath, err)
}
- formatted, _, err := formatter.JSONFormatter(body)
+ manifest, err := packages.ReadPackageManifestFromPackageRoot(packageRoot)
+ if err != nil {
+ return "", fmt.Errorf("reading package manifest failed: %w", err)
+ }
+ specVersion, err := semver.NewVersion(manifest.SpecVersion)
+ if err != nil {
+ return "", fmt.Errorf("parsing format version %q failed: %w", manifest.SpecVersion, err)
+ }
+
+ jsonFormatter := formatter.JSONFormatterBuilder(*specVersion)
+ formatted, _, err := jsonFormatter.Format(body)
if err != nil {
return "", fmt.Errorf("formatting sample event file failed (path: %s): %w", eventPath, err)
}
diff --git a/internal/formatter/formatter.go b/internal/formatter/formatter.go
index c2b408c854..eda77e11bc 100644
--- a/internal/formatter/formatter.go
+++ b/internal/formatter/formatter.go
@@ -8,19 +8,36 @@ import (
"fmt"
"os"
"path/filepath"
+
+ "github.com/Masterminds/semver/v3"
+
+ "github.com/elastic/elastic-package/internal/packages"
)
type formatter func(content []byte) ([]byte, bool, error)
-var formatters = map[string]formatter{
- ".json": JSONFormatter,
- ".yaml": YAMLFormatter,
- ".yml": YAMLFormatter,
+func newFormatter(specVersion semver.Version, ext string) formatter {
+ switch ext {
+ case ".json":
+ return JSONFormatterBuilder(specVersion).Format
+ case ".yaml", ".yml":
+ return YAMLFormatter
+ default:
+ return nil
+ }
}
// Format method formats files inside of the integration directory.
func Format(packageRoot string, failFast bool) error {
- err := filepath.Walk(packageRoot, func(path string, info os.FileInfo, err error) error {
+ manifest, err := packages.ReadPackageManifestFromPackageRoot(packageRoot)
+ if err != nil {
+ return fmt.Errorf("failed to read package manifest: %w", err)
+ }
+ specVersion, err := semver.NewVersion(manifest.SpecVersion)
+ if err != nil {
+ return fmt.Errorf("failed to parse package format version %q: %w", manifest.SpecVersion, err)
+ }
+ err = filepath.Walk(packageRoot, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
@@ -31,7 +48,7 @@ func Format(packageRoot string, failFast bool) error {
if info.IsDir() {
return nil
}
- err = formatFile(path, failFast)
+ err = formatFile(path, failFast, *specVersion)
if err != nil {
return fmt.Errorf("formatting file failed (path: %s): %w", path, err)
}
@@ -44,17 +61,15 @@ func Format(packageRoot string, failFast bool) error {
return nil
}
-func formatFile(path string, failFast bool) error {
- file := filepath.Base(path)
- ext := filepath.Ext(file)
-
+func formatFile(path string, failFast bool, specVersion semver.Version) error {
content, err := os.ReadFile(path)
if err != nil {
return fmt.Errorf("reading file content failed: %w", err)
}
- format, defined := formatters[ext]
- if !defined {
+ ext := filepath.Ext(filepath.Base(path))
+ format := newFormatter(specVersion, ext)
+ if format == nil {
return nil // no errors returned as we have few files that will be never formatted (png, svg, log, etc.)
}
diff --git a/internal/formatter/json_formatter.go b/internal/formatter/json_formatter.go
index b20f30e902..a9c76aa338 100644
--- a/internal/formatter/json_formatter.go
+++ b/internal/formatter/json_formatter.go
@@ -5,13 +5,31 @@
package formatter
import (
+ "bytes"
"encoding/json"
"fmt"
+
+ "github.com/Masterminds/semver/v3"
)
-// JSONFormatter function is responsible for formatting the given JSON input.
-// The function is exposed, so it can be used by other internal packages, e.g. to format sample events in docs.
-func JSONFormatter(content []byte) ([]byte, bool, error) {
+type JSONFormatter interface {
+ Format([]byte) ([]byte, bool, error)
+ Encode(doc any) ([]byte, error)
+}
+
+func JSONFormatterBuilder(specVersion semver.Version) JSONFormatter {
+ if specVersion.LessThan(semver.MustParse("2.12.0")) {
+ return &jsonFormatterWithHTMLEncoding{}
+ }
+
+ return &jsonFormatter{}
+}
+
+// jsonFormatterWithHTMLEncoding function is responsible for formatting the given JSON input.
+// It encodes special HTML characters.
+type jsonFormatterWithHTMLEncoding struct{}
+
+func (jsonFormatterWithHTMLEncoding) Format(content []byte) ([]byte, bool, error) {
var rawMessage json.RawMessage
err := json.Unmarshal(content, &rawMessage)
if err != nil {
@@ -24,3 +42,35 @@ func JSONFormatter(content []byte) ([]byte, bool, error) {
}
return formatted, string(content) == string(formatted), nil
}
+
+func (jsonFormatterWithHTMLEncoding) Encode(doc any) ([]byte, error) {
+ return json.MarshalIndent(doc, "", " ")
+}
+
+// jsonFormatter function is responsible for formatting the given JSON input.
+type jsonFormatter struct{}
+
+func (jsonFormatter) Format(content []byte) ([]byte, bool, error) {
+ var formatted bytes.Buffer
+ err := json.Indent(&formatted, content, "", " ")
+ if err != nil {
+ return nil, false, fmt.Errorf("formatting JSON document failed: %w", err)
+ }
+
+ return formatted.Bytes(), bytes.Equal(content, formatted.Bytes()), nil
+}
+
+func (jsonFormatter) Encode(doc any) ([]byte, error) {
+ var formatted bytes.Buffer
+ enc := json.NewEncoder(&formatted)
+ enc.SetEscapeHTML(false)
+ enc.SetIndent("", " ")
+
+ err := enc.Encode(doc)
+ if err != nil {
+ return nil, err
+ }
+
+ // Trimming to be consistent with MarshalIndent, that seems to trim the result.
+ return bytes.TrimSpace(formatted.Bytes()), nil
+}
diff --git a/internal/formatter/json_formatter_test.go b/internal/formatter/json_formatter_test.go
new file mode 100644
index 0000000000..d71f21a10e
--- /dev/null
+++ b/internal/formatter/json_formatter_test.go
@@ -0,0 +1,106 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package formatter_test
+
+import (
+ "testing"
+
+ "github.com/Masterminds/semver/v3"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+
+ "github.com/elastic/elastic-package/internal/formatter"
+)
+
+func TestJSONFormatterFormat(t *testing.T) {
+ cases := []struct {
+ title string
+ version *semver.Version
+ content string
+ expected string
+ valid bool
+ }{
+ {
+ title: "invalid json 2.0",
+ version: semver.MustParse("2.0.0"),
+ content: `{"foo":}`,
+ valid: false,
+ },
+ {
+ title: "invalid json 3.0",
+ version: semver.MustParse("3.0.0"),
+ content: `{"foo":}`,
+ valid: false,
+ },
+ {
+ title: "encode html in old versions",
+ version: semver.MustParse("2.0.0"),
+ content: `{"a": ""}`,
+ expected: `{
+ "a": "\u003cscript\u003e\u003c/script\u003e"
+}`,
+ valid: true,
+ },
+ {
+ title: "don't encode html since 2.12.0",
+ version: semver.MustParse("2.12.0"),
+ content: `{"a": ""}`,
+ expected: `{
+ "a": ""
+}`,
+ valid: true,
+ },
+ }
+
+ for _, c := range cases {
+ t.Run(c.title, func(t *testing.T) {
+ jsonFormatter := formatter.JSONFormatterBuilder(*c.version)
+ formatted, equal, err := jsonFormatter.Format([]byte(c.content))
+ if !c.valid {
+ assert.Error(t, err)
+ return
+ }
+ require.NoError(t, err)
+
+ assert.Equal(t, c.expected, string(formatted))
+ assert.Equal(t, c.content == c.expected, equal)
+ })
+ }
+}
+
+func TestJSONFormatterEncode(t *testing.T) {
+ cases := []struct {
+ title string
+ version *semver.Version
+ object any
+ expected string
+ }{
+ {
+ title: "encode html in old versions",
+ version: semver.MustParse("2.0.0"),
+ object: map[string]any{"a": ""},
+ expected: `{
+ "a": "\u003cscript\u003e\u003c/script\u003e"
+}`,
+ },
+ {
+ title: "don't encode html since 2.12.0",
+ version: semver.MustParse("2.12.0"),
+ object: map[string]any{"a": ""},
+ expected: `{
+ "a": ""
+}`,
+ },
+ }
+
+ for _, c := range cases {
+ t.Run(c.title, func(t *testing.T) {
+ jsonFormatter := formatter.JSONFormatterBuilder(*c.version)
+ formatted, err := jsonFormatter.Encode(c.object)
+ require.NoError(t, err)
+ assert.Equal(t, c.expected, string(formatted))
+ })
+ }
+}
diff --git a/internal/testrunner/runners/pipeline/runner.go b/internal/testrunner/runners/pipeline/runner.go
index 3c3e721779..b4b64f4740 100644
--- a/internal/testrunner/runners/pipeline/runner.go
+++ b/internal/testrunner/runners/pipeline/runner.go
@@ -14,6 +14,7 @@ import (
"strings"
"time"
+ "github.com/Masterminds/semver/v3"
"gopkg.in/yaml.v3"
"github.com/elastic/elastic-package/internal/common"
@@ -283,10 +284,19 @@ func (r *runner) loadTestCaseFile(testCaseFile string) (*testCase, error) {
func (r *runner) verifyResults(testCaseFile string, config *testConfig, result *testResult, fieldsValidator *fields.Validator) error {
testCasePath := filepath.Join(r.options.TestFolder.Path, testCaseFile)
+ manifest, err := packages.ReadPackageManifestFromPackageRoot(r.options.PackageRootPath)
+ if err != nil {
+ return fmt.Errorf("failed to read package manifest: %w", err)
+ }
+ specVersion, err := semver.NewVersion(manifest.SpecVersion)
+ if err != nil {
+ return fmt.Errorf("failed to parse package format version %q: %w", manifest.SpecVersion, err)
+ }
+
if r.options.GenerateTestResult {
// TODO: Add tests to cover regressive use of json.Unmarshal in writeTestResult.
// See https://github.com/elastic/elastic-package/pull/717.
- err := writeTestResult(testCasePath, result)
+ err := writeTestResult(testCasePath, result, *specVersion)
if err != nil {
return fmt.Errorf("writing test result failed: %w", err)
}
@@ -301,7 +311,7 @@ func (r *runner) verifyResults(testCaseFile string, config *testConfig, result *
if stackConfig.Provider == stack.ProviderServerless {
skipGeoIP = true
}
- err = compareResults(testCasePath, config, result, skipGeoIP)
+ err = compareResults(testCasePath, config, result, skipGeoIP, *specVersion)
if _, ok := err.(testrunner.ErrTestCaseFailed); ok {
return err
}
diff --git a/internal/testrunner/runners/pipeline/test_result.go b/internal/testrunner/runners/pipeline/test_result.go
index f544c9ebd7..5cea3c9181 100644
--- a/internal/testrunner/runners/pipeline/test_result.go
+++ b/internal/testrunner/runners/pipeline/test_result.go
@@ -13,11 +13,12 @@ import (
"os"
"path/filepath"
+ "github.com/Masterminds/semver/v3"
"github.com/google/go-cmp/cmp"
-
"github.com/pmezard/go-difflib/difflib"
"github.com/elastic/elastic-package/internal/common"
+ "github.com/elastic/elastic-package/internal/formatter"
"github.com/elastic/elastic-package/internal/testrunner"
)
@@ -50,11 +51,11 @@ type testResultDefinition struct {
Expected []json.RawMessage `json:"expected"`
}
-func writeTestResult(testCasePath string, result *testResult) error {
+func writeTestResult(testCasePath string, result *testResult, specVersion semver.Version) error {
testCaseDir := filepath.Dir(testCasePath)
testCaseFile := filepath.Base(testCasePath)
- data, err := marshalTestResultDefinition(result)
+ data, err := marshalTestResultDefinition(result, specVersion)
if err != nil {
return fmt.Errorf("marshalling test result failed: %w", err)
}
@@ -65,13 +66,13 @@ func writeTestResult(testCasePath string, result *testResult) error {
return nil
}
-func compareResults(testCasePath string, config *testConfig, result *testResult, skipGeoip bool) error {
+func compareResults(testCasePath string, config *testConfig, result *testResult, skipGeoip bool, specVersion semver.Version) error {
resultsWithoutDynamicFields, err := adjustTestResult(result, config, skipGeoip)
if err != nil {
return fmt.Errorf("can't adjust test results: %w", err)
}
- actual, err := marshalTestResultDefinition(resultsWithoutDynamicFields)
+ actual, err := marshalTestResultDefinition(resultsWithoutDynamicFields, specVersion)
if err != nil {
return fmt.Errorf("marshalling actual test results failed: %w", err)
}
@@ -81,12 +82,12 @@ func compareResults(testCasePath string, config *testConfig, result *testResult,
return fmt.Errorf("reading expected test result failed: %w", err)
}
- expected, err := marshalTestResultDefinition(expectedResults)
+ expected, err := marshalTestResultDefinition(expectedResults, specVersion)
if err != nil {
return fmt.Errorf("marshalling expected test results failed: %w", err)
}
- report, err := diffJson(expected, actual)
+ report, err := diffJson(expected, actual, specVersion)
if err != nil {
return fmt.Errorf("comparing expected test result: %w", err)
}
@@ -123,7 +124,7 @@ func compareJsonNumbers(a, b json.Number) bool {
return false
}
-func diffJson(want, got []byte) (string, error) {
+func diffJson(want, got []byte, specVersion semver.Version) (string, error) {
var gotVal, wantVal interface{}
err := jsonUnmarshalUsingNumber(want, &wantVal)
if err != nil {
@@ -137,11 +138,11 @@ func diffJson(want, got []byte) (string, error) {
return "", nil
}
- got, err = marshalNormalizedJSON(gotVal)
+ got, err = marshalNormalizedJSON(gotVal, specVersion)
if err != nil {
return "", err
}
- want, err = marshalNormalizedJSON(wantVal)
+ want, err = marshalNormalizedJSON(wantVal, specVersion)
if err != nil {
return "", err
}
@@ -259,10 +260,10 @@ func jsonUnmarshalUsingNumber(data []byte, v interface{}) error {
return nil
}
-func marshalTestResultDefinition(result *testResult) ([]byte, error) {
+func marshalTestResultDefinition(result *testResult, specVersion semver.Version) ([]byte, error) {
var trd testResultDefinition
trd.Expected = result.events
- body, err := marshalNormalizedJSON(trd)
+ body, err := marshalNormalizedJSON(trd, specVersion)
if err != nil {
return nil, fmt.Errorf("marshalling test result definition failed: %w", err)
}
@@ -272,17 +273,20 @@ func marshalTestResultDefinition(result *testResult) ([]byte, error) {
// marshalNormalizedJSON marshals test results ensuring that field
// order remains consistent independent of field order returned by
// ES to minimize diff noise during changes.
-func marshalNormalizedJSON(v interface{}) ([]byte, error) {
- msg, err := json.Marshal(v)
+func marshalNormalizedJSON(v interface{}, specVersion semver.Version) ([]byte, error) {
+ jsonFormatter := formatter.JSONFormatterBuilder(specVersion)
+ msg, err := jsonFormatter.Encode(v)
if err != nil {
return msg, err
}
+
var obj interface{}
err = jsonUnmarshalUsingNumber(msg, &obj)
if err != nil {
return msg, err
}
- return json.MarshalIndent(obj, "", " ")
+
+ return jsonFormatter.Encode(obj)
}
func expectedTestResultFile(testFile string) string {
diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go
index e9eaf11e6b..0f5f397a15 100644
--- a/internal/testrunner/runners/system/runner.go
+++ b/internal/testrunner/runners/system/runner.go
@@ -16,6 +16,7 @@ import (
"strings"
"time"
+ "github.com/Masterminds/semver/v3"
"gopkg.in/yaml.v3"
"github.com/elastic/elastic-package/internal/common"
@@ -23,6 +24,7 @@ import (
"github.com/elastic/elastic-package/internal/elasticsearch"
"github.com/elastic/elastic-package/internal/elasticsearch/ingest"
"github.com/elastic/elastic-package/internal/fields"
+ "github.com/elastic/elastic-package/internal/formatter"
"github.com/elastic/elastic-package/internal/kibana"
"github.com/elastic/elastic-package/internal/logger"
"github.com/elastic/elastic-package/internal/multierror"
@@ -720,8 +722,13 @@ func (r *runner) runTest(config *testConfig, ctxt servicedeployer.ServiceContext
}
}
+ specVersion, err := semver.NewVersion(pkgManifest.SpecVersion)
+ if err != nil {
+ return result.WithError(fmt.Errorf("failed to parse format version %q: %w", pkgManifest.SpecVersion, err))
+ }
+
// Write sample events file from first doc, if requested
- if err := r.generateTestResult(docs); err != nil {
+ if err := r.generateTestResult(docs, *specVersion); err != nil {
return result.WithError(err)
}
@@ -1184,8 +1191,9 @@ func filterAgents(allAgents []kibana.Agent, ctx servicedeployer.ServiceContext)
return filtered
}
-func writeSampleEvent(path string, doc common.MapStr) error {
- body, err := json.MarshalIndent(doc, "", " ")
+func writeSampleEvent(path string, doc common.MapStr, specVersion semver.Version) error {
+ jsonFormatter := formatter.JSONFormatterBuilder(specVersion)
+ body, err := jsonFormatter.Encode(doc)
if err != nil {
return fmt.Errorf("marshalling sample event failed: %w", err)
}
@@ -1250,7 +1258,7 @@ func (r *runner) selectVariants(variantsFile *servicedeployer.VariantsFile) []st
return variantNames
}
-func (r *runner) generateTestResult(docs []common.MapStr) error {
+func (r *runner) generateTestResult(docs []common.MapStr, specVersion semver.Version) error {
if !r.options.GenerateTestResult {
return nil
}
@@ -1260,7 +1268,7 @@ func (r *runner) generateTestResult(docs []common.MapStr) error {
rootPath = filepath.Join(rootPath, "data_stream", ds)
}
- if err := writeSampleEvent(rootPath, docs[0]); err != nil {
+ if err := writeSampleEvent(rootPath, docs[0], specVersion); err != nil {
return fmt.Errorf("failed to write sample event file: %w", err)
}