From f5d962027e9a16b50d136c0dfe864b5e678f153d Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 18 May 2023 15:31:50 +0200 Subject: [PATCH 01/34] Add fields into the search response --- internal/testrunner/runners/system/runner.go | 26 +++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index ba019459c3..b7b9fd5758 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -5,8 +5,10 @@ package system import ( + "bytes" "encoding/json" "fmt" + "io" "math/rand" "os" "path/filepath" @@ -228,11 +230,23 @@ func createTestRunID() string { return fmt.Sprintf("%d", rand.Intn(testRunMaxID-testRunMinID)+testRunMinID) } +func buildAllFieldsBody() io.Reader { + var b strings.Builder + b.WriteString("\n") + b.WriteString(`{ + "fields": ["*"] + }`) + b.WriteString("\n") + return strings.NewReader(b.String()) +} + func (r *runner) getDocs(dataStream string) ([]common.MapStr, error) { resp, err := r.options.API.Search( r.options.API.Search.WithIndex(dataStream), r.options.API.Search.WithSort("@timestamp:asc"), r.options.API.Search.WithSize(elasticsearchQuerySize), + r.options.API.Search.WithSource("true"), + r.options.API.Search.WithBody(buildAllFieldsBody()), ) if err != nil { return nil, errors.Wrap(err, "could not search data stream") @@ -246,6 +260,7 @@ func (r *runner) getDocs(dataStream string) ([]common.MapStr, error) { } Hits []struct { Source common.MapStr `json:"_source"` + Fields common.MapStr `json:"fields"` } } Error *struct { @@ -255,7 +270,13 @@ func (r *runner) getDocs(dataStream string) ([]common.MapStr, error) { Status int } - if err := json.NewDecoder(resp.Body).Decode(&results); err != nil { + b, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + logger.Debugf("\n\n\n>>> Response:\n%s\n\n\n", string(b)) + + if err := json.NewDecoder(bytes.NewReader(b)).Decode(&results); err != nil { return nil, errors.Wrap(err, "could not decode search results response") } @@ -269,7 +290,10 @@ func (r *runner) getDocs(dataStream string) ([]common.MapStr, error) { var docs []common.MapStr for _, hit := range results.Hits.Hits { + // docs = append(docs, hit.Fields) docs = append(docs, hit.Source) + logger.Debugf("Checking hit fields:\n%s", hit.Fields) + logger.Debugf("Checking hit _source:\n%s", hit.Source) } return docs, nil From 9d5a6c56f42ad49939a72a1ca0edb8f7805aae38 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Mon, 22 May 2023 12:40:22 +0200 Subject: [PATCH 02/34] Test with fields --- internal/testrunner/runners/system/runner.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index b7b9fd5758..69dd3cbfeb 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -290,8 +290,8 @@ func (r *runner) getDocs(dataStream string) ([]common.MapStr, error) { var docs []common.MapStr for _, hit := range results.Hits.Hits { - // docs = append(docs, hit.Fields) - docs = append(docs, hit.Source) + docs = append(docs, hit.Fields) + // docs = append(docs, hit.Source) logger.Debugf("Checking hit fields:\n%s", hit.Fields) logger.Debugf("Checking hit _source:\n%s", hit.Source) } From fd4074b905aba7bf5c3977e0cef29d62a71667cc Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 30 May 2023 12:01:17 +0200 Subject: [PATCH 03/34] Request whether or not synthetics mode is enabled --- internal/testrunner/runners/system/runner.go | 106 +++++++++++++++++-- 1 file changed, 95 insertions(+), 11 deletions(-) diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index 69dd3cbfeb..6bf8fc3d71 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -240,7 +240,77 @@ func buildAllFieldsBody() io.Reader { return strings.NewReader(b.String()) } -func (r *runner) getDocs(dataStream string) ([]common.MapStr, error) { +func (r *runner) isSyntheticsEnabled(dataStream, componentTemplatePackage string) (bool, error) { + // resp, err := r.options.API.Indices.GetSettings( + // r.options.API.Indices.GetSettings.WithIndex(dataStream), + // r.options.API.Indices.GetSettings.WithFlatSettings(true), + // ) + // if err != nil { + // return false, fmt.Errorf("could not get mappings from data stream %s: %w", err) + // } + // defer resp.Body.Close() + + // fmt.Println("GetSettings >") + // fmt.Println(resp, err) + + // resp1, err := r.options.API.Indices.GetMapping( + // r.options.API.Indices.GetMapping.WithIndex(dataStream), + // ) + // if err != nil { + // return false, fmt.Errorf("could not get mappings from data stream %s: %w", err) + // } + // defer resp1.Body.Close() + // fmt.Println("GetMapping >") + // fmt.Println(resp1, err) + + resp, err := r.options.API.Cluster.GetComponentTemplate( + r.options.API.Cluster.GetComponentTemplate.WithName(componentTemplatePackage), + ) + if err != nil { + return false, fmt.Errorf("could not get component template from data stream %s: %w", err) + } + defer resp.Body.Close() + fmt.Println("GetComponentTemplate>") + fmt.Println(resp, err) + + var results struct { + ComponentTemplates []struct { + Name string `json:"name"` + ComponentTemplate struct { + Template struct { + Mappings struct { + Source *struct { + Mode string `json:"mode"` + } `json:"_source,omitempty"` + } `json:"mappings"` + } `json:"template"` + } `json:"component_template"` + } `json:"component_templates"` + } + b, err := io.ReadAll(resp.Body) + if err != nil { + return false, err + } + logger.Debugf("\n\n\n>>> Response:\n%s\n\n\n", string(b)) + + if err := json.NewDecoder(bytes.NewReader(b)).Decode(&results); err != nil { + return false, fmt.Errorf("could not decode search results response: %w", err) + } + + if len(results.ComponentTemplates) != 1 { + return false, fmt.Errorf("unexpected response, not found component template") + } + + template := results.ComponentTemplates[0] + + if template.ComponentTemplate.Template.Mappings.Source == nil { + return false, nil + } + + return template.ComponentTemplate.Template.Mappings.Source.Mode == "synthetic", nil +} + +func (r *runner) getDocs(dataStream string) ([]common.MapStr, []common.MapStr, error) { resp, err := r.options.API.Search( r.options.API.Search.WithIndex(dataStream), r.options.API.Search.WithSort("@timestamp:asc"), @@ -249,7 +319,7 @@ func (r *runner) getDocs(dataStream string) ([]common.MapStr, error) { r.options.API.Search.WithBody(buildAllFieldsBody()), ) if err != nil { - return nil, errors.Wrap(err, "could not search data stream") + return nil, nil, errors.Wrap(err, "could not search data stream") } defer resp.Body.Close() @@ -272,12 +342,12 @@ func (r *runner) getDocs(dataStream string) ([]common.MapStr, error) { b, err := io.ReadAll(resp.Body) if err != nil { - return nil, err + return nil, nil, err } logger.Debugf("\n\n\n>>> Response:\n%s\n\n\n", string(b)) if err := json.NewDecoder(bytes.NewReader(b)).Decode(&results); err != nil { - return nil, errors.Wrap(err, "could not decode search results response") + return nil, nil, errors.Wrap(err, "could not decode search results response") } numHits := results.Hits.Total.Value @@ -288,15 +358,15 @@ func (r *runner) getDocs(dataStream string) ([]common.MapStr, error) { logger.Debugf("found %d hits in %s data stream", numHits, dataStream) } - var docs []common.MapStr + var docsSource, docsFields []common.MapStr for _, hit := range results.Hits.Hits { - docs = append(docs, hit.Fields) - // docs = append(docs, hit.Source) + docsSource = append(docsSource, hit.Source) + docsFields = append(docsFields, hit.Fields) logger.Debugf("Checking hit fields:\n%s", hit.Fields) logger.Debugf("Checking hit _source:\n%s", hit.Source) } - return docs, nil + return docsSource, docsFields, nil } func (r *runner) runTest(config *testConfig, ctxt servicedeployer.ServiceContext, serviceOptions servicedeployer.FactoryOptions) ([]testrunner.TestResult, error) { @@ -395,6 +465,12 @@ func (r *runner) runTest(config *testConfig, ctxt servicedeployer.ServiceContext ds.Namespace, ) + componentTemplatePackage := fmt.Sprintf( + "%s-%s@package", + ds.Inputs[0].Streams[0].DataStream.Type, + ds.Inputs[0].Streams[0].DataStream.Dataset, + ) + r.wipeDataStreamHandler = func() error { logger.Debugf("deleting data in data stream...") if err := deleteDataStreamDocs(r.options.API, dataStream); err != nil { @@ -412,7 +488,7 @@ func (r *runner) runTest(config *testConfig, ctxt servicedeployer.ServiceContext return true, errors.New("SIGINT: cancel clearing data") } - docs, err := r.getDocs(dataStream) + docs, _, err := r.getDocs(dataStream) return len(docs) == 0, err }, 2*time.Minute) if err != nil || !cleared { @@ -466,14 +542,14 @@ func (r *runner) runTest(config *testConfig, ctxt servicedeployer.ServiceContext // (TODO in future) Optionally exercise service to generate load. logger.Debug("checking for expected data in data stream...") - var docs []common.MapStr + var docs, docsFields []common.MapStr passed, err := waitUntilTrue(func() (bool, error) { if signal.SIGINT() { return true, errors.New("SIGINT: cancel waiting for policy assigned") } var err error - docs, err = r.getDocs(dataStream) + docs, docsFields, err = r.getDocs(dataStream) if config.Assert.HitCount > 0 { return len(docs) >= config.Assert.HitCount, err @@ -490,6 +566,14 @@ func (r *runner) runTest(config *testConfig, ctxt servicedeployer.ServiceContext return result.WithError(fmt.Errorf("%s", result.FailureMsg)) } + logger.Debugf("docs fields number: %d", len(docsFields)) + logger.Debug("check whether or not synthetics is enabled...") + syntheticEnabled, err := r.isSyntheticsEnabled(dataStream, componentTemplatePackage) + if err != nil { + return result.WithError(err) + } + logger.Debugf("data stream %s has synthetics enabled: %t", dataStream, syntheticEnabled) + // Validate fields in docs var expectedDataset string if ds := r.options.TestFolder.DataStream; ds != "" { From 2c41de38e6b0ee7d720a3ae88c83d44a66c75809 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 30 May 2023 12:09:24 +0200 Subject: [PATCH 04/34] Add option to disable normalization if synthetics enabled --- internal/fields/validate.go | 20 ++++++++++++++++---- internal/testrunner/runners/system/runner.go | 1 + 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/internal/fields/validate.go b/internal/fields/validate.go index 76eb564a3c..0100cdbe5a 100644 --- a/internal/fields/validate.go +++ b/internal/fields/validate.go @@ -55,6 +55,8 @@ type Validator struct { allowedCIDRs []*net.IPNet enabledImportAllECSSchema bool + + disabledNormalization bool } // ValidatorOption represents an optional flag that can be passed to CreateValidatorForDirectory. @@ -124,6 +126,14 @@ func WithEnabledImportAllECSSChema(importSchema bool) ValidatorOption { } } +// WithDisableNormalization configures the validator to disable normalization. +func WithDisableNormalization(disabledNormalization bool) ValidatorOption { + return func(v *Validator) error { + v.disabledNormalization = disabledNormalization + return nil + } +} + type packageRootFinder interface { FindPackageRoot() (string, bool, error) } @@ -345,12 +355,14 @@ func (v *Validator) validateScalarElement(key string, val interface{}, doc commo val = fmt.Sprintf("%q", val) } - err := v.validateExpectedNormalization(*definition, val) - if err != nil { - return errors.Wrapf(err, "field %q is not normalized as expected", key) + if !v.disabledNormalization { + err := v.validateExpectedNormalization(*definition, val) + if err != nil { + return errors.Wrapf(err, "field %q is not normalized as expected", key) + } } - err = v.parseElementValue(key, *definition, val, doc) + err := v.parseElementValue(key, *definition, val, doc) if err != nil { return errors.Wrap(err, "parsing field value failed") } diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index 6bf8fc3d71..95fb4048d8 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -586,6 +586,7 @@ func (r *runner) runTest(config *testConfig, ctxt servicedeployer.ServiceContext fields.WithNumericKeywordFields(config.NumericKeywordFields), fields.WithExpectedDataset(expectedDataset), fields.WithEnabledImportAllECSSChema(true), + fields.WithDisableNormalization(syntheticEnabled), ) if err != nil { return result.WithError(errors.Wrapf(err, "creating fields validator for data stream failed (path: %s)", serviceOptions.DataStreamRootPath)) From 59b12526744b764cb3359f622a2d82333aa9abd9 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 30 May 2023 13:03:16 +0200 Subject: [PATCH 05/34] add struct for hits source and fields --- internal/testrunner/runners/system/runner.go | 60 +++++++++++++------- 1 file changed, 38 insertions(+), 22 deletions(-) diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index 95fb4048d8..51b6c71c6f 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -310,7 +310,23 @@ func (r *runner) isSyntheticsEnabled(dataStream, componentTemplatePackage string return template.ComponentTemplate.Template.Mappings.Source.Mode == "synthetic", nil } -func (r *runner) getDocs(dataStream string) ([]common.MapStr, []common.MapStr, error) { +type hits struct { + Source []common.MapStr `json:"_source"` + Fields []common.MapStr `json:"fields"` +} + +func (h hits) getDocs(syntheticsEnabled bool) []common.MapStr { + if syntheticsEnabled { + return h.Fields + } + return h.Source +} + +func (h hits) size() int { + return len(h.Source) +} + +func (r *runner) getDocs(dataStream string) (*hits, error) { resp, err := r.options.API.Search( r.options.API.Search.WithIndex(dataStream), r.options.API.Search.WithSort("@timestamp:asc"), @@ -319,7 +335,7 @@ func (r *runner) getDocs(dataStream string) ([]common.MapStr, []common.MapStr, e r.options.API.Search.WithBody(buildAllFieldsBody()), ) if err != nil { - return nil, nil, errors.Wrap(err, "could not search data stream") + return nil, errors.Wrap(err, "could not search data stream") } defer resp.Body.Close() @@ -340,14 +356,15 @@ func (r *runner) getDocs(dataStream string) ([]common.MapStr, []common.MapStr, e Status int } - b, err := io.ReadAll(resp.Body) - if err != nil { - return nil, nil, err - } - logger.Debugf("\n\n\n>>> Response:\n%s\n\n\n", string(b)) + // b, err := io.ReadAll(resp.Body) + // if err != nil { + // return nil, err + // } + // logger.Debugf("\n\n\n>>> Response:\n%s\n\n\n", string(b)) - if err := json.NewDecoder(bytes.NewReader(b)).Decode(&results); err != nil { - return nil, nil, errors.Wrap(err, "could not decode search results response") + // if err := json.NewDecoder(bytes.NewReader(b)).Decode(&results); err != nil { + if err := json.NewDecoder(resp.Body).Decode(&results); err != nil { + return nil, errors.Wrap(err, "could not decode search results response") } numHits := results.Hits.Total.Value @@ -358,15 +375,13 @@ func (r *runner) getDocs(dataStream string) ([]common.MapStr, []common.MapStr, e logger.Debugf("found %d hits in %s data stream", numHits, dataStream) } - var docsSource, docsFields []common.MapStr + var hits hits for _, hit := range results.Hits.Hits { - docsSource = append(docsSource, hit.Source) - docsFields = append(docsFields, hit.Fields) - logger.Debugf("Checking hit fields:\n%s", hit.Fields) - logger.Debugf("Checking hit _source:\n%s", hit.Source) + hits.Source = append(hits.Source, hit.Source) + hits.Fields = append(hits.Fields, hit.Fields) } - return docsSource, docsFields, nil + return &hits, nil } func (r *runner) runTest(config *testConfig, ctxt servicedeployer.ServiceContext, serviceOptions servicedeployer.FactoryOptions) ([]testrunner.TestResult, error) { @@ -488,8 +503,8 @@ func (r *runner) runTest(config *testConfig, ctxt servicedeployer.ServiceContext return true, errors.New("SIGINT: cancel clearing data") } - docs, _, err := r.getDocs(dataStream) - return len(docs) == 0, err + hits, err := r.getDocs(dataStream) + return hits.size() == 0, err }, 2*time.Minute) if err != nil || !cleared { if err == nil { @@ -542,20 +557,20 @@ func (r *runner) runTest(config *testConfig, ctxt servicedeployer.ServiceContext // (TODO in future) Optionally exercise service to generate load. logger.Debug("checking for expected data in data stream...") - var docs, docsFields []common.MapStr + var hits *hits passed, err := waitUntilTrue(func() (bool, error) { if signal.SIGINT() { return true, errors.New("SIGINT: cancel waiting for policy assigned") } var err error - docs, docsFields, err = r.getDocs(dataStream) + hits, err = r.getDocs(dataStream) if config.Assert.HitCount > 0 { - return len(docs) >= config.Assert.HitCount, err + return hits.size() >= config.Assert.HitCount, err } - return len(docs) > 0, err + return hits.size() > 0, err }, waitForDataTimeout) if err != nil { return result.WithError(err) @@ -566,7 +581,6 @@ func (r *runner) runTest(config *testConfig, ctxt servicedeployer.ServiceContext return result.WithError(fmt.Errorf("%s", result.FailureMsg)) } - logger.Debugf("docs fields number: %d", len(docsFields)) logger.Debug("check whether or not synthetics is enabled...") syntheticEnabled, err := r.isSyntheticsEnabled(dataStream, componentTemplatePackage) if err != nil { @@ -574,6 +588,8 @@ func (r *runner) runTest(config *testConfig, ctxt servicedeployer.ServiceContext } logger.Debugf("data stream %s has synthetics enabled: %t", dataStream, syntheticEnabled) + docs := hits.getDocs(syntheticEnabled) + // Validate fields in docs var expectedDataset string if ds := r.options.TestFolder.DataStream; ds != "" { From abfd2669d7792c99d58ef2a9e383e5e29bc2c984 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 30 May 2023 16:48:55 +0200 Subject: [PATCH 06/34] Add some tests for mapstr --- internal/common/mapstr_test.go | 60 ++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 internal/common/mapstr_test.go diff --git a/internal/common/mapstr_test.go b/internal/common/mapstr_test.go new file mode 100644 index 0000000000..7c35126aea --- /dev/null +++ b/internal/common/mapstr_test.go @@ -0,0 +1,60 @@ +package common + +import ( + "encoding/json" + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestMapStrGetValue(t *testing.T) { + + cases := []struct { + title string + testFile string + fieldKey string + expectedValue interface{} + }{ + { + title: "string value", + testFile: "./testdata/source.json", + fieldKey: "host.architecture", + expectedValue: "x86_64", + }, + { + title: "float64 value", + testFile: "./testdata/source.json", + fieldKey: "metricset.period", + expectedValue: float64(10000), + }, + { + title: "slice value", + testFile: "./testdata/source.json", + fieldKey: "tags", + expectedValue: []interface{}{"apache_tomcat-cache", "forwarded"}, + }, + { + title: "map value", + testFile: "./testdata/source.json", + fieldKey: "data_stream", + expectedValue: map[string]interface{}{"dataset": "apache_tomcat.cache", "namespace": "ep", "type": "metrics"}, + }, + } + + for _, c := range cases { + t.Run(c.title, func(t *testing.T) { + b, err := os.ReadFile("./testdata/source.json") + require.NoError(t, err) + + var given MapStr + err = json.Unmarshal(b, &given) + require.NoError(t, err) + + val, err := given.GetValue(c.fieldKey) + assert.NoError(t, err) + assert.Equal(t, c.expectedValue, val) + }) + } +} From 6e5bd94beef842cc268427c8a886fe77fe5d8fd6 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 30 May 2023 17:07:30 +0200 Subject: [PATCH 07/34] Add helper to get string slice --- internal/common/helpers.go | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/internal/common/helpers.go b/internal/common/helpers.go index 023d527cd2..218b5e712f 100644 --- a/internal/common/helpers.go +++ b/internal/common/helpers.go @@ -4,7 +4,10 @@ package common -import "strings" +import ( + "fmt" + "strings" +) // TrimStringSlice removes whitespace from the beginning and end of the contents of a []string. func TrimStringSlice(slice []string) { @@ -36,3 +39,21 @@ func StringSlicesUnion(slices ...[]string) (result []string) { } return } + +// ToStringSlice returns the list of strings from an interface variable +func ToStringSlice(val interface{}) ([]string, error) { + vals, ok := val.([]interface{}) + if !ok { + return nil, fmt.Errorf("conversion error") + } + + var s []string + for _, v := range vals { + str, ok := v.(string) + if !ok { + return nil, fmt.Errorf("conversion error") + } + s = append(s, str) + } + return s, nil +} From d3d7177f00d5698983d6d51b077fc3aafbc14318 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 30 May 2023 17:07:58 +0200 Subject: [PATCH 08/34] Remove debug --- internal/testrunner/runners/system/runner.go | 41 +------------------- 1 file changed, 2 insertions(+), 39 deletions(-) diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index 51b6c71c6f..3f9b1b2387 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -5,7 +5,6 @@ package system import ( - "bytes" "encoding/json" "fmt" "io" @@ -241,37 +240,13 @@ func buildAllFieldsBody() io.Reader { } func (r *runner) isSyntheticsEnabled(dataStream, componentTemplatePackage string) (bool, error) { - // resp, err := r.options.API.Indices.GetSettings( - // r.options.API.Indices.GetSettings.WithIndex(dataStream), - // r.options.API.Indices.GetSettings.WithFlatSettings(true), - // ) - // if err != nil { - // return false, fmt.Errorf("could not get mappings from data stream %s: %w", err) - // } - // defer resp.Body.Close() - - // fmt.Println("GetSettings >") - // fmt.Println(resp, err) - - // resp1, err := r.options.API.Indices.GetMapping( - // r.options.API.Indices.GetMapping.WithIndex(dataStream), - // ) - // if err != nil { - // return false, fmt.Errorf("could not get mappings from data stream %s: %w", err) - // } - // defer resp1.Body.Close() - // fmt.Println("GetMapping >") - // fmt.Println(resp1, err) - resp, err := r.options.API.Cluster.GetComponentTemplate( r.options.API.Cluster.GetComponentTemplate.WithName(componentTemplatePackage), ) if err != nil { - return false, fmt.Errorf("could not get component template from data stream %s: %w", err) + return false, fmt.Errorf("could not get component template from data stream %s: %w", dataStream, err) } defer resp.Body.Close() - fmt.Println("GetComponentTemplate>") - fmt.Println(resp, err) var results struct { ComponentTemplates []struct { @@ -287,13 +262,8 @@ func (r *runner) isSyntheticsEnabled(dataStream, componentTemplatePackage string } `json:"component_template"` } `json:"component_templates"` } - b, err := io.ReadAll(resp.Body) - if err != nil { - return false, err - } - logger.Debugf("\n\n\n>>> Response:\n%s\n\n\n", string(b)) - if err := json.NewDecoder(bytes.NewReader(b)).Decode(&results); err != nil { + if err := json.NewDecoder(resp.Body).Decode(&results); err != nil { return false, fmt.Errorf("could not decode search results response: %w", err) } @@ -356,13 +326,6 @@ func (r *runner) getDocs(dataStream string) (*hits, error) { Status int } - // b, err := io.ReadAll(resp.Body) - // if err != nil { - // return nil, err - // } - // logger.Debugf("\n\n\n>>> Response:\n%s\n\n\n", string(b)) - - // if err := json.NewDecoder(bytes.NewReader(b)).Decode(&results); err != nil { if err := json.NewDecoder(resp.Body).Decode(&results); err != nil { return nil, errors.Wrap(err, "could not decode search results response") } From f52a36d38de03165f370eaf124f8b7c1350ad216 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 30 May 2023 17:08:19 +0200 Subject: [PATCH 09/34] check dataset value if disabled normalization --- internal/fields/validate.go | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/internal/fields/validate.go b/internal/fields/validate.go index 0100cdbe5a..13468c5564 100644 --- a/internal/fields/validate.go +++ b/internal/fields/validate.go @@ -283,7 +283,24 @@ func (v *Validator) validateDocumentValues(body common.MapStr) multierror.Error if err == common.ErrKeyNotFound { continue } - str, ok := value.(string) + + var str string + var ok bool + if v.disabledNormalization { + ok = true + vals, err := common.ToStringSlice(value) + if err != nil { + ok = false + } + if err == nil && len(vals) != 1 { + ok = false + } + if err == nil && len(vals) != 1 { + str = vals[0] + } + } else { + str, ok = value.(string) + } if !ok || str != v.expectedDataset { err := errors.Errorf("field %q should have value %q, it has \"%v\"", datasetField, v.expectedDataset, value) From 5cbf156e9d5b6303c50ab21e0752c954085a6055 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 30 May 2023 17:17:34 +0200 Subject: [PATCH 10/34] Remove usages of errors.Wrap --- internal/fields/validate.go | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/internal/fields/validate.go b/internal/fields/validate.go index 13468c5564..6aeb774733 100644 --- a/internal/fields/validate.go +++ b/internal/fields/validate.go @@ -16,7 +16,6 @@ import ( "strings" "github.com/Masterminds/semver/v3" - "github.com/pkg/errors" "gopkg.in/yaml.v3" "github.com/elastic/elastic-package/internal/common" @@ -163,7 +162,7 @@ func createValidatorForDirectoryAndPackageRoot(fieldsParentDir string, finder pa fieldsDir := filepath.Join(fieldsParentDir, "fields") v.Schema, err = loadFieldsFromDir(fieldsDir) if err != nil { - return nil, errors.Wrapf(err, "can't load fields from directory (path: %s)", fieldsDir) + return nil, fmt.Errorf("can't load fields from directory (path: %s): %w", fieldsDir, err) } if v.disabledDependencyManagement { @@ -172,7 +171,7 @@ func createValidatorForDirectoryAndPackageRoot(fieldsParentDir string, finder pa packageRoot, found, err := finder.FindPackageRoot() if err != nil { - return nil, errors.Wrap(err, "can't find package root") + return nil, fmt.Errorf("can't find package root: %w", err) } // As every command starts with approximating where is the package root, it isn't required to return an error in case the root is missing. // This is also useful for testing purposes, where we don't have a real package, but just "fields" directory. The package root is always absent. @@ -184,7 +183,7 @@ func createValidatorForDirectoryAndPackageRoot(fieldsParentDir string, finder pa bm, ok, err := buildmanifest.ReadBuildManifest(packageRoot) if err != nil { - return nil, errors.Wrap(err, "can't read build manifest") + return nil, fmt.Errorf("can't read build manifest: %w", err) } if !ok { v.disabledDependencyManagement = true @@ -193,7 +192,7 @@ func createValidatorForDirectoryAndPackageRoot(fieldsParentDir string, finder pa fdm, err := CreateFieldDependencyManager(bm.Dependencies) if err != nil { - return nil, errors.Wrap(err, "can't create field dependency manager") + return nil, fmt.Errorf("can't create field dependency manager: %w", err) } v.FieldDependencyManager = fdm @@ -227,20 +226,20 @@ func initializeAllowedCIDRsList() (cidrs []*net.IPNet) { func loadFieldsFromDir(fieldsDir string) ([]FieldDefinition, error) { files, err := filepath.Glob(filepath.Join(fieldsDir, "*.yml")) if err != nil { - return nil, errors.Wrapf(err, "reading directory with fields failed (path: %s)", fieldsDir) + return nil, fmt.Errorf("reading directory with fields failed (path: %s): %w", fieldsDir, err) } var fields []FieldDefinition for _, file := range files { body, err := os.ReadFile(file) if err != nil { - return nil, errors.Wrap(err, "reading fields file failed") + return nil, fmt.Errorf("reading fields file failed: %w", err) } var u []FieldDefinition err = yaml.Unmarshal(body, &u) if err != nil { - return nil, errors.Wrap(err, "unmarshalling field body failed") + return nil, fmt.Errorf("unmarshalling field body failed: %w", err) } fields = append(fields, u...) } @@ -253,7 +252,7 @@ func (v *Validator) ValidateDocumentBody(body json.RawMessage) multierror.Error err := json.Unmarshal(body, &c) if err != nil { var errs multierror.Error - errs = append(errs, errors.Wrap(err, "unmarshalling document body failed")) + errs = append(errs, fmt.Errorf("unmarshalling document body failed: %w", err)) return errs } @@ -302,7 +301,7 @@ func (v *Validator) validateDocumentValues(body common.MapStr) multierror.Error str, ok = value.(string) } if !ok || str != v.expectedDataset { - err := errors.Errorf("field %q should have value %q, it has \"%v\"", + err := fmt.Errorf("field %q should have value %q, it has \"%v\"", datasetField, v.expectedDataset, value) errs = append(errs, err) } @@ -361,7 +360,7 @@ func (v *Validator) validateScalarElement(key string, val interface{}, doc commo if !v.disabledDependencyManagement && definition.External != "" { def, err := v.FieldDependencyManager.ImportField(definition.External, key) if err != nil { - return errors.Wrapf(err, "can't import field (field: %s)", key) + return fmt.Errorf("can't import field (field: %s): %w", key, err) } definition = &def } @@ -375,13 +374,13 @@ func (v *Validator) validateScalarElement(key string, val interface{}, doc commo if !v.disabledNormalization { err := v.validateExpectedNormalization(*definition, val) if err != nil { - return errors.Wrapf(err, "field %q is not normalized as expected", key) + return fmt.Errorf("field %q is not normalized as expected: %w", key, err) } } err := v.parseElementValue(key, *definition, val, doc) if err != nil { - return errors.Wrap(err, "parsing field value failed") + return fmt.Errorf("parsing field value failed: %w", err) } return nil } @@ -712,7 +711,7 @@ func ensurePatternMatches(key, value, pattern string) error { } valid, err := regexp.MatchString(pattern, value) if err != nil { - return errors.Wrap(err, "invalid pattern") + return fmt.Errorf("invalid pattern: %w", err) } if !valid { return fmt.Errorf("field %q's value, %s, does not match the expected pattern: %s", key, value, pattern) From abb0666af872267cc06bde573e9aa89a21b007f2 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 30 May 2023 17:32:27 +0200 Subject: [PATCH 11/34] Fix condition --- internal/fields/validate.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/internal/fields/validate.go b/internal/fields/validate.go index 6aeb774733..d7b8c41d89 100644 --- a/internal/fields/validate.go +++ b/internal/fields/validate.go @@ -286,6 +286,8 @@ func (v *Validator) validateDocumentValues(body common.MapStr) multierror.Error var str string var ok bool if v.disabledNormalization { + // when synthetics mode is enabled, each field present in the document is an array + // so this check needs to retrieve the first element of the array ok = true vals, err := common.ToStringSlice(value) if err != nil { @@ -294,7 +296,7 @@ func (v *Validator) validateDocumentValues(body common.MapStr) multierror.Error if err == nil && len(vals) != 1 { ok = false } - if err == nil && len(vals) != 1 { + if err == nil && len(vals) == 1 { str = vals[0] } } else { From 3a4facac3c2817a1a993e1343946e765c4de49e0 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 30 May 2023 17:36:31 +0200 Subject: [PATCH 12/34] Add license header --- internal/common/mapstr_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/internal/common/mapstr_test.go b/internal/common/mapstr_test.go index 7c35126aea..c61372a296 100644 --- a/internal/common/mapstr_test.go +++ b/internal/common/mapstr_test.go @@ -1,3 +1,7 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + package common import ( From 5cec1e62190710042a2fe960604e5b7c41ce253b Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 30 May 2023 17:43:05 +0200 Subject: [PATCH 13/34] Add testdata folder --- internal/common/testdata/fields.json | 150 +++++++++++++++++++++++++++ internal/common/testdata/source.json | 92 ++++++++++++++++ 2 files changed, 242 insertions(+) create mode 100644 internal/common/testdata/fields.json create mode 100644 internal/common/testdata/source.json diff --git a/internal/common/testdata/fields.json b/internal/common/testdata/fields.json new file mode 100644 index 0000000000..9d5a8ad81a --- /dev/null +++ b/internal/common/testdata/fields.json @@ -0,0 +1,150 @@ +{ + "elastic_agent.version": [ + "8.8.0" + ], + "event.category": [ + "web" + ], + "host.os.name.text": [ + "Ubuntu" + ], + "host.hostname": [ + "docker-fleet-agent" + ], + "host.mac": [ + "02-42-AC-17-00-07" + ], + "service.type": [ + "prometheus" + ], + "apache_tomcat.cache.application_name": [ + "/" + ], + "host.os.version": [ + "20.04.6 LTS (Focal Fossa)" + ], + "host.os.name": [ + "Ubuntu" + ], + "agent.name": [ + "docker-fleet-agent" + ], + "host.name": [ + "docker-fleet-agent" + ], + "event.agent_id_status": [ + "verified" + ], + "metricset.name.text": [ + "collector" + ], + "event.kind": [ + "metric" + ], + "host.os.type": [ + "linux" + ], + "apache_tomcat.cache.object.size.max.kb": [ + 512.0 + ], + "apache_tomcat.cache.size.max.kb": [ + 10240.0 + ], + "data_stream.type": [ + "metrics" + ], + "tags": [ + "apache_tomcat-cache", + "forwarded" + ], + "host.architecture": [ + "x86_64" + ], + "agent.id": [ + "5cad1bda-ddc8-4333-8d88-333b6ef311b5" + ], + "ecs.version": [ + "8.7.0" + ], + "host.containerized": [ + false + ], + "service.address": [ + "http://elastic-package-service-apache_tomcat-1:9090/metrics" + ], + "agent.version": [ + "8.8.0" + ], + "host.os.family": [ + "debian" + ], + "apache_tomcat.cache.size.current.kb": [ + 19.0 + ], + "apache_tomcat.cache.lookup.count": [ + 30.0 + ], + "host.ip": [ + "172.23.0.7" + ], + "agent.type": [ + "metricbeat" + ], + "event.module": [ + "apache_tomcat" + ], + "host.os.kernel": [ + "5.19.0-42-generic" + ], + "elastic_agent.snapshot": [ + true + ], + "host.id": [ + "4e0d02de87bd4101b7215d3aaf5f623b" + ], + "elastic_agent.id": [ + "5cad1bda-ddc8-4333-8d88-333b6ef311b5" + ], + "data_stream.namespace": [ + "ep" + ], + "metricset.period": [ + 10000 + ], + "host.os.codename": [ + "focal" + ], + "apache_tomcat.cache.ttl.ms": [ + 5000.0 + ], + "metricset.name": [ + "collector" + ], + "event.duration": [ + 136411023 + ], + "event.ingested": [ + "2023-05-30T11:42:09Z" + ], + "@timestamp": [ + "2023-05-30T11:42:07.923Z" + ], + "host.os.platform": [ + "ubuntu" + ], + "data_stream.dataset": [ + "apache_tomcat.cache" + ], + "event.type": [ + "info" + ], + "agent.ephemeral_id": [ + "7dd994f8-d866-4068-9f29-d6c90a28738e" + ], + "apache_tomcat.cache.hit.count": [ + 15.0 + ], + "event.dataset": [ + "apache_tomcat.cache" + ] +} \ No newline at end of file diff --git a/internal/common/testdata/source.json b/internal/common/testdata/source.json new file mode 100644 index 0000000000..598d3ef705 --- /dev/null +++ b/internal/common/testdata/source.json @@ -0,0 +1,92 @@ +{ + "@timestamp": "2023-05-30T11:42:07.923Z", + "agent": { + "ephemeral_id": "7dd994f8-d866-4068-9f29-d6c90a28738e", + "id": "5cad1bda-ddc8-4333-8d88-333b6ef311b5", + "name": "docker-fleet-agent", + "type": "metricbeat", + "version": "8.8.0" + }, + "apache_tomcat": { + "cache": { + "application_name": "/", + "hit": { + "count": 15.0 + }, + "lookup": { + "count": 30.0 + }, + "object": { + "size": { + "max": { + "kb": 512.0 + } + } + }, + "size": { + "current": { + "kb": 19.0 + }, + "max": { + "kb": 10240.0 + } + }, + "ttl": { + "ms": 5000.0 + } + } + }, + "data_stream": { + "dataset": "apache_tomcat.cache", + "namespace": "ep", + "type": "metrics" + }, + "ecs": { + "version": "8.7.0" + }, + "elastic_agent": { + "id": "5cad1bda-ddc8-4333-8d88-333b6ef311b5", + "snapshot": true, + "version": "8.8.0" + }, + "event": { + "agent_id_status": "verified", + "category": "web", + "dataset": "apache_tomcat.cache", + "duration": 136411023, + "ingested": "2023-05-30T11:42:09Z", + "kind": "metric", + "module": "apache_tomcat", + "type": "info" + }, + "host": { + "architecture": "x86_64", + "containerized": false, + "hostname": "docker-fleet-agent", + "id": "4e0d02de87bd4101b7215d3aaf5f623b", + "ip": "172.23.0.7", + "mac": "02-42-AC-17-00-07", + "name": "docker-fleet-agent", + "os": { + "codename": "focal", + "family": "debian", + "kernel": "5.19.0-42-generic", + "name": "Ubuntu", + "platform": "ubuntu", + "type": "linux", + "version": "20.04.6 LTS (Focal Fossa)" + } + }, + "metricset": { + "name": "collector", + "period": 10000 + }, + "service": { + "address": "http://elastic-package-service-apache_tomcat-1:9090/metrics", + "type": "prometheus" + }, + "tags": [ + "apache_tomcat-cache", + "forwarded" + ] +} \ No newline at end of file From fd482c02b53b3ac3e04288d5ef54d0c456ce5d7c Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 30 May 2023 17:48:59 +0200 Subject: [PATCH 14/34] Remove unused test data file --- internal/common/testdata/fields.json | 150 --------------------------- 1 file changed, 150 deletions(-) delete mode 100644 internal/common/testdata/fields.json diff --git a/internal/common/testdata/fields.json b/internal/common/testdata/fields.json deleted file mode 100644 index 9d5a8ad81a..0000000000 --- a/internal/common/testdata/fields.json +++ /dev/null @@ -1,150 +0,0 @@ -{ - "elastic_agent.version": [ - "8.8.0" - ], - "event.category": [ - "web" - ], - "host.os.name.text": [ - "Ubuntu" - ], - "host.hostname": [ - "docker-fleet-agent" - ], - "host.mac": [ - "02-42-AC-17-00-07" - ], - "service.type": [ - "prometheus" - ], - "apache_tomcat.cache.application_name": [ - "/" - ], - "host.os.version": [ - "20.04.6 LTS (Focal Fossa)" - ], - "host.os.name": [ - "Ubuntu" - ], - "agent.name": [ - "docker-fleet-agent" - ], - "host.name": [ - "docker-fleet-agent" - ], - "event.agent_id_status": [ - "verified" - ], - "metricset.name.text": [ - "collector" - ], - "event.kind": [ - "metric" - ], - "host.os.type": [ - "linux" - ], - "apache_tomcat.cache.object.size.max.kb": [ - 512.0 - ], - "apache_tomcat.cache.size.max.kb": [ - 10240.0 - ], - "data_stream.type": [ - "metrics" - ], - "tags": [ - "apache_tomcat-cache", - "forwarded" - ], - "host.architecture": [ - "x86_64" - ], - "agent.id": [ - "5cad1bda-ddc8-4333-8d88-333b6ef311b5" - ], - "ecs.version": [ - "8.7.0" - ], - "host.containerized": [ - false - ], - "service.address": [ - "http://elastic-package-service-apache_tomcat-1:9090/metrics" - ], - "agent.version": [ - "8.8.0" - ], - "host.os.family": [ - "debian" - ], - "apache_tomcat.cache.size.current.kb": [ - 19.0 - ], - "apache_tomcat.cache.lookup.count": [ - 30.0 - ], - "host.ip": [ - "172.23.0.7" - ], - "agent.type": [ - "metricbeat" - ], - "event.module": [ - "apache_tomcat" - ], - "host.os.kernel": [ - "5.19.0-42-generic" - ], - "elastic_agent.snapshot": [ - true - ], - "host.id": [ - "4e0d02de87bd4101b7215d3aaf5f623b" - ], - "elastic_agent.id": [ - "5cad1bda-ddc8-4333-8d88-333b6ef311b5" - ], - "data_stream.namespace": [ - "ep" - ], - "metricset.period": [ - 10000 - ], - "host.os.codename": [ - "focal" - ], - "apache_tomcat.cache.ttl.ms": [ - 5000.0 - ], - "metricset.name": [ - "collector" - ], - "event.duration": [ - 136411023 - ], - "event.ingested": [ - "2023-05-30T11:42:09Z" - ], - "@timestamp": [ - "2023-05-30T11:42:07.923Z" - ], - "host.os.platform": [ - "ubuntu" - ], - "data_stream.dataset": [ - "apache_tomcat.cache" - ], - "event.type": [ - "info" - ], - "agent.ephemeral_id": [ - "7dd994f8-d866-4068-9f29-d6c90a28738e" - ], - "apache_tomcat.cache.hit.count": [ - 15.0 - ], - "event.dataset": [ - "apache_tomcat.cache" - ] -} \ No newline at end of file From 6396fd753c2958821bd0ca2087e45a4c942a7e35 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 30 May 2023 18:20:50 +0200 Subject: [PATCH 15/34] Add condition for no component template --- internal/testrunner/runners/system/runner.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index 3f9b1b2387..154e6e2640 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -240,6 +240,7 @@ func buildAllFieldsBody() io.Reader { } func (r *runner) isSyntheticsEnabled(dataStream, componentTemplatePackage string) (bool, error) { + logger.Debug("check whether or not synthetics is enabled (component template %s)...", componentTemplatePackage) resp, err := r.options.API.Cluster.GetComponentTemplate( r.options.API.Cluster.GetComponentTemplate.WithName(componentTemplatePackage), ) @@ -267,6 +268,10 @@ func (r *runner) isSyntheticsEnabled(dataStream, componentTemplatePackage string return false, fmt.Errorf("could not decode search results response: %w", err) } + if len(results.ComponentTemplates) == 0 { + logger.Debugf("no component template found for data stream %s", dataStream) + return false, nil + } if len(results.ComponentTemplates) != 1 { return false, fmt.Errorf("unexpected response, not found component template") } @@ -544,7 +549,6 @@ func (r *runner) runTest(config *testConfig, ctxt servicedeployer.ServiceContext return result.WithError(fmt.Errorf("%s", result.FailureMsg)) } - logger.Debug("check whether or not synthetics is enabled...") syntheticEnabled, err := r.isSyntheticsEnabled(dataStream, componentTemplatePackage) if err != nil { return result.WithError(err) From 63dc94b073caa1f3a88f5c93e299722830692cf5 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 30 May 2023 18:51:06 +0200 Subject: [PATCH 16/34] fix log message --- internal/testrunner/runners/system/runner.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index 154e6e2640..a02fd4fa4a 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -240,7 +240,7 @@ func buildAllFieldsBody() io.Reader { } func (r *runner) isSyntheticsEnabled(dataStream, componentTemplatePackage string) (bool, error) { - logger.Debug("check whether or not synthetics is enabled (component template %s)...", componentTemplatePackage) + logger.Debugf("check whether or not synthetics is enabled (component template %s)...", componentTemplatePackage) resp, err := r.options.API.Cluster.GetComponentTemplate( r.options.API.Cluster.GetComponentTemplate.WithName(componentTemplatePackage), ) From bfdb279b90a130d9ba091361ece3b770ffc43463 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 30 May 2023 18:52:31 +0200 Subject: [PATCH 17/34] Show status --- scripts/test-check-packages.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/test-check-packages.sh b/scripts/test-check-packages.sh index 9eebcec9c9..6c610a0b39 100755 --- a/scripts/test-check-packages.sh +++ b/scripts/test-check-packages.sh @@ -53,6 +53,8 @@ elastic-package stack update -v # Boot up the stack elastic-package stack up -d -v +elastic-package stack status + if [ "${PACKAGE_TEST_TYPE:-other}" == "with-kind" ]; then # Boot up the kind cluster kind create cluster --config $PWD/scripts/kind-config.yaml From d981f9cf49054981b0e9437a8d0d0d0f7f049318 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 31 May 2023 22:02:14 +0200 Subject: [PATCH 18/34] Add test package for synthetic --- .../packages/other/synthetic_mode/LICENSE.txt | 93 +++++++++++ .../other/synthetic_mode/_dev/build/build.yml | 3 + .../_dev/deploy/docker/docker-compose.yml | 7 + .../synthetic_mode/_dev/deploy/variants.yml | 4 + .../other/synthetic_mode/changelog.yml | 6 + .../_dev/test/system/test-default-config.yml | 5 + .../synthetic/agent/stream/stream.yml.hbs | 53 +++++++ .../synthetic/fields/base-fields.yml | 12 ++ .../data_stream/synthetic/fields/ecs.yml | 7 + .../data_stream/synthetic/fields/fields.yml | 41 +++++ .../data_stream/synthetic/manifest.yml | 145 ++++++++++++++++++ .../data_stream/synthetic/sample_event.json | 72 +++++++++ .../other/synthetic_mode/docs/README.md | 84 ++++++++++ .../other/synthetic_mode/img/sample-logo.svg | 1 + .../synthetic_mode/img/sample-screenshot.png | Bin 0 -> 18849 bytes .../other/synthetic_mode/manifest.yml | 33 ++++ 16 files changed, 566 insertions(+) create mode 100644 test/packages/other/synthetic_mode/LICENSE.txt create mode 100644 test/packages/other/synthetic_mode/_dev/build/build.yml create mode 100644 test/packages/other/synthetic_mode/_dev/deploy/docker/docker-compose.yml create mode 100644 test/packages/other/synthetic_mode/_dev/deploy/variants.yml create mode 100644 test/packages/other/synthetic_mode/changelog.yml create mode 100644 test/packages/other/synthetic_mode/data_stream/synthetic/_dev/test/system/test-default-config.yml create mode 100644 test/packages/other/synthetic_mode/data_stream/synthetic/agent/stream/stream.yml.hbs create mode 100644 test/packages/other/synthetic_mode/data_stream/synthetic/fields/base-fields.yml create mode 100644 test/packages/other/synthetic_mode/data_stream/synthetic/fields/ecs.yml create mode 100644 test/packages/other/synthetic_mode/data_stream/synthetic/fields/fields.yml create mode 100644 test/packages/other/synthetic_mode/data_stream/synthetic/manifest.yml create mode 100644 test/packages/other/synthetic_mode/data_stream/synthetic/sample_event.json create mode 100644 test/packages/other/synthetic_mode/docs/README.md create mode 100644 test/packages/other/synthetic_mode/img/sample-logo.svg create mode 100644 test/packages/other/synthetic_mode/img/sample-screenshot.png create mode 100644 test/packages/other/synthetic_mode/manifest.yml diff --git a/test/packages/other/synthetic_mode/LICENSE.txt b/test/packages/other/synthetic_mode/LICENSE.txt new file mode 100644 index 0000000000..809108b857 --- /dev/null +++ b/test/packages/other/synthetic_mode/LICENSE.txt @@ -0,0 +1,93 @@ +Elastic License 2.0 + +URL: https://www.elastic.co/licensing/elastic-license + +## Acceptance + +By using the software, you agree to all of the terms and conditions below. + +## Copyright License + +The licensor grants you a non-exclusive, royalty-free, worldwide, +non-sublicensable, non-transferable license to use, copy, distribute, make +available, and prepare derivative works of the software, in each case subject to +the limitations and conditions below. + +## Limitations + +You may not provide the software to third parties as a hosted or managed +service, where the service provides users with access to any substantial set of +the features or functionality of the software. + +You may not move, change, disable, or circumvent the license key functionality +in the software, and you may not remove or obscure any functionality in the +software that is protected by the license key. + +You may not alter, remove, or obscure any licensing, copyright, or other notices +of the licensor in the software. Any use of the licensor’s trademarks is subject +to applicable law. + +## Patents + +The licensor grants you a license, under any patent claims the licensor can +license, or becomes able to license, to make, have made, use, sell, offer for +sale, import and have imported the software, in each case subject to the +limitations and conditions in this license. This license does not cover any +patent claims that you cause to be infringed by modifications or additions to +the software. If you or your company make any written claim that the software +infringes or contributes to infringement of any patent, your patent license for +the software granted under these terms ends immediately. If your company makes +such a claim, your patent license ends immediately for work on behalf of your +company. + +## Notices + +You must ensure that anyone who gets a copy of any part of the software from you +also gets a copy of these terms. + +If you modify the software, you must include in any modified copies of the +software prominent notices stating that you have modified the software. + +## No Other Rights + +These terms do not imply any licenses other than those expressly granted in +these terms. + +## Termination + +If you use the software in violation of these terms, such use is not licensed, +and your licenses will automatically terminate. If the licensor provides you +with a notice of your violation, and you cease all violation of this license no +later than 30 days after you receive that notice, your licenses will be +reinstated retroactively. However, if you violate these terms after such +reinstatement, any additional violation of these terms will cause your licenses +to terminate automatically and permanently. + +## No Liability + +*As far as the law allows, the software comes as is, without any warranty or +condition, and the licensor will not be liable to you for any damages arising +out of these terms or the use or nature of the software, under any kind of +legal claim.* + +## Definitions + +The **licensor** is the entity offering these terms, and the **software** is the +software the licensor makes available under these terms, including any portion +of it. + +**you** refers to the individual or entity agreeing to these terms. + +**your company** is any legal entity, sole proprietorship, or other kind of +organization that you work for, plus all organizations that have control over, +are under the control of, or are under common control with that +organization. **control** means ownership of substantially all the assets of an +entity, or the power to direct its management and policies by vote, contract, or +otherwise. Control can be direct or indirect. + +**your licenses** are all the licenses granted to you for the software under +these terms. + +**use** means anything you do with the software requiring one of your licenses. + +**trademark** means trademarks, service marks, and similar rights. diff --git a/test/packages/other/synthetic_mode/_dev/build/build.yml b/test/packages/other/synthetic_mode/_dev/build/build.yml new file mode 100644 index 0000000000..e3c4cca65a --- /dev/null +++ b/test/packages/other/synthetic_mode/_dev/build/build.yml @@ -0,0 +1,3 @@ +dependencies: + ecs: + reference: git@v8.7.0 diff --git a/test/packages/other/synthetic_mode/_dev/deploy/docker/docker-compose.yml b/test/packages/other/synthetic_mode/_dev/deploy/docker/docker-compose.yml new file mode 100644 index 0000000000..133131ed73 --- /dev/null +++ b/test/packages/other/synthetic_mode/_dev/deploy/docker/docker-compose.yml @@ -0,0 +1,7 @@ +version: '3.2' +services: + synthetic_mode: + image: prom/prometheus:${PROMETHEUS_VERSION:-prometheus_2} + ports: + - 9090:9090 + diff --git a/test/packages/other/synthetic_mode/_dev/deploy/variants.yml b/test/packages/other/synthetic_mode/_dev/deploy/variants.yml new file mode 100644 index 0000000000..1058c81e28 --- /dev/null +++ b/test/packages/other/synthetic_mode/_dev/deploy/variants.yml @@ -0,0 +1,4 @@ +variants: + prometheus_2: + PROMETHEUS_VERSION: v2.36.2 +default: prometheus_2 diff --git a/test/packages/other/synthetic_mode/changelog.yml b/test/packages/other/synthetic_mode/changelog.yml new file mode 100644 index 0000000000..bb0320a524 --- /dev/null +++ b/test/packages/other/synthetic_mode/changelog.yml @@ -0,0 +1,6 @@ +# newer versions go on top +- version: "0.0.1" + changes: + - description: Initial draft of the package + type: enhancement + link: https://github.com/elastic/integrations/pull/1 # FIXME Replace with the real PR link diff --git a/test/packages/other/synthetic_mode/data_stream/synthetic/_dev/test/system/test-default-config.yml b/test/packages/other/synthetic_mode/data_stream/synthetic/_dev/test/system/test-default-config.yml new file mode 100644 index 0000000000..10731d5946 --- /dev/null +++ b/test/packages/other/synthetic_mode/data_stream/synthetic/_dev/test/system/test-default-config.yml @@ -0,0 +1,5 @@ +vars: ~ +data_stream: + vars: + hosts: + - "{{Hostname}}:9090" diff --git a/test/packages/other/synthetic_mode/data_stream/synthetic/agent/stream/stream.yml.hbs b/test/packages/other/synthetic_mode/data_stream/synthetic/agent/stream/stream.yml.hbs new file mode 100644 index 0000000000..58f93cc7a4 --- /dev/null +++ b/test/packages/other/synthetic_mode/data_stream/synthetic/agent/stream/stream.yml.hbs @@ -0,0 +1,53 @@ +metricsets: ["collector"] +hosts: +{{#each hosts}} + - {{this}} +{{/each}} +metrics_filters.exclude: +{{#each metrics_filters.exclude}} + - {{this}} +{{/each}} +metrics_filters.include: +{{#each metrics_filters.include}} + - {{this}} +{{/each}} +metrics_path: {{metrics_path}} +period: {{period}} +rate_counters: {{rate_counters}} +{{#if bearer_token_file}} +bearer_token_file: {{bearer_token_file}} +{{/if}} +{{#if ssl.certificate_authorities}} +ssl.certificate_authorities: +{{#each ssl.certificate_authorities}} + - {{this}} +{{/each}} +{{/if}} +use_types: {{use_types}} +username: {{username}} +password: {{password}} +{{#if leaderelection }} +{{#if condition }} +condition: ${kubernetes_leaderelection.leader} == true and {{ condition }} +{{ else }} +condition: ${kubernetes_leaderelection.leader} == true +{{/if}} +{{ else }} +{{#if condition }} +condition: {{ condition }} +{{/if}} +{{/if}} +{{#if query}} +{{query}} +{{/if}} +{{#if headers}} +{{headers}} +{{/if}} +{{#if connect_timeout}} +connect_timeout: {{connect_timeout}} +{{/if}} +{{#if timeout}} +timeout: {{timeout}} +{{/if}} +data_stream: + dataset: {{data_stream.dataset}} diff --git a/test/packages/other/synthetic_mode/data_stream/synthetic/fields/base-fields.yml b/test/packages/other/synthetic_mode/data_stream/synthetic/fields/base-fields.yml new file mode 100644 index 0000000000..7c798f4534 --- /dev/null +++ b/test/packages/other/synthetic_mode/data_stream/synthetic/fields/base-fields.yml @@ -0,0 +1,12 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. diff --git a/test/packages/other/synthetic_mode/data_stream/synthetic/fields/ecs.yml b/test/packages/other/synthetic_mode/data_stream/synthetic/fields/ecs.yml new file mode 100644 index 0000000000..aabd8a9b9c --- /dev/null +++ b/test/packages/other/synthetic_mode/data_stream/synthetic/fields/ecs.yml @@ -0,0 +1,7 @@ +- external: ecs + name: ecs.version +- external: ecs + name: service.address +- external: ecs + name: service.type + diff --git a/test/packages/other/synthetic_mode/data_stream/synthetic/fields/fields.yml b/test/packages/other/synthetic_mode/data_stream/synthetic/fields/fields.yml new file mode 100644 index 0000000000..1f25ea1309 --- /dev/null +++ b/test/packages/other/synthetic_mode/data_stream/synthetic/fields/fields.yml @@ -0,0 +1,41 @@ +- name: prometheus + type: group + fields: + - name: labels.* + type: object + object_type: keyword + description: | + Prometheus metric labels + - name: metrics.* + type: object + object_type: double + object_type_mapping_type: "*" + description: | + Prometheus metric +- name: prometheus.*.value + type: object + object_type: double + object_type_mapping_type: "*" + description: > + Prometheus gauge metric + +- name: prometheus.*.counter + type: object + object_type: double + object_type_mapping_type: "*" + description: > + Prometheus counter metric + +- name: prometheus.*.rate + type: object + object_type: double + object_type_mapping_type: "*" + description: > + Prometheus rated counter metric + +- name: prometheus.*.histogram + type: object + object_type: histogram + object_type_mapping_type: "*" + description: > + Prometheus histogram metric diff --git a/test/packages/other/synthetic_mode/data_stream/synthetic/manifest.yml b/test/packages/other/synthetic_mode/data_stream/synthetic/manifest.yml new file mode 100644 index 0000000000..b6e5e5c12d --- /dev/null +++ b/test/packages/other/synthetic_mode/data_stream/synthetic/manifest.yml @@ -0,0 +1,145 @@ +title: Prometheus collector metrics synthetic +type: metrics +streams: + - input: prometheus/metrics + vars: + - name: hosts + type: text + title: Hosts + multi: true + required: true + show_user: true + default: + - localhost:9090 + - name: metrics_path + type: text + title: Metrics Path + multi: false + required: false + show_user: true + default: /metrics + - name: period + type: text + title: Period + multi: false + required: true + show_user: true + default: 10s + - name: use_types + type: bool + title: Use Types + multi: false + required: true + show_user: true + default: true + - name: rate_counters + type: bool + title: Rate Counters + multi: false + required: true + show_user: true + default: true + - name: leaderelection + type: bool + title: Leader Election + description: Enable leaderelection between a set of Elastic Agents running on Kubernetes + multi: false + required: true + show_user: true + default: false + - name: condition + title: Condition + description: Condition to filter when to apply this datastream + type: text + multi: false + required: false + show_user: true + - name: bearer_token_file + type: text + title: 'HTTP config options: bearer_token_file' + description: If defined, the contents of the file will be read once at initialization and then the value will be used in an HTTP Authorization header. + multi: false + required: false + show_user: false + - name: ssl.certificate_authorities + type: text + title: SSL Certificate Authorities + multi: true + required: false + show_user: false + - name: metrics_filters.exclude + type: text + title: Metrics Filters Exclude + multi: true + required: false + show_user: false + default: [] + - name: metrics_filters.include + type: text + title: Metrics Filters Include + multi: true + required: false + show_user: false + default: [] + - name: username + type: text + title: 'HTTP config options: Username' + description: The username to use for basic authentication. + multi: false + required: false + show_user: false + default: user + - name: password + type: password + title: 'HTTP config options: Password' + description: The password to use for basic authentication. + multi: false + required: false + show_user: false + default: secret + - name: connect_timeout + type: text + title: 'HTTP config options: connect_timeout' + description: Total time limit for an HTTP connection to be completed (Default 2 seconds) + multi: false + required: false + show_user: false + - name: timeout + type: text + title: 'HTTP config options: timeout' + description: Total time limit for HTTP requests made by the module (Default 10 seconds) + multi: false + required: false + show_user: false + - name: headers + type: yaml + title: "HTTP config options: headers" + description: A list of headers to use with the HTTP request + multi: false + required: false + show_user: false + default: | + # headers: + # Cookie: abcdef=123456 + # My-Custom-Header: my-custom-value + - name: query + type: yaml + title: "HTTP config options: query" + description: An optional value to pass common query params in YAML + multi: false + required: false + show_user: false + default: | + # query: + # key: value + - name: data_stream.dataset + type: text + title: 'Datasream Dataset name' + description: Name of Datastream dataset + multi: false + default: synthetic_mode.synthetic + required: true + show_user: true + title: Prometheus collector metrics + enabled: true + description: Collect Prometheus collector metrics diff --git a/test/packages/other/synthetic_mode/data_stream/synthetic/sample_event.json b/test/packages/other/synthetic_mode/data_stream/synthetic/sample_event.json new file mode 100644 index 0000000000..40dea34c86 --- /dev/null +++ b/test/packages/other/synthetic_mode/data_stream/synthetic/sample_event.json @@ -0,0 +1,72 @@ +{ + "@timestamp": "2023-05-31T19:55:04.734Z", + "agent": { + "ephemeral_id": "d8c2f32e-9fa7-419a-bdd0-a93604aea83a", + "id": "63a71457-c239-40ed-b42d-1a0e8b523875", + "name": "docker-fleet-agent", + "type": "metricbeat", + "version": "8.6.2" + }, + "data_stream": { + "dataset": "synthetic_mode.synthetic", + "namespace": "ep", + "type": "metrics" + }, + "ecs": { + "version": "8.0.0" + }, + "elastic_agent": { + "id": "63a71457-c239-40ed-b42d-1a0e8b523875", + "snapshot": false, + "version": "8.6.2" + }, + "event": { + "agent_id_status": "verified", + "dataset": "synthetic_mode.synthetic", + "duration": 3889366, + "ingested": "2023-05-31T19:55:05Z", + "module": "prometheus" + }, + "host": { + "architecture": "x86_64", + "containerized": false, + "hostname": "docker-fleet-agent", + "id": "f91b175388d443fca5c155815dfc2279", + "ip": [ + "172.30.0.7" + ], + "mac": [ + "02-42-AC-1E-00-07" + ], + "name": "docker-fleet-agent", + "os": { + "codename": "focal", + "family": "debian", + "kernel": "5.19.0-42-generic", + "name": "Ubuntu", + "platform": "ubuntu", + "type": "linux", + "version": "20.04.5 LTS (Focal Fossa)" + } + }, + "metricset": { + "name": "collector", + "period": 10000 + }, + "prometheus": { + "labels": { + "event": "update", + "instance": "elastic-package-service-synthetic_mode-1:9090", + "job": "prometheus", + "role": "node" + }, + "prometheus_sd_kubernetes_events_total": { + "counter": 0, + "rate": 0 + } + }, + "service": { + "address": "http://elastic-package-service-synthetic_mode-1:9090/metrics", + "type": "prometheus" + } +} \ No newline at end of file diff --git a/test/packages/other/synthetic_mode/docs/README.md b/test/packages/other/synthetic_mode/docs/README.md new file mode 100644 index 0000000000..355a85cb5f --- /dev/null +++ b/test/packages/other/synthetic_mode/docs/README.md @@ -0,0 +1,84 @@ + + + +# Synthetic Mode + + + +## Data streams + + + + + + + + + + + +## Requirements + +You need Elasticsearch for storing and searching your data and Kibana for visualizing and managing it. +You can use our hosted Elasticsearch Service on Elastic Cloud, which is recommended, or self-manage the Elastic Stack on your own hardware. + + + +## Setup + + + +For step-by-step instructions on how to set up an integration, see the +[Getting started](https://www.elastic.co/guide/en/welcome-to-elastic/current/getting-started-observability.html) guide. + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/test/packages/other/synthetic_mode/img/sample-logo.svg b/test/packages/other/synthetic_mode/img/sample-logo.svg new file mode 100644 index 0000000000..6268dd88f3 --- /dev/null +++ b/test/packages/other/synthetic_mode/img/sample-logo.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/test/packages/other/synthetic_mode/img/sample-screenshot.png b/test/packages/other/synthetic_mode/img/sample-screenshot.png new file mode 100644 index 0000000000000000000000000000000000000000..d7a56a3ecc078c38636698cefba33f86291dd178 GIT binary patch literal 18849 zcmeEu^S~#!E#4Tq;}?6chqwB{?k=6jc5D4>l%v(rleJ2Y%tW zDj9g7px}|*e;{M?LDwiK3@FNS(lDRTd-MJYIyUJCN948~OJk1M(DrJyI#iV;P4k~& zFZo35IfQt0RwlUN`48^6(1dv_wm(y1xhEdMld=Y?!%u=fPT_*{3( zwBwz3#qR}_)t>C*jp5@U)Ti~B)Y;qq*TRxZJ7ZRN_^A3TDAEM*@7Ve%(Ro7=1%1B< zVj6GBUTxXev>_^SFA zgKZ=g4aTS}9>Ofj7cSB0WO?gQ)x=+!hs_)b$6#>ScFZ>XAoIX)%Bc|BDC~JFBk0f0 z0NY}6gb)&!qx^FWC(!ji+Kl$V$2|ocA=vN0TM0Y`U?tX+T)c*C zA!IL(T2Vm%MCLa85^if@J@Kkprx8QN5!6eCR@4Oa5S?4-4|ou?90mFCM8D!;n(5xz zO}-*t!TntN>|a$s(kGQg1P-U?hqvGF2_fGvd&~yZ_l3Qf&j~XWa=;>N3#-~#zjzcc z*m18L`A-K2o!d@J>a8SRbm4P&-q1(H>|JgIymDbnJF&@008`=X!P?4DGgZb>voUl^ zNJKgPR4S={)3vuk_{n@=M8q;;aJL>q+VLdTnO=}`&x;1DKjJA3*f*idS{jP5?+;!W zn-^7021Z4zv`Aq`hmX1aid997RNh3fa-@PG(W7TzKa1W&5^y3|lPeETP7j9qXpo4)7%(W0_2 z^Nmq;t@rb1eP3?%kOkH`P%!zTC7ZHjSfNN3*Sb#=3#jB*KpNGNfnRZ{N(6DrW(;B2Bwom<%m?VQP%K+ zsFeF1-(DY}oP@)w^Kw~gPg03q?N;)Ec6^|nikA34T~RynX*z}H>R~qgT$`Zbhn8wzZs$j2fsGN&rOK-mIBBvzD@a8FgbLpL!h5N^u&0wG} zq!#md3MHITv?3@$37J?lc_5*LWJTTjel;IiU-Yq;(g9I^D&KN_NKVS0O~GvB~FzPM6}=4d%fG4Nw4pZshcyLqK@`b8?RhD38haIyr@+8+0r5TC1*C7^WleJ zZN3_ngTD#RQvNL*;qD2H@cBWJbCC#d!}=oKfod5SE9a?!?j%DVt1z@inN}Iy$r+96 zM@P?AC+(`cM;z6J94BYGJ;+P-N#yj$?`G26ydS&OVH?~JY(N4l()Fh+x+DoJ@r<+i zhm^ck@QP`=fLApr62@KyOef~}zuG;(VbDQmw|Wb+oSHSw=%w9R)=et0cY*~ytX)#M zEXlK^p;zM@vTnXn+C1vwP)~TJv|TvDE2($;;EzC5_5IL#H;u z)#CO8)TSzbt8)wHB8$I8KcIojx&GoE)3QNu{CQ+_xBmQ&`mL5-u=BX(hs^hMY^ zae!!*Q;Tr$@(0~GoBJAohGw*d{l8~!aXop87aaSUb2jm)Tk>#$1*cdo5Sl+?oD!l4Og~yX+soottl4 zp4OartUuAN(dD~yLJ}`A1*!D4-|L^hM;`_DM^1KYs-VF(}h(BjRO``b+xV~%O=-)?p z7ciJH7Fnl?V&=ay_AB{oQoa2iR;6$^tiE|-eRCFy|3F@%j#6gUxkZX@?K`F$u#;T< z4IZORpUthmB?U`;zrOkp?P(Rvd5TFRWrBJmVg;KEZvJ+;Q}FRY%QZ?c^&$oPXW+C5 zdN#c>v%U?QuE+hMQdzxS1Q(BT90;29qu#^A?a^)Ui;{TJ;%`nLgm2ew$J4NvREjCJ z$`C7&?tH$CrVG@M3J1-KJw_*9BKeL*JX{ zN+Vg_TXb9^jJO$ZGkXO6BBFDjt~w5`w2TB*z$&1W5Il3IiDs=ZMDt|9iRtKET*wF6 z0Z+|N87p-5Fh)^(*l>OVr5^aY5LW(@PuM>Qo@&)yj6XRkPm1>eTF#Y_c*aRF^ZY5A z9FAU7lKEHG@i{wJMPg;n6z2|69d-)q9@<7t()d-zPy&X zdXG7{Uw{k23)CzzQAXw#iqj<1u~W@K_Ljc#?ukh;fRKHeJ2l~Z+52b2n^bGiDF2oX zm25FLx|4AP8>rAi@koY03lrtS#X?zK591c?2iZ_jjc>0y>q9>fU<08o6zG%z9WK+S zDwZMW4~28wu#ye#V*@#5t^S@NiAA`3{SF$xINmc_WW^u-C9M=H>RQ1>WM=|R!660{ z6E6%DwX`eu<3pkmz7Z=FCRd$(vhDkc3yMnSr)5C*aho)DZ<12$`$TXj<8Z70)|rK7 zXFD8QzksfWZU`qL2K8X{C~TcF{KVW`3Y{IMb&)T9%1V`tv(HY1 z+LXkLyM|3mtLD{x-#hOw-U?sr-iLeHFA|=-sGZ4#hX)atL!a91(tWJc+og&5W}VfZ zpgE7`{5D`~?yGR++y7~xA&eU0N*ZezDjF$> zUeK&1aTFQRg*?v^Z2e7u<`lk$czR6}b6Cl-qA9%A`#A6q0*zyTu)X`3rhjR86NK3= zLdw{+-F}+b2gxd-qF7>Rla}dFkj|L#c|pg5Ni+MRA|BZH(@ME*o<1ijKcoXb%PVfJ ztp_uf=G%kvU((pHcw90Xut=}atA!giM-5By)f40nKp zv7Wdb{;^<}VRvruH~rYr~wEuYY2ov-5Q|p@u3Da9+z7PeIpBAwi?RxnxN3Kt+N9L(LUS%wxY` z>e&1VV;{CYw8DNRlvBH)>!I49SU4R!t3I4=y;mCevPZh!-}~G+F>6hcL_Rli4r zC4(WN)`j$>^S=~GMGR=^)A6wrqi(-x{xK37&Vx!OS6t=KQ2JVZo#GrSODtTe=TVh%*qfF%91nqsMNLNL^Gp|_ zz%I*HUkMQGqb!1eh{{bp|0GSCDbkG_D_d)8<(0r<6-%Qi7qDa7xZjcdZ$?Rth9L!f z$erCcs3<~mtupywbaT8NWZF#v?iZkvqSz3@p`RiXs7P!GUa~-U9hEG(NgI#3BzO-# z!9JWf(;r!*A=@g$f}>wi|6Q@9z8AmYf~x8G%sp>C5cfuJY;hs1o3Ozu^{pH0AFbs%yU)Xy5>Cf?qXiHn*-PAfKDRiy`U0sFSKFsgEZ6_ z9#ma!<#Izr^}_z*>PRSt564u6We*XmZUx^jv*dK; z4zyFZ*ZFSE!00<6!|+#33&R)@RA8V9YRjp$HS9?CGq*xDSDRbX#i;}mateEF{fqTI zt?X}Efkq_Ap*_ETgaikOBbQ|;47}hwX44K`(DUI@C)QiG&6UJ1UmRn*Q@6%e`+x(gpQp74O{;yli8YLCV}qD z4gIyZd_(8ED~WWaeXOb0^r=9=AiDT}by~+$KVF~M{ywbQl zng-h?a_E;yX?DCr4|_h7JMc7>xgWf7Ek-VmH^hCYunVp3{(d{---&%-GZ=rK#V5Jo zJvP8b!2AA5?9)G8gwzB6ze3TU<5*Pqms^Q-?C9-CN~4hb-`U0D@kAkTWn23``cao^ z8IWAp8h7`%ZA+eI?w$sJktq5m>e&0@mQn>2BdpKAxbj1$m$8Z;`!iFvl9($Lb9Ff? zT^6cTZ~HgIeR6R*;G(rzpgsJP41Fx9Df;G6{;k6T(i}&8hX(jHSC@~#X@70h#)g(( z*9vUC+a*b%oAdf1$}Z3NR;|c5nY4^Z51pfqk(tmJbB;Q#ka#tf5eae;-kq$I{xO3<(TI$0lSe-JQzJ*es;il=Kn_?&?E zfLbs{qErPqm)-*ZfwbA*D-shgb|1;X;cH*yA|q8gS=HiosF=-kbdk6--SR+`F^H_` z0*i`J==@XSe=HT;_``G}ulE=H@*3GU*?gVd@h*`eT^GKjI;C@8+h~;(u3bA#b&bN{ zYw>dJ$(;RfHDLlndS`CWOE=g0jOocCc&;w(dOzrLf4-DK*MD@P_;u&CbfMw=#Q-B` zDq8hGwKN-O7(hQA_bP3f5XrZH+@*FGw~ppmDgNWcf|Lf*Pc%e5dw1DcJ1BWm!z7z3 zr^toEU*P(>G#;_1X}Rz(5lbDtCui%hY^d3lm)kw0vyk zX~K4$AG#7cG`6s2%9g9zsaQ9o?;3yzW4Pt!;NlS zzI#G7tiq&@eV&}qDtY(e$1JwscAfle%Al{3>Nr%``n?`Jac^CdOXUbFgI3;m{RkA~ zokl+lxuw9=%W&MmzA+G%ZdFMMP&N2^6BWjG2Lt|xKx)lMCR@b0n+xgw<)&Dwi?}>- z+$_e|@M;uW@3z6)q&L7bYitZ%huzGqH_qHOr&G5o!?(8TJv_MN1ka|&c6_!Q>#PgHSFoPWiLg|k_{ zQd#Zy&BPkU(0OE5S35!B5qb6%T3Wd#J(zBl8dw6I#xIDDF-LBPi-jXv1E?!gE|1OIdTejK)+U3ooC^otSIRsWZf-`&K}6}s!407Y58zH zK(oYx*7sN1O|Z_1YIJS_H$E@DH(hB4QKNCGQT3PTvwYoe2&8WKi5`5tU-r4!>_V3XUT}N)>8V;+z-!@-IGCKiD>E9RC(K`NMx=;Qp zf$2g^t?)zpU0L!BZi(oE#)^Z_biT*Svh>r#%1=O+Wo37G`Q)4@k#Pe?^mgBIugC)8 zyEICH=`{A~^x#X&%tr-$j|(nXrIrGQYNY+C3M+LO;yUU4-|v>a5#P)XYp>_|C0f0n{_p0mvwWmghfd%!Cm}$qBDxOqA3htLs~ghSA1>6^dVgd~ zVHHBBy6;Pp=El;dkTE=ttp~BoOJ$L@EB3Z37T1kTNG3tm4PY5O-7hP5DA$-k=vV&6 z?RiAm;W~*o)R7!x9>u$&@|&D4xMmJ*y+^-6t!F0u8G~78t&Bs#W>w_NbW>W9M3tXWXRf zI86FWVx%iXXh6MJ>dg#?lNu{K@S#nzMIG4PXQd%!Bvc*H0c7F_Y=adptJr*cHevMQ z%?Xu~q8CFw>^L*S_83kVhq=)hf0%_Lq}SE*g(Da_A{kXVZfAd*YCwp~bG32wi&SNM z#QZ7}Ug5-=+s^uqAh_|}gzya<(&E?XAZ%0ybd9nraj?|z1YfPr*{N?Q{ji}YG`T#| z=uwJZHIMlsmevnenT#-)t$L*=2wh|1EYXW?_36TR?L!sUItJVxaC0$Gb|gq4{|4gA z(v0ODFj!T)jc5>65ys)* z7$aBHfbKdz@QJq1b`NT`344*g()$>5*Ey`TPB7WI;|_8o8t9-_4ikFub|I{66>ge> zHA+6onzFKY*eaiA!77SD*^&LyumAR6gSvxY6Q?;!AvI{rZ##!G$%ZfIgce4F`aF;e z?jVh%+B-vj69ei~bh_zA9w}S4B4rzRKQ1~u$gwVu_x5PlRKDXX2(_2Mm7fs%6{SS7Qh1gWT8xaxc=f8`mW38ukIZxwU;lmHABwFSg50*o zrj%f%j~IKR?N5Dxwrq|sTa?!pd{b3sFM&~{4~_^YH4$bI^Fq2W4-y`))^|7fS?i0) zJ&Z9wY!8%l7@gAr`2{fqA;L;ptQR*X2|xUtrT47KK%XN+dydN$*M?65LuXTRabgERR{n>;E;(&vS0_@COY!p<%5LsRqGpER%~YjkSK zwBo9-2|-ZFiU3TT&S+@}3gDT35t0IXTzX@yHA(v>Y8;-mZNySQ&fE7RJ1^tzJfvdApX& z*!+tE)Y{oR%jk8A)3EiI3i*(TOwP!;B3hAOj?KQ6^h-q~1V^166uYS~mH*2Hh*0}r z`R3u1#^LG9IW|^QT^|61H(T1Jz?n;(Z>52lU0BO>Q6*zgpP*gTFk2Uw)!3zt>3F~_ ztil4!R*-j}wjh%&(kSB%}X=u4RbFRp@^l+$SmM@nW9B;yGbf@nasjFMEE{m9Oe

}qal5$moSACwfNXLXG5|3R0AtBcN` z?%yS)&>O>sqxU64U~C3&Q^>z-Zt}WuX4Wh3dKj9EO zfSbV!c3e;EOeKHQmWEw#NM4;*tw-2o@x&kKT?rsmy-F|$jw-F>WgA7?C@{O1qPg*J zf92|RTBMh&ptHADFc{T+cB?+mOj>h2HKgwkxq6w&XBxPc?>=JKvU2K9aU93@vp-R% z{5T=P$9U}AYZ5QU{3%7}YZ+ACWXw#-U zWyxU(OP#Q9-2AeGmCwcp`zWghf2hvsOjWjDQbU?U`v0&a--f1`v0Bd8HLiLmo)PKz5!A1|XVO+89 zm3h2~6yI~cpWor!_yt-?Lt>z`c0a7cJAW)#d8N8nNIf0H<+v;s4{0guDD(?T7Z<~$ zd`$vpZ_QQgFaMT0_d5&+(jwGU?M1FqUu6wjA-9z?mRM}(CmSdK;2e$Na}F-8jbhgN z9)@AIQeghf{xCC^{9P%VdYW1PP#}2BJwWt z0Hd8%st1NK5%h+)UB^mVwh{e#8TIm$xxgGo6I5;e{~VUeeMGRpM_Z%=eH5$X1}?Z5 z`|*_Vp~K&ziz45-Ih9y>EOr(Buy0&n$dbQ4$5eSr=Ti z#~7^n8dmem;$0D4+6eV7&G2D~d@ z+R#u8+nw_N%7_U_1e53P?~&10^m|ZUXrZhVp04lQLsGos%0fRDhS=@>8TOAAxK;Cy z9GZw_1pfSxD5~xoR!INI?tU0wrKDd6^Tv{jL>`Xb49kBaNPlhMaIfh_nq_)zB7NcX z05XeQKz`@BDUx7*i!V~%dc8XQ#ngBw0A2tSr(npSCrNy5Z7>48v&Zz?0{%FRElh_h zN2|?#EhJL5HQMIu6m1=ypTR?tVymHK)xQvS9ir7FzMp?CjlND39PK`od#GytVhZWp zQ1@>MTE1*Ip>hnXSWa?XbMH#708@j12yPbm`JfcqIgmJepn$5YgkJn_%5I)mr`Q(k z-a0yFR3A`houhvf&|wNpIsV{2p%MqhR@`@R(l6`}iufEgI*UxWq~26?WTpZCV{JtG zYL?&#I98fyf_;2S0?_V{=Aa4t^x%vy$pF$_Lh7W2f*~5uPvGYh;vZhMv|u+Z?2t0~ zcYPXdxbg6OS*LUjR_=jLDt)ab6;?g1IuySLG@UE;jLpt-wjLX&RlY>fnd@f&?0NyT zht5vhP^};k6`U76$%&I)iWPNxG6KPjdh`S6>g9GN@;KObQsLG zKyjfrPR0PU1B0a0=)3@9eCDl?mB9rFdlTMtTAeZv2}F*|@JWleq2+H1bt>>x!^wTk z+I)cgsZwzCMwoRpW_*!3IySTQu!`HWugAXe(Ai(a9Rsu;*0#o6torxwNMxPzEAjt` z>70Vw;HCQ?AnP`RKQ;2R8h%;LI#tx^(MO*lMWJe4_?)Q571P`kTmN#(ez21V!<6+S z@Uap+y%#8&cGgdf+E@y$dUx3g#)=#5k31Vqv0p!%L`*=-PiQAiSg-d9lKRZQDuJ-| zA96zwwomG+4}X$vR*IU=NC!vL<`rUTbf_uRJC4FS;k&HtV<=<)p(qymH)=MDV^aqK z#%sid7K|~!H`J!7hRr~Z!emxgWq6#GpQs%c#BM+scvNGz|Gi4G`;8Z~dP8)+51iB8 zw)0fazNz5(iK$LJeC_4e^8&@wT(DZ~~>SStz3P(>V8CLNlZqgv=2K-|Lu~si@XFwMN>QE^k zVS2U_A?Q$?M`NkU}^!M8m%O&T=kW>dG}1s2I~hxp9Y=a=1XX-(fB5) zej3`e5Et~R^r%?CZK0)UZsF_+tSOGIBMdrtMf#oJjGF9U`*P8t>i*TWed$Z2WNUZ* z_1Qw4Yr+Q0@bD?hD0P-^v}?FpPBg~zz5~g@J#J76C695|P>1l;OS8%~hZh5&-9Ji# z50%&56ZK4FC9}{jHL0!=qo9Yd(GGHCEX2|-F(f}q6@NMT4P3rQd{Q!=bz-8N(Z^!N;;ZzAWRf@C?X>mG=_NgyQX_?Jv$m(9$W>P;+e}O|&w&DjbsJPdWp0A2$yLr*!BY73Z z5d*BCaTI)w=sTlofc>n}@v_tSXIK?8(g`G_06u>SD*fOZJ~visq3lBVS2+cf-r$UQ zZ(8A0g&5M$IV7w5nqL(m$VS0X?=yy-e6>S>Ca3wZNT)b{GF39_gJdONflqc-j$b~o z2l@@h{$KVfC)V?#We*)@xYC;L^<@cHo>8axRMbSzw|eYTl|8pkabsQJ(3`z{>5H}c z`psz_Y6t)hvzL^=}P#++XUl6v`-j)SuXd6BynjNZ!&c2hnyE&4*K$nXn31Zk)cm+lx;> zya{T?{MRtSu?^3Y9bS&O$*mW^vRUpv!J3Tz12?3&Y62b_oiZ$24O(75Z)JWb+Rj)ACbK`f<&tSwtT$|Sy z$41kRPiM-jnPY9PKrLyI`pHm6LusMsrO*HpmE){Kp1^u2t%6nW^;GB|!4k!Ik8oav zjM?DBKh9G@W0gEwiU-M}0B)}olvoM71RccgiZBCs)L?q_GX&JDhegx4k2&cNatr5w zU)1#2USb8&`etO5Vk z?0}K+*2*@a5yt*X{qg0@8jEz~jcylVj>-042p1PBnabI#xUiCRD!ouw3?u-wwsqwF z8(@m8-Lk7q@v154g6yvx_tRDa>}oqpVda)wfI9(;ZVGt1v^{<|X?vC_(i@IJC+2I_lusrT=$h zF1lPc*Neb`;Xgrdf`p$w)~MzQW0M3_FYRKu{2$VU82J^B=X1#^<&P$_`=S$Ey04WU zTxG;hrFNLhWC*p+sH3x=JVcBJ9*7>eO20)n671SxQhZQlHMRP8FyO}yai~OTsbms0 zQ3b$C1Cn!>jMHDq{VX1ab^~_Q!z+f75+_AuwiN0*wA_#M#0|rU{+NlB%>Y+TNT0Gj z`3^LKMSJjz2(?lwg~ixDl_5%rzzZ}o_6Fj9e)T7gpH4=BgT1zmwJpC@g(f%&0`}8B z%7Y&qlP3aFmI#nmT`|R3+Lwzp+PLXt|5g%vlY_$fvse7zjus0D0fA##r+i4G4K-2Y zC#H95NGoYfWP#ZF_v$^Li{PZpm}fc&)aL?5doPcb835Cr6`T+EzzcEvLtmXcbAb<^ zw!_Zgk6Az7YA@*vb)(G{_W-B|zrf76z^`X%jOgqIIaqi~5nUup3vugzzg&rA^w(zR z+qCzvIV~nGR=47pDOcNTzuBw#5a=<=DMvGa)g zPw$^pmq9Fg&b#BZrPSoml(149rZS!fioV*Dy$z440U3MXDJmI?RZqLy0}IKSxN)o( z8+8wIZs#q(|KTg6y;Z(=96>xfpUsr@SP}I^v zN^R;ZVrDaWmNrM5-<X@k6JyjvA3;jHhma|Y|7!Vk& zgf(UK_6~cC;!|b!YTjke=nBiUqQdb#I9TY}!s5P)H+^c;9cW(QO8O%n5J^8Xfktd*qrn)+?-gP`m%B&q zi^}7jKm`yMW8ITFOMN#!QIB6$SWx*75tnCMaNg*_J*WuwBh~AT>0($nS8%&zmFQDp z$dL65niDtTV%!Kg1`6epWoQGNG`$`doy;Zjaa`keyL0F6iJMae6FIgnhAfzU%m@V+ zm5rQihLwS~b6{-bVR1ZSzBI7(Yj+V6T-8V*7I`ptWArGdy~8pnV>fALpi~NQLZ7;^ zpaj35=md<~-(tNmF69UX3?ua}A7UIn)q5i1iPYEGlhYSbkfeX`5epkxtzk3Qbu| zlgA`7ts%IvF4HJ}-98akyRnjCo{u-`A4&b+r?s|o`4wdYAHs-yh91p$7C_|+EdYH5 z10`!*=n+W9g>V&dfU1H!J}ASZi&-?`2IlDOAHnu306rD`y>jT)4^@S(X4XhN2{g9i zj-ym98+RT|d0ejIFJCM5>S{mT-8uGmRRqkJ3sMO_AQDrv77Q zv$t>zaVpVF6eBguE%9M2u?E-Oleft8z5+~W`G}KXD(Yc;7m4{Op>Le(k`g1UK7(1# zt6g}$n=Tdn{T4pu>v!c;xRCd_WI$Ali13x=U_0T!Ga-U~9W88q-lU+RLn2`N8Ouho z^0@SvC>$DguHWx)?^*ms-{PVq%dn(U3vrLj9zITDqQZ`H>Wsp@Gf%}SG=m)Vh}F$ztQAbwVGdDgd!28j&yX9wLW&s! zNR~6`nYg;ULAq8zi<;gUchAV5ib67Y##l2 zy+%gaD(|~G4@||{A;TYDSoS>q2o{t23t-^!NDSDEm8j3ao7Ei>KYLEpb$jz}7ciAM zD}trDN+AVVT_lXW<++~>8>Cj8fzJo@R;>%nGq)6+w?(#mNc#1J4W+!hA}?g$0Xqo? zn67qJmss)e%k(xO*&K@z6+}nHA(lCkb6n-|{pSztys$8HiOWTVR)tCO*Q9~if%3n7`uxGzE+OCu zwcVV|tgQdq60952$>85-GHk$lwM(uI+CU1?i{sVnKd0+UNq#eSSKjUKfDDgLnBG1y z^v?f#MRFkph~TgkoKBvM`L_~we8__xpLcjh`GwV|87q`vazJq?SX=mXhdvK>VqUf~ z4sYoTIpt5S)KrE-?>&=cRoBumD7;b5pq!Y07)#I$`)<@U+mo*dE*P~773p*u^6waO z2#thJahX_ySlYMpjx%h<)i43ao~Is`^Ya zMNZkuChEA7+ZJe6$>-C*dzTYf3#1SY82yFG?S&Q)5rTbKS-XLjckTLEc7>^sFcntQ zBeNXCSg&q1N3Bi^4zlQ%mcEBQ%2ab$?(;t-$HYd2%cnX$uuwU#I_6D3($m zR(>gHzM9ODf;r8b0l5LuEIQVZiQ0-|3Y_xzJkZc*CD=bPJ+&J+>>se%D4uTq?Ny{l z0Z5~og*Wa1O&anlcRWu_%o)(x?IZ0CfUNk_R-ik>GyvdFmpu1wHZaKTDGhL zqxsji)n<+)VKbV0_BRq9E;Kb`f=&vn(BK0Ba-gL?ZN;^^b3YFg6R=!q#zM;tcX0dM zdy5PPx@6pJPXHzH7$dGjM|6@6777nXPWV;CIQdNf(*Znv)sMy&Xcq> zhCq+6h6&v8<0}vd2(sKqU3j>fr7&#Xy%qZHcMU3m{wld^Nstkz8GagB?Y=SI&H z&{&BSA-|(i35$9(l6LpFyLm$0M0fK`Dz!~ezL?yEInsXAFR!bHe;ZL>Gd(#Hv?<$%`^b)oi?x%(jkylCPb=juPlF znMo&o961=NZ_$gd{xp1ZY2dNDOS!=XVj!M^A z+$z`EK4v=m{Bs{&I4W)({`&<5*^BV#z{IBAI_d+9Qx;~ zby?2zEjzUUeZWBDo5cz>%;z||z)<+6UtC)y60yD5J5`oo_zSM;l21@CY<0_|)NME5 zs)kHCMBa5YzB#N=W2aR?y9((~WuYwwf+HAc2mvU>NYlxOTvGf^Ye3za?*f-qUs^`a zT3>RPh9*Jf%3*bf|kqtnD_Buxv!<9N>BbuD#uYv-q^ z%RDnd7a3O4M9Y~TNISS@9K}JDkdg@>x8E6@n8jF=6qiDV+}{!V)(o?ykcr0sxBGEx zo!X;pc=r{H^vw6ztV5VZXBa4~(ujB$rZQ|AaGN@J7#q%2nU9gJ)g6dcj}zYB1& z@iFE0vMQVxa|v7tDHS$gwX$Ihc#M^DXRC>J@Zk?dC(3uB_s~*W&m-01DFMQGWjj5x z5po1@1gPl!v1Yra@qPG{D;$bYLM3qOwpl~7f~l)#n< zP+6`!NYe3EE~4RFR#_e=7YctPRBt6$He@`%e5m}f$M%yzC2S0<1}hRPjO>HJY~ z*dx(nbMbjv*;o&k{qzBdF|lS;UNVKziV=gbLq}UOCwr8GT5E9oRYQ}+>DhbQ1R=lj zgcNJN8|D)$Mx3#c+t@lhqcDUnHGVt0&EyQ{b5)=52B(VTzw=pQ^ba3`JB@BU^lS`_ zJEiLzgU#Acd_!}FMxCWC**FP^i#P}bYzNs78)#uSejEtYLbG>JJ7Igtho2oKQ;XW~ z4eMGO+t!_;G^V6c&R`5Tg+Pz2ToN(aybq4Q0ssie_{`t*DO%V7FaZ`{MBobFc9|pV z70o5ayHGJo9$$&Pgbs)pWNzduAcbh?~U?_P)(ve0S*3H%eNF&a5XR=!J#4c z;t992n7ZJr{*%`^dU1d-ALE8!3i#v;3r4r%j+JFCe=%3Vj=8{aXe zs)jrcUBZ=;LudcTUXj2ub>K5!{HHFHJ}Trx(PYugbQ8yK7&sqX;(;|UWjk3tGs3zuceeX)i4i_jA8Qz2Bc%DxN8 zXw!$+9jBtEHd1y90bYG4f8DcJM)Ab!M39tH5zz94*MAvnhA377@buNupSOUU3j8~> zd6&hk^ENRCp9T?_QUHk<=(&9Q^MJ^pi;nKOYNR@?L=RCSmKMJ5UQJQ`X!i~(gD*P! zs`RobzJG3Ra_Pg+WZUXUmMU$ilpwfcEti6)mw(~MZ0q!^sza>#jv!-+7B6F3QuMWg zVO!rXwD+lF1BBTito?ml-CV3vxuek~TKuOX^N6sol$v*{_%nAuD7i81eXm^Lz(Z~I z2Xj_Dts#G0&C;PV_Wkq*1QvB7+Post4={v;gk7b9u%#DC_bh(iJm$rqog^{JEx6NE zrs5^2SEL$|98#2WV#iG@L6cq|)SuTMSfGocPl65wUd^|5Lbpnb(;t>-Qu2jvANLgv zdte0vED-3C@^BdyHWLL(7{G$WA02z@JG!T-U^Q7HZ(7Bs&vchkh(p&}KvnS{MG^i6 z4r){gJp9p7WyWOEiKA2Cm6EXIn&&gk|Fc6^78OpPrX4ExCFE=SD$xcH;C2eB^{XTI zaxz_Cef*Yj==w_i_BTGXP;8C&f? z*QEM>={jFM8)lWAR870pG4XEWsl%%K|82S5b=9hVz7p_6i-d(Iyvq76&a#PV zR;VbQV|n?mg}&(ehClg%tK%IjgtnTR-u)lxH06XxXqH0soAZbB_Rm)XX=6Nge1uoG7 z9vQM_S~2h53n|W`y{{R9+=08rv~MohI_v4-BU^7fZ0-A}#b5{AOSTJm+(J;9yw%pD zX6u62GJ&@HKX5zQwq~j8T!Hrv-Mk^QSB5cu09L03{ToDO7jikM0WAcsjW>D}^jqCF zT0DEZ@K^KO_MD*%M!+V)lGVU6?LpX)eQVXEmq}R`NIJv;kBitJ!nW?0OxTVlu2ADf zE{A!*0g3%nwVcBD+AgT5bGx@WOnQk{zRpiZ4HhP`3BF%N|HdqPbbiV5)7x)kzC3ID zZ;27>0^mrMgWc7evsbQY`l`l})wr+e;=8U_!2&B77;1qL!N8y)eTJ2lf#CvhR~!Qa mc;sM|90DP5A*JW%f2r=u1xt!e4gwD_V(@hJb6Mw<&;$SznOm^{ literal 0 HcmV?d00001 diff --git a/test/packages/other/synthetic_mode/manifest.yml b/test/packages/other/synthetic_mode/manifest.yml new file mode 100644 index 0000000000..b4895897f4 --- /dev/null +++ b/test/packages/other/synthetic_mode/manifest.yml @@ -0,0 +1,33 @@ +format_version: 2.5.1 +name: synthetic_mode +title: "Synthetic Mode" +version: 0.0.1 +source: + license: "Elastic-2.0" +description: "This package contains data streams with synthetic mode enabled" +type: integration +categories: + - custom +conditions: + kibana.version: "^8.6.0" + elastic.subscription: "basic" +screenshots: + - src: /img/sample-screenshot.png + title: Sample screenshot + size: 600x600 + type: image/png +icons: + - src: /img/sample-logo.svg + title: Sample logo + size: 32x32 + type: image/svg+xml +policy_templates: + - name: prometheus + title: Prometheus metrics + description: Collect metrics from Prometheus instances + inputs: + - type: prometheus/metrics + title: Collect Prometheus metrics + description: Collecting metrics from Prometheus exporters, Prometheus query API as well as able to receive metrics via remote write functionality. +owner: + github: elastic/integrations From 3bd4cf1175f9c74d58822a310a316d90bbb3c210 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 1 Jun 2023 11:30:01 +0200 Subject: [PATCH 19/34] Add sample_event --- .../data_stream/synthetic/fields/ecs.yml | 5 +- .../data_stream/synthetic/manifest.yml | 2 + .../data_stream/synthetic/sample_event.json | 205 ++++++++++++------ 3 files changed, 141 insertions(+), 71 deletions(-) diff --git a/test/packages/other/synthetic_mode/data_stream/synthetic/fields/ecs.yml b/test/packages/other/synthetic_mode/data_stream/synthetic/fields/ecs.yml index aabd8a9b9c..b98c11b26e 100644 --- a/test/packages/other/synthetic_mode/data_stream/synthetic/fields/ecs.yml +++ b/test/packages/other/synthetic_mode/data_stream/synthetic/fields/ecs.yml @@ -4,4 +4,7 @@ name: service.address - external: ecs name: service.type - +- external: ecs + name: event.category +- external: ecs + name: event.type diff --git a/test/packages/other/synthetic_mode/data_stream/synthetic/manifest.yml b/test/packages/other/synthetic_mode/data_stream/synthetic/manifest.yml index b6e5e5c12d..907b04f12d 100644 --- a/test/packages/other/synthetic_mode/data_stream/synthetic/manifest.yml +++ b/test/packages/other/synthetic_mode/data_stream/synthetic/manifest.yml @@ -143,3 +143,5 @@ streams: title: Prometheus collector metrics enabled: true description: Collect Prometheus collector metrics +elasticsearch: + source_mode: synthetic diff --git a/test/packages/other/synthetic_mode/data_stream/synthetic/sample_event.json b/test/packages/other/synthetic_mode/data_stream/synthetic/sample_event.json index 40dea34c86..cd93510924 100644 --- a/test/packages/other/synthetic_mode/data_stream/synthetic/sample_event.json +++ b/test/packages/other/synthetic_mode/data_stream/synthetic/sample_event.json @@ -1,72 +1,137 @@ { - "@timestamp": "2023-05-31T19:55:04.734Z", - "agent": { - "ephemeral_id": "d8c2f32e-9fa7-419a-bdd0-a93604aea83a", - "id": "63a71457-c239-40ed-b42d-1a0e8b523875", - "name": "docker-fleet-agent", - "type": "metricbeat", - "version": "8.6.2" - }, - "data_stream": { - "dataset": "synthetic_mode.synthetic", - "namespace": "ep", - "type": "metrics" - }, - "ecs": { - "version": "8.0.0" - }, - "elastic_agent": { - "id": "63a71457-c239-40ed-b42d-1a0e8b523875", - "snapshot": false, - "version": "8.6.2" - }, - "event": { - "agent_id_status": "verified", - "dataset": "synthetic_mode.synthetic", - "duration": 3889366, - "ingested": "2023-05-31T19:55:05Z", - "module": "prometheus" - }, - "host": { - "architecture": "x86_64", - "containerized": false, - "hostname": "docker-fleet-agent", - "id": "f91b175388d443fca5c155815dfc2279", - "ip": [ - "172.30.0.7" - ], - "mac": [ - "02-42-AC-1E-00-07" - ], - "name": "docker-fleet-agent", - "os": { - "codename": "focal", - "family": "debian", - "kernel": "5.19.0-42-generic", - "name": "Ubuntu", - "platform": "ubuntu", - "type": "linux", - "version": "20.04.5 LTS (Focal Fossa)" - } - }, - "metricset": { - "name": "collector", - "period": 10000 - }, - "prometheus": { - "labels": { - "event": "update", - "instance": "elastic-package-service-synthetic_mode-1:9090", - "job": "prometheus", - "role": "node" - }, - "prometheus_sd_kubernetes_events_total": { - "counter": 0, - "rate": 0 - } - }, - "service": { - "address": "http://elastic-package-service-synthetic_mode-1:9090/metrics", - "type": "prometheus" - } + "@timestamp": [ + "2023-05-31T21:15:05.264Z" + ], + "agent.ephemeral_id": [ + "5dd078c2-0f0d-4eb7-934a-f47d4952cd82" + ], + "agent.id": [ + "cc2b0c0e-a327-446c-afed-cd2fa957bcdb" + ], + "agent.name": [ + "docker-fleet-agent" + ], + "agent.type": [ + "metricbeat" + ], + "agent.version": [ + "8.7.1" + ], + "data_stream.dataset": [ + "synthetic_mode.synthetic" + ], + "data_stream.namespace": [ + "ep" + ], + "data_stream.type": [ + "metrics" + ], + "ecs.version": [ + "8.7.0" + ], + "elastic_agent.id": [ + "cc2b0c0e-a327-446c-afed-cd2fa957bcdb" + ], + "elastic_agent.snapshot": [ + false + ], + "elastic_agent.version": [ + "8.7.1" + ], + "event.agent_id_status": [ + "verified" + ], + "event.category": [ + "web" + ], + "event.dataset": [ + "synthetic_mode.synthetic" + ], + "event.duration": [ + 3904515 + ], + "event.ingested": [ + "2023-05-31T21:15:06Z" + ], + "event.kind": [ + "metric" + ], + "event.module": [ + "apache_tomcat" + ], + "event.type": [ + "info" + ], + "host.architecture": [ + "x86_64" + ], + "host.containerized": [ + false + ], + "host.hostname": [ + "docker-fleet-agent" + ], + "host.id": [ + "fd2c4b0943e444508c12855a04d117c7" + ], + "host.ip": [ + "192.168.160.7" + ], + "host.mac": [ + "02-42-C0-A8-A0-07" + ], + "host.name": [ + "docker-fleet-agent" + ], + "host.os.codename": [ + "focal" + ], + "host.os.family": [ + "debian" + ], + "host.os.kernel": [ + "5.19.0-42-generic" + ], + "host.os.name": [ + "Ubuntu" + ], + "host.os.platform": [ + "ubuntu" + ], + "host.os.type": [ + "linux" + ], + "host.os.version": [ + "20.04.6 LTS (Focal Fossa)" + ], + "metricset.name": [ + "collector" + ], + "metricset.period": [ + 10000 + ], + "prometheus.labels.event": [ + "update" + ], + "prometheus.labels.instance": [ + "elastic-package-service-synthetic_mode-1:9090" + ], + "prometheus.labels.job": [ + "prometheus" + ], + "prometheus.labels.role": [ + "node" + ], + "prometheus.prometheus_sd_kubernetes_events_total.counter": [ + 0 + ], + "prometheus.prometheus_sd_kubernetes_events_total.rate": [ + 0 + ], + "service.address": [ + "http://elastic-package-service-synthetic_mode-1:9090/metrics" + ], + "service.type": [ + "prometheus" + ] } \ No newline at end of file From 1a319cc53e9f1ced5db2a2d9703c8d2910345c02 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Fri, 2 Jun 2023 17:29:48 +0200 Subject: [PATCH 20/34] update settings test package --- test/packages/other/synthetic_mode/_dev/build/build.yml | 1 + test/packages/other/synthetic_mode/manifest.yml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/test/packages/other/synthetic_mode/_dev/build/build.yml b/test/packages/other/synthetic_mode/_dev/build/build.yml index e3c4cca65a..883ced510c 100644 --- a/test/packages/other/synthetic_mode/_dev/build/build.yml +++ b/test/packages/other/synthetic_mode/_dev/build/build.yml @@ -1,3 +1,4 @@ dependencies: ecs: reference: git@v8.7.0 + import_mappings: true diff --git a/test/packages/other/synthetic_mode/manifest.yml b/test/packages/other/synthetic_mode/manifest.yml index b4895897f4..a2a4bd50e9 100644 --- a/test/packages/other/synthetic_mode/manifest.yml +++ b/test/packages/other/synthetic_mode/manifest.yml @@ -1,4 +1,4 @@ -format_version: 2.5.1 +format_version: 2.8.0 name: synthetic_mode title: "Synthetic Mode" version: 0.0.1 From e3c0a6d121ec75efeda809fc27cf48b0f9ebac03 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Fri, 2 Jun 2023 17:39:44 +0200 Subject: [PATCH 21/34] Sanitize docs when synthetic mode is enabled --- internal/fields/validate.go | 60 ++++++++ internal/testrunner/runners/system/runner.go | 8 + .../data_stream/synthetic/sample_event.json | 144 +++++------------- 3 files changed, 109 insertions(+), 103 deletions(-) diff --git a/internal/fields/validate.go b/internal/fields/validate.go index d7b8c41d89..1c0d5a5b4c 100644 --- a/internal/fields/validate.go +++ b/internal/fields/validate.go @@ -387,6 +387,66 @@ func (v *Validator) validateScalarElement(key string, val interface{}, doc commo return nil } +func (v *Validator) SanitizeDocs(docs []common.MapStr) ([]common.MapStr, error) { + for _, doc := range docs { + for key, contents := range doc { + logger.Debugf("Look for field with key %q", key) + + definition := FindElementDefinition(key, v.Schema) + if definition == nil { + continue + } + + // resolve external fields + if !v.disabledDependencyManagement && definition.External != "" { + def, err := v.FieldDependencyManager.ImportField(definition.External, key) + if err != nil { + return nil, fmt.Errorf("can't import field (field: %s): %w", key, err) + } + definition = &def + } + logger.Debugf("Found definition for %s:\n%+v", key, definition) + + shouldBeNormalized := false + if v.disabledNormalization && !v.specVersion.LessThan(semver2_0_0) { + for _, normalize := range definition.Normalize { + switch normalize { + case "array": + shouldBeNormalized = true + break + } + } + } + if shouldBeNormalized { + logger.Debugf("Skip changes key %s must be normalized", key) + continue + } + // is an array of just one element ? + vals, ok := contents.([]interface{}) + if !ok { + logger.Debugf("key %s just has one element: %v", key, contents) + // logger.Debugf("should be normalized: %t", shouldBeNormalized) + // // one element + + // logger.Debugf("Updating key %s to be an array", key) + // _, err := doc.Put(key, []interface{}{contents}) + // if err != nil { + // return nil, fmt.Errorf("key %s was not updated: %w", key, err) + // } + + continue + } + if len(vals) == 1 { + _, err := doc.Put(key, vals[0]) + if err != nil { + return nil, fmt.Errorf("key %s was not updated: %w", key, err) + } + } + } + } + return docs, nil +} + func isNumericKeyword(definition FieldDefinition, val interface{}) bool { _, isNumber := val.(float64) return isNumber && (definition.Type == "keyword" || definition.Type == "constant_keyword") diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index 30af3338e3..07af9cc5fa 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -593,6 +593,13 @@ func (r *runner) runTest(config *testConfig, ctxt servicedeployer.ServiceContext return result.WithError(err) } + if syntheticEnabled { + docs, err = fieldsValidator.SanitizeDocs(docs) + if err != nil { + return result.WithError(err) + } + } + // Write sample events file from first doc, if requested if err := r.generateTestResult(docs); err != nil { return result.WithError(err) @@ -936,6 +943,7 @@ func filterAgents(allAgents []kibana.Agent, ctx servicedeployer.ServiceContext) } func writeSampleEvent(path string, doc common.MapStr) error { + body, err := json.MarshalIndent(doc, "", " ") if err != nil { return errors.Wrap(err, "marshalling sample event failed") diff --git a/test/packages/other/synthetic_mode/data_stream/synthetic/sample_event.json b/test/packages/other/synthetic_mode/data_stream/synthetic/sample_event.json index cd93510924..b2482209bf 100644 --- a/test/packages/other/synthetic_mode/data_stream/synthetic/sample_event.json +++ b/test/packages/other/synthetic_mode/data_stream/synthetic/sample_event.json @@ -1,36 +1,16 @@ { - "@timestamp": [ - "2023-05-31T21:15:05.264Z" - ], - "agent.ephemeral_id": [ - "5dd078c2-0f0d-4eb7-934a-f47d4952cd82" - ], - "agent.id": [ - "cc2b0c0e-a327-446c-afed-cd2fa957bcdb" - ], - "agent.name": [ - "docker-fleet-agent" - ], - "agent.type": [ - "metricbeat" - ], - "agent.version": [ - "8.7.1" - ], - "data_stream.dataset": [ - "synthetic_mode.synthetic" - ], - "data_stream.namespace": [ - "ep" - ], - "data_stream.type": [ - "metrics" - ], - "ecs.version": [ - "8.7.0" - ], + "@timestamp": "2023-06-02T15:21:39.436Z", + "agent.ephemeral_id": "d5a061f5-0aac-4c11-8496-7f2c42f2393c", + "agent.id": "bc2c6d4c-79fd-4368-ae8b-540913be01fe", + "agent.name": "docker-fleet-agent", + "agent.type": "metricbeat", + "agent.version": "8.7.1", + "data_stream.dataset": "synthetic_mode.synthetic", + "data_stream.namespace": "ep", + "data_stream.type": "metrics", + "ecs.version": "8.7.0", "elastic_agent.id": [ - "cc2b0c0e-a327-446c-afed-cd2fa957bcdb" + "bc2c6d4c-79fd-4368-ae8b-540913be01fe" ], "elastic_agent.snapshot": [ false @@ -38,100 +18,58 @@ "elastic_agent.version": [ "8.7.1" ], - "event.agent_id_status": [ - "verified" - ], + "event.agent_id_status": "verified", "event.category": [ "web" ], - "event.dataset": [ - "synthetic_mode.synthetic" - ], - "event.duration": [ - 3904515 - ], - "event.ingested": [ - "2023-05-31T21:15:06Z" - ], - "event.kind": [ - "metric" - ], - "event.module": [ - "apache_tomcat" - ], + "event.dataset": "synthetic_mode.synthetic", + "event.duration": 5392852, + "event.ingested": "2023-06-02T15:21:40Z", + "event.kind": "metric", + "event.module": "apache_tomcat", "event.type": [ "info" ], - "host.architecture": [ - "x86_64" - ], + "host.architecture": "x86_64", "host.containerized": [ false ], - "host.hostname": [ - "docker-fleet-agent" - ], - "host.id": [ - "fd2c4b0943e444508c12855a04d117c7" - ], + "host.hostname": "docker-fleet-agent", + "host.id": "fd2c4b0943e444508c12855a04d117c7", "host.ip": [ - "192.168.160.7" + "192.168.224.7" ], "host.mac": [ - "02-42-C0-A8-A0-07" - ], - "host.name": [ - "docker-fleet-agent" + "02-42-C0-A8-E0-07" ], + "host.name": "docker-fleet-agent", "host.os.codename": [ "focal" ], - "host.os.family": [ - "debian" - ], - "host.os.kernel": [ - "5.19.0-42-generic" - ], - "host.os.name": [ + "host.os.family": "debian", + "host.os.kernel": "5.19.0-42-generic", + "host.os.name": "Ubuntu", + "host.os.name.text": [ "Ubuntu" ], - "host.os.platform": [ - "ubuntu" - ], - "host.os.type": [ - "linux" - ], - "host.os.version": [ - "20.04.6 LTS (Focal Fossa)" - ], + "host.os.platform": "ubuntu", + "host.os.type": "linux", + "host.os.version": "20.04.6 LTS (Focal Fossa)", "metricset.name": [ "collector" ], + "metricset.name.text": [ + "collector" + ], "metricset.period": [ 10000 ], - "prometheus.labels.event": [ - "update" - ], - "prometheus.labels.instance": [ - "elastic-package-service-synthetic_mode-1:9090" - ], - "prometheus.labels.job": [ - "prometheus" - ], - "prometheus.labels.role": [ - "node" - ], - "prometheus.prometheus_sd_kubernetes_events_total.counter": [ - 0 - ], - "prometheus.prometheus_sd_kubernetes_events_total.rate": [ - 0 - ], - "service.address": [ - "http://elastic-package-service-synthetic_mode-1:9090/metrics" - ], - "service.type": [ - "prometheus" - ] + "prometheus.labels.dialer_name": "prometheus", + "prometheus.labels.instance": "elastic-package-service-synthetic_mode-1:9090", + "prometheus.labels.job": "prometheus", + "prometheus.labels.reason": "resolution", + "prometheus.net_conntrack_dialer_conn_failed_total.counter": 0, + "prometheus.net_conntrack_dialer_conn_failed_total.rate": 0, + "service.address": "http://elastic-package-service-synthetic_mode-1:9090/metrics", + "service.type": "prometheus" } \ No newline at end of file From 6afdeb212cac71175014c82a69dd8760066cea31 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Fri, 2 Jun 2023 17:40:45 +0200 Subject: [PATCH 22/34] comment debug messages --- internal/fields/validate.go | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/internal/fields/validate.go b/internal/fields/validate.go index 1c0d5a5b4c..c9c0213e34 100644 --- a/internal/fields/validate.go +++ b/internal/fields/validate.go @@ -390,7 +390,7 @@ func (v *Validator) validateScalarElement(key string, val interface{}, doc commo func (v *Validator) SanitizeDocs(docs []common.MapStr) ([]common.MapStr, error) { for _, doc := range docs { for key, contents := range doc { - logger.Debugf("Look for field with key %q", key) + // logger.Debugf("Look for field with key %q", key) definition := FindElementDefinition(key, v.Schema) if definition == nil { @@ -405,7 +405,7 @@ func (v *Validator) SanitizeDocs(docs []common.MapStr) ([]common.MapStr, error) } definition = &def } - logger.Debugf("Found definition for %s:\n%+v", key, definition) + // logger.Debugf("Found definition for %s:\n%+v", key, definition) shouldBeNormalized := false if v.disabledNormalization && !v.specVersion.LessThan(semver2_0_0) { @@ -418,22 +418,13 @@ func (v *Validator) SanitizeDocs(docs []common.MapStr) ([]common.MapStr, error) } } if shouldBeNormalized { - logger.Debugf("Skip changes key %s must be normalized", key) + // logger.Debugf("Skip changes key %s must be normalized", key) continue } // is an array of just one element ? vals, ok := contents.([]interface{}) if !ok { - logger.Debugf("key %s just has one element: %v", key, contents) - // logger.Debugf("should be normalized: %t", shouldBeNormalized) - // // one element - - // logger.Debugf("Updating key %s to be an array", key) - // _, err := doc.Put(key, []interface{}{contents}) - // if err != nil { - // return nil, fmt.Errorf("key %s was not updated: %w", key, err) - // } - + // logger.Debugf("key %s just has one element: %v", key, contents) continue } if len(vals) == 1 { From bf5523472e860f60ea3bafaa88f995e9244084f3 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Fri, 2 Jun 2023 17:49:45 +0200 Subject: [PATCH 23/34] Solve redundant break statement --- internal/fields/validate.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/internal/fields/validate.go b/internal/fields/validate.go index c9c0213e34..bfb0d134c1 100644 --- a/internal/fields/validate.go +++ b/internal/fields/validate.go @@ -397,7 +397,6 @@ func (v *Validator) SanitizeDocs(docs []common.MapStr) ([]common.MapStr, error) continue } - // resolve external fields if !v.disabledDependencyManagement && definition.External != "" { def, err := v.FieldDependencyManager.ImportField(definition.External, key) if err != nil { @@ -408,20 +407,27 @@ func (v *Validator) SanitizeDocs(docs []common.MapStr) ([]common.MapStr, error) // logger.Debugf("Found definition for %s:\n%+v", key, definition) shouldBeNormalized := false + // normalization should just be checked if synthetic source is enabled and the + // spec version of this package is >= 2.0.0 if v.disabledNormalization && !v.specVersion.LessThan(semver2_0_0) { for _, normalize := range definition.Normalize { switch normalize { case "array": shouldBeNormalized = true + } + if shouldBeNormalized { break } } } + // if it needs to be normalized, the field is kept as it is if shouldBeNormalized { // logger.Debugf("Skip changes key %s must be normalized", key) continue } - // is an array of just one element ? + // in case it is not specified any normalization and that field is an array of + // just one element, the field is going to be updated to remove the array and keep + // that element as a value. vals, ok := contents.([]interface{}) if !ok { // logger.Debugf("key %s just has one element: %v", key, contents) From 14179f6c0340ef1cdc8fb66028b04d897c5bb70b Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Fri, 2 Jun 2023 18:18:15 +0200 Subject: [PATCH 24/34] Fix lint --- .../other/synthetic_mode/_dev/deploy/docker/docker-compose.yml | 1 - .../other/synthetic_mode/data_stream/synthetic/fields/fields.yml | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/test/packages/other/synthetic_mode/_dev/deploy/docker/docker-compose.yml b/test/packages/other/synthetic_mode/_dev/deploy/docker/docker-compose.yml index 133131ed73..c8d97af0bd 100644 --- a/test/packages/other/synthetic_mode/_dev/deploy/docker/docker-compose.yml +++ b/test/packages/other/synthetic_mode/_dev/deploy/docker/docker-compose.yml @@ -4,4 +4,3 @@ services: image: prom/prometheus:${PROMETHEUS_VERSION:-prometheus_2} ports: - 9090:9090 - diff --git a/test/packages/other/synthetic_mode/data_stream/synthetic/fields/fields.yml b/test/packages/other/synthetic_mode/data_stream/synthetic/fields/fields.yml index 1f25ea1309..01fd9ee6b7 100644 --- a/test/packages/other/synthetic_mode/data_stream/synthetic/fields/fields.yml +++ b/test/packages/other/synthetic_mode/data_stream/synthetic/fields/fields.yml @@ -39,3 +39,4 @@ object_type_mapping_type: "*" description: > Prometheus histogram metric + From 597d17c9a79aab70be088c671b9aa433bb67367a Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Mon, 5 Jun 2023 13:03:55 +0200 Subject: [PATCH 25/34] Rename function --- internal/fields/validate.go | 9 ++------- internal/testrunner/runners/system/runner.go | 2 +- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/internal/fields/validate.go b/internal/fields/validate.go index bfb0d134c1..7fbd986022 100644 --- a/internal/fields/validate.go +++ b/internal/fields/validate.go @@ -387,11 +387,9 @@ func (v *Validator) validateScalarElement(key string, val interface{}, doc commo return nil } -func (v *Validator) SanitizeDocs(docs []common.MapStr) ([]common.MapStr, error) { +func (v *Validator) SanitizeSyntheticsDocs(docs []common.MapStr) ([]common.MapStr, error) { for _, doc := range docs { for key, contents := range doc { - // logger.Debugf("Look for field with key %q", key) - definition := FindElementDefinition(key, v.Schema) if definition == nil { continue @@ -404,7 +402,6 @@ func (v *Validator) SanitizeDocs(docs []common.MapStr) ([]common.MapStr, error) } definition = &def } - // logger.Debugf("Found definition for %s:\n%+v", key, definition) shouldBeNormalized := false // normalization should just be checked if synthetic source is enabled and the @@ -422,7 +419,6 @@ func (v *Validator) SanitizeDocs(docs []common.MapStr) ([]common.MapStr, error) } // if it needs to be normalized, the field is kept as it is if shouldBeNormalized { - // logger.Debugf("Skip changes key %s must be normalized", key) continue } // in case it is not specified any normalization and that field is an array of @@ -430,13 +426,12 @@ func (v *Validator) SanitizeDocs(docs []common.MapStr) ([]common.MapStr, error) // that element as a value. vals, ok := contents.([]interface{}) if !ok { - // logger.Debugf("key %s just has one element: %v", key, contents) continue } if len(vals) == 1 { _, err := doc.Put(key, vals[0]) if err != nil { - return nil, fmt.Errorf("key %s was not updated: %w", key, err) + return nil, fmt.Errorf("key %s could not be updated: %w", key, err) } } } diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index 07af9cc5fa..0fded67a31 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -594,7 +594,7 @@ func (r *runner) runTest(config *testConfig, ctxt servicedeployer.ServiceContext } if syntheticEnabled { - docs, err = fieldsValidator.SanitizeDocs(docs) + docs, err = fieldsValidator.SanitizeSyntheticsDocs(docs) if err != nil { return result.WithError(err) } From 538c1c2e9d6a3b2bbb5a3cfebc406c853106b66d Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 6 Jun 2023 10:48:42 +0200 Subject: [PATCH 26/34] Extract function --- internal/fields/validate.go | 34 +++++++++++++++------------------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/internal/fields/validate.go b/internal/fields/validate.go index 7fbd986022..daef58b376 100644 --- a/internal/fields/validate.go +++ b/internal/fields/validate.go @@ -283,25 +283,7 @@ func (v *Validator) validateDocumentValues(body common.MapStr) multierror.Error continue } - var str string - var ok bool - if v.disabledNormalization { - // when synthetics mode is enabled, each field present in the document is an array - // so this check needs to retrieve the first element of the array - ok = true - vals, err := common.ToStringSlice(value) - if err != nil { - ok = false - } - if err == nil && len(vals) != 1 { - ok = false - } - if err == nil && len(vals) == 1 { - str = vals[0] - } - } else { - str, ok = value.(string) - } + str, ok := valueToString(value, v.disabledNormalization) if !ok || str != v.expectedDataset { err := fmt.Errorf("field %q should have value %q, it has \"%v\"", datasetField, v.expectedDataset, value) @@ -312,6 +294,20 @@ func (v *Validator) validateDocumentValues(body common.MapStr) multierror.Error return errs } +func valueToString(value any, disabledNormalization bool) (string, bool) { + if disabledNormalization { + // when synthetics mode is enabled, each field present in the document is an array + // so this check needs to retrieve the first element of the array + vals, err := common.ToStringSlice(value) + if err != nil || len(vals) != 1 { + return "", false + } + return vals[0], true + } + str, ok := value.(string) + return str, ok +} + func (v *Validator) validateMapElement(root string, elem common.MapStr, doc common.MapStr) multierror.Error { var errs multierror.Error for name, val := range elem { From abe7a0f1089faf93fc6c75cbc6673027eb9f88ef Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 6 Jun 2023 10:49:07 +0200 Subject: [PATCH 27/34] Extract function --- internal/fields/validate.go | 2 +- internal/testrunner/runners/system/runner.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/fields/validate.go b/internal/fields/validate.go index daef58b376..0b0594db57 100644 --- a/internal/fields/validate.go +++ b/internal/fields/validate.go @@ -383,7 +383,7 @@ func (v *Validator) validateScalarElement(key string, val interface{}, doc commo return nil } -func (v *Validator) SanitizeSyntheticsDocs(docs []common.MapStr) ([]common.MapStr, error) { +func (v *Validator) SanitizeSyntheticSourceDocs(docs []common.MapStr) ([]common.MapStr, error) { for _, doc := range docs { for key, contents := range doc { definition := FindElementDefinition(key, v.Schema) diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index 50f6c3a718..30579eee15 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -646,7 +646,7 @@ func (r *runner) runTest(config *testConfig, ctxt servicedeployer.ServiceContext } if syntheticEnabled { - docs, err = fieldsValidator.SanitizeSyntheticsDocs(docs) + docs, err = fieldsValidator.SanitizeSyntheticSourceDocs(docs) if err != nil { return result.WithError(err) } From e669293e0b89ca35e91ff876706930f9f2757046 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 6 Jun 2023 10:50:54 +0200 Subject: [PATCH 28/34] Rename --- internal/fields/validate.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/fields/validate.go b/internal/fields/validate.go index 0b0594db57..d4039c93c4 100644 --- a/internal/fields/validate.go +++ b/internal/fields/validate.go @@ -399,22 +399,22 @@ func (v *Validator) SanitizeSyntheticSourceDocs(docs []common.MapStr) ([]common. definition = &def } - shouldBeNormalized := false + shouldBeArray := false // normalization should just be checked if synthetic source is enabled and the // spec version of this package is >= 2.0.0 if v.disabledNormalization && !v.specVersion.LessThan(semver2_0_0) { for _, normalize := range definition.Normalize { switch normalize { case "array": - shouldBeNormalized = true + shouldBeArray = true } - if shouldBeNormalized { + if shouldBeArray { break } } } // if it needs to be normalized, the field is kept as it is - if shouldBeNormalized { + if shouldBeArray { continue } // in case it is not specified any normalization and that field is an array of From e3003d2dd963aecf39ca8a14d9745c7d52997993 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 6 Jun 2023 11:12:48 +0200 Subject: [PATCH 29/34] Move to a constant all fields body --- internal/testrunner/runners/system/runner.go | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index 30579eee15..0ee0211917 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -7,7 +7,6 @@ package system import ( "encoding/json" "fmt" - "io" "math/rand" "os" "path/filepath" @@ -34,6 +33,8 @@ import ( const ( testRunMaxID = 99999 testRunMinID = 10000 + + allFieldsBody = `{"fields": ["*"]}` ) func init() { @@ -296,16 +297,6 @@ func createTestRunID() string { return fmt.Sprintf("%d", rand.Intn(testRunMaxID-testRunMinID)+testRunMinID) } -func buildAllFieldsBody() io.Reader { - var b strings.Builder - b.WriteString("\n") - b.WriteString(`{ - "fields": ["*"] - }`) - b.WriteString("\n") - return strings.NewReader(b.String()) -} - func (r *runner) isSyntheticsEnabled(dataStream, componentTemplatePackage string) (bool, error) { logger.Debugf("check whether or not synthetics is enabled (component template %s)...", componentTemplatePackage) resp, err := r.options.API.Cluster.GetComponentTemplate( @@ -374,7 +365,7 @@ func (r *runner) getDocs(dataStream string) (*hits, error) { r.options.API.Search.WithSort("@timestamp:asc"), r.options.API.Search.WithSize(elasticsearchQuerySize), r.options.API.Search.WithSource("true"), - r.options.API.Search.WithBody(buildAllFieldsBody()), + r.options.API.Search.WithBody(strings.NewReader(allFieldsBody)), ) if err != nil { return nil, errors.Wrap(err, "could not search data stream") From 935beb5b4336adafedba9658fc86950167c0064c Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 6 Jun 2023 13:01:54 +0200 Subject: [PATCH 30/34] Add context to error --- internal/testrunner/runners/system/runner.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index 0ee0211917..931cac1621 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -609,7 +609,7 @@ func (r *runner) runTest(config *testConfig, ctxt servicedeployer.ServiceContext syntheticEnabled, err := r.isSyntheticsEnabled(dataStream, componentTemplatePackage) if err != nil { - return result.WithError(err) + return result.WithError(fmt.Errorf("failed to check if synthetic source is enabled", err)) } logger.Debugf("data stream %s has synthetics enabled: %t", dataStream, syntheticEnabled) From b1d4921cb5c9aadf10b920c46580bd486904afc3 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 6 Jun 2023 13:08:59 +0200 Subject: [PATCH 31/34] Add more context to some erros --- internal/testrunner/runners/system/runner.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index 931cac1621..7368c54abf 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -609,7 +609,7 @@ func (r *runner) runTest(config *testConfig, ctxt servicedeployer.ServiceContext syntheticEnabled, err := r.isSyntheticsEnabled(dataStream, componentTemplatePackage) if err != nil { - return result.WithError(fmt.Errorf("failed to check if synthetic source is enabled", err)) + return result.WithError(fmt.Errorf("failed to check if synthetic source is enabled: %w", err)) } logger.Debugf("data stream %s has synthetics enabled: %t", dataStream, syntheticEnabled) @@ -639,7 +639,7 @@ func (r *runner) runTest(config *testConfig, ctxt servicedeployer.ServiceContext if syntheticEnabled { docs, err = fieldsValidator.SanitizeSyntheticSourceDocs(docs) if err != nil { - return result.WithError(err) + return result.WithError(fmt.Errorf("failed to sanitize synthetic source docs: %w", err)) } } From 4855ce29ca3085c0794985bfaa1098096507fefa Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 7 Jun 2023 10:25:24 +0200 Subject: [PATCH 32/34] Expand objects in the docs retrieved from Elasticsearch Expand objects when docs retrieved from Elasticsearch and synthetic mode is enabled. That would keep the same format of the sample_event.json when that file is generated. --- internal/fields/validate.go | 40 ++++- internal/testrunner/runners/system/runner.go | 1 - .../elasticsearch/ingest_pipeline/default.yml | 17 ++ .../data_stream/synthetic/sample_event.json | 165 ++++++++++-------- 4 files changed, 148 insertions(+), 75 deletions(-) create mode 100644 test/packages/other/synthetic_mode/data_stream/synthetic/elasticsearch/ingest_pipeline/default.yml diff --git a/internal/fields/validate.go b/internal/fields/validate.go index d4039c93c4..bfdcd2e55c 100644 --- a/internal/fields/validate.go +++ b/internal/fields/validate.go @@ -13,6 +13,7 @@ import ( "os" "path/filepath" "regexp" + "sort" "strings" "github.com/Masterminds/semver/v3" @@ -384,6 +385,7 @@ func (v *Validator) validateScalarElement(key string, val interface{}, doc commo } func (v *Validator) SanitizeSyntheticSourceDocs(docs []common.MapStr) ([]common.MapStr, error) { + var newDocs []common.MapStr for _, doc := range docs { for key, contents := range doc { definition := FindElementDefinition(key, v.Schema) @@ -431,10 +433,46 @@ func (v *Validator) SanitizeSyntheticSourceDocs(docs []common.MapStr) ([]common. } } } + expandedDoc, err := createDocExpandingObjects(doc) + if err != nil { + return nil, fmt.Errorf("failure while expanding objects from doc: %w", err) + } + + newDocs = append(newDocs, expandedDoc) } - return docs, nil + return newDocs, nil } +func createDocExpandingObjects(doc common.MapStr) (common.MapStr, error) { + keys := make([]string, 0) + for k, _ := range doc { + keys = append(keys, k) + } + sort.Strings(keys) + + newDoc := make(common.MapStr) + for _, k := range keys { + value, err := doc.GetValue(k) + if err != nil { + return nil, fmt.Errorf("not found key %s: %w", k, err) + } + + _, err = newDoc.Put(k, value) + if err == nil { + continue + } + + // Possible errors found but not limited to those + // - expected map but type is string + // - expected map but type is []interface{} + if strings.HasPrefix(err.Error(), "expected map but type is") { + logger.Warnf("not able to add key %s, is this a multifield?: %s", k, err) + continue + } + return nil, fmt.Errorf("not added key %s with value %s: %w", k, value, err) + } + return newDoc, nil +} func isNumericKeyword(definition FieldDefinition, val interface{}) bool { _, isNumber := val.(float64) return isNumber && (definition.Type == "keyword" || definition.Type == "constant_keyword") diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index 7368c54abf..e54b368a70 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -986,7 +986,6 @@ func filterAgents(allAgents []kibana.Agent, ctx servicedeployer.ServiceContext) } func writeSampleEvent(path string, doc common.MapStr) error { - body, err := json.MarshalIndent(doc, "", " ") if err != nil { return errors.Wrap(err, "marshalling sample event failed") diff --git a/test/packages/other/synthetic_mode/data_stream/synthetic/elasticsearch/ingest_pipeline/default.yml b/test/packages/other/synthetic_mode/data_stream/synthetic/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 0000000000..c1d95b20e3 --- /dev/null +++ b/test/packages/other/synthetic_mode/data_stream/synthetic/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,17 @@ +--- +processors: + - set: + field: ecs.version + value: 8.7.0 + - set: + field: event.kind + value: metric + - set: + field: event.module + value: synthetic_mode + - set: + field: event.type + value: [info] + - set: + field: event.category + value: [web] diff --git a/test/packages/other/synthetic_mode/data_stream/synthetic/sample_event.json b/test/packages/other/synthetic_mode/data_stream/synthetic/sample_event.json index b2482209bf..5d09ff5b44 100644 --- a/test/packages/other/synthetic_mode/data_stream/synthetic/sample_event.json +++ b/test/packages/other/synthetic_mode/data_stream/synthetic/sample_event.json @@ -1,75 +1,94 @@ { - "@timestamp": "2023-06-02T15:21:39.436Z", - "agent.ephemeral_id": "d5a061f5-0aac-4c11-8496-7f2c42f2393c", - "agent.id": "bc2c6d4c-79fd-4368-ae8b-540913be01fe", - "agent.name": "docker-fleet-agent", - "agent.type": "metricbeat", - "agent.version": "8.7.1", - "data_stream.dataset": "synthetic_mode.synthetic", - "data_stream.namespace": "ep", - "data_stream.type": "metrics", - "ecs.version": "8.7.0", - "elastic_agent.id": [ - "bc2c6d4c-79fd-4368-ae8b-540913be01fe" - ], - "elastic_agent.snapshot": [ - false - ], - "elastic_agent.version": [ - "8.7.1" - ], - "event.agent_id_status": "verified", - "event.category": [ - "web" - ], - "event.dataset": "synthetic_mode.synthetic", - "event.duration": 5392852, - "event.ingested": "2023-06-02T15:21:40Z", - "event.kind": "metric", - "event.module": "apache_tomcat", - "event.type": [ - "info" - ], - "host.architecture": "x86_64", - "host.containerized": [ - false - ], - "host.hostname": "docker-fleet-agent", - "host.id": "fd2c4b0943e444508c12855a04d117c7", - "host.ip": [ - "192.168.224.7" - ], - "host.mac": [ - "02-42-C0-A8-E0-07" - ], - "host.name": "docker-fleet-agent", - "host.os.codename": [ - "focal" - ], - "host.os.family": "debian", - "host.os.kernel": "5.19.0-42-generic", - "host.os.name": "Ubuntu", - "host.os.name.text": [ - "Ubuntu" - ], - "host.os.platform": "ubuntu", - "host.os.type": "linux", - "host.os.version": "20.04.6 LTS (Focal Fossa)", - "metricset.name": [ - "collector" - ], - "metricset.name.text": [ - "collector" - ], - "metricset.period": [ - 10000 - ], - "prometheus.labels.dialer_name": "prometheus", - "prometheus.labels.instance": "elastic-package-service-synthetic_mode-1:9090", - "prometheus.labels.job": "prometheus", - "prometheus.labels.reason": "resolution", - "prometheus.net_conntrack_dialer_conn_failed_total.counter": 0, - "prometheus.net_conntrack_dialer_conn_failed_total.rate": 0, - "service.address": "http://elastic-package-service-synthetic_mode-1:9090/metrics", - "service.type": "prometheus" + "@timestamp": "2023-06-07T08:30:02.333Z", + "agent": { + "ephemeral_id": "296ade19-2a9f-47f6-95f9-dcfa40559fd0", + "id": "86e09a11-1c59-4cb9-9eda-2b025a06143e", + "name": "docker-fleet-agent", + "type": "metricbeat", + "version": "8.7.1" + }, + "data_stream": { + "dataset": "synthetic_mode.synthetic", + "namespace": "ep", + "type": "metrics" + }, + "ecs": { + "version": "8.7.0" + }, + "elastic_agent": { + "id": [ + "86e09a11-1c59-4cb9-9eda-2b025a06143e" + ], + "snapshot": [ + false + ], + "version": [ + "8.7.1" + ] + }, + "event": { + "agent_id_status": "verified", + "category": [ + "web" + ], + "dataset": "synthetic_mode.synthetic", + "duration": 7261724, + "ingested": "2023-06-07T08:30:06Z", + "kind": "metric", + "module": "synthetic_mode", + "type": [ + "info" + ] + }, + "host": { + "architecture": "x86_64", + "containerized": [ + false + ], + "hostname": "docker-fleet-agent", + "id": "fd2c4b0943e444508c12855a04d117c7", + "ip": [ + "172.26.0.7" + ], + "mac": [ + "02-42-AC-1A-00-07" + ], + "name": "docker-fleet-agent", + "os": { + "codename": [ + "focal" + ], + "family": "debian", + "kernel": "5.19.0-42-generic", + "name": "Ubuntu", + "platform": "ubuntu", + "type": "linux", + "version": "20.04.6 LTS (Focal Fossa)" + } + }, + "metricset": { + "name": [ + "collector" + ], + "period": [ + 10000 + ] + }, + "prometheus": { + "labels": { + "branch": "HEAD", + "goversion": "go1.18.3", + "instance": "elastic-package-service-synthetic_mode-1:9090", + "job": "prometheus", + "revision": "d7e7b8e04b5ecdc1dd153534ba376a622b72741b", + "version": "2.36.2" + }, + "prometheus_build_info": { + "value": 1 + } + }, + "service": { + "address": "http://elastic-package-service-synthetic_mode-1:9090/metrics", + "type": "prometheus" + } } \ No newline at end of file From 5a1cc242e929f52701fa165b78301ceece38e23d Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 7 Jun 2023 11:02:43 +0200 Subject: [PATCH 33/34] Fix lint --- internal/fields/validate.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/fields/validate.go b/internal/fields/validate.go index bfdcd2e55c..8726c59260 100644 --- a/internal/fields/validate.go +++ b/internal/fields/validate.go @@ -445,7 +445,7 @@ func (v *Validator) SanitizeSyntheticSourceDocs(docs []common.MapStr) ([]common. func createDocExpandingObjects(doc common.MapStr) (common.MapStr, error) { keys := make([]string, 0) - for k, _ := range doc { + for k := range doc { keys = append(keys, k) } sort.Strings(keys) From c3f1a3054b8354b5b444147958732de4fc3bc8c5 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 7 Jun 2023 13:58:28 +0200 Subject: [PATCH 34/34] Remove array for those fields without definition --- internal/fields/validate.go | 49 ++++++++--------- .../data_stream/synthetic/sample_event.json | 53 +++++++------------ 2 files changed, 44 insertions(+), 58 deletions(-) diff --git a/internal/fields/validate.go b/internal/fields/validate.go index 8726c59260..9b541aa5af 100644 --- a/internal/fields/validate.go +++ b/internal/fields/validate.go @@ -388,33 +388,20 @@ func (v *Validator) SanitizeSyntheticSourceDocs(docs []common.MapStr) ([]common. var newDocs []common.MapStr for _, doc := range docs { for key, contents := range doc { - definition := FindElementDefinition(key, v.Schema) - if definition == nil { - continue - } - - if !v.disabledDependencyManagement && definition.External != "" { - def, err := v.FieldDependencyManager.ImportField(definition.External, key) - if err != nil { - return nil, fmt.Errorf("can't import field (field: %s): %w", key, err) - } - definition = &def - } - shouldBeArray := false - // normalization should just be checked if synthetic source is enabled and the - // spec version of this package is >= 2.0.0 - if v.disabledNormalization && !v.specVersion.LessThan(semver2_0_0) { - for _, normalize := range definition.Normalize { - switch normalize { - case "array": - shouldBeArray = true - } - if shouldBeArray { - break + definition := FindElementDefinition(key, v.Schema) + if definition != nil { + if !v.disabledDependencyManagement && definition.External != "" { + def, err := v.FieldDependencyManager.ImportField(definition.External, key) + if err != nil { + return nil, fmt.Errorf("can't import field (field: %s): %w", key, err) } + definition = &def } + + shouldBeArray = v.shouldValueBeArray(definition) } + // if it needs to be normalized, the field is kept as it is if shouldBeArray { continue @@ -443,6 +430,20 @@ func (v *Validator) SanitizeSyntheticSourceDocs(docs []common.MapStr) ([]common. return newDocs, nil } +func (v *Validator) shouldValueBeArray(definition *FieldDefinition) bool { + // normalization should just be checked if synthetic source is enabled and the + // spec version of this package is >= 2.0.0 + if v.disabledNormalization && !v.specVersion.LessThan(semver2_0_0) { + for _, normalize := range definition.Normalize { + switch normalize { + case "array": + return true + } + } + } + return false +} + func createDocExpandingObjects(doc common.MapStr) (common.MapStr, error) { keys := make([]string, 0) for k := range doc { @@ -466,7 +467,7 @@ func createDocExpandingObjects(doc common.MapStr) (common.MapStr, error) { // - expected map but type is string // - expected map but type is []interface{} if strings.HasPrefix(err.Error(), "expected map but type is") { - logger.Warnf("not able to add key %s, is this a multifield?: %s", k, err) + logger.Debugf("not able to add key %s, is this a multifield?: %s", k, err) continue } return nil, fmt.Errorf("not added key %s with value %s: %w", k, value, err) diff --git a/test/packages/other/synthetic_mode/data_stream/synthetic/sample_event.json b/test/packages/other/synthetic_mode/data_stream/synthetic/sample_event.json index 5d09ff5b44..7826070c8f 100644 --- a/test/packages/other/synthetic_mode/data_stream/synthetic/sample_event.json +++ b/test/packages/other/synthetic_mode/data_stream/synthetic/sample_event.json @@ -1,8 +1,8 @@ { - "@timestamp": "2023-06-07T08:30:02.333Z", + "@timestamp": "2023-06-07T11:18:14.475Z", "agent": { - "ephemeral_id": "296ade19-2a9f-47f6-95f9-dcfa40559fd0", - "id": "86e09a11-1c59-4cb9-9eda-2b025a06143e", + "ephemeral_id": "0265eeb1-26e2-452a-8ec0-e52e125ba9f6", + "id": "f1c9ef05-68a9-4eee-b254-89692ad3eacf", "name": "docker-fleet-agent", "type": "metricbeat", "version": "8.7.1" @@ -16,15 +16,9 @@ "version": "8.7.0" }, "elastic_agent": { - "id": [ - "86e09a11-1c59-4cb9-9eda-2b025a06143e" - ], - "snapshot": [ - false - ], - "version": [ - "8.7.1" - ] + "id": "f1c9ef05-68a9-4eee-b254-89692ad3eacf", + "snapshot": false, + "version": "8.7.1" }, "event": { "agent_id_status": "verified", @@ -32,8 +26,8 @@ "web" ], "dataset": "synthetic_mode.synthetic", - "duration": 7261724, - "ingested": "2023-06-07T08:30:06Z", + "duration": 4378129, + "ingested": "2023-06-07T11:18:18Z", "kind": "metric", "module": "synthetic_mode", "type": [ @@ -42,22 +36,18 @@ }, "host": { "architecture": "x86_64", - "containerized": [ - false - ], + "containerized": false, "hostname": "docker-fleet-agent", "id": "fd2c4b0943e444508c12855a04d117c7", "ip": [ - "172.26.0.7" + "192.168.80.7" ], "mac": [ - "02-42-AC-1A-00-07" + "02-42-C0-A8-50-07" ], "name": "docker-fleet-agent", "os": { - "codename": [ - "focal" - ], + "codename": "focal", "family": "debian", "kernel": "5.19.0-42-generic", "name": "Ubuntu", @@ -67,24 +57,19 @@ } }, "metricset": { - "name": [ - "collector" - ], - "period": [ - 10000 - ] + "name": "collector", + "period": 10000 }, "prometheus": { "labels": { - "branch": "HEAD", - "goversion": "go1.18.3", + "event": "delete", "instance": "elastic-package-service-synthetic_mode-1:9090", "job": "prometheus", - "revision": "d7e7b8e04b5ecdc1dd153534ba376a622b72741b", - "version": "2.36.2" + "role": "node" }, - "prometheus_build_info": { - "value": 1 + "prometheus_sd_kubernetes_events_total": { + "counter": 0, + "rate": 0 } }, "service": {