From f958322e10b0e6b4e6db3efa9b7aa934336862a5 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Wed, 7 Sep 2022 19:20:53 +0200 Subject: [PATCH 1/5] Add tests --- internal/fields/testdata/fields/fields.yml | 17 +++++++++ internal/fields/validate.go | 6 +-- internal/fields/validate_test.go | 43 ++++++++++++++++++++++ 3 files changed, 61 insertions(+), 5 deletions(-) diff --git a/internal/fields/testdata/fields/fields.yml b/internal/fields/testdata/fields/fields.yml index b78ddc6554..bed34ea60b 100644 --- a/internal/fields/testdata/fields/fields.yml +++ b/internal/fields/testdata/fields/fields.yml @@ -17,3 +17,20 @@ type: keyword normalize: - array +- name: event.category + type: keyword + allowed_values: + - name: authentication + expected_event_types: + - start + - end + - info + - name: configuration + expected_event_types: + - access + - change + - creation + - deletion + - info +- name: event.type + type: keyword diff --git a/internal/fields/validate.go b/internal/fields/validate.go index 97d7c769a7..26f5136053 100644 --- a/internal/fields/validate.go +++ b/internal/fields/validate.go @@ -195,11 +195,7 @@ func (v *Validator) ValidateDocumentBody(body json.RawMessage) multierror.Error return errs } - errs := v.validateMapElement("", c) - if len(errs) == 0 { - return nil - } - return errs + return v.ValidateDocumentMap(c) } // ValidateDocumentMap validates the provided document as common.MapStr. diff --git a/internal/fields/validate_test.go b/internal/fields/validate_test.go index b1daa2327d..3a0ec5d74d 100644 --- a/internal/fields/validate_test.go +++ b/internal/fields/validate_test.go @@ -9,6 +9,7 @@ import ( "os" "testing" + "github.com/elastic/elastic-package/internal/common" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -113,6 +114,48 @@ func TestValidate_WithSpecVersion(t *testing.T) { require.Empty(t, errs) } +func TestValidate_ExpectedEventType(t *testing.T) { + validator, err := CreateValidatorForDirectory("testdata") + require.NoError(t, err) + require.NotNil(t, validator) + + cases := []struct { + title string + doc common.MapStr + valid bool + }{ + { + title: "valid event type", + doc: common.MapStr{ + "event.category": "authentication", + "event.type": "info", + }, + valid: true, + }, + { + title: "unexpected event type", + doc: common.MapStr{ + "event.category": "authentication", + "event.type": "access", + }, + valid: false, + }, + } + + for _, c := range cases { + t.Run(c.title, func(t *testing.T) { + errs := validator.ValidateDocumentMap(c.doc) + if c.valid { + assert.Empty(t, errs) + } else { + if assert.Len(t, errs, 1) { + assert.Contains(t, errs[0].Error(), "unexpected value for event.type") + } + } + }) + } +} + func Test_parseElementValue(t *testing.T) { for _, test := range []struct { key string From 85a926b9139cfca658ed43d6fe3c11a604f8ef9f Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Wed, 7 Sep 2022 20:00:44 +0200 Subject: [PATCH 2/5] Add validation --- internal/fields/model.go | 29 +++++++++++++++++++- internal/fields/validate.go | 47 +++++++++++++++++++++++--------- internal/fields/validate_test.go | 6 ++-- 3 files changed, 65 insertions(+), 17 deletions(-) diff --git a/internal/fields/model.go b/internal/fields/model.go index 20823224a5..38b28a8805 100644 --- a/internal/fields/model.go +++ b/internal/fields/model.go @@ -205,7 +205,7 @@ func cleanNested(parent *FieldDefinition) (base []FieldDefinition) { // AllowedValues is the list of allowed values for a field. type AllowedValues []AllowedValue -// Allowed returns true if a given value is allowed. +// IsAllowed returns true if a given value is allowed. func (avs AllowedValues) IsAllowed(value string) bool { if len(avs) == 0 { // No configured allowed values, any value is allowed. @@ -214,6 +214,21 @@ func (avs AllowedValues) IsAllowed(value string) bool { return common.StringSliceContains(avs.Values(), value) } +// IsExpectedEventType returns true if the event type is allowed for the given value. +func (avs AllowedValues) IsExpectedEventType(value string, eventType interface{}) bool { + expected := avs.ExpectedEventTypes(value) + if len(expected) == 0 { + // No restrictions defined, all good to go. + return true + } + eventTypeStr, ok := eventType.(string) + if !ok { + // It must be a string. + return false + } + return common.StringSliceContains(expected, eventTypeStr) +} + // Values returns the list of allowed values. func (avs AllowedValues) Values() []string { var values []string @@ -223,6 +238,18 @@ func (avs AllowedValues) Values() []string { return values } +// ExpectedEventTypes returns the list of expected event types for a given value. +func (avs AllowedValues) ExpectedEventTypes(value string) []string { + for _, v := range avs { + if v.Name == value { + return v.ExpectedEventTypes + } + } + + // If we are here, IsAllowed(value) is also false. + return nil +} + // AllowedValue is one of the allowed values for a field. type AllowedValue struct { Name string `yaml:"name"` diff --git a/internal/fields/validate.go b/internal/fields/validate.go index 26f5136053..606f538081 100644 --- a/internal/fields/validate.go +++ b/internal/fields/validate.go @@ -200,14 +200,14 @@ func (v *Validator) ValidateDocumentBody(body json.RawMessage) multierror.Error // ValidateDocumentMap validates the provided document as common.MapStr. func (v *Validator) ValidateDocumentMap(body common.MapStr) multierror.Error { - errs := v.validateMapElement("", body) + errs := v.validateMapElement("", body, body) if len(errs) == 0 { return nil } return errs } -func (v *Validator) validateMapElement(root string, elem common.MapStr) multierror.Error { +func (v *Validator) validateMapElement(root string, elem common.MapStr, doc common.MapStr) multierror.Error { var errs multierror.Error for name, val := range elem { key := strings.TrimLeft(root+"."+name, ".") @@ -215,7 +215,7 @@ func (v *Validator) validateMapElement(root string, elem common.MapStr) multierr switch val := val.(type) { case []map[string]interface{}: for _, m := range val { - err := v.validateMapElement(key, m) + err := v.validateMapElement(key, m, doc) if err != nil { errs = append(errs, err...) } @@ -226,12 +226,12 @@ func (v *Validator) validateMapElement(root string, elem common.MapStr) multierr // because the entire object is mapped as a single field. continue } - err := v.validateMapElement(key, val) + err := v.validateMapElement(key, val, doc) if err != nil { errs = append(errs, err...) } default: - err := v.validateScalarElement(key, val) + err := v.validateScalarElement(key, val, doc) if err != nil { errs = append(errs, err) } @@ -240,7 +240,7 @@ func (v *Validator) validateMapElement(root string, elem common.MapStr) multierr return errs } -func (v *Validator) validateScalarElement(key string, val interface{}) error { +func (v *Validator) validateScalarElement(key string, val interface{}, doc common.MapStr) error { if key == "" { return nil // root key is always valid } @@ -272,7 +272,7 @@ func (v *Validator) validateScalarElement(key string, val interface{}) error { return errors.Wrapf(err, "field %q is not normalized as expected", key) } - err = v.parseElementValue(key, *definition, val) + err = v.parseElementValue(key, *definition, val, doc) if err != nil { return errors.Wrap(err, "parsing field value failed") } @@ -429,11 +429,11 @@ func validSubField(def FieldDefinition, extraPart string) bool { // parseElementValue checks that the value stored in a field matches the field definition. For // arrays it checks it for each Element. -func (v *Validator) parseElementValue(key string, definition FieldDefinition, val interface{}) error { - return forEachElementValue(key, definition, val, v.parseSingleElementValue) +func (v *Validator) parseElementValue(key string, definition FieldDefinition, val interface{}, doc common.MapStr) error { + return forEachElementValue(key, definition, val, doc, v.parseSingleElementValue) } -func (v *Validator) parseSingleElementValue(key string, definition FieldDefinition, val interface{}) error { +func (v *Validator) parseSingleElementValue(key string, definition FieldDefinition, val interface{}, doc common.MapStr) error { invalidTypeError := func() error { return fmt.Errorf("field %q's Go type, %T, does not match the expected field type: %s (field value: %v)", key, val, definition.Type, val) } @@ -457,6 +457,11 @@ func (v *Validator) parseSingleElementValue(key string, definition FieldDefiniti if err := ensureAllowedValues(key, valStr, definition); err != nil { return err } + if !v.specVersion.LessThan(semver.MustParse("2.0.0")) { + if err := ensureExpectedEventType(key, valStr, definition, doc); err != nil { + return err + } + } // Normal text fields should be of type string. // If a pattern is provided, it checks if the value matches. case "keyword", "text": @@ -471,6 +476,11 @@ func (v *Validator) parseSingleElementValue(key string, definition FieldDefiniti if err := ensureAllowedValues(key, valStr, definition); err != nil { return err } + if !v.specVersion.LessThan(semver.MustParse("2.0.0")) { + if err := ensureExpectedEventType(key, valStr, definition, doc); err != nil { + return err + } + } // Dates are expected to be formatted as strings or as seconds or milliseconds // since epoch. // If it is a string and a pattern is provided, it checks if the value matches. @@ -559,13 +569,13 @@ func (v *Validator) isAllowedIPValue(s string) bool { // forEachElementValue visits a function for each element in the given value if // it is an array. If it is not an array, it calls the function with it. -func forEachElementValue(key string, definition FieldDefinition, val interface{}, fn func(string, FieldDefinition, interface{}) error) error { +func forEachElementValue(key string, definition FieldDefinition, val interface{}, doc common.MapStr, fn func(string, FieldDefinition, interface{}, common.MapStr) error) error { arr, isArray := val.([]interface{}) if !isArray { - return fn(key, definition, val) + return fn(key, definition, val, doc) } for _, element := range arr { - err := fn(key, definition, element) + err := fn(key, definition, element, doc) if err != nil { return err } @@ -612,3 +622,14 @@ func ensureAllowedValues(key, value string, definition FieldDefinition) error { } return nil } + +// ensureExpectedEventType validates that the document's `event.type` field is one of the expected +// one for the given value. +func ensureExpectedEventType(key, value string, definition FieldDefinition, doc common.MapStr) error { + eventType, _ := doc.GetValue("event.type") + if !definition.AllowedValues.IsExpectedEventType(value, eventType) { + expected := definition.AllowedValues.ExpectedEventTypes(value) + return fmt.Errorf("field \"event.type\" value \"%v\" (%T) is not one of the expected values (%s) for %s=%q", eventType, eventType, strings.Join(expected, ", "), key, value) + } + return nil +} diff --git a/internal/fields/validate_test.go b/internal/fields/validate_test.go index 3a0ec5d74d..69fe24863f 100644 --- a/internal/fields/validate_test.go +++ b/internal/fields/validate_test.go @@ -115,7 +115,7 @@ func TestValidate_WithSpecVersion(t *testing.T) { } func TestValidate_ExpectedEventType(t *testing.T) { - validator, err := CreateValidatorForDirectory("testdata") + validator, err := CreateValidatorForDirectory("testdata", WithSpecVersion("2.0.0")) require.NoError(t, err) require.NotNil(t, validator) @@ -149,7 +149,7 @@ func TestValidate_ExpectedEventType(t *testing.T) { assert.Empty(t, errs) } else { if assert.Len(t, errs, 1) { - assert.Contains(t, errs[0].Error(), "unexpected value for event.type") + assert.Contains(t, errs[0].Error(), "is not one of the expected values") } } }) @@ -476,7 +476,7 @@ func Test_parseElementValue(t *testing.T) { } t.Run(test.key, func(t *testing.T) { - err := v.parseElementValue(test.key, test.definition, test.value) + err := v.parseElementValue(test.key, test.definition, test.value, common.MapStr{}) if test.fail { require.Error(t, err) } else { From 70c186fbb59b63da3a204237b6028654dc55d9cc Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Thu, 8 Sep 2022 10:42:24 +0200 Subject: [PATCH 3/5] Review feedback --- internal/fields/validate.go | 8 +++++--- internal/fields/validate_test.go | 3 ++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/internal/fields/validate.go b/internal/fields/validate.go index 606f538081..e5583c366e 100644 --- a/internal/fields/validate.go +++ b/internal/fields/validate.go @@ -26,6 +26,8 @@ import ( "github.com/elastic/elastic-package/internal/packages/buildmanifest" ) +var semver2_0_0 = semver.MustParse("2.0.0") + // Validator is responsible for fields validation. type Validator struct { // Schema contains definition records. @@ -380,7 +382,7 @@ func compareKeys(key string, def FieldDefinition, searchedKey string) bool { func (v *Validator) validateExpectedNormalization(definition FieldDefinition, val interface{}) error { // Validate expected normalization starting with packages following spec v2 format. - if v.specVersion.LessThan(semver.MustParse("2.0.0")) { + if v.specVersion.LessThan(semver2_0_0) { return nil } for _, normalize := range definition.Normalize { @@ -457,7 +459,7 @@ func (v *Validator) parseSingleElementValue(key string, definition FieldDefiniti if err := ensureAllowedValues(key, valStr, definition); err != nil { return err } - if !v.specVersion.LessThan(semver.MustParse("2.0.0")) { + if !v.specVersion.LessThan(semver2_0_0) { if err := ensureExpectedEventType(key, valStr, definition, doc); err != nil { return err } @@ -476,7 +478,7 @@ func (v *Validator) parseSingleElementValue(key string, definition FieldDefiniti if err := ensureAllowedValues(key, valStr, definition); err != nil { return err } - if !v.specVersion.LessThan(semver.MustParse("2.0.0")) { + if !v.specVersion.LessThan(semver2_0_0) { if err := ensureExpectedEventType(key, valStr, definition, doc); err != nil { return err } diff --git a/internal/fields/validate_test.go b/internal/fields/validate_test.go index 69fe24863f..4b820609cc 100644 --- a/internal/fields/validate_test.go +++ b/internal/fields/validate_test.go @@ -9,9 +9,10 @@ import ( "os" "testing" - "github.com/elastic/elastic-package/internal/common" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + "github.com/elastic/elastic-package/internal/common" ) type results struct { From aba1d143747c7d76d321b5b99f64fa7e7493a4c2 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Thu, 8 Sep 2022 11:45:33 +0200 Subject: [PATCH 4/5] event.type must be actually be an array --- internal/fields/model.go | 24 ++++++++++++++--- internal/fields/testdata/fields/fields.yml | 30 ++++++++++++++++++++++ internal/fields/validate_test.go | 12 +++++++-- 3 files changed, 60 insertions(+), 6 deletions(-) diff --git a/internal/fields/model.go b/internal/fields/model.go index 38b28a8805..359aff9ab8 100644 --- a/internal/fields/model.go +++ b/internal/fields/model.go @@ -215,18 +215,34 @@ func (avs AllowedValues) IsAllowed(value string) bool { } // IsExpectedEventType returns true if the event type is allowed for the given value. +// This method can be used to check single values of event type or arrays of them. func (avs AllowedValues) IsExpectedEventType(value string, eventType interface{}) bool { expected := avs.ExpectedEventTypes(value) if len(expected) == 0 { // No restrictions defined, all good to go. return true } - eventTypeStr, ok := eventType.(string) - if !ok { - // It must be a string. + switch eventType := eventType.(type) { + case string: + return common.StringSliceContains(expected, eventType) + case []interface{}: + if len(eventType) == 0 { + return false + } + for _, elem := range eventType { + elem, ok := elem.(string) + if !ok { + return false + } + if !common.StringSliceContains(expected, elem) { + return false + } + } + return true + default: + // It must be a string, or an array of strings. return false } - return common.StringSliceContains(expected, eventTypeStr) } // Values returns the list of allowed values. diff --git a/internal/fields/testdata/fields/fields.yml b/internal/fields/testdata/fields/fields.yml index bed34ea60b..d160c96fa4 100644 --- a/internal/fields/testdata/fields/fields.yml +++ b/internal/fields/testdata/fields/fields.yml @@ -32,5 +32,35 @@ - creation - deletion - info + - name: network + expected_event_types: + - access + - allowed + - connection + - denied + - end + - info + - protocol + - start - name: event.type type: keyword + normalize: + - array + allowed_values: + - name: access + - name: admin + - name: allowed + - name: change + - name: connection + - name: creation + - name: deletion + - name: denied + - name: end + - name: error + - name: group + - name: indicator + - name: info + - name: installation + - name: protocol + - name: start + - name: user diff --git a/internal/fields/validate_test.go b/internal/fields/validate_test.go index 4b820609cc..bb4c2ce360 100644 --- a/internal/fields/validate_test.go +++ b/internal/fields/validate_test.go @@ -129,7 +129,15 @@ func TestValidate_ExpectedEventType(t *testing.T) { title: "valid event type", doc: common.MapStr{ "event.category": "authentication", - "event.type": "info", + "event.type": []interface{}{"info"}, + }, + valid: true, + }, + { + title: "multiple valid event type", + doc: common.MapStr{ + "event.category": "network", + "event.type": []interface{}{"protocol", "connection", "end"}, }, valid: true, }, @@ -137,7 +145,7 @@ func TestValidate_ExpectedEventType(t *testing.T) { title: "unexpected event type", doc: common.MapStr{ "event.category": "authentication", - "event.type": "access", + "event.type": []interface{}{"access"}, }, valid: false, }, From df4521bc7c38aa204a0f89f056fe56d4c268ac1e Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Thu, 8 Sep 2022 12:07:00 +0200 Subject: [PATCH 5/5] Fix apache test package --- .../test-access-basic.log-expected.json | 49 ++++++++++----- .../test-access-darwin.log-expected.json | 42 +++++++++---- .../test-access-ssl-request.log-expected.json | 14 +++-- .../test-access-ubuntu.log-expected.json | 63 +++++++++++++------ .../test-access-vhost.log-expected.json | 7 ++- .../elasticsearch/ingest_pipeline/default.yml | 3 + .../data_stream/access/sample_event.json | 21 ++++--- 7 files changed, 140 insertions(+), 59 deletions(-) diff --git a/test/packages/parallel/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log-expected.json b/test/packages/parallel/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log-expected.json index 80604eeeb6..9040732867 100644 --- a/test/packages/parallel/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log-expected.json +++ b/test/packages/parallel/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log-expected.json @@ -13,10 +13,13 @@ "web" ], "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-09-06T07:11:46.614845014Z", + "ingested": "2022-09-08T10:05:05.863598691Z", "kind": "event", "original": "::1 - - [26/Dec/2016:16:16:29 +0200] \"GET /favicon.ico HTTP/1.1\" 404 209", - "outcome": "failure" + "outcome": "failure", + "type": [ + "access" + ] }, "http": { "request": { @@ -59,10 +62,13 @@ "web" ], "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-09-06T07:11:46.614847398Z", + "ingested": "2022-09-08T10:05:05.863610730Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:13 +0000] \"GET /hello HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", - "outcome": "failure" + "outcome": "failure", + "type": [ + "access" + ] }, "http": { "request": { @@ -118,10 +124,13 @@ "web" ], "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-09-06T07:11:46.614848507Z", + "ingested": "2022-09-08T10:05:05.863613299Z", "kind": "event", "original": "::1 - - [26/Dec/2016:16:16:48 +0200] \"-\" 408 -", - "outcome": "failure" + "outcome": "failure", + "type": [ + "access" + ] }, "http": { "response": { @@ -152,10 +161,13 @@ "web" ], "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-09-06T07:11:46.614849470Z", + "ingested": "2022-09-08T10:05:05.863614844Z", "kind": "event", "original": "172.17.0.1 - - [29/May/2017:19:02:48 +0000] \"GET /stringpatch HTTP/1.1\" 404 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", - "outcome": "failure" + "outcome": "failure", + "type": [ + "access" + ] }, "http": { "request": { @@ -211,10 +223,13 @@ "web" ], "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-09-06T07:11:46.614850424Z", + "ingested": "2022-09-08T10:05:05.863616333Z", "kind": "event", "original": "monitoring-server - - [29/May/2017:19:02:48 +0000] \"GET /status HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", - "outcome": "success" + "outcome": "success", + "type": [ + "access" + ] }, "http": { "request": { @@ -270,10 +285,13 @@ "web" ], "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-09-06T07:11:46.614854678Z", + "ingested": "2022-09-08T10:05:05.863617808Z", "kind": "event", "original": "127.0.0.1 - - [02/Feb/2019:05:38:45 +0100] \"-\" 408 152 \"-\" \"-\"", - "outcome": "failure" + "outcome": "failure", + "type": [ + "access" + ] }, "http": { "request": { @@ -317,10 +335,13 @@ "web" ], "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-09-06T07:11:46.614855729Z", + "ingested": "2022-09-08T10:05:05.863619268Z", "kind": "event", "original": "monitoring-server - - [29/May/2017:19:02:48 +0000] \"GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", - "outcome": "success" + "outcome": "success", + "type": [ + "access" + ] }, "http": { "request": { diff --git a/test/packages/parallel/apache/data_stream/access/_dev/test/pipeline/test-access-darwin.log-expected.json b/test/packages/parallel/apache/data_stream/access/_dev/test/pipeline/test-access-darwin.log-expected.json index 173c06e28e..a88a6b3df6 100644 --- a/test/packages/parallel/apache/data_stream/access/_dev/test/pipeline/test-access-darwin.log-expected.json +++ b/test/packages/parallel/apache/data_stream/access/_dev/test/pipeline/test-access-darwin.log-expected.json @@ -13,10 +13,13 @@ "web" ], "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-09-06T07:11:46.659642334Z", + "ingested": "2022-09-08T10:05:05.950483828Z", "kind": "event", "original": "::1 - - [26/Dec/2016:16:16:28 +0200] \"GET / HTTP/1.1\" 200 45", - "outcome": "success" + "outcome": "success", + "type": [ + "access" + ] }, "http": { "request": { @@ -58,10 +61,13 @@ "web" ], "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-09-06T07:11:46.659644823Z", + "ingested": "2022-09-08T10:05:05.950496473Z", "kind": "event", "original": "::1 - - [26/Dec/2016:16:16:29 +0200] \"GET /favicon.ico HTTP/1.1\" 404 209", - "outcome": "failure" + "outcome": "failure", + "type": [ + "access" + ] }, "http": { "request": { @@ -104,10 +110,13 @@ "web" ], "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-09-06T07:11:46.659645835Z", + "ingested": "2022-09-08T10:05:05.950497950Z", "kind": "event", "original": "::1 - - [26/Dec/2016:16:16:48 +0200] \"-\" 408 -", - "outcome": "failure" + "outcome": "failure", + "type": [ + "access" + ] }, "http": { "response": { @@ -138,10 +147,13 @@ "web" ], "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-09-06T07:11:46.659646801Z", + "ingested": "2022-09-08T10:05:05.950499298Z", "kind": "event", "original": "89.160.20.156 - - [26/Dec/2016:18:23:35 +0200] \"GET / HTTP/1.1\" 200 45", - "outcome": "success" + "outcome": "success", + "type": [ + "access" + ] }, "http": { "request": { @@ -201,10 +213,13 @@ "web" ], "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-09-06T07:11:46.659647759Z", + "ingested": "2022-09-08T10:05:05.950500622Z", "kind": "event", "original": "89.160.20.156 - - [26/Dec/2016:18:23:41 +0200] \"GET /notfound HTTP/1.1\" 404 206", - "outcome": "failure" + "outcome": "failure", + "type": [ + "access" + ] }, "http": { "request": { @@ -264,10 +279,13 @@ "web" ], "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-09-06T07:11:46.659648717Z", + "ingested": "2022-09-08T10:05:05.950501935Z", "kind": "event", "original": "89.160.20.156 - - [26/Dec/2016:18:23:45 +0200] \"GET /hmm HTTP/1.1\" 404 201", - "outcome": "failure" + "outcome": "failure", + "type": [ + "access" + ] }, "http": { "request": { diff --git a/test/packages/parallel/apache/data_stream/access/_dev/test/pipeline/test-access-ssl-request.log-expected.json b/test/packages/parallel/apache/data_stream/access/_dev/test/pipeline/test-access-ssl-request.log-expected.json index 4b56cde975..d2a6f20aff 100644 --- a/test/packages/parallel/apache/data_stream/access/_dev/test/pipeline/test-access-ssl-request.log-expected.json +++ b/test/packages/parallel/apache/data_stream/access/_dev/test/pipeline/test-access-ssl-request.log-expected.json @@ -18,9 +18,12 @@ "web" ], "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-09-06T07:11:46.704516601Z", + "ingested": "2022-09-08T10:05:06.029928962Z", "kind": "event", - "original": "[10/Aug/2018:09:45:56 +0200] 172.30.0.119 TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 \"GET /nagiosxi/ajaxhelper.php?cmd=getxicoreajax\u0026amp;opts=%7B%22func%22%3A%22get_admin_tasks_html%22%2C%22args%22%3A%22%22%7D\u0026amp;nsp=b5c7d5d4b6f7d0cf0c92f9cbdf737f6a5c838218425e6ae21 HTTP/1.1\" 1375" + "original": "[10/Aug/2018:09:45:56 +0200] 172.30.0.119 TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 \"GET /nagiosxi/ajaxhelper.php?cmd=getxicoreajax\u0026amp;opts=%7B%22func%22%3A%22get_admin_tasks_html%22%2C%22args%22%3A%22%22%7D\u0026amp;nsp=b5c7d5d4b6f7d0cf0c92f9cbdf737f6a5c838218425e6ae21 HTTP/1.1\" 1375", + "type": [ + "access" + ] }, "http": { "request": { @@ -70,9 +73,12 @@ "web" ], "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-09-06T07:11:46.704520327Z", + "ingested": "2022-09-08T10:05:06.029934826Z", "kind": "event", - "original": "[16/Oct/2019:11:53:47 +0200] 89.160.20.156 TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 \"GET /appl/ajaxhelper.php?cmd=getxicoreajax\u0026opts=%7B%22func%22%3A%22get_pagetop_alert_content_html%22%2C%22args%22%3A%22%22%7D\u0026nsp=c2700eab9797eda8a9f65a3ab17a6adbceccd60a6cca7708650a5923950d HTTP/1.1\" -" + "original": "[16/Oct/2019:11:53:47 +0200] 89.160.20.156 TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 \"GET /appl/ajaxhelper.php?cmd=getxicoreajax\u0026opts=%7B%22func%22%3A%22get_pagetop_alert_content_html%22%2C%22args%22%3A%22%22%7D\u0026nsp=c2700eab9797eda8a9f65a3ab17a6adbceccd60a6cca7708650a5923950d HTTP/1.1\" -", + "type": [ + "access" + ] }, "http": { "request": { diff --git a/test/packages/parallel/apache/data_stream/access/_dev/test/pipeline/test-access-ubuntu.log-expected.json b/test/packages/parallel/apache/data_stream/access/_dev/test/pipeline/test-access-ubuntu.log-expected.json index d1f392cd6c..0aab255bcd 100644 --- a/test/packages/parallel/apache/data_stream/access/_dev/test/pipeline/test-access-ubuntu.log-expected.json +++ b/test/packages/parallel/apache/data_stream/access/_dev/test/pipeline/test-access-ubuntu.log-expected.json @@ -13,10 +13,13 @@ "web" ], "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-09-06T07:11:46.762449415Z", + "ingested": "2022-09-08T10:05:06.078187403Z", "kind": "event", "original": "127.0.0.1 - - [26/Dec/2016:16:18:09 +0000] \"GET / HTTP/1.1\" 200 491 \"-\" \"Wget/1.13.4 (linux-gnu)\"", - "outcome": "success" + "outcome": "success", + "type": [ + "access" + ] }, "http": { "request": { @@ -70,10 +73,13 @@ "web" ], "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-09-06T07:11:46.762453903Z", + "ingested": "2022-09-08T10:05:06.078193659Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:00 +0000] \"GET / HTTP/1.1\" 200 484 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"", - "outcome": "success" + "outcome": "success", + "type": [ + "access" + ] }, "http": { "request": { @@ -129,10 +135,13 @@ "web" ], "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-09-06T07:11:46.762455848Z", + "ingested": "2022-09-08T10:05:06.078195340Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:00 +0000] \"GET /favicon.ico HTTP/1.1\" 404 504 \"http://192.168.33.72/\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"", - "outcome": "failure" + "outcome": "failure", + "type": [ + "access" + ] }, "http": { "request": { @@ -189,10 +198,13 @@ "web" ], "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-09-06T07:11:46.762457571Z", + "ingested": "2022-09-08T10:05:06.078196730Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:08 +0000] \"GET / HTTP/1.1\" 200 484 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", - "outcome": "success" + "outcome": "success", + "type": [ + "access" + ] }, "http": { "request": { @@ -248,10 +260,13 @@ "web" ], "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-09-06T07:11:46.762459219Z", + "ingested": "2022-09-08T10:05:06.078198115Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:08 +0000] \"GET /favicon.ico HTTP/1.1\" 404 504 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", - "outcome": "failure" + "outcome": "failure", + "type": [ + "access" + ] }, "http": { "request": { @@ -308,10 +323,13 @@ "web" ], "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-09-06T07:11:46.762460903Z", + "ingested": "2022-09-08T10:05:06.078199480Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:08 +0000] \"GET /favicon.ico HTTP/1.1\" 404 504 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", - "outcome": "failure" + "outcome": "failure", + "type": [ + "access" + ] }, "http": { "request": { @@ -368,10 +386,13 @@ "web" ], "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-09-06T07:11:46.762462605Z", + "ingested": "2022-09-08T10:05:06.078200843Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:10 +0000] \"GET /test HTTP/1.1\" 404 498 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", - "outcome": "failure" + "outcome": "failure", + "type": [ + "access" + ] }, "http": { "request": { @@ -427,10 +448,13 @@ "web" ], "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-09-06T07:11:46.762464295Z", + "ingested": "2022-09-08T10:05:06.078202210Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:13 +0000] \"GET /hello HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", - "outcome": "failure" + "outcome": "failure", + "type": [ + "access" + ] }, "http": { "request": { @@ -486,10 +510,13 @@ "web" ], "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-09-06T07:11:46.762465914Z", + "ingested": "2022-09-08T10:05:06.078203552Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:17 +0000] \"GET /crap HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", - "outcome": "failure" + "outcome": "failure", + "type": [ + "access" + ] }, "http": { "request": { diff --git a/test/packages/parallel/apache/data_stream/access/_dev/test/pipeline/test-access-vhost.log-expected.json b/test/packages/parallel/apache/data_stream/access/_dev/test/pipeline/test-access-vhost.log-expected.json index 120247e442..0e6ac6978e 100644 --- a/test/packages/parallel/apache/data_stream/access/_dev/test/pipeline/test-access-vhost.log-expected.json +++ b/test/packages/parallel/apache/data_stream/access/_dev/test/pipeline/test-access-vhost.log-expected.json @@ -16,10 +16,13 @@ "web" ], "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-09-06T07:11:46.823078522Z", + "ingested": "2022-09-08T10:05:06.135584519Z", "kind": "event", "original": "vhost1.domaine.fr 192.168.33.2 - - [26/Dec/2016:16:22:14 +0000] \"GET /hello HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", - "outcome": "failure" + "outcome": "failure", + "type": [ + "access" + ] }, "http": { "request": { diff --git a/test/packages/parallel/apache/data_stream/access/elasticsearch/ingest_pipeline/default.yml b/test/packages/parallel/apache/data_stream/access/elasticsearch/ingest_pipeline/default.yml index 38d5c041d1..48763c70d8 100644 --- a/test/packages/parallel/apache/data_stream/access/elasticsearch/ingest_pipeline/default.yml +++ b/test/packages/parallel/apache/data_stream/access/elasticsearch/ingest_pipeline/default.yml @@ -47,6 +47,9 @@ processors: - set: field: event.category value: ["web"] + - set: + field: event.type + value: ["access"] - set: field: event.outcome value: success diff --git a/test/packages/parallel/apache/data_stream/access/sample_event.json b/test/packages/parallel/apache/data_stream/access/sample_event.json index 7885d8d9dc..167963b4de 100644 --- a/test/packages/parallel/apache/data_stream/access/sample_event.json +++ b/test/packages/parallel/apache/data_stream/access/sample_event.json @@ -1,8 +1,8 @@ { - "@timestamp": "2022-09-06T07:48:09.000Z", + "@timestamp": "2022-09-08T10:05:46.000Z", "agent": { - "ephemeral_id": "357ad51a-c816-46b4-a094-8da1fe2b47af", - "id": "9a83e2ce-8ade-4cc3-ba6a-6305c90b3022", + "ephemeral_id": "62fff331-1942-4ca6-b1f4-7b09d72e4e6b", + "id": "488aa68b-25f3-4a6c-aeaf-bbdacf341968", "name": "docker-fleet-agent", "type": "filebeat", "version": "8.3.3" @@ -19,7 +19,7 @@ "version": "1.12.0" }, "elastic_agent": { - "id": "9a83e2ce-8ade-4cc3-ba6a-6305c90b3022", + "id": "488aa68b-25f3-4a6c-aeaf-bbdacf341968", "snapshot": false, "version": "8.3.3" }, @@ -28,21 +28,24 @@ "category": [ "web" ], - "created": "2022-09-06T07:48:30.073Z", + "created": "2022-09-08T10:06:04.881Z", "dataset": "apache.access", - "ingested": "2022-09-06T07:48:31Z", + "ingested": "2022-09-08T10:06:05Z", "kind": "event", - "outcome": "success" + "outcome": "success", + "type": [ + "access" + ] }, "host": { "architecture": "x86_64", "containerized": false, "hostname": "docker-fleet-agent", "ip": [ - "172.21.0.7" + "192.168.128.4" ], "mac": [ - "02:42:ac:15:00:07" + "02:42:c0:a8:80:04" ], "name": "docker-fleet-agent", "os": {