From beb09c66c70c079332d80f54453307414cc7a526 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Fri, 6 Oct 2023 16:10:37 +0200 Subject: [PATCH 1/3] Allow to override ECS objects to group or nested --- internal/fields/dependency_manager.go | 30 +++++++++++++++-- internal/fields/dependency_manager_test.go | 38 ++++++++++++++++++++++ 2 files changed, 65 insertions(+), 3 deletions(-) diff --git a/internal/fields/dependency_manager.go b/internal/fields/dependency_manager.go index 4221f064e5..3499f1f1b9 100644 --- a/internal/fields/dependency_manager.go +++ b/internal/fields/dependency_manager.go @@ -201,9 +201,8 @@ func (dm *DependencyManager) injectFieldsWithOptions(defs []common.MapStr, optio transformed.Delete("external") } - // Allow to override the type only from keyword to constant_keyword, - // to support the case of setting the value already in the mappings. - if ttype, _ := transformed["type"].(string); ttype != "constant_keyword" || imported.Type != "keyword" { + // Set the type back to the one imported, unless it it one of the allowed overrides. + if ttype, _ := transformed["type"].(string); !allowedTypeOverride(imported.Type, ttype) { transformed["type"] = imported.Type } @@ -264,6 +263,31 @@ func skipField(def common.MapStr) bool { return false } +func allowedTypeOverride(fromType, toType string) bool { + allowed := []struct { + from string + to string + }{ + // Support the case of setting the value already in the mappings. + {"keyword", "constant_keyword"}, + + // Not sure why, but was allowed in legacy implementations. + {"long", "keyword"}, + + // Support objects in ECS where the developer must decide if using + // a group or nested object. + {"object", "group"}, + {"object", "nested"}, + } + + for _, a := range allowed { + if a.from == fromType && a.to == toType { + return true + } + } + return false +} + // importField method resolves dependency on a single external field using available schemas. func (dm *DependencyManager) importField(schemaName, fieldPath string) (FieldDefinition, error) { if dm == nil { diff --git a/internal/fields/dependency_manager_test.go b/internal/fields/dependency_manager_test.go index 6ff488b2b4..15cc6d1ad2 100644 --- a/internal/fields/dependency_manager_test.go +++ b/internal/fields/dependency_manager_test.go @@ -725,6 +725,44 @@ func TestDependencyManagerWithECS(t *testing.T) { }, }, }, + { + title: "object to nested override", + defs: []common.MapStr{ + { + "name": "dns.answers", + "external": "ecs", + "type": "nested", + }, + }, + options: InjectFieldsOptions{}, + valid: true, + result: []common.MapStr{ + { + "name": "dns.answers", + "description": "An array containing an object for each answer section returned by the server.\nThe main keys that should be present in these objects are defined by ECS. Records that have more information may contain more keys than what ECS defines.\nNot all DNS data sources give all details about DNS answers. At minimum, answer objects must contain the `data` key. If more information is available, map as much of it to ECS as possible, and add any additional fields to the answer objects as custom fields.", + "type": "nested", + }, + }, + }, + { + title: "object to group override", + defs: []common.MapStr{ + { + "name": "dns.answers", + "external": "ecs", + "type": "group", + }, + }, + options: InjectFieldsOptions{}, + valid: true, + result: []common.MapStr{ + { + "name": "dns.answers", + "description": "An array containing an object for each answer section returned by the server.\nThe main keys that should be present in these objects are defined by ECS. Records that have more information may contain more keys than what ECS defines.\nNot all DNS data sources give all details about DNS answers. At minimum, answer objects must contain the `data` key. If more information is available, map as much of it to ECS as possible, and add any additional fields to the answer objects as custom fields.", + "type": "group", + }, + }, + }, } for _, c := range cases { From d24805f6a2af5b4add26288351ed96cb377ab5d5 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Fri, 6 Oct 2023 16:34:07 +0200 Subject: [PATCH 2/3] Remove condition actually not needed --- internal/fields/dependency_manager.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/internal/fields/dependency_manager.go b/internal/fields/dependency_manager.go index 3499f1f1b9..6b03cf8c16 100644 --- a/internal/fields/dependency_manager.go +++ b/internal/fields/dependency_manager.go @@ -271,9 +271,6 @@ func allowedTypeOverride(fromType, toType string) bool { // Support the case of setting the value already in the mappings. {"keyword", "constant_keyword"}, - // Not sure why, but was allowed in legacy implementations. - {"long", "keyword"}, - // Support objects in ECS where the developer must decide if using // a group or nested object. {"object", "group"}, From 29dfbd26dfaa6b099567c7c2cd63d16afa61f551 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Fri, 6 Oct 2023 16:35:34 +0200 Subject: [PATCH 3/3] Typo --- internal/fields/dependency_manager.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/fields/dependency_manager.go b/internal/fields/dependency_manager.go index 6b03cf8c16..d5a6e9d0e9 100644 --- a/internal/fields/dependency_manager.go +++ b/internal/fields/dependency_manager.go @@ -201,7 +201,7 @@ func (dm *DependencyManager) injectFieldsWithOptions(defs []common.MapStr, optio transformed.Delete("external") } - // Set the type back to the one imported, unless it it one of the allowed overrides. + // Set the type back to the one imported, unless it is one of the allowed overrides. if ttype, _ := transformed["type"].(string); !allowedTypeOverride(imported.Type, ttype) { transformed["type"] = imported.Type }