Merge pull request #10600 from Security-Onion-Solutions/fix/dataset_tags

Change format of event dataset and assign dataset to tags
Security-Onion-Solutions · Jun 21, 2023 · 6ba9e05 · 6ba9e05
2 parents b02c381 + 3a34da3
commit 6ba9e05
Show file tree

Hide file tree

Showing 4 changed files with 112 additions and 107 deletions.
diff --git a/salt/elasticsearch/files/ingest-dynamic/common b/salt/elasticsearch/files/ingest-dynamic/common
@@ -57,8 +57,11 @@
     { "convert":         { "field": "log.id.uid", "type": "string",  "ignore_failure": true,  "ignore_missing": true  } },
     { "convert":         { "field": "agent.id", "type": "string",  "ignore_failure": true,  "ignore_missing": true  } },
     { "convert":         { "field": "event.severity", "type": "integer",  "ignore_failure": true,  "ignore_missing": true  } },
-    { "set":             { "field": "event.dataset", "ignore_empty_value":true, "copy_from":"event.dataset_temp"   }},
-    { "remove":          { "field": [ "message2", "type", "fields", "category", "module", "dataset", "event.dataset_temp" ], "ignore_missing": true, "ignore_failure": true } }
+    { "set":             { "field": "event.dataset", "ignore_empty_value":true, "copy_from": "event.dataset_temp"   }},
+    { "set":             { "if": "ctx.event?.dataset != null && !ctx.event.dataset.contains('.')", "field": "event.dataset", "value": "{{event.module}}.{{event.dataset}}" } },
+    { "split":      { "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')", "field": "event.dataset", "separator": "\\.", "target_field": "dataset_tag_temp" } },
+    { "append":        { "if": "ctx.dataset_tag_temp != null", "field": "tags", "value": "{{dataset_tag_temp.1}}"  }},
+    { "remove":          { "field": [ "message2", "type", "fields", "category", "module", "dataset", "dataset_tag_temp", "event.dataset_temp" ], "ignore_missing": true, "ignore_failure": true } }
 {%- endraw %}
 {%- if HIGHLANDER %}
     ,

diff --git a/salt/elasticsearch/files/ingest/.fleet_final_pipeline-1 b/salt/elasticsearch/files/ingest/.fleet_final_pipeline-1
@@ -71,8 +71,10 @@
     },
     { "set":        { "ignore_failure": true,  "field": "event.module", "value": "elastic_agent" } },
     { "split":      { "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')", "field": "event.dataset", "separator": "\\.", "target_field": "module_temp"  } },
-    { "set":        { "if": "ctx.module_temp != null", "override": true, "field": "event.module", "value": "{{module_temp.0}}"  }},
-    { "remove":     { "field": [ "module_temp" ], "ignore_missing": true, "ignore_failure": true } }
+    { "set":        { "if": "ctx.module_temp != null", "override": true, "field": "event.module", "value": "{{module_temp.0}}" } },
+    { "split":      { "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')", "field": "event.dataset", "separator": "\\.", "target_field": "dataset_tag_temp" } },
+    { "append":        { "if": "ctx.dataset_tag_temp != null", "field": "tags", "value": "{{dataset_tag_temp.1}}"  } },
+    { "remove":     { "field": [ "message2", "type", "fields", "category", "module", "dataset", "event.dataset_temp", "dataset_tag_temp", "module_temp" ], "ignore_missing": true, "ignore_failure": true } } 
   ],
   "on_failure": [
     {