From 92bc19cf494ae323c19cbe58cafcff66bae3fb37 Mon Sep 17 00:00:00 2001 From: "Freddy R. Laffita Almaguer" Date: Fri, 29 Dec 2023 05:21:38 -0500 Subject: [PATCH 1/4] Adding support for filter ordering issues (mutate) --- filters/privafy/privafy.conf | 37 +++++++++++++++++++++++++----------- 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/filters/privafy/privafy.conf b/filters/privafy/privafy.conf index 9ab93255e..a00e70568 100644 --- a/filters/privafy/privafy.conf +++ b/filters/privafy/privafy.conf @@ -1,6 +1,6 @@ filter { -# Privafy filter version 1.1.0 +# Privafy filter version 1.1.1 # Based on (User Doc) https://docs.progress.com/es-ES/bundle/loadmaster-technical-note-common-event-format-cef-logs-ga/page/Common-Event-Format-CEF-Logs.html (December, 2023) # and (User Doc) https://help.deepsecurity.trendmicro.com/20_0/on-premise/event-syslog-message-formats.html (December, 2023) # and example logs provided by user during POC @@ -28,10 +28,20 @@ filter { } } - if ![dataType] { +#......................................................................# +# Creating privafy message field from syslog message + if [logx][syslog][message] { + mutate { + add_field => { "prvf_message" => "%{[logx][syslog][message]}" } + } + } else { + mutate { + add_field => { "prvf_message" => "%{message}" } + } + } #......................................................................# # Privafy Entry point - if [message] and (("CEF:" in [message] or "LEEF:" in [message]) and [message] =~/\|(\w+)?(\s)?Privafy(\s)?(\w+)?\|/ ) { + if [prvf_message] and (("CEF:" in [prvf_message] or "LEEF:" in [prvf_message]) and [prvf_message] =~/\|(\w+)?(\s)?Privafy(\s)?(\w+)?\|/ ) { #......................................................................# #Generating dataSource field required by CurrelationRulesEngine #Checks if exists, if not evaluate to the host variable @@ -42,23 +52,29 @@ filter { } #......................................................................# #Generating dataType field required by CurrelationRulesEngine + if (![dataType]){ mutate { add_field => { "dataType" => "privafy" } } + } else { + mutate { + update => { "dataType" => "privafy" } + } + } #......................................................................# -#If CEF or LEEF formatted log do the parsing of the message mark as undefined syslog format - if ("CEF:" in [message] or "LEEF:" in [message] ) { +#If CEF or LEEF formatted log do the parsing of the prvf_message mark as undefined syslog format + if ("CEF:" in [prvf_message] or "LEEF:" in [prvf_message] ) { #......................................................................# -#Using grok to parse header of the message +#Using grok to parse header of the prvf_message grok { match => { - "message" => [ + "prvf_message" => [ "(%{INT:not_defined})?(\s)?(<%{NUMBER:priority}>)?(%{INT:syslog_version})?((\s)%{GREEDYDATA:syslog_date_host}(\s))?(?(CEF|LEEF)):(\s)?(?(%{INT}\.%{INT}|%{INT}))%{GREEDYDATA:cef_or_leef_msg_all}" ] } } } - if ("CEF:" in [message] ) { + if ("CEF:" in [prvf_message] ) { #......................................................................# #Using grok to parse components of the cef_or_leef_msg_all if [cef_or_leef_msg_all] { @@ -72,7 +88,7 @@ filter { } } } - } else if ("LEEF:" in [message] ) { + } else if ("LEEF:" in [prvf_message] ) { #......................................................................# #Using grok to parse components of the leef_message if [cef_or_leef_msg_all] { @@ -277,12 +293,11 @@ if [kv_field][severity]{ #......................................................................# #Finally, remove unnecessary fields mutate { - remove_field => ["@version","path","tags","type","syslog_version","kv_field", + remove_field => ["@version","path","tags","type","syslog_version","kv_field","prvf_message","[logx][syslog][message]", "not_defined","cef_or_leef_msg_all","cef_or_leef_msg","syslog_date_host","irrelevant","init_msg"] } } # End CEF entrypoint - } #Also, remove unwanted fields if the message not match with conditions mutate { From 9357e05d67225fe8167535b5f558b3173e20a528 Mon Sep 17 00:00:00 2001 From: "Freddy R. Laffita Almaguer" Date: Fri, 29 Dec 2023 06:54:35 -0500 Subject: [PATCH 2/4] Adding support multiple filters in the same pipeline --- ...1_updating_syslog_json_generic_filters.xml | 161 ++++++++++++++++++ .../resources/config/liquibase/master.xml | 2 + filters/generic/generic.conf | 5 +- filters/json/json-input.conf | 8 +- filters/syslog/syslog-generic.conf | 5 +- 5 files changed, 174 insertions(+), 7 deletions(-) create mode 100644 backend/src/main/resources/config/liquibase/changelog/20231229001_updating_syslog_json_generic_filters.xml diff --git a/backend/src/main/resources/config/liquibase/changelog/20231229001_updating_syslog_json_generic_filters.xml b/backend/src/main/resources/config/liquibase/changelog/20231229001_updating_syslog_json_generic_filters.xml new file mode 100644 index 000000000..bb4e4b61d --- /dev/null +++ b/backend/src/main/resources/config/liquibase/changelog/20231229001_updating_syslog_json_generic_filters.xml @@ -0,0 +1,161 @@ + + + + + + + "message" + terminator => "" +} + +#Looking for datasource generated by an agent and parse original message +if [message]=~/\[utm_stack_agent_ds=(.+)\]-(.+)/ { + grok { + match => { + "message" => [ "\[utm_stack_agent_ds=%{DATA:dataSource}\]-%{GREEDYDATA:original_log_message}" ] + } + } +} +if [original_log_message] { + mutate { + update => { "message" => "%{[original_log_message]}" } + } +} +#......................................................................# +#Generating dataSource and dataType fields required by CurrelationRulesEngine +if ![dataSource] { + mutate { + add_field => { "dataSource" => "%{host}" } + } +} +if ![dataType] { + mutate { + add_field => {"dataType" => "generic"} + } +} +#......................................................................# +#Adding json support +if [message] =~/^\{/ { + json { + source => "message" + } +} + +#Remove unwanted fields if the message not match with conditions + mutate { + remove_field => ["@version","original_log_message","headers"] + } +} +',filter_version='1.0.1' + WHERE id=1521; + +-- Updating Syslog filter + UPDATE utm_logstash_filter + SET logstash_filter='filter { +# Syslog filter, version 1.0.1 + +split { + field => "message" + terminator => "" +} + +#Looking for datasource generated by an agent and parse original message +if [message]=~/\[utm_stack_agent_ds=(.+)\]-(.+)/ { + grok { + match => { + "message" => [ "\[utm_stack_agent_ds=%{DATA:dataSource}\]-%{GREEDYDATA:original_log_message}" ] + } + } +} +if [original_log_message] { + mutate { + update => { "message" => "%{[original_log_message]}" } + } +} +#......................................................................# +#Generating dataSource field required by CurrelationRulesEngine +if ![dataSource] { + mutate { + add_field => { "dataSource" => "%{host}" } + } +} +#......................................................................# +#Generating logx structure +if ![dataType] { + mutate { + add_field => {"dataType" => "syslog"} + rename => ["message", "[logx][syslog][message]"] + } +} +#Remove unwanted fields if the message not match with conditions + mutate { + remove_field => ["@version","original_log_message","headers"] + } +} +',filter_version='1.0.1' + WHERE id=1520; + +-- Updating json input filter + UPDATE utm_logstash_filter + SET logstash_filter='filter { +#Filter version 1.0.2 +#Used to format generic json files + + if [message] { + split { + field => "message" + terminator => "" + } + json { + source => "message" + target => "json_input" + } + + #Create logx structure + mutate { + rename => { "[message]" => "[logx][json_input][message]" } + rename => { "[json_input]" => "[logx][json_input]" } + } + + #Generating dataType and dataSource fields + if ![dataType] { + mutate { + add_field => { "dataType" => "json-input" } + } + } + if ![dataSource] { + if [logx][json_input][dataSource]{ + mutate { + rename => { "[logx][json_input][dataSource]" => "[dataSource]" } + } + } else { + mutate { + add_field => { "dataSource" => "%{host}" } + } + } + } + } + + #Finally remove unused fields + mutate { + remove_field => ["path","@version","host","headers"] + } +} +',filter_version='1.0.2' + WHERE id=1515; + + ]]> + + + diff --git a/backend/src/main/resources/config/liquibase/master.xml b/backend/src/main/resources/config/liquibase/master.xml index 94f4689fe..c22d2aebb 100644 --- a/backend/src/main/resources/config/liquibase/master.xml +++ b/backend/src/main/resources/config/liquibase/master.xml @@ -26,4 +26,6 @@ + + diff --git a/filters/generic/generic.conf b/filters/generic/generic.conf index 3a6d89aa1..6b563e024 100644 --- a/filters/generic/generic.conf +++ b/filters/generic/generic.conf @@ -1,5 +1,5 @@ filter { -# Generic pipeline filter, version 1.0.0 +# Generic pipeline filter, version 1.0.1 # Supports plain logs and simple json logs split { @@ -27,10 +27,11 @@ if ![dataSource] { add_field => { "dataSource" => "%{host}" } } } +if ![dataType] { mutate { add_field => {"dataType" => "generic"} } - +} #......................................................................# #Adding json support if [message] =~/^\{/ { diff --git a/filters/json/json-input.conf b/filters/json/json-input.conf index 7dd4f83b5..a765843dd 100644 --- a/filters/json/json-input.conf +++ b/filters/json/json-input.conf @@ -1,5 +1,5 @@ filter { -#Filter version 1.0.1 +#Filter version 1.0.2 #Used to format generic json files if [message] { @@ -24,14 +24,16 @@ filter { add_field => { "dataType" => "json-input" } } } - if [logx][json_input][dataSource]{ + if ![dataSource] { + if [logx][json_input][dataSource]{ mutate { rename => { "[logx][json_input][dataSource]" => "[dataSource]" } } - } else { + } else { mutate { add_field => { "dataSource" => "%{host}" } } + } } } diff --git a/filters/syslog/syslog-generic.conf b/filters/syslog/syslog-generic.conf index b33074ce1..6b25c72ec 100644 --- a/filters/syslog/syslog-generic.conf +++ b/filters/syslog/syslog-generic.conf @@ -1,5 +1,5 @@ filter { -# Syslog filter, version 1.0.0 +# Syslog filter, version 1.0.1 split { field => "message" @@ -28,11 +28,12 @@ if ![dataSource] { } #......................................................................# #Generating logx structure +if ![dataType] { mutate { add_field => {"dataType" => "syslog"} rename => ["message", "[logx][syslog][message]"] } - +} #Remove unwanted fields if the message not match with conditions mutate { remove_field => ["@version","original_log_message","headers"] From 208f046a682ff466c18a94c638727305fdb7b957 Mon Sep 17 00:00:00 2001 From: "Freddy R. Laffita Almaguer" Date: Tue, 2 Jan 2024 07:57:05 -0500 Subject: [PATCH 3/4] Adding .gitignore to web-pdf --- web-pdf/.gitignore | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 web-pdf/.gitignore diff --git a/web-pdf/.gitignore b/web-pdf/.gitignore new file mode 100644 index 000000000..549e00a2a --- /dev/null +++ b/web-pdf/.gitignore @@ -0,0 +1,33 @@ +HELP.md +target/ +!.mvn/wrapper/maven-wrapper.jar +!**/src/main/**/target/ +!**/src/test/**/target/ + +### STS ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans +.sts4-cache + +### IntelliJ IDEA ### +.idea +*.iws +*.iml +*.ipr + +### NetBeans ### +/nbproject/private/ +/nbbuild/ +/dist/ +/nbdist/ +/.nb-gradle/ +build/ +!**/src/main/**/build/ +!**/src/test/**/build/ + +### VS Code ### +.vscode/ From 3bbcb9506455bf6f2ab49766eb3a055948f8f47b Mon Sep 17 00:00:00 2001 From: "Freddy R. Laffita Almaguer" Date: Tue, 2 Jan 2024 08:56:31 -0500 Subject: [PATCH 4/4] Fixing command_line field issues with kv transformation --- filters/privafy/privafy.conf | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/filters/privafy/privafy.conf b/filters/privafy/privafy.conf index a00e70568..1c5337923 100644 --- a/filters/privafy/privafy.conf +++ b/filters/privafy/privafy.conf @@ -1,6 +1,6 @@ filter { -# Privafy filter version 1.1.1 +# Privafy filter version 1.1.2 # Based on (User Doc) https://docs.progress.com/es-ES/bundle/loadmaster-technical-note-common-event-format-cef-logs-ga/page/Common-Event-Format-CEF-Logs.html (December, 2023) # and (User Doc) https://help.deepsecurity.trendmicro.com/20_0/on-premise/event-syslog-message-formats.html (December, 2023) # and example logs provided by user during POC @@ -132,15 +132,15 @@ filter { #Using grok to parse kv issued fields grok { match => { - "cef_msg" => [ - "command_line=%{DATA:command_line} %{WORD}=(%{GREEDYDATA:irrelevant})?" + "cef_or_leef_msg" => [ + "command_line=%{DATA:command_line}\s(\b([a-zA-Z0-9_]+)\b)=(%{GREEDYDATA:irrelevant})?" ] } } grok { match => { - "cef_msg" => [ + "cef_or_leef_msg" => [ "username=%{DATA:src_user} %{WORD}=(%{GREEDYDATA:irrelevant})?" ] } @@ -148,7 +148,7 @@ filter { grok { match => { - "cef_msg" => [ + "cef_or_leef_msg" => [ "activity=%{DATA:activity} %{WORD}=(%{GREEDYDATA:irrelevant})?" ] } @@ -156,7 +156,7 @@ filter { grok { match => { - "cef_msg" => [ + "cef_or_leef_msg" => [ "description=%{DATA:description} %{WORD}=(%{GREEDYDATA:irrelevant})?" ] } @@ -164,7 +164,7 @@ filter { grok { match => { - "cef_msg" => [ + "cef_or_leef_msg" => [ "parent_path=%{DATA:parent_path} %{WORD}=(%{GREEDYDATA:irrelevant})?" ] } @@ -172,7 +172,7 @@ filter { grok { match => { - "cef_msg" => [ + "cef_or_leef_msg" => [ "path=%{DATA:path} %{WORD}=(%{GREEDYDATA:irrelevant})?" ] } @@ -192,7 +192,7 @@ filter { rename => { "[format_version]" => "[kv_field][format_version]" } rename => { "[format_type]" => "[kv_field][format_type]" } rename => { "[end_msg]" => "[kv_field][end_msg]" } - rename => { "[message]" => "[kv_field][message]" } + rename => { "[prvf_message]" => "[kv_field][message]" } #Generating other fields rename => { "[kv_field][srcIP]" => "[kv_field][src_ip]" } @@ -279,6 +279,10 @@ if [kv_field][severity]{ event.get("[kv_field]").each do |k, v| if (v == "X0X") event.set("[logx][privafy][#{k}]",nil) + elsif k.start_with?("-") + event.remove(k) + elsif k =~ /\W(.*)?/ + event.remove(k) elsif !(v.kind_of?(Array)) new_v = v.to_s.gsub(/\"/, "") new_v = new_v.gsub(/\'/, "") @@ -293,7 +297,7 @@ if [kv_field][severity]{ #......................................................................# #Finally, remove unnecessary fields mutate { - remove_field => ["@version","path","tags","type","syslog_version","kv_field","prvf_message","[logx][syslog][message]", + remove_field => ["@version","path","tags","type","syslog_version","kv_field","prvf_message","[logx][syslog]", "not_defined","cef_or_leef_msg_all","cef_or_leef_msg","syslog_date_host","irrelevant","init_msg"] } }