tweaks for idaholab#419, testing ja4+ merge

mmguero-dev · Jun 25, 2024 · dbf3367 · dbf3367
1 parent d44b353
commit dbf3367
Show file tree

Hide file tree

Showing 3 changed files with 22 additions and 41 deletions.
diff --git a/arkime/wise/source.zeeklogs.js b/arkime/wise/source.zeeklogs.js
@@ -715,10 +715,6 @@ class MalcolmSource extends WISESource {
       "threat.technique.id",
       "threat.technique.name",
       "threat.technique.reference",
-      "tcp.client.ja4l",
-      "tcp.server.ja4ls",
-      "tcp.client.ja4t",
-      "tcp.server.ja4ts",
       "tls.cipher",
       "tls.client.issuer",
       "tls.client.ja4",

diff --git a/dashboards/templates/composable/component/malcolm_common.json b/dashboards/templates/composable/component/malcolm_common.json
@@ -71,7 +71,9 @@
             "url": { "type": "keyword" },
             "details": { "type": "nested" }
           }
-        }
+        },
+        "tls.client.ja4": { "type": "keyword" },
+        "tls.server.ja4s": { "type": "keyword" }
       }
     }
   }

diff --git a/logstash/pipelines/zeek/12_zeek_mutate.conf b/logstash/pipelines/zeek/12_zeek_mutate.conf
@@ -180,6 +180,7 @@ filter {
       }
     }
 
+    # rename conn.log's TCP JA4 fields to what Arkime uses
     if ([zeek][conn][ja4l]) {
       mutate { id => "mutate_merge_zeek_tcp_ja4l"
                merge => { "[tcp][ja4l]" => "[zeek][conn][ja4l]" } }
@@ -200,27 +201,6 @@ filter {
                merge => { "[tcp][ja4ts]" => "[zeek][conn][ja4ts]" } }
     }
 
-    # ECS uses the client/server convention with JA4. This is parallelled with the JA4 location fields
-    if ([zeek][conn][ja4l]) {
-      mutate { id => "mutate_add_field_zeek_tcp_ja4l"
-               add_field => { "[tcp][client][ja4l]" => "[zeek][conn][ja4l]" } }
-    }
-
-    if ([zeek][conn][ja4ls]) {
-      mutate { id => "mutate_add_field_zeek_tcp_ja4ls"
-               add_field => { "[tcp][server][ja4ls]" => "[zeek][conn][ja4ls]" } }
-    }
-
-    if ([zeek][conn][ja4t]) {
-      mutate { id => "mutate_add_field_zeek_tcp_ja4t"
-               add_field => { "[tcp][client][ja4t]" => "[zeek][conn][ja4t]" } }
-    }
-
-    if ([zeek][conn][ja4ts]) {
-      mutate { id => "mutate_add_field_zeek_tcp_ja4ts"
-               add_field => { "[tcp][server][ja4ts]" => "[zeek][conn][ja4ts]" } }
-    }
-
     # aggregate total bytes and packets
     ruby {
       id => "ruby_zeek_bytes_and_packets_calc"
@@ -920,18 +900,23 @@ filter {
     #############################################################################################################################
     # ja4ssh.log specific logic
 
-    mutate {
-      id => "mutate_rename_ja4ssh_fields"
-      rename => { "[zeek][ja4ssh][ja4ssh]" => "[zeek][ssh][ja4ssh]" }
-    }
-
-    mutate {
-      id => "mutate_merge_ja4ssh_fields"
-      merge => { "[ssh][ja4ssh]" => "[zeek][ssh][ja4ssh]" }
+    if ([zeek][ja4ssh][ja4ssh]) {
+      mutate {
+        id => "mutate_rename_ja4ssh_fields"
+        rename => { "[zeek][ja4ssh][ja4ssh]" => "[zeek][ssh][ja4ssh]" }
+      }
+      mutate {
+        id => "mutate_merge_ja4ssh_fields"
+        merge => { "[ssh][ja4ssh]" => "[zeek][ssh][ja4ssh]" }
+      }
     }
 
     mutate { id => "mutate_remove_fields_zeek_ja4ssh"
-             remove_field => [ "[zeek][ja4ssh]" ] }
+             remove_field => [ "[zeek][ja4ssh][is_ssh]",
+                               "[zeek][ja4ssh][orig_pack_len]",
+                               "[zeek][ja4ssh][resp_pack_len]",
+                               "[zeek][ja4ssh][orig_ack]",
+                               "[zeek][ja4ssh][resp_ack]" ] }
 
   } else if ([log_source] == "kerberos") {
     #############################################################################################################################
@@ -2070,8 +2055,6 @@ filter {
     if ([zeek][ssl][ja4]) {
       mutate { id => "mutate_merge_zeek_ssl_ja4"
                merge => { "[tls][ja4]" => "[zeek][ssl][ja4]" } }
-      mutate { id => "mutate_merge_zeek_ssl_ja4_ecs"
-              merge => { "[tls][ja4]" => "[zeek][ssl][ja4]" } }
     }
 
     if ([zeek][ssl][ja4s]) {
@@ -2131,8 +2114,8 @@ filter {
                                                      add_field => { "[tls][curve]" => "%{[zeek][ssl][curve]}" } } }
 
     # ECS - zeek.ssl.ja4 -> tls.client.ja4
-    if ([zeek][ssl][ja4]) {                  mutate { id => "mutate_add_field_ecs_zeek_tls_client_ja4"
-                                                     add_field => { "[tls][client][ja4]" => "%{[zeek][ssl][ja4]}" } } }
+    if ([zeek][ssl][ja4]) {                  mutate { id => "mutate_rename_ecs_zeek_tls_client_ja4"
+                                                      rename => { "[zeek][ssl][ja4]" => "[tls][client][ja4]" } } }
 
     # ECS - zeek.ssl.client_issuer_full -> tls.client.issuer
     if ([zeek][ssl][client_issuer_full]) {   mutate { id => "mutate_add_field_ecs_zeek_tls_client_issuer_full"
@@ -2155,8 +2138,8 @@ filter {
                                                      add_field => { "[tls][server][issuer]" => "%{[zeek][ssl][issuer_full]}" } } }
 
     # ECS - zeek.ssl.ja4s -> tls.server.ja4s
-    if ([zeek][ssl][ja4s]) {                 mutate { id => "mutate_add_field_ecs_zeek_tls_server_ja4s"
-                                                     add_field => { "[tls][server][ja4s]" => "%{[zeek][ssl][ja4s]}" } } }
+    if ([zeek][ssl][ja4s]) {                 mutate { id => "mutate_rename_field_ecs_zeek_tls_server_ja4s"
+                                                      rename => { "[zeek][ssl][ja4s]" => "[tls][server][ja4s]" } } }
 
     # ECS - zeek.ssl.subject_full -> tls.server.subject
     if ([zeek][ssl][subject_full]) {         mutate { id => "mutate_add_field_ecs_zeek_tls_subject_full"