tweaks for idaholab#419, testing ja4+ merge

arkime/etc/config.ini

@@ -2672,7 +2672,7 @@ o_zeek_files=require:zeek.files;title:Zeek files.log;fields:zeek.files.tx_hosts,
 o_zeek_ftp=require:zeek.ftp;title:Zeek ftp.log;fields:zeek.ftp.command,zeek.ftp.arg,zeek.ftp.mime_type,zeek.ftp.file_size,zeek.ftp.reply_code,zeek.ftp.reply_msg,zeek.ftp.data_channel.passive,zeek.ftp.data_channel.orig_h,zeek.ftp.data_channel.resp_h,zeek.ftp.data_channel.resp_p
 o_zeek_genisys=require:zeek.genisys;title:Zeek genisys.log;fields:zeek.genisys.header,zeek.genisys.server,zeek.genisys.direction,zeek.genisys.crc_transmitted,zeek.genisys.crc_calculated,zeek.genisys.payload.address,zeek.genisys.payload.data
 o_zeek_gquic=require:zeek.gquic;title:Zeek gquic.log;fields:zeek.gquic.version,zeek.gquic.server_name,zeek.gquic.user_agent,zeek.gquic.tag_count,zeek.gquic.cyu,zeek.gquic.cyutags
-o_zeek_http=require:zeek.http;title:Zeek http.log;fields:zeek.http.trans_depth,zeek.http.method,zeek.http.host,zeek.http.uri,zeek.http.origin,zeek.http.post_password_plain,zeek.http.post_username,zeek.http.referrer,zeek.http.version,zeek.http.user_agent,zeek.http.request_body_len,zeek.http.response_body_len,zeek.http.status_code,zeek.http.status_msg,zeek.http.info_code,zeek.http.info_msg,zeek.http.tags,zeek.http.proxied,zeek.http.orig_fuids,zeek.http.orig_filenames,zeek.http.orig_mime_types,zeek.http.resp_fuids,zeek.http.resp_filenames,zeek.http.resp_mime_types,zeek.http.ja4
+o_zeek_http=require:zeek.http;title:Zeek http.log;fields:zeek.http.trans_depth,zeek.http.method,zeek.http.host,zeek.http.uri,zeek.http.origin,zeek.http.post_password_plain,zeek.http.post_username,zeek.http.referrer,zeek.http.version,zeek.http.user_agent,zeek.http.request_body_len,zeek.http.response_body_len,zeek.http.status_code,zeek.http.status_msg,zeek.http.info_code,zeek.http.info_msg,zeek.http.tags,zeek.http.proxied,zeek.http.orig_fuids,zeek.http.orig_filenames,zeek.http.orig_mime_types,zeek.http.resp_fuids,zeek.http.resp_filenames,zeek.http.resp_mime_types,zeek.http.ja4h
 o_zeek_intel=require:zeek.intel;title:Zeek intel.log;fields:zeek.intel.cif_tags,zeek.intel.cif_confidence,zeek.intel.cif_source,zeek.intel.cif_description,zeek.intel.cif_firstseen,zeek.intel.cif_lastseen,zeek.intel.seen_indicator,zeek.intel.seen_indicator_type,zeek.intel.seen_where,zeek.intel.seen_node,zeek.intel.matched,zeek.intel.sources,zeek.intel.file_mime_type,zeek.intel.file_description
 o_zeek_ipsec=require:zeek.ipsec;title:Zeek ipsec.log;fields:zeek.ipsec.initiator_spi,zeek.ipsec.responder_spi,zeek.ipsec.maj_ver,zeek.ipsec.min_ver,zeek.ipsec.exchange_type,zeek.ipsec.doi,zeek.ipsec.flag_e,zeek.ipsec.flag_c,zeek.ipsec.flag_a,zeek.ipsec.flag_i,zeek.ipsec.flag_v,zeek.ipsec.flag_r,zeek.ipsec.flags,zeek.ipsec.message_id,zeek.ipsec.vendor_ids,zeek.ipsec.notify_messages,zeek.ipsec.transforms,zeek.ipsec.ke_dh_groups,zeek.ipsec.proposals,zeek.ipsec.protocol_id,zeek.ipsec.certificates,zeek.ipsec.situation,zeek.ipsec.transform_attributes,zeek.ipsec.length,zeek.ipsec.hash
 o_zeek_irc=require:zeek.irc;title:Zeek irc.log;fields:zeek.irc.nick,zeek.irc.command,zeek.irc.value,zeek.irc.addl,zeek.irc.dcc_file_name,zeek.irc.dcc_file_size,zeek.irc.dcc_mime_type

arkime/wise/source.zeeklogs.js

@@ -2010,8 +2010,6 @@ class MalcolmSource extends WISESource {
       "zeek.ssl.issuer.ST",
       "zeek.ssl.issuer.title",
       "zeek.ssl.issuer_full",
-      "zeek.ssl.ja4",
-      "zeek.ssl.ja4s",
       "zeek.ssl.last_alert",
       "zeek.ssl.next_protocol",
       "zeek.ssl.resumed",

dashboards/dashboards/0ad3d7c2-3441-485e-9dfe-dbb22e84e576.json

@@ -170,7 +170,7 @@
       "version": "WzYzLDFd",
       "attributes": {
         "title": "Log Type",
-        "visState": "{\"title\":\"Log Type\",\"type\":\"table\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"event.provider\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"unknown\",\"customLabel\":\"Data Source\"},\"schema\":\"bucket\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"event.dataset\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Log Type\"},\"schema\":\"bucket\"}],\"params\":{\"perPage\":10,\"showPartialRows\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"showMetricsAtAllLevels\":false,\"percentageCol\":\"\"}}",
+        "visState": "{\"title\":\"Log Type\",\"type\":\"table\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"event.provider\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":50,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"unknown\",\"customLabel\":\"Data Source\"},\"schema\":\"bucket\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"event.dataset\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":250,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Log Type\"},\"schema\":\"bucket\"}],\"params\":{\"perPage\":10,\"showPartialRows\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"showMetricsAtAllLevels\":false,\"percentageCol\":\"\"}}",
         "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":2,\"direction\":\"desc\"}}}}",
         "description": "",
         "version": 1,
@@ -229,7 +229,7 @@
       "version": "WzY1LDFd",
       "attributes": {
         "title": "DNS - Queries",
-        "visState": "{\"title\":\"DNS - Queries\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"zeek.dns.query\",\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":100,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Query\"}}]}",
+        "visState": "{\"title\":\"DNS - Queries\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"zeek.dns.query\",\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":250,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Query\"}}]}",
         "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}",
         "description": "",
         "version": 1,
@@ -288,7 +288,7 @@
       "version": "WzY3LDFd",
       "attributes": {
         "title": "Application Protocol",
-        "visState": "{\"title\":\"Application Protocol\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":0,\"direction\":\"asc\"},\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\",\"dimensions\":{\"metrics\":[{\"accessor\":2,\"format\":{\"id\":\"number\"},\"params\":{},\"label\":\"Count\",\"aggType\":\"count\"}],\"buckets\":[{\"accessor\":0,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\"}},\"params\":{},\"label\":\"Application Protocol\",\"aggType\":\"terms\"},{\"accessor\":1,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\"}},\"params\":{},\"label\":\"Protocol Version\",\"aggType\":\"terms\"}]}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"network.protocol\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Application Protocol\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"network.protocol_version\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Protocol Version\"}}]}",
+        "visState": "{\"title\":\"Application Protocol\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":0,\"direction\":\"asc\"},\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\",\"dimensions\":{\"metrics\":[{\"accessor\":2,\"format\":{\"id\":\"number\"},\"params\":{},\"label\":\"Count\",\"aggType\":\"count\"}],\"buckets\":[{\"accessor\":0,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\"}},\"params\":{},\"label\":\"Application Protocol\",\"aggType\":\"terms\"},{\"accessor\":1,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\"}},\"params\":{},\"label\":\"Protocol Version\",\"aggType\":\"terms\"}]}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"network.protocol\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":250,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Application Protocol\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"network.protocol_version\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Protocol Version\"}}]}",
         "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":2,\"direction\":\"desc\"}}}}",
         "description": "",
         "version": 1,
@@ -317,7 +317,7 @@
       "version": "WzY5MywxXQ==",
       "attributes": {
         "title": "Actions and Results",
-        "visState": "{\"title\":\"Actions and Results\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":2,\"direction\":\"desc\"},\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\",\"dimensions\":{\"metrics\":[{\"accessor\":3,\"format\":{\"id\":\"number\"},\"params\":{},\"label\":\"Count\",\"aggType\":\"count\"}],\"buckets\":[{\"accessor\":0,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Unknown\"}},\"params\":{},\"label\":\"Protocol\",\"aggType\":\"terms\"},{\"accessor\":1,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"-\"}},\"params\":{},\"label\":\"Action\",\"aggType\":\"terms\"},{\"accessor\":2,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"-\"}},\"params\":{},\"label\":\"Result\",\"aggType\":\"terms\"}]}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"network.protocol\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Unknown\",\"customLabel\":\"Protocol\"}},{\"id\":\"4\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"event.action\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Action\"}},{\"id\":\"5\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"event.result\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Result\"}}]}",
+        "visState": "{\"title\":\"Actions and Results\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":2,\"direction\":\"desc\"},\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\",\"dimensions\":{\"metrics\":[{\"accessor\":3,\"format\":{\"id\":\"number\"},\"params\":{},\"label\":\"Count\",\"aggType\":\"count\"}],\"buckets\":[{\"accessor\":0,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Unknown\"}},\"params\":{},\"label\":\"Protocol\",\"aggType\":\"terms\"},{\"accessor\":1,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"-\"}},\"params\":{},\"label\":\"Action\",\"aggType\":\"terms\"},{\"accessor\":2,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"-\"}},\"params\":{},\"label\":\"Result\",\"aggType\":\"terms\"}]}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"network.protocol\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":250,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Unknown\",\"customLabel\":\"Protocol\"}},{\"id\":\"4\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"event.action\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":250,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Action\"}},{\"id\":\"5\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"event.result\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":250,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Result\"}}]}",
         "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":3,\"direction\":\"desc\"}}}}",
         "description": "",
         "version": 1,

dashboards/templates/composable/component/zeek.json

@@ -545,8 +545,6 @@
         "zeek.ssl.issuer.ST": { "type": "keyword" },
         "zeek.ssl.issuer.title": { "type": "keyword" },
         "zeek.ssl.issuer_full": { "type": "keyword", "ignore_above": 1024, "fields": { "text": { "type": "text" } } },
-        "zeek.ssl.ja4": { "type": "keyword" },
-        "zeek.ssl.ja4s": { "type": "keyword" },
         "zeek.ssl.last_alert": { "type": "keyword" },
         "zeek.ssl.next_protocol": { "type": "keyword" },
         "zeek.ssl.resumed": { "type": "keyword" },

logstash/pipelines/zeek/12_zeek_mutate.conf

@@ -764,9 +764,9 @@ filter {
     }
 
     # There is no good ECS equivalent for this field so it will map to Arkime's field name only
-    if ([zeek][http][ja4]) {
-      mutate { id => "mutate_merge_zeek_http_ja4"
-               merge => { "[http][ja4]" => "[zeek][http][ja4]" } }
+    if ([zeek][http][ja4h]) {
+      mutate { id => "mutate_merge_zeek_http_ja4h"
+               merge => { "[http][ja4h]" => "[zeek][http][ja4h]" } }
     }
 
     if ([zeek][http][post_username]) {

logstash/pipelines/zeek/13_zeek_normalize.conf

@@ -1369,8 +1369,8 @@ filter {
                                                           merge => { "[related][hash]" => "[zeek][files][sha1]" } } }
   if ([zeek][files][sha256]) {                   mutate { id => "mutate_merge_field_related_hash_files_sha256"
                                                           merge => { "[related][hash]" => "[zeek][files][sha256]" } } }
-  if ([zeek][http][ja4]) {                        mutate { id => "mutate_merge_field_related_hash_http_ja4"
-                                                          merge => { "[related][hash]" => "[zeek][http][ja4]" } } }                                                        
+  if ([zeek][http][ja4]h) {                      mutate { id => "mutate_merge_field_related_hash_http_ja4h"
+                                                         merge => { "[related][hash]" => "[zeek][http][ja4h]" } } }
   if ([zeek][ssh][hassh]) {                      mutate { id => "mutate_merge_field_related_hash_ssh_hassh"
                                                           merge => { "[related][hash]" => "[zeek][ssh][hassh]" } } }
   if ([zeek][ssh][hasshServer]) {                mutate { id => "mutate_merge_field_related_hash_ssh_hasshServer"