From 52269e094552d8932361d74d8d22583b941fa475 Mon Sep 17 00:00:00 2001 From: Krishna Chaitanya Reddy Burri Date: Mon, 14 Aug 2023 13:20:52 +0530 Subject: [PATCH] [Juniper SRX] Fix grok patterns for system logs (#7280) * Fix grok patterns for Juniper System logs * update pr num * update negotiation grok * Fix FW groks * Fix rtslib_dfwsm_get_async_cb * Add reth_scan * Non pid structured * Fix other patterns * Refactor * Add required fields * update readme * PR comments * Address PR comments --- packages/juniper_srx/changelog.yml | 5 + .../log/_dev/test/pipeline/test-system.log | 5 + .../pipeline/test-system.log-expected.json | 339 +++++++++++++++-- .../elasticsearch/ingest_pipeline/default.yml | 25 +- .../elasticsearch/ingest_pipeline/system.yml | 359 ++++++++++++------ .../data_stream/log/fields/fields.yml | 65 +++- packages/juniper_srx/docs/README.md | 35 +- packages/juniper_srx/manifest.yml | 2 +- 8 files changed, 674 insertions(+), 161 deletions(-) diff --git a/packages/juniper_srx/changelog.yml b/packages/juniper_srx/changelog.yml index 27aecd829d17..dae8fe730c36 100644 --- a/packages/juniper_srx/changelog.yml +++ b/packages/juniper_srx/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.14.1" + changes: + - description: Fix system logs grok + type: bugfix + link: https://github.com/elastic/integrations/pull/7280 - version: "1.14.0" changes: - description: Update package to ECS 8.9.0. diff --git a/packages/juniper_srx/data_stream/log/_dev/test/pipeline/test-system.log b/packages/juniper_srx/data_stream/log/_dev/test/pipeline/test-system.log index f7e182771cbb..2a3dadbf708b 100644 --- a/packages/juniper_srx/data_stream/log/_dev/test/pipeline/test-system.log +++ b/packages/juniper_srx/data_stream/log/_dev/test/pipeline/test-system.log @@ -1,6 +1,10 @@ <30>1 2023-05-04T15:27:30.846+10:00 AB1234-ABC2-AB-AB01C-ABC kmd 8961 KMD_PM_SA_ESTABLISHED [junos@1111.1.1.1.1.111 local-address="89.160.20.112" remote-address="67.43.156.0" local-initiator="ipv4(89.160.20.112-89.160.20.114)" remote-responder="ipv4(67.43.156.0)" argument1="outbound" index1="36090046" index2="0" mode="Tunnel" type="dynamic" traffic-selector-name="ASJLKN_JKHA" first-forwarding-class=""] <30>1 2023-05-04T15:27:26.461+10:00 AB1234-A-AB-AB01C-ABC kmd 13862 KMD_PM_SA_ESTABLISHED [junos@1111.1.1.1.1.111 local-address="89.160.20.112" remote-address="67.43.156.0" local-initiator="ipv4_subnet(any:0,[0..7\]=89.160.20.112/29)" remote-responder="ipv4_subnet(any:0,[0..7\]=67.43.156.0/24)" argument1="outbound" index1="3700499780" index2="0" mode="Tunnel" type="dynamic" traffic-selector-name="" first-forwarding-class=""] Local gateway: 89.160.20.115, Remote gateway: 67.43.156.1, Local ID: ipv4_subnet(any:0,[0..7]=89.160.20.114/29), Remote ID: ipv4_subnet(any:0,[0..7]=67.43.156.1/24), Direction: outbound, SPI: 0xdc912544, AUX-SPI: 0, Mode: Tunnel, Type: dynamic, Traffic-selector: FC Name: <27>1 2023-05-04T15:19:33.984+10:00 AB1234-A-AB-AB01C-ABC kmd 9159 - - IKE negotiation failed with error: Timed out. IKE Version: 1, VPN: IPSEC-AAAAA-AAA1-PROD-VPN Gateway: IKE-AAAAA-AAA1-GW, Local: 89.160.20.112/500, Remote: 67.43.156.1/500, Local IKE-ID: Not-Available, Remote IKE-ID: Not-Available, VR-ID: 5: Role: Initiator +<27>1 2023-05-04T15:19:33.984+10:00 AB1234-A-AB-AB01C-ABC kmd 9159 asd2 - IKE negotiation failed with error: Timed out. IKE Version: 1, VPN: IPSEC-AAAAA-AAA1-PROD-VPN Gateway: IKE-AAAAA-AAA1-GW, Local: 89.160.20.112/500, Remote: 67.43.156.1/500, Local IKE-ID: Not-Available, Remote IKE-ID: Not-Available, VR-ID: 5: Role: Initiator +<27>1 2023-07-04T12:22:36.461+10:00 AC004-PR-VPN01-DMZ kmd 9812 - - IKE negotiation failed with error: Timed out. IKE Version: 1, VPN: IPSEC-HORSEFACTS-TUN1-PROD-VPN Gateway: IKE-HORSEFACTS-TUN1-GW, Local: 10.11.22.444/500, Remote: 198.1.124.8/500, Local IKE-ID: Not-Available, Remote IKE-ID: Not-Available, VR-ID: 5, Role: Initiator +<30>1 2023-07-04T10:21:11.590+10:00 AC004-PR-VPN01-DMZ kmd 9812 - - IKE negotiation successfully completed. IKE Version: 1, VPN: IPSEC-NIKON-TUN1-PROD-VPN Gateway: IKE-NIKON-TUN1-GW, Local: 10.8.10.115/9001, Remote: 89.160.20.112/9001, Local IKE-ID: 81.2.69.192, Remote IKE-ID: 89.160.20.112, VR-ID: 6, Role: Responder +<27>1 2023-07-04T11:48:31.702+10:00 AC004-PR-VPN01-DMZ kmd 9812 - - IPSec negotiation failed with error: Peer proposed traffic-selectors are not in configured range. IKE Version: 2, VPN: IPSEC-INT-ORMB-TUN2-VPN Gateway: IKE-INT-ORMB-TUN2-GW, Local: 10.32.64.128/9001, Remote: 89.160.20.112/9001, Local IKE-ID: 89.160.20.112, Remote IKE-ID: 89.160.20.112, VR-ID: 6 <158>1 2023-05-04T15:21:01.102+10:00 AB1234-ABC2-AB-AB01C-ABC kernel - - - FW: gr-0/0/0.14 A udp 127.0.0.1 89.160.20.112 49153 49153 <158>1 2023-05-04T15:18:05.010+10:00 AB1234-ABC2-AB-AB01C-ABC - - - - node1.fpc0 PFE_FW_SYSLOG_IP: FW: reth5.175 A pim 67.43.156.1 89.160.20.113 0 0 (1 packets) <158>1 2023-05-09T12:20:23.180+10:00 AAAA-A-AA-AAAAAA-AAAAAA-AAA - - - - AAAA-A-AA-AAAAAA-AAAAAA-AAA PFE_FW_SYSLOG_IP: FW: reth2.605 A udp 67.43.156.2 89.160.20.112 0 0 (1 packets) @@ -15,3 +19,4 @@ <166>1 2023-05-08T10:54:24.821+10:00 AB1234-A-AB-AB01C-ABC - - - - AB1234-A-AB-AB01C-ABC dpdk_eth_devstart (pid=0x4c6a1bc0): port 7 ifd xe-0/0/7, new dpdk_port_state=2 dpdk_swt_port_state 1 <166>1 2023-05-08T10:54:24.756+10:00 AB1234-A-AB-AB01C-ABC - - - - AB1234-A-AB-AB01C-ABC nh_fabric_fill_jnhinfo: Storing nh_id as 0x2dd and jnh as 0x58e302 <167>1 2023-05-08T10:54:24.704+10:00 AB1234-A-AB-AB01C-ABC - - - - AB1234-A-AB-AB01C-ABC Copying remote chassis chassis 1, IP: 81.2.69.192 +<166>1 2023-05-08T10:54:24.756+10:00 AB1234-A-AB-AB01C-ABC - - - - AB1234-A-AB-AB01C-ABC nh_fabric_fill_jnhinfo: ABCDE: Test default message 123456 diff --git a/packages/juniper_srx/data_stream/log/_dev/test/pipeline/test-system.log-expected.json b/packages/juniper_srx/data_stream/log/_dev/test/pipeline/test-system.log-expected.json index 5c4245795f2f..a5ab1e900c25 100644 --- a/packages/juniper_srx/data_stream/log/_dev/test/pipeline/test-system.log-expected.json +++ b/packages/juniper_srx/data_stream/log/_dev/test/pipeline/test-system.log-expected.json @@ -94,6 +94,18 @@ "mode": "Tunnel", "process": "kmd", "remote_responder": "ipv4_subnet(any:0,[0..7\\]=67.43.156.0/24)", + "system": { + "aux_spi": 0, + "direction": "outbound", + "local_gateway": "89.160.20.115", + "local_id": "ipv4_subnet(any:0,[0..7]=89.160.20.114/29)", + "mode": "Tunnel", + "remote_gateway": "67.43.156.1", + "remote_id": "ipv4_subnet(any:0,[0..7]=67.43.156.1/24)", + "spi": "0xdc912544", + "traffic_selector": "FC Name:", + "type": "dynamic" + }, "tag": "KMD_PM_SA_ESTABLISHED", "traffic_selector_name": "", "type": "dynamic" @@ -115,6 +127,8 @@ }, "related": { "ip": [ + "89.160.20.115", + "67.43.156.1", "89.160.20.112", "67.43.156.0" ] @@ -145,13 +159,25 @@ "juniper": { "srx": { "log_type": "system", + "negotiation": { + "err_msg": "Timed out", + "message": "failed with error: Timed out. IKE Version: 1, VPN: IPSEC-AAAAA-AAA1-PROD-VPN Gateway: IKE-AAAAA-AAA1-GW, Local: 89.160.20.112/500, Remote: 67.43.156.1/500, Local IKE-ID: Not-Available, Remote IKE-ID: Not-Available, VR-ID: 5: Role: Initiator", + "type": "IKE" + }, "process": "kmd", - "tag": "Role" + "system": { + "ike_version": 1, + "local": "89.160.20.112/500", + "remote": "67.43.156.1/500", + "vpn": "IPSEC-AAAAA-AAA1-PROD-VPN Gateway: IKE-AAAAA-AAA1-GW", + "vr_id": "5: Role: Initiator" + } } }, "log": { "level": "error" }, + "message": "IKE Version: 1, VPN: IPSEC-AAAAA-AAA1-PROD-VPN Gateway: IKE-AAAAA-AAA1-GW, Local: 89.160.20.112/500, Remote: 67.43.156.1/500, Local IKE-ID: Not-Available, Remote IKE-ID: Not-Available, VR-ID: 5: Role: Initiator", "observer": { "name": "AB1234-A-AB-AB01C-ABC", "product": "SRX", @@ -166,6 +192,207 @@ "preserve_original_event" ] }, + { + "@timestamp": "2023-05-04T05:19:33.984Z", + "ecs": { + "version": "8.9.0" + }, + "event": { + "category": [ + "network" + ], + "kind": "event", + "original": "\u003c27\u003e1 2023-05-04T15:19:33.984+10:00 AB1234-A-AB-AB01C-ABC kmd 9159 asd2 - IKE negotiation failed with error: Timed out. IKE Version: 1, VPN: IPSEC-AAAAA-AAA1-PROD-VPN Gateway: IKE-AAAAA-AAA1-GW, Local: 89.160.20.112/500, Remote: 67.43.156.1/500, Local IKE-ID: Not-Available, Remote IKE-ID: Not-Available, VR-ID: 5: Role: Initiator", + "severity": 27 + }, + "juniper": { + "srx": { + "log_type": "system", + "negotiation": { + "err_msg": "Timed out", + "message": "failed with error: Timed out. IKE Version: 1, VPN: IPSEC-AAAAA-AAA1-PROD-VPN Gateway: IKE-AAAAA-AAA1-GW, Local: 89.160.20.112/500, Remote: 67.43.156.1/500, Local IKE-ID: Not-Available, Remote IKE-ID: Not-Available, VR-ID: 5: Role: Initiator", + "type": "IKE" + }, + "process": "kmd", + "system": { + "ike_version": 1, + "local": "89.160.20.112/500", + "remote": "67.43.156.1/500", + "vpn": "IPSEC-AAAAA-AAA1-PROD-VPN Gateway: IKE-AAAAA-AAA1-GW", + "vr_id": "5: Role: Initiator" + } + } + }, + "log": { + "level": "error" + }, + "message": "IKE Version: 1, VPN: IPSEC-AAAAA-AAA1-PROD-VPN Gateway: IKE-AAAAA-AAA1-GW, Local: 89.160.20.112/500, Remote: 67.43.156.1/500, Local IKE-ID: Not-Available, Remote IKE-ID: Not-Available, VR-ID: 5: Role: Initiator", + "observer": { + "name": "AB1234-A-AB-AB01C-ABC", + "product": "SRX", + "type": "firewall", + "vendor": "Juniper" + }, + "process": { + "name": "kmd", + "pid": 9159 + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2023-07-04T02:22:36.461Z", + "ecs": { + "version": "8.9.0" + }, + "event": { + "category": [ + "network" + ], + "kind": "event", + "original": "\u003c27\u003e1 2023-07-04T12:22:36.461+10:00 AC004-PR-VPN01-DMZ kmd 9812 - - IKE negotiation failed with error: Timed out. IKE Version: 1, VPN: IPSEC-HORSEFACTS-TUN1-PROD-VPN Gateway: IKE-HORSEFACTS-TUN1-GW, Local: 10.11.22.444/500, Remote: 198.1.124.8/500, Local IKE-ID: Not-Available, Remote IKE-ID: Not-Available, VR-ID: 5, Role: Initiator", + "severity": 27 + }, + "juniper": { + "srx": { + "log_type": "system", + "negotiation": { + "err_msg": "Timed out", + "message": "failed with error: Timed out. IKE Version: 1, VPN: IPSEC-HORSEFACTS-TUN1-PROD-VPN Gateway: IKE-HORSEFACTS-TUN1-GW, Local: 10.11.22.444/500, Remote: 198.1.124.8/500, Local IKE-ID: Not-Available, Remote IKE-ID: Not-Available, VR-ID: 5, Role: Initiator", + "type": "IKE" + }, + "process": "kmd", + "system": { + "ike_version": 1, + "local": "10.11.22.444/500", + "remote": "198.1.124.8/500", + "role": "Initiator", + "vpn": "IPSEC-HORSEFACTS-TUN1-PROD-VPN Gateway: IKE-HORSEFACTS-TUN1-GW", + "vr_id": "5" + } + } + }, + "log": { + "level": "error" + }, + "message": "IKE Version: 1, VPN: IPSEC-HORSEFACTS-TUN1-PROD-VPN Gateway: IKE-HORSEFACTS-TUN1-GW, Local: 10.11.22.444/500, Remote: 198.1.124.8/500, Local IKE-ID: Not-Available, Remote IKE-ID: Not-Available, VR-ID: 5, Role: Initiator", + "observer": { + "name": "AC004-PR-VPN01-DMZ", + "product": "SRX", + "type": "firewall", + "vendor": "Juniper" + }, + "process": { + "name": "kmd", + "pid": 9812 + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2023-07-04T00:21:11.590Z", + "ecs": { + "version": "8.9.0" + }, + "event": { + "category": [ + "network" + ], + "kind": "event", + "original": "\u003c30\u003e1 2023-07-04T10:21:11.590+10:00 AC004-PR-VPN01-DMZ kmd 9812 - - IKE negotiation successfully completed. IKE Version: 1, VPN: IPSEC-NIKON-TUN1-PROD-VPN Gateway: IKE-NIKON-TUN1-GW, Local: 10.8.10.115/9001, Remote: 89.160.20.112/9001, Local IKE-ID: 81.2.69.192, Remote IKE-ID: 89.160.20.112, VR-ID: 6, Role: Responder", + "severity": 30 + }, + "juniper": { + "srx": { + "log_type": "system", + "negotiation": { + "message": "successfully completed. IKE Version: 1, VPN: IPSEC-NIKON-TUN1-PROD-VPN Gateway: IKE-NIKON-TUN1-GW, Local: 10.8.10.115/9001, Remote: 89.160.20.112/9001, Local IKE-ID: 81.2.69.192, Remote IKE-ID: 89.160.20.112, VR-ID: 6, Role: Responder", + "type": "IKE" + }, + "process": "kmd", + "system": { + "ike_version": 1, + "local": "10.8.10.115/9001", + "local_ike_id": "81.2.69.192", + "remote": "89.160.20.112/9001", + "remote_ike_id": "89.160.20.112", + "role": "Responder", + "vpn": "IPSEC-NIKON-TUN1-PROD-VPN Gateway: IKE-NIKON-TUN1-GW", + "vr_id": "6" + } + } + }, + "log": { + "level": "informational" + }, + "message": "IKE Version: 1, VPN: IPSEC-NIKON-TUN1-PROD-VPN Gateway: IKE-NIKON-TUN1-GW, Local: 10.8.10.115/9001, Remote: 89.160.20.112/9001, Local IKE-ID: 81.2.69.192, Remote IKE-ID: 89.160.20.112, VR-ID: 6, Role: Responder", + "observer": { + "name": "AC004-PR-VPN01-DMZ", + "product": "SRX", + "type": "firewall", + "vendor": "Juniper" + }, + "process": { + "name": "kmd", + "pid": 9812 + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2023-07-04T01:48:31.702Z", + "ecs": { + "version": "8.9.0" + }, + "event": { + "category": [ + "network" + ], + "kind": "event", + "original": "\u003c27\u003e1 2023-07-04T11:48:31.702+10:00 AC004-PR-VPN01-DMZ kmd 9812 - - IPSec negotiation failed with error: Peer proposed traffic-selectors are not in configured range. IKE Version: 2, VPN: IPSEC-INT-ORMB-TUN2-VPN Gateway: IKE-INT-ORMB-TUN2-GW, Local: 10.32.64.128/9001, Remote: 89.160.20.112/9001, Local IKE-ID: 89.160.20.112, Remote IKE-ID: 89.160.20.112, VR-ID: 6", + "severity": 27 + }, + "juniper": { + "srx": { + "log_type": "system", + "negotiation": { + "err_msg": "Peer proposed traffic-selectors are not in configured range", + "message": "failed with error: Peer proposed traffic-selectors are not in configured range. IKE Version: 2, VPN: IPSEC-INT-ORMB-TUN2-VPN Gateway: IKE-INT-ORMB-TUN2-GW, Local: 10.32.64.128/9001, Remote: 89.160.20.112/9001, Local IKE-ID: 89.160.20.112, Remote IKE-ID: 89.160.20.112, VR-ID: 6", + "type": "IPSec" + }, + "process": "kmd", + "system": { + "ike_version": 2, + "local": "10.32.64.128/9001", + "local_ike_id": "89.160.20.112", + "remote": "89.160.20.112/9001", + "remote_ike_id": "89.160.20.112", + "vpn": "IPSEC-INT-ORMB-TUN2-VPN Gateway: IKE-INT-ORMB-TUN2-GW", + "vr_id": "6" + } + } + }, + "log": { + "level": "error" + }, + "message": "IKE Version: 2, VPN: IPSEC-INT-ORMB-TUN2-VPN Gateway: IKE-INT-ORMB-TUN2-GW, Local: 10.32.64.128/9001, Remote: 89.160.20.112/9001, Local IKE-ID: 89.160.20.112, Remote IKE-ID: 89.160.20.112, VR-ID: 6", + "observer": { + "name": "AC004-PR-VPN01-DMZ", + "product": "SRX", + "type": "firewall", + "vendor": "Juniper" + }, + "process": { + "name": "kmd", + "pid": 9812 + }, + "tags": [ + "preserve_original_event" + ] + }, { "@timestamp": "2023-05-04T05:21:01.102Z", "client": { @@ -193,13 +420,13 @@ "juniper": { "srx": { "log_type": "system", - "process": "kernel", - "tag": "FW" + "process": "kernel" } }, "log": { "level": "informational" }, + "message": "FW: gr-0/0/0.14 A udp 127.0.0.1 89.160.20.112 49153 49153 ", "network": { "transport": "udp" }, @@ -263,13 +490,13 @@ "juniper": { "srx": { "log_type": "system", - "process": "PFE_FW_SYSLOG_IP", - "tag": "FW" + "tag": "PFE_FW_SYSLOG_IP" } }, "log": { "level": "informational" }, + "message": "FW: reth5.175 A pim 67.43.156.1 89.160.20.113 0 0 (1 packets) ", "network": { "transport": "pim" }, @@ -284,9 +511,6 @@ "type": "firewall", "vendor": "Juniper" }, - "process": { - "name": "PFE_FW_SYSLOG_IP" - }, "related": { "ip": [ "67.43.156.1", @@ -334,13 +558,13 @@ "juniper": { "srx": { "log_type": "system", - "process": "PFE_FW_SYSLOG_IP", - "tag": "FW" + "tag": "PFE_FW_SYSLOG_IP" } }, "log": { "level": "informational" }, + "message": "FW: reth2.605 A udp 67.43.156.2 89.160.20.112 0 0 (1 packets)", "network": { "transport": "udp" }, @@ -355,9 +579,6 @@ "type": "firewall", "vendor": "Juniper" }, - "process": { - "name": "PFE_FW_SYSLOG_IP" - }, "related": { "ip": [ "67.43.156.2", @@ -398,7 +619,7 @@ "log_type": "system", "process": "mib2d", "rtslib_dfwsm": { - "k_usr_d": "6a38769", + "k_usr_d": "6a38769 ", "u_data": "7a33678" }, "tag": "rtslib_dfwsm_get_async_cb" @@ -407,6 +628,7 @@ "log": { "level": "debug" }, + "message": "rtslib_dfwsm_get_async_cb:u_data:7a33678 k_usr_d:6a38769 ", "observer": { "name": "AB1234-A-AB-AB01C-ABC", "product": "SRX", @@ -494,14 +716,23 @@ }, "juniper": { "srx": { - "log_type": "system" + "ip_mon_reth_scan": { + "trigger": "reth_scan" + }, + "log_type": "system", + "tag": "ip_mon_reth_scan" } }, "log": { "level": "informational" }, - "message": "- - - - AB1234-A-AB-AB01C-ABC ip_mon_reth_scan: interface st0.60 trigger reth_scan", + "message": "interface st0.60 trigger reth_scan", "observer": { + "ingress": { + "interface": { + "name": "st0.60" + } + }, "name": "AB1234-A-AB-AB01C-ABC", "product": "SRX", "type": "firewall", @@ -526,13 +757,14 @@ }, "juniper": { "srx": { - "log_type": "system" + "log_type": "system", + "tag": "ha_rto_stats_handler" } }, "log": { "level": "debug" }, - "message": "- - - - AB1234-ABC2-AB-AB01C-ABC ha_rto_stats_handler: Sending RTO counters to RE ", + "message": "Sending RTO counters to RE ", "observer": { "name": "AB1234-ABC2-AB-AB01C-ABC", "product": "SRX", @@ -701,19 +933,28 @@ }, "juniper": { "srx": { - "log_type": "system" + "dpdk": { + "port_number": 8, + "port_state": 2, + "swt_port_state": 1 + }, + "log_type": "system", + "tag": "dpdk_eth_devstart" } }, "log": { "level": "informational" }, - "message": "- - - - AB1234-A-AB-AB01C-ABC dpdk_eth_devstart (pid=0x4c6b17c0): port 8 has already been started, dpdk_port_state=2 dpdk_swt_port_state 1", + "message": "port 8 has already been started, dpdk_port_state=2 dpdk_swt_port_state 1", "observer": { "name": "AB1234-A-AB-AB01C-ABC", "product": "SRX", "type": "firewall", "vendor": "Juniper" }, + "process": { + "pid": 1282086848 + }, "tags": [ "preserve_original_event" ] @@ -733,19 +974,33 @@ }, "juniper": { "srx": { - "log_type": "system" + "dpdk": { + "port_number": 7, + "port_state": 2, + "swt_port_state": 1 + }, + "log_type": "system", + "tag": "dpdk_eth_devstart" } }, "log": { "level": "informational" }, - "message": "- - - - AB1234-A-AB-AB01C-ABC dpdk_eth_devstart (pid=0x4c6a1bc0): port 7 ifd xe-0/0/7, new dpdk_port_state=2 dpdk_swt_port_state 1", + "message": "port 7 ifd xe-0/0/7, new dpdk_port_state=2 dpdk_swt_port_state 1", "observer": { + "ingress": { + "interface": { + "name": "xe-0/0/7" + } + }, "name": "AB1234-A-AB-AB01C-ABC", "product": "SRX", "type": "firewall", "vendor": "Juniper" }, + "process": { + "pid": 1282022336 + }, "tags": [ "preserve_original_event" ] @@ -765,13 +1020,14 @@ }, "juniper": { "srx": { - "log_type": "system" + "log_type": "system", + "tag": "nh_fabric_fill_jnhinfo" } }, "log": { "level": "informational" }, - "message": "- - - - AB1234-A-AB-AB01C-ABC nh_fabric_fill_jnhinfo: Storing nh_id as 0x2dd and jnh as 0x58e302", + "message": "Storing nh_id as 0x2dd and jnh as 0x58e302", "observer": { "name": "AB1234-A-AB-AB01C-ABC", "product": "SRX", @@ -803,7 +1059,40 @@ "log": { "level": "debug" }, - "message": "- - - - AB1234-A-AB-AB01C-ABC Copying remote chassis chassis 1, IP: 81.2.69.192", + "message": "Copying remote chassis chassis 1, IP: 81.2.69.192", + "observer": { + "name": "AB1234-A-AB-AB01C-ABC", + "product": "SRX", + "type": "firewall", + "vendor": "Juniper" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2023-05-08T00:54:24.756Z", + "ecs": { + "version": "8.9.0" + }, + "event": { + "category": [ + "network" + ], + "kind": "event", + "original": "\u003c166\u003e1 2023-05-08T10:54:24.756+10:00 AB1234-A-AB-AB01C-ABC - - - - AB1234-A-AB-AB01C-ABC nh_fabric_fill_jnhinfo: ABCDE: Test default message 123456", + "severity": 166 + }, + "juniper": { + "srx": { + "log_type": "system", + "tag": "nh_fabric_fill_jnhinfo" + } + }, + "log": { + "level": "informational" + }, + "message": "ABCDE: Test default message 123456", "observer": { "name": "AB1234-A-AB-AB01C-ABC", "product": "SRX", diff --git a/packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/default.yml index 4f6ec247decc..0c8c0310e9d9 100644 --- a/packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/default.yml +++ b/packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/default.yml @@ -13,22 +13,35 @@ processors: - grok: field: event.original patterns: - # SRX Traffic log pattern - - '^<%{POSINT:syslog_pri}>(\d{1,3}\s)?(?:%{TIMESTAMP_ISO8601:_temp_.raw_date})\s%{SYSLOGHOST:syslog_hostname}\s%{PROG:syslog_program}\s(?:%{POSINT:syslog_pid}|-)?\s%{WORD:tag}\s\[([^=]+?\s)?%{GREEDYDATA:_temp_.traffic_structured}\]\s?$' - # SRX System log patterns (further parsing done in system.yml) - - '^<%{POSINT:syslog_pri}>(\d{1,3}\s)?(?:%{CUSTOM_DATE:_temp_.raw_date})\s%{SYSLOGHOST:syslog_hostname}\s%{GREEDYDATA:_temp_.unparsed.message}$' + # 1. SRX Traffic structured log pattern + - '^%{SYSLOG_PREFIX}?%{TIMESTAMP_ISO8601:_temp_.raw_date}\s%{SYSLOGHOST:syslog_hostname}\s%{JUNIPER_TRAFFIC_PROCESS:syslog_program}\s(?:%{POSINT:syslog_pid}|-)?\s%{WORD:tag}\s\[([^=]+?\s)?%{GREEDYDATA:_temp_.traffic_structured}\]\s?$' + # 2. SRX System structured log pattern (captures all structured logs when syslog_program not in JUNIPER_TRAFFIC_PROCESS) + - '^%{SYSLOG_PREFIX}?%{CUSTOM_DATE:_temp_.raw_date}\s%{SYSLOGHOST:syslog_hostname}\s%{PROG:syslog_program}\s(?:%{POSINT:syslog_pid}|-)?\s%{WORD:tag}\s\[([^=]+?\s)?%{GREEDYDATA:_temp_.system_structured}\](?!=)\s?%{DATA:_temp_.unparsed.message}\s?$' + # 3. SRX System structured-brief and unstructured log patterns (further parsing done in system.yml) + - '^%{SYSLOG_PREFIX}?%{CUSTOM_DATE:_temp_.raw_date}\s%{SYSLOGHOST:syslog_hostname}\s%{GREEDYDATA:_temp_.unparsed.message}$' pattern_definitions: + SYSLOG_PREFIX: '<%{POSINT:syslog_pri}>(?:\d{1,3}\s)' CUSTOM_DATE: "%{TIMESTAMP_ISO8601}|(%{MONTH}%{SPACE}+%{MONTHDAY}%{SPACE}+%{TIME})" + JUNIPER_TRAFFIC_PROCESS: "RT_FLOW|RT_UTM|RT_IDP|RT_IDS|RT_AAMW|RT_SECINTEL" # split Juniper-SRX fields - kv: field: _temp_.traffic_structured - field_split: " (?=[a-z0-9\\_\\-]+=)" + field_split: ' (?=[a-z0-9\_\-]+=)' value_split: "=" prefix: "juniper.srx." ignore_missing: true ignore_failure: false - trim_value: "\"" + trim_value: '"' +# split Juniper-SRX fields + - kv: + field: _temp_.system_structured + field_split: ' (?=[a-z0-9\_\-]+=)' + value_split: "=" + prefix: "juniper.srx." + ignore_missing: true + ignore_failure: false + trim_value: '"' - rename: field: syslog_program diff --git a/packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/system.yml b/packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/system.yml index 1284a9bf6db1..7b5ae8e6f090 100644 --- a/packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/system.yml +++ b/packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/system.yml @@ -8,84 +8,74 @@ processors: if: ctx._temp_?.unparsed?.message != null && ctx._temp_?.unparsed?.message != "" tag: grok_unparsed_message patterns: -# # SRX System log patterns - - '^(?:%{PROG:syslog_program}|-)?\s(?:%{POSINT:syslog_pid}|-)?\s(?:%{WORD:tag}|-)?\s?((\[([^=]+?\s)?%{DATA:_temp_.system_structured}\"\])|-)?\s?(%{CUSTOM_PROG} )?%{CUSTOM_SYSTEM_TAG:_temp_.sub_tag}:%{DATA:_temp_.unparsed.syslog_structured}\s?$' - - '^(?:%{PROG:syslog_program})?\s(?:%{POSINT:syslog_pid})?\s(?:%{WORD:tag})?\s?(\[([^=]+?\s)?%{DATA:_temp_.system_structured}\"\])?\s?%{DATA:message}\s?$' - - '^(%{CUSTOM_PROG} )?%{CUSTOM_SYSTEM_TAG:_temp_.sub_tag}:%{DATA:_temp_.unparsed.syslog_structured}\s?$' + # 1. SRX System Structured Brief + - '^(?:%{PROG:syslog_program}|-)?\s(?:%{POSINT:syslog_pid}|-)?\s(?:%{WORD:tag}|-)?\s([-]+\s)?%{GREEDYDATA:_temp_.unparsed.system_structured_brief}\s?$' + # 2. SRX System Unstructured - '^%{GREEDYDATA:message}$' - pattern_definitions: - CUSTOM_DATE: "%{TIMESTAMP_ISO8601}|(%{MONTH}%{SPACE}+%{MONTHDAY}%{SPACE}+%{TIME})" - CUSTOM_PROG: "(%{DATA:_temp_.to_be_parsed})?%{PROG:_temp_.syslog_sub_program}(\\[%{POSINT:syslog_pid}\\]|\\s*\\(pid=%{DATA:syslog_pid}\\))?:" - CUSTOM_SYSTEM_TAG: "[A-Za-z_]+" - -# split k-v fields inside structured data -- kv: - field: _temp_.system_structured - if: 'ctx._temp_?.system_structured != null && ctx._temp_?.system_structured != ""' - field_split: " (?=[a-z0-9\\_\\-]+=)" - value_split: "=" - prefix: "juniper.srx." - ignore_missing: true - ignore_failure: false - trim_value: "\"" - - -# Converts all kebab-case key names to snake_case -- script: - lang: painless - if: ctx.juniper?.srx != null - source: >- - ctx.juniper.srx = ctx?.juniper?.srx.entrySet().stream().collect(Collectors.toMap(e -> e.getKey().replace('-', '_'), e -> e.getValue())); +# Parse different types of system_structured_brief messages +- grok: + if: "ctx._temp_?.unparsed?.system_structured_brief != null && ctx._temp_.unparsed.system_structured_brief != ''" + tag: "grok_system_structured_brief" + field: "_temp_.unparsed.system_structured_brief" + patterns: + # 1. Has word 'negotiation' + - '^%{WORD:_temp_.negotiation.type} negotiation %{GREEDYDATA:_temp_.negotiation.message}$' + # 2. Optionally contain hostname, tag, pid, and rest of message is parsed into _temp_.message_brief + - '^(%{SYSLOGHOST:syslog_hostname}\s)?(%{CUSTOM_TAG_BRIEF:_temp_.tag_brief}(\s\(pid=%{DATA:syslog_pid}\))?(:\s))?%{GREEDYDATA:_temp_.message_brief}$' + # 3. Match all + - '^%{GREEDYDATA:message}$' + pattern_definitions: + CUSTOM_TAG_BRIEF: "(?!FW)[A-Za-z_]+" + on_failure: + - append: + field: error.message + value: "Processor {{ _ingest.on_failure_processor_type }} with tag {{ _ingest.on_failure_processor_tag }} in pipeline {{ _ingest.on_failure_pipeline }} failed with message: {{ _ingest.on_failure_message }}" -# Removes all empty fields -- script: - lang: painless - if: ctx.juniper?.srx != null - params: - values: - - "None" - - "UNKNOWN" - - "N/A" - - "-" - source: >- - ctx?.juniper?.srx.entrySet().removeIf(entry -> params.values.contains(entry.getValue())); - -# Clean up syslog_program & tag +# Cleanup process and tag fields. +# If juniper.srx.process and juniper.srx.tag fields are not already set, or has `-`, then set them using newly parsed grok pattern in system_structured_brief - set: field: juniper.srx.process value: "{{{syslog_program}}}" if: 'ctx.syslog_program != null && (ctx.juniper?.srx?.process == null || ctx.juniper?.srx?.process == "-")' -- set: - field: juniper.srx.process - value: "{{{_temp_.syslog_sub_program}}}" - if: 'ctx._temp_?.syslog_sub_program != null && (ctx.juniper?.srx?.process == null || ctx.juniper?.srx?.process == "-")' -- set: - field: juniper.srx.tag - value: "{{{tag}}}" - if: 'ctx.tag != null && (ctx.juniper?.srx?.tag == null || ctx.juniper?.srx?.tag == "-")' - set: field: juniper.srx.tag - value: "{{{_temp_.sub_tag}}}" - if: 'ctx._temp_?.sub_tag != null && (ctx.juniper?.srx?.tag == null || ctx.juniper?.srx?.tag == "-")' + value: "{{{_temp_.tag_brief}}}" + if: 'ctx._temp_?.tag_brief != null && (ctx.juniper?.srx?.tag == null || ctx.juniper.srx.tag == "-")' -#### Firewall (tag = FW) -# if: tag = FW && process = PFE_FW_SYSLOG_IP +# Handle negotiation messages +# 1. if: negotiation.message ~ /^failed.*/ - dissect: - if: "ctx._temp_?.unparsed?.syslog_structured != null && ctx.juniper?.srx?.tag == 'FW' && ctx.juniper?.srx?.process == 'PFE_FW_SYSLOG_IP'" - tag: "dissect_sub_tag_fw_packets" - field: "_temp_.unparsed.syslog_structured" - pattern: " %{_temp_.fw.interface_name} %{_temp_.fw.filter_action} %{_temp_.fw.packet_protocol} %{_temp_.fw.src_addr} %{_temp_.fw.dst_addr} %{_temp_.fw.src_port} %{_temp_.fw.dst_port} (%{_temp_.fw.packets_num} packets)" + if: "ctx._temp_?.negotiation?.message != null && ctx._temp_.negotiation.message.startsWith('failed')" + tag: "dissect_neg_failed" + field: "_temp_.negotiation.message" + pattern: "failed with error: %{_temp_.negotiation.err_msg}. %{message}" on_failure: - append: field: error.message value: "Processor {{ _ingest.on_failure_processor_type }} with tag {{ _ingest.on_failure_processor_tag }} in pipeline {{ _ingest.on_failure_pipeline }} failed with message: {{ _ingest.on_failure_message }}" -# if: tag = FW && process != PFE_FW_SYSLOG_IP +# 2. if: negotiation.message ~ /^success.*/ - dissect: - if: "ctx._temp_?.unparsed?.syslog_structured != null && ctx.juniper?.srx?.tag == 'FW' && ctx.juniper?.srx?.process != 'PFE_FW_SYSLOG_IP'" - tag: "dissect_sub_tag_fw" - field: "_temp_.unparsed.syslog_structured" - pattern: " %{_temp_.fw.interface_name} %{_temp_.fw.filter_action} %{_temp_.fw.packet_protocol} %{_temp_.fw.src_addr} %{_temp_.fw.dst_addr} %{_temp_.fw.src_port} %{_temp_.fw.dst_port}" + if: "ctx._temp_?.negotiation?.message != null && ctx._temp_.negotiation.message.startsWith('success')" + tag: "dissect_neg_success" + field: "_temp_.negotiation.message" + pattern: "successfully completed. %{message}" + on_failure: + - append: + field: error.message + value: "Processor {{ _ingest.on_failure_processor_type }} with tag {{ _ingest.on_failure_processor_tag }} in pipeline {{ _ingest.on_failure_pipeline }} failed with message: {{ _ingest.on_failure_message }}" +- rename: + field: _temp_.negotiation + target_field: juniper.srx.negotiation + if: ctx._temp_?.negotiation != null + +# Handle Firewall messages (message_brief ~ /^FW:/) +- grok: + if: "ctx._temp_?.message_brief != null && ctx._temp_.message_brief.startsWith('FW:')" + tag: "grok_message_brief" + field: "_temp_.message_brief" + patterns: + - '^FW:\s%{NOTSPACE:_temp_.fw.interface_name}\s%{NOTSPACE:_temp_.fw.filter_action}\s%{NOTSPACE:_temp_.fw.packet_protocol}\s%{NOTSPACE:_temp_.fw.src_addr}\s%{NOTSPACE:_temp_.fw.dst_addr}\s%{NOTSPACE:_temp_.fw.src_port}\s%{NOTSPACE:_temp_.fw.dst_port}\s(\(%{NOTSPACE:_temp_.fw.packets_num} packets\))?\s?$' on_failure: - append: field: error.message @@ -93,17 +83,102 @@ processors: - rename: field: _temp_.fw target_field: juniper.srx.firewall - if: "ctx._temp_?.fw != null && ctx.juniper?.srx?.tag == 'FW' " + if: ctx._temp_?.fw != null - rename: field: juniper.srx.firewall.interface_name target_field: juniper.srx.interface_name if: "ctx.juniper?.srx?.firewall?.interface_name != null" +# Handle rtslib_dfwsm_get_async_cb messages (message_brief ~ /^rtslib_dfwsm_get_async_cb:/) +- dissect: + if: "ctx._temp_?.message_brief != null && ctx._temp_.message_brief.startsWith('rtslib_dfwsm_get_async_cb:')" + tag: "dissect_rtslib_dfwsmr" + field: "_temp_.message_brief" + pattern: "rtslib_dfwsm_get_async_cb:u_data:%{_temp_.rtslib_dfwsm.u_data} k_usr_d:%{_temp_.rtslib_dfwsm.k_usr_d}" + on_failure: + - append: + field: error.message + value: "Processor {{ _ingest.on_failure_processor_type }} with tag {{ _ingest.on_failure_processor_tag }} in pipeline {{ _ingest.on_failure_pipeline }} failed with message: {{ _ingest.on_failure_message }}" +- rename: + field: _temp_.rtslib_dfwsm + target_field: juniper.srx.rtslib_dfwsm + if: ctx._temp_?.rtslib_dfwsm != null + +# Handle ip_mon_reth_scan messages (tag_brief == ip_mon_reth_scan) +- dissect: + if: "ctx._temp_?.tag_brief != null && ctx._temp_.tag_brief == 'ip_mon_reth_scan'" + tag: "dissect_ip_mon_reth_scan" + field: "_temp_.message_brief" + pattern: "interface %{_temp_.ip_mon_reth_scan.interface_name} trigger %{_temp_.ip_mon_reth_scan.trigger}" + on_failure: + - append: + field: error.message + value: "Processor {{ _ingest.on_failure_processor_type }} with tag {{ _ingest.on_failure_processor_tag }} in pipeline {{ _ingest.on_failure_pipeline }} failed with message: {{ _ingest.on_failure_message }}" +- rename: + field: _temp_.ip_mon_reth_scan + target_field: juniper.srx.ip_mon_reth_scan + if: ctx._temp_?.ip_mon_reth_scan != null +- rename: + field: juniper.srx.ip_mon_reth_scan.interface_name + target_field: juniper.srx.interface_name + if: "ctx.juniper?.srx?.ip_mon_reth_scan?.interface_name != null" + +# Handle dpdk_eth_devstart messages (tag_brief == dpdk_eth_devstart) +- grok: + if: "ctx._temp_?.tag_brief != null && ctx._temp_.tag_brief == 'dpdk_eth_devstart'" + tag: "grok_dpdk_eth_devstart" + field: "_temp_.message_brief" + patterns: + - '^port %{POSINT:_temp_.dpdk.port_number} (has already been started|ifd %{DATA:_temp_.dpdk.interface_name}), (new\s)?dpdk_port_state=%{POSINT:_temp_.dpdk.port_state} dpdk_swt_port_state %{POSINT:_temp_.dpdk.swt_port_state}$' + on_failure: + - append: + field: error.message + value: "Processor {{ _ingest.on_failure_processor_type }} with tag {{ _ingest.on_failure_processor_tag }} in pipeline {{ _ingest.on_failure_pipeline }} failed with message: {{ _ingest.on_failure_message }}" +- convert: + field: _temp_.dpdk.port_number + type: integer + tag: convert_dpdk_port_number_to_int + ignore_missing: true + if: "ctx._temp_?.dpdk?.port_number != null" + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' +- convert: + field: _temp_.dpdk.port_state + type: integer + tag: convert_dpdk_port_state_to_int + ignore_missing: true + if: "ctx._temp_?.dpdk?.port_state != null" + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' +- convert: + field: _temp_.dpdk.swt_port_state + type: integer + tag: convert_swt_port_state_to_int + ignore_missing: true + if: "ctx._temp_?.dpdk?.swt_port_state != null" + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' +- rename: + field: _temp_.dpdk + target_field: juniper.srx.dpdk + if: ctx._temp_?.dpdk != null +- rename: + field: juniper.srx.dpdk.interface_name + target_field: juniper.srx.interface_name + if: "ctx.juniper?.srx?.dpdk?.interface_name != null" + + ### RTLOG_CONN_ERROR (tag = RTLOG_CONN_ERROR) - dissect: - if: "ctx._temp_?.unparsed?.syslog_structured != null && ctx.juniper?.srx?.tag == 'RTLOG_CONN_ERROR'" - tag: "dissect_sub_tag_rtlog_conn_err" - field: "_temp_.unparsed.syslog_structured" + if: "ctx._temp_?.unparsed?.system_structured_brief != null && ctx.juniper?.srx?.tag == 'RTLOG_CONN_ERROR'" + tag: "dissect_tag_rtlog_conn_err" + field: "_temp_.unparsed.system_structured_brief" pattern: " Connection error %{_temp_.rtlog_conn_error.stream_name} %{_temp_.rtlog_conn_error.err_msg}" on_failure: - append: @@ -163,9 +238,9 @@ processors: # PING_TEST_COMPLETED # if: tag = PING_TEST_COMPLETED - dissect: - if: "ctx._temp_?.unparsed?.syslog_structured != null && ctx.juniper?.srx?.tag == 'PING_TEST_COMPLETED'" - tag: "dissect_sub_tag_ping_test" - field: "_temp_.unparsed.syslog_structured" + if: "ctx._temp_?.unparsed?.system_structured_brief != null && ctx.juniper?.srx?.tag == 'PING_TEST_COMPLETED'" + tag: "dissect_tag_ping_test" + field: "_temp_.unparsed.system_structured_brief" pattern: " pingCtlOwnerIndex = %{_temp_.ping_test.owner}, pingCtlTestName = %{_temp_.ping_test.name}" on_failure: - append: @@ -176,28 +251,12 @@ processors: target_field: juniper.srx.ping_test if: "ctx._temp_?.ping_test != null && ctx.juniper?.srx?.tag == 'PING_TEST_COMPLETED' " -# IKE negotiation -# if: tag = IKE negotiation -- dissect: - if: "ctx._temp_?.unparsed?.syslog_structured != null && ctx.juniper?.srx?.tag == 'IKE negotiation'" - tag: "dissect_sub_tag_ike_neg" - field: "_temp_.unparsed.syslog_structured" - pattern: " failed with error: %{_temp_.ike_negotiation.err_msg}\\. IKE Version: %{_temp_.ike_negotiation.version}, VPN: %{_temp_.ike_negotiation.vpn} Gateway: %{_temp_.ike_negotiation.gateway}, Local: %{_temp_.ike_negotiation.local.ip_range}, Remote: %{_temp_.ike_negotiation.remote.ip_range}, Local IKE-ID: %{_temp_.ike_negotiation.local.ike_id}, Remote IKE-ID: %{_temp_.ike_negotiation.remote.ike_id}, VR-ID: %{_temp_.ike_negotiation.vr_id}, Role: %{_temp_.ike_negotiation.role}" - on_failure: - - append: - field: error.message - value: "Processor {{ _ingest.on_failure_processor_type }} with tag {{ _ingest.on_failure_processor_tag }} in pipeline {{ _ingest.on_failure_pipeline }} failed with message: {{ _ingest.on_failure_message }}" -- rename: - field: _temp_.ike_negotiation - target_field: juniper.srx.ike_negotiation - if: "ctx._temp_?.ike_negotiation != null && ctx.juniper?.srx?.tag == 'IKE negotiation' " - # KERN_ARP_ADDR_CHANGE # if: tag = KERN_ARP_ADDR_CHANGE - dissect: - if: "ctx._temp_?.unparsed?.syslog_structured != null && ctx.juniper?.srx?.tag == 'KERN_ARP_ADDR_CHANGE'" - tag: "dissect_sub_tag_kern_arp_addr" - field: "_temp_.unparsed.syslog_structured" + if: "ctx._temp_?.unparsed?.system_structured_brief != null && ctx.juniper?.srx?.tag == 'KERN_ARP_ADDR_CHANGE'" + tag: "dissect_tag_kern_arp_addr" + field: "_temp_.unparsed.system_structured_brief" pattern: " arp info overwritten for %{_temp_.kern_arp_addr_change.ip} from %{_temp_.kern_arp_addr_change.mac1} to %{_temp_.kern_arp_addr_change.mac2}" on_failure: - append: @@ -219,26 +278,91 @@ processors: value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' -# rtslib_dfwsm_get_async_cb -# if: tag = rtslib_dfwsm_get_async_cb -- dissect: - if: "ctx._temp_?.unparsed?.syslog_structured != null && ctx.juniper?.srx?.tag == 'rtslib_dfwsm_get_async_cb'" - tag: "dissect_sub_tag_rtslib_dfwsmr" - field: "_temp_.unparsed.syslog_structured" - pattern: "u_data:%{_temp_.rtslib_dfwsm.u_data} k_usr_d:%{_temp_.rtslib_dfwsm.k_usr_d}" +# After System Structured, Structured-Brief, and Unstructured messages: there are optional key-value pairs seperated by ", ". These should now be in `message` field +# split k-v fields inside message field. +- kv: + field: message + if: 'ctx.message != null && ctx.message != ""' + tag: "kv_message" + field_split: ',\s(?=[a-zA-Z0-9\_\-\s]+:)' + value_split: ":" + prefix: "juniper.srx.system." + ignore_missing: true + ignore_failure: true + trim_value: '"' + +# Cleanup parsed k-v root field `juniper.srx.system`. Removes spaces, lowercases, and converts fields into snake_case +- script: + lang: painless + tag: "script_cleanup_system" + if: ctx.juniper?.srx?.system != null + source: >- + ctx.juniper.srx.system = ctx.juniper.srx.system.entrySet().stream().collect(Collectors.toMap(e -> e.getKey().replace(' ', '_').replace('-', '_').toLowerCase(), e -> e.getValue().trim())); + +####################### +## SRX System Fields ## +####################### +- convert: + field: juniper.srx.system.aux_spi + type: integer + tag: convert_aux_spi_to_int + ignore_missing: true + if: "ctx.juniper?.srx?.system?.aux_spi != null" on_failure: - - append: - field: error.message - value: "Processor {{ _ingest.on_failure_processor_type }} with tag {{ _ingest.on_failure_processor_tag }} in pipeline {{ _ingest.on_failure_pipeline }} failed with message: {{ _ingest.on_failure_message }}" -- rename: - field: _temp_.rtslib_dfwsm - target_field: juniper.srx.rtslib_dfwsm - if: "ctx._temp_?.rtslib_dfwsm != null && ctx.juniper?.srx?.tag == 'rtslib_dfwsm_get_async_cb' " + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' +- convert: + field: juniper.srx.system.ike_version + type: integer + tag: convert_ike_version_to_int + ignore_missing: true + if: "ctx.juniper?.srx?.system?.ike_version != null" + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' +- convert: + field: juniper.srx.system.local_gateway + type: ip + tag: convert_local_gateway_to_ip + ignore_missing: true + if: "ctx.juniper?.srx?.system?.local_gateway != null" + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' +- convert: + field: juniper.srx.system.remote_gateway + type: ip + tag: convert_remote_gateway_to_ip + ignore_missing: true + if: "ctx.juniper?.srx?.system?.remote_gateway != null" + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' +# Removes all empty fields +- script: + lang: painless + if: ctx.juniper?.srx?.system != null + params: + values: + - "None" + - "UNKNOWN" + - "N/A" + - "-" + - "Not-Available" + source: >- + ctx?.juniper?.srx?.system.entrySet().removeIf(entry -> params.values.contains(entry.getValue())); + +# Copy `message_brief` into `message` if not already exists. This ensures unpased/unstructured data to be present in `message`. +- set: + field: message + copy_from: _temp_.message_brief + if: "ctx._temp_?.message_brief != null && ctx.message == null" -######################### -## SRX System fields ## -######################### ####################### ## ECS Event Mapping ## @@ -604,6 +728,20 @@ processors: ignore_missing: true if: "ctx.juniper?.srx?.username != null" +######################### +## ECS Related Mapping ## +######################### +- append: + if: 'ctx.juniper?.srx?.system?.local_gateway != null' + field: related.ip + value: '{{juniper.srx.system.local_gateway}}' + allow_duplicates: false +- append: + if: 'ctx.juniper?.srx?.system?.remote_gateway != null' + field: related.ip + value: '{{juniper.srx.system.remote_gateway}}' + allow_duplicates: false + ###################### ## ECS Observer Mapping ## ###################### @@ -611,7 +749,11 @@ processors: field: juniper.srx.interface_name target_field: observer.ingress.interface.name ignore_missing: true - +- rename: + field: syslog_hostname + target_field: observer.name + ignore_missing: true + if: "ctx.syslog_hostname != null && ctx.observer?.name == null" ###################### ## ECS Rule Mapping ## ###################### @@ -649,6 +791,10 @@ processors: ignore_missing: true if: "ctx.juniper?.srx?.message != null" +- remove: + field: juniper.srx.process + if: "ctx.juniper?.srx?.process != null && ['-','N/A','UNKNOWN','None'].contains(ctx.juniper.srx.process)" + ############# ## Cleanup ## ############# @@ -657,6 +803,7 @@ processors: # message field can be removed if all dissect patterns are added. # - message - syslog_program + - syslog_hostname - tag - juniper.srx.destination_port - juniper.srx.nat_destination_port diff --git a/packages/juniper_srx/data_stream/log/fields/fields.yml b/packages/juniper_srx/data_stream/log/fields/fields.yml index 62292ca437f4..3f5e6c1f2ef2 100644 --- a/packages/juniper_srx/data_stream/log/fields/fields.yml +++ b/packages/juniper_srx/data_stream/log/fields/fields.yml @@ -416,36 +416,75 @@ type: keyword - name: first_forwarding_class type: keyword - - name: rtslib_dfwsm + - name: system type: group fields: - - name: k_usr_d + - name: aux_spi + type: integer + - name: direction type: keyword - - name: u_data + - name: ike_version + type: integer + - name: local type: keyword - - name: ike_negotiation - type: group - fields: - - name: err_msg + - name: local_gateway + type: ip + - name: local_id type: keyword - - name: gateway + - name: local_ike_id type: keyword - - name: local.ike_id + - name: mode type: keyword - - name: local.ip_range + - name: remote type: keyword - - name: remote.ike_id + - name: remote_gateway type: keyword - - name: remote.ip_range + - name: remote_id + type: keyword + - name: remote_ike_id type: keyword - name: role type: keyword - - name: version + - name: spi + type: keyword + - name: traffic_selector + type: keyword + - name: type type: keyword - name: vpn type: keyword - name: vr_id type: keyword + - name: rtslib_dfwsm + type: group + fields: + - name: k_usr_d + type: keyword + - name: u_data + type: keyword + - name: negotiation + type: group + fields: + - name: err_msg + type: keyword + - name: message + type: keyword + - name: type + type: keyword + - name: ip_mon_reth_scan + type: group + fields: + - name: trigger + type: keyword + - name: dpdk + type: group + fields: + - name: port_number + type: integer + - name: port_state + type: integer + - name: swt_port_state + type: integer - name: kern_arp_addr_change type: group fields: diff --git a/packages/juniper_srx/docs/README.md b/packages/juniper_srx/docs/README.md index 80e71d3adde7..bda140a2608f 100644 --- a/packages/juniper_srx/docs/README.md +++ b/packages/juniper_srx/docs/README.md @@ -357,6 +357,9 @@ The following processes and tags are supported: | juniper.srx.context_value | context value | keyword | | juniper.srx.context_value_hit_rate | context value hit rate | integer | | juniper.srx.ddos_application_name | ddos application name | keyword | +| juniper.srx.dpdk.port_number | | integer | +| juniper.srx.dpdk.port_state | | integer | +| juniper.srx.dpdk.swt_port_state | | integer | | juniper.srx.dscp_value | apbr rule type | integer | | juniper.srx.dst_nat_rule_name | dst nat rule name | keyword | | juniper.srx.dst_nat_rule_type | dst nat rule type | keyword | @@ -376,21 +379,12 @@ The following processes and tags are supported: | juniper.srx.function_name | | keyword | | juniper.srx.hostname | hostname | keyword | | juniper.srx.icmp_type | icmp type | integer | -| juniper.srx.ike_negotiation.err_msg | | keyword | -| juniper.srx.ike_negotiation.gateway | | keyword | -| juniper.srx.ike_negotiation.local.ike_id | | keyword | -| juniper.srx.ike_negotiation.local.ip_range | | keyword | -| juniper.srx.ike_negotiation.remote.ike_id | | keyword | -| juniper.srx.ike_negotiation.remote.ip_range | | keyword | -| juniper.srx.ike_negotiation.role | | keyword | -| juniper.srx.ike_negotiation.version | | keyword | -| juniper.srx.ike_negotiation.vpn | | keyword | -| juniper.srx.ike_negotiation.vr_id | | keyword | | juniper.srx.inbound_bytes | bytes from server | integer | | juniper.srx.inbound_packets | packets from server | integer | | juniper.srx.index | index | keyword | | juniper.srx.index1 | | keyword | | juniper.srx.index2 | | keyword | +| juniper.srx.ip_mon_reth_scan.trigger | | keyword | | juniper.srx.kern_arp_addr_change.ip | | ip | | juniper.srx.kern_arp_addr_change.mac1 | | keyword | | juniper.srx.kern_arp_addr_change.mac2 | | keyword | @@ -403,6 +397,9 @@ The following processes and tags are supported: | juniper.srx.mode | | keyword | | juniper.srx.name | name | keyword | | juniper.srx.nat_connection_tag | nat connection tag | keyword | +| juniper.srx.negotiation.err_msg | | keyword | +| juniper.srx.negotiation.message | | keyword | +| juniper.srx.negotiation.type | | keyword | | juniper.srx.nested_application | nested application | keyword | | juniper.srx.obj | url path | keyword | | juniper.srx.occur_count | occur count | integer | @@ -453,6 +450,24 @@ The following processes and tags are supported: | juniper.srx.state | state | keyword | | juniper.srx.status | status | keyword | | juniper.srx.sub_category | sub category | keyword | +| juniper.srx.system.aux_spi | | integer | +| juniper.srx.system.direction | | keyword | +| juniper.srx.system.ike_version | | integer | +| juniper.srx.system.local | | keyword | +| juniper.srx.system.local_gateway | | ip | +| juniper.srx.system.local_id | | keyword | +| juniper.srx.system.local_ike_id | | keyword | +| juniper.srx.system.mode | | keyword | +| juniper.srx.system.remote | | keyword | +| juniper.srx.system.remote_gateway | | keyword | +| juniper.srx.system.remote_id | | keyword | +| juniper.srx.system.remote_ike_id | | keyword | +| juniper.srx.system.role | | keyword | +| juniper.srx.system.spi | | keyword | +| juniper.srx.system.traffic_selector | | keyword | +| juniper.srx.system.type | | keyword | +| juniper.srx.system.vpn | | keyword | +| juniper.srx.system.vr_id | | keyword | | juniper.srx.tag | system log message tag, which uniquely identifies the message. | keyword | | juniper.srx.temporary_filename | temporary_filename | keyword | | juniper.srx.tenant_id | tenant id | keyword | diff --git a/packages/juniper_srx/manifest.yml b/packages/juniper_srx/manifest.yml index 684d1125275d..3e26700f9307 100644 --- a/packages/juniper_srx/manifest.yml +++ b/packages/juniper_srx/manifest.yml @@ -1,7 +1,7 @@ format_version: 2.7.0 name: juniper_srx title: Juniper SRX -version: "1.14.0" +version: "1.14.1" description: Collect logs from Juniper SRX devices with Elastic Agent. categories: ["network", "security", "firewall_security"] type: integration