[Juniper SRX] Fix grok patterns for system logs (#7280)

* Fix grok patterns for Juniper System logs * update pr num * update negotiation grok * Fix FW groks * Fix rtslib_dfwsm_get_async_cb * Add reth_scan * Non pid structured * Fix other patterns * Refactor * Add required fields * update readme * PR comments * Address PR comments
elastic · Aug 14, 2023 · 52269e0 · 52269e0
1 parent 01c94bf
commit 52269e0
Show file tree

Hide file tree

Showing 8 changed files with 674 additions and 161 deletions.
diff --git a/packages/juniper_srx/changelog.yml b/packages/juniper_srx/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.14.1"
+  changes:
+    - description: Fix system logs grok
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/7280
 - version: "1.14.0"
   changes:
     - description: Update package to ECS 8.9.0.

diff --git a/packages/juniper_srx/data_stream/log/_dev/test/pipeline/test-system.log b/packages/juniper_srx/data_stream/log/_dev/test/pipeline/test-system.log
@@ -1,6 +1,10 @@
 <30>1 2023-05-04T15:27:30.846+10:00 AB1234-ABC2-AB-AB01C-ABC kmd 8961 KMD_PM_SA_ESTABLISHED [junos@1111.1.1.1.1.111 local-address="89.160.20.112" remote-address="67.43.156.0" local-initiator="ipv4(89.160.20.112-89.160.20.114)" remote-responder="ipv4(67.43.156.0)" argument1="outbound" index1="36090046" index2="0" mode="Tunnel" type="dynamic" traffic-selector-name="ASJLKN_JKHA" first-forwarding-class=""] 
 <30>1 2023-05-04T15:27:26.461+10:00 AB1234-A-AB-AB01C-ABC kmd 13862 KMD_PM_SA_ESTABLISHED [junos@1111.1.1.1.1.111 local-address="89.160.20.112" remote-address="67.43.156.0" local-initiator="ipv4_subnet(any:0,[0..7\]=89.160.20.112/29)" remote-responder="ipv4_subnet(any:0,[0..7\]=67.43.156.0/24)" argument1="outbound" index1="3700499780" index2="0" mode="Tunnel" type="dynamic" traffic-selector-name="" first-forwarding-class=""] Local gateway: 89.160.20.115, Remote gateway: 67.43.156.1, Local ID: ipv4_subnet(any:0,[0..7]=89.160.20.114/29), Remote ID: ipv4_subnet(any:0,[0..7]=67.43.156.1/24), Direction: outbound, SPI: 0xdc912544, AUX-SPI: 0, Mode: Tunnel, Type: dynamic, Traffic-selector: FC Name:
 <27>1 2023-05-04T15:19:33.984+10:00 AB1234-A-AB-AB01C-ABC kmd 9159 - - IKE negotiation failed with error: Timed out. IKE Version: 1, VPN: IPSEC-AAAAA-AAA1-PROD-VPN Gateway: IKE-AAAAA-AAA1-GW, Local: 89.160.20.112/500, Remote: 67.43.156.1/500, Local IKE-ID: Not-Available, Remote IKE-ID: Not-Available, VR-ID: 5: Role: Initiator
+<27>1 2023-05-04T15:19:33.984+10:00 AB1234-A-AB-AB01C-ABC kmd 9159 asd2 - IKE negotiation failed with error: Timed out. IKE Version: 1, VPN: IPSEC-AAAAA-AAA1-PROD-VPN Gateway: IKE-AAAAA-AAA1-GW, Local: 89.160.20.112/500, Remote: 67.43.156.1/500, Local IKE-ID: Not-Available, Remote IKE-ID: Not-Available, VR-ID: 5: Role: Initiator
+<27>1 2023-07-04T12:22:36.461+10:00 AC004-PR-VPN01-DMZ kmd 9812 - - IKE negotiation failed with error: Timed out. IKE Version: 1, VPN: IPSEC-HORSEFACTS-TUN1-PROD-VPN Gateway: IKE-HORSEFACTS-TUN1-GW, Local: 10.11.22.444/500, Remote: 198.1.124.8/500, Local IKE-ID: Not-Available, Remote IKE-ID: Not-Available, VR-ID: 5, Role: Initiator
+<30>1 2023-07-04T10:21:11.590+10:00 AC004-PR-VPN01-DMZ kmd 9812 - - IKE negotiation successfully completed. IKE Version: 1, VPN: IPSEC-NIKON-TUN1-PROD-VPN Gateway: IKE-NIKON-TUN1-GW, Local: 10.8.10.115/9001, Remote: 89.160.20.112/9001, Local IKE-ID: 81.2.69.192, Remote IKE-ID: 89.160.20.112, VR-ID: 6, Role: Responder
+<27>1 2023-07-04T11:48:31.702+10:00 AC004-PR-VPN01-DMZ kmd 9812 - - IPSec negotiation failed with error: Peer proposed traffic-selectors are not in configured range. IKE Version: 2, VPN: IPSEC-INT-ORMB-TUN2-VPN Gateway: IKE-INT-ORMB-TUN2-GW, Local: 10.32.64.128/9001, Remote: 89.160.20.112/9001, Local IKE-ID: 89.160.20.112, Remote IKE-ID: 89.160.20.112, VR-ID: 6
 <158>1 2023-05-04T15:21:01.102+10:00 AB1234-ABC2-AB-AB01C-ABC kernel - - - FW: gr-0/0/0.14 A udp 127.0.0.1 89.160.20.112 49153 49153 
 <158>1 2023-05-04T15:18:05.010+10:00 AB1234-ABC2-AB-AB01C-ABC - - - - node1.fpc0 PFE_FW_SYSLOG_IP: FW: reth5.175 A pim 67.43.156.1 89.160.20.113 0 0 (1 packets) 
 <158>1 2023-05-09T12:20:23.180+10:00 AAAA-A-AA-AAAAAA-AAAAAA-AAA - - - - AAAA-A-AA-AAAAAA-AAAAAA-AAA PFE_FW_SYSLOG_IP: FW: reth2.605 A udp 67.43.156.2 89.160.20.112 0 0 (1 packets)
@@ -15,3 +19,4 @@
 <166>1 2023-05-08T10:54:24.821+10:00 AB1234-A-AB-AB01C-ABC - - - - AB1234-A-AB-AB01C-ABC dpdk_eth_devstart (pid=0x4c6a1bc0): port 7 ifd xe-0/0/7, new dpdk_port_state=2 dpdk_swt_port_state 1
 <166>1 2023-05-08T10:54:24.756+10:00 AB1234-A-AB-AB01C-ABC - - - - AB1234-A-AB-AB01C-ABC nh_fabric_fill_jnhinfo: Storing nh_id as 0x2dd and jnh as 0x58e302
 <167>1 2023-05-08T10:54:24.704+10:00 AB1234-A-AB-AB01C-ABC - - - - AB1234-A-AB-AB01C-ABC Copying remote chassis chassis 1, IP: 81.2.69.192
+<166>1 2023-05-08T10:54:24.756+10:00 AB1234-A-AB-AB01C-ABC - - - - AB1234-A-AB-AB01C-ABC nh_fabric_fill_jnhinfo: ABCDE: Test default message 123456
diff --git a/packages/juniper_srx/data_stream/log/_dev/test/pipeline/test-system.log-expected.json b/packages/juniper_srx/data_stream/log/_dev/test/pipeline/test-system.log-expected.json
diff --git a/packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/default.yml
@@ -13,22 +13,35 @@ processors:
   - grok:
       field: event.original
       patterns:
-      # SRX Traffic log pattern
-      - '^<%{POSINT:syslog_pri}>(\d{1,3}\s)?(?:%{TIMESTAMP_ISO8601:_temp_.raw_date})\s%{SYSLOGHOST:syslog_hostname}\s%{PROG:syslog_program}\s(?:%{POSINT:syslog_pid}|-)?\s%{WORD:tag}\s\[([^=]+?\s)?%{GREEDYDATA:_temp_.traffic_structured}\]\s?$'
-      # SRX System log patterns (further parsing done in system.yml)
-      - '^<%{POSINT:syslog_pri}>(\d{1,3}\s)?(?:%{CUSTOM_DATE:_temp_.raw_date})\s%{SYSLOGHOST:syslog_hostname}\s%{GREEDYDATA:_temp_.unparsed.message}$'
+      # 1. SRX Traffic structured log pattern
+      - '^%{SYSLOG_PREFIX}?%{TIMESTAMP_ISO8601:_temp_.raw_date}\s%{SYSLOGHOST:syslog_hostname}\s%{JUNIPER_TRAFFIC_PROCESS:syslog_program}\s(?:%{POSINT:syslog_pid}|-)?\s%{WORD:tag}\s\[([^=]+?\s)?%{GREEDYDATA:_temp_.traffic_structured}\]\s?$'
+      # 2. SRX System structured log pattern (captures all structured logs when syslog_program not in JUNIPER_TRAFFIC_PROCESS)
+      - '^%{SYSLOG_PREFIX}?%{CUSTOM_DATE:_temp_.raw_date}\s%{SYSLOGHOST:syslog_hostname}\s%{PROG:syslog_program}\s(?:%{POSINT:syslog_pid}|-)?\s%{WORD:tag}\s\[([^=]+?\s)?%{GREEDYDATA:_temp_.system_structured}\](?!=)\s?%{DATA:_temp_.unparsed.message}\s?$'
+      # 3. SRX System structured-brief and unstructured log patterns (further parsing done in system.yml)
+      - '^%{SYSLOG_PREFIX}?%{CUSTOM_DATE:_temp_.raw_date}\s%{SYSLOGHOST:syslog_hostname}\s%{GREEDYDATA:_temp_.unparsed.message}$'
       pattern_definitions:
+        SYSLOG_PREFIX: '<%{POSINT:syslog_pri}>(?:\d{1,3}\s)'
         CUSTOM_DATE: "%{TIMESTAMP_ISO8601}|(%{MONTH}%{SPACE}+%{MONTHDAY}%{SPACE}+%{TIME})"
+        JUNIPER_TRAFFIC_PROCESS: "RT_FLOW|RT_UTM|RT_IDP|RT_IDS|RT_AAMW|RT_SECINTEL"
 
 # split Juniper-SRX fields
   - kv:
       field: _temp_.traffic_structured
-      field_split: " (?=[a-z0-9\\_\\-]+=)"
+      field_split: ' (?=[a-z0-9\_\-]+=)'
       value_split: "="
       prefix: "juniper.srx."
       ignore_missing: true
       ignore_failure: false
-      trim_value: "\""
+      trim_value: '"'
+# split Juniper-SRX fields
+  - kv:
+      field: _temp_.system_structured
+      field_split: ' (?=[a-z0-9\_\-]+=)'
+      value_split: "="
+      prefix: "juniper.srx."
+      ignore_missing: true
+      ignore_failure: false
+      trim_value: '"'
 
   - rename:
       field: syslog_program