Skip to content

Commit

Permalink
[Cisco Meraki] Simplify ipflows pipeline to cover ICMP events (#8354)
Browse files Browse the repository at this point in the history 
Several dissect processors have been replaced by a kv processor that makes the processing of ip_flow events less restrictive, so now ICMP events are also processed as well as TCP/UDP ones.
  • Loading branch information
@chemamartinez
chemamartinez committed Nov 2, 2023
1 parent b93b686 commit 5d5aec7
Show file tree
Hide file tree
Showing 10 changed files with 359 additions and 252 deletions.
5 changes: 5 additions & 0 deletions packages/cisco_meraki/changelog.yml
View file
@@ -1,4 +1,9 @@
# newer versions go on top
- version: "1.18.0"
changes:
- description: Simplify IPflows pipeline to cover ICMP events.
type: enhancement
link: https://github.com/elastic/integrations/pull/8354
- version: "1.17.1"
changes:
- description: Add missing `client.as.*` field definitions.
Expand Down
350 changes: 175 additions & 175 deletions .../cisco_meraki/data_stream/log/_dev/test/pipeline/test-airmarshal-events.log-expected.json
View file

Large diffs are not rendered by default.

62 changes: 31 additions & 31 deletions packages/cisco_meraki/data_stream/log/_dev/test/pipeline/test-events.log-expected.json
View file

Large diffs are not rendered by default.

36 changes: 18 additions & 18 deletions packages/cisco_meraki/data_stream/log/_dev/test/pipeline/test-flows.log-expected.json
View file
Expand Up @@ -21,7 +21,7 @@
"category": [
"network"
],
"original": "\u003c134\u003e1 1647478988.289402144 MX84_4 flows allow src=10.0.2.170 dst=10.0.0.34 mac=00:7C:2D:BD:76:F2 protocol=udp sport=54841 dport=15600",
"original": "<134>1 1647478988.289402144 MX84_4 flows allow src=10.0.2.170 dst=10.0.0.34 mac=00:7C:2D:BD:76:F2 protocol=udp sport=54841 dport=15600",
"type": [
"info",
"connection",
Expand Down Expand Up @@ -80,7 +80,7 @@
"category": [
"network"
],
"original": "\u003c134\u003e1 1647478988.476061795 MX84 flows src=216.160.83.57 dst=216.160.83.61 protocol=tcp sport=54445 dport=44210 pattern: 1 all",
"original": "<134>1 1647478988.476061795 MX84 flows src=216.160.83.57 dst=216.160.83.61 protocol=tcp sport=54445 dport=44210 pattern: 1 all",
"type": [
"info",
"access",
Expand Down Expand Up @@ -138,7 +138,7 @@
"category": [
"network"
],
"original": "\u003c134\u003e1 1647478988.596151424 MX84_7 flows allow src=10.0.0.34 dst=10.0.0.234 mac=64:1C:B0:BA:F0:EC protocol=tcp sport=49761 dport=15500",
"original": "<134>1 1647478988.596151424 MX84_7 flows allow src=10.0.0.34 dst=10.0.0.234 mac=64:1C:B0:BA:F0:EC protocol=tcp sport=49761 dport=15500",
"type": [
"info",
"connection",
Expand Down Expand Up @@ -181,7 +181,7 @@
"category": [
"network"
],
"original": "\u003c134\u003e1 1664382879.496990921 AP_XXXX flows allow src=fe80::1021:83ca:b68:4cd8 dst=ff02::1:ffb6:a227 mac=28:FF:3C:AB:DB:AA protocol=icmp6 type=135",
"original": "<134>1 1664382879.496990921 AP_XXXX flows allow src=fe80::1021:83ca:b68:4cd8 dst=ff02::1:ffb6:a227 mac=28:FF:3C:AB:DB:AA protocol=icmp6 type=135",
"type": [
"info",
"connection",
Expand Down Expand Up @@ -223,7 +223,7 @@
"category": [
"network"
],
"original": "\u003c134\u003e1 1664385452.707589827 AP_XXXX flows allow src=172.16.12.23 dst=224.0.0.2 mac=4C:AB:4F:0D:3D:AA protocol=2",
"original": "<134>1 1664385452.707589827 AP_XXXX flows allow src=172.16.12.23 dst=224.0.0.2 mac=4C:AB:4F:0D:3D:AA protocol=2",
"type": [
"info",
"connection",
Expand Down Expand Up @@ -277,7 +277,7 @@
"category": [
"network"
],
"original": "\u003c134\u003e1 1664385453.129104346 AP_XXXX flows allow src=172.16.10.14 dst=81.2.69.144 mac=EC:63:D7:0F:6B:AA protocol=icmp type=8",
"original": "<134>1 1664385453.129104346 AP_XXXX flows allow src=172.16.10.14 dst=81.2.69.144 mac=EC:63:D7:0F:6B:AA protocol=icmp type=8",
"type": [
"info",
"connection",
Expand Down Expand Up @@ -306,7 +306,7 @@
"event_type": "flows",
"firewall": {
"action": "allow",
"rule": "(dst 10.0.0.0/8) \u0026\u0026 (src 10.241.0.0/16)"
"rule": "(dst 10.0.0.0/8) && (src 10.241.0.0/16)"
}
},
"destination": {
Expand All @@ -321,7 +321,7 @@
"category": [
"network"
],
"original": "\u003c134\u003e1 1674604848.429996761 MX84 flows src=10.241.77.11 dst=10.241.77.255 mac=24:2F:FA:1E:B7:E6 protocol=udp sport=138 dport=138 pattern: allow (dst 10.0.0.0/8) \u0026\u0026 (src 10.241.0.0/16)",
"original": "<134>1 1674604848.429996761 MX84 flows src=10.241.77.11 dst=10.241.77.255 mac=24:2F:FA:1E:B7:E6 protocol=udp sport=138 dport=138 pattern: allow (dst 10.0.0.0/8) && (src 10.241.0.0/16)",
"type": [
"info",
"access",
Expand Down Expand Up @@ -380,7 +380,7 @@
"category": [
"network"
],
"original": "\u003c134\u003e1 1674604848.429996761 MX84 flows src=192.168.222.3 dst=216.160.83.57 mac=00:17:55:76:EC:12 protocol=tcp sport=61403 dport=9998 pattern: Group Policy Allow",
"original": "<134>1 1674604848.429996761 MX84 flows src=192.168.222.3 dst=216.160.83.57 mac=00:17:55:76:EC:12 protocol=tcp sport=61403 dport=9998 pattern: Group Policy Allow",
"type": [
"info",
"access",
Expand Down Expand Up @@ -424,7 +424,7 @@
"category": [
"network"
],
"original": "\u003c134\u003e1 1674604848.429996761 MX84 flows src=10.8.6.10 dst=172.28.1.14 protocol=icmp type=8 pattern: allow all",
"original": "<134>1 1674604848.429996761 MX84 flows src=10.8.6.10 dst=172.28.1.14 protocol=icmp type=8 pattern: allow all",
"type": [
"info",
"access",
Expand Down Expand Up @@ -482,7 +482,7 @@
"category": [
"network"
],
"original": "\u003c134\u003e1 1674604848.429996761 MX84 flows src=172.28.1.9 dst=216.160.83.61 mac=98:18:88:7C:45:BF protocol=udp sport=45713 dport=53 pattern: allow udp",
"original": "<134>1 1674604848.429996761 MX84 flows src=172.28.1.9 dst=216.160.83.61 mac=98:18:88:7C:45:BF protocol=udp sport=45713 dport=53 pattern: allow udp",
"type": [
"info",
"access",
Expand Down Expand Up @@ -527,7 +527,7 @@
"category": [
"network"
],
"original": "\u003c134\u003e1 1674604848.429996761 MX84 flows src=10.10.10.11 dst=172.16.12.23 mac=9C:7B:EF:A9:6C:D8 protocol=udp sport=64138 dport=3289 pattern: deny (src 10.10.0.0/16)",
"original": "<134>1 1674604848.429996761 MX84 flows src=10.10.10.11 dst=172.16.12.23 mac=9C:7B:EF:A9:6C:D8 protocol=udp sport=64138 dport=3289 pattern: deny (src 10.10.0.0/16)",
"type": [
"info",
"access",
Expand Down Expand Up @@ -572,7 +572,7 @@
"category": [
"network"
],
"original": "\u003c134\u003e1 1674604848.429996761 MX84 flows src=10.241.192.11 dst=10.8.2.6 mac=9C:7B:EF:A5:9C:9B protocol=tcp sport=54791 dport=80 pattern: deny all",
"original": "<134>1 1674604848.429996761 MX84 flows src=10.241.192.11 dst=10.8.2.6 mac=9C:7B:EF:A5:9C:9B protocol=tcp sport=54791 dport=80 pattern: deny all",
"type": [
"info",
"access",
Expand Down Expand Up @@ -617,7 +617,7 @@
"category": [
"network"
],
"original": "\u003c134\u003e1 1674604848.429996761 MX84 flows src=192.168.201.81 dst=10.8.2.4 mac=B4:6B:FC:6A:E0:5A protocol=udp sport=60288 dport=53 pattern: allow all",
"original": "<134>1 1674604848.429996761 MX84 flows src=192.168.201.81 dst=10.8.2.4 mac=B4:6B:FC:6A:E0:5A protocol=udp sport=60288 dport=53 pattern: allow all",
"type": [
"info",
"access",
Expand Down Expand Up @@ -662,7 +662,7 @@
"category": [
"network"
],
"original": "\u003c134\u003e1 948136486.721741837 MX60 firewall src=10.10.10.11 dst=10.241.77.255 mac=24:2F:FA:1E:B7:E6 protocol=udp sport=9562 dport=53 pattern: allow all",
"original": "<134>1 948136486.721741837 MX60 firewall src=10.10.10.11 dst=10.241.77.255 mac=24:2F:FA:1E:B7:E6 protocol=udp sport=9562 dport=53 pattern: allow all",
"type": [
"info",
"access",
Expand Down Expand Up @@ -707,7 +707,7 @@
"category": [
"network"
],
"original": "\u003c134\u003e1 948136486.721741837 MX60 vpn_firewall src=10.241.192.1 dst=10.241.77.255 mac=24:2F:FA:1E:B7:E6 protocol=udp sport=9562 dport=53 pattern: allow all",
"original": "<134>1 948136486.721741837 MX60 vpn_firewall src=10.241.192.1 dst=10.241.77.255 mac=24:2F:FA:1E:B7:E6 protocol=udp sport=9562 dport=53 pattern: allow all",
"type": [
"info",
"access",
Expand Down Expand Up @@ -752,7 +752,7 @@
"category": [
"network"
],
"original": "\u003c134\u003e1 948136486.721741837 MX60 cellular_firewall src=10.10.10.11 dst=10.241.77.255 mac=24:2F:FA:1E:B7:E6 protocol=udp sport=9562 dport=53 pattern: allow all",
"original": "<134>1 948136486.721741837 MX60 cellular_firewall src=10.10.10.11 dst=10.241.77.255 mac=24:2F:FA:1E:B7:E6 protocol=udp sport=9562 dport=53 pattern: allow all",
"type": [
"info",
"access",
Expand Down Expand Up @@ -797,7 +797,7 @@
"category": [
"network"
],
"original": "\u003c134\u003e1 948136486.721741837 MX60 bridge_anyconnect_client_vpn_firewall src=10.241.192.1 dst=10.241.77.255 mac=24:2F:FA:1E:B7:E6 protocol=udp sport=9562 dport=53 pattern: allow all",
"original": "<134>1 948136486.721741837 MX60 bridge_anyconnect_client_vpn_firewall src=10.241.192.1 dst=10.241.77.255 mac=24:2F:FA:1E:B7:E6 protocol=udp sport=9562 dport=53 pattern: allow all",
"type": [
"info",
"access",
Expand Down
2 changes: 2 additions & 0 deletions packages/cisco_meraki/data_stream/log/_dev/test/pipeline/test-ip-flow.log
View file
Expand Up @@ -6,3 +6,5 @@
<134>1 1647479325.842384731 MX84 ip_flow_end src=10.0.3.116 dst=67.43.156.14 protocol=udp sport=38422 dport=443 translated_src_ip=216.160.83.61 translated_port=38422
<134>1 1647479325.842377481 MX84 ip_flow_end src=10.0.2.99 dst=10.0.0.1 protocol=udp sport=29534 dport=53 translated_dst_ip=89.160.20.112 translated_port=53
<134>1 1647479325.755292025 MX100 ip_flow_end src=10.0.0.234 dst=81.2.69.144 protocol=tcp sport=36498 dport=80 translated_src_ip=1.128.3.4 translated_port=36498
<134>1 1647479325.755292025 MX100 ip_flow_start src=10.0.0.234 dst=81.2.69.145 protocol=icmp translated_src_ip=1.128.3.4
<134>1 1647479325.755292025 MX100 ip_flow_end src=10.0.2.99 dst=10.0.0.1 protocol=icmp translated_dst_ip=89.160.20.112
120 changes: 112 additions & 8 deletions packages/cisco_meraki/data_stream/log/_dev/test/pipeline/test-ip-flow.log-expected.json
View file
Expand Up @@ -28,7 +28,7 @@
"category": [
"network"
],
"original": "\u003c134\u003e1 1647479278.997155282 MX100 ip_flow_start src=10.0.0.234 dst=81.2.69.145 protocol=tcp sport=34294 dport=80 translated_src_ip=1.128.3.4 translated_port=34294",
"original": "<134>1 1647479278.997155282 MX100 ip_flow_start src=10.0.0.234 dst=81.2.69.145 protocol=tcp sport=34294 dport=80 translated_src_ip=1.128.3.4 translated_port=34294",
"type": [
"info"
]
Expand Down Expand Up @@ -82,7 +82,7 @@
"category": [
"network"
],
"original": "\u003c134\u003e1 1647479278.995279215 MX100 ip_flow_start src=10.0.0.234 dst=81.2.69.143 protocol=udp sport=45061 dport=53 translated_src_ip=1.128.3.4 translated_port=45061",
"original": "<134>1 1647479278.995279215 MX100 ip_flow_start src=10.0.0.234 dst=81.2.69.143 protocol=udp sport=45061 dport=53 translated_src_ip=1.128.3.4 translated_port=45061",
"type": [
"info"
]
Expand Down Expand Up @@ -136,7 +136,7 @@
"category": [
"network"
],
"original": "\u003c134\u003e1 1647479278.974067126 MX100 ip_flow_start src=10.0.0.234 dst=81.2.69.143 protocol=udp sport=37401 dport=53 translated_src_ip=1.128.3.4 translated_port=37401",
"original": "<134>1 1647479278.974067126 MX100 ip_flow_start src=10.0.0.234 dst=81.2.69.143 protocol=udp sport=37401 dport=53 translated_src_ip=1.128.3.4 translated_port=37401",
"type": [
"info"
]
Expand Down Expand Up @@ -196,7 +196,7 @@
"category": [
"network"
],
"original": "\u003c134\u003e1 1647479278.911594876 MX84 ip_flow_start src=10.0.3.138 dst=89.160.20.156 protocol=tcp sport=61272 dport=443 translated_src_ip=216.160.83.61 translated_port=61272",
"original": "<134>1 1647479278.911594876 MX84 ip_flow_start src=10.0.3.138 dst=89.160.20.156 protocol=tcp sport=61272 dport=443 translated_src_ip=216.160.83.61 translated_port=61272",
"type": [
"info"
]
Expand Down Expand Up @@ -265,7 +265,7 @@
"category": [
"network"
],
"original": "\u003c134\u003e1 1647479325.891451682 MX84 ip_flow_end src=10.0.2.249 dst=10.0.0.1 protocol=udp sport=7421 dport=53 translated_dst_ip=89.160.20.112 translated_port=53",
"original": "<134>1 1647479325.891451682 MX84 ip_flow_end src=10.0.2.249 dst=10.0.0.1 protocol=udp sport=7421 dport=53 translated_dst_ip=89.160.20.112 translated_port=53",
"type": [
"info"
]
Expand Down Expand Up @@ -313,7 +313,7 @@
"category": [
"network"
],
"original": "\u003c134\u003e1 1647479325.842384731 MX84 ip_flow_end src=10.0.3.116 dst=67.43.156.14 protocol=udp sport=38422 dport=443 translated_src_ip=216.160.83.61 translated_port=38422",
"original": "<134>1 1647479325.842384731 MX84 ip_flow_end src=10.0.3.116 dst=67.43.156.14 protocol=udp sport=38422 dport=443 translated_src_ip=216.160.83.61 translated_port=38422",
"type": [
"info"
]
Expand Down Expand Up @@ -382,7 +382,7 @@
"category": [
"network"
],
"original": "\u003c134\u003e1 1647479325.842377481 MX84 ip_flow_end src=10.0.2.99 dst=10.0.0.1 protocol=udp sport=29534 dport=53 translated_dst_ip=89.160.20.112 translated_port=53",
"original": "<134>1 1647479325.842377481 MX84 ip_flow_end src=10.0.2.99 dst=10.0.0.1 protocol=udp sport=29534 dport=53 translated_dst_ip=89.160.20.112 translated_port=53",
"type": [
"info"
]
Expand Down Expand Up @@ -430,7 +430,7 @@
"category": [
"network"
],
"original": "\u003c134\u003e1 1647479325.755292025 MX100 ip_flow_end src=10.0.0.234 dst=81.2.69.144 protocol=tcp sport=36498 dport=80 translated_src_ip=1.128.3.4 translated_port=36498",
"original": "<134>1 1647479325.755292025 MX100 ip_flow_end src=10.0.0.234 dst=81.2.69.144 protocol=tcp sport=36498 dport=80 translated_src_ip=1.128.3.4 translated_port=36498",
"type": [
"info"
]
Expand All @@ -455,6 +455,110 @@
"forwarded",
"preserve_original_event"
]
},
{
"@timestamp": "2022-03-17T01:08:45.755Z",
"cisco_meraki": {
"event_type": "ip_flow_start"
},
"destination": {
"geo": {
"city_name": "London",
"continent_name": "Europe",
"country_iso_code": "GB",
"country_name": "United Kingdom",
"location": {
"lat": 51.5142,
"lon": -0.0931
},
"region_iso_code": "GB-ENG",
"region_name": "England"
},
"ip": "81.2.69.145"
},
"ecs": {
"version": "8.10.0"
},
"event": {
"category": [
"network"
],
"original": "<134>1 1647479325.755292025 MX100 ip_flow_start src=10.0.0.234 dst=81.2.69.145 protocol=icmp translated_src_ip=1.128.3.4",
"type": [
"info"
]
},
"network": {
"protocol": "icmp"
},
"observer": {
"hostname": "MX100"
},
"source": {
"as": {
"number": 1221,
"organization": {
"name": "Telstra Pty Ltd"
}
},
"ip": "1.128.3.4"
},
"tags": [
"forwarded",
"preserve_original_event"
]
},
{
"@timestamp": "2022-03-17T01:08:45.755Z",
"cisco_meraki": {
"event_type": "ip_flow_end"
},
"destination": {
"as": {
"number": 29518,
"organization": {
"name": "Bredband2 AB"
}
},
"geo": {
"city_name": "Linköping",
"continent_name": "Europe",
"country_iso_code": "SE",
"country_name": "Sweden",
"location": {
"lat": 58.4167,
"lon": 15.6167
},
"region_iso_code": "SE-E",
"region_name": "Östergötland County"
},
"ip": "89.160.20.112"
},
"ecs": {
"version": "8.10.0"
},
"event": {
"category": [
"network"
],
"original": "<134>1 1647479325.755292025 MX100 ip_flow_end src=10.0.2.99 dst=10.0.0.1 protocol=icmp translated_dst_ip=89.160.20.112",
"type": [
"info"
]
},
"network": {
"protocol": "icmp"
},
"observer": {
"hostname": "MX100"
},
"source": {
"ip": "10.0.2.99"
},
"tags": [
"forwarded",
"preserve_original_event"
]
}
]
}

0 comments on commit 5d5aec7

Please sign in to comment.