Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[Cisco Meraki] Simplify ipflows pipeline to cover ICMP events #8354

Merged
merged 3 commits into from Nov 2, 2023
Merged

[Cisco Meraki] Simplify ipflows pipeline to cover ICMP events #8354

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
1888f3b
Simplify ipflows pipeline to cover ICMP events
chemamartinez Oct 31, 2023
be09d69
Update changelog
chemamartinez Oct 31, 2023
48eca25
Fix requested changes
chemamartinez Nov 1, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
5 changes: 5 additions & 0 deletions packages/cisco_meraki/changelog.yml
View file
Open in desktop
@@ -1,4 +1,9 @@
# newer versions go on top
- version: "1.18.0"
changes:
- description: Simplify IPflows pipeline to cover ICMP events.
type: enhancement
link: https://github.com/elastic/integrations/pull/8354
- version: "1.17.1"
changes:
- description: Add missing `client.as.*` field definitions.
Expand Down
350 changes: 175 additions & 175 deletions .../cisco_meraki/data_stream/log/_dev/test/pipeline/test-airmarshal-events.log-expected.json
View file
Open in desktop

Large diffs are not rendered by default.

62 changes: 31 additions & 31 deletions packages/cisco_meraki/data_stream/log/_dev/test/pipeline/test-events.log-expected.json
View file
Open in desktop

Large diffs are not rendered by default.

36 changes: 18 additions & 18 deletions packages/cisco_meraki/data_stream/log/_dev/test/pipeline/test-flows.log-expected.json
View file
Open in desktop
Expand Up @@ -21,7 +21,7 @@
"category": [
"network"
],
"original": "\u003c134\u003e1 1647478988.289402144 MX84_4 flows allow src=10.0.2.170 dst=10.0.0.34 mac=00:7C:2D:BD:76:F2 protocol=udp sport=54841 dport=15600",
"original": "<134>1 1647478988.289402144 MX84_4 flows allow src=10.0.2.170 dst=10.0.0.34 mac=00:7C:2D:BD:76:F2 protocol=udp sport=54841 dport=15600",
"type": [
"info",
"connection",
Expand Down Expand Up @@ -80,7 +80,7 @@
"category": [
"network"
],
"original": "\u003c134\u003e1 1647478988.476061795 MX84 flows src=216.160.83.57 dst=216.160.83.61 protocol=tcp sport=54445 dport=44210 pattern: 1 all",
"original": "<134>1 1647478988.476061795 MX84 flows src=216.160.83.57 dst=216.160.83.61 protocol=tcp sport=54445 dport=44210 pattern: 1 all",
"type": [
"info",
"access",
Expand Down Expand Up @@ -138,7 +138,7 @@
"category": [
"network"
],
"original": "\u003c134\u003e1 1647478988.596151424 MX84_7 flows allow src=10.0.0.34 dst=10.0.0.234 mac=64:1C:B0:BA:F0:EC protocol=tcp sport=49761 dport=15500",
"original": "<134>1 1647478988.596151424 MX84_7 flows allow src=10.0.0.34 dst=10.0.0.234 mac=64:1C:B0:BA:F0:EC protocol=tcp sport=49761 dport=15500",
"type": [
"info",
"connection",
Expand Down Expand Up @@ -181,7 +181,7 @@
"category": [
"network"
],
"original": "\u003c134\u003e1 1664382879.496990921 AP_XXXX flows allow src=fe80::1021:83ca:b68:4cd8 dst=ff02::1:ffb6:a227 mac=28:FF:3C:AB:DB:AA protocol=icmp6 type=135",
"original": "<134>1 1664382879.496990921 AP_XXXX flows allow src=fe80::1021:83ca:b68:4cd8 dst=ff02::1:ffb6:a227 mac=28:FF:3C:AB:DB:AA protocol=icmp6 type=135",
"type": [
"info",
"connection",
Expand Down Expand Up @@ -223,7 +223,7 @@
"category": [
"network"
],
"original": "\u003c134\u003e1 1664385452.707589827 AP_XXXX flows allow src=172.16.12.23 dst=224.0.0.2 mac=4C:AB:4F:0D:3D:AA protocol=2",
"original": "<134>1 1664385452.707589827 AP_XXXX flows allow src=172.16.12.23 dst=224.0.0.2 mac=4C:AB:4F:0D:3D:AA protocol=2",
"type": [
"info",
"connection",
Expand Down Expand Up @@ -277,7 +277,7 @@
"category": [
"network"
],
"original": "\u003c134\u003e1 1664385453.129104346 AP_XXXX flows allow src=172.16.10.14 dst=81.2.69.144 mac=EC:63:D7:0F:6B:AA protocol=icmp type=8",
"original": "<134>1 1664385453.129104346 AP_XXXX flows allow src=172.16.10.14 dst=81.2.69.144 mac=EC:63:D7:0F:6B:AA protocol=icmp type=8",
"type": [
"info",
"connection",
Expand Down Expand Up @@ -306,7 +306,7 @@
"event_type": "flows",
"firewall": {
"action": "allow",
"rule": "(dst 10.0.0.0/8) \u0026\u0026 (src 10.241.0.0/16)"
"rule": "(dst 10.0.0.0/8) && (src 10.241.0.0/16)"
}
},
"destination": {
Expand All @@ -321,7 +321,7 @@
"category": [
"network"
],
"original": "\u003c134\u003e1 1674604848.429996761 MX84 flows src=10.241.77.11 dst=10.241.77.255 mac=24:2F:FA:1E:B7:E6 protocol=udp sport=138 dport=138 pattern: allow (dst 10.0.0.0/8) \u0026\u0026 (src 10.241.0.0/16)",
"original": "<134>1 1674604848.429996761 MX84 flows src=10.241.77.11 dst=10.241.77.255 mac=24:2F:FA:1E:B7:E6 protocol=udp sport=138 dport=138 pattern: allow (dst 10.0.0.0/8) && (src 10.241.0.0/16)",
"type": [
"info",
"access",
Expand Down Expand Up @@ -380,7 +380,7 @@
"category": [
"network"
],
"original": "\u003c134\u003e1 1674604848.429996761 MX84 flows src=192.168.222.3 dst=216.160.83.57 mac=00:17:55:76:EC:12 protocol=tcp sport=61403 dport=9998 pattern: Group Policy Allow",
"original": "<134>1 1674604848.429996761 MX84 flows src=192.168.222.3 dst=216.160.83.57 mac=00:17:55:76:EC:12 protocol=tcp sport=61403 dport=9998 pattern: Group Policy Allow",
"type": [
"info",
"access",
Expand Down Expand Up @@ -424,7 +424,7 @@
"category": [
"network"
],
"original": "\u003c134\u003e1 1674604848.429996761 MX84 flows src=10.8.6.10 dst=172.28.1.14 protocol=icmp type=8 pattern: allow all",
"original": "<134>1 1674604848.429996761 MX84 flows src=10.8.6.10 dst=172.28.1.14 protocol=icmp type=8 pattern: allow all",
"type": [
"info",
"access",
Expand Down Expand Up @@ -482,7 +482,7 @@
"category": [
"network"
],
"original": "\u003c134\u003e1 1674604848.429996761 MX84 flows src=172.28.1.9 dst=216.160.83.61 mac=98:18:88:7C:45:BF protocol=udp sport=45713 dport=53 pattern: allow udp",
"original": "<134>1 1674604848.429996761 MX84 flows src=172.28.1.9 dst=216.160.83.61 mac=98:18:88:7C:45:BF protocol=udp sport=45713 dport=53 pattern: allow udp",
"type": [
"info",
"access",
Expand Down Expand Up @@ -527,7 +527,7 @@
"category": [
"network"
],
"original": "\u003c134\u003e1 1674604848.429996761 MX84 flows src=10.10.10.11 dst=172.16.12.23 mac=9C:7B:EF:A9:6C:D8 protocol=udp sport=64138 dport=3289 pattern: deny (src 10.10.0.0/16)",
"original": "<134>1 1674604848.429996761 MX84 flows src=10.10.10.11 dst=172.16.12.23 mac=9C:7B:EF:A9:6C:D8 protocol=udp sport=64138 dport=3289 pattern: deny (src 10.10.0.0/16)",
"type": [
"info",
"access",
Expand Down Expand Up @@ -572,7 +572,7 @@
"category": [
"network"
],
"original": "\u003c134\u003e1 1674604848.429996761 MX84 flows src=10.241.192.11 dst=10.8.2.6 mac=9C:7B:EF:A5:9C:9B protocol=tcp sport=54791 dport=80 pattern: deny all",
"original": "<134>1 1674604848.429996761 MX84 flows src=10.241.192.11 dst=10.8.2.6 mac=9C:7B:EF:A5:9C:9B protocol=tcp sport=54791 dport=80 pattern: deny all",
"type": [
"info",
"access",
Expand Down Expand Up @@ -617,7 +617,7 @@
"category": [
"network"
],
"original": "\u003c134\u003e1 1674604848.429996761 MX84 flows src=192.168.201.81 dst=10.8.2.4 mac=B4:6B:FC:6A:E0:5A protocol=udp sport=60288 dport=53 pattern: allow all",
"original": "<134>1 1674604848.429996761 MX84 flows src=192.168.201.81 dst=10.8.2.4 mac=B4:6B:FC:6A:E0:5A protocol=udp sport=60288 dport=53 pattern: allow all",
"type": [
"info",
"access",
Expand Down Expand Up @@ -662,7 +662,7 @@
"category": [
"network"
],
"original": "\u003c134\u003e1 948136486.721741837 MX60 firewall src=10.10.10.11 dst=10.241.77.255 mac=24:2F:FA:1E:B7:E6 protocol=udp sport=9562 dport=53 pattern: allow all",
"original": "<134>1 948136486.721741837 MX60 firewall src=10.10.10.11 dst=10.241.77.255 mac=24:2F:FA:1E:B7:E6 protocol=udp sport=9562 dport=53 pattern: allow all",
"type": [
"info",
"access",
Expand Down Expand Up @@ -707,7 +707,7 @@
"category": [
"network"
],
"original": "\u003c134\u003e1 948136486.721741837 MX60 vpn_firewall src=10.241.192.1 dst=10.241.77.255 mac=24:2F:FA:1E:B7:E6 protocol=udp sport=9562 dport=53 pattern: allow all",
"original": "<134>1 948136486.721741837 MX60 vpn_firewall src=10.241.192.1 dst=10.241.77.255 mac=24:2F:FA:1E:B7:E6 protocol=udp sport=9562 dport=53 pattern: allow all",
"type": [
"info",
"access",
Expand Down Expand Up @@ -752,7 +752,7 @@
"category": [
"network"
],
"original": "\u003c134\u003e1 948136486.721741837 MX60 cellular_firewall src=10.10.10.11 dst=10.241.77.255 mac=24:2F:FA:1E:B7:E6 protocol=udp sport=9562 dport=53 pattern: allow all",
"original": "<134>1 948136486.721741837 MX60 cellular_firewall src=10.10.10.11 dst=10.241.77.255 mac=24:2F:FA:1E:B7:E6 protocol=udp sport=9562 dport=53 pattern: allow all",
"type": [
"info",
"access",
Expand Down Expand Up @@ -797,7 +797,7 @@
"category": [
"network"
],
"original": "\u003c134\u003e1 948136486.721741837 MX60 bridge_anyconnect_client_vpn_firewall src=10.241.192.1 dst=10.241.77.255 mac=24:2F:FA:1E:B7:E6 protocol=udp sport=9562 dport=53 pattern: allow all",
"original": "<134>1 948136486.721741837 MX60 bridge_anyconnect_client_vpn_firewall src=10.241.192.1 dst=10.241.77.255 mac=24:2F:FA:1E:B7:E6 protocol=udp sport=9562 dport=53 pattern: allow all",
"type": [
"info",
"access",
Expand Down
2 changes: 2 additions & 0 deletions packages/cisco_meraki/data_stream/log/_dev/test/pipeline/test-ip-flow.log
View file
Open in desktop
Expand Up @@ -6,3 +6,5 @@
<134>1 1647479325.842384731 MX84 ip_flow_end src=10.0.3.116 dst=67.43.156.14 protocol=udp sport=38422 dport=443 translated_src_ip=216.160.83.61 translated_port=38422
<134>1 1647479325.842377481 MX84 ip_flow_end src=10.0.2.99 dst=10.0.0.1 protocol=udp sport=29534 dport=53 translated_dst_ip=89.160.20.112 translated_port=53
<134>1 1647479325.755292025 MX100 ip_flow_end src=10.0.0.234 dst=81.2.69.144 protocol=tcp sport=36498 dport=80 translated_src_ip=1.128.3.4 translated_port=36498
<134>1 1647479325.755292025 MX100 ip_flow_start src=10.0.0.234 dst=81.2.69.145 protocol=icmp translated_src_ip=1.128.3.4
<134>1 1647479325.755292025 MX100 ip_flow_end src=10.0.2.99 dst=10.0.0.1 protocol=icmp translated_dst_ip=89.160.20.112
120 changes: 112 additions & 8 deletions packages/cisco_meraki/data_stream/log/_dev/test/pipeline/test-ip-flow.log-expected.json
View file
Open in desktop
Expand Up @@ -28,7 +28,7 @@
"category": [
"network"
],
"original": "\u003c134\u003e1 1647479278.997155282 MX100 ip_flow_start src=10.0.0.234 dst=81.2.69.145 protocol=tcp sport=34294 dport=80 translated_src_ip=1.128.3.4 translated_port=34294",
"original": "<134>1 1647479278.997155282 MX100 ip_flow_start src=10.0.0.234 dst=81.2.69.145 protocol=tcp sport=34294 dport=80 translated_src_ip=1.128.3.4 translated_port=34294",
"type": [
"info"
]
Expand Down Expand Up @@ -82,7 +82,7 @@
"category": [
"network"
],
"original": "\u003c134\u003e1 1647479278.995279215 MX100 ip_flow_start src=10.0.0.234 dst=81.2.69.143 protocol=udp sport=45061 dport=53 translated_src_ip=1.128.3.4 translated_port=45061",
"original": "<134>1 1647479278.995279215 MX100 ip_flow_start src=10.0.0.234 dst=81.2.69.143 protocol=udp sport=45061 dport=53 translated_src_ip=1.128.3.4 translated_port=45061",
"type": [
"info"
]
Expand Down Expand Up @@ -136,7 +136,7 @@
"category": [
"network"
],
"original": "\u003c134\u003e1 1647479278.974067126 MX100 ip_flow_start src=10.0.0.234 dst=81.2.69.143 protocol=udp sport=37401 dport=53 translated_src_ip=1.128.3.4 translated_port=37401",
"original": "<134>1 1647479278.974067126 MX100 ip_flow_start src=10.0.0.234 dst=81.2.69.143 protocol=udp sport=37401 dport=53 translated_src_ip=1.128.3.4 translated_port=37401",
"type": [
"info"
]
Expand Down Expand Up @@ -196,7 +196,7 @@
"category": [
"network"
],
"original": "\u003c134\u003e1 1647479278.911594876 MX84 ip_flow_start src=10.0.3.138 dst=89.160.20.156 protocol=tcp sport=61272 dport=443 translated_src_ip=216.160.83.61 translated_port=61272",
"original": "<134>1 1647479278.911594876 MX84 ip_flow_start src=10.0.3.138 dst=89.160.20.156 protocol=tcp sport=61272 dport=443 translated_src_ip=216.160.83.61 translated_port=61272",
"type": [
"info"
]
Expand Down Expand Up @@ -265,7 +265,7 @@
"category": [
"network"
],
"original": "\u003c134\u003e1 1647479325.891451682 MX84 ip_flow_end src=10.0.2.249 dst=10.0.0.1 protocol=udp sport=7421 dport=53 translated_dst_ip=89.160.20.112 translated_port=53",
"original": "<134>1 1647479325.891451682 MX84 ip_flow_end src=10.0.2.249 dst=10.0.0.1 protocol=udp sport=7421 dport=53 translated_dst_ip=89.160.20.112 translated_port=53",
"type": [
"info"
]
Expand Down Expand Up @@ -313,7 +313,7 @@
"category": [
"network"
],
"original": "\u003c134\u003e1 1647479325.842384731 MX84 ip_flow_end src=10.0.3.116 dst=67.43.156.14 protocol=udp sport=38422 dport=443 translated_src_ip=216.160.83.61 translated_port=38422",
"original": "<134>1 1647479325.842384731 MX84 ip_flow_end src=10.0.3.116 dst=67.43.156.14 protocol=udp sport=38422 dport=443 translated_src_ip=216.160.83.61 translated_port=38422",
"type": [
"info"
]
Expand Down Expand Up @@ -382,7 +382,7 @@
"category": [
"network"
],
"original": "\u003c134\u003e1 1647479325.842377481 MX84 ip_flow_end src=10.0.2.99 dst=10.0.0.1 protocol=udp sport=29534 dport=53 translated_dst_ip=89.160.20.112 translated_port=53",
"original": "<134>1 1647479325.842377481 MX84 ip_flow_end src=10.0.2.99 dst=10.0.0.1 protocol=udp sport=29534 dport=53 translated_dst_ip=89.160.20.112 translated_port=53",
"type": [
"info"
]
Expand Down Expand Up @@ -430,7 +430,7 @@
"category": [
"network"
],
"original": "\u003c134\u003e1 1647479325.755292025 MX100 ip_flow_end src=10.0.0.234 dst=81.2.69.144 protocol=tcp sport=36498 dport=80 translated_src_ip=1.128.3.4 translated_port=36498",
"original": "<134>1 1647479325.755292025 MX100 ip_flow_end src=10.0.0.234 dst=81.2.69.144 protocol=tcp sport=36498 dport=80 translated_src_ip=1.128.3.4 translated_port=36498",
"type": [
"info"
]
Expand All @@ -455,6 +455,110 @@
"forwarded",
"preserve_original_event"
]
},
{
"@timestamp": "2022-03-17T01:08:45.755Z",
"cisco_meraki": {
"event_type": "ip_flow_start"
},
"destination": {
"geo": {
"city_name": "London",
"continent_name": "Europe",
"country_iso_code": "GB",
"country_name": "United Kingdom",
"location": {
"lat": 51.5142,
"lon": -0.0931
},
"region_iso_code": "GB-ENG",
"region_name": "England"
},
"ip": "81.2.69.145"
},
"ecs": {
"version": "8.10.0"
},
"event": {
"category": [
"network"
],
"original": "<134>1 1647479325.755292025 MX100 ip_flow_start src=10.0.0.234 dst=81.2.69.145 protocol=icmp translated_src_ip=1.128.3.4",
"type": [
"info"
]
},
"network": {
"protocol": "icmp"
},
"observer": {
"hostname": "MX100"
},
"source": {
"as": {
"number": 1221,
"organization": {
"name": "Telstra Pty Ltd"
}
},
"ip": "1.128.3.4"
},
"tags": [
"forwarded",
"preserve_original_event"
]
},
{
"@timestamp": "2022-03-17T01:08:45.755Z",
"cisco_meraki": {
"event_type": "ip_flow_end"
},
"destination": {
"as": {
"number": 29518,
"organization": {
"name": "Bredband2 AB"
}
},
"geo": {
"city_name": "Linköping",
"continent_name": "Europe",
"country_iso_code": "SE",
"country_name": "Sweden",
"location": {
"lat": 58.4167,
"lon": 15.6167
},
"region_iso_code": "SE-E",
"region_name": "Östergötland County"
},
"ip": "89.160.20.112"
},
"ecs": {
"version": "8.10.0"
},
"event": {
"category": [
"network"
],
"original": "<134>1 1647479325.755292025 MX100 ip_flow_end src=10.0.2.99 dst=10.0.0.1 protocol=icmp translated_dst_ip=89.160.20.112",
"type": [
"info"
]
},
"network": {
"protocol": "icmp"
},
"observer": {
"hostname": "MX100"
},
"source": {
"ip": "10.0.2.99"
},
"tags": [
"forwarded",
"preserve_original_event"
]
}
]
}