Skip to content

Commit

Permalink
add another test and update the pipeline to handle nat better
Browse files Browse the repository at this point in the history
  • Loading branch information
@jrmolin
jrmolin committed Jun 7, 2024
1 parent aa04de5 commit 9e8a8a4
Show file tree
Hide file tree
Showing 5 changed files with 138 additions and 53 deletions.
1 change: 1 addition & 0 deletions packages/stormshield/data_stream/log/_dev/test/pipeline/test-firewall.log
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,3 +28,4 @@ id=firewall time="2024-03-08 15:14:57" fw="stormy-1" tz=+0000 startime="2024-03-
id=firewall time="2024-03-08 15:14:57" fw="stormy-1" tz=+0000 startime="2024-03-08 10:14:51" pri=5 confid=01 slotlevel=0 ruleid=7 ipproto=tcp dstif="Ethernet0" dstifname="out" proto=https src=192.168.197.134 srcport=19883 srcname=Firewall_out dst=192.168.36.20 dstport=443 dstportname=https dstname=update2-sns.stormshieldcs.eu dstcontinent="eu" dstcountry="fr" modsrc=192.168.197.134 modsrcport=19883 origdst=192.168.36.20 origdstport=443 ipv=4 sent=3721 rcvd=921861 duration=5.52 action=pass logtype="connection"
id=firewall time="2024-03-08 15:14:57" fw="stormy-1" tz=+0000 startime="2024-03-08 10:14:54" pri=5 confid=01 slotlevel=0 ruleid=7 ipproto=tcp dstif="Ethernet0" dstifname="out" proto=https src=192.168.197.134 srcport=15908 srcname=Firewall_out dst=192.168.36.4 dstport=443 dstportname=https dstname=update1-sns.stormshieldcs.eu dstcontinent="eu" dstcountry="fr" modsrc=192.168.197.134 modsrcport=15908 origdst=192.168.36.4 origdstport=443 ipv=4 sent=1467 rcvd=4863757 duration=2.98 action=pass logtype="connection"
id=firewall time="2024-03-08 15:14:58" fw="stormy-1" tz=+0000 startime="2024-03-08 10:14:56" pri=5 msg="Active Update: update successful CompromisedUrls" service=sysevent logtype="system" alarmid=41
id=firewall time="2024-06-07 18:46:18" fw="stormy-1" tz=+0000 startime="2024-06-07 10:40:42" pri=5 confid=01 slotlevel=2 ruleid=7 srcif="Ethernet1" srcifname="segment0" ipproto=tcp dstif="Ethernet0" dstifname="out" proto=http src=89.160.20.128 srcport=55008 srcportname=ephemeral_fw_tcp srcname=vm-internal srcmac=00:0c:29:8d:6c:55 srccontinent="eu" srccountry="se" dst=1.128.91.48 dstport=80 dstportname=http dstname=connectivity-check.ubuntu.com dstcontinent="na" dstcountry="us" modsrc=192.168.197.134 modsrcport=55008 origdst=1.128.91.48 origdstport=80 ipv=4 sent=77 rcvd=177 duration=0.06 action=pass op=GET result=204 arg="/" logtype="plugin"
139 changes: 99 additions & 40 deletions packages/stormshield/data_stream/log/_dev/test/pipeline/test-firewall.log-expected.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -176,16 +176,13 @@
"dstcountry": "fr",
"dstportname": "ntp",
"id": "firewall",
"modsrc": "192.168.197.134",
"modsrcport": "123",
"pri": "5",
"rcvd": "48",
"sent": "48",
"slotlevel": "0",
"srcname": "Firewall_out",
"srcportname": "ntp"
},
"origdstport": "123",
"startime": "2024-03-08 12:27:28",
"time": "2024-03-08 17:29:30",
"tz": "+0000"
Expand Down Expand Up @@ -420,7 +417,9 @@
"@timestamp": "2019-02-24T16:38:00.000+01:00",
"destination": {
"ip": "10.10.10.10",
"port": 1900
"nat": {
"port": 1900
}
},
"ecs": {
"version": "8.11.0"
Expand Down Expand Up @@ -547,9 +546,6 @@
"destination": {
"domain": "Firewall_out",
"ip": "192.168.197.134",
"nat": {
"ip": "192.168.197.134"
},
"port": 443
},
"ecs": {
Expand Down Expand Up @@ -586,16 +582,13 @@
"confid": "00",
"dstportname": "https",
"id": "firewall",
"modsrc": "192.168.197.1",
"modsrcport": "61549",
"pri": "5",
"rcvd": "3614",
"sent": "605",
"slotlevel": "0",
"srcportname": "ad2008-dyn_tcp",
"version": "TLSv1.3"
},
"origdstport": "443",
"srcif": "Ethernet0",
"srcifname": "out",
"startime": "2024-03-08 10:14:08",
Expand Down Expand Up @@ -679,9 +672,6 @@
"destination": {
"domain": "Firewall_out",
"ip": "192.168.197.134",
"nat": {
"ip": "192.168.197.134"
},
"port": 443
},
"ecs": {
Expand Down Expand Up @@ -718,16 +708,13 @@
"confid": "00",
"dstportname": "https",
"id": "firewall",
"modsrc": "192.168.197.1",
"modsrcport": "61548",
"pri": "5",
"rcvd": "3614",
"sent": "573",
"slotlevel": "0",
"srcportname": "ad2008-dyn_tcp",
"version": "TLSv1.3"
},
"origdstport": "443",
"srcif": "Ethernet0",
"srcifname": "out",
"startime": "2024-03-08 10:14:08",
Expand All @@ -743,9 +730,6 @@
"destination": {
"domain": "Firewall_out",
"ip": "192.168.197.134",
"nat": {
"ip": "192.168.197.134"
},
"port": 443
},
"ecs": {
Expand Down Expand Up @@ -782,16 +766,13 @@
"confid": "00",
"dstportname": "https",
"id": "firewall",
"modsrc": "192.168.197.1",
"modsrcport": "61550",
"pri": "5",
"rcvd": "3614",
"sent": "605",
"slotlevel": "0",
"srcportname": "ad2008-dyn_tcp",
"version": "TLSv1.3"
},
"origdstport": "443",
"srcif": "Ethernet0",
"srcifname": "out",
"startime": "2024-03-08 10:14:08",
Expand All @@ -806,9 +787,6 @@
"@timestamp": "2024-03-08T10:12:09.000Z",
"destination": {
"ip": "192.168.236.254",
"nat": {
"ip": "192.168.236.254"
},
"port": 67
},
"ecs": {
Expand Down Expand Up @@ -846,16 +824,13 @@
"confid": "01",
"dstportname": "bootps",
"id": "firewall",
"modsrc": "192.168.236.131",
"modsrcport": "68",
"pri": "5",
"rcvd": "300",
"sent": "300",
"slotlevel": "0",
"srcname": "Firewall_in",
"srcportname": "bootpc"
},
"origdstport": "67",
"startime": "2024-03-08 10:12:09",
"time": "2024-03-08 10:14:09",
"tz": "+0000"
Expand Down Expand Up @@ -1342,9 +1317,6 @@
"country_iso_code": "fr"
},
"ip": "192.168.36.20",
"nat": {
"ip": "192.168.36.20"
},
"port": 443
},
"ecs": {
Expand Down Expand Up @@ -1383,15 +1355,12 @@
"dstcontinent": "eu",
"dstportname": "https",
"id": "firewall",
"modsrc": "192.168.197.134",
"modsrcport": "19883",
"pri": "5",
"rcvd": "921861",
"sent": "3721",
"slotlevel": "0",
"srcname": "Firewall_out"
},
"origdstport": "443",
"startime": "2024-03-08 10:14:51",
"time": "2024-03-08 15:14:57",
"tz": "+0000"
Expand All @@ -1408,9 +1377,6 @@
"country_iso_code": "fr"
},
"ip": "192.168.36.4",
"nat": {
"ip": "192.168.36.4"
},
"port": 443
},
"ecs": {
Expand Down Expand Up @@ -1449,15 +1415,12 @@
"dstcontinent": "eu",
"dstportname": "https",
"id": "firewall",
"modsrc": "192.168.197.134",
"modsrcport": "15908",
"pri": "5",
"rcvd": "4863757",
"sent": "1467",
"slotlevel": "0",
"srcname": "Firewall_out"
},
"origdstport": "443",
"startime": "2024-03-08 10:14:54",
"time": "2024-03-08 15:14:57",
"tz": "+0000"
Expand Down Expand Up @@ -1496,6 +1459,102 @@
"tags": [
"preserve_original_event"
]
},
{
"@timestamp": "2024-06-07T10:40:42.000Z",
"destination": {
"as": {
"number": 1221,
"organization": {
"name": "Telstra Pty Ltd"
}
},
"domain": "connectivity-check.ubuntu.com",
"geo": {
"country_iso_code": "us"
},
"ip": "1.128.91.48",
"port": 80
},
"ecs": {
"version": "8.11.0"
},
"event": {
"action": "pass",
"created": "2024-06-07T10:40:42.000Z",
"duration": 60000000,
"original": "id=firewall time=\"2024-06-07 18:46:18\" fw=\"stormy-1\" tz=+0000 startime=\"2024-06-07 10:40:42\" pri=5 confid=01 slotlevel=2 ruleid=7 srcif=\"Ethernet1\" srcifname=\"segment0\" ipproto=tcp dstif=\"Ethernet0\" dstifname=\"out\" proto=http src=89.160.20.128 srcport=55008 srcportname=ephemeral_fw_tcp srcname=vm-internal srcmac=00:0c:29:8d:6c:55 srccontinent=\"eu\" srccountry=\"se\" dst=1.128.91.48 dstport=80 dstportname=http dstname=connectivity-check.ubuntu.com dstcontinent=\"na\" dstcountry=\"us\" modsrc=192.168.197.134 modsrcport=55008 origdst=1.128.91.48 origdstport=80 ipv=4 sent=77 rcvd=177 duration=0.06 action=pass op=GET result=204 arg=\"/\" logtype=\"plugin\"",
"timezone": "+00:00"
},
"network": {
"protocol": "http",
"transport": "tcp",
"type": "ipv4"
},
"observer": {
"vendor": "Stormshield"
},
"rule": {
"id": "7"
},
"source": {
"as": {
"number": 29518,
"organization": {
"name": "Bredband2 AB"
}
},
"bytes": 77,
"geo": {
"city_name": "Linköping",
"continent_name": "Europe",
"country_iso_code": "SE",
"country_name": "Sweden",
"location": {
"lat": 58.4167,
"lon": 15.6167
},
"region_iso_code": "SE-E",
"region_name": "Östergötland County"
},
"ip": "89.160.20.128",
"mac": "00-0C-29-8D-6C-55",
"nat": {
"ip": "192.168.197.134"
},
"port": 55008
},
"stormshield": {
"dstif": "Ethernet0",
"dstifname": "out",
"fw": "stormy-1",
"logtype": "plugin",
"metadata": {
"arg": "/",
"confid": "01",
"dstcontinent": "na",
"dstportname": "http",
"id": "firewall",
"op": "GET",
"pri": "5",
"rcvd": "177",
"result": "204",
"sent": "77",
"slotlevel": "2",
"srccontinent": "eu",
"srccountry": "se",
"srcname": "vm-internal",
"srcportname": "ephemeral_fw_tcp"
},
"srcif": "Ethernet1",
"srcifname": "segment0",
"startime": "2024-06-07 10:40:42",
"time": "2024-06-07 18:46:18",
"tz": "+0000"
},
"tags": [
"preserve_original_event"
]
}
]
}
48 changes: 35 additions & 13 deletions packages/stormshield/data_stream/log/elasticsearch/ingest_pipeline/default.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -102,26 +102,38 @@ processors:
if: ctx.stormshield?.logtype == 'count'

- convert:
field: stormshield.origsrc
target_field: source.ip
field: stormshield.modsrc
target_field: source.nat.ip
type: ip
ignore_missing: true
if: ctx.stormshield?.modsrc != null && ctx.stormshield?.modsrc != ctx.stormshield?.src

- convert:
field: stormshield.src
target_field: source.ip
type: ip
if: ctx.stormshield?.origsrc == null && ctx.stormshield?.src != null
- convert:
field: stormshield.src
target_field: source.nat.ip
type: ip
if: ctx.stormshield?.origsrc != null && ctx.stormshield?.src != null
ignore_missing: true
- remove:
field: stormshield.src
ignore_missing: true
- remove:
field: stormshield.origsrc
field: stormshield.modsrc
ignore_missing: true

- convert:
field: stormshield.modsrcport
target_field: source.nat.port
type: long
if: ctx.stormshield?.modsrcport != null && ctx.stormshield?.modsrcport != ctx.stormshield?.srcport
- convert:
field: stormshield.srcport
target_field: source.port
type: long
ignore_missing: true
- remove:
field: stormshield.srcport
ignore_missing: true
- remove:
field: stormshield.modsrcport
ignore_missing: true

- convert:
Expand All @@ -138,7 +150,7 @@ processors:
field: stormshield.dst
target_field: destination.nat.ip
type: ip
if: ctx.stormshield?.origdst != null && ctx.stormshield?.dst != null
if: ctx.stormshield?.origdst != null && ctx.stormshield?.dst != null && ctx.stormshield?.origdst != ctx.stormshield?.dst
- remove:
field: stormshield.dst
ignore_missing: true
Expand Down Expand Up @@ -248,15 +260,22 @@ processors:
if: ctx.destination?.mac == null

- convert:
field: stormshield.dstport
field: stormshield.origdstport
target_field: destination.port
type: long
ignore_missing: true
if: ctx.destination?.port == null
- convert:
field: stormshield.dstport
target_field: destination.nat.port
type: long
if: ctx.stormshield?.dstport != null && ctx.stormshield?.dstport != ctx.stormshield?.origdstport

- remove:
field: stormshield.dstport
ignore_missing: true
- remove:
field: stormshield.origdstport
ignore_missing: true

- rename:
field: stormshield.dstcountry
Expand Down Expand Up @@ -351,6 +370,7 @@ processors:
- Pvm
- address
- alarmid
- arg
- auth
- authcaptive
- authconsole
Expand All @@ -368,8 +388,10 @@ processors:
- mem
- modsrc
- modsrcport
- op
- pri
- rcvd
- result
- security
- sent
- sessionid
Expand Down
Loading

0 comments on commit 9e8a8a4

Please sign in to comment.