Skip to content

Commit

Permalink
[zscaler_zia] Fix mapping of source.ip and source.nat.ip (elastic#9727)
Browse files Browse the repository at this point in the history 
* Fix mapping of source.ip and source.nat.ip

* Update changelog

* updated web datastream pipeline tests

---------

Co-authored-by: Shourie Ganguly <shourie.ganguly@elastic.co>
  • Loading branch information
@chemamartinez @ShourieG
chemamartinez and ShourieG committed Apr 27, 2024
1 parent 4750ea8 commit c7bc530
Show file tree
Hide file tree
Showing 8 changed files with 58 additions and 75 deletions.
2 changes: 1 addition & 1 deletion packages/zscaler_zia/_dev/deploy/docker/sample_logs/web-http_endpoint.log
View file
@@ -1 +1 @@
{ "sourcetype" : "zscalernss-web", "event" :{"time":"2021-12-31 08:08:08","login":"test@example.com","proto":"HTTP_PROXY","eurl":"www.example.com","action":"Blocked","appname":"General Browsing","appclass":"General Browsing","reqsize":"600","respsize":"65","stime":"0","ctime":"0","urlclass":"Business Use","urlsupercat":"Information Technology","urlcat":"Web Search","malwarecat":"None","threatname":"None","riskscore":"0","dlpeng":"None","dlpdict":"None","location":"Test DB","dept":"Unknown","cip":"1.128.3.4","sip":"1.128.3.4","reqmethod":"CONNECT","respcode":"200","eua":"Windows Microsoft Windows 10 Pro ZTunnel/1.0","ereferer":"None","ruletype":"FwFilter","rulelabel":"Zscaler Proxy Traffic","contenttype":"Other","unscannabletype":"None","deviceowner":"administrator1","devicehostname":"TestMachine35"}}
{ "sourcetype" : "zscalernss-web", "event" :{"time":"2021-12-31 08:08:08","login":"test@example.com","proto":"HTTP_PROXY","eurl":"www.example.com","action":"Blocked","appname":"General Browsing","appclass":"General Browsing","reqsize":"600","respsize":"65","stime":"0","ctime":"0","urlclass":"Business Use","urlsupercat":"Information Technology","urlcat":"Web Search","malwarecat":"None","threatname":"None","riskscore":"0","dlpeng":"None","dlpdict":"None","location":"Test DB","dept":"Unknown","cip":"192.168.1.35","cintip":"203.0.113.5","sip":"1.128.3.4","reqmethod":"CONNECT","respcode":"200","eua":"Windows Microsoft Windows 10 Pro ZTunnel/1.0","ereferer":"None","ruletype":"FwFilter","rulelabel":"Zscaler Proxy Traffic","contenttype":"Other","unscannabletype":"None","deviceowner":"administrator1","devicehostname":"TestMachine35"}}
5 changes: 5 additions & 0 deletions packages/zscaler_zia/changelog.yml
View file
@@ -1,4 +1,9 @@
# newer versions go on top
- version: "2.19.1"
changes:
- description: Fix mapping of source.ip and source.nat.ip
type: bugfix
link: https://github.com/elastic/integrations/pull/9727
- version: "2.19.0"
changes:
- description: Set sensitive values as secret.
Expand Down
16 changes: 6 additions & 10 deletions ...s/zscaler_zia/data_stream/web/_dev/test/pipeline/test-web-http-endpoint.log-expected.json
View file
Expand Up @@ -43,8 +43,8 @@
"TestMachine35"
],
"ip": [
"81.2.69.193",
"81.2.69.145"
"81.2.69.145",
"81.2.69.193"
],
"user": [
"test",
Expand All @@ -57,9 +57,7 @@
"ruleset": "FwFilter"
},
"source": {
"nat": {
"ip": "81.2.69.193"
}
"ip": "81.2.69.193"
},
"tags": [
"preserve_original_event"
Expand Down Expand Up @@ -169,8 +167,8 @@
"TestMachine35"
],
"ip": [
"81.2.69.193",
"81.2.69.145"
"81.2.69.145",
"81.2.69.193"
],
"user": [
"test",
Expand All @@ -183,9 +181,7 @@
"ruleset": "FwFilter"
},
"source": {
"nat": {
"ip": "81.2.69.193"
}
"ip": "81.2.69.193"
},
"tags": [
"preserve_original_event"
Expand Down
88 changes: 33 additions & 55 deletions packages/zscaler_zia/data_stream/web/_dev/test/pipeline/test-web.log-expected.json
View file
Expand Up @@ -43,8 +43,8 @@
"TestMachine35"
],
"ip": [
"81.2.69.193",
"81.2.69.145"
"81.2.69.145",
"81.2.69.193"
],
"user": [
"test",
Expand All @@ -57,9 +57,7 @@
"ruleset": "FwFilter"
},
"source": {
"nat": {
"ip": "81.2.69.193"
}
"ip": "81.2.69.193"
},
"tags": [
"preserve_original_event"
Expand Down Expand Up @@ -171,8 +169,8 @@
"TestMachine35"
],
"ip": [
"81.2.69.193",
"89.160.20.156"
"89.160.20.156",
"81.2.69.193"
],
"user": [
"test",
Expand All @@ -185,9 +183,7 @@
"ruleset": "SSLPol"
},
"source": {
"nat": {
"ip": "81.2.69.193"
}
"ip": "81.2.69.193"
},
"tags": [
"preserve_original_event"
Expand Down Expand Up @@ -294,8 +290,8 @@
"TestMachine35"
],
"ip": [
"81.2.69.193",
"89.160.20.112"
"89.160.20.112",
"81.2.69.193"
],
"user": [
"test",
Expand All @@ -308,9 +304,7 @@
"ruleset": "FwFilter"
},
"source": {
"nat": {
"ip": "81.2.69.193"
}
"ip": "81.2.69.193"
},
"tags": [
"preserve_original_event"
Expand Down Expand Up @@ -422,8 +416,8 @@
"TestMachine35"
],
"ip": [
"81.2.69.193",
"81.2.69.144"
"81.2.69.144",
"81.2.69.193"
],
"user": [
"test",
Expand All @@ -436,9 +430,7 @@
"ruleset": "FwFilter"
},
"source": {
"nat": {
"ip": "81.2.69.193"
}
"ip": "81.2.69.193"
},
"tags": [
"preserve_original_event"
Expand Down Expand Up @@ -550,8 +542,8 @@
"TestMachine35"
],
"ip": [
"81.2.69.193",
"81.2.69.143"
"81.2.69.143",
"81.2.69.193"
],
"user": [
"test",
Expand All @@ -564,9 +556,7 @@
"ruleset": "FwFilter"
},
"source": {
"nat": {
"ip": "81.2.69.193"
}
"ip": "81.2.69.193"
},
"tags": [
"preserve_original_event"
Expand Down Expand Up @@ -674,8 +664,8 @@
"TestMachine35"
],
"ip": [
"81.2.69.193",
"89.160.20.112"
"89.160.20.112",
"81.2.69.193"
],
"user": [
"test",
Expand All @@ -688,9 +678,7 @@
"ruleset": "None"
},
"source": {
"nat": {
"ip": "81.2.69.193"
}
"ip": "81.2.69.193"
},
"tags": [
"preserve_original_event"
Expand Down Expand Up @@ -802,8 +790,8 @@
"TestMachine35"
],
"ip": [
"81.2.69.193",
"81.2.69.143"
"81.2.69.143",
"81.2.69.193"
],
"user": [
"test",
Expand All @@ -816,9 +804,7 @@
"ruleset": "FwFilter"
},
"source": {
"nat": {
"ip": "81.2.69.193"
}
"ip": "81.2.69.193"
},
"tags": [
"preserve_original_event"
Expand Down Expand Up @@ -930,8 +916,8 @@
"TestMachine35"
],
"ip": [
"81.2.69.193",
"81.2.69.143"
"81.2.69.143",
"81.2.69.193"
],
"user": [
"test"
Expand All @@ -942,9 +928,7 @@
"ruleset": "FwFilter"
},
"source": {
"nat": {
"ip": "81.2.69.193"
}
"ip": "81.2.69.193"
},
"tags": [
"preserve_original_event"
Expand Down Expand Up @@ -1050,8 +1034,8 @@
"TestMachine35"
],
"ip": [
"81.2.69.193",
"81.2.69.143"
"81.2.69.143",
"81.2.69.193"
],
"user": [
"test",
Expand All @@ -1063,9 +1047,7 @@
"ruleset": "FwFilter"
},
"source": {
"nat": {
"ip": "81.2.69.193"
}
"ip": "81.2.69.193"
},
"tags": [
"preserve_original_event"
Expand Down Expand Up @@ -1173,8 +1155,8 @@
"TestMachine35"
],
"ip": [
"81.2.69.193",
"81.2.69.143"
"81.2.69.143",
"81.2.69.193"
],
"user": [
"test",
Expand All @@ -1187,9 +1169,7 @@
"ruleset": "None"
},
"source": {
"nat": {
"ip": "81.2.69.193"
}
"ip": "81.2.69.193"
},
"tags": [
"preserve_original_event"
Expand Down Expand Up @@ -1289,8 +1269,8 @@
},
"related": {
"ip": [
"81.2.69.193",
"81.2.69.143"
"81.2.69.143",
"81.2.69.193"
],
"user": [
"test",
Expand All @@ -1302,9 +1282,7 @@
"ruleset": "None"
},
"source": {
"nat": {
"ip": "81.2.69.193"
}
"ip": "81.2.69.193"
},
"tags": [
"preserve_original_event"
Expand Down
8 changes: 4 additions & 4 deletions packages/zscaler_zia/data_stream/web/elasticsearch/ingest_pipeline/default.yml
View file
Expand Up @@ -78,14 +78,14 @@ processors:
field: event.type
value: info
- convert:
field: json.cip
field: json.cintip
target_field: source.nat.ip
if: ctx.json?.cip != ctx.json?.cintip
type: ip
ignore_missing: true
on_failure:
- remove:
field: json.cip
field: json.cintip
- append:
field: error.message
value: '{{{_ingest.on_failure_message}}}'
Expand Down Expand Up @@ -364,13 +364,13 @@ processors:
target_field: zscaler_zia.web.bandwidth_throttle
ignore_missing: true
- convert:
field: json.cintip
field: json.cip
target_field: source.ip
type: ip
ignore_missing: true
on_failure:
- remove:
field: json.cintip
field: json.cip
- append:
field: error.message
value: '{{{_ingest.on_failure_message}}}'
Expand Down
6 changes: 4 additions & 2 deletions packages/zscaler_zia/data_stream/web/sample_event.json
View file
Expand Up @@ -63,6 +63,7 @@
"TestMachine35"
],
"ip": [
"203.0.113.5",
"1.128.3.4"
],
"user": [
Expand All @@ -76,8 +77,9 @@
},
"source": {
"nat": {
"ip": "1.128.3.4"
}
"ip": "203.0.113.5"
},
"ip": "192.168.1.35"
},
"tags": [
"forwarded",
Expand Down
6 changes: 4 additions & 2 deletions packages/zscaler_zia/docs/README.md
View file
Expand Up @@ -767,6 +767,7 @@ An example event for `web` looks as following:
"TestMachine35"
],
"ip": [
"203.0.113.5",
"1.128.3.4"
],
"user": [
Expand All @@ -780,8 +781,9 @@ An example event for `web` looks as following:
},
"source": {
"nat": {
"ip": "1.128.3.4"
}
"ip": "203.0.113.5"
},
"ip": "192.168.1.35"
},
"tags": [
"forwarded",
Expand Down
2 changes: 1 addition & 1 deletion packages/zscaler_zia/manifest.yml
View file
@@ -1,7 +1,7 @@
format_version: "3.0.2"
name: zscaler_zia
title: Zscaler Internet Access
version: "2.19.0"
version: "2.19.1"
description: Collect logs from Zscaler Internet Access (ZIA) with Elastic Agent.
type: integration
categories:
Expand Down

0 comments on commit c7bc530

Please sign in to comment.