From cddc8556608b7b7f6646000f893e033aadbc3df0 Mon Sep 17 00:00:00 2001 From: Taylor Swanson Date: Tue, 22 Feb 2022 13:39:36 -0600 Subject: [PATCH 1/5] Update cef to ECS 8.2 - Breaking change: remapping email ECS fields --- packages/cef/_dev/build/build.yml | 2 +- packages/cef/_dev/build/docs/README.md | 8 +- packages/cef/changelog.yml | 5 + .../test/pipeline/test-cef.log-expected.json | 40 +++--- .../test-checkpoint.log-expected.json | 30 ++-- .../test-fp-ngfw-smc.log-expected.json | 130 +++++++++--------- .../ingest_pipeline/cp-pipeline.yml | 21 ++- .../elasticsearch/ingest_pipeline/default.yml | 2 +- packages/cef/data_stream/log/fields/ecs.yml | 6 + .../cef/data_stream/log/fields/fields.yml | 3 - .../cef/data_stream/log/sample_event.json | 2 +- packages/cef/docs/README.md | 14 +- packages/cef/manifest.yml | 2 +- 13 files changed, 146 insertions(+), 119 deletions(-) diff --git a/packages/cef/_dev/build/build.yml b/packages/cef/_dev/build/build.yml index 809e76063e90..d61527283ec8 100644 --- a/packages/cef/_dev/build/build.yml +++ b/packages/cef/_dev/build/build.yml @@ -1,3 +1,3 @@ dependencies: ecs: - reference: git@8.0 + reference: git@8.2 diff --git a/packages/cef/_dev/build/docs/README.md b/packages/cef/_dev/build/docs/README.md index 574dc3f8647a..3a4017b567e5 100644 --- a/packages/cef/_dev/build/docs/README.md +++ b/packages/cef/_dev/build/docs/README.md @@ -62,8 +62,8 @@ Check Point CEF extensions are mapped as follows: | requestCookies | - | - | checkpoint.cookie | | sourceNtDomain | - | dns.question.name | - | | Signature | - | vulnerability.id | - | -| Recipient | - | destination.user.email | - | -| Sender | - | source.user.email | - | +| Recipient | - | email.to.address | - | +| Sender | - | email.from.address | - | | deviceCustomFloatingPoint1 | update version | observer.version | - | | deviceCustomIPv6Address2 | source ipv6 address | source.ip | - | | deviceCustomIPv6Address3 | destination ipv6 address | destination.ip | - | @@ -80,7 +80,7 @@ Check Point CEF extensions are mapped as follows: | deviceCustomString1 | dlp rule name | rule.name | - | | deviceCustomString1 | email id | - | checkpoint.email_id | | deviceCustomString2 | category | - | checkpoint.category | -| deviceCustomString2 | email subject | - | checkpoint.email_subject | +| deviceCustomString2 | email subject | email.subject | checkpoint.email_subject | | deviceCustomString2 | sensor mode | - | checkpoint.sensor_mode | | deviceCustomString2 | protection id | - | checkpoint.protection_id | | deviceCustomString2 | scan invoke type | - | checkpoint.integrity_av_invoke_type | @@ -108,7 +108,7 @@ Check Point CEF extensions are mapped as follows: | deviceCustomString5 | matched category | rule.category | - | | deviceCustomString5 | vlan id | network.vlan.id | - | | deviceCustomString5 | authentication method | - | checkpoint.auth_method | -| deviceCustomString5 | email session id | - | checkpoint.email_session_id | +| deviceCustomString5 | email session id | email.message_id | checkpoint.email_session_id | | deviceCustomDate2 | subscription expiration | - | checkpoint.subs_exp | | deviceFlexNumber1 | confidence | - | checkpoint.confidence_level | | deviceFlexNumber2 | performance impact | - | checkpoint.performance_impact | diff --git a/packages/cef/changelog.yml b/packages/cef/changelog.yml index d3d856cbf6ad..f3f622704618 100644 --- a/packages/cef/changelog.yml +++ b/packages/cef/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "2.0.0" + changes: + - description: Update to ECS 8.2 + type: enhancement + link: https://github.com/elastic/integrations/pull/xxxx - version: "1.4.1" changes: - description: Append pipeline errors to error.message instead of overwriting existing errors. diff --git a/packages/cef/data_stream/log/_dev/test/pipeline/test-cef.log-expected.json b/packages/cef/data_stream/log/_dev/test/pipeline/test-cef.log-expected.json index 49cdc4336369..2bc33da3210c 100644 --- a/packages/cef/data_stream/log/_dev/test/pipeline/test-cef.log-expected.json +++ b/packages/cef/data_stream/log/_dev/test/pipeline/test-cef.log-expected.json @@ -1,49 +1,49 @@ { "expected": [ { - "message": "CEF:0|Elastic|Vaporware|1.0.0-alpha|18|Web request|low|eventId=3457 requestMethod=POST slat=38.915 slong=-77.511 proto=TCP sourceServiceName=httpd requestContext=https://www.google.com src=89.160.20.156 spt=33876 dst=192.168.10.1 dpt=443 request=https://www.example.com/cart", - "event": { - "ingested": "2021-12-25T05:20:14.195117374Z" - }, "ecs": { - "version": "8.0.0" + "version": "8.2.0" }, + "event": { + "ingested": "2022-03-09T20:17:07.566330Z" + }, + "message": "CEF:0|Elastic|Vaporware|1.0.0-alpha|18|Web request|low|eventId=3457 requestMethod=POST slat=38.915 slong=-77.511 proto=TCP sourceServiceName=httpd requestContext=https://www.google.com src=89.160.20.156 spt=33876 dst=192.168.10.1 dpt=443 request=https://www.example.com/cart", "tags": [ "preserve_original_event" ] }, { - "message": "CEF:0|Elastic|Vaporware|1.0.0-alpha|18|Authentication|low|eventId=123 src=89.160.20.156 spt=33876 dst=89.160.20.156 dpt=443 duser=alice suser=bob destinationTranslatedAddress=10.10.10.10 fileHash=bc8bbe52f041fd17318f08a0f73762ce oldFileHash=a9796280592f86b74b27e370662d41eb", - "event": { - "ingested": "2021-12-25T05:20:14.195127930Z" - }, "ecs": { - "version": "8.0.0" + "version": "8.2.0" }, + "event": { + "ingested": "2022-03-09T20:17:07.566340Z" + }, + "message": "CEF:0|Elastic|Vaporware|1.0.0-alpha|18|Authentication|low|eventId=123 src=89.160.20.156 spt=33876 dst=89.160.20.156 dpt=443 duser=alice suser=bob destinationTranslatedAddress=10.10.10.10 fileHash=bc8bbe52f041fd17318f08a0f73762ce oldFileHash=a9796280592f86b74b27e370662d41eb", "tags": [ "preserve_original_event" ] }, { - "message": "CEF:0|Elastic|Vaporware|1.0.0-alpha|18|Authentication|low|spriv=user dpriv=root", - "event": { - "ingested": "2021-12-25T05:20:14.195130507Z" - }, "ecs": { - "version": "8.0.0" + "version": "8.2.0" }, + "event": { + "ingested": "2022-03-09T20:17:07.566345800Z" + }, + "message": "CEF:0|Elastic|Vaporware|1.0.0-alpha|18|Authentication|low|spriv=user dpriv=root", "tags": [ "preserve_original_event" ] }, { - "message": "CEF:0|Elastic|Vaporware|1.0.0-alpha|18|Authentication|low|message=This event is padded with whitespace dst=192.168.1.2 src=192.168.3.4 ", - "event": { - "ingested": "2021-12-25T05:20:14.195132750Z" - }, "ecs": { - "version": "8.0.0" + "version": "8.2.0" }, + "event": { + "ingested": "2022-03-09T20:17:07.566350400Z" + }, + "message": "CEF:0|Elastic|Vaporware|1.0.0-alpha|18|Authentication|low|message=This event is padded with whitespace dst=192.168.1.2 src=192.168.3.4 ", "tags": [ "preserve_original_event" ] diff --git a/packages/cef/data_stream/log/_dev/test/pipeline/test-checkpoint.log-expected.json b/packages/cef/data_stream/log/_dev/test/pipeline/test-checkpoint.log-expected.json index 219ab1fd49e4..bcb9dc2bef5d 100644 --- a/packages/cef/data_stream/log/_dev/test/pipeline/test-checkpoint.log-expected.json +++ b/packages/cef/data_stream/log/_dev/test/pipeline/test-checkpoint.log-expected.json @@ -1,37 +1,37 @@ { "expected": [ { - "message": "CEF:0|Check Point|VPN-1 \u0026 FireWall-1|Check Point|Log|https|Unknown|act=Accept destinationTranslatedAddress=0.0.0.0 destinationTranslatedPort=0 deviceDirection=0 rt=1543270652000 sourceTranslatedAddress=192.168.103.254 sourceTranslatedPort=35398 spt=49363 dpt=443 cs2Label=Rule Name layer_name=Network layer_uuid=b406b732-2437-4848-9741-6eae1f5bf112 match_id=4 parent_rule=0 rule_action=Accept rule_uid=9e5e6e74-aa9a-4693-b9fe-53712dd27bea ifname=eth0 logid=0 loguid={0x5bfc70fc,0x1,0xfe65a8c0,0xc0000001} origin=192.168.101.254 originsicname=CN\\=R80,O\\=R80_M..6u6bdo sequencenum=1 version=5 dst=89.160.20.156 inzone=Internal nat_addtnl_rulenum=1 nat_rulenum=4 outzone=External product=VPN-1 \u0026 FireWall-1 proto=6 service_id=https src=192.168.101.100 cs5Label=Matched Category cs5=Business / Economy deviceCustomDate2=1508150533713 deviceCustomDate2Label=This field is made up", - "event": { - "ingested": "2021-12-25T05:20:14.900578496Z" - }, "ecs": { - "version": "8.0.0" + "version": "8.2.0" + }, + "event": { + "ingested": "2022-03-09T20:17:07.649902300Z" }, + "message": "CEF:0|Check Point|VPN-1 \u0026 FireWall-1|Check Point|Log|https|Unknown|act=Accept destinationTranslatedAddress=0.0.0.0 destinationTranslatedPort=0 deviceDirection=0 rt=1543270652000 sourceTranslatedAddress=192.168.103.254 sourceTranslatedPort=35398 spt=49363 dpt=443 cs2Label=Rule Name layer_name=Network layer_uuid=b406b732-2437-4848-9741-6eae1f5bf112 match_id=4 parent_rule=0 rule_action=Accept rule_uid=9e5e6e74-aa9a-4693-b9fe-53712dd27bea ifname=eth0 logid=0 loguid={0x5bfc70fc,0x1,0xfe65a8c0,0xc0000001} origin=192.168.101.254 originsicname=CN\\=R80,O\\=R80_M..6u6bdo sequencenum=1 version=5 dst=89.160.20.156 inzone=Internal nat_addtnl_rulenum=1 nat_rulenum=4 outzone=External product=VPN-1 \u0026 FireWall-1 proto=6 service_id=https src=192.168.101.100 cs5Label=Matched Category cs5=Business / Economy deviceCustomDate2=1508150533713 deviceCustomDate2Label=This field is made up", "tags": [ "preserve_original_event" ] }, { - "message": "CEF:0|Check Point|VPN-1 \u0026 FireWall-1|Check Point|Log|https|Unknown|act=Bypass cn1Label=Email Recipients Number cs1Label=Email ID cs4Label=Email Control cs4=SMTP Policy Restrictions cs5Label=Email Session ID deviceDirection=0 msg=Encrypted session rt=1545211330000 spt=4001 dpt=25 fileHash=55f4a511e6f630a6b1319505414f114e7bcaf13d deviceCustomDate2=Apr 11 2020 10:42:13 deviceCustomDate2Label=Subscription expiration", - "event": { - "ingested": "2021-12-25T05:20:14.900585817Z" - }, "ecs": { - "version": "8.0.0" + "version": "8.2.0" }, + "event": { + "ingested": "2022-03-09T20:17:07.649913Z" + }, + "message": "CEF:0|Check Point|VPN-1 \u0026 FireWall-1|Check Point|Log|https|Unknown|act=Bypass cn1Label=Email Recipients Number cs1Label=Email ID cs4Label=Email Control cs4=SMTP Policy Restrictions cs5Label=Email Session ID deviceDirection=0 msg=Encrypted session rt=1545211330000 spt=4001 dpt=25 fileHash=55f4a511e6f630a6b1319505414f114e7bcaf13d deviceCustomDate2=Apr 11 2020 10:42:13 deviceCustomDate2Label=Subscription expiration", "tags": [ "preserve_original_event" ] }, { - "message": "CEF:0|Check Point|VPN-1 \u0026 FireWall-1|Check Point|Log|https|Unknown|act=Drop cp_app_risk=High cp_severity=Very-High baseEventCount=12 deviceFacility=4 c6a2=fd00::555 c6a2Label=Source IPv6 Address c6a3=::1 c6a3Label=Destination IPv6 Address fileHash=580a783c1cb2b20613323f715d231a69 cn2=5 cn2Label=Duration in Seconds", - "event": { - "ingested": "2021-12-25T05:20:14.900587839Z" - }, "ecs": { - "version": "8.0.0" + "version": "8.2.0" + }, + "event": { + "ingested": "2022-03-09T20:17:07.649920400Z" }, + "message": "CEF:0|Check Point|VPN-1 \u0026 FireWall-1|Check Point|Log|https|Unknown|act=Drop cp_app_risk=High cp_severity=Very-High baseEventCount=12 deviceFacility=4 c6a2=fd00::555 c6a2Label=Source IPv6 Address c6a3=::1 c6a3Label=Destination IPv6 Address fileHash=580a783c1cb2b20613323f715d231a69 cn2=5 cn2Label=Duration in Seconds", "tags": [ "preserve_original_event" ] diff --git a/packages/cef/data_stream/log/_dev/test/pipeline/test-fp-ngfw-smc.log-expected.json b/packages/cef/data_stream/log/_dev/test/pipeline/test-fp-ngfw-smc.log-expected.json index ba8fee34ec41..366b336bd466 100644 --- a/packages/cef/data_stream/log/_dev/test/pipeline/test-fp-ngfw-smc.log-expected.json +++ b/packages/cef/data_stream/log/_dev/test/pipeline/test-fp-ngfw-smc.log-expected.json @@ -1,157 +1,157 @@ { "expected": [ { - "message": "CEF:0|FORCEPOINT|Firewall|6.6.1|0|Generic|0|deviceExternalId=Master FW node 1 dvc=10.1.1.40 dvchost=10.1.1.40 msg=log server connection established deviceFacility=Logging System rt=Jan 17 2020 08:52:10", - "event": { - "ingested": "2021-12-25T05:20:15.084186867Z" - }, "ecs": { - "version": "8.0.0" + "version": "8.2.0" + }, + "event": { + "ingested": "2022-03-09T20:17:07.729070500Z" }, + "message": "CEF:0|FORCEPOINT|Firewall|6.6.1|0|Generic|0|deviceExternalId=Master FW node 1 dvc=10.1.1.40 dvchost=10.1.1.40 msg=log server connection established deviceFacility=Logging System rt=Jan 17 2020 08:52:10", "tags": [ "preserve_original_event" ] }, { - "message": "CEF:0|FORCEPOINT|Firewall|6.6.1|9005|FW_Communication-Communication-Error|0|deviceExternalId=Master FW node 1 dvc=10.1.1.40 dvchost=10.1.1.40 msg=Communication error: No route to host (-3, 5, 0) deviceFacility=Management rt=Jan 17 2020 08:52:09", - "event": { - "ingested": "2021-12-25T05:20:15.084194409Z" - }, "ecs": { - "version": "8.0.0" + "version": "8.2.0" + }, + "event": { + "ingested": "2022-03-09T20:17:07.729075900Z" }, + "message": "CEF:0|FORCEPOINT|Firewall|6.6.1|9005|FW_Communication-Communication-Error|0|deviceExternalId=Master FW node 1 dvc=10.1.1.40 dvchost=10.1.1.40 msg=Communication error: No route to host (-3, 5, 0) deviceFacility=Management rt=Jan 17 2020 08:52:09", "tags": [ "preserve_original_event" ] }, { - "message": "CEF:0|FORCEPOINT|Firewall|6.6.1|70018|Connection_Allowed|0|deviceExternalId=Master FW node 1 dvc=10.1.1.40 dvchost=10.1.1.40 src=10.37.205.252 dst=10.1.1.40 proto=1 deviceOutboundInterface=255 act=Allow msg=Referred connection: 10.1.1.40 -\u003e 10.37.133.35 frag\\=0x4000 TCP 47413-\u003e3020 deviceFacility=Packet Filtering rt=Jan 17 2020 08:52:09 app=Dest. Unreachable (Host Unreachable) cs1Label=RuleID cs1=2097157.1", - "event": { - "ingested": "2021-12-25T05:20:15.084196316Z" - }, "ecs": { - "version": "8.0.0" + "version": "8.2.0" }, + "event": { + "ingested": "2022-03-09T20:17:07.729082600Z" + }, + "message": "CEF:0|FORCEPOINT|Firewall|6.6.1|70018|Connection_Allowed|0|deviceExternalId=Master FW node 1 dvc=10.1.1.40 dvchost=10.1.1.40 src=10.37.205.252 dst=10.1.1.40 proto=1 deviceOutboundInterface=255 act=Allow msg=Referred connection: 10.1.1.40 -\u003e 10.37.133.35 frag\\=0x4000 TCP 47413-\u003e3020 deviceFacility=Packet Filtering rt=Jan 17 2020 08:52:09 app=Dest. Unreachable (Host Unreachable) cs1Label=RuleID cs1=2097157.1", "tags": [ "preserve_original_event" ] }, { - "message": "CEF:0|FORCEPOINT|Firewall|unknown|70019|Connection_Discarded|0|deviceExternalId=Firewall-10 node 1 dvc=10.1.1.10 dvchost=10.1.1.10 src=172.16.1.1 dst=89.160.20.156 spt=68 dpt=67 proto=17 deviceOutboundInterface=255 deviceFacility=Packet Filtering rt=Jan 17 2020 08:56:21 app=BOOTPS (UDP) cs1Label=RuleID cs1=605.0", - "event": { - "ingested": "2021-12-25T05:20:15.084197989Z" - }, "ecs": { - "version": "8.0.0" + "version": "8.2.0" + }, + "event": { + "ingested": "2022-03-09T20:17:07.729087800Z" }, + "message": "CEF:0|FORCEPOINT|Firewall|unknown|70019|Connection_Discarded|0|deviceExternalId=Firewall-10 node 1 dvc=10.1.1.10 dvchost=10.1.1.10 src=172.16.1.1 dst=89.160.20.156 spt=68 dpt=67 proto=17 deviceOutboundInterface=255 deviceFacility=Packet Filtering rt=Jan 17 2020 08:56:21 app=BOOTPS (UDP) cs1Label=RuleID cs1=605.0", "tags": [ "preserve_original_event" ] }, { - "message": "CEF:0|FORCEPOINT|Firewall|unknown|70020|Connection_Refused|0|deviceExternalId=Firewall-1 node 1 dvc=10.1.1.1 dvchost=10.1.1.1 src=172.16.1.1 dst=192.168.1.1 proto=1 deviceOutboundInterface=255 act=Refuse deviceFacility=Packet Filtering rt=Jan 17 2020 08:56:23 app=Echo Request (No Code) cs1Label=RuleID cs1=601.0", - "event": { - "ingested": "2021-12-25T05:20:15.084199637Z" - }, "ecs": { - "version": "8.0.0" + "version": "8.2.0" }, + "event": { + "ingested": "2022-03-09T20:17:07.729093300Z" + }, + "message": "CEF:0|FORCEPOINT|Firewall|unknown|70020|Connection_Refused|0|deviceExternalId=Firewall-1 node 1 dvc=10.1.1.1 dvchost=10.1.1.1 src=172.16.1.1 dst=192.168.1.1 proto=1 deviceOutboundInterface=255 act=Refuse deviceFacility=Packet Filtering rt=Jan 17 2020 08:56:23 app=Echo Request (No Code) cs1Label=RuleID cs1=601.0", "tags": [ "preserve_original_event" ] }, { - "message": "CEF:0|FORCEPOINT|Firewall|unknown|70021|Connection_Closed|0|deviceExternalId=Firewall-6 node 1 dvc=10.1.1.6 dvchost=10.1.1.6 proto=6 deviceOutboundInterface=255 destinationServiceName=YouTube suser=alice deviceFacility=Packet Filtering rt=Jan 17 2020 08:56:20 app=TCP in=32526 out=27366", - "event": { - "ingested": "2021-12-25T05:20:15.084201294Z" - }, "ecs": { - "version": "8.0.0" + "version": "8.2.0" + }, + "event": { + "ingested": "2022-03-09T20:17:07.729098600Z" }, + "message": "CEF:0|FORCEPOINT|Firewall|unknown|70021|Connection_Closed|0|deviceExternalId=Firewall-6 node 1 dvc=10.1.1.6 dvchost=10.1.1.6 proto=6 deviceOutboundInterface=255 destinationServiceName=YouTube suser=alice deviceFacility=Packet Filtering rt=Jan 17 2020 08:56:20 app=TCP in=32526 out=27366", "tags": [ "preserve_original_event" ] }, { - "message": "CEF:0|FORCEPOINT|Firewall|unknown|72714|ECA_Metadata_login|0|deviceExternalId=Firewall-3 node 1 dvc=10.1.1.3 dvchost=10.1.1.3 src=192.168.1.1 suser=bob deviceFacility=Endpoint Context Agent rt=Jan 17 2020 08:56:33", - "event": { - "ingested": "2021-12-25T05:20:15.084202918Z" - }, "ecs": { - "version": "8.0.0" + "version": "8.2.0" }, + "event": { + "ingested": "2022-03-09T20:17:07.729104Z" + }, + "message": "CEF:0|FORCEPOINT|Firewall|unknown|72714|ECA_Metadata_login|0|deviceExternalId=Firewall-3 node 1 dvc=10.1.1.3 dvchost=10.1.1.3 src=192.168.1.1 suser=bob deviceFacility=Endpoint Context Agent rt=Jan 17 2020 08:56:33", "tags": [ "preserve_original_event" ] }, { - "message": "CEF:0|FORCEPOINT|Firewall|unknown|72715|ECA_Metadata_logout|0|deviceExternalId=Firewall-10 node 1 dvc=10.1.1.10 dvchost=10.1.1.10 src=192.168.1.1 suser=bob deviceFacility=Endpoint Context Agent rt=Jan 17 2020 08:56:31", - "event": { - "ingested": "2021-12-25T05:20:15.084204554Z" - }, "ecs": { - "version": "8.0.0" + "version": "8.2.0" + }, + "event": { + "ingested": "2022-03-09T20:17:07.729109200Z" }, + "message": "CEF:0|FORCEPOINT|Firewall|unknown|72715|ECA_Metadata_logout|0|deviceExternalId=Firewall-10 node 1 dvc=10.1.1.10 dvchost=10.1.1.10 src=192.168.1.1 suser=bob deviceFacility=Endpoint Context Agent rt=Jan 17 2020 08:56:31", "tags": [ "preserve_original_event" ] }, { - "message": "CEF:0|FORCEPOINT|Firewall|unknown|72716|ECA_Metadata_system_metadata_received|0|deviceExternalId=Firewall-8 node 1 dvc=10.1.1.8 dvchost=10.1.1.8 src=172.16.2.1 suser=alice deviceFacility=Endpoint Context Agent rt=Jan 17 2020 08:56:26", - "event": { - "ingested": "2021-12-25T05:20:15.084206222Z" - }, "ecs": { - "version": "8.0.0" + "version": "8.2.0" }, + "event": { + "ingested": "2022-03-09T20:17:07.729114200Z" + }, + "message": "CEF:0|FORCEPOINT|Firewall|unknown|72716|ECA_Metadata_system_metadata_received|0|deviceExternalId=Firewall-8 node 1 dvc=10.1.1.8 dvchost=10.1.1.8 src=172.16.2.1 suser=alice deviceFacility=Endpoint Context Agent rt=Jan 17 2020 08:56:26", "tags": [ "preserve_original_event" ] }, { - "message": "CEF:0|FORCEPOINT|Firewall|6.6.1|78002|TLS connection state|0|deviceExternalId=Master FW node 1 dvc=10.1.1.40 dvchost=10.1.1.40 msg=TLS: Couldn't establish TLS connection (11, N/A) deviceFacility=Management rt=Jan 17 2020 08:52:09", - "event": { - "ingested": "2021-12-25T05:20:15.084207816Z" - }, "ecs": { - "version": "8.0.0" + "version": "8.2.0" + }, + "event": { + "ingested": "2022-03-09T20:17:07.729119200Z" }, + "message": "CEF:0|FORCEPOINT|Firewall|6.6.1|78002|TLS connection state|0|deviceExternalId=Master FW node 1 dvc=10.1.1.40 dvchost=10.1.1.40 msg=TLS: Couldn't establish TLS connection (11, N/A) deviceFacility=Management rt=Jan 17 2020 08:52:09", "tags": [ "preserve_original_event" ] }, { - "message": "", - "event": { - "ingested": "2021-12-25T05:20:15.084209443Z" - }, "ecs": { - "version": "8.0.0" + "version": "8.2.0" + }, + "event": { + "ingested": "2022-03-09T20:17:07.729124900Z" }, + "message": "", "tags": [ "preserve_original_event" ] }, { - "message": "", - "event": { - "ingested": "2021-12-25T05:20:15.084211214Z" - }, "ecs": { - "version": "8.0.0" + "version": "8.2.0" }, + "event": { + "ingested": "2022-03-09T20:17:07.729131500Z" + }, + "message": "", "tags": [ "preserve_original_event" ] }, { - "message": "", - "event": { - "ingested": "2021-12-25T05:20:15.084212839Z" - }, "ecs": { - "version": "8.0.0" + "version": "8.2.0" + }, + "event": { + "ingested": "2022-03-09T20:17:07.729138400Z" }, + "message": "", "tags": [ "preserve_original_event" ] diff --git a/packages/cef/data_stream/log/elasticsearch/ingest_pipeline/cp-pipeline.yml b/packages/cef/data_stream/log/elasticsearch/ingest_pipeline/cp-pipeline.yml index b9c6cf1106e7..857bd2e5c282 100644 --- a/packages/cef/data_stream/log/elasticsearch/ingest_pipeline/cp-pipeline.yml +++ b/packages/cef/data_stream/log/elasticsearch/ingest_pipeline/cp-pipeline.yml @@ -120,10 +120,10 @@ processors: to: vulnerability.id - name: Recipient - to: destination.user.email + to: email.to.address - name: Sender - to: source.user.email + to: email.from.address - name: deviceCustomFloatingPoint1 labels: @@ -285,6 +285,23 @@ processors: - remove: field: _tmp_copy + - set: + field: email.to.address + value: ["{{email.to.address}}"] + if: "ctx.email?.to?.address != null" + - set: + field: email.from.address + value: ["{{email.from.address}}"] + if: "ctx.email?.from?.address != null" + - set: + field: email.subject + copy_from: checkpoint.email_subject + if: "ctx.checkpoint?.email_subject != null" + - set: + field: email.message_id + copy_from: checkpoint.email_session_id + if: "ctx.checkpoint?.email_session_id != null" + # event.duration is a string and contains seconds. Convert to long nanos. - script: params: diff --git a/packages/cef/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/cef/data_stream/log/elasticsearch/ingest_pipeline/default.yml index b00e8fa3c583..32ab30f5ee49 100644 --- a/packages/cef/data_stream/log/elasticsearch/ingest_pipeline/default.yml +++ b/packages/cef/data_stream/log/elasticsearch/ingest_pipeline/default.yml @@ -7,7 +7,7 @@ processors: value: '{{{_ingest.timestamp}}}' - set: field: ecs.version - value: '8.0.0' + value: '8.2.0' # IP Geolocation Lookup - geoip: diff --git a/packages/cef/data_stream/log/fields/ecs.yml b/packages/cef/data_stream/log/fields/ecs.yml index 9e2f1df34fcc..db280287367f 100644 --- a/packages/cef/data_stream/log/fields/ecs.yml +++ b/packages/cef/data_stream/log/fields/ecs.yml @@ -40,6 +40,12 @@ name: destination.user.name - external: ecs name: ecs.version +- external: ecs + name: email.from.address +- external: ecs + name: email.to.address +- external: ecs + name: email.subject - external: ecs name: event.ingested - external: ecs diff --git a/packages/cef/data_stream/log/fields/fields.yml b/packages/cef/data_stream/log/fields/fields.yml index 0d1a9f79ec87..7abb860e3ad9 100644 --- a/packages/cef/data_stream/log/fields/fields.yml +++ b/packages/cef/data_stream/log/fields/fields.yml @@ -60,9 +60,6 @@ - name: email_spool_id type: keyword description: Internal email spool ID. - - name: email_subject - type: keyword - description: Email subject. - name: event_count type: long description: Number of events associated with the log. diff --git a/packages/cef/data_stream/log/sample_event.json b/packages/cef/data_stream/log/sample_event.json index 5ba416d64bdb..646c57c770b3 100644 --- a/packages/cef/data_stream/log/sample_event.json +++ b/packages/cef/data_stream/log/sample_event.json @@ -27,7 +27,7 @@ "type": "logs" }, "ecs": { - "version": "8.0.0" + "version": "8.2.0" }, "elastic_agent": { "id": "4ef3d20e-66f0-4723-b86d-512327135b90", diff --git a/packages/cef/docs/README.md b/packages/cef/docs/README.md index 22a881a68461..2807edc0a497 100644 --- a/packages/cef/docs/README.md +++ b/packages/cef/docs/README.md @@ -62,8 +62,8 @@ Check Point CEF extensions are mapped as follows: | requestCookies | - | - | checkpoint.cookie | | sourceNtDomain | - | dns.question.name | - | | Signature | - | vulnerability.id | - | -| Recipient | - | destination.user.email | - | -| Sender | - | source.user.email | - | +| Recipient | - | email.to.address | - | +| Sender | - | email.from.address | - | | deviceCustomFloatingPoint1 | update version | observer.version | - | | deviceCustomIPv6Address2 | source ipv6 address | source.ip | - | | deviceCustomIPv6Address3 | destination ipv6 address | destination.ip | - | @@ -80,7 +80,7 @@ Check Point CEF extensions are mapped as follows: | deviceCustomString1 | dlp rule name | rule.name | - | | deviceCustomString1 | email id | - | checkpoint.email_id | | deviceCustomString2 | category | - | checkpoint.category | -| deviceCustomString2 | email subject | - | checkpoint.email_subject | +| deviceCustomString2 | email subject | email.subject | checkpoint.email_subject | | deviceCustomString2 | sensor mode | - | checkpoint.sensor_mode | | deviceCustomString2 | protection id | - | checkpoint.protection_id | | deviceCustomString2 | scan invoke type | - | checkpoint.integrity_av_invoke_type | @@ -108,7 +108,7 @@ Check Point CEF extensions are mapped as follows: | deviceCustomString5 | matched category | rule.category | - | | deviceCustomString5 | vlan id | network.vlan.id | - | | deviceCustomString5 | authentication method | - | checkpoint.auth_method | -| deviceCustomString5 | email session id | - | checkpoint.email_session_id | +| deviceCustomString5 | email session id | email.message_id | checkpoint.email_session_id | | deviceCustomDate2 | subscription expiration | - | checkpoint.subs_exp | | deviceFlexNumber1 | confidence | - | checkpoint.confidence_level | | deviceFlexNumber2 | performance impact | - | checkpoint.performance_impact | @@ -160,7 +160,7 @@ An example event for `log` looks as following: "type": "logs" }, "ecs": { - "version": "8.0.0" + "version": "8.2.0" }, "elastic_agent": { "id": "4ef3d20e-66f0-4723-b86d-512327135b90", @@ -312,7 +312,6 @@ An example event for `log` looks as following: | checkpoint.email_recipients_num | Number of recipients. | long | | checkpoint.email_session_id | Internal email session ID. | keyword | | checkpoint.email_spool_id | Internal email spool ID. | keyword | -| checkpoint.email_subject | Email subject. | keyword | | checkpoint.event_count | Number of events associated with the log. | long | | checkpoint.frequency | Scan frequency. | keyword | | checkpoint.icmp_code | ICMP code. | long | @@ -376,6 +375,9 @@ An example event for `log` looks as following: | destination.user.id | Unique identifier of the user. | keyword | | destination.user.name | Short name or login of the user. | keyword | | ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | +| email.from.address | The email address of the sender, typically from the RFC 5322 `From:` header field. | keyword | +| email.subject | A brief summary of the topic of the message. | keyword | +| email.to.address | The email address of recipient | keyword | | event.dataset | Event dataset | constant_keyword | | event.ingested | Timestamp when an event arrived in the central data store. This is different from `@timestamp`, which is when the event originally occurred. It's also different from `event.created`, which is meant to capture the first time an agent saw the event. In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` \< `event.created` \< `event.ingested`. | date | | event.module | Event module | constant_keyword | diff --git a/packages/cef/manifest.yml b/packages/cef/manifest.yml index 787dcc1a4c5d..f6fac3895c88 100644 --- a/packages/cef/manifest.yml +++ b/packages/cef/manifest.yml @@ -1,6 +1,6 @@ name: cef title: CEF Logs -version: 1.4.1 +version: 2.0.0 release: ga description: Collect logs from CEF Logs with Elastic Agent. type: integration From 1720b3230049e7cae96f410af42f63665c319653 Mon Sep 17 00:00:00 2001 From: Taylor Swanson Date: Tue, 15 Mar 2022 08:46:49 -0500 Subject: [PATCH 2/5] Review comments --- packages/cef/changelog.yml | 4 ++-- .../elasticsearch/ingest_pipeline/cp-pipeline.yml | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/cef/changelog.yml b/packages/cef/changelog.yml index f3f622704618..f485293a970b 100644 --- a/packages/cef/changelog.yml +++ b/packages/cef/changelog.yml @@ -1,9 +1,9 @@ # newer versions go on top - version: "2.0.0" changes: - - description: Update to ECS 8.2 + - description: Update to ECS 8.2 by modifying Check Point events to use the new email field set. type: enhancement - link: https://github.com/elastic/integrations/pull/xxxx + link: https://github.com/elastic/integrations/pull/2804 - version: "1.4.1" changes: - description: Append pipeline errors to error.message instead of overwriting existing errors. diff --git a/packages/cef/data_stream/log/elasticsearch/ingest_pipeline/cp-pipeline.yml b/packages/cef/data_stream/log/elasticsearch/ingest_pipeline/cp-pipeline.yml index 857bd2e5c282..a6d1b03e96ca 100644 --- a/packages/cef/data_stream/log/elasticsearch/ingest_pipeline/cp-pipeline.yml +++ b/packages/cef/data_stream/log/elasticsearch/ingest_pipeline/cp-pipeline.yml @@ -287,20 +287,20 @@ processors: - set: field: email.to.address - value: ["{{email.to.address}}"] - if: "ctx.email?.to?.address != null" + value: ["{{{email.to.address}}}"] + if: "ctx?.email?.to?.address != null" - set: field: email.from.address - value: ["{{email.from.address}}"] - if: "ctx.email?.from?.address != null" + value: ["{{{email.from.address}}}"] + if: "ctx?.email?.from?.address != null" - set: field: email.subject copy_from: checkpoint.email_subject - if: "ctx.checkpoint?.email_subject != null" + if: "ctx?.checkpoint?.email_subject != null" - set: field: email.message_id copy_from: checkpoint.email_session_id - if: "ctx.checkpoint?.email_session_id != null" + if: "ctx?.checkpoint?.email_session_id != null" # event.duration is a string and contains seconds. Convert to long nanos. - script: From a24065e9785770234b7f7425515a0583dd4b3499 Mon Sep 17 00:00:00 2001 From: Taylor Swanson Date: Tue, 15 Mar 2022 08:50:08 -0500 Subject: [PATCH 3/5] Remove event.ingested --- .../test/pipeline/test-cef.log-expected.json | 12 ------ .../test-checkpoint.log-expected.json | 9 ----- .../test-fp-ngfw-smc.log-expected.json | 39 ------------------- .../elasticsearch/ingest_pipeline/default.yml | 3 -- 4 files changed, 63 deletions(-) diff --git a/packages/cef/data_stream/log/_dev/test/pipeline/test-cef.log-expected.json b/packages/cef/data_stream/log/_dev/test/pipeline/test-cef.log-expected.json index 2bc33da3210c..ee5c143d445a 100644 --- a/packages/cef/data_stream/log/_dev/test/pipeline/test-cef.log-expected.json +++ b/packages/cef/data_stream/log/_dev/test/pipeline/test-cef.log-expected.json @@ -4,9 +4,6 @@ "ecs": { "version": "8.2.0" }, - "event": { - "ingested": "2022-03-09T20:17:07.566330Z" - }, "message": "CEF:0|Elastic|Vaporware|1.0.0-alpha|18|Web request|low|eventId=3457 requestMethod=POST slat=38.915 slong=-77.511 proto=TCP sourceServiceName=httpd requestContext=https://www.google.com src=89.160.20.156 spt=33876 dst=192.168.10.1 dpt=443 request=https://www.example.com/cart", "tags": [ "preserve_original_event" @@ -16,9 +13,6 @@ "ecs": { "version": "8.2.0" }, - "event": { - "ingested": "2022-03-09T20:17:07.566340Z" - }, "message": "CEF:0|Elastic|Vaporware|1.0.0-alpha|18|Authentication|low|eventId=123 src=89.160.20.156 spt=33876 dst=89.160.20.156 dpt=443 duser=alice suser=bob destinationTranslatedAddress=10.10.10.10 fileHash=bc8bbe52f041fd17318f08a0f73762ce oldFileHash=a9796280592f86b74b27e370662d41eb", "tags": [ "preserve_original_event" @@ -28,9 +22,6 @@ "ecs": { "version": "8.2.0" }, - "event": { - "ingested": "2022-03-09T20:17:07.566345800Z" - }, "message": "CEF:0|Elastic|Vaporware|1.0.0-alpha|18|Authentication|low|spriv=user dpriv=root", "tags": [ "preserve_original_event" @@ -40,9 +31,6 @@ "ecs": { "version": "8.2.0" }, - "event": { - "ingested": "2022-03-09T20:17:07.566350400Z" - }, "message": "CEF:0|Elastic|Vaporware|1.0.0-alpha|18|Authentication|low|message=This event is padded with whitespace dst=192.168.1.2 src=192.168.3.4 ", "tags": [ "preserve_original_event" diff --git a/packages/cef/data_stream/log/_dev/test/pipeline/test-checkpoint.log-expected.json b/packages/cef/data_stream/log/_dev/test/pipeline/test-checkpoint.log-expected.json index bcb9dc2bef5d..3582b158271d 100644 --- a/packages/cef/data_stream/log/_dev/test/pipeline/test-checkpoint.log-expected.json +++ b/packages/cef/data_stream/log/_dev/test/pipeline/test-checkpoint.log-expected.json @@ -4,9 +4,6 @@ "ecs": { "version": "8.2.0" }, - "event": { - "ingested": "2022-03-09T20:17:07.649902300Z" - }, "message": "CEF:0|Check Point|VPN-1 \u0026 FireWall-1|Check Point|Log|https|Unknown|act=Accept destinationTranslatedAddress=0.0.0.0 destinationTranslatedPort=0 deviceDirection=0 rt=1543270652000 sourceTranslatedAddress=192.168.103.254 sourceTranslatedPort=35398 spt=49363 dpt=443 cs2Label=Rule Name layer_name=Network layer_uuid=b406b732-2437-4848-9741-6eae1f5bf112 match_id=4 parent_rule=0 rule_action=Accept rule_uid=9e5e6e74-aa9a-4693-b9fe-53712dd27bea ifname=eth0 logid=0 loguid={0x5bfc70fc,0x1,0xfe65a8c0,0xc0000001} origin=192.168.101.254 originsicname=CN\\=R80,O\\=R80_M..6u6bdo sequencenum=1 version=5 dst=89.160.20.156 inzone=Internal nat_addtnl_rulenum=1 nat_rulenum=4 outzone=External product=VPN-1 \u0026 FireWall-1 proto=6 service_id=https src=192.168.101.100 cs5Label=Matched Category cs5=Business / Economy deviceCustomDate2=1508150533713 deviceCustomDate2Label=This field is made up", "tags": [ "preserve_original_event" @@ -16,9 +13,6 @@ "ecs": { "version": "8.2.0" }, - "event": { - "ingested": "2022-03-09T20:17:07.649913Z" - }, "message": "CEF:0|Check Point|VPN-1 \u0026 FireWall-1|Check Point|Log|https|Unknown|act=Bypass cn1Label=Email Recipients Number cs1Label=Email ID cs4Label=Email Control cs4=SMTP Policy Restrictions cs5Label=Email Session ID deviceDirection=0 msg=Encrypted session rt=1545211330000 spt=4001 dpt=25 fileHash=55f4a511e6f630a6b1319505414f114e7bcaf13d deviceCustomDate2=Apr 11 2020 10:42:13 deviceCustomDate2Label=Subscription expiration", "tags": [ "preserve_original_event" @@ -28,9 +22,6 @@ "ecs": { "version": "8.2.0" }, - "event": { - "ingested": "2022-03-09T20:17:07.649920400Z" - }, "message": "CEF:0|Check Point|VPN-1 \u0026 FireWall-1|Check Point|Log|https|Unknown|act=Drop cp_app_risk=High cp_severity=Very-High baseEventCount=12 deviceFacility=4 c6a2=fd00::555 c6a2Label=Source IPv6 Address c6a3=::1 c6a3Label=Destination IPv6 Address fileHash=580a783c1cb2b20613323f715d231a69 cn2=5 cn2Label=Duration in Seconds", "tags": [ "preserve_original_event" diff --git a/packages/cef/data_stream/log/_dev/test/pipeline/test-fp-ngfw-smc.log-expected.json b/packages/cef/data_stream/log/_dev/test/pipeline/test-fp-ngfw-smc.log-expected.json index 366b336bd466..dbc4f7cbc682 100644 --- a/packages/cef/data_stream/log/_dev/test/pipeline/test-fp-ngfw-smc.log-expected.json +++ b/packages/cef/data_stream/log/_dev/test/pipeline/test-fp-ngfw-smc.log-expected.json @@ -4,9 +4,6 @@ "ecs": { "version": "8.2.0" }, - "event": { - "ingested": "2022-03-09T20:17:07.729070500Z" - }, "message": "CEF:0|FORCEPOINT|Firewall|6.6.1|0|Generic|0|deviceExternalId=Master FW node 1 dvc=10.1.1.40 dvchost=10.1.1.40 msg=log server connection established deviceFacility=Logging System rt=Jan 17 2020 08:52:10", "tags": [ "preserve_original_event" @@ -16,9 +13,6 @@ "ecs": { "version": "8.2.0" }, - "event": { - "ingested": "2022-03-09T20:17:07.729075900Z" - }, "message": "CEF:0|FORCEPOINT|Firewall|6.6.1|9005|FW_Communication-Communication-Error|0|deviceExternalId=Master FW node 1 dvc=10.1.1.40 dvchost=10.1.1.40 msg=Communication error: No route to host (-3, 5, 0) deviceFacility=Management rt=Jan 17 2020 08:52:09", "tags": [ "preserve_original_event" @@ -28,9 +22,6 @@ "ecs": { "version": "8.2.0" }, - "event": { - "ingested": "2022-03-09T20:17:07.729082600Z" - }, "message": "CEF:0|FORCEPOINT|Firewall|6.6.1|70018|Connection_Allowed|0|deviceExternalId=Master FW node 1 dvc=10.1.1.40 dvchost=10.1.1.40 src=10.37.205.252 dst=10.1.1.40 proto=1 deviceOutboundInterface=255 act=Allow msg=Referred connection: 10.1.1.40 -\u003e 10.37.133.35 frag\\=0x4000 TCP 47413-\u003e3020 deviceFacility=Packet Filtering rt=Jan 17 2020 08:52:09 app=Dest. Unreachable (Host Unreachable) cs1Label=RuleID cs1=2097157.1", "tags": [ "preserve_original_event" @@ -40,9 +31,6 @@ "ecs": { "version": "8.2.0" }, - "event": { - "ingested": "2022-03-09T20:17:07.729087800Z" - }, "message": "CEF:0|FORCEPOINT|Firewall|unknown|70019|Connection_Discarded|0|deviceExternalId=Firewall-10 node 1 dvc=10.1.1.10 dvchost=10.1.1.10 src=172.16.1.1 dst=89.160.20.156 spt=68 dpt=67 proto=17 deviceOutboundInterface=255 deviceFacility=Packet Filtering rt=Jan 17 2020 08:56:21 app=BOOTPS (UDP) cs1Label=RuleID cs1=605.0", "tags": [ "preserve_original_event" @@ -52,9 +40,6 @@ "ecs": { "version": "8.2.0" }, - "event": { - "ingested": "2022-03-09T20:17:07.729093300Z" - }, "message": "CEF:0|FORCEPOINT|Firewall|unknown|70020|Connection_Refused|0|deviceExternalId=Firewall-1 node 1 dvc=10.1.1.1 dvchost=10.1.1.1 src=172.16.1.1 dst=192.168.1.1 proto=1 deviceOutboundInterface=255 act=Refuse deviceFacility=Packet Filtering rt=Jan 17 2020 08:56:23 app=Echo Request (No Code) cs1Label=RuleID cs1=601.0", "tags": [ "preserve_original_event" @@ -64,9 +49,6 @@ "ecs": { "version": "8.2.0" }, - "event": { - "ingested": "2022-03-09T20:17:07.729098600Z" - }, "message": "CEF:0|FORCEPOINT|Firewall|unknown|70021|Connection_Closed|0|deviceExternalId=Firewall-6 node 1 dvc=10.1.1.6 dvchost=10.1.1.6 proto=6 deviceOutboundInterface=255 destinationServiceName=YouTube suser=alice deviceFacility=Packet Filtering rt=Jan 17 2020 08:56:20 app=TCP in=32526 out=27366", "tags": [ "preserve_original_event" @@ -76,9 +58,6 @@ "ecs": { "version": "8.2.0" }, - "event": { - "ingested": "2022-03-09T20:17:07.729104Z" - }, "message": "CEF:0|FORCEPOINT|Firewall|unknown|72714|ECA_Metadata_login|0|deviceExternalId=Firewall-3 node 1 dvc=10.1.1.3 dvchost=10.1.1.3 src=192.168.1.1 suser=bob deviceFacility=Endpoint Context Agent rt=Jan 17 2020 08:56:33", "tags": [ "preserve_original_event" @@ -88,9 +67,6 @@ "ecs": { "version": "8.2.0" }, - "event": { - "ingested": "2022-03-09T20:17:07.729109200Z" - }, "message": "CEF:0|FORCEPOINT|Firewall|unknown|72715|ECA_Metadata_logout|0|deviceExternalId=Firewall-10 node 1 dvc=10.1.1.10 dvchost=10.1.1.10 src=192.168.1.1 suser=bob deviceFacility=Endpoint Context Agent rt=Jan 17 2020 08:56:31", "tags": [ "preserve_original_event" @@ -100,9 +76,6 @@ "ecs": { "version": "8.2.0" }, - "event": { - "ingested": "2022-03-09T20:17:07.729114200Z" - }, "message": "CEF:0|FORCEPOINT|Firewall|unknown|72716|ECA_Metadata_system_metadata_received|0|deviceExternalId=Firewall-8 node 1 dvc=10.1.1.8 dvchost=10.1.1.8 src=172.16.2.1 suser=alice deviceFacility=Endpoint Context Agent rt=Jan 17 2020 08:56:26", "tags": [ "preserve_original_event" @@ -112,9 +85,6 @@ "ecs": { "version": "8.2.0" }, - "event": { - "ingested": "2022-03-09T20:17:07.729119200Z" - }, "message": "CEF:0|FORCEPOINT|Firewall|6.6.1|78002|TLS connection state|0|deviceExternalId=Master FW node 1 dvc=10.1.1.40 dvchost=10.1.1.40 msg=TLS: Couldn't establish TLS connection (11, N/A) deviceFacility=Management rt=Jan 17 2020 08:52:09", "tags": [ "preserve_original_event" @@ -124,9 +94,6 @@ "ecs": { "version": "8.2.0" }, - "event": { - "ingested": "2022-03-09T20:17:07.729124900Z" - }, "message": "", "tags": [ "preserve_original_event" @@ -136,9 +103,6 @@ "ecs": { "version": "8.2.0" }, - "event": { - "ingested": "2022-03-09T20:17:07.729131500Z" - }, "message": "", "tags": [ "preserve_original_event" @@ -148,9 +112,6 @@ "ecs": { "version": "8.2.0" }, - "event": { - "ingested": "2022-03-09T20:17:07.729138400Z" - }, "message": "", "tags": [ "preserve_original_event" diff --git a/packages/cef/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/cef/data_stream/log/elasticsearch/ingest_pipeline/default.yml index 32ab30f5ee49..76808ec84051 100644 --- a/packages/cef/data_stream/log/elasticsearch/ingest_pipeline/default.yml +++ b/packages/cef/data_stream/log/elasticsearch/ingest_pipeline/default.yml @@ -2,9 +2,6 @@ description: Pipeline for CEF logs. CEF decoding happens in the Agent. This performs additional enrichment and vendor specific transformations. processors: - - set: - field: event.ingested - value: '{{{_ingest.timestamp}}}' - set: field: ecs.version value: '8.2.0' From c4a46cc635565a95d6bc7425481ccebfec151633 Mon Sep 17 00:00:00 2001 From: Taylor Swanson Date: Tue, 22 Mar 2022 08:31:05 -0500 Subject: [PATCH 4/5] Make changes be minor - Copy values to new ECS fields and retain older fields for now - Version bump is now minor - Remove obsolete expected files - Regen README --- packages/cef/changelog.yml | 2 +- .../test/pipeline/test-cef.log-expected.json | 40 ------ .../test-checkpoint.log-expected.json | 31 ----- .../test-fp-ngfw-smc.log-expected.json | 121 ------------------ .../ingest_pipeline/cp-pipeline.yml | 12 +- .../cef/data_stream/log/fields/fields.yml | 3 + packages/cef/docs/README.md | 1 + packages/cef/manifest.yml | 2 +- 8 files changed, 12 insertions(+), 200 deletions(-) delete mode 100644 packages/cef/data_stream/log/_dev/test/pipeline/test-cef.log-expected.json delete mode 100644 packages/cef/data_stream/log/_dev/test/pipeline/test-checkpoint.log-expected.json delete mode 100644 packages/cef/data_stream/log/_dev/test/pipeline/test-fp-ngfw-smc.log-expected.json diff --git a/packages/cef/changelog.yml b/packages/cef/changelog.yml index 27b1342ac512..460e7981ccf2 100644 --- a/packages/cef/changelog.yml +++ b/packages/cef/changelog.yml @@ -1,5 +1,5 @@ # newer versions go on top -- version: "2.0.0" +- version: "1.5.0" changes: - description: Update to ECS 8.2 by modifying Check Point events to use the new email field set. type: enhancement diff --git a/packages/cef/data_stream/log/_dev/test/pipeline/test-cef.log-expected.json b/packages/cef/data_stream/log/_dev/test/pipeline/test-cef.log-expected.json deleted file mode 100644 index ee5c143d445a..000000000000 --- a/packages/cef/data_stream/log/_dev/test/pipeline/test-cef.log-expected.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "expected": [ - { - "ecs": { - "version": "8.2.0" - }, - "message": "CEF:0|Elastic|Vaporware|1.0.0-alpha|18|Web request|low|eventId=3457 requestMethod=POST slat=38.915 slong=-77.511 proto=TCP sourceServiceName=httpd requestContext=https://www.google.com src=89.160.20.156 spt=33876 dst=192.168.10.1 dpt=443 request=https://www.example.com/cart", - "tags": [ - "preserve_original_event" - ] - }, - { - "ecs": { - "version": "8.2.0" - }, - "message": "CEF:0|Elastic|Vaporware|1.0.0-alpha|18|Authentication|low|eventId=123 src=89.160.20.156 spt=33876 dst=89.160.20.156 dpt=443 duser=alice suser=bob destinationTranslatedAddress=10.10.10.10 fileHash=bc8bbe52f041fd17318f08a0f73762ce oldFileHash=a9796280592f86b74b27e370662d41eb", - "tags": [ - "preserve_original_event" - ] - }, - { - "ecs": { - "version": "8.2.0" - }, - "message": "CEF:0|Elastic|Vaporware|1.0.0-alpha|18|Authentication|low|spriv=user dpriv=root", - "tags": [ - "preserve_original_event" - ] - }, - { - "ecs": { - "version": "8.2.0" - }, - "message": "CEF:0|Elastic|Vaporware|1.0.0-alpha|18|Authentication|low|message=This event is padded with whitespace dst=192.168.1.2 src=192.168.3.4 ", - "tags": [ - "preserve_original_event" - ] - } - ] -} \ No newline at end of file diff --git a/packages/cef/data_stream/log/_dev/test/pipeline/test-checkpoint.log-expected.json b/packages/cef/data_stream/log/_dev/test/pipeline/test-checkpoint.log-expected.json deleted file mode 100644 index 3582b158271d..000000000000 --- a/packages/cef/data_stream/log/_dev/test/pipeline/test-checkpoint.log-expected.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "expected": [ - { - "ecs": { - "version": "8.2.0" - }, - "message": "CEF:0|Check Point|VPN-1 \u0026 FireWall-1|Check Point|Log|https|Unknown|act=Accept destinationTranslatedAddress=0.0.0.0 destinationTranslatedPort=0 deviceDirection=0 rt=1543270652000 sourceTranslatedAddress=192.168.103.254 sourceTranslatedPort=35398 spt=49363 dpt=443 cs2Label=Rule Name layer_name=Network layer_uuid=b406b732-2437-4848-9741-6eae1f5bf112 match_id=4 parent_rule=0 rule_action=Accept rule_uid=9e5e6e74-aa9a-4693-b9fe-53712dd27bea ifname=eth0 logid=0 loguid={0x5bfc70fc,0x1,0xfe65a8c0,0xc0000001} origin=192.168.101.254 originsicname=CN\\=R80,O\\=R80_M..6u6bdo sequencenum=1 version=5 dst=89.160.20.156 inzone=Internal nat_addtnl_rulenum=1 nat_rulenum=4 outzone=External product=VPN-1 \u0026 FireWall-1 proto=6 service_id=https src=192.168.101.100 cs5Label=Matched Category cs5=Business / Economy deviceCustomDate2=1508150533713 deviceCustomDate2Label=This field is made up", - "tags": [ - "preserve_original_event" - ] - }, - { - "ecs": { - "version": "8.2.0" - }, - "message": "CEF:0|Check Point|VPN-1 \u0026 FireWall-1|Check Point|Log|https|Unknown|act=Bypass cn1Label=Email Recipients Number cs1Label=Email ID cs4Label=Email Control cs4=SMTP Policy Restrictions cs5Label=Email Session ID deviceDirection=0 msg=Encrypted session rt=1545211330000 spt=4001 dpt=25 fileHash=55f4a511e6f630a6b1319505414f114e7bcaf13d deviceCustomDate2=Apr 11 2020 10:42:13 deviceCustomDate2Label=Subscription expiration", - "tags": [ - "preserve_original_event" - ] - }, - { - "ecs": { - "version": "8.2.0" - }, - "message": "CEF:0|Check Point|VPN-1 \u0026 FireWall-1|Check Point|Log|https|Unknown|act=Drop cp_app_risk=High cp_severity=Very-High baseEventCount=12 deviceFacility=4 c6a2=fd00::555 c6a2Label=Source IPv6 Address c6a3=::1 c6a3Label=Destination IPv6 Address fileHash=580a783c1cb2b20613323f715d231a69 cn2=5 cn2Label=Duration in Seconds", - "tags": [ - "preserve_original_event" - ] - } - ] -} \ No newline at end of file diff --git a/packages/cef/data_stream/log/_dev/test/pipeline/test-fp-ngfw-smc.log-expected.json b/packages/cef/data_stream/log/_dev/test/pipeline/test-fp-ngfw-smc.log-expected.json deleted file mode 100644 index dbc4f7cbc682..000000000000 --- a/packages/cef/data_stream/log/_dev/test/pipeline/test-fp-ngfw-smc.log-expected.json +++ /dev/null @@ -1,121 +0,0 @@ -{ - "expected": [ - { - "ecs": { - "version": "8.2.0" - }, - "message": "CEF:0|FORCEPOINT|Firewall|6.6.1|0|Generic|0|deviceExternalId=Master FW node 1 dvc=10.1.1.40 dvchost=10.1.1.40 msg=log server connection established deviceFacility=Logging System rt=Jan 17 2020 08:52:10", - "tags": [ - "preserve_original_event" - ] - }, - { - "ecs": { - "version": "8.2.0" - }, - "message": "CEF:0|FORCEPOINT|Firewall|6.6.1|9005|FW_Communication-Communication-Error|0|deviceExternalId=Master FW node 1 dvc=10.1.1.40 dvchost=10.1.1.40 msg=Communication error: No route to host (-3, 5, 0) deviceFacility=Management rt=Jan 17 2020 08:52:09", - "tags": [ - "preserve_original_event" - ] - }, - { - "ecs": { - "version": "8.2.0" - }, - "message": "CEF:0|FORCEPOINT|Firewall|6.6.1|70018|Connection_Allowed|0|deviceExternalId=Master FW node 1 dvc=10.1.1.40 dvchost=10.1.1.40 src=10.37.205.252 dst=10.1.1.40 proto=1 deviceOutboundInterface=255 act=Allow msg=Referred connection: 10.1.1.40 -\u003e 10.37.133.35 frag\\=0x4000 TCP 47413-\u003e3020 deviceFacility=Packet Filtering rt=Jan 17 2020 08:52:09 app=Dest. Unreachable (Host Unreachable) cs1Label=RuleID cs1=2097157.1", - "tags": [ - "preserve_original_event" - ] - }, - { - "ecs": { - "version": "8.2.0" - }, - "message": "CEF:0|FORCEPOINT|Firewall|unknown|70019|Connection_Discarded|0|deviceExternalId=Firewall-10 node 1 dvc=10.1.1.10 dvchost=10.1.1.10 src=172.16.1.1 dst=89.160.20.156 spt=68 dpt=67 proto=17 deviceOutboundInterface=255 deviceFacility=Packet Filtering rt=Jan 17 2020 08:56:21 app=BOOTPS (UDP) cs1Label=RuleID cs1=605.0", - "tags": [ - "preserve_original_event" - ] - }, - { - "ecs": { - "version": "8.2.0" - }, - "message": "CEF:0|FORCEPOINT|Firewall|unknown|70020|Connection_Refused|0|deviceExternalId=Firewall-1 node 1 dvc=10.1.1.1 dvchost=10.1.1.1 src=172.16.1.1 dst=192.168.1.1 proto=1 deviceOutboundInterface=255 act=Refuse deviceFacility=Packet Filtering rt=Jan 17 2020 08:56:23 app=Echo Request (No Code) cs1Label=RuleID cs1=601.0", - "tags": [ - "preserve_original_event" - ] - }, - { - "ecs": { - "version": "8.2.0" - }, - "message": "CEF:0|FORCEPOINT|Firewall|unknown|70021|Connection_Closed|0|deviceExternalId=Firewall-6 node 1 dvc=10.1.1.6 dvchost=10.1.1.6 proto=6 deviceOutboundInterface=255 destinationServiceName=YouTube suser=alice deviceFacility=Packet Filtering rt=Jan 17 2020 08:56:20 app=TCP in=32526 out=27366", - "tags": [ - "preserve_original_event" - ] - }, - { - "ecs": { - "version": "8.2.0" - }, - "message": "CEF:0|FORCEPOINT|Firewall|unknown|72714|ECA_Metadata_login|0|deviceExternalId=Firewall-3 node 1 dvc=10.1.1.3 dvchost=10.1.1.3 src=192.168.1.1 suser=bob deviceFacility=Endpoint Context Agent rt=Jan 17 2020 08:56:33", - "tags": [ - "preserve_original_event" - ] - }, - { - "ecs": { - "version": "8.2.0" - }, - "message": "CEF:0|FORCEPOINT|Firewall|unknown|72715|ECA_Metadata_logout|0|deviceExternalId=Firewall-10 node 1 dvc=10.1.1.10 dvchost=10.1.1.10 src=192.168.1.1 suser=bob deviceFacility=Endpoint Context Agent rt=Jan 17 2020 08:56:31", - "tags": [ - "preserve_original_event" - ] - }, - { - "ecs": { - "version": "8.2.0" - }, - "message": "CEF:0|FORCEPOINT|Firewall|unknown|72716|ECA_Metadata_system_metadata_received|0|deviceExternalId=Firewall-8 node 1 dvc=10.1.1.8 dvchost=10.1.1.8 src=172.16.2.1 suser=alice deviceFacility=Endpoint Context Agent rt=Jan 17 2020 08:56:26", - "tags": [ - "preserve_original_event" - ] - }, - { - "ecs": { - "version": "8.2.0" - }, - "message": "CEF:0|FORCEPOINT|Firewall|6.6.1|78002|TLS connection state|0|deviceExternalId=Master FW node 1 dvc=10.1.1.40 dvchost=10.1.1.40 msg=TLS: Couldn't establish TLS connection (11, N/A) deviceFacility=Management rt=Jan 17 2020 08:52:09", - "tags": [ - "preserve_original_event" - ] - }, - { - "ecs": { - "version": "8.2.0" - }, - "message": "", - "tags": [ - "preserve_original_event" - ] - }, - { - "ecs": { - "version": "8.2.0" - }, - "message": "", - "tags": [ - "preserve_original_event" - ] - }, - { - "ecs": { - "version": "8.2.0" - }, - "message": "", - "tags": [ - "preserve_original_event" - ] - } - ] -} \ No newline at end of file diff --git a/packages/cef/data_stream/log/elasticsearch/ingest_pipeline/cp-pipeline.yml b/packages/cef/data_stream/log/elasticsearch/ingest_pipeline/cp-pipeline.yml index 29666878907b..ddca09121102 100644 --- a/packages/cef/data_stream/log/elasticsearch/ingest_pipeline/cp-pipeline.yml +++ b/packages/cef/data_stream/log/elasticsearch/ingest_pipeline/cp-pipeline.yml @@ -120,10 +120,10 @@ processors: to: vulnerability.id - name: Recipient - to: email.to.address + to: destination.user.email - name: Sender - to: email.from.address + to: source.user.email - name: deviceCustomFloatingPoint1 labels: @@ -287,12 +287,12 @@ processors: - set: field: email.to.address - value: ["{{{email.to.address}}}"] - if: "ctx?.email?.to?.address != null" + value: ["{{{destination.user.email}}}"] + if: "ctx?.destination?.user?.email != null" - set: field: email.from.address - value: ["{{{email.from.address}}}"] - if: "ctx?.email?.from?.address != null" + value: ["{{{source.user.email}}}"] + if: "ctx?.source?.user?.email != null" - set: field: email.subject copy_from: checkpoint.email_subject diff --git a/packages/cef/data_stream/log/fields/fields.yml b/packages/cef/data_stream/log/fields/fields.yml index 7abb860e3ad9..0d1a9f79ec87 100644 --- a/packages/cef/data_stream/log/fields/fields.yml +++ b/packages/cef/data_stream/log/fields/fields.yml @@ -60,6 +60,9 @@ - name: email_spool_id type: keyword description: Internal email spool ID. + - name: email_subject + type: keyword + description: Email subject. - name: event_count type: long description: Number of events associated with the log. diff --git a/packages/cef/docs/README.md b/packages/cef/docs/README.md index 81660e309e6a..54dc75eae338 100644 --- a/packages/cef/docs/README.md +++ b/packages/cef/docs/README.md @@ -312,6 +312,7 @@ An example event for `log` looks as following: | checkpoint.email_recipients_num | Number of recipients. | long | | checkpoint.email_session_id | Internal email session ID. | keyword | | checkpoint.email_spool_id | Internal email spool ID. | keyword | +| checkpoint.email_subject | Email subject. | keyword | | checkpoint.event_count | Number of events associated with the log. | long | | checkpoint.frequency | Scan frequency. | keyword | | checkpoint.icmp_code | ICMP code. | long | diff --git a/packages/cef/manifest.yml b/packages/cef/manifest.yml index f6fac3895c88..b1f905ebeb89 100644 --- a/packages/cef/manifest.yml +++ b/packages/cef/manifest.yml @@ -1,6 +1,6 @@ name: cef title: CEF Logs -version: 2.0.0 +version: 1.5.0 release: ga description: Collect logs from CEF Logs with Elastic Agent. type: integration From 94a269fe9a261f0c7d7672bee09a9e2d5b97a8c6 Mon Sep 17 00:00:00 2001 From: Taylor Swanson Date: Tue, 5 Apr 2022 09:26:29 -0500 Subject: [PATCH 5/5] Regen files after merge --- packages/cef/docs/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/cef/docs/README.md b/packages/cef/docs/README.md index b398bfb5320c..d5d17a86d44f 100644 --- a/packages/cef/docs/README.md +++ b/packages/cef/docs/README.md @@ -380,6 +380,7 @@ An example event for `log` looks as following: | ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | email.from.address | The email address of the sender, typically from the RFC 5322 `From:` header field. | keyword | | email.subject | A brief summary of the topic of the message. | keyword | +| email.subject.text | Multi-field of `email.subject`. | match_only_text | | email.to.address | The email address of recipient | keyword | | event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | | event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword |