diff --git a/packages/f5_bigip/changelog.yml b/packages/f5_bigip/changelog.yml index b2ca095397a..9ddebe926f4 100644 --- a/packages/f5_bigip/changelog.yml +++ b/packages/f5_bigip/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.28.0" + changes: + - description: Handle Bot Defense and DoS event with quoted device_product value. + type: enhancement + link: https://github.com/elastic/integrations/pull/18890 - version: "1.27.3" changes: - description: Update README to clarify Bot Defense and DoS event handling. diff --git a/packages/f5_bigip/data_stream/log/_dev/test/pipeline/test-pipeline-bigip-afm.log-expected.json b/packages/f5_bigip/data_stream/log/_dev/test/pipeline/test-pipeline-bigip-afm.log-expected.json index 1b43c1384b8..3a396ee1c2c 100644 --- a/packages/f5_bigip/data_stream/log/_dev/test/pipeline/test-pipeline-bigip-afm.log-expected.json +++ b/packages/f5_bigip/data_stream/log/_dev/test/pipeline/test-pipeline-bigip-afm.log-expected.json @@ -453,4 +453,4 @@ ] } ] -} \ No newline at end of file +} diff --git a/packages/f5_bigip/data_stream/log/_dev/test/pipeline/test-pipeline-bigip-apm.log-expected.json b/packages/f5_bigip/data_stream/log/_dev/test/pipeline/test-pipeline-bigip-apm.log-expected.json index 1a080385473..bf40f0852f4 100644 --- a/packages/f5_bigip/data_stream/log/_dev/test/pipeline/test-pipeline-bigip-apm.log-expected.json +++ b/packages/f5_bigip/data_stream/log/_dev/test/pipeline/test-pipeline-bigip-apm.log-expected.json @@ -342,4 +342,4 @@ } } ] -} \ No newline at end of file +} diff --git a/packages/f5_bigip/data_stream/log/_dev/test/pipeline/test-pipeline-bigip-asm.log-expected.json b/packages/f5_bigip/data_stream/log/_dev/test/pipeline/test-pipeline-bigip-asm.log-expected.json index a6ef8955224..019b2e00cf5 100644 --- a/packages/f5_bigip/data_stream/log/_dev/test/pipeline/test-pipeline-bigip-asm.log-expected.json +++ b/packages/f5_bigip/data_stream/log/_dev/test/pipeline/test-pipeline-bigip-asm.log-expected.json @@ -766,4 +766,4 @@ ] } ] -} \ No newline at end of file +} diff --git a/packages/f5_bigip/data_stream/log/_dev/test/pipeline/test-pipeline-bigip-avr.log-expected.json b/packages/f5_bigip/data_stream/log/_dev/test/pipeline/test-pipeline-bigip-avr.log-expected.json index 3b040f1de0b..3b99b3a13d4 100644 --- a/packages/f5_bigip/data_stream/log/_dev/test/pipeline/test-pipeline-bigip-avr.log-expected.json +++ b/packages/f5_bigip/data_stream/log/_dev/test/pipeline/test-pipeline-bigip-avr.log-expected.json @@ -744,4 +744,4 @@ ] } ] -} \ No newline at end of file +} diff --git a/packages/f5_bigip/data_stream/log/_dev/test/pipeline/test-pipeline-bigip-dos-and-bot.log b/packages/f5_bigip/data_stream/log/_dev/test/pipeline/test-pipeline-bigip-dos-and-bot.log index 2040a2339ce..67665857033 100644 --- a/packages/f5_bigip/data_stream/log/_dev/test/pipeline/test-pipeline-bigip-dos-and-bot.log +++ b/packages/f5_bigip/data_stream/log/_dev/test/pipeline/test-pipeline-bigip-dos-and-bot.log @@ -1,2 +1,3 @@ hostname="asm.example.com",bigip_mgmt_ip="10.10.100.100",bigip_mgmt_ip2="::",client_ip="192.168.10.10",client_ip_geo_location="NA",client_port="54389",client_request_uri="/test/picture.jpg",configuration_date_time="Apr 15 2018 11:30:26",context_name="/Common/v1",context_type="Virtual Server",dest_ip="10.10.10.10",dest_port="80",device_product="Application Security Module",device_vendor="F5",device_version="no_pgo x86_64 padc TMM Version 0.0.0.0.0.0",errdefs_msgno="23003147",http_method="GET",http_protocol_indication="HTTP",route_domain="0",timestamp="Apr 15 2018 18:32:36",virtual_server_name="/Common/v1",device_id="4734097073bff",request_date_time="Apr 15 2018 11:32:36",profile_name="/Common/bot-defense",support_id="3161892955527053449",request_status="illegal",action="undetermined",reason="",previous_action="NA",previous_support_id="3161892955527053433",previous_request_date_time="Apr 15 2018 11:32:36",bot_signature="",bot_signature_category="",bot_name="Presenting as CHROME",session_id="8549049561352296353",class="Suspicious Browser",anomaly_categories="Suspicious Browsers and Extensions",anomalies="Suspicious HTTP Headers Presence or Order",additional_bot_signatures="",micro_service_name="",micro_service_type="N/A",micro_service_matched_wildcard_url="",configured_mitigation_action="CAPTCHA",configured_mitigation_action_reason="/Common/Suspicious HTTP Headers Presence or Order",actual_mitigation_action="Alarm",actual_mitigation_action_reason="CAPTCHA valid",browser_configured_verification_action="Verify after Access (Blocking)",browser_actual_verification_action="Challenge-Free Verification",browser_actual_verification_action_reason="URL Not Qualified for Injection",captcha_status="Correct CAPTCHA Challenge Answer",browser_verification_status="None",device_id_status="Device ID Is Valid",device_id_action="None",previous_initiated_action="HTTP 307 redirect to the same domain",previous_initiated_action_status="Valid",classification_reason="NA",client_type="Browser",application_display_name="",application_version="",mobile_in_emulation_mode="NA",os_name="NA",jailbroken_or_rooted_device="NA",imei="NA",human_behaviour="NA",http_request="GET /test/picture.jpg HTTP/1.1\r\nHost: 10.10.10.10\r\nConnection: keep-alive\r\nPragma: no-cache\r\nCache-Control: no-cache\r\nUser-Agent: Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36\r\nAccept: image/webp,image/apng,image/*,*/*;q=0.8\r\nReferer: http://10.10.10.20/test.php\r\nAccept-Encoding: gzip, deflate\r\nAccept-Language: en-US,en;q=0.9\r\nCookie: TSPD_101_R0=08d4cb6559ab20004e07e455224b442635febc4bfd400cf12348476d6c5348915bbd05a9456945e9086f640741142800b55a0e04163a367ba2782565a133955ee09ead23f9b0c84da482a1191ca53624a2bfd23b83d58590; TSe82ee8e4076=08d4cb6559ab2800647a6b8f5174fa4d117dc35be98cfad9b381201bbf35732267060629f224f65324567ec0fcca30ba086a800c2b09e800cfb30067be291566a1e2d84c768dc6e536ac52c9a4de7165f17a887228eb0eb6089982af41b9283d6ba32bed2da397ca38d3a89a7d88bdf9cee3702abe62bd541be5fef1261da1bb9cc67809a40b7b4279a3c5d7ceb92135042c03803e96aeb3f1df4a0dae85eeb28a90bf29eb5ff89c4e28a057e98d280e2d7029d90848b9ffdff469863790a706a1236fdf21ee05a2a3090745d09f144d9a5b41f94bd82f0b705f87786113ad42a8060b47be91fe00da527a4766c19bd055212d902114be23d1d5851552a609c62b8b423276bd6fe7f6f03554ca79e141402add2f17fe0055e1d1c4b98f9178c3; TSe82ee8e4077=08d4cb6559ab2800647a6b8f5174fa4d117dc35be98cfad9b381201bbf35732267060629f22325678b5d7ec0fcca30ba086a800c2b17180019367ecd339e5ecdf1ed0f21e443c527622b755de5e5970d;TSPD_101_DID=08d4cb6559ab2800647a6b8f5174fa4d117dc35be98cfad9b381201bbf35443267060629f224f65aab5d7ec0fcca30ba086a800c2b06380042193d5e0ebcc52ddeac6779552cfb347b37163f8c670444e3afe3cdefbc44b886970c27c4ac8a943279d5558fea9a6ca6c141054ddf8c5e; TSe82ee8e4073=08d4cb6559ab2800647a6b8f5174fa4d117dc35be98cfad9b381201bbf35733908670629f224f65aab5d7ec0fcca30ba086a800c2b021800ee042b654fbac3e9108ae78264b721a99e4d973e35aa4c3c:086a800c2b0c1800bf61079db313f0ffd28df8c27561a3f0db295765d2bd1313;TS0f9815ea027=08d4cb6559ab200041b71ae213e47cf5a8c7b1c26a79c7ff49295e7142f62c14234568b32def3a7a0800d6fc831128001535352e47895a92b8278e1c9306950f50771797c8ede03b81af7d940f5afc272cb77365117040b9\r\n\r\n" -action=Blocking;hostname=ziv-45-196.f5net.com;bigip_mgmt_ip=10.10.10.10;client_ip_geo_location=N/A;client_request_uri=N/A;configuration_date_time=Nov 23 2016 02:03:02;context_name=/Common/v1;context_type=Virtual Server;date_time=Nov 23 2016 02:03:22;device_product=ASM;device_vendor=F5;device_version=13.0.0;device_blade=0;dos_attack_detection_mode=TPS Increased;dos_attack_event=Suspicious entity;dos_attack_id=2843816221;dos_attack_latency=test;dos_attack_name=DOS L7 attack;dos_attack_tps=4 tps;dos_baseline_latency=test;dos_baseline_tps=4 tps;dos_baseline_traffic_percent=test;dos_current_traffic_percent=test;dos_dropped_requests_count=12;dos_incoming_requests_count=27;dos_mitigation_action=Source IP-Based Rate Limiting;dos_mitigation_reason=Abnormal volume;errdefs_msgno=23003140;errdefs_msg_name=Application DoS Event;severity=3;partition_name=Common;profile_name=/Common/dos;reported_entity_type=Source IP;source_ip=10.10.10.10;timestamp=Nov 23 2016 07:03:22;device_id=test;event_id=0;dos_detection_threshold=1 tps;dos_mitigate_to_threshold=1 tps;dos_detection_condition=Absolute Manual Threshold; \ No newline at end of file +action=Blocking;hostname=ziv-45-196.f5net.com;bigip_mgmt_ip=10.10.10.10;client_ip_geo_location=N/A;client_request_uri=N/A;configuration_date_time=Nov 23 2016 02:03:02;context_name=/Common/v1;context_type=Virtual Server;date_time=Nov 23 2016 02:03:22;device_product=ASM;device_vendor=F5;device_version=13.0.0;device_blade=0;dos_attack_detection_mode=TPS Increased;dos_attack_event=Suspicious entity;dos_attack_id=2843816221;dos_attack_latency=test;dos_attack_name=DOS L7 attack;dos_attack_tps=4 tps;dos_baseline_latency=test;dos_baseline_tps=4 tps;dos_baseline_traffic_percent=test;dos_current_traffic_percent=test;dos_dropped_requests_count=12;dos_incoming_requests_count=27;dos_mitigation_action=Source IP-Based Rate Limiting;dos_mitigation_reason=Abnormal volume;errdefs_msgno=23003140;errdefs_msg_name=Application DoS Event;severity=3;partition_name=Common;profile_name=/Common/dos;reported_entity_type=Source IP;source_ip=10.10.10.10;timestamp=Nov 23 2016 07:03:22;device_id=test;event_id=0;dos_detection_threshold=1 tps;dos_mitigate_to_threshold=1 tps;dos_detection_condition=Absolute Manual Threshold; +action=Blocking;hostname=ziv-45-196.f5net.com;bigip_mgmt_ip=10.10.10.10;client_ip_geo_location=N/A;client_request_uri=N/A;configuration_date_time=Nov 23 2016 02:03:02;context_name=/Common/v1;context_type=Virtual Server;date_time=Nov 23 2016 02:03:22;device_product="ASM";device_vendor=F5;device_version=13.0.0;device_blade=0;dos_attack_detection_mode=TPS Increased;dos_attack_event=Suspicious entity;dos_attack_id=2843816221;dos_attack_latency=test;dos_attack_name=DOS L7 attack;dos_attack_tps=4 tps;dos_baseline_latency=test;dos_baseline_tps=4 tps;dos_baseline_traffic_percent=test;dos_current_traffic_percent=test;dos_dropped_requests_count=12;dos_incoming_requests_count=27;dos_mitigation_action=Source IP-Based Rate Limiting;dos_mitigation_reason=Abnormal volume;errdefs_msgno=23003140;errdefs_msg_name=Application DoS Event;severity=3;partition_name=Common;profile_name=/Common/dos;reported_entity_type=Source IP;source_ip=10.10.10.10;timestamp=Nov 23 2016 07:03:22;device_id=test;event_id=0;dos_detection_threshold=1 tps;dos_mitigate_to_threshold=1 tps;dos_detection_condition=Absolute Manual Threshold; \ No newline at end of file diff --git a/packages/f5_bigip/data_stream/log/_dev/test/pipeline/test-pipeline-bigip-dos-and-bot.log-expected.json b/packages/f5_bigip/data_stream/log/_dev/test/pipeline/test-pipeline-bigip-dos-and-bot.log-expected.json index f612bbfe54a..ad313258b7a 100644 --- a/packages/f5_bigip/data_stream/log/_dev/test/pipeline/test-pipeline-bigip-dos-and-bot.log-expected.json +++ b/packages/f5_bigip/data_stream/log/_dev/test/pipeline/test-pipeline-bigip-dos-and-bot.log-expected.json @@ -306,6 +306,130 @@ "preserve_original_event", "preserve_duplicate_custom_fields" ] + }, + { + "@timestamp": "2016-11-23T07:03:22.000Z", + "device": { + "id": "test" + }, + "ecs": { + "version": "8.11.0" + }, + "error": { + "id": "23003140" + }, + "event": { + "action": "blocking", + "category": [ + "network" + ], + "created": "2016-11-23T02:03:02.000Z", + "id": "0", + "kind": "alert", + "original": "action=Blocking;hostname=ziv-45-196.f5net.com;bigip_mgmt_ip=10.10.10.10;client_ip_geo_location=N/A;client_request_uri=N/A;configuration_date_time=Nov 23 2016 02:03:02;context_name=/Common/v1;context_type=Virtual Server;date_time=Nov 23 2016 02:03:22;device_product=\"ASM\";device_vendor=F5;device_version=13.0.0;device_blade=0;dos_attack_detection_mode=TPS Increased;dos_attack_event=Suspicious entity;dos_attack_id=2843816221;dos_attack_latency=test;dos_attack_name=DOS L7 attack;dos_attack_tps=4 tps;dos_baseline_latency=test;dos_baseline_tps=4 tps;dos_baseline_traffic_percent=test;dos_current_traffic_percent=test;dos_dropped_requests_count=12;dos_incoming_requests_count=27;dos_mitigation_action=Source IP-Based Rate Limiting;dos_mitigation_reason=Abnormal volume;errdefs_msgno=23003140;errdefs_msg_name=Application DoS Event;severity=3;partition_name=Common;profile_name=/Common/dos;reported_entity_type=Source IP;source_ip=10.10.10.10;timestamp=Nov 23 2016 07:03:22;device_id=test;event_id=0;dos_detection_threshold=1 tps;dos_mitigate_to_threshold=1 tps;dos_detection_condition=Absolute Manual Threshold;", + "severity": 3, + "start": "2016-11-23T02:03:22.000Z", + "type": [ + "info" + ] + }, + "f5_bigip": { + "log": { + "action": "Blocking", + "bigip_management": { + "ip": "10.10.10.10" + }, + "configuration_date_time": "2016-11-23T02:03:02.000Z", + "context": { + "name": "/Common/v1", + "type": "Virtual Server" + }, + "date_time": "2016-11-23T02:03:22.000Z", + "device": { + "blade": "0", + "id": "test", + "product": "ASM", + "vendor": "F5", + "version": "13.0.0" + }, + "dos": { + "attack": { + "detection_mode": "TPS Increased", + "event": "Suspicious entity", + "id": "2843816221", + "latency": "test", + "name": "DOS L7 attack", + "tps": "4 tps" + }, + "baseline": { + "latency": "test", + "tps": "4 tps", + "traffic_percent": "test" + }, + "current_traffic_percent": "test", + "dropped_requests_count": 12, + "incoming_requests_count": 27 + }, + "dos_detection": { + "condition": "Absolute Manual Threshold", + "threshold": "1 tps" + }, + "dos_mitigate_to_threshold": "1 tps", + "dos_mitigation": { + "action": "Source IP-Based Rate Limiting", + "reason": "Abnormal volume" + }, + "errdefs": { + "msg_name": "Application DoS Event", + "msgno": "23003140" + }, + "event": { + "id": "0" + }, + "hostname": "ziv-45-196.f5net.com", + "http": { + "request": "ng;hostname=ziv-45-196.f5net.com;bigip_mgmt_ip=10.10.10.10;client_ip_geo_location=N/A;client_request_uri=N/A;configuration_date_time=Nov 23 2016 02:03:02;context_name=/Common/v1;context_type=Virtual Server;date_time=Nov 23 2016 02:03:22;device_product=" + }, + "partition_name": "Common", + "profile_name": "/Common/dos", + "reported_entity_type": "Source IP", + "severity": { + "code": 3 + }, + "source": { + "ip": "10.10.10.10" + }, + "timestamp": "2016-11-23T07:03:22.000Z" + } + }, + "host": { + "hostname": "ziv-45-196.f5net.com", + "ip": [ + "10.10.10.10" + ] + }, + "observer": { + "product": "ASM", + "vendor": "F5", + "version": "13.0.0" + }, + "related": { + "hosts": [ + "ziv-45-196.f5net.com" + ], + "ip": [ + "10.10.10.10" + ] + }, + "source": { + "ip": [ + "10.10.10.10" + ] + }, + "tags": [ + "preserve_original_event", + "preserve_duplicate_custom_fields" + ] } ] -} \ No newline at end of file +} diff --git a/packages/f5_bigip/data_stream/log/_dev/test/pipeline/test-pipeline-bigip-ihealth.log-expected.json b/packages/f5_bigip/data_stream/log/_dev/test/pipeline/test-pipeline-bigip-ihealth.log-expected.json index b287a6029d0..3981262d283 100644 --- a/packages/f5_bigip/data_stream/log/_dev/test/pipeline/test-pipeline-bigip-ihealth.log-expected.json +++ b/packages/f5_bigip/data_stream/log/_dev/test/pipeline/test-pipeline-bigip-ihealth.log-expected.json @@ -771,4 +771,4 @@ } } ] -} \ No newline at end of file +} diff --git a/packages/f5_bigip/data_stream/log/_dev/test/pipeline/test-pipeline-bigip-ltm.log-expected.json b/packages/f5_bigip/data_stream/log/_dev/test/pipeline/test-pipeline-bigip-ltm.log-expected.json index 1f3de19566d..3be25416826 100644 --- a/packages/f5_bigip/data_stream/log/_dev/test/pipeline/test-pipeline-bigip-ltm.log-expected.json +++ b/packages/f5_bigip/data_stream/log/_dev/test/pipeline/test-pipeline-bigip-ltm.log-expected.json @@ -334,4 +334,4 @@ ] } ] -} \ No newline at end of file +} diff --git a/packages/f5_bigip/data_stream/log/_dev/test/pipeline/test-pipeline-bigip-system.log-expected.json b/packages/f5_bigip/data_stream/log/_dev/test/pipeline/test-pipeline-bigip-system.log-expected.json index ea93e08140a..7c6b57f590d 100644 --- a/packages/f5_bigip/data_stream/log/_dev/test/pipeline/test-pipeline-bigip-system.log-expected.json +++ b/packages/f5_bigip/data_stream/log/_dev/test/pipeline/test-pipeline-bigip-system.log-expected.json @@ -475,4 +475,4 @@ ] } ] -} \ No newline at end of file +} diff --git a/packages/f5_bigip/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/f5_bigip/data_stream/log/elasticsearch/ingest_pipeline/default.yml index 264dd3f6eaf..0ccee2953fe 100644 --- a/packages/f5_bigip/data_stream/log/elasticsearch/ingest_pipeline/default.yml +++ b/packages/f5_bigip/data_stream/log/elasticsearch/ingest_pipeline/default.yml @@ -23,7 +23,8 @@ processors: if: >- ctx.event?.original != null && !( ctx.event.original.contains('device_product="Application Security Module"') || - ctx.event.original.contains('device_product=ASM') + ctx.event.original.contains('device_product=ASM') || + ctx.event.original.contains('device_product="ASM"') ) on_failure: - append: @@ -115,6 +116,7 @@ processors: if: >- ctx.event?.original != null && ( ctx.event.original.contains('device_product="Application Security Module"') || + ctx.event.original.contains('device_product="ASM"') || ctx.event.original.contains('device_product=ASM') ) - script: diff --git a/packages/f5_bigip/data_stream/log/elasticsearch/ingest_pipeline/pipeline_bigip_bot_and_dos.yml b/packages/f5_bigip/data_stream/log/elasticsearch/ingest_pipeline/pipeline_bigip_bot_and_dos.yml index 5d6809c8cd0..02e75946a13 100644 --- a/packages/f5_bigip/data_stream/log/elasticsearch/ingest_pipeline/pipeline_bigip_bot_and_dos.yml +++ b/packages/f5_bigip/data_stream/log/elasticsearch/ingest_pipeline/pipeline_bigip_bot_and_dos.yml @@ -16,7 +16,8 @@ processors: field_split: ';' value_split: = target_field: kv - if: ctx.event.original.contains('device_product=ASM') + strip_brackets: true + if: ctx.event.original.contains('device_product=ASM') || ctx.event.original.contains('device_product="ASM"') on_failure: - append: field: error.message diff --git a/packages/f5_bigip/manifest.yml b/packages/f5_bigip/manifest.yml index 7bd35734349..d45b45135bc 100644 --- a/packages/f5_bigip/manifest.yml +++ b/packages/f5_bigip/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.2" name: f5_bigip title: F5 BIG-IP -version: "1.27.3" +version: "1.28.0" description: Collect logs from F5 BIG-IP with Elastic Agent. type: integration categories: