f5_bigip: handle Bot Defense and DoS event with quoted device_product value

packages/f5_bigip/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.28.0"
+  changes:
+    - description: Handle Bot Defense and DoS event with quoted device_product value.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/18890
 - version: "1.27.3"
   changes:
     - description: Update README to clarify Bot Defense and DoS event handling.

packages/f5_bigip/data_stream/log/_dev/test/pipeline/test-pipeline-bigip-afm.log-expected.json

@@ -453,4 +453,4 @@
             ]
         }
     ]
-}
+}

packages/f5_bigip/data_stream/log/_dev/test/pipeline/test-pipeline-bigip-apm.log-expected.json

@@ -342,4 +342,4 @@
             }
         }
     ]
-}
+}

packages/f5_bigip/data_stream/log/_dev/test/pipeline/test-pipeline-bigip-asm.log-expected.json

@@ -766,4 +766,4 @@
             ]
         }
     ]
-}
+}

packages/f5_bigip/data_stream/log/_dev/test/pipeline/test-pipeline-bigip-avr.log-expected.json

@@ -744,4 +744,4 @@
             ]
         }
     ]
-}
+}

packages/f5_bigip/data_stream/log/_dev/test/pipeline/test-pipeline-bigip-dos-and-bot.log

@@ -1,2 +1,3 @@
 hostname="asm.example.com",bigip_mgmt_ip="10.10.100.100",bigip_mgmt_ip2="::",client_ip="192.168.10.10",client_ip_geo_location="NA",client_port="54389",client_request_uri="/test/picture.jpg",configuration_date_time="Apr 15 2018 11:30:26",context_name="/Common/v1",context_type="Virtual Server",dest_ip="10.10.10.10",dest_port="80",device_product="Application Security Module",device_vendor="F5",device_version="no_pgo x86_64 padc TMM Version 0.0.0.0.0.0",errdefs_msgno="23003147",http_method="GET",http_protocol_indication="HTTP",route_domain="0",timestamp="Apr 15 2018 18:32:36",virtual_server_name="/Common/v1",device_id="4734097073bff",request_date_time="Apr 15 2018 11:32:36",profile_name="/Common/bot-defense",support_id="3161892955527053449",request_status="illegal",action="undetermined",reason="",previous_action="NA",previous_support_id="3161892955527053433",previous_request_date_time="Apr 15 2018 11:32:36",bot_signature="",bot_signature_category="",bot_name="Presenting as CHROME",session_id="8549049561352296353",class="Suspicious Browser",anomaly_categories="Suspicious Browsers and Extensions",anomalies="Suspicious HTTP Headers Presence or Order",additional_bot_signatures="",micro_service_name="",micro_service_type="N/A",micro_service_matched_wildcard_url="",configured_mitigation_action="CAPTCHA",configured_mitigation_action_reason="/Common/Suspicious HTTP Headers Presence or Order",actual_mitigation_action="Alarm",actual_mitigation_action_reason="CAPTCHA valid",browser_configured_verification_action="Verify after Access (Blocking)",browser_actual_verification_action="Challenge-Free Verification",browser_actual_verification_action_reason="URL Not Qualified for Injection",captcha_status="Correct CAPTCHA Challenge Answer",browser_verification_status="None",device_id_status="Device ID Is Valid",device_id_action="None",previous_initiated_action="HTTP 307 redirect to the same domain",previous_initiated_action_status="Valid",classification_reason="NA",client_type="Browser",application_display_name="",application_version="",mobile_in_emulation_mode="NA",os_name="NA",jailbroken_or_rooted_device="NA",imei="NA",human_behaviour="NA",http_request="GET /test/picture.jpg HTTP/1.1\r\nHost: 10.10.10.10\r\nConnection: keep-alive\r\nPragma: no-cache\r\nCache-Control: no-cache\r\nUser-Agent: Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36\r\nAccept: image/webp,image/apng,image/*,*/*;q=0.8\r\nReferer: http://10.10.10.20/test.php\r\nAccept-Encoding: gzip, deflate\r\nAccept-Language: en-US,en;q=0.9\r\nCookie: TSPD_101_R0=08d4cb6559ab20004e07e455224b442635febc4bfd400cf12348476d6c5348915bbd05a9456945e9086f640741142800b55a0e04163a367ba2782565a133955ee09ead23f9b0c84da482a1191ca53624a2bfd23b83d58590; TSe82ee8e4076=08d4cb6559ab2800647a6b8f5174fa4d117dc35be98cfad9b381201bbf35732267060629f224f65324567ec0fcca30ba086a800c2b09e800cfb30067be291566a1e2d84c768dc6e536ac52c9a4de7165f17a887228eb0eb6089982af41b9283d6ba32bed2da397ca38d3a89a7d88bdf9cee3702abe62bd541be5fef1261da1bb9cc67809a40b7b4279a3c5d7ceb92135042c03803e96aeb3f1df4a0dae85eeb28a90bf29eb5ff89c4e28a057e98d280e2d7029d90848b9ffdff469863790a706a1236fdf21ee05a2a3090745d09f144d9a5b41f94bd82f0b705f87786113ad42a8060b47be91fe00da527a4766c19bd055212d902114be23d1d5851552a609c62b8b423276bd6fe7f6f03554ca79e141402add2f17fe0055e1d1c4b98f9178c3; TSe82ee8e4077=08d4cb6559ab2800647a6b8f5174fa4d117dc35be98cfad9b381201bbf35732267060629f22325678b5d7ec0fcca30ba086a800c2b17180019367ecd339e5ecdf1ed0f21e443c527622b755de5e5970d;TSPD_101_DID=08d4cb6559ab2800647a6b8f5174fa4d117dc35be98cfad9b381201bbf35443267060629f224f65aab5d7ec0fcca30ba086a800c2b06380042193d5e0ebcc52ddeac6779552cfb347b37163f8c670444e3afe3cdefbc44b886970c27c4ac8a943279d5558fea9a6ca6c141054ddf8c5e; TSe82ee8e4073=08d4cb6559ab2800647a6b8f5174fa4d117dc35be98cfad9b381201bbf35733908670629f224f65aab5d7ec0fcca30ba086a800c2b021800ee042b654fbac3e9108ae78264b721a99e4d973e35aa4c3c:086a800c2b0c1800bf61079db313f0ffd28df8c27561a3f0db295765d2bd1313;TS0f9815ea027=08d4cb6559ab200041b71ae213e47cf5a8c7b1c26a79c7ff49295e7142f62c14234568b32def3a7a0800d6fc831128001535352e47895a92b8278e1c9306950f50771797c8ede03b81af7d940f5afc272cb77365117040b9\r\n\r\n"
-action=Blocking;hostname=ziv-45-196.f5net.com;bigip_mgmt_ip=10.10.10.10;client_ip_geo_location=N/A;client_request_uri=N/A;configuration_date_time=Nov 23 2016 02:03:02;context_name=/Common/v1;context_type=Virtual Server;date_time=Nov 23 2016 02:03:22;device_product=ASM;device_vendor=F5;device_version=13.0.0;device_blade=0;dos_attack_detection_mode=TPS Increased;dos_attack_event=Suspicious entity;dos_attack_id=2843816221;dos_attack_latency=test;dos_attack_name=DOS L7 attack;dos_attack_tps=4 tps;dos_baseline_latency=test;dos_baseline_tps=4 tps;dos_baseline_traffic_percent=test;dos_current_traffic_percent=test;dos_dropped_requests_count=12;dos_incoming_requests_count=27;dos_mitigation_action=Source IP-Based Rate Limiting;dos_mitigation_reason=Abnormal volume;errdefs_msgno=23003140;errdefs_msg_name=Application DoS Event;severity=3;partition_name=Common;profile_name=/Common/dos;reported_entity_type=Source IP;source_ip=10.10.10.10;timestamp=Nov 23 2016 07:03:22;device_id=test;event_id=0;dos_detection_threshold=1 tps;dos_mitigate_to_threshold=1 tps;dos_detection_condition=Absolute Manual Threshold;
+action=Blocking;hostname=ziv-45-196.f5net.com;bigip_mgmt_ip=10.10.10.10;client_ip_geo_location=N/A;client_request_uri=N/A;configuration_date_time=Nov 23 2016 02:03:02;context_name=/Common/v1;context_type=Virtual Server;date_time=Nov 23 2016 02:03:22;device_product=ASM;device_vendor=F5;device_version=13.0.0;device_blade=0;dos_attack_detection_mode=TPS Increased;dos_attack_event=Suspicious entity;dos_attack_id=2843816221;dos_attack_latency=test;dos_attack_name=DOS L7 attack;dos_attack_tps=4 tps;dos_baseline_latency=test;dos_baseline_tps=4 tps;dos_baseline_traffic_percent=test;dos_current_traffic_percent=test;dos_dropped_requests_count=12;dos_incoming_requests_count=27;dos_mitigation_action=Source IP-Based Rate Limiting;dos_mitigation_reason=Abnormal volume;errdefs_msgno=23003140;errdefs_msg_name=Application DoS Event;severity=3;partition_name=Common;profile_name=/Common/dos;reported_entity_type=Source IP;source_ip=10.10.10.10;timestamp=Nov 23 2016 07:03:22;device_id=test;event_id=0;dos_detection_threshold=1 tps;dos_mitigate_to_threshold=1 tps;dos_detection_condition=Absolute Manual Threshold;
+action=Blocking;hostname=ziv-45-196.f5net.com;bigip_mgmt_ip=10.10.10.10;client_ip_geo_location=N/A;client_request_uri=N/A;configuration_date_time=Nov 23 2016 02:03:02;context_name=/Common/v1;context_type=Virtual Server;date_time=Nov 23 2016 02:03:22;device_product="ASM";device_vendor=F5;device_version=13.0.0;device_blade=0;dos_attack_detection_mode=TPS Increased;dos_attack_event=Suspicious entity;dos_attack_id=2843816221;dos_attack_latency=test;dos_attack_name=DOS L7 attack;dos_attack_tps=4 tps;dos_baseline_latency=test;dos_baseline_tps=4 tps;dos_baseline_traffic_percent=test;dos_current_traffic_percent=test;dos_dropped_requests_count=12;dos_incoming_requests_count=27;dos_mitigation_action=Source IP-Based Rate Limiting;dos_mitigation_reason=Abnormal volume;errdefs_msgno=23003140;errdefs_msg_name=Application DoS Event;severity=3;partition_name=Common;profile_name=/Common/dos;reported_entity_type=Source IP;source_ip=10.10.10.10;timestamp=Nov 23 2016 07:03:22;device_id=test;event_id=0;dos_detection_threshold=1 tps;dos_mitigate_to_threshold=1 tps;dos_detection_condition=Absolute Manual Threshold;

packages/f5_bigip/data_stream/log/_dev/test/pipeline/test-pipeline-bigip-dos-and-bot.log-expected.json

@@ -306,6 +306,130 @@
                 "preserve_original_event",
                 "preserve_duplicate_custom_fields"
             ]
+        },
+        {
+            "@timestamp": "2016-11-23T07:03:22.000Z",
+            "device": {
+                "id": "test"
+            },
+            "ecs": {
+                "version": "8.11.0"
+            },
+            "error": {
+                "id": "23003140"
+            },
+            "event": {
+                "action": "blocking",
+                "category": [
+                    "network"
+                ],
+                "created": "2016-11-23T02:03:02.000Z",
+                "id": "0",
+                "kind": "alert",
+                "original": "action=Blocking;hostname=ziv-45-196.f5net.com;bigip_mgmt_ip=10.10.10.10;client_ip_geo_location=N/A;client_request_uri=N/A;configuration_date_time=Nov 23 2016 02:03:02;context_name=/Common/v1;context_type=Virtual Server;date_time=Nov 23 2016 02:03:22;device_product=\"ASM\";device_vendor=F5;device_version=13.0.0;device_blade=0;dos_attack_detection_mode=TPS Increased;dos_attack_event=Suspicious entity;dos_attack_id=2843816221;dos_attack_latency=test;dos_attack_name=DOS L7 attack;dos_attack_tps=4 tps;dos_baseline_latency=test;dos_baseline_tps=4 tps;dos_baseline_traffic_percent=test;dos_current_traffic_percent=test;dos_dropped_requests_count=12;dos_incoming_requests_count=27;dos_mitigation_action=Source IP-Based Rate Limiting;dos_mitigation_reason=Abnormal volume;errdefs_msgno=23003140;errdefs_msg_name=Application DoS Event;severity=3;partition_name=Common;profile_name=/Common/dos;reported_entity_type=Source IP;source_ip=10.10.10.10;timestamp=Nov 23 2016 07:03:22;device_id=test;event_id=0;dos_detection_threshold=1 tps;dos_mitigate_to_threshold=1 tps;dos_detection_condition=Absolute Manual Threshold;",
+                "severity": 3,
+                "start": "2016-11-23T02:03:22.000Z",
+                "type": [
+                    "info"
+                ]
+            },
+            "f5_bigip": {
+                "log": {
+                    "action": "Blocking",
+                    "bigip_management": {
+                        "ip": "10.10.10.10"
+                    },
+                    "configuration_date_time": "2016-11-23T02:03:02.000Z",
+                    "context": {
+                        "name": "/Common/v1",
+                        "type": "Virtual Server"
+                    },
+                    "date_time": "2016-11-23T02:03:22.000Z",
+                    "device": {
+                        "blade": "0",
+                        "id": "test",
+                        "product": "ASM",
+                        "vendor": "F5",
+                        "version": "13.0.0"
+                    },
+                    "dos": {
+                        "attack": {
+                            "detection_mode": "TPS Increased",
+                            "event": "Suspicious entity",
+                            "id": "2843816221",
+                            "latency": "test",
+                            "name": "DOS L7 attack",
+                            "tps": "4 tps"
+                        },
+                        "baseline": {
+                            "latency": "test",
+                            "tps": "4 tps",
+                            "traffic_percent": "test"
+                        },
+                        "current_traffic_percent": "test",
+                        "dropped_requests_count": 12,
+                        "incoming_requests_count": 27
+                    },
+                    "dos_detection": {
+                        "condition": "Absolute Manual Threshold",
+                        "threshold": "1 tps"
+                    },
+                    "dos_mitigate_to_threshold": "1 tps",
+                    "dos_mitigation": {
+                        "action": "Source IP-Based Rate Limiting",
+                        "reason": "Abnormal volume"
+                    },
+                    "errdefs": {
+                        "msg_name": "Application DoS Event",
+                        "msgno": "23003140"
+                    },
+                    "event": {
+                        "id": "0"
+                    },
+                    "hostname": "ziv-45-196.f5net.com",
+                    "http": {
+                        "request": "ng;hostname=ziv-45-196.f5net.com;bigip_mgmt_ip=10.10.10.10;client_ip_geo_location=N/A;client_request_uri=N/A;configuration_date_time=Nov 23 2016 02:03:02;context_name=/Common/v1;context_type=Virtual Server;date_time=Nov 23 2016 02:03:22;device_product="
+                    },
+                    "partition_name": "Common",
+                    "profile_name": "/Common/dos",
+                    "reported_entity_type": "Source IP",
+                    "severity": {
+                        "code": 3
+                    },
+                    "source": {
+                        "ip": "10.10.10.10"
+                    },
+                    "timestamp": "2016-11-23T07:03:22.000Z"
+                }
+            },
+            "host": {
+                "hostname": "ziv-45-196.f5net.com",
+                "ip": [
+                    "10.10.10.10"
+                ]
+            },
+            "observer": {
+                "product": "ASM",
+                "vendor": "F5",
+                "version": "13.0.0"
+            },
+            "related": {
+                "hosts": [
+                    "ziv-45-196.f5net.com"
+                ],
+                "ip": [
+                    "10.10.10.10"
+                ]
+            },
+            "source": {
+                "ip": [
+                    "10.10.10.10"
+                ]
+            },
+            "tags": [
+                "preserve_original_event",
+                "preserve_duplicate_custom_fields"
+            ]
         }
     ]
-}
+}

packages/f5_bigip/data_stream/log/_dev/test/pipeline/test-pipeline-bigip-ihealth.log-expected.json

@@ -771,4 +771,4 @@
             }
         }
     ]
-}
+}

packages/f5_bigip/data_stream/log/_dev/test/pipeline/test-pipeline-bigip-ltm.log-expected.json

@@ -334,4 +334,4 @@
             ]
         }
     ]
-}
+}

packages/f5_bigip/data_stream/log/_dev/test/pipeline/test-pipeline-bigip-system.log-expected.json

@@ -475,4 +475,4 @@
             ]
         }
     ]
-}
+}

packages/f5_bigip/data_stream/log/elasticsearch/ingest_pipeline/default.yml

@@ -23,7 +23,8 @@ processors:
       if: >-
         ctx.event?.original != null && !(
           ctx.event.original.contains('device_product="Application Security Module"') ||
-          ctx.event.original.contains('device_product=ASM')
+          ctx.event.original.contains('device_product=ASM') ||
+          ctx.event.original.contains('device_product="ASM"')
         )
       on_failure:
         - append:
@@ -115,6 +116,7 @@ processors:
       if: >-
         ctx.event?.original != null && (
           ctx.event.original.contains('device_product="Application Security Module"') ||
+          ctx.event.original.contains('device_product="ASM"') ||
           ctx.event.original.contains('device_product=ASM')
         )
   - script:

packages/f5_bigip/data_stream/log/elasticsearch/ingest_pipeline/pipeline_bigip_bot_and_dos.yml

@@ -16,7 +16,8 @@ processors:
       field_split: ';'
       value_split: =
       target_field: kv
-      if: ctx.event.original.contains('device_product=ASM')
+      strip_brackets: true
+      if: ctx.event.original.contains('device_product=ASM') || ctx.event.original.contains('device_product="ASM"')
       on_failure:
         - append:
             field: error.message

packages/f5_bigip/manifest.yml

@@ -1,7 +1,7 @@
 format_version: "3.0.2"
 name: f5_bigip
 title: F5 BIG-IP
-version: "1.27.3"
+version: "1.28.0"
 description: Collect logs from F5 BIG-IP with Elastic Agent.
 type: integration
 categories: