cisco_meraki: remove incorrect event.category:threat and event.type:i…

…ndicator values (#8508)
elastic · Nov 23, 2023 · 05d757c · 05d757c
1 parent a19f4a5
commit 05d757c
Show file tree

Hide file tree

Showing 11 changed files with 431 additions and 1,763 deletions.
diff --git a/packages/cisco_meraki/changelog.yml b/packages/cisco_meraki/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: 1.20.1
+  changes:
+    - description: Remove incorrect event.category:threat and event.type:indicator values.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/8508
 - version: 1.20.0
   changes:
     - description: Record port state changes.

diff --git a/.../cisco_meraki/data_stream/log/_dev/test/pipeline/test-airmarshal-events.log-expected.json b/.../cisco_meraki/data_stream/log/_dev/test/pipeline/test-airmarshal-events.log-expected.json
diff --git a/packages/cisco_meraki/data_stream/log/_dev/test/pipeline/test-generated.log-expected.json b/packages/cisco_meraki/data_stream/log/_dev/test/pipeline/test-generated.log-expected.json
diff --git a/...es/cisco_meraki/data_stream/log/_dev/test/pipeline/test-security-events.log-expected.json b/...es/cisco_meraki/data_stream/log/_dev/test/pipeline/test-security-events.log-expected.json
@@ -23,14 +23,14 @@
                 "action": "ids-signature-matched",
                 "category": [
                     "network",
-                    "threat"
+                    "intrusion_detection"
                 ],
                 "original": "<134>1 1637691198.348361125 MX84 security_event ids_alerted signature=1:29708:4 priority=1 timestamp=1637691198.330873 dhost=D0:AB:D5:7B:43:73 direction=ingress protocol=tcp/ip src=67.43.156.12:80 dst=10.0.3.162:56391 decision=allowed message: BROWSER-IE Microsoft Internet Explorer CSS uninitialized object access attempt detected",
                 "type": [
-                    "info",
-                    "indicator"
+                    "info"
                 ]
             },
+            "message": "BROWSER-IE Microsoft Internet Explorer CSS uninitialized object access attempt detected",
             "network": {
                 "direction": "ingress",
                 "protocol": "tcp/ip"
@@ -57,13 +57,7 @@
             "tags": [
                 "forwarded",
                 "preserve_original_event"
-            ],
-            "threat": {
-                "indicator": {
-                    "description": "BROWSER-IE Microsoft Internet Explorer CSS uninitialized object access attempt detected",
-                    "last_seen": "2021-11-23T18:13:18.330Z"
-                }
-            }
+            ]
         },
         {
             "@timestamp": "2023-10-23T12:58:11.323Z",
@@ -88,14 +82,14 @@
                 "action": "ids-signature-matched",
                 "category": [
                     "network",
-                    "threat"
+                    "intrusion_detection"
                 ],
                 "original": "<134>1 1698065891.323413683 MX84 security_event ids_alerted signature=1:45749:2 priority=1 timestamp=1698065891.322786 direction=ingress protocol=tcp/ip src=67.43.156.12:80 dst=10.0.3.162:56391 decision=blocked action=allow message: SERVER-WEBAPP PHPUnit PHP remote code execution attempt",
                 "type": [
-                    "info",
-                    "indicator"
+                    "info"
                 ]
             },
+            "message": "SERVER-WEBAPP PHPUnit PHP remote code execution attempt",
             "network": {
                 "direction": "ingress",
                 "protocol": "tcp/ip"
@@ -122,13 +116,7 @@
             "tags": [
                 "forwarded",
                 "preserve_original_event"
-            ],
-            "threat": {
-                "indicator": {
-                    "description": "SERVER-WEBAPP PHPUnit PHP remote code execution attempt",
-                    "last_seen": "2023-10-23T12:58:11.322Z"
-                }
-            }
+            ]
         },
         {
             "@timestamp": "2021-11-23T18:14:58.984Z",
@@ -164,16 +152,20 @@
                 "action": "malicious-file-actioned",
                 "category": [
                     "network",
-                    "threat",
+                    "file",
                     "malware"
                 ],
                 "original": "<134>1 1637691298.984398273 MX84 security_event security_filtering_file_scanned url=http://www.eicar.org/download/eicar.com.txt src=192.168.128.2:53150 dst=67.43.156.15:80 mac=98:5A:EB:E1:81:2F name='EICAR:EICAR_Test_file_not_a_virus-tpd' sha256=275a021bbfb6489e54d471899f7db9d1663fc695ec2fe2a2c4538aabf651fd0f disposition=malicious action=block",
                 "type": [
-                    "info",
-                    "indicator",
                     "info"
                 ]
             },
+            "file": {
+                "hash": {
+                    "sha256": "275a021bbfb6489e54d471899f7db9d1663fc695ec2fe2a2c4538aabf651fd0f"
+                },
+                "name": "EICAR:EICAR_Test_file_not_a_virus-tpd"
+            },
             "observer": {
                 "hostname": "MX84"
             },
@@ -185,16 +177,12 @@
                 "forwarded",
                 "preserve_original_event"
             ],
-            "threat": {
-                "indicator": {
-                    "file": {
-                        "hash": {
-                            "sha256": "275a021bbfb6489e54d471899f7db9d1663fc695ec2fe2a2c4538aabf651fd0f"
-                        },
-                        "name": "EICAR:EICAR_Test_file_not_a_virus-tpd"
-                    },
-                    "reference": "http://www.eicar.org/download/eicar.com.txt"
-                }
+            "url": {
+                "domain": "www.eicar.org",
+                "extension": "txt",
+                "original": "http://www.eicar.org/download/eicar.com.txt",
+                "path": "/download/eicar.com.txt",
+                "scheme": "http"
             }
         },
         {
@@ -214,33 +202,27 @@
                 "action": "issued-retrospective-malicious-disposition",
                 "category": [
                     "network",
-                    "threat",
+                    "file",
                     "malware"
                 ],
                 "original": "<134>1 1637783435.239819833 MX84 security_event security_filtering_disposition_change name=EICAR:EICAR_Test_file_not_a_virus-tpd sha256=275a021bbfb6489e54d471899f7db9d1663fc695ec2fe2a2c4538aabf651fd0f disposition=malicious action=allow",
                 "type": [
-                    "info",
-                    "indicator",
                     "info"
                 ]
             },
+            "file": {
+                "hash": {
+                    "sha256": "275a021bbfb6489e54d471899f7db9d1663fc695ec2fe2a2c4538aabf651fd0f"
+                },
+                "name": "EICAR:EICAR_Test_file_not_a_virus-tpd"
+            },
             "observer": {
                 "hostname": "MX84"
             },
             "tags": [
                 "forwarded",
                 "preserve_original_event"
-            ],
-            "threat": {
-                "indicator": {
-                    "file": {
-                        "hash": {
-                            "sha256": "275a021bbfb6489e54d471899f7db9d1663fc695ec2fe2a2c4538aabf651fd0f"
-                        },
-                        "name": "EICAR:EICAR_Test_file_not_a_virus-tpd"
-                    }
-                }
-            }
+            ]
         },
         {
             "@timestamp": "2021-11-24T19:58:11.345Z",
@@ -259,12 +241,11 @@
                 "action": "ids-signature-matched",
                 "category": [
                     "network",
-                    "threat"
+                    "intrusion_detection"
                 ],
                 "original": "<134>1 1637783891.345984502 MX84 ids-alerts signature=129:4:1 priority=3 timestamp=1637783891.512569 direction=ingress protocol=tcp/ip src=67.43.156.15:80",
                 "type": [
-                    "info",
-                    "indicator"
+                    "info"
                 ]
             },
             "network": {
@@ -293,12 +274,7 @@
             "tags": [
                 "forwarded",
                 "preserve_original_event"
-            ],
-            "threat": {
-                "indicator": {
-                    "last_seen": "2021-11-24T19:58:11.512Z"
-                }
-            }
+            ]
         },
         {
             "@timestamp": "2021-11-24T21:43:21.246Z",
@@ -317,12 +293,11 @@
                 "action": "ids-signature-matched",
                 "category": [
                     "network",
-                    "threat"
+                    "intrusion_detection"
                 ],
                 "original": "<134>1 1637790201.246576346 MX84 ids-alerts signature=119:15:1 priority=2 timestamp=1637790201.238064 direction=egress protocol=tcp/ip src=192.168.111.254:56240",
                 "type": [
-                    "info",
-                    "indicator"
+                    "info"
                 ]
             },
             "network": {
@@ -339,12 +314,7 @@
             "tags": [
                 "forwarded",
                 "preserve_original_event"
-            ],
-            "threat": {
-                "indicator": {
-                    "last_seen": "2021-11-24T21:43:21.238Z"
-                }
-            }
+            ]
         }
     ]
 }
diff --git a/packages/cisco_meraki/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/cisco_meraki/data_stream/log/elasticsearch/ingest_pipeline/default.yml
@@ -122,25 +122,17 @@ processors:
             action: http-access-error
           "ids_alerted":
             category:
-              - threat
-            type:
-              - indicator
+              - intrusion_detection
             action: ids-signature-matched
           "security_filtering_file_scanned":
             category:
-              - threat
+              - file
               - malware
-            type:
-              - indicator
-              - info
             action: malicious-file-actioned
           "security_filtering_disposition_change":
             category:
-              - threat
+              - file
               - malware
-            type:
-              - indicator
-              - info
             action: issued-retrospective-malicious-disposition
           "association":
             type:
@@ -195,22 +187,10 @@ processors:
               - start
             action: splash-authentication
           "device_packet_flood":
-            category:
-              - threat
-            type:
-              - indicator
             action: wireless-packet-flood-detected
           "rogue_ssid_detected":
-            category:
-              - threat
-            type:
-              - indicator
             action: rogue-ssid-detected
           "ssid_spoofing_detected":
-            category:
-              - threat
-            type:
-              - indicator
             action: ssid-spoofing-detected
           "multiple_dhcp_servers_detected":
             type:

diff --git a/packages/cisco_meraki/data_stream/log/elasticsearch/ingest_pipeline/idsalerts.yml b/packages/cisco_meraki/data_stream/log/elasticsearch/ingest_pipeline/idsalerts.yml
@@ -14,10 +14,6 @@ processors:
 - rename:
     field: signature
     target_field: cisco_meraki.security.signature
-- date:
-    field: timestamp
-    target_field: threat.indicator.last_seen
-    formats: ['UNIX']
 - rename:
     field: direction
     target_field: network.direction

diff --git a/packages/cisco_meraki/data_stream/log/elasticsearch/ingest_pipeline/security.yml b/packages/cisco_meraki/data_stream/log/elasticsearch/ingest_pipeline/security.yml
@@ -29,11 +29,6 @@ processors:
     field: signature
     target_field: cisco_meraki.security.signature
     ignore_missing: true
-- date:
-    field: timestamp
-    target_field: threat.indicator.last_seen
-    formats: ['UNIX']
-    if: ctx.timestamp != null
 - gsub:
     field: dhost
     target_field: cisco_meraki.security.dhost
@@ -48,21 +43,16 @@ processors:
     field: protocol
     target_field: network.protocol
     ignore_missing: true
-- rename:
-    field: message
-    target_field: threat.indicator.description
-    ignore_missing: true
-    if: ctx?.cisco_meraki?.event_subtype == 'ids_alerted'
 - rename:
     field: decision
     target_field: cisco_meraki.security.decision
     ignore_missing: true
 
 # handle fields of security_filtering_file_scanned or security_filtering_disposition_change type
-- rename:
+- uri_parts:
     field: url
-    target_field: threat.indicator.reference
-    ignore_missing: true
+    if: ctx.url != null
+    ignore_failure: true
 - gsub:
     field: mac
     target_field: cisco_meraki.security.mac
@@ -71,11 +61,11 @@ processors:
     ignore_missing: true
 - rename:
     field: name
-    target_field: threat.indicator.file.name
+    target_field: file.name
     ignore_missing: true
 - rename:
     field: sha256
-    target_field: threat.indicator.file.hash.sha256
+    target_field: file.hash.sha256
     ignore_missing: true
 - rename:
     field: disposition

diff --git a/packages/cisco_meraki/data_stream/log/fields/ecs.yml b/packages/cisco_meraki/data_stream/log/fields/ecs.yml
@@ -86,6 +86,8 @@
   name: file.directory
 - external: ecs
   name: file.extension
+- external: ecs
+  name: file.hash.sha256
 - external: ecs
   name: file.name
 - external: ecs
@@ -220,6 +222,8 @@
   name: source.subdomain
 - external: ecs
   name: source.top_level_domain
+- external: ecs
+  name: url.extension
 - external: ecs
   name: url.domain
 - external: ecs
@@ -230,6 +234,8 @@
   name: url.query
 - external: ecs
   name: url.registered_domain
+- external: ecs
+  name: url.scheme
 - external: ecs
   name: url.top_level_domain
 - external: ecs
@@ -274,16 +280,6 @@
   name: source.geo.region_name
 - external: ecs
   name: network.vlan.id
-- external: ecs
-  name: threat.indicator.last_seen
-- external: ecs
-  name: threat.indicator.description
-- external: ecs
-  name: threat.indicator.reference
-- external: ecs
-  name: threat.indicator.file.name
-- external: ecs
-  name: threat.indicator.file.hash.sha256
 - external: ecs
   name: client.geo.city_name
 - external: ecs