address pr comments

elastic · Nov 21, 2023 · 5895aa5 · 5895aa5
1 parent 282b934
commit 5895aa5
Show file tree

Hide file tree

Showing 7 changed files with 55 additions and 1,012 deletions.
diff --git a/packages/cisco_meraki/data_stream/log/_dev/test/pipeline/test-generated.log-expected.json b/packages/cisco_meraki/data_stream/log/_dev/test/pipeline/test-generated.log-expected.json
diff --git a/...es/cisco_meraki/data_stream/log/_dev/test/pipeline/test-security-events.log-expected.json b/...es/cisco_meraki/data_stream/log/_dev/test/pipeline/test-security-events.log-expected.json
@@ -30,6 +30,7 @@
                     "info"
                 ]
             },
+            "message": "BROWSER-IE Microsoft Internet Explorer CSS uninitialized object access attempt detected",
             "network": {
                 "direction": "ingress",
                 "protocol": "tcp/ip"
@@ -56,13 +57,7 @@
             "tags": [
                 "forwarded",
                 "preserve_original_event"
-            ],
-            "threat": {
-                "indicator": {
-                    "description": "BROWSER-IE Microsoft Internet Explorer CSS uninitialized object access attempt detected",
-                    "last_seen": "2021-11-23T18:13:18.330Z"
-                }
-            }
+            ]
         },
         {
             "@timestamp": "2023-10-23T12:58:11.323Z",
@@ -94,6 +89,7 @@
                     "info"
                 ]
             },
+            "message": "SERVER-WEBAPP PHPUnit PHP remote code execution attempt",
             "network": {
                 "direction": "ingress",
                 "protocol": "tcp/ip"
@@ -120,13 +116,7 @@
             "tags": [
                 "forwarded",
                 "preserve_original_event"
-            ],
-            "threat": {
-                "indicator": {
-                    "description": "SERVER-WEBAPP PHPUnit PHP remote code execution attempt",
-                    "last_seen": "2023-10-23T12:58:11.322Z"
-                }
-            }
+            ]
         },
         {
             "@timestamp": "2021-11-23T18:14:58.984Z",
@@ -170,6 +160,12 @@
                     "info"
                 ]
             },
+            "file": {
+                "hash": {
+                    "sha256": "275a021bbfb6489e54d471899f7db9d1663fc695ec2fe2a2c4538aabf651fd0f"
+                },
+                "name": "EICAR:EICAR_Test_file_not_a_virus-tpd"
+            },
             "observer": {
                 "hostname": "MX84"
             },
@@ -181,16 +177,12 @@
                 "forwarded",
                 "preserve_original_event"
             ],
-            "threat": {
-                "indicator": {
-                    "file": {
-                        "hash": {
-                            "sha256": "275a021bbfb6489e54d471899f7db9d1663fc695ec2fe2a2c4538aabf651fd0f"
-                        },
-                        "name": "EICAR:EICAR_Test_file_not_a_virus-tpd"
-                    },
-                    "reference": "http://www.eicar.org/download/eicar.com.txt"
-                }
+            "url": {
+                "domain": "www.eicar.org",
+                "extension": "txt",
+                "original": "http://www.eicar.org/download/eicar.com.txt",
+                "path": "/download/eicar.com.txt",
+                "scheme": "http"
             }
         },
         {
@@ -218,23 +210,19 @@
                     "info"
                 ]
             },
+            "file": {
+                "hash": {
+                    "sha256": "275a021bbfb6489e54d471899f7db9d1663fc695ec2fe2a2c4538aabf651fd0f"
+                },
+                "name": "EICAR:EICAR_Test_file_not_a_virus-tpd"
+            },
             "observer": {
                 "hostname": "MX84"
             },
             "tags": [
                 "forwarded",
                 "preserve_original_event"
-            ],
-            "threat": {
-                "indicator": {
-                    "file": {
-                        "hash": {
-                            "sha256": "275a021bbfb6489e54d471899f7db9d1663fc695ec2fe2a2c4538aabf651fd0f"
-                        },
-                        "name": "EICAR:EICAR_Test_file_not_a_virus-tpd"
-                    }
-                }
-            }
+            ]
         },
         {
             "@timestamp": "2021-11-24T19:58:11.345Z",
@@ -286,12 +274,7 @@
             "tags": [
                 "forwarded",
                 "preserve_original_event"
-            ],
-            "threat": {
-                "indicator": {
-                    "last_seen": "2021-11-24T19:58:11.512Z"
-                }
-            }
+            ]
         },
         {
             "@timestamp": "2021-11-24T21:43:21.246Z",
@@ -331,12 +314,7 @@
             "tags": [
                 "forwarded",
                 "preserve_original_event"
-            ],
-            "threat": {
-                "indicator": {
-                    "last_seen": "2021-11-24T21:43:21.238Z"
-                }
-            }
+            ]
         }
     ]
 }
diff --git a/packages/cisco_meraki/data_stream/log/elasticsearch/ingest_pipeline/idsalerts.yml b/packages/cisco_meraki/data_stream/log/elasticsearch/ingest_pipeline/idsalerts.yml
@@ -14,10 +14,6 @@ processors:
 - rename:
     field: signature
     target_field: cisco_meraki.security.signature
-- date:
-    field: timestamp
-    target_field: threat.indicator.last_seen
-    formats: ['UNIX']
 - rename:
     field: direction
     target_field: network.direction

diff --git a/packages/cisco_meraki/data_stream/log/elasticsearch/ingest_pipeline/security.yml b/packages/cisco_meraki/data_stream/log/elasticsearch/ingest_pipeline/security.yml
@@ -29,11 +29,6 @@ processors:
     field: signature
     target_field: cisco_meraki.security.signature
     ignore_missing: true
-- date:
-    field: timestamp
-    target_field: threat.indicator.last_seen
-    formats: ['UNIX']
-    if: ctx.timestamp != null
 - gsub:
     field: dhost
     target_field: cisco_meraki.security.dhost
@@ -48,20 +43,14 @@ processors:
     field: protocol
     target_field: network.protocol
     ignore_missing: true
-- rename:
-    field: message
-    target_field: threat.indicator.description
-    ignore_missing: true
-    if: ctx?.cisco_meraki?.event_subtype == 'ids_alerted'
 - rename:
     field: decision
     target_field: cisco_meraki.security.decision
     ignore_missing: true
 
 # handle fields of security_filtering_file_scanned or security_filtering_disposition_change type
-- rename:
+- uri_parts:
     field: url
-    target_field: threat.indicator.reference
     ignore_missing: true
 - gsub:
     field: mac
@@ -71,11 +60,11 @@ processors:
     ignore_missing: true
 - rename:
     field: name
-    target_field: threat.indicator.file.name
+    target_field: file.name
     ignore_missing: true
 - rename:
     field: sha256
-    target_field: threat.indicator.file.hash.sha256
+    target_field: file.hash.sha256
     ignore_missing: true
 - rename:
     field: disposition

diff --git a/packages/cisco_meraki/data_stream/log/fields/ecs.yml b/packages/cisco_meraki/data_stream/log/fields/ecs.yml
@@ -86,6 +86,8 @@
   name: file.directory
 - external: ecs
   name: file.extension
+- external: ecs
+  name: file.hash.sha256
 - external: ecs
   name: file.name
 - external: ecs
@@ -220,6 +222,8 @@
   name: source.subdomain
 - external: ecs
   name: source.top_level_domain
+- external: ecs
+  name: url.extension
 - external: ecs
   name: url.domain
 - external: ecs
@@ -230,6 +234,8 @@
   name: url.query
 - external: ecs
   name: url.registered_domain
+- external: ecs
+  name: url.scheme
 - external: ecs
   name: url.top_level_domain
 - external: ecs
@@ -274,16 +280,6 @@
   name: source.geo.region_name
 - external: ecs
   name: network.vlan.id
-- external: ecs
-  name: threat.indicator.last_seen
-- external: ecs
-  name: threat.indicator.description
-- external: ecs
-  name: threat.indicator.reference
-- external: ecs
-  name: threat.indicator.file.name
-- external: ecs
-  name: threat.indicator.file.hash.sha256
 - external: ecs
   name: client.geo.city_name
 - external: ecs

diff --git a/packages/cisco_meraki/data_stream/log/sample_event.json b/packages/cisco_meraki/data_stream/log/sample_event.json
@@ -1,11 +1,11 @@
 {
     "@timestamp": "2021-11-23T18:13:18.348Z",
     "agent": {
-        "ephemeral_id": "66ed7cfa-f0ac-4350-9746-94eb916618bf",
-        "id": "0e259c68-d228-45c1-a61f-7ba14f07253b",
+        "ephemeral_id": "bd9fe1e0-a3cd-42b7-9b0b-e0946be0c276",
+        "id": "234cd698-ca4b-4fd7-8a3f-8617e423274a",
         "name": "docker-fleet-agent",
         "type": "filebeat",
-        "version": "8.0.0"
+        "version": "8.11.0"
     },
     "cisco_meraki": {
         "event_subtype": "ids_alerted",
@@ -30,9 +30,9 @@
         "version": "8.11.0"
     },
     "elastic_agent": {
-        "id": "0e259c68-d228-45c1-a61f-7ba14f07253b",
+        "id": "234cd698-ca4b-4fd7-8a3f-8617e423274a",
         "snapshot": false,
-        "version": "8.0.0"
+        "version": "8.11.0"
     },
     "event": {
         "action": "ids-signature-matched",
@@ -42,7 +42,7 @@
             "intrusion_detection"
         ],
         "dataset": "cisco_meraki.log",
-        "ingested": "2023-11-15T04:07:05Z",
+        "ingested": "2023-11-21T20:46:12Z",
         "original": "<134>1 1637691198.348361125 MX84 security_event ids_alerted signature=1:29708:4 priority=1 timestamp=1637691198.330873 dhost=D0:AB:D5:7B:43:73 direction=ingress protocol=tcp/ip src=67.43.156.12:80 dst=10.0.3.162:56391 decision=allowed message: BROWSER-IE Microsoft Internet Explorer CSS uninitialized object access attempt detected",
         "type": [
             "info"
@@ -53,9 +53,10 @@
     },
     "log": {
         "source": {
-            "address": "192.168.240.4:53108"
+            "address": "192.168.160.4:52334"
         }
     },
+    "message": "BROWSER-IE Microsoft Internet Explorer CSS uninitialized object access attempt detected",
     "network": {
         "direction": "ingress",
         "protocol": "tcp/ip"
@@ -83,11 +84,5 @@
         "preserve_original_event",
         "cisco-meraki",
         "forwarded"
-    ],
-    "threat": {
-        "indicator": {
-            "description": "BROWSER-IE Microsoft Internet Explorer CSS uninitialized object access attempt detected",
-            "last_seen": "2021-11-23T18:13:18.330Z"
-        }
-    }
+    ]
 }
diff --git a/packages/cisco_meraki/docs/README.md b/packages/cisco_meraki/docs/README.md
@@ -170,6 +170,7 @@ The `cisco_meraki.log` dataset provides events from the configured syslog server
 | file.attributes | Array of file attributes. Attributes names will vary by platform. Here's a non-exhaustive list of values that are expected in this field: archive, compressed, directory, encrypted, execute, hidden, read, readonly, system, write. | keyword |
 | file.directory | Directory where the file is located. It should include the drive letter, when appropriate. | keyword |
 | file.extension | File extension, excluding the leading dot. Note that when the file name has multiple extensions (example.tar.gz), only the last one should be captured ("gz", not "tar.gz"). | keyword |
+| file.hash.sha256 | SHA256 hash. | keyword |
 | file.name | Name of the file including the extension, without the directory. | keyword |
 | file.path | Full path to the file, including the file name. It should include the drive letter, when appropriate. | keyword |
 | file.path.text | Multi-field of `file.path`. | match_only_text |
@@ -265,17 +266,14 @@ The `cisco_meraki.log` dataset provides events from the configured syslog server
 | source.subdomain | The subdomain portion of a fully qualified domain name includes all of the names except the host name under the registered_domain.  In a partially qualified domain, or if the the qualification level of the full name cannot be determined, subdomain contains all of the names below the registered domain. For example the subdomain portion of "www.east.mydomain.co.uk" is "east". If the domain has multiple levels of subdomain, such as "sub2.sub1.example.com", the subdomain field should contain "sub2.sub1", with no trailing period. | keyword |
 | source.top_level_domain | The effective top level domain (eTLD), also known as the domain suffix, is the last part of the domain name. For example, the top level domain for example.com is "com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last label will not work well for effective TLDs such as "co.uk". | keyword |
 | tags | List of keywords used to tag each event. | keyword |
-| threat.indicator.description | Describes the type of action conducted by the threat. | keyword |
-| threat.indicator.file.hash.sha256 | SHA256 hash. | keyword |
-| threat.indicator.file.name | Name of the file including the extension, without the directory. | keyword |
-| threat.indicator.last_seen | The date and time when intelligence source last reported sighting this indicator. | date |
-| threat.indicator.reference | Reference URL linking to additional information about this indicator. | keyword |
 | url.domain | Domain of the url, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. If the URL contains a literal IPv6 address enclosed by `[` and `]` (IETF RFC 2732), the `[` and `]` characters should also be captured in the `domain` field. | keyword |
+| url.extension | The field contains the file extension from the original request url, excluding the leading dot. The file extension is only set if it exists, as not every url has a file extension. The leading period must not be included. For example, the value must be "png", not ".png". Note that when the file name has multiple extensions (example.tar.gz), only the last one should be captured ("gz", not "tar.gz"). | keyword |
 | url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | wildcard |
 | url.original.text | Multi-field of `url.original`. | match_only_text |
 | url.path | Path of the request, such as "/search". | wildcard |
 | url.query | The query field describes the query string of the request, such as "q=elasticsearch". The `?` is excluded from the query string. If a URL contains no `?`, there is no query field. If there is a `?` but no query, the query field exists with an empty string. The `exists` query can be used to differentiate between the two cases. | keyword |
 | url.registered_domain | The highest registered url domain, stripped of the subdomain. For example, the registered domain for "foo.example.com" is "example.com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last two labels will not work well for TLDs such as "co.uk". | keyword |
+| url.scheme | Scheme of the request, such as "https". Note: The `:` is not part of the scheme. | keyword |
 | url.top_level_domain | The effective top level domain (eTLD), also known as the domain suffix, is the last part of the domain name. For example, the top level domain for example.com is "com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last label will not work well for effective TLDs such as "co.uk". | keyword |
 | user.domain | Name of the directory the user is a member of. For example, an LDAP or Active Directory domain name. | keyword |
 | user.full_name | User's full name, if available. | keyword |
@@ -301,11 +299,11 @@ An example event for `log` looks as following:
 {
     "@timestamp": "2021-11-23T18:13:18.348Z",
     "agent": {
-        "ephemeral_id": "66ed7cfa-f0ac-4350-9746-94eb916618bf",
-        "id": "0e259c68-d228-45c1-a61f-7ba14f07253b",
+        "ephemeral_id": "bd9fe1e0-a3cd-42b7-9b0b-e0946be0c276",
+        "id": "234cd698-ca4b-4fd7-8a3f-8617e423274a",
         "name": "docker-fleet-agent",
         "type": "filebeat",
-        "version": "8.0.0"
+        "version": "8.11.0"
     },
     "cisco_meraki": {
         "event_subtype": "ids_alerted",
@@ -330,9 +328,9 @@ An example event for `log` looks as following:
         "version": "8.11.0"
     },
     "elastic_agent": {
-        "id": "0e259c68-d228-45c1-a61f-7ba14f07253b",
+        "id": "234cd698-ca4b-4fd7-8a3f-8617e423274a",
         "snapshot": false,
-        "version": "8.0.0"
+        "version": "8.11.0"
     },
     "event": {
         "action": "ids-signature-matched",
@@ -342,7 +340,7 @@ An example event for `log` looks as following:
             "intrusion_detection"
         ],
         "dataset": "cisco_meraki.log",
-        "ingested": "2023-11-15T04:07:05Z",
+        "ingested": "2023-11-21T20:46:12Z",
         "original": "<134>1 1637691198.348361125 MX84 security_event ids_alerted signature=1:29708:4 priority=1 timestamp=1637691198.330873 dhost=D0:AB:D5:7B:43:73 direction=ingress protocol=tcp/ip src=67.43.156.12:80 dst=10.0.3.162:56391 decision=allowed message: BROWSER-IE Microsoft Internet Explorer CSS uninitialized object access attempt detected",
         "type": [
             "info"
@@ -353,9 +351,10 @@ An example event for `log` looks as following:
     },
     "log": {
         "source": {
-            "address": "192.168.240.4:53108"
+            "address": "192.168.160.4:52334"
         }
     },
+    "message": "BROWSER-IE Microsoft Internet Explorer CSS uninitialized object access attempt detected",
     "network": {
         "direction": "ingress",
         "protocol": "tcp/ip"
@@ -383,13 +382,7 @@ An example event for `log` looks as following:
         "preserve_original_event",
         "cisco-meraki",
         "forwarded"
-    ],
-    "threat": {
-        "indicator": {
-            "description": "BROWSER-IE Microsoft Internet Explorer CSS uninitialized object access attempt detected",
-            "last_seen": "2021-11-23T18:13:18.330Z"
-        }
-    }
+    ]
 }
 ```