[ECS] Updating aws/guardduty to ECS 8.10 (#8002)

* Bumping ecs version and  aws/guardduty ecs update for 8.10

* Updating sample events

* changelog

* formatting

* bumping version

packages/aws/_dev/build/build.yml

@@ -1,3 +1,3 @@
 dependencies:
   ecs:
-    reference: git@v8.2.0
+    reference: git@v8.10.0

packages/aws/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.6.0"
+  changes:
+    - description: ECS version updated to 8.10.0.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/8002
 - version: "2.5.0"
   changes:
     - description: Update Cloudtrail datastream to support tlsDetails field

packages/aws/data_stream/ebs/sample_event.json

@@ -44,7 +44,7 @@
         ],
         "name": "docker-fleet-agent",
         "mac": [
-            "02:42:ac:12:00:07"
+            "02-42-AC-12-00-07"
         ],
         "architecture": "x86_64"
     },

packages/aws/data_stream/ecs_metrics/sample_event.json

@@ -48,7 +48,7 @@
         ],
         "name": "4b4f1fd6f3ff",
         "mac": [
-            "02:42:ac:13:00:04"
+            "02-42-AC-13-00-04"
         ],
         "architecture": "aarch64"
     },

packages/aws/data_stream/elb_metrics/sample_event.json

@@ -48,7 +48,7 @@
         ],
         "name": "docker-fleet-agent",
         "mac": [
-            "02:42:c0:a8:60:07"
+            "02-42-C0-A8-60-07"
         ],
         "architecture": "x86_64"
     },

packages/aws/data_stream/guardduty/_dev/test/pipeline/test-guardduty.log-expected.json

@@ -178,7 +178,7 @@
                 }
             },
             "ecs": {
-                "version": "8.2.0"
+                "version": "8.10.0"
             },
             "event": {
                 "action": "DNS_REQUEST",
@@ -335,7 +335,7 @@
                 }
             },
             "ecs": {
-                "version": "8.2.0"
+                "version": "8.10.0"
             },
             "event": {
                 "action": "KUBERNETES_API_CALL",
@@ -548,7 +548,7 @@
                 }
             },
             "ecs": {
-                "version": "8.2.0"
+                "version": "8.10.0"
             },
             "event": {
                 "action": "KUBERNETES_API_CALL",
@@ -745,7 +745,7 @@
                 }
             },
             "ecs": {
-                "version": "8.2.0"
+                "version": "8.10.0"
             },
             "event": {
                 "action": "RDS_LOGIN_ATTEMPT",

packages/aws/data_stream/guardduty/elasticsearch/ingest_pipeline/default.yml

@@ -3,7 +3,7 @@ description: Pipeline for processing Amazon GuardDuty Findings logs.
 processors:
   - set:
       field: ecs.version
-      value: '8.2.0'
+      value: '8.10.0'
   - set:
       field: event.kind
       value: [event]
@@ -127,6 +127,10 @@ processors:
         - append:
             field: error.message
             value: '{{{_ingest.on_failure_message}}}'
+  - set:
+      field: container.security_context.privileged
+      copy_from: aws.guardduty.resource.container_details.security_context.privileged
+      ignore_empty_value: true
   - foreach:
       field: json.resource.containerDetails.volumeMounts
       if: ctx.json?.resource?.containerDetails?.volumeMounts instanceof List

packages/aws/data_stream/guardduty/fields/ecs.yml

@@ -138,3 +138,5 @@
   name: user.name
 - external: ecs
   name: user.roles
+- external: ecs
+  name: container.security_context.privileged

packages/aws/data_stream/natgateway/sample_event.json

@@ -44,7 +44,7 @@
         ],
         "name": "a3fc2d7bc1c5",
         "mac": [
-            "02:42:ac:14:00:07"
+            "02-42-AC-14-00-07"
         ],
         "architecture": "aarch64"
     },

packages/aws/data_stream/redshift/sample_event.json

@@ -109,7 +109,7 @@
             "192.168.112.7"
         ],
         "mac": [
-            "02:42:c0:a8:70:07"
+            "02-42-C0-A8-70-07"
         ],
         "name": "docker-fleet-agent",
         "os": {

packages/aws/data_stream/securityhub_findings/sample_event.json

@@ -362,7 +362,7 @@
         "type": "httpjson"
     },
     "network": {
-        "direction": "IN",
+        "direction": "ingress",
         "protocol": "tcp"
     },
     "organization": {
@@ -401,7 +401,7 @@
     "threat": {
         "indicator": {
             "last_seen": "2018-09-27T23:37:31.000Z",
-            "type": "IPV4_ADDRESS"
+            "type": "ipv4-addr"
         }
     },
     "url": {

packages/aws/data_stream/transitgateway/sample_event.json

@@ -48,7 +48,7 @@
         ],
         "name": "a20ad158868c",
         "mac": [
-            "02:42:ac:14:00:07"
+            "02-42-AC-14-00-07"
         ],
         "architecture": "aarch64"
     },

packages/aws/docs/apigateway.md

@@ -217,7 +217,7 @@ An example event for `apigateway` looks as following:
 | host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword |  |  |
 | host.ip | Host ip addresses. | ip |  |  |
 | host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword |  |  |
-| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword |  |  |
+| host.name | Name of the host. It can contain what hostname returns on Unix systems, the fully qualified domain name (FQDN), or a name specified by the user. The recommended value is the lowercase FQDN of the host. | keyword |  |  |
 | host.os.build | OS build information. | keyword |  |  |
 | host.os.codename | OS codename, if any. | keyword |  |  |
 | host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword |  |  |
@@ -313,7 +313,7 @@ An example event for `apigateway` looks as following:
 | host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword |
 | host.ip | Host ip addresses. | ip |
 | host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword |
-| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword |
+| host.name | Name of the host. It can contain what hostname returns on Unix systems, the fully qualified domain name (FQDN), or a name specified by the user. The recommended value is the lowercase FQDN of the host. | keyword |
 | host.os.build | OS build information. | keyword |
 | host.os.codename | OS codename, if any. | keyword |
 | host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword |

packages/aws/docs/billing.md

@@ -170,7 +170,7 @@ An example event for `billing` looks as following:
 | host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword |  |
 | host.ip | Host ip addresses. | ip |  |
 | host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword |  |
-| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword |  |
+| host.name | Name of the host. It can contain what hostname returns on Unix systems, the fully qualified domain name (FQDN), or a name specified by the user. The recommended value is the lowercase FQDN of the host. | keyword |  |
 | host.os.build | OS build information. | keyword |  |
 | host.os.codename | OS codename, if any. | keyword |  |
 | host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword |  |

packages/aws/docs/cloudfront.md

@@ -80,7 +80,7 @@ CloudFront standard logs provide detailed records about every request that’s m
 | ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword |
 | error.message | Error message. | match_only_text |
 | event.dataset | Event dataset | constant_keyword |
-| event.duration | Duration of the event in nanoseconds. If event.start and event.end are known this value should be the difference between the end and start time. | long |
+| event.duration | Duration of the event in nanoseconds. If `event.start` and `event.end` are known this value should be the difference between the end and start time. | long |
 | event.module | Event module | constant_keyword |
 | host.architecture | Operating system architecture. | keyword |
 | host.containerized | If the host is a container. | boolean |
@@ -89,7 +89,7 @@ CloudFront standard logs provide detailed records about every request that’s m
 | host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword |
 | host.ip | Host ip addresses. | ip |
 | host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword |
-| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword |
+| host.name | Name of the host. It can contain what hostname returns on Unix systems, the fully qualified domain name (FQDN), or a name specified by the user. The recommended value is the lowercase FQDN of the host. | keyword |
 | host.os.build | OS build information. | keyword |
 | host.os.codename | OS codename, if any. | keyword |
 | host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword |

packages/aws/docs/cloudtrail.md

@@ -138,10 +138,10 @@ If blank, CloudTrail Digest logs will be skipped.
 | ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword |
 | error.message | Error message. | match_only_text |
 | event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword |
-| event.created | event.created contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from @timestamp in that @timestamp typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, @timestamp should be used. | date |
+| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date |
 | event.dataset | Event dataset | constant_keyword |
 | event.ingested | Timestamp when an event arrived in the central data store. This is different from `@timestamp`, which is when the event originally occurred.  It's also different from `event.created`, which is meant to capture the first time an agent saw the event. In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` \< `event.created` \< `event.ingested`. | date |
-| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data coming in at a regular interval or not. | keyword |
+| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword |
 | event.module | Event module | constant_keyword |
 | event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword |
 | event.provider | Source of the event. Event transports such as Syslog or the Windows Event Log typically mention the source of an event. It can be the name of the software that generated the event (e.g. Sysmon, httpd), or of a subsystem of the operating system (kernel, Microsoft-Windows-Security-Auditing). | keyword |
@@ -161,7 +161,7 @@ If blank, CloudTrail Digest logs will be skipped.
 | host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword |
 | host.ip | Host ip addresses. | ip |
 | host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword |
-| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword |
+| host.name | Name of the host. It can contain what hostname returns on Unix systems, the fully qualified domain name (FQDN), or a name specified by the user. The recommended value is the lowercase FQDN of the host. | keyword |
 | host.os.build | OS build information. | keyword |
 | host.os.codename | OS codename, if any. | keyword |
 | host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword |

packages/aws/docs/cloudwatch.md

@@ -97,7 +97,7 @@ CloudWatch logs to monitor, store, and access log files from different sources.
 | host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword |
 | host.ip | Host ip addresses. | ip |
 | host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword |
-| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword |
+| host.name | Name of the host. It can contain what hostname returns on Unix systems, the fully qualified domain name (FQDN), or a name specified by the user. The recommended value is the lowercase FQDN of the host. | keyword |
 | host.os.build | OS build information. | keyword |
 | host.os.codename | OS codename, if any. | keyword |
 | host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword |
@@ -240,7 +240,7 @@ An example event for `cloudwatch` looks as following:
 | host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword |  |
 | host.ip | Host ip addresses. | ip |  |
 | host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword |  |
-| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword |  |
+| host.name | Name of the host. It can contain what hostname returns on Unix systems, the fully qualified domain name (FQDN), or a name specified by the user. The recommended value is the lowercase FQDN of the host. | keyword |  |
 | host.os.build | OS build information. | keyword |  |
 | host.os.codename | OS codename, if any. | keyword |  |
 | host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword |  |

packages/aws/docs/dynamodb.md

@@ -200,7 +200,7 @@ An example event for `dynamodb` looks as following:
 | host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword |  |
 | host.ip | Host ip addresses. | ip |  |
 | host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword |  |
-| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword |  |
+| host.name | Name of the host. It can contain what hostname returns on Unix systems, the fully qualified domain name (FQDN), or a name specified by the user. The recommended value is the lowercase FQDN of the host. | keyword |  |
 | host.os.build | OS build information. | keyword |  |
 | host.os.codename | OS codename, if any. | keyword |  |
 | host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword |  |