Add CloudTrail standalone S3 option (#6915)

* Add CloudTrail standalone S3 option

* Changelog and manifest

* Add S3 fields Cloudtrail

packages/aws/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.48.0"
+  changes:
+    - description: Adding missing fields for the CloudTrail datastream - add option for standalone S3 bucket
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/6915
 - version: "1.47.1"
   changes:
     - description: Migrate AWS Redshift dashboard input controls.

packages/aws/data_stream/cloudtrail/agent/stream/aws-s3.yml.hbs

@@ -1,4 +1,35 @@
+{{#if collect_s3_logs}} 
+
+{{#if bucket_arn}}
+bucket_arn: {{bucket_arn}}
+{{/if}}
+{{#if number_of_workers}}
+number_of_workers: {{number_of_workers}}
+{{/if}}
+{{#if interval}}
+bucket_list_interval: {{interval}}
+{{/if}}
+{{#if bucket_list_prefix}}
+bucket_list_prefix: {{bucket_list_prefix}}
+{{/if}}
+
+{{else}}
+
+{{#if queue_url}}
 queue_url: {{queue_url}}
+{{/if}}
+{{#if visibility_timeout}}
+visibility_timeout: {{visibility_timeout}}
+{{/if}}
+{{#if api_timeout}}
+api_timeout: {{api_timeout}}
+{{/if}}
+{{#if max_number_of_messages}}
+max_number_of_messages: {{max_number_of_messages}}
+{{/if}}
+
+{{/if}}
+
 file_selectors:
 {{#if cloudtrail_regex}}
   - regex: {{cloudtrail_regex}}
@@ -18,15 +49,8 @@ credential_profile_name: {{credential_profile_name}}
 {{#if shared_credential_file}}
 shared_credential_file: {{shared_credential_file}}
 {{/if}}
-{{#if visibility_timeout}}
-visibility_timeout: {{visibility_timeout}}
-{{/if}}
-{{#if api_timeout}}
-api_timeout: {{api_timeout}}
-{{/if}}
-{{#if max_number_of_messages}}
-max_number_of_messages: {{max_number_of_messages}}
-{{/if}}
+
+
 {{#if default_region}}
 default_region: {{default_region}}
 {{/if}}

packages/aws/data_stream/cloudtrail/fields/fields.yml

@@ -1,3 +1,22 @@
+- name: aws.s3
+  type: group
+  fields:
+    - name: bucket.name
+      type: keyword
+      description: |
+        Name of a S3 bucket.
+    - name: bucket.arn
+      type: keyword
+      description: |
+        ARN of the S3 bucket that this log retrieved from.
+    - name: object.key
+      type: keyword
+      description: |
+        Name of the S3 object that this log retrieved from.
+    - name: metadata
+      type: flattened
+      description: |
+        AWS S3 object metadata values.
 - name: aws.cloudtrail
   type: group
   fields:

packages/aws/data_stream/cloudtrail/manifest.yml

@@ -6,27 +6,65 @@ streams:
     title: AWS CloudTrail Logs
     description: Collect AWS CloudTrail logs using s3 input
     vars:
+      - name: collect_s3_logs
+        required: true
+        show_user: true
+        title: Collect logs via S3 Bucket
+        description: To Collect logs via S3 bucket enable the toggle switch. By default, it will collect logs via SQS Queue.
+        type: bool
+        multi: false
+        default: false
+      - name: bucket_arn
+        type: text
+        title: "[S3] Bucket ARN"
+        multi: false
+        required: false
+        show_user: true
+        description: Mandatory if the "Collect logs via S3 Bucket" switch is on. It is a required parameter for collecting logs via the AWS S3 Bucket.
+      - name: queue_url
+        type: text
+        title: "[SQS] Queue URL"
+        multi: false
+        required: false
+        show_user: true
+        description: Mandatory if the "Collect logs via S3 Bucket" switch is off. URL of the AWS SQS queue that messages will be received from.
+      - name: bucket_list_prefix
+        type: text
+        title: "[S3] Bucket Prefix"
+        multi: false
+        required: false
+        show_user: false
+        description: Prefix to apply for the list request to the S3 bucket.
+      - name: interval
+        type: text
+        title: "[S3] Interval"
+        multi: false
+        required: false
+        show_user: false
+        default: 1m
+        description: "Time interval for polling listing of the S3 bucket. NOTE: Supported units for this parameter are h/m/s."
+      - name: number_of_workers
+        type: integer
+        title: "[S3] Number of Workers"
+        multi: false
+        required: false
+        show_user: false
+        default: 5
+        description: Number of workers that will process the S3 objects listed.
       - name: visibility_timeout
         type: text
-        title: Visibility Timeout
+        title: "[SQS] Visibility Timeout"
         multi: false
         required: false
         show_user: false
         description: The duration that the received messages are hidden from subsequent retrieve requests after being retrieved by a ReceiveMessage request.  The maximum is 12 hours.
       - name: api_timeout
         type: text
-        title: API Timeout
+        title: "[SQS] API Timeout"
         multi: false
         required: false
         show_user: false
         description: The maximum duration of AWS API can take. The maximum is half of the visibility timeout value.
-      - name: queue_url
-        type: text
-        title: Queue URL
-        multi: false
-        required: true
-        show_user: true
-        description: URL of the AWS SQS queue that messages will be received from.
       - name: fips_enabled
         type: bool
         title: Enable S3 FIPS
@@ -89,7 +127,7 @@ streams:
           blank CloudTrail Insight logs will be skipped.
       - name: max_number_of_messages
         type: integer
-        title: Maximum Concurrent SQS Messages
+        title: "[SQS] Maximum Concurrent SQS Messages"
         description: The maximum number of SQS messages that can be inflight at any time.
         default: 5
         required: false

packages/aws/docs/cloudtrail.md

@@ -115,6 +115,10 @@ If blank, CloudTrail Digest logs will be skipped.
 | aws.cloudtrail.user_identity.session_context.session_issuer.type | The source of the temporary security credentials, such as Root, IAMUser, or Role. | keyword |
 | aws.cloudtrail.user_identity.type | The type of the identity | keyword |
 | aws.cloudtrail.vpc_endpoint_id | Identifies the VPC endpoint in which requests were made from a VPC to another AWS service, such as Amazon S3. | keyword |
+| aws.s3.bucket.arn | ARN of the S3 bucket that this log retrieved from. | keyword |
+| aws.s3.bucket.name | Name of a S3 bucket. | keyword |
+| aws.s3.metadata | AWS S3 object metadata values. | flattened |
+| aws.s3.object.key | Name of the S3 object that this log retrieved from. | keyword |
 | cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword |
 | cloud.availability_zone | Availability zone in which this host, resource, or service is located. | keyword |
 | cloud.image.id | Image ID for the cloud instance. | keyword |

packages/aws/manifest.yml

@@ -1,7 +1,7 @@
 format_version: 1.0.0
 name: aws
 title: AWS
-version: 1.47.1
+version: 1.48.0
 license: basic
 description: Collect logs and metrics from Amazon Web Services (AWS) with Elastic Agent.
 type: integration