From 0890c70de64ae603d58fdd13805b431489d65994 Mon Sep 17 00:00:00 2001 From: Tamara Dancheva <122360116+tdancheva@users.noreply.github.com> Date: Wed, 19 Jul 2023 16:59:41 +0200 Subject: [PATCH] Add CloudTrail standalone S3 option (#6915) * Add CloudTrail standalone S3 option * Changelog and manifest * Add S3 fields Cloudtrail --- packages/aws/changelog.yml | 5 ++ .../cloudtrail/agent/stream/aws-s3.yml.hbs | 42 +++++++++++--- .../data_stream/cloudtrail/fields/fields.yml | 19 ++++++ .../aws/data_stream/cloudtrail/manifest.yml | 58 +++++++++++++++---- packages/aws/docs/cloudtrail.md | 4 ++ packages/aws/manifest.yml | 2 +- 6 files changed, 110 insertions(+), 20 deletions(-) diff --git a/packages/aws/changelog.yml b/packages/aws/changelog.yml index df1cde7a9bec..58dda527bd2c 100644 --- a/packages/aws/changelog.yml +++ b/packages/aws/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.48.0" + changes: + - description: Adding missing fields for the CloudTrail datastream - add option for standalone S3 bucket + type: enhancement + link: https://github.com/elastic/integrations/pull/6915 - version: "1.47.1" changes: - description: Migrate AWS Redshift dashboard input controls. diff --git a/packages/aws/data_stream/cloudtrail/agent/stream/aws-s3.yml.hbs b/packages/aws/data_stream/cloudtrail/agent/stream/aws-s3.yml.hbs index 39d17cc88ef5..b54c36bb530d 100644 --- a/packages/aws/data_stream/cloudtrail/agent/stream/aws-s3.yml.hbs +++ b/packages/aws/data_stream/cloudtrail/agent/stream/aws-s3.yml.hbs @@ -1,4 +1,35 @@ +{{#if collect_s3_logs}} + +{{#if bucket_arn}} +bucket_arn: {{bucket_arn}} +{{/if}} +{{#if number_of_workers}} +number_of_workers: {{number_of_workers}} +{{/if}} +{{#if interval}} +bucket_list_interval: {{interval}} +{{/if}} +{{#if bucket_list_prefix}} +bucket_list_prefix: {{bucket_list_prefix}} +{{/if}} + +{{else}} + +{{#if queue_url}} queue_url: {{queue_url}} +{{/if}} +{{#if visibility_timeout}} +visibility_timeout: {{visibility_timeout}} +{{/if}} +{{#if api_timeout}} +api_timeout: {{api_timeout}} +{{/if}} +{{#if max_number_of_messages}} +max_number_of_messages: {{max_number_of_messages}} +{{/if}} + +{{/if}} + file_selectors: {{#if cloudtrail_regex}} - regex: {{cloudtrail_regex}} @@ -18,15 +49,8 @@ credential_profile_name: {{credential_profile_name}} {{#if shared_credential_file}} shared_credential_file: {{shared_credential_file}} {{/if}} -{{#if visibility_timeout}} -visibility_timeout: {{visibility_timeout}} -{{/if}} -{{#if api_timeout}} -api_timeout: {{api_timeout}} -{{/if}} -{{#if max_number_of_messages}} -max_number_of_messages: {{max_number_of_messages}} -{{/if}} + + {{#if default_region}} default_region: {{default_region}} {{/if}} diff --git a/packages/aws/data_stream/cloudtrail/fields/fields.yml b/packages/aws/data_stream/cloudtrail/fields/fields.yml index 5b59153c9bf1..68fbc0ee8cee 100644 --- a/packages/aws/data_stream/cloudtrail/fields/fields.yml +++ b/packages/aws/data_stream/cloudtrail/fields/fields.yml @@ -1,3 +1,22 @@ +- name: aws.s3 + type: group + fields: + - name: bucket.name + type: keyword + description: | + Name of a S3 bucket. + - name: bucket.arn + type: keyword + description: | + ARN of the S3 bucket that this log retrieved from. + - name: object.key + type: keyword + description: | + Name of the S3 object that this log retrieved from. + - name: metadata + type: flattened + description: | + AWS S3 object metadata values. - name: aws.cloudtrail type: group fields: diff --git a/packages/aws/data_stream/cloudtrail/manifest.yml b/packages/aws/data_stream/cloudtrail/manifest.yml index e6bd6a284ec7..132031cfece2 100644 --- a/packages/aws/data_stream/cloudtrail/manifest.yml +++ b/packages/aws/data_stream/cloudtrail/manifest.yml @@ -6,27 +6,65 @@ streams: title: AWS CloudTrail Logs description: Collect AWS CloudTrail logs using s3 input vars: + - name: collect_s3_logs + required: true + show_user: true + title: Collect logs via S3 Bucket + description: To Collect logs via S3 bucket enable the toggle switch. By default, it will collect logs via SQS Queue. + type: bool + multi: false + default: false + - name: bucket_arn + type: text + title: "[S3] Bucket ARN" + multi: false + required: false + show_user: true + description: Mandatory if the "Collect logs via S3 Bucket" switch is on. It is a required parameter for collecting logs via the AWS S3 Bucket. + - name: queue_url + type: text + title: "[SQS] Queue URL" + multi: false + required: false + show_user: true + description: Mandatory if the "Collect logs via S3 Bucket" switch is off. URL of the AWS SQS queue that messages will be received from. + - name: bucket_list_prefix + type: text + title: "[S3] Bucket Prefix" + multi: false + required: false + show_user: false + description: Prefix to apply for the list request to the S3 bucket. + - name: interval + type: text + title: "[S3] Interval" + multi: false + required: false + show_user: false + default: 1m + description: "Time interval for polling listing of the S3 bucket. NOTE: Supported units for this parameter are h/m/s." + - name: number_of_workers + type: integer + title: "[S3] Number of Workers" + multi: false + required: false + show_user: false + default: 5 + description: Number of workers that will process the S3 objects listed. - name: visibility_timeout type: text - title: Visibility Timeout + title: "[SQS] Visibility Timeout" multi: false required: false show_user: false description: The duration that the received messages are hidden from subsequent retrieve requests after being retrieved by a ReceiveMessage request. The maximum is 12 hours. - name: api_timeout type: text - title: API Timeout + title: "[SQS] API Timeout" multi: false required: false show_user: false description: The maximum duration of AWS API can take. The maximum is half of the visibility timeout value. - - name: queue_url - type: text - title: Queue URL - multi: false - required: true - show_user: true - description: URL of the AWS SQS queue that messages will be received from. - name: fips_enabled type: bool title: Enable S3 FIPS @@ -89,7 +127,7 @@ streams: blank CloudTrail Insight logs will be skipped. - name: max_number_of_messages type: integer - title: Maximum Concurrent SQS Messages + title: "[SQS] Maximum Concurrent SQS Messages" description: The maximum number of SQS messages that can be inflight at any time. default: 5 required: false diff --git a/packages/aws/docs/cloudtrail.md b/packages/aws/docs/cloudtrail.md index 4a29fbccfabc..ff2f854432f7 100644 --- a/packages/aws/docs/cloudtrail.md +++ b/packages/aws/docs/cloudtrail.md @@ -115,6 +115,10 @@ If blank, CloudTrail Digest logs will be skipped. | aws.cloudtrail.user_identity.session_context.session_issuer.type | The source of the temporary security credentials, such as Root, IAMUser, or Role. | keyword | | aws.cloudtrail.user_identity.type | The type of the identity | keyword | | aws.cloudtrail.vpc_endpoint_id | Identifies the VPC endpoint in which requests were made from a VPC to another AWS service, such as Amazon S3. | keyword | +| aws.s3.bucket.arn | ARN of the S3 bucket that this log retrieved from. | keyword | +| aws.s3.bucket.name | Name of a S3 bucket. | keyword | +| aws.s3.metadata | AWS S3 object metadata values. | flattened | +| aws.s3.object.key | Name of the S3 object that this log retrieved from. | keyword | | cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | | cloud.availability_zone | Availability zone in which this host, resource, or service is located. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | diff --git a/packages/aws/manifest.yml b/packages/aws/manifest.yml index 512aca212fee..1f00d83e0dc5 100644 --- a/packages/aws/manifest.yml +++ b/packages/aws/manifest.yml @@ -1,7 +1,7 @@ format_version: 1.0.0 name: aws title: AWS -version: 1.47.1 +version: 1.48.0 license: basic description: Collect logs and metrics from Amazon Web Services (AWS) with Elastic Agent. type: integration