From dd7729d837d6491957c1a6ec945cd4befb35bd75 Mon Sep 17 00:00:00 2001 From: Alex Resnick Date: Sun, 28 May 2023 18:51:14 +0000 Subject: [PATCH 1/5] [AWS] Support Cloudtrail tlsDetails field --- packages/aws/changelog.yml | 5 + .../test/pipeline/test-tls-details-json.log | 1 + .../test-tls-details-json.log-expected.json | 96 +++++++++++++++++++ .../elasticsearch/ingest_pipeline/default.yml | 15 +++ .../aws/data_stream/cloudtrail/fields/ecs.yml | 8 ++ packages/aws/manifest.yml | 2 +- 6 files changed, 126 insertions(+), 1 deletion(-) create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-tls-details-json.log create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-tls-details-json.log-expected.json diff --git a/packages/aws/changelog.yml b/packages/aws/changelog.yml index b44cec8115f..62413371219 100644 --- a/packages/aws/changelog.yml +++ b/packages/aws/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.38.0" + changes: + - description: Update Cloudtrail datastream to support tlsDetails field + type: enhancement + link: https://github.com/elastic/integrations/pull/ - version: "1.37.1" changes: - description: Migrate AWS SNS dashboard visualizations to lenses. diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-tls-details-json.log b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-tls-details-json.log new file mode 100644 index 00000000000..fd941b72544 --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-tls-details-json.log @@ -0,0 +1 @@ +{"eventVersion":"1.05","userIdentity":{"type":"IAMUser","principalId":"EXAMPLE_ID","arn":"arn:aws:iam::0123456789012:user/Alice","accountId":"0123456789012","accessKeyId":"EXAMPLE_KEY","userName":"Alice","sessionContext":{"attributes":{"mfaAuthenticated":"true","creationDate":"2020-01-10T14:38:30Z"}},"invokedBy":"signin.amazonaws.com"},"eventTime":"2020-01-10T16:06:40Z","eventSource":"iam.amazonaws.com","eventName":"UploadSSHPublicKey","awsRegion":"us-east-1","sourceIPAddress":"127.0.0.1","userAgent":"signin.amazonaws.com","requestParameters":{"sSHPublicKeyBody":"ssh-rsa AAAAdeadcodedeadcode Alice@localhost.domain","userName":"Alice"},"responseElements":{"sSHPublicKey":{"fingerprint":"de:ad:c0:de:de:ad:c0:de:de:ad:c0:de:de:ad:c0:de","status":"Active","uploadDate":"Jan 10, 2020 4:06:40 PM","userName":"Alice","sSHPublicKeyId":"EXAMPLE_KEY_ID","sSHPublicKeyBody":"ssh-rsa AAAAdeadcodedeadcode Alice@localhost.domain"}},"requestID":"EXAMPLE-44b9-41cd-90f2-EXAMPLE","eventID":"EXAMPLE-9a9d-4da4-9998-EXAMPLE","eventType":"AwsApiCall","recipientAccountId":"0123456789012","tlsDetails":{"tlsVersion":"TLSv1.2","cipherSuite":"ECDHE-RSA-AES128-GCM-SHA256","clientProvidedHostHeader":"ssm.us-west-2.amazonaws.com"}} \ No newline at end of file diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-tls-details-json.log-expected.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-tls-details-json.log-expected.json new file mode 100644 index 00000000000..3cb61cdf8b8 --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-tls-details-json.log-expected.json @@ -0,0 +1,96 @@ +{ + "expected": [ + { + "@timestamp": "2020-01-10T16:06:40.000Z", + "aws": { + "cloudtrail": { + "event_type": "AwsApiCall", + "event_version": "1.05", + "flattened": { + "request_parameters": { + "sSHPublicKeyBody": "ssh-rsa AAAAdeadcodedeadcode Alice@localhost.domain", + "userName": "Alice" + }, + "response_elements": { + "sSHPublicKey": { + "fingerprint": "de:ad:c0:de:de:ad:c0:de:de:ad:c0:de:de:ad:c0:de", + "sSHPublicKeyBody": "ssh-rsa AAAAdeadcodedeadcode Alice@localhost.domain", + "sSHPublicKeyId": "EXAMPLE_KEY_ID", + "status": "Active", + "uploadDate": "Jan 10, 2020 4:06:40 PM", + "userName": "Alice" + } + } + }, + "recipient_account_id": "0123456789012", + "request_id": "EXAMPLE-44b9-41cd-90f2-EXAMPLE", + "request_parameters": "{sSHPublicKeyBody=ssh-rsa AAAAdeadcodedeadcode Alice@localhost.domain, userName=Alice}", + "response_elements": "{sSHPublicKey={sSHPublicKeyBody=ssh-rsa AAAAdeadcodedeadcode Alice@localhost.domain, sSHPublicKeyId=EXAMPLE_KEY_ID, uploadDate=Jan 10, 2020 4:06:40 PM, fingerprint=de:ad:c0:de:de:ad:c0:de:de:ad:c0:de:de:ad:c0:de, userName=Alice, status=Active}}", + "user_identity": { + "access_key_id": "EXAMPLE_KEY", + "arn": "arn:aws:iam::0123456789012:user/Alice", + "invoked_by": "signin.amazonaws.com", + "session_context": { + "creation_date": "2020-01-10T14:38:30.000Z", + "mfa_authenticated": "true" + }, + "type": "IAMUser" + } + } + }, + "cloud": { + "account": { + "id": "0123456789012" + }, + "region": "us-east-1" + }, + "ecs": { + "version": "8.0.0" + }, + "event": { + "action": "UploadSSHPublicKey", + "created": "2021-11-11T01:02:03.123456789Z", + "id": "EXAMPLE-9a9d-4da4-9998-EXAMPLE", + "kind": "event", + "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EXAMPLE_ID\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY\",\"userName\":\"Alice\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"true\",\"creationDate\":\"2020-01-10T14:38:30Z\"}},\"invokedBy\":\"signin.amazonaws.com\"},\"eventTime\":\"2020-01-10T16:06:40Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"UploadSSHPublicKey\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"signin.amazonaws.com\",\"requestParameters\":{\"sSHPublicKeyBody\":\"ssh-rsa AAAAdeadcodedeadcode Alice@localhost.domain\",\"userName\":\"Alice\"},\"responseElements\":{\"sSHPublicKey\":{\"fingerprint\":\"de:ad:c0:de:de:ad:c0:de:de:ad:c0:de:de:ad:c0:de\",\"status\":\"Active\",\"uploadDate\":\"Jan 10, 2020 4:06:40 PM\",\"userName\":\"Alice\",\"sSHPublicKeyId\":\"EXAMPLE_KEY_ID\",\"sSHPublicKeyBody\":\"ssh-rsa AAAAdeadcodedeadcode Alice@localhost.domain\"}},\"requestID\":\"EXAMPLE-44b9-41cd-90f2-EXAMPLE\",\"eventID\":\"EXAMPLE-9a9d-4da4-9998-EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\",\"tlsDetails\":{\"tlsVersion\":\"TLSv1.2\",\"cipherSuite\":\"ECDHE-RSA-AES128-GCM-SHA256\",\"clientProvidedHostHeader\":\"ssm.us-west-2.amazonaws.com\"}}", + "outcome": "success", + "provider": "iam.amazonaws.com", + "type": "info" + }, + "related": { + "user": [ + "Alice" + ] + }, + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, + "tags": [ + "preserve_original_event" + ], + "tls": { + "cipher": "ECDHE-RSA-AES128-GCM-SHA256", + "client": { + "server_name": "ssm.us-west-2.amazonaws.com" + }, + "version": "1.2", + "version_protocol": "tls" + }, + "user": { + "id": "EXAMPLE_ID", + "name": "Alice", + "target": { + "name": "Alice" + } + }, + "user_agent": { + "device": { + "name": "Other" + }, + "name": "Other", + "original": "signin.amazonaws.com" + } + } + ] +} \ No newline at end of file diff --git a/packages/aws/data_stream/cloudtrail/elasticsearch/ingest_pipeline/default.yml b/packages/aws/data_stream/cloudtrail/elasticsearch/ingest_pipeline/default.yml index 162d2c8c664..1736d2ff828 100644 --- a/packages/aws/data_stream/cloudtrail/elasticsearch/ingest_pipeline/default.yml +++ b/packages/aws/data_stream/cloudtrail/elasticsearch/ingest_pipeline/default.yml @@ -743,6 +743,21 @@ processors: field: aws.cloudtrail.insight_details target_field: aws.cloudtrail.flattened.insight_details ignore_missing: true + - dissect: + field: json.tlsDetails.tlsVersion + pattern: "%{tls.version_protocol}v%{tls.version}" + ignore_missing: true + - lowercase: + field: tls.version_protocol + ignore_missing: true + - rename: + field: json.tlsDetails.cipherSuite + target_field: tls.cipher + ignore_missing: true + - rename: + field: json.tlsDetails.clientProvidedHostHeader + target_field: tls.client.server_name + ignore_missing: true - remove: field: json ignore_missing: true diff --git a/packages/aws/data_stream/cloudtrail/fields/ecs.yml b/packages/aws/data_stream/cloudtrail/fields/ecs.yml index eb00fdb764f..4a294d8733d 100644 --- a/packages/aws/data_stream/cloudtrail/fields/ecs.yml +++ b/packages/aws/data_stream/cloudtrail/fields/ecs.yml @@ -134,3 +134,11 @@ name: container.labels - external: ecs name: container.name +- external: ecs + name: tls.version +- external: ecs + name: tls.version_protocol +- external: ecs + name: tls.cipher +- external: ecs + name: tls.client.server_name diff --git a/packages/aws/manifest.yml b/packages/aws/manifest.yml index 4ccc409afc3..361819e435b 100644 --- a/packages/aws/manifest.yml +++ b/packages/aws/manifest.yml @@ -1,7 +1,7 @@ format_version: 1.0.0 name: aws title: AWS -version: 1.37.1 +version: 1.38.0 license: basic description: Collect logs and metrics from Amazon Web Services with Elastic Agent. type: integration From 36a5f2fd1fc50e5633b09ec1d7e8317becddbc67 Mon Sep 17 00:00:00 2001 From: Alex Resnick Date: Sun, 28 May 2023 18:52:28 +0000 Subject: [PATCH 2/5] update log --- packages/aws/changelog.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/aws/changelog.yml b/packages/aws/changelog.yml index 62413371219..90317fa056b 100644 --- a/packages/aws/changelog.yml +++ b/packages/aws/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Update Cloudtrail datastream to support tlsDetails field type: enhancement - link: https://github.com/elastic/integrations/pull/ + link: https://github.com/elastic/integrations/pull/6352 - version: "1.37.1" changes: - description: Migrate AWS SNS dashboard visualizations to lenses. From 13494d19a17213453279f498e490fd9686d56faa Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Wed, 27 Sep 2023 11:32:23 -0600 Subject: [PATCH 3/5] Update manifest.yml version field --- packages/aws/manifest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/aws/manifest.yml b/packages/aws/manifest.yml index d07fe11eac1..1ab7f948af0 100644 --- a/packages/aws/manifest.yml +++ b/packages/aws/manifest.yml @@ -1,7 +1,7 @@ format_version: 1.0.0 name: aws title: AWS -version: 2.4.0 +version: 2.5.0 license: basic description: Collect logs and metrics from Amazon Web Services (AWS) with Elastic Agent. type: integration From 970d30d95d685d83c36f704f216c060f25a090a1 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Wed, 27 Sep 2023 13:43:25 -0600 Subject: [PATCH 4/5] run elastic-package build --- packages/aws/docs/cloudtrail.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/aws/docs/cloudtrail.md b/packages/aws/docs/cloudtrail.md index ff2f854432f..1faf172856a 100644 --- a/packages/aws/docs/cloudtrail.md +++ b/packages/aws/docs/cloudtrail.md @@ -186,6 +186,10 @@ If blank, CloudTrail Digest logs will be skipped. | source.geo.region_name | Region name. | keyword | | source.ip | IP address of the source (IPv4 or IPv6). | ip | | tags | List of keywords used to tag each event. | keyword | +| tls.cipher | String indicating the cipher used during the current connection. | keyword | +| tls.client.server_name | Also called an SNI, this tells the server which hostname to which the client is attempting to connect to. When this value is available, it should get copied to `destination.domain`. | keyword | +| tls.version | Numeric part of the version parsed from the original string. | keyword | +| tls.version_protocol | Normalized lowercase protocol name parsed from original string. | keyword | | user.changes.name | Short name or login of the user. | keyword | | user.changes.name.text | Multi-field of `user.changes.name`. | match_only_text | | user.id | Unique identifier of the user. | keyword | From 654d4d546b46cfdaebc5c120e708e8a2e661d5ab Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Wed, 27 Sep 2023 15:10:31 -0600 Subject: [PATCH 5/5] add link in changelog --- packages/aws/changelog.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/aws/changelog.yml b/packages/aws/changelog.yml index 69854764d0a..487c2473dae 100644 --- a/packages/aws/changelog.yml +++ b/packages/aws/changelog.yml @@ -3,6 +3,7 @@ changes: - description: Update Cloudtrail datastream to support tlsDetails field type: enhancement + link: https://github.com/elastic/integrations/pull/6352 - version: "2.4.1" changes: - description: Fix Security Hub Findings to abide by ECS allowed values.