From da5e0498d451a6612354026c06a66fc9cdd53299 Mon Sep 17 00:00:00 2001 From: Sean Rathier Date: Tue, 4 Aug 2026 14:55:30 -0400 Subject: [PATCH 1/3] [aws] Enable Identity Federation for SecurityHub data streams (ingest-dev#8812) Migrates securityhub_findings, securityhub_findings_full_posture, and securityhub_insights HTTPJSON templates from hand-rolled SigV4 signing to the input's auth.aws configuration. Removes the identity_federation gate from the securityhub policy template. Co-Authored-By: Claude Sonnet 4.6 --- packages/aws/changelog.yml | 5 ++ .../agent/stream/httpjson.yml.hbs | 43 ++++++++++++----- .../agent/stream/httpjson.yml.hbs | 47 +++++++++++++------ .../agent/stream/httpjson.yml.hbs | 43 ++++++++++++----- packages/aws/manifest.yml | 4 +- 5 files changed, 101 insertions(+), 41 deletions(-) diff --git a/packages/aws/changelog.yml b/packages/aws/changelog.yml index 79206a5ee87..81275a0afc6 100644 --- a/packages/aws/changelog.yml +++ b/packages/aws/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "7.2.0" + changes: + - description: Enable Identity Federation (Cloud Connectors) for the SecurityHub data streams (securityhub_findings, securityhub_findings_full_posture, securityhub_insights) by migrating their HTTPJSON templates from hand-rolled SigV4 signing to the input's auth.aws configuration. Fixes silent session-token breakage in the hand-rolled path. + type: enhancement + link: https://github.com/elastic/integrations/pull/20527 - version: "7.1.1" changes: - description: Add `data_stream.namespace` to the Amazon Inspector vulnerability latest transform's unique key so findings are tracked per namespace, preventing findings ingested into non-default namespaces from being dropped or conflated in the latest index. Bump transform's destination suffix to `-v2`. diff --git a/packages/aws/data_stream/securityhub_findings/agent/stream/httpjson.yml.hbs b/packages/aws/data_stream/securityhub_findings/agent/stream/httpjson.yml.hbs index 66c6eae172a..bb1f66586bb 100644 --- a/packages/aws/data_stream/securityhub_findings/agent/stream/httpjson.yml.hbs +++ b/packages/aws/data_stream/securityhub_findings/agent/stream/httpjson.yml.hbs @@ -16,9 +16,6 @@ request.ssl: {{ssl}} request.url: https://securityhub.{{aws_region}}.{{tld}}/findings request.transforms: - - set: - target: header.X-Amz-Date - value: '[[formatDate (now) "20060102T150405Z"]]' - set: target: body.MaxResults value: 100 @@ -36,21 +33,12 @@ request.transforms: value: '[{ "Start": "[[formatDate (parseDate .cursor.last_execution_datetime "RFC3339") "2006-01-02T15"]]", "End": "2099-01-01T00" }]' default: '[{ "Start": "[[formatDate (now (parseDuration "-{{initial_interval}}")) "2006-01-02T15"]]", "End": "2099-01-01T00" }]' value_type: json - - set: - target: header.Authorization - value: '[[$now := (now)]][[(sprintf "AWS4-HMAC-SHA256 Credential={{access_key_id}}/%s/{{aws_region}}/securityhub/aws4_request, SignedHeaders=host;x-amz-date, Signature=%s" (formatDate ($now) "20060102") (hmac "sha256" (hexDecode (hmac "sha256" (hexDecode (hmac "sha256" (hexDecode (hmac "sha256" (hexDecode (hmac "sha256" "AWS4{{secret_access_key}}" (formatDate ($now) "20060102"))) "{{aws_region}}")) "securityhub")) "aws4_request")) "AWS4-HMAC-SHA256\n" (formatDate ($now) "20060102T150405Z") "\n" (sprintf "%s/%s\n" (formatDate ($now) "20060102") "{{aws_region}}/securityhub/aws4_request") (hash "sha256" "POST\n" "/findings\n" "\n" "host:securityhub.{{aws_region}}.{{tld}}\n" (sprintf "x-amz-date:%s\n\n" (formatDate ($now) "20060102T150405Z")) "host;x-amz-date\n" (hash "sha256" (sprintf `%s` .body)))))]]' -{{!-- https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-auth-using-authorization-header.html --}} response.pagination: - set: target: body.NextToken value: '[[if (eq (len .last_response.body.Findings) 100)]][[.last_response.body.NextToken]][[end]]' fail_on_template_error: true do_not_log_failure: true - - delete: - target: header.Authorization - - set: - target: header.Authorization - value: '[[$now := (now)]][[(sprintf "AWS4-HMAC-SHA256 Credential={{access_key_id}}/%s/{{aws_region}}/securityhub/aws4_request, SignedHeaders=host;x-amz-date, Signature=%s" (formatDate ($now) "20060102") (hmac "sha256" (hexDecode (hmac "sha256" (hexDecode (hmac "sha256" (hexDecode (hmac "sha256" (hexDecode (hmac "sha256" "AWS4{{secret_access_key}}" (formatDate ($now) "20060102"))) "{{aws_region}}")) "securityhub")) "aws4_request")) "AWS4-HMAC-SHA256\n" (formatDate ($now) "20060102T150405Z") "\n" (sprintf "%s/%s\n" (formatDate ($now) "20060102") "{{aws_region}}/securityhub/aws4_request") (hash "sha256" "POST\n" "/findings\n" "\n" "host:securityhub.{{aws_region}}.{{tld}}\n" (sprintf "x-amz-date:%s\n\n" (formatDate ($now) "20060102T150405Z")) "host;x-amz-date\n" (hash "sha256" (sprintf `%s` .body)))))]]' cursor: last_execution_datetime: value: '[[if (ne (len .last_response.body.Findings) 100)]][[.last_event.UpdatedAt]][[end]]' @@ -74,3 +62,34 @@ publisher_pipeline.disable_host: true processors: {{processors}} {{/if}} +auth.aws: +{{#if access_key_id}} + access_key_id: {{access_key_id}} +{{/if}} +{{#if secret_access_key}} + secret_access_key: {{secret_access_key}} +{{/if}} +{{#if session_token}} + session_token: {{session_token}} +{{/if}} +{{#if shared_credential_file}} + shared_credential_file: {{shared_credential_file}} +{{/if}} +{{#if credential_profile_name}} + credential_profile_name: {{credential_profile_name}} +{{/if}} +{{#if role_arn}} + role_arn: {{role_arn}} +{{/if}} +{{#if external_id}} + external_id: {{external_id}} +{{/if}} +{{#if assume_role_duration}} + assume_role.duration: {{assume_role_duration}} +{{/if}} +{{#if assume_role_expiry_window}} + assume_role.expiry_window: {{assume_role_expiry_window}} +{{/if}} +{{#if supports_identity_federation}} + use_cloud_connectors: {{supports_identity_federation}} +{{/if}} diff --git a/packages/aws/data_stream/securityhub_findings_full_posture/agent/stream/httpjson.yml.hbs b/packages/aws/data_stream/securityhub_findings_full_posture/agent/stream/httpjson.yml.hbs index 7ad191e7b42..2f4bac97a23 100644 --- a/packages/aws/data_stream/securityhub_findings_full_posture/agent/stream/httpjson.yml.hbs +++ b/packages/aws/data_stream/securityhub_findings_full_posture/agent/stream/httpjson.yml.hbs @@ -16,36 +16,24 @@ request.ssl: {{ssl}} request.url: https://securityhub.{{aws_region}}.{{tld}}/findings request.transforms: - - set: - target: header.X-Amz-Date - value: '[[formatDate (now) "20060102T150405Z"]]' - set: target: body.MaxResults value: 100 value_type: int - set: - target: body.Filters.RecordState + target: body.Filters.RecordState value: '[{"Comparison":"NOT_EQUALS","Value":"ARCHIVED"}]' value_type: json - set: - target: body.Filters.WorkflowStatus + target: body.Filters.WorkflowStatus value: '[{"Comparison":"NOT_EQUALS","Value":"SUPPRESSED"}]' value_type: json - - set: - target: header.Authorization - value: '[[$now := (now)]][[(sprintf "AWS4-HMAC-SHA256 Credential={{access_key_id}}/%s/{{aws_region}}/securityhub/aws4_request, SignedHeaders=host;x-amz-date, Signature=%s" (formatDate ($now) "20060102") (hmac "sha256" (hexDecode (hmac "sha256" (hexDecode (hmac "sha256" (hexDecode (hmac "sha256" (hexDecode (hmac "sha256" "AWS4{{secret_access_key}}" (formatDate ($now) "20060102"))) "{{aws_region}}")) "securityhub")) "aws4_request")) "AWS4-HMAC-SHA256\n" (formatDate ($now) "20060102T150405Z") "\n" (sprintf "%s/%s\n" (formatDate ($now) "20060102") "{{aws_region}}/securityhub/aws4_request") (hash "sha256" "POST\n" "/findings\n" "\n" "host:securityhub.{{aws_region}}.{{tld}}\n" (sprintf "x-amz-date:%s\n\n" (formatDate ($now) "20060102T150405Z")) "host;x-amz-date\n" (hash "sha256" (sprintf `%s` .body)))))]]' -{{!-- https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-auth-using-authorization-header.html --}} response.pagination: - set: target: body.NextToken value: '[[if (eq (len .last_response.body.Findings) 100)]][[.last_response.body.NextToken]][[end]]' fail_on_template_error: true do_not_log_failure: true - - delete: - target: header.Authorization - - set: - target: header.Authorization - value: '[[$now := (now)]][[(sprintf "AWS4-HMAC-SHA256 Credential={{access_key_id}}/%s/{{aws_region}}/securityhub/aws4_request, SignedHeaders=host;x-amz-date, Signature=%s" (formatDate ($now) "20060102") (hmac "sha256" (hexDecode (hmac "sha256" (hexDecode (hmac "sha256" (hexDecode (hmac "sha256" (hexDecode (hmac "sha256" "AWS4{{secret_access_key}}" (formatDate ($now) "20060102"))) "{{aws_region}}")) "securityhub")) "aws4_request")) "AWS4-HMAC-SHA256\n" (formatDate ($now) "20060102T150405Z") "\n" (sprintf "%s/%s\n" (formatDate ($now) "20060102") "{{aws_region}}/securityhub/aws4_request") (hash "sha256" "POST\n" "/findings\n" "\n" "host:securityhub.{{aws_region}}.{{tld}}\n" (sprintf "x-amz-date:%s\n\n" (formatDate ($now) "20060102T150405Z")) "host;x-amz-date\n" (hash "sha256" (sprintf `%s` .body)))))]]' response.split: target: body.Findings ignore_empty_value: true @@ -66,3 +54,34 @@ publisher_pipeline.disable_host: true processors: {{processors}} {{/if}} +auth.aws: +{{#if access_key_id}} + access_key_id: {{access_key_id}} +{{/if}} +{{#if secret_access_key}} + secret_access_key: {{secret_access_key}} +{{/if}} +{{#if session_token}} + session_token: {{session_token}} +{{/if}} +{{#if shared_credential_file}} + shared_credential_file: {{shared_credential_file}} +{{/if}} +{{#if credential_profile_name}} + credential_profile_name: {{credential_profile_name}} +{{/if}} +{{#if role_arn}} + role_arn: {{role_arn}} +{{/if}} +{{#if external_id}} + external_id: {{external_id}} +{{/if}} +{{#if assume_role_duration}} + assume_role.duration: {{assume_role_duration}} +{{/if}} +{{#if assume_role_expiry_window}} + assume_role.expiry_window: {{assume_role_expiry_window}} +{{/if}} +{{#if supports_identity_federation}} + use_cloud_connectors: {{supports_identity_federation}} +{{/if}} diff --git a/packages/aws/data_stream/securityhub_insights/agent/stream/httpjson.yml.hbs b/packages/aws/data_stream/securityhub_insights/agent/stream/httpjson.yml.hbs index dd39f6dbe3d..04fff917584 100644 --- a/packages/aws/data_stream/securityhub_insights/agent/stream/httpjson.yml.hbs +++ b/packages/aws/data_stream/securityhub_insights/agent/stream/httpjson.yml.hbs @@ -16,9 +16,6 @@ request.ssl: {{ssl}} request.url: https://securityhub.{{aws_region}}.{{tld}}/insights/get request.transforms: - - set: - target: header.X-Amz-Date - value: '[[(formatDate (now) "20060102T150405Z")]]' - set: target: body.MaxResults value: 100 @@ -69,10 +66,6 @@ request.transforms: "arn:aws:securityhub:::insight/securityhub/default/39", "arn:aws:securityhub:::insight/securityhub/default/40" ] - - set: - target: header.Authorization - value: '[[$now := (now)]][[(sprintf "AWS4-HMAC-SHA256 Credential={{access_key_id}}/%s/{{aws_region}}/securityhub/aws4_request, SignedHeaders=host;x-amz-date, Signature=%s" (formatDate ($now) "20060102") (hmac "sha256" (hexDecode (hmac "sha256" (hexDecode (hmac "sha256" (hexDecode (hmac "sha256" (hexDecode (hmac "sha256" "AWS4{{secret_access_key}}" (formatDate ($now) "20060102"))) "{{aws_region}}")) "securityhub")) "aws4_request")) "AWS4-HMAC-SHA256\n" (formatDate ($now) "20060102T150405Z") "\n" (sprintf "%s/%s\n" (formatDate ($now) "20060102") "{{aws_region}}/securityhub/aws4_request") (hash "sha256" "POST\n" "/insights/get\n" "\n" "host:securityhub.{{aws_region}}.{{tld}}\n" (sprintf "x-amz-date:%s\n\n" (formatDate ($now) "20060102T150405Z")) "host;x-amz-date\n" (hash "sha256" (sprintf `%s` .body)))))]]' -{{!-- https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-auth-using-authorization-header.html --}} response.pagination: - delete: # Only request specific ARNs in the first request (for managed insights). @@ -97,11 +90,6 @@ response.pagination: value: '[[with (index .last_response.body "NextToken")]][[.]][[end]]' fail_on_template_error: false do_not_log_failure: true - - delete: - target: header.Authorization - - set: - target: header.Authorization - value: '[[$now := (now)]][[(sprintf "AWS4-HMAC-SHA256 Credential={{access_key_id}}/%s/{{aws_region}}/securityhub/aws4_request, SignedHeaders=host;x-amz-date, Signature=%s" (formatDate ($now) "20060102") (hmac "sha256" (hexDecode (hmac "sha256" (hexDecode (hmac "sha256" (hexDecode (hmac "sha256" (hexDecode (hmac "sha256" "AWS4{{secret_access_key}}" (formatDate ($now) "20060102"))) "{{aws_region}}")) "securityhub")) "aws4_request")) "AWS4-HMAC-SHA256\n" (formatDate ($now) "20060102T150405Z") "\n" (sprintf "%s/%s\n" (formatDate ($now) "20060102") "{{aws_region}}/securityhub/aws4_request") (hash "sha256" "POST\n" "/insights/get\n" "\n" "host:securityhub.{{aws_region}}.{{tld}}\n" (sprintf "x-amz-date:%s\n\n" (formatDate ($now) "20060102T150405Z")) "host;x-amz-date\n" (hash "sha256" (sprintf `%s` .body)))))]]' response.split: target: body.Insights ignore_empty_value: true @@ -122,3 +110,34 @@ publisher_pipeline.disable_host: true processors: {{processors}} {{/if}} +auth.aws: +{{#if access_key_id}} + access_key_id: {{access_key_id}} +{{/if}} +{{#if secret_access_key}} + secret_access_key: {{secret_access_key}} +{{/if}} +{{#if session_token}} + session_token: {{session_token}} +{{/if}} +{{#if shared_credential_file}} + shared_credential_file: {{shared_credential_file}} +{{/if}} +{{#if credential_profile_name}} + credential_profile_name: {{credential_profile_name}} +{{/if}} +{{#if role_arn}} + role_arn: {{role_arn}} +{{/if}} +{{#if external_id}} + external_id: {{external_id}} +{{/if}} +{{#if assume_role_duration}} + assume_role.duration: {{assume_role_duration}} +{{/if}} +{{#if assume_role_expiry_window}} + assume_role.expiry_window: {{assume_role_expiry_window}} +{{/if}} +{{#if supports_identity_federation}} + use_cloud_connectors: {{supports_identity_federation}} +{{/if}} diff --git a/packages/aws/manifest.yml b/packages/aws/manifest.yml index 6aea6328c5e..0560ac1e78e 100644 --- a/packages/aws/manifest.yml +++ b/packages/aws/manifest.yml @@ -1,7 +1,7 @@ format_version: 3.6.1 name: aws title: AWS -version: 7.1.1 +version: 7.2.0 description: Collect logs and metrics from Amazon Web Services (AWS) with Elastic Agent. type: integration categories: @@ -1017,8 +1017,6 @@ policy_templates: - type: httpjson title: Collect AWS Security Hub CSPM logs via API description: Collecting AWS Security Hub CSPM logs via API. - hide_in_var_group_options: - credential_type: [identity_federation] screenshots: - src: /img/securityhub_cspm_findings_insights_dashboard.png title: Security Hub CSPM Findings and Insights dashboard screenshot From 35791932c4f20b6459d7b3b3a4fbb2e9c731a96b Mon Sep 17 00:00:00 2001 From: Sean Rathier Date: Tue, 4 Aug 2026 15:05:48 -0400 Subject: [PATCH 2/3] Fix version to 7.5.0 (follows 7.4.0 from #20527) Co-Authored-By: Claude Sonnet 4.6 --- packages/aws/changelog.yml | 4 ++-- packages/aws/manifest.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/aws/changelog.yml b/packages/aws/changelog.yml index 81275a0afc6..3423a406d1b 100644 --- a/packages/aws/changelog.yml +++ b/packages/aws/changelog.yml @@ -1,9 +1,9 @@ # newer versions go on top -- version: "7.2.0" +- version: "7.5.0" changes: - description: Enable Identity Federation (Cloud Connectors) for the SecurityHub data streams (securityhub_findings, securityhub_findings_full_posture, securityhub_insights) by migrating their HTTPJSON templates from hand-rolled SigV4 signing to the input's auth.aws configuration. Fixes silent session-token breakage in the hand-rolled path. type: enhancement - link: https://github.com/elastic/integrations/pull/20527 + link: https://github.com/elastic/integrations/pull/20529 - version: "7.1.1" changes: - description: Add `data_stream.namespace` to the Amazon Inspector vulnerability latest transform's unique key so findings are tracked per namespace, preventing findings ingested into non-default namespaces from being dropped or conflated in the latest index. Bump transform's destination suffix to `-v2`. diff --git a/packages/aws/manifest.yml b/packages/aws/manifest.yml index 0560ac1e78e..a056d0f173a 100644 --- a/packages/aws/manifest.yml +++ b/packages/aws/manifest.yml @@ -1,7 +1,7 @@ format_version: 3.6.1 name: aws title: AWS -version: 7.2.0 +version: 7.5.0 description: Collect logs and metrics from Amazon Web Services (AWS) with Elastic Agent. type: integration categories: From 0d97cd3a115fda187c60233ddf17b8eb94d98058 Mon Sep 17 00:00:00 2001 From: Sean Rathier Date: Tue, 4 Aug 2026 15:18:28 -0400 Subject: [PATCH 3/3] Fix version to 7.3.0 (follows consolidated 7.2.0 from #20527) Co-Authored-By: Claude Fable 5 --- packages/aws/changelog.yml | 2 +- packages/aws/manifest.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/aws/changelog.yml b/packages/aws/changelog.yml index 3423a406d1b..5b477843c71 100644 --- a/packages/aws/changelog.yml +++ b/packages/aws/changelog.yml @@ -1,5 +1,5 @@ # newer versions go on top -- version: "7.5.0" +- version: "7.3.0" changes: - description: Enable Identity Federation (Cloud Connectors) for the SecurityHub data streams (securityhub_findings, securityhub_findings_full_posture, securityhub_insights) by migrating their HTTPJSON templates from hand-rolled SigV4 signing to the input's auth.aws configuration. Fixes silent session-token breakage in the hand-rolled path. type: enhancement diff --git a/packages/aws/manifest.yml b/packages/aws/manifest.yml index a056d0f173a..4ecd0f1b9d6 100644 --- a/packages/aws/manifest.yml +++ b/packages/aws/manifest.yml @@ -1,7 +1,7 @@ format_version: 3.6.1 name: aws title: AWS -version: 7.5.0 +version: 7.3.0 description: Collect logs and metrics from Amazon Web Services (AWS) with Elastic Agent. type: integration categories: