From 0aef50fcfb9d50addf579621cb83ee6dab9fcb69 Mon Sep 17 00:00:00 2001 From: Sean Rathier Date: Fri, 31 Jul 2026 16:13:28 -0400 Subject: [PATCH 1/3] [aws] Enable Identity Federation for the Config data stream MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Migrate the Config CEL program from hand-rolled SigV4 signing to the input's auth.aws configuration. The program previously derived signing keys and built canonical requests inline, with access keys passed through CEL state — which limited the stream to static access keys (the credential-validation gate rejected Role ARN and every other method). With auth.aws the input signs requests transparently, so the program keeps only its real logic: rule pagination, service-linked rule filtering, per-rule compliance collection with non-fatal errors. - Add the full auth.aws block including the use_cloud_connectors hook driven by supports_identity_federation - Drop credentials from CEL state and the redact list - Remove the hide_in_var_group_options gate from the config policy template, making Identity Federation selectable on the agentless path - Bonus: Role ARN, shared credentials, and IAM profiles now work for Config, which the hand-rolled signing never supported Part of elastic/ingest-dev#8802. Permissions for the static fallback template are mirrored in the paired cloudbeat PR from the patch sets in elastic/integrations#20240. Co-Authored-By: Claude Fable 5 --- packages/aws/changelog.yml | 5 + .../config/agent/stream/cel.yml.hbs | 363 +++++------------- packages/aws/manifest.yml | 4 +- 3 files changed, 100 insertions(+), 272 deletions(-) diff --git a/packages/aws/changelog.yml b/packages/aws/changelog.yml index 79206a5ee87..1bda46352c0 100644 --- a/packages/aws/changelog.yml +++ b/packages/aws/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "7.2.0" + changes: + - description: Enable Identity Federation (Cloud Connectors) for the AWS Config data stream by migrating its CEL program from hand-rolled SigV4 signing to the input's `auth.aws` configuration. This also adds support for the Role ARN and shared-credential authentication methods that the hand-rolled signing could not use. + type: enhancement + link: https://github.com/elastic/integrations/pull/99999 - version: "7.1.1" changes: - description: Add `data_stream.namespace` to the Amazon Inspector vulnerability latest transform's unique key so findings are tracked per namespace, preventing findings ingested into non-default namespaces from being dropped or conflated in the latest index. Bump transform's destination suffix to `-v2`. diff --git a/packages/aws/data_stream/config/agent/stream/cel.yml.hbs b/packages/aws/data_stream/config/agent/stream/cel.yml.hbs index e44a8ffec17..73d6e4feff6 100644 --- a/packages/aws/data_stream/config/agent/stream/cel.yml.hbs +++ b/packages/aws/data_stream/config/agent/stream/cel.yml.hbs @@ -30,37 +30,49 @@ resource.retry.wait_min: {{retry_wait_min}} resource.retry.wait_max: {{retry_wait_max}} {{/if}} resource.url: https://config.{{aws_region}}.{{tld}}/ -state: - access_key: {{access_key_id}} - secret_key: {{secret_access_key}} + +auth.aws: +{{#if access_key_id}} + access_key_id: {{access_key_id}} +{{/if}} +{{#if secret_access_key}} + secret_access_key: {{secret_access_key}} +{{/if}} +{{#if session_token}} session_token: {{session_token}} - aws_region: {{aws_region}} +{{/if}} +{{#if shared_credential_file}} + shared_credential_file: {{shared_credential_file}} +{{/if}} +{{#if credential_profile_name}} + credential_profile_name: {{credential_profile_name}} +{{/if}} +{{#if role_arn}} + role_arn: {{role_arn}} +{{/if}} +{{#if external_id}} + external_id: {{external_id}} +{{/if}} +{{#if assume_role_duration}} + assume_role.duration: {{assume_role_duration}} +{{/if}} +{{#if assume_role_expiry_window}} + assume_role.expiry_window: {{assume_role_expiry_window}} +{{/if}} +{{#if supports_identity_federation}} + use_cloud_connectors: {{supports_identity_federation}} +{{/if}} + +state: batch_size: {{batch_size}} - tld: {{tld}} redact: - fields: - - access_key - - secret_key - - session_token + fields: ~ program: | ( - // Credential validation: access_key_id and secret_access_key are required - // for hand-rolled SigV4 signing. Role ARN, IAM instance profiles, and - // other SDK-based credential methods are not supported. - state.?secret_key.orValue("") == "" || state.?access_key.orValue("") == "" ? - { - "events": { - "error": { - "code": "configuration_error", - "id": "missing_credentials", - "message": "AWS Config requires access_key_id and secret_access_key. Role ARN, IAM instance profiles, and other credential methods are not supported for this data stream.", - }, - }, - "want_more": false, - } - : - // Stage 1: Check if we have existing config rules to process - // If we have worklist with ConfigRules, we already fetched rules and now need to get compliance details + // Stage 1: Check if we have existing config rules to process. + // If we have worklist with ConfigRules, we already fetched rules and now + // need to get compliance details. Request signing (SigV4) is handled by + // the input's auth.aws configuration. has(state.?worklist.ConfigRules) && state.worklist.ConfigRules != null && size(state.worklist.ConfigRules) > 0 ? { // State management: preserve existing state from previous iteration @@ -70,219 +82,68 @@ program: | "has_next": state.has_next, "next": state.next, "next_page": state.next_page, - - // AWS SigV4 Signing: Generate signing key for GetComplianceDetailsByConfigRule request - // Perform a succession of keyed hash operations (HMAC) on the request date, Region, and service, - // with the AWS secret access key as the key for the initial hashing operation. - // Refer https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_sigv-create-signed-request.html#derive-signing-key - - "signing_key": "aws4_request".hmac("sha256", - "config".hmac("sha256", - state.aws_region.hmac("sha256", - now.format("20060102").hmac("sha256", - bytes("AWS4" + state.secret_key) - ) - ) - ) - ), - // Create a string_to_sign that includes the algorithm, request timestamp, credential scope, and hashed canonical request. - // Refer https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_sigv-create-signed-request.html#create-string-to-sign - - "string_to_sign": [ - 'AWS4-HMAC-SHA256', - now.format("20060102T150405Z"), - now.format("20060102") + '/' + state.aws_region + '/config/aws4_request', - [ - 'POST', - '/', - '', - "content-type:application/x-amz-json-1.1", - "host:config." + state.aws_region + "." + state.tld, - "x-amz-date:" + now.format("20060102T150405Z"), - "x-amz-target:StarlingDoveService.GetComplianceDetailsByConfigRule", - '', - "content-type;host;x-amz-date;x-amz-target", - { - ?"NextToken": state.?next_page.result_token, - "ConfigRuleName": state.worklist.ConfigRules[int(state.next)].ConfigRuleName, - "Limit": int(state.batch_size) - }.encode_json().sha256().hex() - ].join("\n").sha256().hex() - ].join("\n"), - ?"session_token": state.?session_token, - "access_key": state.access_key, - "secret_key": state.secret_key, - "aws_region": state.aws_region, - "tld": state.tld, } : // Stage 2: First time execution - need to fetch config rules first - ( - state.with({ - // AWS SigV4 Signing: Generate signing key for DescribeConfigRules request - // Perform a succession of keyed hash operations (HMAC) on the request date, Region, and service, - // with the AWS secret access key as the key for the initial hashing operation. - // Refer https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_sigv-create-signed-request.html#derive-signing-key - - "signing_key": "aws4_request".hmac("sha256", - "config".hmac("sha256", - state.aws_region.hmac("sha256", - now.format("20060102").hmac("sha256", - bytes("AWS4" + state.secret_key) - ) - ) - ) - ), - // Create a string_to_sign that includes the algorithm, request timestamp, credential scope, and hashed canonical request - // Refer https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_sigv-create-signed-request.html#create-string-to-sign - - "string_to_sign": [ - 'AWS4-HMAC-SHA256', - now.format("20060102T150405Z"), - now.format("20060102") + '/' + state.aws_region + '/config/aws4_request', - [ - 'POST', - '/', - '', - "content-type:application/x-amz-json-1.1", - "host:config." + state.aws_region + "." + state.tld, - "x-amz-date:" + now.format("20060102T150405Z"), - "x-amz-target:StarlingDoveService.DescribeConfigRules", - '', - "content-type;host;x-amz-date;x-amz-target", - { - ?"NextToken": state.?next_page.rule_token - }.encode_json().sha256().hex() - ].join("\n").sha256().hex() - ].join("\n"), - }) - ).as(state, - // API Call: Execute DescribeConfigRules request to get list of config rules - post_request( - state.url.trim_right("/"), - "application/json", - { - ?"NextToken": has(state.next_page) && has(state.next_page.rule_token) ? optional.of(state.next_page.rule_token) : optional.none(), - }.encode_json() - ).with( + // API Call: Execute DescribeConfigRules request to get list of config rules + post_request( + state.url.trim_right("/"), + "application/x-amz-json-1.1", + { + ?"NextToken": has(state.next_page) && has(state.next_page.rule_token) ? optional.of(state.next_page.rule_token) : optional.none(), + }.encode_json() + ).with( + { + "Header": { + "Content-Type": ["application/x-amz-json-1.1"], + "X-Amz-Target": ["StarlingDoveService.DescribeConfigRules"], + }, + } + ).do_request().as(resp, (resp.StatusCode == 200) ? + // Response Processing: Successfully got config rules list + resp.Body.decode_json().as(raw_body, + // Skip AWS service-linked rules. AWS owns these rules and blocks + // GetComplianceDetailsByConfigRule for them with an AccessDeniedException + // ("An AWS service owns ServiceLinkedConfigRule"). They are identified by a + // populated CreatedBy field, which AWS sets only for service-linked rules. + // Filtering them out of the worklist here avoids the blocked call entirely. + raw_body.with({ + "ConfigRules": raw_body.?ConfigRules.orValue([]).filter(r, + r.?CreatedBy.orValue("") == "" + ), + }).as(body, { - "Header": { - "Content-Type": ["application/x-amz-json-1.1"], - "X-Amz-Date": [now.format("20060102T150405Z")], - - // Perform a keyed hash operation on the string to sign using the derived signing key as the hash key to calculate the signature - // Refer https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_sigv-create-signed-request.html#calculate-signature - // and construct the Authorization header by combining the algorithm, credential scope, signed headers, and calculated signature. - // Refer https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_sigv-create-signed-request.html#add-signature-to-request - - "Authorization": [ - "AWS4-HMAC-SHA256 Credential=" - + state.access_key - + "/" + now.format("20060102") - + "/" + state.aws_region - + "/config/aws4_request, SignedHeaders=content-type;host;x-amz-date;x-amz-target, Signature=" - + (state.string_to_sign).hmac("sha256", state.signing_key).hex() - ], - "X-Amz-Target": ["StarlingDoveService.DescribeConfigRules"], - ?"X-Amz-Security-Token": has(state.session_token) ? optional.of([state.session_token]) : optional.none(), + // State Update: Store the fetched config rules as worklist + "worklist": body, + "url": state.url, + "next": 0, + "has_next": has(body.NextToken), + "batch_size": state.batch_size, + "next_page": { + ?"rule_token": body.?NextToken, }, - } - ).do_request().as(resp, (resp.StatusCode == 200) ? - // Response Processing: Successfully got config rules list - resp.Body.decode_json().as(raw_body, - // Skip AWS service-linked rules. AWS owns these rules and blocks - // GetComplianceDetailsByConfigRule for them with an AccessDeniedException - // ("An AWS service owns ServiceLinkedConfigRule"). They are identified by a - // populated CreatedBy field, which AWS sets only for service-linked rules. - // Filtering them out of the worklist here avoids the blocked call entirely. - raw_body.with({ - "ConfigRules": raw_body.?ConfigRules.orValue([]).filter(r, - r.?CreatedBy.orValue("") == "" - ), - }).as(body, - { - // State Update: Store the fetched config rules as worklist - "worklist": body, - "url": state.url, - "next": 0, - "has_next": has(body.NextToken), - "batch_size": state.batch_size, - "next_page": { - ?"rule_token": body.?NextToken, - }, - - // AWS SigV4 Signing: Prepare signing key for next GetComplianceDetailsByConfigRule request - // Perform a succession of keyed hash operations (HMAC) on the request date, Region, and service, - // with the AWS secret access key as the key for the initial hashing operation. - // Refer https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_sigv-create-signed-request.html#derive-signing-key - - "signing_key": "aws4_request".hmac("sha256", - "config".hmac("sha256", - state.aws_region.hmac("sha256", - now.format("20060102").hmac("sha256", - bytes("AWS4" + state.secret_key) - ) - ) - ) + }) + ) + : + // Error Handling: DescribeConfigRules request failed + { + "events": { + "error": { + "code": string(resp.StatusCode), + "id": string(resp.Status), + "message": "DescribeConfigRules: POST " + state.url.trim_right("/") + " " + + ( + (size(resp.Body) != 0) ? + string(resp.Body) + : + string(resp.Status) + " (" + string(resp.StatusCode) + ")" ), - // Create a string_to_sign that includes the algorithm, request timestamp, credential scope, and hashed canonical request. - // Refer https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_sigv-create-signed-request.html#create-string-to-sign - - "string_to_sign": [ - 'AWS4-HMAC-SHA256', - now.format("20060102T150405Z"), - now.format("20060102") + '/' + state.aws_region + '/config/aws4_request', - [ - 'POST', - '/', - '', - "content-type:application/x-amz-json-1.1", - "host:config." + state.aws_region + "." + state.tld, - "x-amz-date:" + now.format("20060102T150405Z"), - "x-amz-target:StarlingDoveService.GetComplianceDetailsByConfigRule", - '', - "content-type;host;x-amz-date;x-amz-target", - { - ?"NextToken": state.?next_page.result_token, - ?"ConfigRuleName": body.?ConfigRules[0].optMap(r, r.ConfigRuleName), - "Limit": int(state.batch_size) - }.encode_json().sha256().hex() - ].join("\n").sha256().hex() - ].join("\n"), - ?"session_token": state.?session_token, - "url": state.url, - "access_key": state.access_key, - "secret_key": state.secret_key, - "aws_region": state.aws_region, - "tld": state.tld, - }) - ) - : - // Error Handling: DescribeConfigRules request failed - { - "events": { - "error": { - "code": string(resp.StatusCode), - "id": string(resp.Status), - "message": "DescribeConfigRules: POST " + state.url.trim_right("/") + " " + - ( - (size(resp.Body) != 0) ? - string(resp.Body) - : - string(resp.Status) + " (" + string(resp.StatusCode) + ")" - ), - }, }, - "want_more": false, - "batch_size": state.batch_size, - ?"session_token": state.?session_token, - "url": state.url, - "access_key": state.access_key, - "secret_key": state.secret_key, - "aws_region": state.aws_region, - "tld": state.tld, - } - ) + }, + "want_more": false, + "batch_size": state.batch_size, + "url": state.url, + } ) ).as(config_rules, // Stage 3: Process results - either exit early on failure or fetch compliance details @@ -292,7 +153,7 @@ program: | // API Call: Execute GetComplianceDetailsByConfigRule for specific config rule post_request( state.url.trim_right("/"), - "application/json", + "application/x-amz-json-1.1", { ?"NextToken": has(config_rules.next_page) && has(config_rules.next_page.result_token) ? optional.of(config_rules.next_page.result_token) : optional.none(), "ConfigRuleName": config_rules.worklist.ConfigRules[int(config_rules.next)].ConfigRuleName, @@ -302,23 +163,7 @@ program: | { "Header": { "Content-Type": ["application/x-amz-json-1.1"], - "X-Amz-Date": [now.format("20060102T150405Z")], - - // Perform a keyed hash operation on the string to sign using the derived signing key as the hash key to calculate the signature - // Refer https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_sigv-create-signed-request.html#calculate-signature - // and construct the Authorization header by combining the algorithm, credential scope, signed headers, and calculated signature. - // Refer https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_sigv-create-signed-request.html#add-signature-to-request - - "Authorization": [ - "AWS4-HMAC-SHA256 Credential=" - + config_rules.access_key - + "/" + now.format("20060102") - + "/" + config_rules.aws_region - + "/config/aws4_request, SignedHeaders=content-type;host;x-amz-date;x-amz-target, Signature=" - + (config_rules.string_to_sign).hmac("sha256", config_rules.signing_key).hex() - ], "X-Amz-Target": ["StarlingDoveService.GetComplianceDetailsByConfigRule"], - ?"X-Amz-Security-Token": has(config_rules.session_token) ? optional.of([config_rules.session_token]) : optional.none(), }, } ).do_request().as(resp, (resp.StatusCode == 200) ? @@ -357,12 +202,7 @@ program: | {}, "has_next": config_rules.has_next, "batch_size": config_rules.batch_size, - ?"session_token": config_rules.?session_token, "url": config_rules.url, - "access_key": config_rules.access_key, - "secret_key": config_rules.secret_key, - "aws_region": config_rules.aws_region, - "tld": config_rules.tld, } ) : @@ -403,12 +243,7 @@ program: | {}, "has_next": config_rules.has_next, "batch_size": config_rules.batch_size, - ?"session_token": config_rules.?session_token, "url": config_rules.url, - "access_key": config_rules.access_key, - "secret_key": config_rules.secret_key, - "aws_region": config_rules.aws_region, - "tld": config_rules.tld, } ) : @@ -428,24 +263,14 @@ program: | "worklist": {}, "has_next": config_rules.?has_next.orValue(false), "batch_size": config_rules.batch_size, - ?"session_token": config_rules.?session_token, "url": config_rules.url, - "access_key": config_rules.access_key, - "secret_key": config_rules.secret_key, - "aws_region": config_rules.aws_region, - "tld": config_rules.tld, } : { "events": [], "want_more": false, "batch_size": config_rules.batch_size, - ?"session_token": config_rules.?session_token, "url": config_rules.url, - "access_key": config_rules.access_key, - "secret_key": config_rules.secret_key, - "aws_region": config_rules.aws_region, - "tld": config_rules.tld, } ) tags: diff --git a/packages/aws/manifest.yml b/packages/aws/manifest.yml index 6aea6328c5e..d0425cc4d46 100644 --- a/packages/aws/manifest.yml +++ b/packages/aws/manifest.yml @@ -1,7 +1,7 @@ format_version: 3.6.1 name: aws title: AWS -version: 7.1.1 +version: 7.2.0 description: Collect logs and metrics from Amazon Web Services (AWS) with Elastic Agent. type: integration categories: @@ -301,8 +301,6 @@ policy_templates: - type: cel title: Collect AWS Config logs via API description: Collecting AWS Config logs via API. - hide_in_var_group_options: - credential_type: [identity_federation] icons: - src: /img/logo-aws-config.svg title: AWS Config logo From b481b6fad499cb1d0493873ebd5f109454508af8 Mon Sep 17 00:00:00 2001 From: Sean Rathier Date: Fri, 31 Jul 2026 16:14:39 -0400 Subject: [PATCH 2/3] Set real PR link in changelog Co-Authored-By: Claude Fable 5 --- packages/aws/changelog.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/aws/changelog.yml b/packages/aws/changelog.yml index 1bda46352c0..764bfb005df 100644 --- a/packages/aws/changelog.yml +++ b/packages/aws/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Enable Identity Federation (Cloud Connectors) for the AWS Config data stream by migrating its CEL program from hand-rolled SigV4 signing to the input's `auth.aws` configuration. This also adds support for the Role ARN and shared-credential authentication methods that the hand-rolled signing could not use. type: enhancement - link: https://github.com/elastic/integrations/pull/99999 + link: https://github.com/elastic/integrations/pull/20437 - version: "7.1.1" changes: - description: Add `data_stream.namespace` to the Amazon Inspector vulnerability latest transform's unique key so findings are tracked per namespace, preventing findings ingested into non-default namespaces from being dropped or conflated in the latest index. Bump transform's destination suffix to `-v2`. From 9370eda41236a948bf43d82794407deb6ba56162 Mon Sep 17 00:00:00 2001 From: Sean Rathier Date: Fri, 31 Jul 2026 17:13:54 -0400 Subject: [PATCH 3/3] Update the missing-credentials script test for the auth.aws contract The old test asserted the CEL program's pre-request credential gate ('access_key_id and secret_access_key' required), which this branch deliberately removed: with auth.aws, keyless configurations are valid (Role ARN, shared credentials, instance profiles, cloud connectors), so the program cannot know at evaluation time whether credentials exist. The renamed test (unauthenticated_error) asserts the new contract observed in CI build 46979: the unauthenticated request fails at the AWS API and the program emits its DescribeConfigRules error wrapper as an error event, with no data events produced. The assertion matches the program's stable message prefix rather than the environment- dependent AWS exception text. Co-Authored-By: Claude Fable 5 --- ...edentials.txt => unauthenticated_error.txt} | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) rename packages/aws/data_stream/config/_dev/test/scripts/{missing_credentials.txt => unauthenticated_error.txt} (58%) diff --git a/packages/aws/data_stream/config/_dev/test/scripts/missing_credentials.txt b/packages/aws/data_stream/config/_dev/test/scripts/unauthenticated_error.txt similarity index 58% rename from packages/aws/data_stream/config/_dev/test/scripts/missing_credentials.txt rename to packages/aws/data_stream/config/_dev/test/scripts/unauthenticated_error.txt index 49867017720..e817ba0f797 100644 --- a/packages/aws/data_stream/config/_dev/test/scripts/missing_credentials.txt +++ b/packages/aws/data_stream/config/_dev/test/scripts/unauthenticated_error.txt @@ -1,5 +1,9 @@ -# Test that the AWS Config data stream emits a clear error event when -# access_key_id and secret_access_key are not provided. +# Test that the AWS Config data stream surfaces a clear error event when no +# usable AWS credentials are configured. With auth.aws the input no longer +# hard-requires static access keys (Role ARN, shared credentials, instance +# profiles, and cloud connectors are all valid), so there is no pre-request +# credential gate: the request is sent, fails to authenticate against AWS, +# and the program emits the API error as an error event. [!external_stack] skip 'Skipping external stack test.' [!exec:jq] skip 'Skipping test requiring absent jq command' @@ -10,14 +14,16 @@ install_agent -profile ${CONFIG_PROFILES}/${PROFILE} -network_name NETWORK_NAME add_package -profile ${CONFIG_PROFILES}/${PROFILE} add_package_policy -profile ${CONFIG_PROFILES}/${PROFILE} test_config.yaml DATA_STREAM_NAME -# The credential check fires before any HTTP request, so exactly one error -# event should be indexed per evaluation cycle. Wait for at least 1. +# The unauthenticated request fails at the API and the failure is emitted as +# an error event per evaluation cycle. Wait for at least 1. get_docs -profile ${CONFIG_PROFILES}/${PROFILE} -want 1 -timeout 5m ${DATA_STREAM_NAME} cp stdout got_docs.json -# Verify the error message names the missing credentials. +# Verify the error event carries the program's DescribeConfigRules error +# wrapper (the exact AWS exception text is environment-dependent; the wrapper +# prefix is ours and stable). exec jq -r '[.hits.hits[]._source.error.message // empty] | flatten | .[]' got_docs.json -stdout 'access_key_id and secret_access_key' +stdout 'DescribeConfigRules: POST' # Verify no data events were produced — only error events. exec jq '[.hits.hits[]._source | select(.aws.config != null)] | length' got_docs.json