diff --git a/packages/aws/changelog.yml b/packages/aws/changelog.yml index 79206a5ee87..764bfb005df 100644 --- a/packages/aws/changelog.yml +++ b/packages/aws/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "7.2.0" + changes: + - description: Enable Identity Federation (Cloud Connectors) for the AWS Config data stream by migrating its CEL program from hand-rolled SigV4 signing to the input's `auth.aws` configuration. This also adds support for the Role ARN and shared-credential authentication methods that the hand-rolled signing could not use. + type: enhancement + link: https://github.com/elastic/integrations/pull/20437 - version: "7.1.1" changes: - description: Add `data_stream.namespace` to the Amazon Inspector vulnerability latest transform's unique key so findings are tracked per namespace, preventing findings ingested into non-default namespaces from being dropped or conflated in the latest index. Bump transform's destination suffix to `-v2`. diff --git a/packages/aws/data_stream/config/_dev/test/scripts/missing_credentials.txt b/packages/aws/data_stream/config/_dev/test/scripts/unauthenticated_error.txt similarity index 58% rename from packages/aws/data_stream/config/_dev/test/scripts/missing_credentials.txt rename to packages/aws/data_stream/config/_dev/test/scripts/unauthenticated_error.txt index 49867017720..e817ba0f797 100644 --- a/packages/aws/data_stream/config/_dev/test/scripts/missing_credentials.txt +++ b/packages/aws/data_stream/config/_dev/test/scripts/unauthenticated_error.txt @@ -1,5 +1,9 @@ -# Test that the AWS Config data stream emits a clear error event when -# access_key_id and secret_access_key are not provided. +# Test that the AWS Config data stream surfaces a clear error event when no +# usable AWS credentials are configured. With auth.aws the input no longer +# hard-requires static access keys (Role ARN, shared credentials, instance +# profiles, and cloud connectors are all valid), so there is no pre-request +# credential gate: the request is sent, fails to authenticate against AWS, +# and the program emits the API error as an error event. [!external_stack] skip 'Skipping external stack test.' [!exec:jq] skip 'Skipping test requiring absent jq command' @@ -10,14 +14,16 @@ install_agent -profile ${CONFIG_PROFILES}/${PROFILE} -network_name NETWORK_NAME add_package -profile ${CONFIG_PROFILES}/${PROFILE} add_package_policy -profile ${CONFIG_PROFILES}/${PROFILE} test_config.yaml DATA_STREAM_NAME -# The credential check fires before any HTTP request, so exactly one error -# event should be indexed per evaluation cycle. Wait for at least 1. +# The unauthenticated request fails at the API and the failure is emitted as +# an error event per evaluation cycle. Wait for at least 1. get_docs -profile ${CONFIG_PROFILES}/${PROFILE} -want 1 -timeout 5m ${DATA_STREAM_NAME} cp stdout got_docs.json -# Verify the error message names the missing credentials. +# Verify the error event carries the program's DescribeConfigRules error +# wrapper (the exact AWS exception text is environment-dependent; the wrapper +# prefix is ours and stable). exec jq -r '[.hits.hits[]._source.error.message // empty] | flatten | .[]' got_docs.json -stdout 'access_key_id and secret_access_key' +stdout 'DescribeConfigRules: POST' # Verify no data events were produced — only error events. exec jq '[.hits.hits[]._source | select(.aws.config != null)] | length' got_docs.json diff --git a/packages/aws/data_stream/config/agent/stream/cel.yml.hbs b/packages/aws/data_stream/config/agent/stream/cel.yml.hbs index e44a8ffec17..73d6e4feff6 100644 --- a/packages/aws/data_stream/config/agent/stream/cel.yml.hbs +++ b/packages/aws/data_stream/config/agent/stream/cel.yml.hbs @@ -30,37 +30,49 @@ resource.retry.wait_min: {{retry_wait_min}} resource.retry.wait_max: {{retry_wait_max}} {{/if}} resource.url: https://config.{{aws_region}}.{{tld}}/ -state: - access_key: {{access_key_id}} - secret_key: {{secret_access_key}} + +auth.aws: +{{#if access_key_id}} + access_key_id: {{access_key_id}} +{{/if}} +{{#if secret_access_key}} + secret_access_key: {{secret_access_key}} +{{/if}} +{{#if session_token}} session_token: {{session_token}} - aws_region: {{aws_region}} +{{/if}} +{{#if shared_credential_file}} + shared_credential_file: {{shared_credential_file}} +{{/if}} +{{#if credential_profile_name}} + credential_profile_name: {{credential_profile_name}} +{{/if}} +{{#if role_arn}} + role_arn: {{role_arn}} +{{/if}} +{{#if external_id}} + external_id: {{external_id}} +{{/if}} +{{#if assume_role_duration}} + assume_role.duration: {{assume_role_duration}} +{{/if}} +{{#if assume_role_expiry_window}} + assume_role.expiry_window: {{assume_role_expiry_window}} +{{/if}} +{{#if supports_identity_federation}} + use_cloud_connectors: {{supports_identity_federation}} +{{/if}} + +state: batch_size: {{batch_size}} - tld: {{tld}} redact: - fields: - - access_key - - secret_key - - session_token + fields: ~ program: | ( - // Credential validation: access_key_id and secret_access_key are required - // for hand-rolled SigV4 signing. Role ARN, IAM instance profiles, and - // other SDK-based credential methods are not supported. - state.?secret_key.orValue("") == "" || state.?access_key.orValue("") == "" ? - { - "events": { - "error": { - "code": "configuration_error", - "id": "missing_credentials", - "message": "AWS Config requires access_key_id and secret_access_key. Role ARN, IAM instance profiles, and other credential methods are not supported for this data stream.", - }, - }, - "want_more": false, - } - : - // Stage 1: Check if we have existing config rules to process - // If we have worklist with ConfigRules, we already fetched rules and now need to get compliance details + // Stage 1: Check if we have existing config rules to process. + // If we have worklist with ConfigRules, we already fetched rules and now + // need to get compliance details. Request signing (SigV4) is handled by + // the input's auth.aws configuration. has(state.?worklist.ConfigRules) && state.worklist.ConfigRules != null && size(state.worklist.ConfigRules) > 0 ? { // State management: preserve existing state from previous iteration @@ -70,219 +82,68 @@ program: | "has_next": state.has_next, "next": state.next, "next_page": state.next_page, - - // AWS SigV4 Signing: Generate signing key for GetComplianceDetailsByConfigRule request - // Perform a succession of keyed hash operations (HMAC) on the request date, Region, and service, - // with the AWS secret access key as the key for the initial hashing operation. - // Refer https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_sigv-create-signed-request.html#derive-signing-key - - "signing_key": "aws4_request".hmac("sha256", - "config".hmac("sha256", - state.aws_region.hmac("sha256", - now.format("20060102").hmac("sha256", - bytes("AWS4" + state.secret_key) - ) - ) - ) - ), - // Create a string_to_sign that includes the algorithm, request timestamp, credential scope, and hashed canonical request. - // Refer https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_sigv-create-signed-request.html#create-string-to-sign - - "string_to_sign": [ - 'AWS4-HMAC-SHA256', - now.format("20060102T150405Z"), - now.format("20060102") + '/' + state.aws_region + '/config/aws4_request', - [ - 'POST', - '/', - '', - "content-type:application/x-amz-json-1.1", - "host:config." + state.aws_region + "." + state.tld, - "x-amz-date:" + now.format("20060102T150405Z"), - "x-amz-target:StarlingDoveService.GetComplianceDetailsByConfigRule", - '', - "content-type;host;x-amz-date;x-amz-target", - { - ?"NextToken": state.?next_page.result_token, - "ConfigRuleName": state.worklist.ConfigRules[int(state.next)].ConfigRuleName, - "Limit": int(state.batch_size) - }.encode_json().sha256().hex() - ].join("\n").sha256().hex() - ].join("\n"), - ?"session_token": state.?session_token, - "access_key": state.access_key, - "secret_key": state.secret_key, - "aws_region": state.aws_region, - "tld": state.tld, } : // Stage 2: First time execution - need to fetch config rules first - ( - state.with({ - // AWS SigV4 Signing: Generate signing key for DescribeConfigRules request - // Perform a succession of keyed hash operations (HMAC) on the request date, Region, and service, - // with the AWS secret access key as the key for the initial hashing operation. - // Refer https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_sigv-create-signed-request.html#derive-signing-key - - "signing_key": "aws4_request".hmac("sha256", - "config".hmac("sha256", - state.aws_region.hmac("sha256", - now.format("20060102").hmac("sha256", - bytes("AWS4" + state.secret_key) - ) - ) - ) - ), - // Create a string_to_sign that includes the algorithm, request timestamp, credential scope, and hashed canonical request - // Refer https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_sigv-create-signed-request.html#create-string-to-sign - - "string_to_sign": [ - 'AWS4-HMAC-SHA256', - now.format("20060102T150405Z"), - now.format("20060102") + '/' + state.aws_region + '/config/aws4_request', - [ - 'POST', - '/', - '', - "content-type:application/x-amz-json-1.1", - "host:config." + state.aws_region + "." + state.tld, - "x-amz-date:" + now.format("20060102T150405Z"), - "x-amz-target:StarlingDoveService.DescribeConfigRules", - '', - "content-type;host;x-amz-date;x-amz-target", - { - ?"NextToken": state.?next_page.rule_token - }.encode_json().sha256().hex() - ].join("\n").sha256().hex() - ].join("\n"), - }) - ).as(state, - // API Call: Execute DescribeConfigRules request to get list of config rules - post_request( - state.url.trim_right("/"), - "application/json", - { - ?"NextToken": has(state.next_page) && has(state.next_page.rule_token) ? optional.of(state.next_page.rule_token) : optional.none(), - }.encode_json() - ).with( + // API Call: Execute DescribeConfigRules request to get list of config rules + post_request( + state.url.trim_right("/"), + "application/x-amz-json-1.1", + { + ?"NextToken": has(state.next_page) && has(state.next_page.rule_token) ? optional.of(state.next_page.rule_token) : optional.none(), + }.encode_json() + ).with( + { + "Header": { + "Content-Type": ["application/x-amz-json-1.1"], + "X-Amz-Target": ["StarlingDoveService.DescribeConfigRules"], + }, + } + ).do_request().as(resp, (resp.StatusCode == 200) ? + // Response Processing: Successfully got config rules list + resp.Body.decode_json().as(raw_body, + // Skip AWS service-linked rules. AWS owns these rules and blocks + // GetComplianceDetailsByConfigRule for them with an AccessDeniedException + // ("An AWS service owns ServiceLinkedConfigRule"). They are identified by a + // populated CreatedBy field, which AWS sets only for service-linked rules. + // Filtering them out of the worklist here avoids the blocked call entirely. + raw_body.with({ + "ConfigRules": raw_body.?ConfigRules.orValue([]).filter(r, + r.?CreatedBy.orValue("") == "" + ), + }).as(body, { - "Header": { - "Content-Type": ["application/x-amz-json-1.1"], - "X-Amz-Date": [now.format("20060102T150405Z")], - - // Perform a keyed hash operation on the string to sign using the derived signing key as the hash key to calculate the signature - // Refer https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_sigv-create-signed-request.html#calculate-signature - // and construct the Authorization header by combining the algorithm, credential scope, signed headers, and calculated signature. - // Refer https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_sigv-create-signed-request.html#add-signature-to-request - - "Authorization": [ - "AWS4-HMAC-SHA256 Credential=" - + state.access_key - + "/" + now.format("20060102") - + "/" + state.aws_region - + "/config/aws4_request, SignedHeaders=content-type;host;x-amz-date;x-amz-target, Signature=" - + (state.string_to_sign).hmac("sha256", state.signing_key).hex() - ], - "X-Amz-Target": ["StarlingDoveService.DescribeConfigRules"], - ?"X-Amz-Security-Token": has(state.session_token) ? optional.of([state.session_token]) : optional.none(), + // State Update: Store the fetched config rules as worklist + "worklist": body, + "url": state.url, + "next": 0, + "has_next": has(body.NextToken), + "batch_size": state.batch_size, + "next_page": { + ?"rule_token": body.?NextToken, }, - } - ).do_request().as(resp, (resp.StatusCode == 200) ? - // Response Processing: Successfully got config rules list - resp.Body.decode_json().as(raw_body, - // Skip AWS service-linked rules. AWS owns these rules and blocks - // GetComplianceDetailsByConfigRule for them with an AccessDeniedException - // ("An AWS service owns ServiceLinkedConfigRule"). They are identified by a - // populated CreatedBy field, which AWS sets only for service-linked rules. - // Filtering them out of the worklist here avoids the blocked call entirely. - raw_body.with({ - "ConfigRules": raw_body.?ConfigRules.orValue([]).filter(r, - r.?CreatedBy.orValue("") == "" - ), - }).as(body, - { - // State Update: Store the fetched config rules as worklist - "worklist": body, - "url": state.url, - "next": 0, - "has_next": has(body.NextToken), - "batch_size": state.batch_size, - "next_page": { - ?"rule_token": body.?NextToken, - }, - - // AWS SigV4 Signing: Prepare signing key for next GetComplianceDetailsByConfigRule request - // Perform a succession of keyed hash operations (HMAC) on the request date, Region, and service, - // with the AWS secret access key as the key for the initial hashing operation. - // Refer https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_sigv-create-signed-request.html#derive-signing-key - - "signing_key": "aws4_request".hmac("sha256", - "config".hmac("sha256", - state.aws_region.hmac("sha256", - now.format("20060102").hmac("sha256", - bytes("AWS4" + state.secret_key) - ) - ) - ) + }) + ) + : + // Error Handling: DescribeConfigRules request failed + { + "events": { + "error": { + "code": string(resp.StatusCode), + "id": string(resp.Status), + "message": "DescribeConfigRules: POST " + state.url.trim_right("/") + " " + + ( + (size(resp.Body) != 0) ? + string(resp.Body) + : + string(resp.Status) + " (" + string(resp.StatusCode) + ")" ), - // Create a string_to_sign that includes the algorithm, request timestamp, credential scope, and hashed canonical request. - // Refer https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_sigv-create-signed-request.html#create-string-to-sign - - "string_to_sign": [ - 'AWS4-HMAC-SHA256', - now.format("20060102T150405Z"), - now.format("20060102") + '/' + state.aws_region + '/config/aws4_request', - [ - 'POST', - '/', - '', - "content-type:application/x-amz-json-1.1", - "host:config." + state.aws_region + "." + state.tld, - "x-amz-date:" + now.format("20060102T150405Z"), - "x-amz-target:StarlingDoveService.GetComplianceDetailsByConfigRule", - '', - "content-type;host;x-amz-date;x-amz-target", - { - ?"NextToken": state.?next_page.result_token, - ?"ConfigRuleName": body.?ConfigRules[0].optMap(r, r.ConfigRuleName), - "Limit": int(state.batch_size) - }.encode_json().sha256().hex() - ].join("\n").sha256().hex() - ].join("\n"), - ?"session_token": state.?session_token, - "url": state.url, - "access_key": state.access_key, - "secret_key": state.secret_key, - "aws_region": state.aws_region, - "tld": state.tld, - }) - ) - : - // Error Handling: DescribeConfigRules request failed - { - "events": { - "error": { - "code": string(resp.StatusCode), - "id": string(resp.Status), - "message": "DescribeConfigRules: POST " + state.url.trim_right("/") + " " + - ( - (size(resp.Body) != 0) ? - string(resp.Body) - : - string(resp.Status) + " (" + string(resp.StatusCode) + ")" - ), - }, }, - "want_more": false, - "batch_size": state.batch_size, - ?"session_token": state.?session_token, - "url": state.url, - "access_key": state.access_key, - "secret_key": state.secret_key, - "aws_region": state.aws_region, - "tld": state.tld, - } - ) + }, + "want_more": false, + "batch_size": state.batch_size, + "url": state.url, + } ) ).as(config_rules, // Stage 3: Process results - either exit early on failure or fetch compliance details @@ -292,7 +153,7 @@ program: | // API Call: Execute GetComplianceDetailsByConfigRule for specific config rule post_request( state.url.trim_right("/"), - "application/json", + "application/x-amz-json-1.1", { ?"NextToken": has(config_rules.next_page) && has(config_rules.next_page.result_token) ? optional.of(config_rules.next_page.result_token) : optional.none(), "ConfigRuleName": config_rules.worklist.ConfigRules[int(config_rules.next)].ConfigRuleName, @@ -302,23 +163,7 @@ program: | { "Header": { "Content-Type": ["application/x-amz-json-1.1"], - "X-Amz-Date": [now.format("20060102T150405Z")], - - // Perform a keyed hash operation on the string to sign using the derived signing key as the hash key to calculate the signature - // Refer https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_sigv-create-signed-request.html#calculate-signature - // and construct the Authorization header by combining the algorithm, credential scope, signed headers, and calculated signature. - // Refer https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_sigv-create-signed-request.html#add-signature-to-request - - "Authorization": [ - "AWS4-HMAC-SHA256 Credential=" - + config_rules.access_key - + "/" + now.format("20060102") - + "/" + config_rules.aws_region - + "/config/aws4_request, SignedHeaders=content-type;host;x-amz-date;x-amz-target, Signature=" - + (config_rules.string_to_sign).hmac("sha256", config_rules.signing_key).hex() - ], "X-Amz-Target": ["StarlingDoveService.GetComplianceDetailsByConfigRule"], - ?"X-Amz-Security-Token": has(config_rules.session_token) ? optional.of([config_rules.session_token]) : optional.none(), }, } ).do_request().as(resp, (resp.StatusCode == 200) ? @@ -357,12 +202,7 @@ program: | {}, "has_next": config_rules.has_next, "batch_size": config_rules.batch_size, - ?"session_token": config_rules.?session_token, "url": config_rules.url, - "access_key": config_rules.access_key, - "secret_key": config_rules.secret_key, - "aws_region": config_rules.aws_region, - "tld": config_rules.tld, } ) : @@ -403,12 +243,7 @@ program: | {}, "has_next": config_rules.has_next, "batch_size": config_rules.batch_size, - ?"session_token": config_rules.?session_token, "url": config_rules.url, - "access_key": config_rules.access_key, - "secret_key": config_rules.secret_key, - "aws_region": config_rules.aws_region, - "tld": config_rules.tld, } ) : @@ -428,24 +263,14 @@ program: | "worklist": {}, "has_next": config_rules.?has_next.orValue(false), "batch_size": config_rules.batch_size, - ?"session_token": config_rules.?session_token, "url": config_rules.url, - "access_key": config_rules.access_key, - "secret_key": config_rules.secret_key, - "aws_region": config_rules.aws_region, - "tld": config_rules.tld, } : { "events": [], "want_more": false, "batch_size": config_rules.batch_size, - ?"session_token": config_rules.?session_token, "url": config_rules.url, - "access_key": config_rules.access_key, - "secret_key": config_rules.secret_key, - "aws_region": config_rules.aws_region, - "tld": config_rules.tld, } ) tags: diff --git a/packages/aws/manifest.yml b/packages/aws/manifest.yml index 6aea6328c5e..d0425cc4d46 100644 --- a/packages/aws/manifest.yml +++ b/packages/aws/manifest.yml @@ -1,7 +1,7 @@ format_version: 3.6.1 name: aws title: AWS -version: 7.1.1 +version: 7.2.0 description: Collect logs and metrics from Amazon Web Services (AWS) with Elastic Agent. type: integration categories: @@ -301,8 +301,6 @@ policy_templates: - type: cel title: Collect AWS Config logs via API description: Collecting AWS Config logs via API. - hide_in_var_group_options: - credential_type: [identity_federation] icons: - src: /img/logo-aws-config.svg title: AWS Config logo