aws - add credential caching for aws assume role sessions (#37788)

Add caching so that AWS `AssumeRole` session credentials are not requested for every single request. Sessions are valid for 15m by default but without caching that does not matter. This will speed up requests for users of `role_arn` by removing the overhead of most STS (session token service) calls and stop users from hitting rate-limiting issues with the STS.

Fixes #37787

(cherry picked from commit a6e5b04)

# Conflicts:
#	x-pack/libbeat/common/aws/credentials.go
#	x-pack/libbeat/docs/aws-credentials-config.asciidoc

CHANGELOG.next.asciidoc

@@ -32,6 +32,20 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 
 *Affecting all Beats*
 
+- Support for multiline zookeeper logs {issue}2496[2496]
+- Add checks to ensure reloading of units if the configuration actually changed. {pull}34346[34346]
+- Fix namespacing on self-monitoring {pull}32336[32336]
+- Fix namespacing on self-monitoring {pull}32336[32336]
+- Fix Beats started by agent do not respect the allow_older_versions: true configuration flag {issue}34227[34227] {pull}34964[34964]
+- Fix performance issues when we have a lot of inputs starting and stopping by allowing to disable global processors under fleet. {issue}35000[35000] {pull}35031[35031]
+- 'add_cloud_metadata' processor - add cloud.region field for GCE cloud provider
+- 'add_cloud_metadata' processor - update azure metadata api version to get missing `cloud.account.id` field
+- Upgraded apache arrow library used in x-pack/libbeat/reader/parquet from v11 to v12.0.1 in order to fix cross-compilation issues {pull}35640[35640]
+- Fix panic when MaxRetryInterval is specified, but RetryInterval is not {pull}35820[35820]
+- Support build of projects outside of beats directory {pull}36126[36126]
+- Support Elastic Agent control protocol chunking support {pull}37343[37343]
+- Upgrade elastic-agent-libs to v0.7.5. Removes obsolete "Treating the CommonName field on X.509 certificates as a host name..." deprecation warning for 8.0. {pull}37755[37755]
+- aws: Add credential caching for `AssumeRole` session tokens. {issue}37787[37787]
 
 *Auditbeat*
 

x-pack/libbeat/common/aws/credentials.go

@@ -7,6 +7,15 @@ package aws
 import (
 	"fmt"
 	"net/http"
+<<<<<<< HEAD
+=======
+	"net/url"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/service/sts"
+
+	"github.com/aws/aws-sdk-go-v2/credentials/stscreds"
+>>>>>>> a6e5b04fad (aws - add credential caching for aws assume role sessions (#37788))
 
 	awssdk "github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/aws/defaults"
@@ -21,6 +30,7 @@ import (
 
 // ConfigAWS is a structure defined for AWS credentials
 type ConfigAWS struct {
+<<<<<<< HEAD
 	AccessKeyID          string `config:"access_key_id"`
 	SecretAccessKey      string `config:"secret_access_key"`
 	SessionToken         string `config:"session_token"`
@@ -30,6 +40,27 @@ type ConfigAWS struct {
 	RoleArn              string `config:"role_arn"`
 	AWSPartition         string `config:"aws_partition"` // Deprecated.
 	ProxyUrl             string `config:"proxy_url"`
+=======
+	AccessKeyID          string            `config:"access_key_id"`
+	SecretAccessKey      string            `config:"secret_access_key"`
+	SessionToken         string            `config:"session_token"`
+	ProfileName          string            `config:"credential_profile_name"`
+	SharedCredentialFile string            `config:"shared_credential_file"`
+	Endpoint             string            `config:"endpoint"`
+	RoleArn              string            `config:"role_arn"`
+	ExternalID           string            `config:"external_id"`
+	ProxyUrl             string            `config:"proxy_url"`
+	FIPSEnabled          bool              `config:"fips_enabled"`
+	TLS                  *tlscommon.Config `config:"ssl" yaml:"ssl,omitempty" json:"ssl,omitempty"`
+	DefaultRegion        string            `config:"default_region"`
+
+	// The duration of the role session. Defaults to 15m when not set.
+	AssumeRoleDuration time.Duration `config:"assume_role.duration"`
+
+	// AssumeRoleExpiryWindow will allow the credentials to trigger refreshing prior to the credentials
+	// actually expiring. If expiry_window is less than or equal to zero, the setting is ignored.
+	AssumeRoleExpiryWindow time.Duration `config:"assume_role.expiry_window"`
+>>>>>>> a6e5b04fad (aws - add credential caching for aws assume role sessions (#37788))
 }
 
 // InitializeAWSConfig function creates the awssdk.Config object from the provided config
@@ -126,10 +157,27 @@ func getSharedCredentialProfile(config ConfigAWS) (awssdk.Config, error) {
 func switchToAssumeRoleProvider(config ConfigAWS, awsConfig awssdk.Config) awssdk.Config {
 	logger := logp.NewLogger("switchToAssumeRoleProvider")
 	logger.Debug("Switching credentials provider to AssumeRoleProvider")
+<<<<<<< HEAD
 	stsSvc := sts.New(awsConfig)
 	stsCredProvider := stscreds.NewAssumeRoleProvider(stsSvc, config.RoleArn)
 	awsConfig.Credentials = stsCredProvider
 	return awsConfig
+=======
+	stsSvc := sts.NewFromConfig(*awsConfig)
+	stsCredProvider := stscreds.NewAssumeRoleProvider(stsSvc, config.RoleArn, func(aro *stscreds.AssumeRoleOptions) {
+		if config.ExternalID != "" {
+			aro.ExternalID = awssdk.String(config.ExternalID)
+		}
+		if config.AssumeRoleDuration > 0 {
+			aro.Duration = config.AssumeRoleDuration
+		}
+	})
+	awsConfig.Credentials = awssdk.NewCredentialsCache(stsCredProvider, func(options *awssdk.CredentialsCacheOptions) {
+		if config.AssumeRoleExpiryWindow > 0 {
+			options.ExpiryWindow = config.AssumeRoleExpiryWindow
+		}
+	})
+>>>>>>> a6e5b04fad (aws - add credential caching for aws assume role sessions (#37788))
 }
 
 // EnrichAWSConfigWithEndpoint function enabled endpoint resolver for AWS

x-pack/libbeat/docs/aws-credentials-config.asciidoc

@@ -18,6 +18,15 @@ services do not include a region. In `aws` module, `endpoint` config is to set
 the `endpoint-code` part, such as `amazonaws.com`, `amazonaws.com.cn`, `c2s.ic.gov`,
 `sc2s.sgov.gov`.
 * *proxy_url*: URL of the proxy to use to connect to AWS web services. The syntax is `http(s)://<IP/Hostname>:<port>`
+<<<<<<< HEAD
+=======
+* *fips_enabled*: Enabling this option instructs {beatname_uc} to use the FIPS endpoint of a service. All services used by {beatname_uc} are FIPS compatible except for `tagging` but only certain regions are FIPS compatible. See https://aws.amazon.com/compliance/fips/ or the appropriate service page, https://docs.aws.amazon.com/general/latest/gr/aws-service-information.html, for a full list of FIPS endpoints and regions.
+* *ssl*: This specifies SSL/TLS configuration. If the ssl section is missing, the host's CAs are used for HTTPS connections. See <<configuration-ssl>> for more information.
+* *default_region*: Default region to query if no other region is set. Most AWS services offer a regional endpoint that can be used to make requests. Some services, such as IAM, do not support regions. If a region is not provided by any other way (environment variable, credential or instance profile), the value set here will be used.
+* *assume_role.duration*: The duration of the requested assume role session. Defaults to 15m when not set. AWS allows a maximum session duration between 1h and 12h depending on your maximum session duration policies.
+* *assume_role.expiry_window*: The expiry_window will allow refreshing the session prior to its expiration.
+  This is beneficial to prevent expiring tokens from causing requests to fail with an ExpiredTokenException.
+>>>>>>> a6e5b04fad (aws - add credential caching for aws assume role sessions (#37788))
 
 [float]
 ==== Supported Formats