Add GetAWSCredentials function in libbeat common

CHANGELOG.next.asciidoc

@@ -278,6 +278,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add Oracle Module {pull}11890[11890]
 - Add Oracle Tablespaces Dashboard {pull}12736[12736]
 - Collect client provided name for rabbitmq connection. {issue}12851[12851] {pull}12852[12852]
+- Add support to load default aws config file to get credentials. {pull}12727[12727] {issue}12708[12708]
 
 *Packetbeat*
 

metricbeat/docs/index.asciidoc

@@ -1,6 +1,7 @@
 = Metricbeat Reference
 
 :libbeat-dir: ../../libbeat
+:libbeat-xpack-dir: ../../../x-pack/libbeat
 
 include::{libbeat-dir}/docs/version.asciidoc[]
 

metricbeat/docs/modules/aws.asciidoc

@@ -14,38 +14,16 @@ The default metricsets are `ec2`, `sqs`, `s3_request`, `s3_daily_storage`, `clou
 [float]
 === Module-specific configuration notes
 
-This module uses environment variable `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_SESSION_TOKEN` and `AWS_REGION`
-that are referenced in the config file to set values that need to be configurable during deployment. It also
-accept optional configuration `regions` to specify what are the AWS regions to query metrics from. If `regions`
-parameter is not set in the config file, then by default, aws module will query metrics from all available
+The `aws` module requires AWS credentials configuration in order to make AWS API calls.
+Users can either use `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` and/or
+`AWS_SESSION_TOKEN`, or use shared AWS credentials file.
+Please see <<aws-credentials-config>> for more details.
+
+This module also accepts optional configuration `regions` to specify which
+AWS regions to query metrics from. If the `regions` parameter is not set in the
+config file, then by default, the `aws` module will query metrics from all available
 AWS regions.
 
-There are two different kinds of AWS credentials can be used here: `access keys` and `temporary security credentials`.
-`AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` are the two parts of `access keys`. They are long-term credentials for
-an IAM user or the AWS account root user. Please see
-https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys[AWS Access Keys
- and Secret Access Keys] for more details. `temporary security credentials` has a limited lifetime and consists of an access key ID,
-a secret access key, and a security token which typically returned from `GetSessionToken`. MFA-enabled IAM users would
-need to submit an MFA code while calling `GetSessionToken`. `default_region` identifies the AWS Region whose servers you want to send
-your first API request to by default. This is typically the Region closest to you, but it can be any Region.
-Please see https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp.html[Temporary Security Credentials] for more details.
-`sts get-session-token` AWS CLI can be used to generate temporary credentials. For example. with MFA-enabled:
-
-----
-aws> sts get-session-token --serial-number arn:aws:iam::1234:mfa/your-email@example.com --token-code 456789 --duration-seconds 129600
-----
-
-Because temporary security credentials are short term, after they expire, the user needs to generate new ones and modify
-the aws.yml config file with the new credentials. Unless https://www.elastic.co/guide/en/beats/metricbeat/current/_live_reloading.html[live reloading]
-feature is enabled for Metricbeat, the user needs to manually restart Metricbeat after updating the config file in order
-to continue collecting Cloudwatch metrics. This will cause data loss if the config file is not updated with new
-credentials before the old ones expire. For Metricbeat, we recommend users to use access keys in config file to enable
-aws module making AWS api calls without have to generate new temporary credentials and update the config frequently.
-
-IAM policy is an entity that defines permissions to an object within your AWS environment. Specific permissions needs
-to be added into the IAM user's policy to authorize Metricbeat to collect AWS monitoring metrics. Please see documentation
-under each metricset for required permissions.
-
 The aws module comes with a predefined dashboard. For example:
 
 image::./images/metricbeat-aws-overview.png[]
@@ -109,6 +87,9 @@ for a list of AWS services that publish metrics to CloudWatch.
 `period` for `rds` metricset is recommended to be `60s` or multiples of `60s` because Amazon RDS sends metrics and
 dimensions to Amazon CloudWatch every minute.
 
+[id="aws-credentials-config"]
+include::{libbeat-xpack-dir}/docs/aws-credentials-config.asciidoc[]
+
 
 [float]
 === Example configuration

x-pack/libbeat/common/aws/credentials.go

@@ -0,0 +1,53 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package aws
+
+import (
+	awssdk "github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/aws/defaults"
+	"github.com/aws/aws-sdk-go-v2/aws/external"
+)
+
+// ConfigAWS is a structure defined for AWS credentials
+type ConfigAWS struct {
+	AccessKeyID     string `config:"access_key_id"`
+	SecretAccessKey string `config:"secret_access_key"`
+	SessionToken    string `config:"session_token"`
+	ProfileName     string `config:"credential_profile_name"`
+}
+
+// GetAWSCredentials function gets aws credentials from the config.
+// If access_key_id and secret_access_key are given, then use them as credentials.
+// If not, then load from aws config file. If credential_profile_name is not
+// given, then load default profile from the aws config file.
+func GetAWSCredentials(config ConfigAWS) (awssdk.Config, error) {
+	// Check if accessKeyID or secretAccessKey or sessionToken is given from configuration
+	if config.AccessKeyID != "" || config.SecretAccessKey != "" || config.SessionToken != "" {
+		awsConfig := defaults.Config()
+		awsCredentials := awssdk.Credentials{
+			AccessKeyID:     config.AccessKeyID,
+			SecretAccessKey: config.SecretAccessKey,
+		}
+
+		if config.SessionToken != "" {
+			awsCredentials.SessionToken = config.SessionToken
+		}
+
+		awsConfig.Credentials = awssdk.StaticCredentialsProvider{
+			Value: awsCredentials,
+		}
+		return awsConfig, nil
+	}
+
+	// If accessKeyID, secretAccessKey or sessionToken is not given, then load from default config
+	// Please see https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-profiles.html
+	// with more details.
+	if config.ProfileName != "" {
+		return external.LoadDefaultAWSConfig(
+			external.WithSharedConfigProfile(config.ProfileName),
+		)
+	}
+	return external.LoadDefaultAWSConfig()
+}

x-pack/libbeat/common/aws/credentials_test.go

@@ -0,0 +1,28 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package aws
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGetAWSCredentials(t *testing.T) {
+	inputConfig := ConfigAWS{
+		AccessKeyID:     "123",
+		SecretAccessKey: "abc",
+		SessionToken:    "fake-session-token",
+	}
+	awsConfig, err := GetAWSCredentials(inputConfig)
+	assert.NoError(t, err)
+
+	retrievedAWSConfig, err := awsConfig.Credentials.Retrieve()
+	assert.NoError(t, err)
+
+	assert.Equal(t, inputConfig.AccessKeyID, retrievedAWSConfig.AccessKeyID)
+	assert.Equal(t, inputConfig.SecretAccessKey, retrievedAWSConfig.SecretAccessKey)
+	assert.Equal(t, inputConfig.SessionToken, retrievedAWSConfig.SessionToken)
+}

x-pack/libbeat/docs/aws-credentials-config.asciidoc

@@ -0,0 +1,96 @@
+[float]
+=== AWS Credentials Configuration
+To configure AWS credentials, there are two different ways supported:
+
+[float]
+==== Supported Formats
+* Use `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` and/or `AWS_SESSION_TOKEN`
+
+Users can either put the credentials into metricbeat module configuration or use
+environment variable `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` and/or
+`AWS_SESSION_TOKEN` instead.
+
+[source,yaml]
+----
+metricbeat.modules:
+- module: aws
+  period: 300s
+  metricsets:
+    - ec2
+  access_key_id: '<access_key_id>'
+  secret_access_key: '<secret_access_key>'
+  session_token: '<session_token>'
+----
+
+or
+
+[source,yaml]
+----
+metricbeat.modules:
+- module: aws
+  period: 300s
+  metricsets:
+    - ec2
+  access_key_id: '${AWS_ACCESS_KEY_ID:""}'
+  secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}'
+  session_token: '${AWS_SESSION_TOKEN:""}'
+----
+
+* Use shared AWS credentials file
+
+[source,yaml]
+----
+metricbeat.modules:
+- module: aws
+  period: 300s
+  metricsets:
+    - ec2
+  credential_profile_name: test-mb
+----
+`credential_profile_name` is optional. If there is no `credential_profile_name`
+given, the default profile will be used.
+In Windows, shared credentials file is at `C:\Users\<yourUserName>\.aws\credentials`.
+For Linux, macOS or Unix, the file is located at `~/.aws/credentials`. Please see
+https://docs.aws.amazon.com/ses/latest/DeveloperGuide/create-shared-credentials-file.html[Create Shared Credentials File]
+for more details.
+
+[float]
+==== AWS Credentials Types
+There are two different types of AWS credentials can be used:
+access keys and temporary security credentials.
+
+* Access keys
+
+`AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` are the two parts of access keys.
+They are long-term credentials for an IAM user or the AWS account root user.
+Please see
+https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys[AWS Access Keys
+ and Secret Access Keys]
+for more details.
+
+* Temporary security credentials
+
+temporary security credentials has a limited lifetime and consists of an
+access key ID, a secret access key, and a security token which typically returned
+from `GetSessionToken`. MFA-enabled IAM users would need to submit an MFA code
+while calling `GetSessionToken`. `default_region` identifies the AWS Region
+whose servers you want to send your first API request to by default. This is
+typically the Region closest to you, but it can be any Region. Please see
+https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp.html[Temporary Security Credentials]
+for more details.
+`sts get-session-token` AWS CLI can be used to generate temporary credentials. For example. with MFA-enabled:
+
+----
+aws> sts get-session-token --serial-number arn:aws:iam::1234:mfa/your-email@example.com --token-code 456789 --duration-seconds 129600
+----
+
+Because temporary security credentials are short term, after they expire, the user needs to generate new ones and modify
+the aws.yml config file with the new credentials. Unless https://www.elastic.co/guide/en/beats/metricbeat/current/_live_reloading.html[live reloading]
+feature is enabled for Metricbeat, the user needs to manually restart Metricbeat after updating the config file in order
+to continue collecting Cloudwatch metrics. This will cause data loss if the config file is not updated with new
+credentials before the old ones expire. For Metricbeat, we recommend users to use access keys in config file to enable
+aws module making AWS api calls without have to generate new temporary credentials and update the config frequently.
+
+IAM policy is an entity that defines permissions to an object within your AWS environment. Specific permissions needs
+to be added into the IAM user's policy to authorize Metricbeat to collect AWS monitoring metrics. Please see documentation
+under each metricset for required permissions.

x-pack/metricbeat/module/aws/_meta/config.yml

@@ -2,20 +2,12 @@
   period: 300s
   metricsets:
     - ec2
-  access_key_id: '${AWS_ACCESS_KEY_ID:""}'
-  secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}'
-  session_token: '${AWS_SESSION_TOKEN:""}'
-  default_region: '${AWS_REGION:us-west-1}'
   regions:
     - us-east-1
 - module: aws
   period: 300s
   metricsets:
     - cloudwatch
-  access_key_id: '${AWS_ACCESS_KEY_ID:""}'
-  secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}'
-  session_token: '${AWS_SESSION_TOKEN:""}'
-  default_region: '${AWS_REGION:us-west-1}'
   cloudwatch_metrics:
     - namespace: AWS/EBS
     - namespace: AWS/ELB

x-pack/metricbeat/module/aws/_meta/docs.asciidoc

@@ -7,38 +7,16 @@ The default metricsets are `ec2`, `sqs`, `s3_request`, `s3_daily_storage`, `clou
 [float]
 === Module-specific configuration notes
 
-This module uses environment variable `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_SESSION_TOKEN` and `AWS_REGION`
-that are referenced in the config file to set values that need to be configurable during deployment. It also
-accept optional configuration `regions` to specify what are the AWS regions to query metrics from. If `regions`
-parameter is not set in the config file, then by default, aws module will query metrics from all available
+The `aws` module requires AWS credentials configuration in order to make AWS API calls.
+Users can either use `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` and/or
+`AWS_SESSION_TOKEN`, or use shared AWS credentials file.
+Please see <<aws-credentials-config>> for more details.
+
+This module also accepts optional configuration `regions` to specify which
+AWS regions to query metrics from. If the `regions` parameter is not set in the
+config file, then by default, the `aws` module will query metrics from all available
 AWS regions.
 
-There are two different kinds of AWS credentials can be used here: `access keys` and `temporary security credentials`.
-`AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` are the two parts of `access keys`. They are long-term credentials for
-an IAM user or the AWS account root user. Please see
-https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys[AWS Access Keys
- and Secret Access Keys] for more details. `temporary security credentials` has a limited lifetime and consists of an access key ID,
-a secret access key, and a security token which typically returned from `GetSessionToken`. MFA-enabled IAM users would
-need to submit an MFA code while calling `GetSessionToken`. `default_region` identifies the AWS Region whose servers you want to send
-your first API request to by default. This is typically the Region closest to you, but it can be any Region.
-Please see https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp.html[Temporary Security Credentials] for more details.
-`sts get-session-token` AWS CLI can be used to generate temporary credentials. For example. with MFA-enabled:
-
-----
-aws> sts get-session-token --serial-number arn:aws:iam::1234:mfa/your-email@example.com --token-code 456789 --duration-seconds 129600
-----
-
-Because temporary security credentials are short term, after they expire, the user needs to generate new ones and modify
-the aws.yml config file with the new credentials. Unless https://www.elastic.co/guide/en/beats/metricbeat/current/_live_reloading.html[live reloading]
-feature is enabled for Metricbeat, the user needs to manually restart Metricbeat after updating the config file in order
-to continue collecting Cloudwatch metrics. This will cause data loss if the config file is not updated with new
-credentials before the old ones expire. For Metricbeat, we recommend users to use access keys in config file to enable
-aws module making AWS api calls without have to generate new temporary credentials and update the config frequently.
-
-IAM policy is an entity that defines permissions to an object within your AWS environment. Specific permissions needs
-to be added into the IAM user's policy to authorize Metricbeat to collect AWS monitoring metrics. Please see documentation
-under each metricset for required permissions.
-
 The aws module comes with a predefined dashboard. For example:
 
 image::./images/metricbeat-aws-overview.png[]
@@ -101,3 +79,6 @@ for a list of AWS services that publish metrics to CloudWatch.
 === `rds`
 `period` for `rds` metricset is recommended to be `60s` or multiples of `60s` because Amazon RDS sends metrics and
 dimensions to Amazon CloudWatch every minute.
+
+[id="aws-credentials-config"]
+include::{libbeat-xpack-dir}/docs/aws-credentials-config.asciidoc[]

x-pack/metricbeat/module/aws/aws.go

@@ -8,25 +8,21 @@ import (
 	"context"
 	"time"
 
-	"github.com/elastic/beats/libbeat/common"
-
 	awssdk "github.com/aws/aws-sdk-go-v2/aws"
-	"github.com/aws/aws-sdk-go-v2/aws/defaults"
 	"github.com/aws/aws-sdk-go-v2/service/ec2"
 	"github.com/aws/aws-sdk-go-v2/service/ec2/ec2iface"
 	"github.com/pkg/errors"
 
+	"github.com/elastic/beats/libbeat/common"
 	"github.com/elastic/beats/metricbeat/mb"
+	awscommon "github.com/elastic/beats/x-pack/libbeat/common/aws"
 )
 
 // Config defines all required and optional parameters for aws metricsets
 type Config struct {
-	Period          time.Duration `config:"period" validate:"nonzero,required"`
-	AccessKeyID     string        `config:"access_key_id" validate:"nonzero,required"`
-	SecretAccessKey string        `config:"secret_access_key" validate:"nonzero,required"`
-	SessionToken    string        `config:"session_token"`
-	DefaultRegion   string        `config:"default_region"`
-	Regions         []string      `config:"regions"`
+	Period    time.Duration `config:"period" validate:"nonzero,required"`
+	Regions   []string      `config:"regions"`
+	AWSConfig awscommon.ConfigAWS
 }
 
 // MetricSet is the base metricset for all aws metricsets
@@ -62,21 +58,11 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) {
 		return nil, err
 	}
 
-	awsConfig := defaults.Config()
-	awsCredentials := awssdk.Credentials{
-		AccessKeyID:     config.AccessKeyID,
-		SecretAccessKey: config.SecretAccessKey,
-	}
-	if config.SessionToken != "" {
-		awsCredentials.SessionToken = config.SessionToken
-	}
-
-	awsConfig.Credentials = awssdk.StaticCredentialsProvider{
-		Value: awsCredentials,
+	awsConfig, err := awscommon.GetAWSCredentials(config.AWSConfig)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to get aws credentials")
 	}
 
-	awsConfig.Region = config.DefaultRegion
-
 	metricSet := MetricSet{
 		BaseMetricSet: base,
 		Period:        config.Period,