From 4d8b3908558f9545f13c02176f85e053c7e65f5c Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Thu, 8 Sep 2022 09:21:23 +0200 Subject: [PATCH] [8.4](backport #32921) Refactor endpoint resolver for AWS services (#33014) * Refactor endpoint resolver for AWS services (#32921) * Add signing region * Add changelog entry * remove awscommon.EnrichAWSConfigWithEndpoint * add nonAWSBucketResolver when recreating the s3 client for new region * remove unused functions * changelog * fix changelog * fix docs Co-authored-by: Andrea Spacca (cherry picked from commit d0bc413a50eeebb4b183919429ecaaae7d989234) # Conflicts: # x-pack/libbeat/common/aws/credentials.go # x-pack/libbeat/common/aws/credentials_test.go * fix merge Co-authored-by: Mario Castro Co-authored-by: Andrea Spacca --- CHANGELOG.next.asciidoc | 2 +- .../docs/inputs/input-aws-s3.asciidoc | 3 +- x-pack/filebeat/input/awscloudwatch/input.go | 8 +- x-pack/filebeat/input/awss3/input.go | 44 +++++-- .../input/awss3/input_integration_test.go | 4 +- .../providers/aws/ec2/provider.go | 18 ++- .../providers/aws/elb/provider.go | 19 ++- x-pack/libbeat/common/aws/credentials.go | 38 ------ x-pack/libbeat/common/aws/credentials_test.go | 113 ------------------ .../docs/aws-credentials-config.asciidoc | 11 +- x-pack/metricbeat/module/aws/aws.go | 31 ++--- .../metricbeat/module/aws/billing/billing.go | 23 ++-- .../module/aws/cloudwatch/cloudwatch.go | 18 +-- .../module/aws/cloudwatch/metadata.go | 8 +- .../module/aws/cloudwatch/metadata/ec2/ec2.go | 12 +- .../module/aws/cloudwatch/metadata/rds/rds.go | 10 +- .../module/aws/cloudwatch/metadata/sqs/sqs.go | 10 +- 17 files changed, 136 insertions(+), 236 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 8281dc88769..6b9e3941387 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -35,7 +35,7 @@ https://github.com/elastic/beats/compare/v8.2.0\...main[Check the HEAD diff] ==== Bugfixes *Affecting all Beats* - +- Fix in AWS related services initialisation relying on custom endpoint resolver. {issue}32888[32888] {pull}32921[32921] *Auditbeat* diff --git a/x-pack/filebeat/docs/inputs/input-aws-s3.asciidoc b/x-pack/filebeat/docs/inputs/input-aws-s3.asciidoc index e771258562c..19a47e3e543 100644 --- a/x-pack/filebeat/docs/inputs/input-aws-s3.asciidoc +++ b/x-pack/filebeat/docs/inputs/input-aws-s3.asciidoc @@ -65,8 +65,7 @@ Listing of the S3 bucket will be polled according the time interval defined by The `aws-s3` input can also poll 3rd party S3 compatible services such as the self hosted Minio. Using non-AWS S3 compatible buckets requires the use of `access_key_id` and `secret_access_key` for authentication. To specify the S3 bucket name, use the `non_aws_bucket_name` config and the `endpoint` must be set to replace the default API endpoint. -`endpoint` should be a full URI in the form of `https(s)://`, that will be used as the API endpoint of the service, or a single domain. -If a domain is provided, the full endpoint URI will be constructed with the region name in the standard form of `https://s3..` supported by AWS and several 3rd party providers. +`endpoint` should be a full URI in the form of `https(s)://` in the case of `non_aws_bucket_name`, that will be used as the API endpoint of the service. No `endpoint` is needed if using the native AWS S3 service hosted at `amazonaws.com`. Please see <> for alternate AWS domains that require a different endpoint. diff --git a/x-pack/filebeat/input/awscloudwatch/input.go b/x-pack/filebeat/input/awscloudwatch/input.go index dd4a09ee2eb..02e8d936ee5 100644 --- a/x-pack/filebeat/input/awscloudwatch/input.go +++ b/x-pack/filebeat/input/awscloudwatch/input.go @@ -120,9 +120,11 @@ func (in *cloudwatchInput) Run(inputContext v2.Context, pipeline beat.Pipeline) } defer client.Close() - logsServiceName := awscommon.CreateServiceName("logs", in.config.AWSConfig.FIPSEnabled, in.config.RegionName) - cwConfig := awscommon.EnrichAWSConfigWithEndpoint(in.config.AWSConfig.Endpoint, logsServiceName, in.config.RegionName, in.awsConfig) - svc := cloudwatchlogs.NewFromConfig(cwConfig) + svc := cloudwatchlogs.NewFromConfig(in.awsConfig, func(o *cloudwatchlogs.Options) { + if in.config.AWSConfig.FIPSEnabled { + o.EndpointOptions.UseFIPSEndpoint = awssdk.FIPSEndpointStateEnabled + } + }) logGroupNames, err := getLogGroupNames(svc, in.config.LogGroupNamePrefix, in.config.LogGroupName) if err != nil { diff --git a/x-pack/filebeat/input/awss3/input.go b/x-pack/filebeat/input/awss3/input.go index c116af21bc9..04d96db2b1e 100644 --- a/x-pack/filebeat/input/awss3/input.go +++ b/x-pack/filebeat/input/awss3/input.go @@ -155,11 +155,12 @@ func (in *s3Input) Run(inputContext v2.Context, pipeline beat.Pipeline) error { } func (in *s3Input) createSQSReceiver(ctx v2.Context, client beat.Client) (*sqsReader, error) { - s3ServiceName := awscommon.CreateServiceName("s3", in.config.AWSConfig.FIPSEnabled, in.awsConfig.Region) - sqsServiceName := awscommon.CreateServiceName("sqs", in.config.AWSConfig.FIPSEnabled, in.awsConfig.Region) - sqsAPI := &awsSQSAPI{ - client: sqs.NewFromConfig(awscommon.EnrichAWSConfigWithEndpoint(in.config.AWSConfig.Endpoint, sqsServiceName, in.awsConfig.Region, in.awsConfig)), + client: sqs.NewFromConfig(in.awsConfig, func(o *sqs.Options) { + if in.config.AWSConfig.FIPSEnabled { + o.EndpointOptions.UseFIPSEndpoint = awssdk.FIPSEndpointStateEnabled + } + }), queueURL: in.config.QueueURL, apiTimeout: in.config.APITimeout, visibilityTimeout: in.config.VisibilityTimeout, @@ -167,7 +168,11 @@ func (in *s3Input) createSQSReceiver(ctx v2.Context, client beat.Client) (*sqsRe } s3API := &awsS3API{ - client: s3.NewFromConfig(awscommon.EnrichAWSConfigWithEndpoint(in.config.AWSConfig.Endpoint, s3ServiceName, in.awsConfig.Region, in.awsConfig)), + client: s3.NewFromConfig(in.awsConfig, func(o *s3.Options) { + if in.config.AWSConfig.FIPSEnabled { + o.EndpointOptions.UseFIPSEndpoint = awssdk.FIPSEndpointStateEnabled + } + }), } log := ctx.Logger.With("queue_url", in.config.QueueURL) @@ -175,7 +180,6 @@ func (in *s3Input) createSQSReceiver(ctx v2.Context, client beat.Client) (*sqsRe log.Infof("AWS region is set to %v.", in.awsConfig.Region) log.Infof("AWS SQS visibility_timeout is set to %v.", in.config.VisibilityTimeout) log.Infof("AWS SQS max_number_of_messages is set to %v.", in.config.MaxNumberOfMessages) - log.Debugf("AWS S3 service name is %v.", s3ServiceName) metricRegistry := monitoring.GetNamespace("dataset").GetRegistry() metrics := newInputMetrics(metricRegistry, ctx.ID) @@ -195,8 +199,15 @@ func (in *s3Input) createSQSReceiver(ctx v2.Context, client beat.Client) (*sqsRe return sqsReader, nil } +type nonAWSBucketResolver struct { + endpoint string +} + +func (n nonAWSBucketResolver) ResolveEndpoint(region string, options s3.EndpointResolverOptions) (awssdk.Endpoint, error) { + return awssdk.Endpoint{URL: n.endpoint, SigningRegion: region, HostnameImmutable: true, Source: awssdk.EndpointSourceCustom}, nil +} + func (in *s3Input) createS3Lister(ctx v2.Context, cancelCtx context.Context, client beat.Client, persistentStore *statestore.Store, states *states) (*s3Poller, error) { - s3ServiceName := awscommon.CreateServiceName("s3", in.config.AWSConfig.FIPSEnabled, in.awsConfig.Region) var bucketName string var bucketID string if in.config.NonAWSBucketName != "" { @@ -207,7 +218,14 @@ func (in *s3Input) createS3Lister(ctx v2.Context, cancelCtx context.Context, cli bucketID = in.config.BucketARN } - s3Client := s3.NewFromConfig(awscommon.EnrichAWSConfigWithEndpoint(in.config.AWSConfig.Endpoint, s3ServiceName, in.awsConfig.Region, in.awsConfig), func(o *s3.Options) { + s3Client := s3.NewFromConfig(in.awsConfig, func(o *s3.Options) { + if in.config.NonAWSBucketName != "" { + o.EndpointResolver = nonAWSBucketResolver{endpoint: in.config.AWSConfig.Endpoint} + } + + if in.config.AWSConfig.FIPSEnabled { + o.EndpointOptions.UseFIPSEndpoint = awssdk.FIPSEndpointStateEnabled + } o.UsePathStyle = in.config.PathStyle }) regionName, err := getRegionForBucket(cancelCtx, s3Client, bucketName) @@ -220,7 +238,14 @@ func (in *s3Input) createS3Lister(ctx v2.Context, cancelCtx context.Context, cli in.awsConfig.Region = regionName if regionName != originalAwsConfigRegion { - s3Client = s3.NewFromConfig(awscommon.EnrichAWSConfigWithEndpoint(in.config.AWSConfig.Endpoint, s3ServiceName, in.awsConfig.Region, in.awsConfig), func(o *s3.Options) { + s3Client = s3.NewFromConfig(in.awsConfig, func(o *s3.Options) { + if in.config.NonAWSBucketName != "" { + o.EndpointResolver = nonAWSBucketResolver{endpoint: in.config.AWSConfig.Endpoint} + } + + if in.config.AWSConfig.FIPSEnabled { + o.EndpointOptions.UseFIPSEndpoint = awssdk.FIPSEndpointStateEnabled + } o.UsePathStyle = in.config.PathStyle }) } @@ -234,7 +259,6 @@ func (in *s3Input) createS3Lister(ctx v2.Context, cancelCtx context.Context, cli log.Infof("bucket_list_interval is set to %v.", in.config.BucketListInterval) log.Infof("bucket_list_prefix is set to %v.", in.config.BucketListPrefix) log.Infof("AWS region is set to %v.", in.awsConfig.Region) - log.Debugf("AWS S3 service name is %v.", s3ServiceName) metricRegistry := monitoring.GetNamespace("dataset").GetRegistry() metrics := newInputMetrics(metricRegistry, ctx.ID) diff --git a/x-pack/filebeat/input/awss3/input_integration_test.go b/x-pack/filebeat/input/awss3/input_integration_test.go index 6fc3da78d46..024caa23ab5 100644 --- a/x-pack/filebeat/input/awss3/input_integration_test.go +++ b/x-pack/filebeat/input/awss3/input_integration_test.go @@ -398,7 +398,7 @@ func TestGetRegionForBucketARN(t *testing.T) { t.Fatal(err) } - s3Client := s3.NewFromConfig(awscommon.EnrichAWSConfigWithEndpoint("", "s3", "", cfg)) + s3Client := s3.NewFromConfig(cfg) regionName, err := getRegionForBucket(context.Background(), s3Client, getBucketNameFromARN(tfConfig.BucketName)) assert.NoError(t, err) @@ -428,7 +428,7 @@ func TestPaginatorListPrefix(t *testing.T) { t.Fatal(err) } - s3Client := s3.NewFromConfig(awscommon.EnrichAWSConfigWithEndpoint("", "s3", "", cfg)) + s3Client := s3.NewFromConfig(cfg) s3API := &awsS3API{ client: s3Client, diff --git a/x-pack/libbeat/autodiscover/providers/aws/ec2/provider.go b/x-pack/libbeat/autodiscover/providers/aws/ec2/provider.go index 1f16f64e82b..0c4d7720974 100644 --- a/x-pack/libbeat/autodiscover/providers/aws/ec2/provider.go +++ b/x-pack/libbeat/autodiscover/providers/aws/ec2/provider.go @@ -7,6 +7,8 @@ package ec2 import ( "fmt" + awssdk "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/gofrs/uuid" @@ -63,9 +65,11 @@ func AutodiscoverBuilder( if config.Regions == nil { // set default region to make initial aws api call awsCfg.Region = "us-west-1" - ec2ServiceName := awscommon.CreateServiceName("ec2", config.AWSConfig.FIPSEnabled, awsCfg.Region) - svcEC2 := ec2.NewFromConfig(awscommon.EnrichAWSConfigWithEndpoint( - config.AWSConfig.Endpoint, ec2ServiceName, awsCfg.Region, awsCfg)) + svcEC2 := ec2.NewFromConfig(awsCfg, func(o *ec2.Options) { + if config.AWSConfig.FIPSEnabled { + o.EndpointOptions.UseFIPSEndpoint = awssdk.FIPSEndpointStateEnabled + } + }) completeRegionsList, err := awsauto.GetRegions(svcEC2) if err != nil { @@ -81,9 +85,11 @@ func AutodiscoverBuilder( logp.Error(fmt.Errorf("error loading AWS config for aws_ec2 autodiscover provider: %w", err)) } awsCfg.Region = region - ec2ServiceName := awscommon.CreateServiceName("ec2", config.AWSConfig.FIPSEnabled, region) - clients = append(clients, ec2.NewFromConfig(awscommon.EnrichAWSConfigWithEndpoint( - config.AWSConfig.Endpoint, ec2ServiceName, region, awsCfg))) + clients = append(clients, ec2.NewFromConfig(awsCfg, func(o *ec2.Options) { + if config.AWSConfig.FIPSEnabled { + o.EndpointOptions.UseFIPSEndpoint = awssdk.FIPSEndpointStateEnabled + } + })) } return internalBuilder(uuid, bus, config, newAPIFetcher(clients), keystore) diff --git a/x-pack/libbeat/autodiscover/providers/aws/elb/provider.go b/x-pack/libbeat/autodiscover/providers/aws/elb/provider.go index 496b71d7080..b76d6050c32 100644 --- a/x-pack/libbeat/autodiscover/providers/aws/elb/provider.go +++ b/x-pack/libbeat/autodiscover/providers/aws/elb/provider.go @@ -5,6 +5,7 @@ package elb import ( + awssdk "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2" "github.com/gofrs/uuid" @@ -64,9 +65,12 @@ func AutodiscoverBuilder( // Construct MetricSet with a full regions list if there is no region specified. if config.Regions == nil { - ec2ServiceName := awscommon.CreateServiceName("ec2", config.AWSConfig.FIPSEnabled, awsCfg.Region) - svcEC2 := ec2.NewFromConfig(awscommon.EnrichAWSConfigWithEndpoint( - config.AWSConfig.Endpoint, ec2ServiceName, awsCfg.Region, awsCfg)) + svcEC2 := ec2.NewFromConfig(awsCfg, func(o *ec2.Options) { + if config.AWSConfig.FIPSEnabled { + o.EndpointOptions.UseFIPSEndpoint = awssdk.FIPSEndpointStateEnabled + } + + }) completeRegionsList, err := awsauto.GetRegions(svcEC2) if err != nil { @@ -88,9 +92,12 @@ func AutodiscoverBuilder( logp.Err("error loading AWS config for aws_elb autodiscover provider: %s", err) } awsCfg.Region = region - elbServiceName := awscommon.CreateServiceName("elasticloadbalancing", config.AWSConfig.FIPSEnabled, region) - clients = append(clients, elasticloadbalancingv2.NewFromConfig(awscommon.EnrichAWSConfigWithEndpoint( - config.AWSConfig.Endpoint, elbServiceName, region, awsCfg))) + clients = append(clients, elasticloadbalancingv2.NewFromConfig(awsCfg, func(o *elasticloadbalancingv2.Options) { + if config.AWSConfig.FIPSEnabled { + o.EndpointOptions.UseFIPSEndpoint = awssdk.FIPSEndpointStateEnabled + } + + })) } return internalBuilder(uuid, bus, config, newAPIFetcher(clients), keystore) diff --git a/x-pack/libbeat/common/aws/credentials.go b/x-pack/libbeat/common/aws/credentials.go index 0467c02df2c..3667e72c798 100644 --- a/x-pack/libbeat/common/aws/credentials.go +++ b/x-pack/libbeat/common/aws/credentials.go @@ -10,7 +10,6 @@ import ( "fmt" "net/http" "net/url" - "strings" "github.com/aws/aws-sdk-go-v2/service/sts" @@ -164,40 +163,3 @@ func addStaticCredentialsProviderToAwsConfig(beatsConfig ConfigAWS, awsConfig *a awsConfig.Credentials = staticCredentialsProvider } - -// EnrichAWSConfigWithEndpoint function enabled endpoint resolver for AWS service clients when endpoint is given in config. -func EnrichAWSConfigWithEndpoint(endpoint string, serviceName string, regionName string, beatsConfig awssdk.Config) awssdk.Config { - var eurl string - if endpoint != "" { - parsedEndpoint, _ := url.Parse(endpoint) - - // Beats uses the provided endpoint if the scheme is present or... - if parsedEndpoint.Scheme != "" { - eurl = endpoint - } else { - // ...build one by using the scheme, service and region names. - if regionName == "" { - eurl = "https://" + serviceName + "." + endpoint - } else { - eurl = "https://" + serviceName + "." + regionName + "." + endpoint - } - } - - beatsConfig.EndpointResolverWithOptions = awssdk.EndpointResolverWithOptionsFunc( - func(service, region string, options ...interface{}) (awssdk.Endpoint, error) { - return awssdk.Endpoint{URL: eurl}, nil - }) - } - return beatsConfig -} - -// CreateServiceName based on Service name, Region and FIPS. Returns service name if Fips is not enabled. -func CreateServiceName(serviceName string, fipsEnabled bool, region string) string { - if fipsEnabled { - _, found := OptionalGovCloudFIPS[serviceName] - if !strings.HasPrefix(region, "us-gov-") || found { - return serviceName + "-fips" - } - } - return serviceName -} diff --git a/x-pack/libbeat/common/aws/credentials_test.go b/x-pack/libbeat/common/aws/credentials_test.go index 2a7d7a31ea0..43bbc642bc5 100644 --- a/x-pack/libbeat/common/aws/credentials_test.go +++ b/x-pack/libbeat/common/aws/credentials_test.go @@ -9,7 +9,6 @@ import ( "net/http" "testing" - awssdk "github.com/aws/aws-sdk-go-v2/aws" "github.com/stretchr/testify/assert" "github.com/elastic/elastic-agent-libs/transport/tlscommon" @@ -53,118 +52,6 @@ func TestGetAWSCredentials(t *testing.T) { assert.Equal(t, inputConfig.SessionToken, retrievedAWSConfig.SessionToken) } -func TestEnrichAWSConfigWithEndpoint(t *testing.T) { - cases := []struct { - title string - endpoint string - serviceName string - region string - awsConfig awssdk.Config - expectedEndpointResolverWithOptionsFunc awssdk.EndpointResolverWithOptionsFunc - }{ - { - "endpoint and serviceName given", - "amazonaws.com", - "ec2", - "", - awssdk.Config{}, - getEndpointResolverWithOptionsFunc("https://ec2.amazonaws.com"), - }, - { - "endpoint, serviceName and region given", - "amazonaws.com", - "cloudwatch", - "us-west-1", - awssdk.Config{}, - getEndpointResolverWithOptionsFunc("https://cloudwatch.us-west-1.amazonaws.com"), - }, - { - "full URI endpoint", - "https://s3.test.com:9000", - "s3", - "", - awssdk.Config{}, - - getEndpointResolverWithOptionsFunc("https://s3.test.com:9000"), - }, - { - "full non HTTPS URI endpoint", - "http://testobjects.com:9000", - "s3", - "", - awssdk.Config{}, - - getEndpointResolverWithOptionsFunc("http://testobjects.com:9000"), - }, - } - for _, c := range cases { - t.Run(c.title, func(t *testing.T) { - enrichedAWSConfig := EnrichAWSConfigWithEndpoint(c.endpoint, c.serviceName, c.region, c.awsConfig) - expectedEndpointResolved, _ := c.expectedEndpointResolverWithOptionsFunc(c.serviceName, c.region) - enrichedAWSConfigexpectedEndpointResolved, _ := enrichedAWSConfig.EndpointResolverWithOptions.ResolveEndpoint(c.serviceName, c.region) - assert.EqualValues(t, expectedEndpointResolved, enrichedAWSConfigexpectedEndpointResolved) - }) - } -} - -func getEndpointResolverWithOptionsFunc(e string) awssdk.EndpointResolverWithOptionsFunc { - return func(service, region string, options ...interface{}) (awssdk.Endpoint, error) { - return awssdk.Endpoint{URL: e}, nil - } -} - -func TestCreateServiceName(t *testing.T) { - cases := []struct { - title string - serviceName string - fips_enabled bool - region string - expectedServiceName string - }{ - { - "S3 - non-fips - us-east-1", - "s3", - false, - "us-east-1", - "s3", - }, - { - "S3 - non-fips - us-gov-east-1", - "s3", - false, - "us-gov-east-1", - "s3", - }, - { - "S3 - fips - us-gov-east-1", - "s3", - true, - "us-gov-east-1", - "s3-fips", - }, - { - "EC2 - fips - us-gov-east-1", - "ec2", - true, - "us-gov-east-1", - "ec2", - }, - { - "EC2 - fips - us-east-1", - "ec2", - true, - "us-east-1", - "ec2-fips", - }, - } - for _, c := range cases { - t.Run(c.title, func(t *testing.T) { - serviceName := CreateServiceName(c.serviceName, c.fips_enabled, c.region) - assert.Equal(t, c.expectedServiceName, serviceName) - }) - } -} - func TestDefaultRegion(t *testing.T) { cases := []struct { title string diff --git a/x-pack/libbeat/docs/aws-credentials-config.asciidoc b/x-pack/libbeat/docs/aws-credentials-config.asciidoc index 3ccfd09a84d..99fe445b3b1 100644 --- a/x-pack/libbeat/docs/aws-credentials-config.asciidoc +++ b/x-pack/libbeat/docs/aws-credentials-config.asciidoc @@ -10,17 +10,10 @@ To configure AWS credentials, either put the credentials into the {beatname_uc} * *credential_profile_name*: profile name in shared credentials file. * *shared_credential_file*: directory of the shared credentials file. * *role_arn*: AWS IAM Role to assume. -* *endpoint*: URL of the entry point for an AWS web service. -Most AWS services offer a regional endpoint that can be used to make requests. -The general syntax of a regional endpoint is `protocol://service-code.region-code.endpoint-code`. -Some services, such as IAM, do not support regions. The endpoints for these -services do not include a region. In `aws` module, `endpoint` config is to set -the `endpoint-code` part, such as `amazonaws.com`, `amazonaws.com.cn`, `c2s.ic.gov`, -`sc2s.sgov.gov`. * *proxy_url*: URL of the proxy to use to connect to AWS web services. The syntax is `http(s)://:` -* *fips_enabled*: Enabling this option changes the service names from `s3` to `s3-fips` for connecting to the correct service endpoint. For example: `s3-fips.us-gov-east-1.amazonaws.com`. All services used by Beats are FIPS compatible except for `tagging` but only certain regions are FIPS compatible. See https://aws.amazon.com/compliance/fips/ or the appropriate service page, https://docs.aws.amazon.com/general/latest/gr/aws-service-information.html, for a full list of FIPS endpoints and regions. +* *fips_enabled*: Enabling this option instructs {beatname_uc} to use the FIPS endpoint of a service. All services used by {beatname_uc} are FIPS compatible except for `tagging` but only certain regions are FIPS compatible. See https://aws.amazon.com/compliance/fips/ or the appropriate service page, https://docs.aws.amazon.com/general/latest/gr/aws-service-information.html, for a full list of FIPS endpoints and regions. * *ssl*: This specifies SSL/TLS configuration. If the ssl section is missing, the host's CAs are used for HTTPS connections. See <> for more information. -* *default_region*: Default region to query if no other region is set. +* *default_region*: Default region to query if no other region is set. Most AWS services offer a regional endpoint that can be used to make requests. Some services, such as IAM, do not support regions. If a region is not provided by any other way (environment variable, credential or instance profile), the value set here will be used. [float] ==== Supported Formats diff --git a/x-pack/metricbeat/module/aws/aws.go b/x-pack/metricbeat/module/aws/aws.go index 46d4aa6b8d5..3e286f66560 100644 --- a/x-pack/metricbeat/module/aws/aws.go +++ b/x-pack/metricbeat/module/aws/aws.go @@ -8,7 +8,6 @@ import ( "context" "fmt" - "strings" "time" awssdk "github.com/aws/aws-sdk-go-v2/aws" @@ -110,12 +109,12 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) { awsConfig.Region = config.Regions[0] } - stsServiceName := awscommon.CreateServiceName("sts", config.AWSConfig.FIPSEnabled, awsConfig.Region) - iamServiceName := awscommon.CreateServiceName("iam", config.AWSConfig.FIPSEnabled, awsConfig.Region) - // Get IAM account id - svcSts := sts.NewFromConfig(awscommon.EnrichAWSConfigWithEndpoint( - config.AWSConfig.Endpoint, stsServiceName, "", awsConfig)) + svcSts := sts.NewFromConfig(awsConfig, func(o *sts.Options) { + if config.AWSConfig.FIPSEnabled { + o.EndpointOptions.UseFIPSEndpoint = awssdk.FIPSEndpointStateEnabled + } + }) outputIdentity, err := svcSts.GetCallerIdentity(context.TODO(), &sts.GetCallerIdentityInput{}) if err != nil { base.Logger().Warn("failed to get caller identity, please check permission setting: ", err) @@ -123,20 +122,22 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) { metricSet.AccountID = *outputIdentity.Account base.Logger().Debug("AWS Credentials belong to account ID: ", metricSet.AccountID) } - iamRegion := "" - if strings.HasPrefix(awsConfig.Region, "us-gov-") { - iamRegion = "us-gov" - } // Get account name/alias - svcIam := iam.NewFromConfig(awscommon.EnrichAWSConfigWithEndpoint( - config.AWSConfig.Endpoint, iamServiceName, iamRegion, awsConfig)) + svcIam := iam.NewFromConfig(awsConfig, func(o *iam.Options) { + if config.AWSConfig.FIPSEnabled { + o.EndpointOptions.UseFIPSEndpoint = awssdk.FIPSEndpointStateEnabled + } + + }) metricSet.AccountName = getAccountName(svcIam, base, metricSet) // Construct MetricSet with a full regions list if config.Regions == nil { - ec2ServiceName := awscommon.CreateServiceName("ec2", config.AWSConfig.FIPSEnabled, awsConfig.Region) - svcEC2 := ec2.NewFromConfig(awscommon.EnrichAWSConfigWithEndpoint( - config.AWSConfig.Endpoint, ec2ServiceName, "", awsConfig)) + svcEC2 := ec2.NewFromConfig(awsConfig, func(o *ec2.Options) { + if config.AWSConfig.FIPSEnabled { + o.EndpointOptions.UseFIPSEndpoint = awssdk.FIPSEndpointStateEnabled + } + }) completeRegionsList, err := getRegions(svcEC2) if err != nil { return nil, err diff --git a/x-pack/metricbeat/module/aws/billing/billing.go b/x-pack/metricbeat/module/aws/billing/billing.go index 796c5eccef0..e8510c4de7c 100644 --- a/x-pack/metricbeat/module/aws/billing/billing.go +++ b/x-pack/metricbeat/module/aws/billing/billing.go @@ -22,7 +22,6 @@ import ( "github.com/aws/aws-sdk-go-v2/service/organizations" "github.com/elastic/beats/v7/metricbeat/mb" - awscommon "github.com/elastic/beats/v7/x-pack/libbeat/common/aws" "github.com/elastic/beats/v7/x-pack/metricbeat/module/aws" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" @@ -119,7 +118,6 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { if err != nil { return err } - monitoringServiceName := awscommon.CreateServiceName("monitoring", config.AWSConfig.FIPSEnabled, regionName) // Get startDate and endDate startDate, endDate := getStartDateEndDate(m.Period) @@ -128,10 +126,19 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { // get cost metrics from cost explorer awsBeatsConfig := m.MetricSet.AwsConfig.Copy() - svcCostExplorer := costexplorer.NewFromConfig(awscommon.EnrichAWSConfigWithEndpoint(m.Endpoint, monitoringServiceName, "", awsBeatsConfig)) + svcCostExplorer := costexplorer.NewFromConfig(awsBeatsConfig, func(o *costexplorer.Options) { + if config.AWSConfig.FIPSEnabled { + o.EndpointOptions.UseFIPSEndpoint = awssdk.FIPSEndpointStateEnabled + } + + }) awsBeatsConfig.Region = regionName - svcCloudwatch := cloudwatch.NewFromConfig(awscommon.EnrichAWSConfigWithEndpoint(m.Endpoint, monitoringServiceName, regionName, awsBeatsConfig)) + svcCloudwatch := cloudwatch.NewFromConfig(awsBeatsConfig, func(o *cloudwatch.Options) { + if config.AWSConfig.FIPSEnabled { + o.EndpointOptions.UseFIPSEndpoint = awssdk.FIPSEndpointStateEnabled + } + }) timePeriod := costexplorertypes.DateInterval{ Start: awssdk.String(startDate), @@ -223,10 +230,12 @@ func (m *MetricSet) getCostGroupBy(svcCostExplorer *costexplorer.Client, groupBy } if ok, _ := aws.StringInSlice("LINKED_ACCOUNT", groupByDimKeys); ok { awsConfig := m.MetricSet.AwsConfig.Copy() - organizationsServiceName := awscommon.CreateServiceName("organizations", config.AWSConfig.FIPSEnabled, regionName) - svcOrg := organizations.NewFromConfig(awscommon.EnrichAWSConfigWithEndpoint( - m.Endpoint, organizationsServiceName, regionName, awsConfig)) + svcOrg := organizations.NewFromConfig(awsConfig, func(o *organizations.Options) { + if config.AWSConfig.FIPSEnabled { + o.EndpointOptions.UseFIPSEndpoint = awssdk.FIPSEndpointStateEnabled + } + }) accounts = m.getAccountName(svcOrg) } diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go index ccee583a460..3ed35ff6576 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go @@ -19,7 +19,6 @@ import ( "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/metricbeat/mb" - awscommon "github.com/elastic/beats/v7/x-pack/libbeat/common/aws" "github.com/elastic/beats/v7/x-pack/metricbeat/module/aws" "github.com/elastic/elastic-agent-libs/logp" ) @@ -206,7 +205,7 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { m.logger.Debugf("Collected number of metrics = %d", len(eventsWithIdentifier)) - events, err := addMetadata(namespace, m.Endpoint, regionName, beatsConfig, config.AWSConfig.FIPSEnabled, eventsWithIdentifier) + events, err := addMetadata(namespace, regionName, beatsConfig, config.AWSConfig.FIPSEnabled, eventsWithIdentifier) if err != nil { // TODO What to do if add metadata fails? I guess to continue, probably we have an 90% of reliable data m.Logger().Warn("could not add metadata to events: %w", err) @@ -224,13 +223,18 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { func (m *MetricSet) createAwsRequiredClients(beatsConfig awssdk.Config, regionName string, config aws.Config) (*cloudwatch.Client, *resourcegroupstaggingapi.Client, error) { m.logger.Debugf("Collecting metrics from AWS region %s", regionName) - monitoringServiceName := awscommon.CreateServiceName("monitoring", config.AWSConfig.FIPSEnabled, regionName) + svcCloudwatchClient := cloudwatch.NewFromConfig(beatsConfig, func(o *cloudwatch.Options) { + if config.AWSConfig.FIPSEnabled { + o.EndpointOptions.UseFIPSEndpoint = awssdk.FIPSEndpointStateEnabled + } - svcCloudwatchClient := cloudwatch.NewFromConfig(awscommon.EnrichAWSConfigWithEndpoint( - m.Endpoint, monitoringServiceName, regionName, beatsConfig)) + }) - svcResourceAPIClient := resourcegroupstaggingapi.NewFromConfig(awscommon.EnrichAWSConfigWithEndpoint( - m.Endpoint, "tagging", regionName, beatsConfig)) //Does not support FIPS + svcResourceAPIClient := resourcegroupstaggingapi.NewFromConfig(beatsConfig, func(o *resourcegroupstaggingapi.Options) { + if config.AWSConfig.FIPSEnabled { + o.EndpointOptions.UseFIPSEndpoint = awssdk.FIPSEndpointStateEnabled + } + }) return svcCloudwatchClient, svcResourceAPIClient, nil } diff --git a/x-pack/metricbeat/module/aws/cloudwatch/metadata.go b/x-pack/metricbeat/module/aws/cloudwatch/metadata.go index f35b49487c1..4e2d8f31545 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/metadata.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/metadata.go @@ -23,20 +23,20 @@ const ( ) // addMetadata adds metadata to the given events map based on namespace -func addMetadata(namespace string, endpoint string, regionName string, awsConfig awssdk.Config, fipsEnabled bool, events map[string]mb.Event) (map[string]mb.Event, error) { +func addMetadata(namespace string, regionName string, awsConfig awssdk.Config, fipsEnabled bool, events map[string]mb.Event) (map[string]mb.Event, error) { switch namespace { case namespaceEC2: - events, err := ec2.AddMetadata(endpoint, regionName, awsConfig, fipsEnabled, events) + events, err := ec2.AddMetadata(regionName, awsConfig, fipsEnabled, events) if err != nil { return events, fmt.Errorf("error adding metadata to ec2: %w", err) } case namespaceRDS: - events, err := rds.AddMetadata(endpoint, regionName, awsConfig, fipsEnabled, events) + events, err := rds.AddMetadata(regionName, awsConfig, fipsEnabled, events) if err != nil { return events, fmt.Errorf("error adding metadata to rds: %w", err) } case namespaceSQS: - events, err := sqs.AddMetadata(endpoint, regionName, awsConfig, fipsEnabled, events) + events, err := sqs.AddMetadata(regionName, awsConfig, fipsEnabled, events) if err != nil { return events, fmt.Errorf("error adding metadata to sqs: %w", err) } diff --git a/x-pack/metricbeat/module/aws/cloudwatch/metadata/ec2/ec2.go b/x-pack/metricbeat/module/aws/cloudwatch/metadata/ec2/ec2.go index 40ba6291209..fc6f6d83f28 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/metadata/ec2/ec2.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/metadata/ec2/ec2.go @@ -14,17 +14,19 @@ import ( ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" "github.com/elastic/beats/v7/metricbeat/mb" - awscommon "github.com/elastic/beats/v7/x-pack/libbeat/common/aws" "github.com/elastic/elastic-agent-libs/logp" ) const metadataPrefix = "aws.ec2.instance." // AddMetadata adds metadata for EC2 instances from a specific region -func AddMetadata(endpoint string, regionName string, awsConfig awssdk.Config, fips_enabled bool, events map[string]mb.Event) (map[string]mb.Event, error) { - ec2ServiceName := awscommon.CreateServiceName("ec2", fips_enabled, regionName) - svcEC2 := ec2.NewFromConfig(awscommon.EnrichAWSConfigWithEndpoint( - endpoint, ec2ServiceName, regionName, awsConfig)) +func AddMetadata(regionName string, awsConfig awssdk.Config, fips_enabled bool, events map[string]mb.Event) (map[string]mb.Event, error) { + svcEC2 := ec2.NewFromConfig(awsConfig, func(o *ec2.Options) { + if fips_enabled { + o.EndpointOptions.UseFIPSEndpoint = awssdk.FIPSEndpointStateEnabled + } + + }) instancesOutputs, err := getInstancesPerRegion(svcEC2) if err != nil { diff --git a/x-pack/metricbeat/module/aws/cloudwatch/metadata/rds/rds.go b/x-pack/metricbeat/module/aws/cloudwatch/metadata/rds/rds.go index cab6803f241..2b8402b659f 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/metadata/rds/rds.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/metadata/rds/rds.go @@ -14,16 +14,18 @@ import ( "github.com/aws/aws-sdk-go-v2/service/rds" "github.com/elastic/beats/v7/metricbeat/mb" - awscommon "github.com/elastic/beats/v7/x-pack/libbeat/common/aws" "github.com/elastic/elastic-agent-libs/logp" ) const metadataPrefix = "aws.rds.db_instance." // AddMetadata adds metadata for RDS instances from a specific region -func AddMetadata(endpoint string, regionName string, awsConfig awssdk.Config, fips_enabled bool, events map[string]mb.Event) (map[string]mb.Event, error) { - rdsServiceName := awscommon.CreateServiceName("rds", fips_enabled, regionName) - svc := rds.NewFromConfig(awscommon.EnrichAWSConfigWithEndpoint(endpoint, rdsServiceName, regionName, awsConfig)) +func AddMetadata(regionName string, awsConfig awssdk.Config, fips_enabled bool, events map[string]mb.Event) (map[string]mb.Event, error) { + svc := rds.NewFromConfig(awsConfig, func(o *rds.Options) { + if fips_enabled { + o.EndpointOptions.UseFIPSEndpoint = awssdk.FIPSEndpointStateEnabled + } + }) // Get DBInstance IDs per region dbDetailsMap, err := getDBInstancesPerRegion(svc) diff --git a/x-pack/metricbeat/module/aws/cloudwatch/metadata/sqs/sqs.go b/x-pack/metricbeat/module/aws/cloudwatch/metadata/sqs/sqs.go index 276a5aa31b1..9a24f0ba47f 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/metadata/sqs/sqs.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/metadata/sqs/sqs.go @@ -13,17 +13,19 @@ import ( "github.com/aws/aws-sdk-go-v2/service/sqs" "github.com/elastic/beats/v7/metricbeat/mb" - awscommon "github.com/elastic/beats/v7/x-pack/libbeat/common/aws" "github.com/elastic/elastic-agent-libs/logp" ) const metadataPrefix = "aws.sqs.queue" // AddMetadata adds metadata for SQS queues from a specific region -func AddMetadata(endpoint string, regionName string, awsConfig awssdk.Config, fips_enabled bool, events map[string]mb.Event) (map[string]mb.Event, error) { - sqsServiceName := awscommon.CreateServiceName("sqs", fips_enabled, regionName) +func AddMetadata(regionName string, awsConfig awssdk.Config, fips_enabled bool, events map[string]mb.Event) (map[string]mb.Event, error) { + svc := sqs.NewFromConfig(awsConfig, func(o *sqs.Options) { + if fips_enabled { + o.EndpointOptions.UseFIPSEndpoint = awssdk.FIPSEndpointStateEnabled + } - svc := sqs.NewFromConfig(awscommon.EnrichAWSConfigWithEndpoint(endpoint, sqsServiceName, regionName, awsConfig)) + }) // Get queueUrls for each region queueURLs, err := getQueueUrls(svc)