From 774bc1b99ad4fb60e22154d3facc8d924cdc4f68 Mon Sep 17 00:00:00 2001 From: Tiago Silva Date: Wed, 28 Jun 2023 10:36:48 +0100 Subject: [PATCH] Support specifying `assume_role_arn` for Kube cluster matchers (#28282) * Support specifiying `assume_role_arn` for Kube cluster matchers This PR allows users to assume different AWS roles when interacting with AWS EKS API. It allows users to proxy EKS clusters in different AWS accounts using the same Teleport Kubernetes Service. Example configuration: ```yaml kubernetes_service: enabled: true resources: - labels: 'a': 'b' aws: assume_role_arn: "arn:aws:iam::0987654321:role/KubeAccess" external_id: "0987654321" - labels: 'c': 'd' aws: assume_role_arn: "arn:aws:iam::123456789012:role/KubeAccess" external_id: "123456789012" ``` * reuse eks token validation --- lib/cloud/clients.go | 2 +- lib/cloud/mocks/aws.go | 69 +++++- lib/cloud/mocks/azure.go | 57 +++++ lib/cloud/mocks/gcp.go | 31 +++ lib/config/configuration.go | 7 +- lib/config/configuration_test.go | 21 +- lib/kube/proxy/cluster_details.go | 107 +++++++--- lib/kube/proxy/kube_creds.go | 69 ++++-- lib/kube/proxy/kube_creds_test.go | 338 ++++++++++++++++++++++++++++++ lib/kube/proxy/watcher.go | 22 +- 10 files changed, 664 insertions(+), 59 deletions(-) create mode 100644 lib/cloud/mocks/azure.go create mode 100644 lib/kube/proxy/kube_creds_test.go diff --git a/lib/cloud/clients.go b/lib/cloud/clients.go index 1966d65869ae6..df8eca26f05f1 100644 --- a/lib/cloud/clients.go +++ b/lib/cloud/clients.go @@ -987,7 +987,7 @@ func (c *TestCloudClients) GetAzurePostgresClient(subscription string) (azure.DB // GetAzureKubernetesClient returns an AKS client for the specified subscription func (c *TestCloudClients) GetAzureKubernetesClient(subscription string) (azure.AKSClient, error) { - if len(c.AzurePostgresPerSub) != 0 { + if len(c.AzureAKSClientPerSub) != 0 { return c.AzureAKSClientPerSub[subscription], nil } return c.AzureAKSClient, nil diff --git a/lib/cloud/mocks/aws.go b/lib/cloud/mocks/aws.go index 0c21dc3f3918b..1fb0d979743de 100644 --- a/lib/cloud/mocks/aws.go +++ b/lib/cloud/mocks/aws.go @@ -18,11 +18,15 @@ package mocks import ( "context" + "net/http" + "net/url" "sync" "time" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/request" + "github.com/aws/aws-sdk-go/service/eks" + "github.com/aws/aws-sdk-go/service/eks/eksiface" "github.com/aws/aws-sdk-go/service/elasticache" "github.com/aws/aws-sdk-go/service/elasticache/elasticacheiface" "github.com/aws/aws-sdk-go/service/iam" @@ -46,6 +50,7 @@ import ( type STSMock struct { stsiface.STSAPI ARN string + URL *url.URL assumedRoleARNs []string assumedRoleExternalIDs []string mu sync.Mutex @@ -98,6 +103,21 @@ func (m *STSMock) AssumeRoleWithContext(ctx aws.Context, in *sts.AssumeRoleInput }, nil } +func (m *STSMock) GetCallerIdentityRequest(req *sts.GetCallerIdentityInput) (*request.Request, *sts.GetCallerIdentityOutput) { + return &request.Request{ + HTTPRequest: &http.Request{ + Header: http.Header{}, + URL: m.URL, + }, + Operation: &request.Operation{ + Name: "GetCallerIdentity", + HTTPMethod: "POST", + HTTPPath: "/", + }, + Handlers: request.Handlers{}, + }, nil +} + // RDSMock mocks AWS RDS API. type RDSMock struct { rdsiface.RDSAPI @@ -210,6 +230,7 @@ func (m *RDSMock) ModifyDBClusterWithContext(ctx aws.Context, input *rds.ModifyD } return nil, trace.NotFound("cluster %v not found", aws.StringValue(input.DBClusterIdentifier)) } + func (m *RDSMock) DescribeDBProxiesWithContext(ctx aws.Context, input *rds.DescribeDBProxiesInput, options ...request.Option) (*rds.DescribeDBProxiesOutput, error) { if aws.StringValue(input.DBProxyName) == "" { return &rds.DescribeDBProxiesOutput{ @@ -225,6 +246,7 @@ func (m *RDSMock) DescribeDBProxiesWithContext(ctx aws.Context, input *rds.Descr } return nil, trace.NotFound("proxy %v not found", aws.StringValue(input.DBProxyName)) } + func (m *RDSMock) DescribeDBProxyEndpointsWithContext(ctx aws.Context, input *rds.DescribeDBProxyEndpointsInput, options ...request.Option) (*rds.DescribeDBProxyEndpointsOutput, error) { inputProxyName := aws.StringValue(input.DBProxyName) inputProxyEndpointName := aws.StringValue(input.DBProxyEndpointName) @@ -254,6 +276,7 @@ func (m *RDSMock) DescribeDBProxyEndpointsWithContext(ctx aws.Context, input *rd } return &rds.DescribeDBProxyEndpointsOutput{DBProxyEndpoints: endpoints}, nil } + func (m *RDSMock) DescribeDBProxyTargetsWithContext(ctx aws.Context, input *rds.DescribeDBProxyTargetsInput, options ...request.Option) (*rds.DescribeDBProxyTargetsOutput, error) { // only mocking to return a port here return &rds.DescribeDBProxyTargetsOutput{ @@ -262,18 +285,21 @@ func (m *RDSMock) DescribeDBProxyTargetsWithContext(ctx aws.Context, input *rds. }}, }, nil } + func (m *RDSMock) DescribeDBProxiesPagesWithContext(ctx aws.Context, input *rds.DescribeDBProxiesInput, fn func(*rds.DescribeDBProxiesOutput, bool) bool, options ...request.Option) error { fn(&rds.DescribeDBProxiesOutput{ DBProxies: m.DBProxies, }, true) return nil } + func (m *RDSMock) DescribeDBProxyEndpointsPagesWithContext(ctx aws.Context, input *rds.DescribeDBProxyEndpointsInput, fn func(*rds.DescribeDBProxyEndpointsOutput, bool) bool, options ...request.Option) error { fn(&rds.DescribeDBProxyEndpointsOutput{ DBProxyEndpoints: m.DBProxyEndpoints, }, true) return nil } + func (m *RDSMock) ListTagsForResourceWithContext(ctx aws.Context, input *rds.ListTagsForResourceInput, options ...request.Option) (*rds.ListTagsForResourceOutput, error) { return &rds.ListTagsForResourceOutput{}, nil } @@ -381,6 +407,7 @@ func (m *RedshiftMock) GetClusterCredentialsWithContext(aws.Context, *redshift.G } return m.GetClusterCredentialsOutput, nil } + func (m *RedshiftMock) DescribeClustersWithContext(ctx aws.Context, input *redshift.DescribeClustersInput, options ...request.Option) (*redshift.DescribeClustersOutput, error) { if aws.StringValue(input.ClusterIdentifier) == "" { return &redshift.DescribeClustersOutput{ @@ -432,12 +459,15 @@ func (m *RDSMockUnauth) DescribeDBInstancesPagesWithContext(ctx aws.Context, inp func (m *RDSMockUnauth) DescribeDBClustersPagesWithContext(aws aws.Context, input *rds.DescribeDBClustersInput, fn func(*rds.DescribeDBClustersOutput, bool) bool, options ...request.Option) error { return trace.AccessDenied("unauthorized") } + func (m *RDSMockUnauth) DescribeDBProxiesWithContext(ctx aws.Context, input *rds.DescribeDBProxiesInput, options ...request.Option) (*rds.DescribeDBProxiesOutput, error) { return nil, trace.AccessDenied("unauthorized") } + func (m *RDSMockUnauth) DescribeDBProxyEndpointsWithContext(ctx aws.Context, input *rds.DescribeDBProxyEndpointsInput, options ...request.Option) (*rds.DescribeDBProxyEndpointsOutput, error) { return nil, trace.AccessDenied("unauthorized") } + func (m *RDSMockUnauth) DescribeDBProxiesPagesWithContext(ctx aws.Context, input *rds.DescribeDBProxiesInput, fn func(*rds.DescribeDBProxiesOutput, bool) bool, options ...request.Option) error { return trace.AccessDenied("unauthorized") } @@ -453,9 +483,11 @@ type RDSMockByDBType struct { func (m *RDSMockByDBType) DescribeDBInstancesWithContext(ctx aws.Context, input *rds.DescribeDBInstancesInput, options ...request.Option) (*rds.DescribeDBInstancesOutput, error) { return m.DBInstances.DescribeDBInstancesWithContext(ctx, input, options...) } + func (m *RDSMockByDBType) ModifyDBInstanceWithContext(ctx aws.Context, input *rds.ModifyDBInstanceInput, options ...request.Option) (*rds.ModifyDBInstanceOutput, error) { return m.DBInstances.ModifyDBInstanceWithContext(ctx, input, options...) } + func (m *RDSMockByDBType) DescribeDBInstancesPagesWithContext(ctx aws.Context, input *rds.DescribeDBInstancesInput, fn func(*rds.DescribeDBInstancesOutput, bool) bool, options ...request.Option) error { return m.DBInstances.DescribeDBInstancesPagesWithContext(ctx, input, fn, options...) } @@ -463,18 +495,23 @@ func (m *RDSMockByDBType) DescribeDBInstancesPagesWithContext(ctx aws.Context, i func (m *RDSMockByDBType) DescribeDBClustersWithContext(ctx aws.Context, input *rds.DescribeDBClustersInput, options ...request.Option) (*rds.DescribeDBClustersOutput, error) { return m.DBClusters.DescribeDBClustersWithContext(ctx, input, options...) } + func (m *RDSMockByDBType) ModifyDBClusterWithContext(ctx aws.Context, input *rds.ModifyDBClusterInput, options ...request.Option) (*rds.ModifyDBClusterOutput, error) { return m.DBClusters.ModifyDBClusterWithContext(ctx, input, options...) } + func (m *RDSMockByDBType) DescribeDBClustersPagesWithContext(aws aws.Context, input *rds.DescribeDBClustersInput, fn func(*rds.DescribeDBClustersOutput, bool) bool, options ...request.Option) error { return m.DBClusters.DescribeDBClustersPagesWithContext(aws, input, fn, options...) } + func (m *RDSMockByDBType) DescribeDBProxiesWithContext(ctx aws.Context, input *rds.DescribeDBProxiesInput, options ...request.Option) (*rds.DescribeDBProxiesOutput, error) { return m.DBProxies.DescribeDBProxiesWithContext(ctx, input, options...) } + func (m *RDSMockByDBType) DescribeDBProxyEndpointsWithContext(ctx aws.Context, input *rds.DescribeDBProxyEndpointsInput, options ...request.Option) (*rds.DescribeDBProxyEndpointsOutput, error) { return m.DBProxies.DescribeDBProxyEndpointsWithContext(ctx, input, options...) } + func (m *RDSMockByDBType) DescribeDBProxiesPagesWithContext(ctx aws.Context, input *rds.DescribeDBProxiesInput, fn func(*rds.DescribeDBProxiesOutput, bool) bool, options ...request.Option) error { return m.DBProxies.DescribeDBProxiesPagesWithContext(ctx, input, fn, options...) } @@ -539,6 +576,7 @@ func (m *ElastiCacheMock) AddMockUser(user *elasticache.User, tagsMap map[string m.Users = append(m.Users, user) m.addTags(aws.StringValue(user.ARN), tagsMap) } + func (m *ElastiCacheMock) addTags(arn string, tagsMap map[string]string) { if m.TagsByARN == nil { m.TagsByARN = make(map[string][]*elasticache.Tag) @@ -582,6 +620,7 @@ func (m *ElastiCacheMock) DescribeReplicationGroupsWithContext(_ aws.Context, in } return nil, trace.NotFound("ElastiCache %v not found", aws.StringValue(input.ReplicationGroupId)) } + func (m *ElastiCacheMock) DescribeReplicationGroupsPagesWithContext(_ aws.Context, _ *elasticache.DescribeReplicationGroupsInput, fn func(*elasticache.DescribeReplicationGroupsOutput, bool) bool, _ ...request.Option) error { if m.Unauth { return trace.AccessDenied("unauthorized") @@ -591,6 +630,7 @@ func (m *ElastiCacheMock) DescribeReplicationGroupsPagesWithContext(_ aws.Contex }, true) return nil } + func (m *ElastiCacheMock) DescribeUsersPagesWithContext(_ aws.Context, _ *elasticache.DescribeUsersInput, fn func(*elasticache.DescribeUsersOutput, bool) bool, _ ...request.Option) error { if m.Unauth { return trace.AccessDenied("unauthorized") @@ -607,12 +647,14 @@ func (m *ElastiCacheMock) DescribeCacheClustersPagesWithContext(aws.Context, *el } return trace.NotImplemented("elasticache:DescribeCacheClustersPagesWithContext is not implemented") } + func (m *ElastiCacheMock) DescribeCacheSubnetGroupsPagesWithContext(aws.Context, *elasticache.DescribeCacheSubnetGroupsInput, func(*elasticache.DescribeCacheSubnetGroupsOutput, bool) bool, ...request.Option) error { if m.Unauth { return trace.AccessDenied("unauthorized") } return trace.NotImplemented("elasticache:DescribeCacheSubnetGroupsPagesWithContext is not implemented") } + func (m *ElastiCacheMock) ListTagsForResourceWithContext(_ aws.Context, input *elasticache.ListTagsForResourceInput, _ ...request.Option) (*elasticache.TagListMessage, error) { if m.Unauth { return nil, trace.AccessDenied("unauthorized") @@ -630,6 +672,7 @@ func (m *ElastiCacheMock) ListTagsForResourceWithContext(_ aws.Context, input *e TagList: tags, }, nil } + func (m *ElastiCacheMock) ModifyUserWithContext(_ aws.Context, input *elasticache.ModifyUserInput, opts ...request.Option) (*elasticache.ModifyUserOutput, error) { if m.Unauth { return nil, trace.AccessDenied("unauthorized") @@ -687,6 +730,7 @@ func (m *MemoryDBMock) AddMockUser(user *memorydb.User, tagsMap map[string]strin m.Users = append(m.Users, user) m.addTags(aws.StringValue(user.ARN), tagsMap) } + func (m *MemoryDBMock) addTags(arn string, tagsMap map[string]string) { if m.TagsByARN == nil { m.TagsByARN = make(map[string][]*memorydb.Tag) @@ -701,11 +745,12 @@ func (m *MemoryDBMock) addTags(arn string, tagsMap map[string]string) { } m.TagsByARN[arn] = tags } + func (m *MemoryDBMock) DescribeSubnetGroupsWithContext(aws.Context, *memorydb.DescribeSubnetGroupsInput, ...request.Option) (*memorydb.DescribeSubnetGroupsOutput, error) { return nil, trace.AccessDenied("unauthorized") } -func (m *MemoryDBMock) DescribeClustersWithContext(_ aws.Context, input *memorydb.DescribeClustersInput, _ ...request.Option) (*memorydb.DescribeClustersOutput, error) { +func (m *MemoryDBMock) DescribeClustersWithContext(_ aws.Context, input *memorydb.DescribeClustersInput, _ ...request.Option) (*memorydb.DescribeClustersOutput, error) { if aws.StringValue(input.ClusterName) == "" { return &memorydb.DescribeClustersOutput{ Clusters: m.Clusters, @@ -721,6 +766,7 @@ func (m *MemoryDBMock) DescribeClustersWithContext(_ aws.Context, input *memoryd } return nil, trace.NotFound("cluster %v not found", aws.StringValue(input.ClusterName)) } + func (m *MemoryDBMock) ListTagsWithContext(_ aws.Context, input *memorydb.ListTagsInput, _ ...request.Option) (*memorydb.ListTagsOutput, error) { if m.TagsByARN == nil { return nil, trace.NotFound("no tags") @@ -735,11 +781,13 @@ func (m *MemoryDBMock) ListTagsWithContext(_ aws.Context, input *memorydb.ListTa TagList: tags, }, nil } + func (m *MemoryDBMock) DescribeUsersWithContext(aws.Context, *memorydb.DescribeUsersInput, ...request.Option) (*memorydb.DescribeUsersOutput, error) { return &memorydb.DescribeUsersOutput{ Users: m.Users, }, nil } + func (m *MemoryDBMock) UpdateUserWithContext(_ aws.Context, input *memorydb.UpdateUserInput, opts ...request.Option) (*memorydb.UpdateUserOutput, error) { for _, user := range m.Users { if aws.StringValue(user.Name) == aws.StringValue(input.UserName) { @@ -838,3 +886,22 @@ func RedshiftGetClusterCredentialsOutput(user, password string, clock clockwork. Expiration: aws.Time(clock.Now().Add(15 * time.Minute)), } } + +// EKSMock is a mock EKS client. +type EKSMock struct { + eksiface.EKSAPI + Clusters []*eks.Cluster + Notify chan struct{} +} + +func (e *EKSMock) DescribeClusterWithContext(_ aws.Context, req *eks.DescribeClusterInput, _ ...request.Option) (*eks.DescribeClusterOutput, error) { + defer func() { + e.Notify <- struct{}{} + }() + for _, cluster := range e.Clusters { + if aws.StringValue(req.Name) == aws.StringValue(cluster.Name) { + return &eks.DescribeClusterOutput{Cluster: cluster}, nil + } + } + return nil, trace.NotFound("cluster %v not found", aws.StringValue(req.Name)) +} diff --git a/lib/cloud/mocks/azure.go b/lib/cloud/mocks/azure.go new file mode 100644 index 0000000000000..37385572dd439 --- /dev/null +++ b/lib/cloud/mocks/azure.go @@ -0,0 +1,57 @@ +/* +Copyright 2023 Gravitational, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mocks + +import ( + "context" + "time" + + "github.com/gravitational/trace" + "github.com/jonboulle/clockwork" + "k8s.io/client-go/rest" + + "github.com/gravitational/teleport/lib/cloud/azure" +) + +// AKSClusterEntry is an entry in the AKSMock.Clusters list. +type AKSClusterEntry struct { + azure.ClusterCredentialsConfig + Config *rest.Config + TTL time.Duration +} + +// AKSMock implements the azure.AKSClient interface for tests. +type AKSMock struct { + azure.AKSClient + Clusters []AKSClusterEntry + Notify chan struct{} + Clock clockwork.Clock +} + +func (a *AKSMock) ClusterCredentials(ctx context.Context, cfg azure.ClusterCredentialsConfig) (*rest.Config, time.Time, error) { + defer func() { + a.Notify <- struct{}{} + }() + for _, cluster := range a.Clusters { + if cluster.ClusterCredentialsConfig.ResourceGroup == cfg.ResourceGroup && + cluster.ClusterCredentialsConfig.ResourceName == cfg.ResourceName && + cluster.ClusterCredentialsConfig.TenantID == cfg.TenantID { + return cluster.Config, a.Clock.Now().Add(cluster.TTL), nil + } + } + return nil, time.Now(), trace.NotFound("cluster not found") +} diff --git a/lib/cloud/mocks/gcp.go b/lib/cloud/mocks/gcp.go index 58aaedb55c6a9..3dc076aca4fa6 100644 --- a/lib/cloud/mocks/gcp.go +++ b/lib/cloud/mocks/gcp.go @@ -19,8 +19,12 @@ package mocks import ( "context" "crypto/tls" + "time" + "github.com/gravitational/trace" + "github.com/jonboulle/clockwork" sqladmin "google.golang.org/api/sqladmin/v1beta4" + "k8s.io/client-go/rest" "github.com/gravitational/teleport/api/types" "github.com/gravitational/teleport/lib/cloud/gcp" @@ -48,3 +52,30 @@ func (g *GCPSQLAdminClientMock) GetDatabaseInstance(ctx context.Context, db type func (g *GCPSQLAdminClientMock) GenerateEphemeralCert(ctx context.Context, db types.Database, identity tlsca.Identity) (*tls.Certificate, error) { return g.EphemeralCert, nil } + +// GKEClusterEntry is an entry in the GKEMock.Clusters list. +type GKEClusterEntry struct { + gcp.ClusterDetails + Config *rest.Config + TTL time.Duration +} + +// GKEMock implements the gcp.GKEClient interface for tests. +type GKEMock struct { + gcp.GKEClient + Clusters []GKEClusterEntry + Notify chan struct{} + Clock clockwork.Clock +} + +func (g *GKEMock) GetClusterRestConfig(ctx context.Context, cfg gcp.ClusterDetails) (*rest.Config, time.Time, error) { + defer func() { + g.Notify <- struct{}{} + }() + for _, cluster := range g.Clusters { + if cluster.ClusterDetails == cfg { + return cluster.Config, g.Clock.Now().Add(cluster.TTL), nil + } + } + return nil, time.Now(), trace.NotFound("cluster not found") +} diff --git a/lib/config/configuration.go b/lib/config/configuration.go index 614139b2d3718..7dbb2c035e83d 100644 --- a/lib/config/configuration.go +++ b/lib/config/configuration.go @@ -1342,12 +1342,13 @@ func applyKubeConfig(fc *FileConfig, cfg *servicecfg.Config) error { } for _, matcher := range fc.Kube.ResourceMatchers { - if matcher.AWS.AssumeRoleARN != "" { - return trace.NotImplemented("assume_role_arn is not supported for kube resource matchers") - } cfg.Kube.ResourceMatchers = append(cfg.Kube.ResourceMatchers, services.ResourceMatcher{ Labels: matcher.Labels, + AWS: services.ResourceMatcherAWS{ + AssumeRoleARN: matcher.AWS.AssumeRoleARN, + ExternalID: matcher.AWS.ExternalID, + }, }) } diff --git a/lib/config/configuration_test.go b/lib/config/configuration_test.go index 56eafdc99447c..cfa8f3940c419 100644 --- a/lib/config/configuration_test.go +++ b/lib/config/configuration_test.go @@ -3443,7 +3443,7 @@ func TestApplyConfig_JamfService(t *testing.T) { const password = "supersecret!!1!" passwordFile := filepath.Join(tempDir, "test_jamf_password.txt") require.NoError(t, - os.WriteFile(passwordFile, []byte(password+"\n"), 0400), + os.WriteFile(passwordFile, []byte(password+"\n"), 0o400), "WriteFile(%q) failed", passwordFile) minimalYAML := fmt.Sprintf(` @@ -4030,7 +4030,24 @@ func TestApplyKubeConfig(t *testing.T) { }, }, }, - wantError: true, + wantError: false, + wantServiceConfig: servicecfg.KubeConfig{ + ListenAddr: utils.MustParseAddr("0.0.0.0:8888"), + KubeconfigPath: "path-to-kubeconfig", + ResourceMatchers: []services.ResourceMatcher{ + { + Labels: map[string]apiutils.Strings{"a": {"b"}}, + AWS: services.ResourceMatcherAWS{ + AssumeRoleARN: "arn:aws:iam::123456789012:role/KubeAccess", + ExternalID: "externalID123", + }, + }, + }, + Limiter: limiter.Config{ + MaxConnections: defaults.LimiterMaxConnections, + MaxNumberOfUsers: 250, + }, + }, }, { name: "valid", diff --git a/lib/kube/proxy/cluster_details.go b/lib/kube/proxy/cluster_details.go index 14bca0250cfa0..dda7ff3b4e918 100644 --- a/lib/kube/proxy/cluster_details.go +++ b/lib/kube/proxy/cluster_details.go @@ -24,6 +24,7 @@ import ( "github.com/aws/aws-sdk-go/service/sts" "github.com/aws/aws-sdk-go/service/sts/stsiface" "github.com/gravitational/trace" + "github.com/jonboulle/clockwork" "github.com/sirupsen/logrus" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" @@ -34,6 +35,7 @@ import ( "github.com/gravitational/teleport/lib/cloud/gcp" "github.com/gravitational/teleport/lib/labels" "github.com/gravitational/teleport/lib/service/servicecfg" + "github.com/gravitational/teleport/lib/services" ) // kubeDetails contain the cluster-related details including authentication. @@ -45,21 +47,36 @@ type kubeDetails struct { kubeCluster types.KubeCluster } +// clusterDetailsConfig contains the configuration for creating a proxied cluster. +type clusterDetailsConfig struct { + // cloudClients is the cloud clients to use for dynamic clusters. + cloudClients cloud.Clients + // cluster is the cluster to create a proxied cluster for. + cluster types.KubeCluster + // log is the logger to use. + log *logrus.Entry + // checker is the permissions checker to use. + checker servicecfg.ImpersonationPermissionsChecker + // resourceMatchers is the list of resource matchers to match the cluster against + // to determine if we should assume the role or not for AWS. + resourceMatchers []services.ResourceMatcher +} + // newClusterDetails creates a proxied kubeDetails structure given a dynamic cluster. -func newClusterDetails(ctx context.Context, cloudClients cloud.Clients, cluster types.KubeCluster, log *logrus.Entry, checker servicecfg.ImpersonationPermissionsChecker) (*kubeDetails, error) { +func newClusterDetails(ctx context.Context, cfg clusterDetailsConfig) (*kubeDetails, error) { var dynLabels *labels.Dynamic - creds, err := getKubeClusterCredentials(ctx, cloudClients, cluster, log, checker) + creds, err := getKubeClusterCredentials(ctx, cfg) if err != nil { return nil, trace.Wrap(err) } - if len(cluster.GetDynamicLabels()) > 0 { + if len(cfg.cluster.GetDynamicLabels()) > 0 { dynLabels, err = labels.NewDynamic( ctx, &labels.DynamicConfig{ - Labels: cluster.GetDynamicLabels(), - Log: log, + Labels: cfg.cluster.GetDynamicLabels(), + Log: cfg.log, }) if err != nil { return nil, trace.Wrap(err) @@ -71,7 +88,7 @@ func newClusterDetails(ctx context.Context, cloudClients cloud.Clients, cluster return &kubeDetails{ kubeCreds: creds, dynamicLabels: dynLabels, - kubeCluster: cluster, + kubeCluster: cfg.cluster, }, nil } @@ -84,27 +101,31 @@ func (k *kubeDetails) Close() { } // getKubeClusterCredentials generates kube credentials for dynamic clusters. -func getKubeClusterCredentials(ctx context.Context, cloudClients cloud.Clients, cluster types.KubeCluster, log *logrus.Entry, checker servicecfg.ImpersonationPermissionsChecker) (kubeCreds, error) { +func getKubeClusterCredentials(ctx context.Context, cfg clusterDetailsConfig) (kubeCreds, error) { + dynCredsCfg := dynamicCredsConfig{kubeCluster: cfg.cluster, log: cfg.log, checker: cfg.checker, resourceMatchers: cfg.resourceMatchers} switch { - case cluster.IsKubeconfig(): - return getStaticCredentialsFromKubeconfig(ctx, cluster, log, checker) - case cluster.IsAzure(): - return getAzureCredentials(ctx, cloudClients, cluster, log, checker) - case cluster.IsAWS(): - return getAWSCredentials(ctx, cloudClients, cluster, log, checker) - case cluster.IsGCP(): - return getGCPCredentials(ctx, cloudClients, cluster, log, checker) + case cfg.cluster.IsKubeconfig(): + return getStaticCredentialsFromKubeconfig(ctx, cfg.cluster, cfg.log, cfg.checker) + case cfg.cluster.IsAzure(): + return getAzureCredentials(ctx, cfg.cloudClients, dynCredsCfg) + case cfg.cluster.IsAWS(): + return getAWSCredentials(ctx, cfg.cloudClients, dynCredsCfg) + case cfg.cluster.IsGCP(): + return getGCPCredentials(ctx, cfg.cloudClients, dynCredsCfg) default: - return nil, trace.BadParameter("authentication method provided for cluster %q not supported", cluster.GetName()) + return nil, trace.BadParameter("authentication method provided for cluster %q not supported", cfg.cluster.GetName()) } } // getAzureCredentials creates a dynamicCreds that generates and updates the access credentials to a AKS Kubernetes cluster. -func getAzureCredentials(ctx context.Context, cloudClients cloud.Clients, cluster types.KubeCluster, log *logrus.Entry, checker servicecfg.ImpersonationPermissionsChecker) (*dynamicKubeCreds, error) { +func getAzureCredentials(ctx context.Context, cloudClients cloud.Clients, cfg dynamicCredsConfig) (*dynamicKubeCreds, error) { // create a client that returns the credentials for kubeCluster - client := azureRestConfigClient(cloudClients) + cfg.client = azureRestConfigClient(cloudClients) - creds, err := newDynamicKubeCreds(ctx, cluster, log, client, checker) + creds, err := newDynamicKubeCreds( + ctx, + cfg, + ) return creds, trace.Wrap(err) } @@ -126,19 +147,41 @@ func azureRestConfigClient(cloudClients cloud.Clients) dynamicCredsClient { } // getAWSCredentials creates a dynamicKubeCreds that generates and updates the access credentials to a EKS kubernetes cluster. -func getAWSCredentials(ctx context.Context, cloudClients cloud.Clients, cluster types.KubeCluster, log *logrus.Entry, checker servicecfg.ImpersonationPermissionsChecker) (*dynamicKubeCreds, error) { +func getAWSCredentials(ctx context.Context, cloudClients cloud.Clients, cfg dynamicCredsConfig) (*dynamicKubeCreds, error) { // create a client that returns the credentials for kubeCluster - client := getAWSClientRestConfig(cloudClients) - creds, err := newDynamicKubeCreds(ctx, cluster, log, client, checker) + cfg.client = getAWSClientRestConfig(cloudClients, cfg.clock, cfg.resourceMatchers) + creds, err := newDynamicKubeCreds(ctx, cfg) return creds, trace.Wrap(err) } +// getAWSResourceMatcherToCluster returns the AWS assume role ARN and external ID for the cluster that matches the kubeCluster. +// If no match is found, nil is returned, which means that we should not attempt to assume a role. +func getAWSResourceMatcherToCluster(kubeCluster types.KubeCluster, resourceMatchers []services.ResourceMatcher) *services.ResourceMatcherAWS { + if !kubeCluster.IsAWS() { + return nil + } + for _, matcher := range resourceMatchers { + if len(matcher.Labels) == 0 || matcher.AWS.AssumeRoleARN == "" { + continue + } + if match, _, _ := services.MatchLabels(matcher.Labels, kubeCluster.GetAllLabels()); !match { + continue + } + + return &(matcher.AWS) + } + return nil +} + // getAWSClientRestConfig creates a dynamicCredsClient that generates returns credentials to EKS clusters. -func getAWSClientRestConfig(cloudClients cloud.Clients) dynamicCredsClient { +func getAWSClientRestConfig(cloudClients cloud.Clients, clock clockwork.Clock, resourceMatchers []services.ResourceMatcher) dynamicCredsClient { return func(ctx context.Context, cluster types.KubeCluster) (*rest.Config, time.Time, error) { - // TODO(gavin): support assume_role_arn for AWS EKS. region := cluster.GetAWSConfig().Region - regionalClient, err := cloudClients.GetAWSEKSClient(ctx, region) + var opts []cloud.AWSAssumeRoleOptionFn + if awsAssume := getAWSResourceMatcherToCluster(cluster, resourceMatchers); awsAssume != nil { + opts = append(opts, cloud.WithAssumeRole(awsAssume.AssumeRoleARN, awsAssume.ExternalID)) + } + regionalClient, err := cloudClients.GetAWSEKSClient(ctx, region, opts...) if err != nil { return nil, time.Time{}, trace.Wrap(err) } @@ -160,12 +203,12 @@ func getAWSClientRestConfig(cloudClients cloud.Clients) dynamicCredsClient { return nil, time.Time{}, trace.BadParameter("invalid api endpoint for cluster %q", cluster.GetAWSConfig().Name) } - stsClient, err := cloudClients.GetAWSSTSClient(ctx, region) + stsClient, err := cloudClients.GetAWSSTSClient(ctx, region, opts...) if err != nil { return nil, time.Time{}, trace.Wrap(err) } - token, exp, err := genAWSToken(stsClient, cluster.GetAWSConfig().Name) + token, exp, err := genAWSToken(stsClient, cluster.GetAWSConfig().Name, clock) if err != nil { return nil, time.Time{}, trace.Wrap(err) } @@ -182,7 +225,7 @@ func getAWSClientRestConfig(cloudClients cloud.Clients) dynamicCredsClient { // genAWSToken creates an AWS token to access EKS clusters. // Logic from https://github.com/aws/aws-cli/blob/6c0d168f0b44136fc6175c57c090d4b115437ad1/awscli/customizations/eks/get_token.py#L211-L229 -func genAWSToken(stsClient stsiface.STSAPI, clusterID string) (string, time.Time, error) { +func genAWSToken(stsClient stsiface.STSAPI, clusterID string, clock clockwork.Clock) (string, time.Time, error) { const ( // The sts GetCallerIdentity request is valid for 15 minutes regardless of this parameters value after it has been // signed. @@ -209,7 +252,7 @@ func genAWSToken(stsClient stsiface.STSAPI, clusterID string) (string, time.Time } // Set token expiration to 1 minute before the presigned URL expires for some cushion - tokenExpiration := time.Now().Local().Add(presignedURLExpiration - 1*time.Minute) + tokenExpiration := clock.Now().Add(presignedURLExpiration - 1*time.Minute) return v1Prefix + base64.RawURLEncoding.EncodeToString([]byte(presignedURLString)), tokenExpiration, nil } @@ -237,10 +280,10 @@ func getStaticCredentialsFromKubeconfig(ctx context.Context, cluster types.KubeC } // getGCPCredentials creates a dynamicKubeCreds that generates and updates the access credentials to a GKE kubernetes cluster. -func getGCPCredentials(ctx context.Context, cloudClients cloud.Clients, cluster types.KubeCluster, log *logrus.Entry, checker servicecfg.ImpersonationPermissionsChecker) (*dynamicKubeCreds, error) { +func getGCPCredentials(ctx context.Context, cloudClients cloud.Clients, cfg dynamicCredsConfig) (*dynamicKubeCreds, error) { // create a client that returns the credentials for kubeCluster - client := gcpRestConfigClient(cloudClients) - creds, err := newDynamicKubeCreds(ctx, cluster, log, client, checker) + cfg.client = gcpRestConfigClient(cloudClients) + creds, err := newDynamicKubeCreds(ctx, cfg) return creds, trace.Wrap(err) } diff --git a/lib/kube/proxy/kube_creds.go b/lib/kube/proxy/kube_creds.go index 08c86acefa44a..be677ae261105 100644 --- a/lib/kube/proxy/kube_creds.go +++ b/lib/kube/proxy/kube_creds.go @@ -24,6 +24,7 @@ import ( "time" "github.com/gravitational/trace" + "github.com/jonboulle/clockwork" "github.com/sirupsen/logrus" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" @@ -31,6 +32,7 @@ import ( "github.com/gravitational/teleport/api/types" "github.com/gravitational/teleport/lib/service/servicecfg" + "github.com/gravitational/teleport/lib/services" ) type kubeCreds interface { @@ -158,39 +160,79 @@ type dynamicCredsClient func(ctx context.Context, cluster types.KubeCluster) (cf // function and renews them whenever they are about to expire. type dynamicKubeCreds struct { ctx context.Context - renewTicker *time.Ticker + renewTicker clockwork.Ticker staticCreds *staticKubeCreds log logrus.FieldLogger closeC chan struct{} client dynamicCredsClient checker servicecfg.ImpersonationPermissionsChecker + clock clockwork.Clock sync.RWMutex + wg sync.WaitGroup +} + +// dynamicCredsConfig contains configuration for dynamicKubeCreds. +type dynamicCredsConfig struct { + kubeCluster types.KubeCluster + log logrus.FieldLogger + client dynamicCredsClient + checker servicecfg.ImpersonationPermissionsChecker + clock clockwork.Clock + initialRenewInterval time.Duration + resourceMatchers []services.ResourceMatcher +} + +func (d *dynamicCredsConfig) checkAndSetDefaults() error { + if d.kubeCluster == nil { + return trace.BadParameter("missing kubeCluster") + } + if d.log == nil { + return trace.BadParameter("missing log") + } + if d.client == nil { + return trace.BadParameter("missing client") + } + if d.checker == nil { + return trace.BadParameter("missing checker") + } + if d.clock == nil { + d.clock = clockwork.NewRealClock() + } + if d.initialRenewInterval == 0 { + d.initialRenewInterval = time.Hour + } + return nil } // newDynamicKubeCreds creates a new dynamicKubeCreds refresher and starts the // credentials refresher mechanism to renew them once they are about to expire. -func newDynamicKubeCreds(ctx context.Context, kubeCluster types.KubeCluster, log logrus.FieldLogger, client dynamicCredsClient, checker servicecfg.ImpersonationPermissionsChecker) (*dynamicKubeCreds, error) { +func newDynamicKubeCreds(ctx context.Context, cfg dynamicCredsConfig) (*dynamicKubeCreds, error) { + if err := cfg.checkAndSetDefaults(); err != nil { + return nil, trace.Wrap(err) + } dyn := &dynamicKubeCreds{ ctx: ctx, - log: log, + log: cfg.log, closeC: make(chan struct{}), - client: client, - renewTicker: time.NewTicker(time.Hour), - checker: checker, + client: cfg.client, + renewTicker: cfg.clock.NewTicker(cfg.initialRenewInterval), + checker: cfg.checker, + clock: cfg.clock, } - if err := dyn.renewClientset(kubeCluster); err != nil { + if err := dyn.renewClientset(cfg.kubeCluster); err != nil { return nil, trace.Wrap(err) } - + dyn.wg.Add(1) go func() { + defer dyn.wg.Done() for { select { case <-dyn.closeC: return - case <-dyn.renewTicker.C: - if err := dyn.renewClientset(kubeCluster); err != nil { - log.WithError(err).Warnf("Unable to renew cluster %q credentials.", kubeCluster.GetName()) + case <-dyn.renewTicker.Chan(): + if err := dyn.renewClientset(cfg.kubeCluster); err != nil { + logrus.WithError(err).Warnf("Unable to renew cluster %q credentials.", cfg.kubeCluster.GetName()) } } } @@ -237,6 +279,8 @@ func (d *dynamicKubeCreds) wrapTransport(rt http.RoundTripper) (http.RoundTrippe func (d *dynamicKubeCreds) close() error { close(d.closeC) + d.wg.Wait() + d.renewTicker.Stop() return nil } @@ -263,7 +307,8 @@ func (d *dynamicKubeCreds) renewClientset(cluster types.KubeCluster) error { d.staticCreds = creds // prepares the next renew cycle if !exp.IsZero() { - d.renewTicker.Reset(time.Until(exp) / 2) + reset := exp.Sub(d.clock.Now()) / 2 + d.renewTicker.Reset(reset) } return nil } diff --git a/lib/kube/proxy/kube_creds_test.go b/lib/kube/proxy/kube_creds_test.go new file mode 100644 index 0000000000000..5bda7e4713f2a --- /dev/null +++ b/lib/kube/proxy/kube_creds_test.go @@ -0,0 +1,338 @@ +/* +Copyright 2023 Gravitational, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package proxy + +import ( + "context" + "encoding/base64" + "net/url" + "strings" + "testing" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/eks" + "github.com/gravitational/trace" + "github.com/jonboulle/clockwork" + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/require" + authztypes "k8s.io/client-go/kubernetes/typed/authorization/v1" + "k8s.io/client-go/rest" + + "github.com/gravitational/teleport/api/types" + "github.com/gravitational/teleport/api/utils" + "github.com/gravitational/teleport/lib/cloud" + "github.com/gravitational/teleport/lib/cloud/azure" + "github.com/gravitational/teleport/lib/cloud/gcp" + "github.com/gravitational/teleport/lib/cloud/mocks" + "github.com/gravitational/teleport/lib/fixtures" + "github.com/gravitational/teleport/lib/services" +) + +// Test_DynamicKubeCreds tests the dynamic kube credrentials generator for +// AWS, GCP, and Azure clusters accessed using their respective IAM credentials. +// This test mocks the cloud provider clients and the STS client to generate +// rest.Config objects for each cluster. It also tests the renewal of the +// credentials when they expire. +func Test_DynamicKubeCreds(t *testing.T) { + t.Parallel() + var ( + fakeClock = clockwork.NewFakeClock() + log = logrus.New() + notify = make(chan struct{}, 1) + ttl = 14 * time.Minute + ) + + awsKube, err := types.NewKubernetesClusterV3( + types.Metadata{ + Name: "aws", + }, + types.KubernetesClusterSpecV3{ + AWS: types.KubeAWS{ + Region: "us-west-2", + AccountID: "1234567890", + Name: "eks", + }, + }, + ) + require.NoError(t, err) + gkeKube, err := types.NewKubernetesClusterV3( + types.Metadata{ + Name: "gke", + }, + types.KubernetesClusterSpecV3{ + GCP: types.KubeGCP{ + Location: "us-west-2", + ProjectID: "1234567890", + Name: "gke", + }, + }, + ) + require.NoError(t, err) + aksKube, err := types.NewKubernetesClusterV3( + types.Metadata{ + Name: "aks", + }, + types.KubernetesClusterSpecV3{ + Azure: types.KubeAzure{ + TenantID: "id", + ResourceGroup: "1234567890", + ResourceName: "aks-name", + SubscriptionID: "12345", + }, + }, + ) + require.NoError(t, err) + + // mock sts client + u := &url.URL{ + Scheme: "https", + Host: "sts.amazonaws.com", + Path: "/?Action=GetCallerIdentity&Version=2011-06-15", + } + sts := &mocks.STSMock{ + // u is used to presign the request + // here we just verify the pre-signed request includes this url. + URL: u, + } + // mock clients + cloudclients := &cloud.TestCloudClients{ + STS: sts, + EKS: &mocks.EKSMock{ + Notify: notify, + Clusters: []*eks.Cluster{ + { + Endpoint: aws.String("https://api.eks.us-west-2.amazonaws.com"), + Name: aws.String(awsKube.GetAWSConfig().Name), + CertificateAuthority: &eks.Certificate{ + Data: aws.String(base64.RawStdEncoding.EncodeToString([]byte(fixtures.TLSCACertPEM))), + }, + }, + }, + }, + GCPGKE: &mocks.GKEMock{ + Notify: notify, + Clock: fakeClock, + Clusters: []mocks.GKEClusterEntry{ + { + Config: &rest.Config{ + Host: "https://api.gke.google.com", + TLSClientConfig: rest.TLSClientConfig{ + CAData: []byte(fixtures.TLSCACertPEM), + }, + }, + ClusterDetails: gcp.ClusterDetails{ + Name: gkeKube.GetGCPConfig().Name, + ProjectID: gkeKube.GetGCPConfig().ProjectID, + Location: gkeKube.GetGCPConfig().Location, + }, + TTL: ttl, + }, + }, + }, + AzureAKSClientPerSub: map[string]azure.AKSClient{ + "12345": &mocks.AKSMock{ + Notify: notify, + Clock: fakeClock, + Clusters: []mocks.AKSClusterEntry{ + { + Config: &rest.Config{ + Host: "https://api.aks.microsoft.com", + TLSClientConfig: rest.TLSClientConfig{ + CAData: []byte(fixtures.TLSCACertPEM), + }, + }, + TTL: ttl, + ClusterCredentialsConfig: azure.ClusterCredentialsConfig{ + ResourceName: aksKube.GetAzureConfig().ResourceName, + ResourceGroup: aksKube.GetAzureConfig().ResourceGroup, + TenantID: aksKube.GetAzureConfig().TenantID, + }, + }, + }, + }, + }, + } + validateEKSToken := func(token string) error { + if token == "" { + return trace.BadParameter("missing bearer token") + } + tokens := strings.Split(token, ".") + if len(tokens) != 2 { + return trace.BadParameter("invalid bearer token") + } + if tokens[0] != "k8s-aws-v1" { + return trace.BadParameter("token must start with k8s-aws-v1") + } + dec, err := base64.RawStdEncoding.DecodeString(tokens[1]) + if err != nil { + return trace.Wrap(err) + } + if string(dec) != u.String() { + return trace.BadParameter("invalid token payload") + } + return nil + } + type args struct { + cluster types.KubeCluster + client dynamicCredsClient + validateBearerToken func(string) error + } + tests := []struct { + name string + args args + wantAddr string + wantAssumedRole []string + wantExternalIds []string + }{ + { + name: "aws eks cluster without assume role", + args: args{ + cluster: awsKube, + client: getAWSClientRestConfig(cloudclients, fakeClock, nil), + validateBearerToken: validateEKSToken, + }, + wantAddr: "api.eks.us-west-2.amazonaws.com:443", + }, + { + name: "aws eks cluster with unmatched assume role", + args: args{ + cluster: awsKube, + client: getAWSClientRestConfig(cloudclients, fakeClock, []services.ResourceMatcher{ + { + Labels: types.Labels{ + "rand": []string{"value"}, + }, + AWS: services.ResourceMatcherAWS{ + AssumeRoleARN: "arn:aws:iam::123456789012:role/eks-role", + ExternalID: "1234567890", + }, + }, + }), + validateBearerToken: validateEKSToken, + }, + wantAddr: "api.eks.us-west-2.amazonaws.com:443", + }, + { + name: "aws eks cluster with assume role", + args: args{ + cluster: awsKube, + client: getAWSClientRestConfig( + cloudclients, + fakeClock, + []services.ResourceMatcher{ + { + Labels: types.Labels{ + types.Wildcard: []string{types.Wildcard}, + }, + AWS: services.ResourceMatcherAWS{ + AssumeRoleARN: "arn:aws:iam::123456789012:role/eks-role", + ExternalID: "1234567890", + }, + }, + }, + ), + validateBearerToken: func(token string) error { + if token == "" { + return trace.BadParameter("missing bearer token") + } + tokens := strings.Split(token, ".") + if len(tokens) != 2 { + return trace.BadParameter("invalid bearer token") + } + if tokens[0] != "k8s-aws-v1" { + return trace.BadParameter("token must start with k8s-aws-v1") + } + dec, err := base64.RawStdEncoding.DecodeString(tokens[1]) + if err != nil { + return trace.Wrap(err) + } + if string(dec) != u.String() { + return trace.BadParameter("invalid token payload") + } + return nil + }, + }, + wantAddr: "api.eks.us-west-2.amazonaws.com:443", + wantAssumedRole: []string{"arn:aws:iam::123456789012:role/eks-role"}, + wantExternalIds: []string{"1234567890"}, + }, + { + name: "gcp gke cluster", + args: args{ + cluster: gkeKube, + client: gcpRestConfigClient(cloudclients), + validateBearerToken: func(_ string) error { return nil }, + }, + wantAddr: "api.gke.google.com:443", + }, + { + name: "azure aks cluster", + args: args{ + cluster: aksKube, + client: azureRestConfigClient(cloudclients), + validateBearerToken: func(_ string) error { return nil }, + }, + wantAddr: "api.aks.microsoft.com:443", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := newDynamicKubeCreds( + context.Background(), + dynamicCredsConfig{ + clock: fakeClock, + checker: func(_ context.Context, _ string, + _ authztypes.SelfSubjectAccessReviewInterface, + ) error { + return nil + }, + log: log, + kubeCluster: tt.args.cluster, + client: tt.args.client, + initialRenewInterval: ttl / 2, + }, + ) + require.NoError(t, err) + select { + case <-notify: + case <-time.After(5 * time.Second): + t.Fatalf("timeout waiting for cluster to be ready") + } + for i := 0; i < 10; i++ { + require.Equal(t, got.getKubeRestConfig().CAData, []byte(fixtures.TLSCACertPEM)) + require.NoError(t, tt.args.validateBearerToken(got.getKubeRestConfig().BearerToken)) + require.Equal(t, got.getTargetAddr(), tt.wantAddr) + fakeClock.BlockUntil(1) + fakeClock.Advance(ttl / 2) + // notify receives a signal when the cloud client is invoked. + // this is used to test that the credentials are refreshed each time + // they are about to expire. + select { + case <-notify: + case <-time.After(5 * time.Second): + t.Fatalf("timeout waiting for cluster to be ready, i=%d", i) + } + } + require.NoError(t, got.close()) + + require.Equal(t, tt.wantAssumedRole, utils.Deduplicate(sts.GetAssumedRoleARNs())) + require.Equal(t, tt.wantExternalIds, utils.Deduplicate(sts.GetAssumedRoleExternalIDs())) + sts.ResetAssumeRoleHistory() + }) + } +} diff --git a/lib/kube/proxy/watcher.go b/lib/kube/proxy/watcher.go index 04373f33d02be..7ab5f8d32893b 100644 --- a/lib/kube/proxy/watcher.go +++ b/lib/kube/proxy/watcher.go @@ -181,10 +181,13 @@ func (m *monitoredKubeClusters) get() types.ResourcesWithLabelsMap { func (s *TLSServer) registerKubeCluster(ctx context.Context, cluster types.KubeCluster) error { clusterDetails, err := newClusterDetails( ctx, - s.CloudClients, - cluster, - s.log, - s.CheckImpersonationPermissions, + clusterDetailsConfig{ + cloudClients: s.CloudClients, + cluster: cluster, + log: s.log, + checker: s.CheckImpersonationPermissions, + resourceMatchers: s.ResourceMatchers, + }, ) if err != nil { return trace.Wrap(err) @@ -196,10 +199,13 @@ func (s *TLSServer) registerKubeCluster(ctx context.Context, cluster types.KubeC func (s *TLSServer) updateKubeCluster(ctx context.Context, cluster types.KubeCluster) error { clusterDetails, err := newClusterDetails( ctx, - s.CloudClients, - cluster, - s.log, - s.CheckImpersonationPermissions, + clusterDetailsConfig{ + cloudClients: s.CloudClients, + cluster: cluster, + log: s.log, + checker: s.CheckImpersonationPermissions, + resourceMatchers: s.ResourceMatchers, + }, ) if err != nil { return trace.Wrap(err)