Skip to content

Commit

Permalink
Support specifying assume_role_arn for Kube cluster matchers (#28282)
Browse files Browse the repository at this point in the history 
* Support specifiying `assume_role_arn` for Kube cluster matchers

This PR allows users to assume different AWS roles when interacting with
AWS EKS API. It allows users to proxy EKS clusters in different AWS
accounts using the same Teleport Kubernetes Service.

Example configuration:

```yaml

kubernetes_service:
  enabled: true
  resources:
  - labels:
      'a': 'b'
    aws:
      assume_role_arn: "arn:aws:iam::0987654321:role/KubeAccess"
      external_id: "0987654321"
  - labels:
      'c': 'd'
    aws:
      assume_role_arn: "arn:aws:iam::123456789012:role/KubeAccess"
      external_id: "123456789012"

```

* reuse eks token validation
  • Loading branch information
@tigrato
tigrato committed Jun 28, 2023
1 parent 2e47bf7 commit 73cfb29
Show file tree
Hide file tree
Showing 6 changed files with 183 additions and 46 deletions.
7 changes: 4 additions & 3 deletions lib/config/configuration.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1323,12 +1323,13 @@ func applyKubeConfig(fc *FileConfig, cfg *servicecfg.Config) error {
}

for _, matcher := range fc.Kube.ResourceMatchers {
if matcher.AWS.AssumeRoleARN != "" {
return trace.NotImplemented("assume_role_arn is not supported for kube resource matchers")
}
cfg.Kube.ResourceMatchers = append(cfg.Kube.ResourceMatchers,
services.ResourceMatcher{
Labels: matcher.Labels,
AWS: services.ResourceMatcherAWS{
AssumeRoleARN: matcher.AWS.AssumeRoleARN,
ExternalID: matcher.AWS.ExternalID,
},
})
}

Expand Down
21 changes: 19 additions & 2 deletions lib/config/configuration_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3439,7 +3439,7 @@ func TestApplyConfig_JamfService(t *testing.T) {
const password = "supersecret!!1!"
passwordFile := filepath.Join(tempDir, "test_jamf_password.txt")
require.NoError(t,
os.WriteFile(passwordFile, []byte(password+"\n"), 0400),
os.WriteFile(passwordFile, []byte(password+"\n"), 0o400),
"WriteFile(%q) failed", passwordFile)

minimalYAML := fmt.Sprintf(`
Expand Down Expand Up @@ -4026,7 +4026,24 @@ func TestApplyKubeConfig(t *testing.T) {
},
},
},
wantError: true,
wantError: false,
wantServiceConfig: servicecfg.KubeConfig{
ListenAddr: utils.MustParseAddr("0.0.0.0:8888"),
KubeconfigPath: "path-to-kubeconfig",
ResourceMatchers: []services.ResourceMatcher{
{
Labels: map[string]apiutils.Strings{"a": {"b"}},
AWS: services.ResourceMatcherAWS{
AssumeRoleARN: "arn:aws:iam::123456789012:role/KubeAccess",
ExternalID: "externalID123",
},
},
},
Limiter: limiter.Config{
MaxConnections: defaults.LimiterMaxConnections,
MaxNumberOfUsers: 250,
},
},
},
{
name: "valid",
Expand Down
81 changes: 60 additions & 21 deletions lib/kube/proxy/cluster_details.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ import (
"github.com/gravitational/teleport/lib/cloud/gcp"
"github.com/gravitational/teleport/lib/labels"
"github.com/gravitational/teleport/lib/service/servicecfg"
"github.com/gravitational/teleport/lib/services"
)

// kubeDetails contain the cluster-related details including authentication.
Expand All @@ -46,21 +47,36 @@ type kubeDetails struct {
kubeCluster types.KubeCluster
}

// clusterDetailsConfig contains the configuration for creating a proxied cluster.
type clusterDetailsConfig struct {
// cloudClients is the cloud clients to use for dynamic clusters.
cloudClients cloud.Clients
// cluster is the cluster to create a proxied cluster for.
cluster types.KubeCluster
// log is the logger to use.
log *logrus.Entry
// checker is the permissions checker to use.
checker servicecfg.ImpersonationPermissionsChecker
// resourceMatchers is the list of resource matchers to match the cluster against
// to determine if we should assume the role or not for AWS.
resourceMatchers []services.ResourceMatcher
}

// newClusterDetails creates a proxied kubeDetails structure given a dynamic cluster.
func newClusterDetails(ctx context.Context, cloudClients cloud.Clients, cluster types.KubeCluster, log *logrus.Entry, checker servicecfg.ImpersonationPermissionsChecker) (*kubeDetails, error) {
func newClusterDetails(ctx context.Context, cfg clusterDetailsConfig) (*kubeDetails, error) {
var dynLabels *labels.Dynamic

creds, err := getKubeClusterCredentials(ctx, cloudClients, cluster, log, checker)
creds, err := getKubeClusterCredentials(ctx, cfg)
if err != nil {
return nil, trace.Wrap(err)
}

if len(cluster.GetDynamicLabels()) > 0 {
if len(cfg.cluster.GetDynamicLabels()) > 0 {
dynLabels, err = labels.NewDynamic(
ctx,
&labels.DynamicConfig{
Labels: cluster.GetDynamicLabels(),
Log: log,
Labels: cfg.cluster.GetDynamicLabels(),
Log: cfg.log,
})
if err != nil {
return nil, trace.Wrap(err)
Expand All @@ -72,7 +88,7 @@ func newClusterDetails(ctx context.Context, cloudClients cloud.Clients, cluster
return &kubeDetails{
kubeCreds: creds,
dynamicLabels: dynLabels,
kubeCluster: cluster,
kubeCluster: cfg.cluster,
}, nil
}

Expand All @@ -85,18 +101,19 @@ func (k *kubeDetails) Close() {
}

// getKubeClusterCredentials generates kube credentials for dynamic clusters.
func getKubeClusterCredentials(ctx context.Context, cloudClients cloud.Clients, cluster types.KubeCluster, log *logrus.Entry, checker servicecfg.ImpersonationPermissionsChecker) (kubeCreds, error) {
func getKubeClusterCredentials(ctx context.Context, cfg clusterDetailsConfig) (kubeCreds, error) {
dynCredsCfg := dynamicCredsConfig{kubeCluster: cfg.cluster, log: cfg.log, checker: cfg.checker, resourceMatchers: cfg.resourceMatchers}
switch {
case cluster.IsKubeconfig():
return getStaticCredentialsFromKubeconfig(ctx, cluster, log, checker)
case cluster.IsAzure():
return getAzureCredentials(ctx, cloudClients, dynamicCredsConfig{kubeCluster: cluster, log: log, checker: checker})
case cluster.IsAWS():
return getAWSCredentials(ctx, cloudClients, dynamicCredsConfig{kubeCluster: cluster, log: log, checker: checker})
case cluster.IsGCP():
return getGCPCredentials(ctx, cloudClients, dynamicCredsConfig{kubeCluster: cluster, log: log, checker: checker})
case cfg.cluster.IsKubeconfig():
return getStaticCredentialsFromKubeconfig(ctx, cfg.cluster, cfg.log, cfg.checker)
case cfg.cluster.IsAzure():
return getAzureCredentials(ctx, cfg.cloudClients, dynCredsCfg)
case cfg.cluster.IsAWS():
return getAWSCredentials(ctx, cfg.cloudClients, dynCredsCfg)
case cfg.cluster.IsGCP():
return getGCPCredentials(ctx, cfg.cloudClients, dynCredsCfg)
default:
return nil, trace.BadParameter("authentication method provided for cluster %q not supported", cluster.GetName())
return nil, trace.BadParameter("authentication method provided for cluster %q not supported", cfg.cluster.GetName())
}
}

Expand Down Expand Up @@ -132,17 +149,39 @@ func azureRestConfigClient(cloudClients cloud.Clients) dynamicCredsClient {
// getAWSCredentials creates a dynamicKubeCreds that generates and updates the access credentials to a EKS kubernetes cluster.
func getAWSCredentials(ctx context.Context, cloudClients cloud.Clients, cfg dynamicCredsConfig) (*dynamicKubeCreds, error) {
// create a client that returns the credentials for kubeCluster
cfg.client = getAWSClientRestConfig(cloudClients, cfg.clock)
cfg.client = getAWSClientRestConfig(cloudClients, cfg.clock, cfg.resourceMatchers)
creds, err := newDynamicKubeCreds(ctx, cfg)
return creds, trace.Wrap(err)
}

// getAWSResourceMatcherToCluster returns the AWS assume role ARN and external ID for the cluster that matches the kubeCluster.
// If no match is found, nil is returned, which means that we should not attempt to assume a role.
func getAWSResourceMatcherToCluster(kubeCluster types.KubeCluster, resourceMatchers []services.ResourceMatcher) *services.ResourceMatcherAWS {
if !kubeCluster.IsAWS() {
return nil
}
for _, matcher := range resourceMatchers {
if len(matcher.Labels) == 0 || matcher.AWS.AssumeRoleARN == "" {
continue
}
if match, _, _ := services.MatchLabels(matcher.Labels, kubeCluster.GetAllLabels()); !match {
continue
}

return &(matcher.AWS)
}
return nil
}

// getAWSClientRestConfig creates a dynamicCredsClient that generates returns credentials to EKS clusters.
func getAWSClientRestConfig(cloudClients cloud.Clients, clock clockwork.Clock) dynamicCredsClient {
func getAWSClientRestConfig(cloudClients cloud.Clients, clock clockwork.Clock, resourceMatchers []services.ResourceMatcher) dynamicCredsClient {
return func(ctx context.Context, cluster types.KubeCluster) (*rest.Config, time.Time, error) {
// TODO(gavin): support assume_role_arn for AWS EKS.
region := cluster.GetAWSConfig().Region
regionalClient, err := cloudClients.GetAWSEKSClient(ctx, region)
var opts []cloud.AWSAssumeRoleOptionFn
if awsAssume := getAWSResourceMatcherToCluster(cluster, resourceMatchers); awsAssume != nil {
opts = append(opts, cloud.WithAssumeRole(awsAssume.AssumeRoleARN, awsAssume.ExternalID))
}
regionalClient, err := cloudClients.GetAWSEKSClient(ctx, region, opts...)
if err != nil {
return nil, time.Time{}, trace.Wrap(err)
}
Expand All @@ -164,7 +203,7 @@ func getAWSClientRestConfig(cloudClients cloud.Clients, clock clockwork.Clock) d
return nil, time.Time{}, trace.BadParameter("invalid api endpoint for cluster %q", cluster.GetAWSConfig().Name)
}

stsClient, err := cloudClients.GetAWSSTSClient(ctx, region)
stsClient, err := cloudClients.GetAWSSTSClient(ctx, region, opts...)
if err != nil {
return nil, time.Time{}, trace.Wrap(err)
}
Expand Down
2 changes: 2 additions & 0 deletions lib/kube/proxy/kube_creds.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ import (

"github.com/gravitational/teleport/api/types"
"github.com/gravitational/teleport/lib/service/servicecfg"
"github.com/gravitational/teleport/lib/services"
)

type kubeCreds interface {
Expand Down Expand Up @@ -171,6 +172,7 @@ type dynamicCredsConfig struct {
checker servicecfg.ImpersonationPermissionsChecker
clock clockwork.Clock
initialRenewInterval time.Duration
resourceMatchers []services.ResourceMatcher
}

func (d *dynamicCredsConfig) checkAndSetDefaults() error {
Expand Down
96 changes: 84 additions & 12 deletions lib/kube/proxy/kube_creds_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,11 +34,13 @@ import (
"k8s.io/client-go/rest"

"github.com/gravitational/teleport/api/types"
"github.com/gravitational/teleport/api/utils"
"github.com/gravitational/teleport/lib/cloud"
"github.com/gravitational/teleport/lib/cloud/azure"
"github.com/gravitational/teleport/lib/cloud/gcp"
"github.com/gravitational/teleport/lib/cloud/mocks"
"github.com/gravitational/teleport/lib/fixtures"
"github.com/gravitational/teleport/lib/services"
)

// Test_DynamicKubeCreds tests the dynamic kube credrentials generator for
Expand Down Expand Up @@ -102,13 +104,14 @@ func Test_DynamicKubeCreds(t *testing.T) {
Host: "sts.amazonaws.com",
Path: "/?Action=GetCallerIdentity&Version=2011-06-15",
}
sts := &mocks.STSMock{
// u is used to presign the request
// here we just verify the pre-signed request includes this url.
URL: u,
}
// mock clients
cloudclients := &cloud.TestCloudClients{
STS: &mocks.STSMock{
// u is used to presign the request
// here we just verify the pre-signed request includes this url.
URL: u,
},
STS: sts,
EKS: &mocks.EKSMock{
Notify: notify,
Clusters: []*eks.Cluster{
Expand Down Expand Up @@ -164,22 +167,85 @@ func Test_DynamicKubeCreds(t *testing.T) {
},
},
}

validateEKSToken := func(token string) error {
if token == "" {
return trace.BadParameter("missing bearer token")
}
tokens := strings.Split(token, ".")
if len(tokens) != 2 {
return trace.BadParameter("invalid bearer token")
}
if tokens[0] != "k8s-aws-v1" {
return trace.BadParameter("token must start with k8s-aws-v1")
}
dec, err := base64.RawStdEncoding.DecodeString(tokens[1])
if err != nil {
return trace.Wrap(err)
}
if string(dec) != u.String() {
return trace.BadParameter("invalid token payload")
}
return nil
}
type args struct {
cluster types.KubeCluster
client dynamicCredsClient
validateBearerToken func(string) error
}
tests := []struct {
name string
args args
wantAddr string
name string
args args
wantAddr string
wantAssumedRole []string
wantExternalIds []string
}{
{
name: "aws eks cluster",
name: "aws eks cluster without assume role",
args: args{
cluster: awsKube,
client: getAWSClientRestConfig(cloudclients, fakeClock, nil),
validateBearerToken: validateEKSToken,
},
wantAddr: "api.eks.us-west-2.amazonaws.com:443",
},
{
name: "aws eks cluster with unmatched assume role",
args: args{
cluster: awsKube,
client: getAWSClientRestConfig(cloudclients, fakeClock),
client: getAWSClientRestConfig(cloudclients, fakeClock, []services.ResourceMatcher{
{
Labels: types.Labels{
"rand": []string{"value"},
},
AWS: services.ResourceMatcherAWS{
AssumeRoleARN: "arn:aws:iam::123456789012:role/eks-role",
ExternalID: "1234567890",
},
},
}),
validateBearerToken: validateEKSToken,
},
wantAddr: "api.eks.us-west-2.amazonaws.com:443",
},
{
name: "aws eks cluster with assume role",
args: args{
cluster: awsKube,
client: getAWSClientRestConfig(
cloudclients,
fakeClock,
[]services.ResourceMatcher{
{
Labels: types.Labels{
types.Wildcard: []string{types.Wildcard},
},
AWS: services.ResourceMatcherAWS{
AssumeRoleARN: "arn:aws:iam::123456789012:role/eks-role",
ExternalID: "1234567890",
},
},
},
),
validateBearerToken: func(token string) error {
if token == "" {
return trace.BadParameter("missing bearer token")
Expand All @@ -201,7 +267,9 @@ func Test_DynamicKubeCreds(t *testing.T) {
return nil
},
},
wantAddr: "api.eks.us-west-2.amazonaws.com:443",
wantAddr: "api.eks.us-west-2.amazonaws.com:443",
wantAssumedRole: []string{"arn:aws:iam::123456789012:role/eks-role"},
wantExternalIds: []string{"1234567890"},
},
{
name: "gcp gke cluster",
Expand Down Expand Up @@ -261,6 +329,10 @@ func Test_DynamicKubeCreds(t *testing.T) {
}
}
require.NoError(t, got.close())

require.Equal(t, tt.wantAssumedRole, utils.Deduplicate(sts.GetAssumedRoleARNs()))
require.Equal(t, tt.wantExternalIds, utils.Deduplicate(sts.GetAssumedRoleExternalIDs()))
sts.ResetAssumeRoleHistory()
})
}
}
22 changes: 14 additions & 8 deletions lib/kube/proxy/watcher.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -181,10 +181,13 @@ func (m *monitoredKubeClusters) get() types.ResourcesWithLabelsMap {
func (s *TLSServer) registerKubeCluster(ctx context.Context, cluster types.KubeCluster) error {
clusterDetails, err := newClusterDetails(
ctx,
s.CloudClients,
cluster,
s.log,
s.CheckImpersonationPermissions,
clusterDetailsConfig{
cloudClients: s.CloudClients,
cluster: cluster,
log: s.log,
checker: s.CheckImpersonationPermissions,
resourceMatchers: s.ResourceMatchers,
},
)
if err != nil {
return trace.Wrap(err)
Expand All @@ -196,10 +199,13 @@ func (s *TLSServer) registerKubeCluster(ctx context.Context, cluster types.KubeC
func (s *TLSServer) updateKubeCluster(ctx context.Context, cluster types.KubeCluster) error {
clusterDetails, err := newClusterDetails(
ctx,
s.CloudClients,
cluster,
s.log,
s.CheckImpersonationPermissions,
clusterDetailsConfig{
cloudClients: s.CloudClients,
cluster: cluster,
log: s.log,
checker: s.CheckImpersonationPermissions,
resourceMatchers: s.ResourceMatchers,
},
)
if err != nil {
return trace.Wrap(err)
Expand Down

0 comments on commit 73cfb29

Please sign in to comment.