Merge pull request rancher#112 from cmurphy/unique-cattle-token

Ensure cattle token secret has unique name

pkg/agent/cluster/cluster.go

@@ -74,9 +74,9 @@ func getTokenFromAPI() ([]byte, []byte, error) {
 		}
 		return secret.Data[coreV1.ServiceAccountRootCAKey], secret.Data[coreV1.ServiceAccountTokenKey], nil
 	}
-	secret, err := serviceaccounttoken.CreateSecretForServiceAccount(context.Background(), k8s, sa)
+	secret, err := serviceaccounttoken.EnsureSecretForServiceAccount(context.Background(), nil, k8s, sa)
 	if err != nil {
-		return nil, nil, fmt.Errorf("failed to create secret for service account %s/%s: %w", namespace.System, "cattle", err)
+		return nil, nil, fmt.Errorf("failed to ensure secret for service account %s/%s: %w", namespace.System, "cattle", err)
 	}
 	return []byte(cm.Data["ca.crt"]), []byte(secret.Data[coreV1.ServiceAccountTokenKey]), nil
 }

pkg/api/norman/customization/monitor/handler_utils.go

@@ -187,9 +187,9 @@ func getAuthToken(userContext *config.UserContext, appName, namespace string) (s
 		return "", fmt.Errorf("get service account %s:%s for monitor failed, %v", namespace, appName, err)
 	}
 
-	secret, err := serviceaccounttoken.CreateSecretForServiceAccount(context.TODO(), userContext.K8sClient, sa)
+	secret, err := serviceaccounttoken.EnsureSecretForServiceAccount(context.Background(), nil, userContext.K8sClient, sa)
 	if err != nil {
-		return "", fmt.Errorf("get secret from service account %s:%s for monitor failed: %v", namespace, appName, err)
+		return "", fmt.Errorf("ensure secret from service account %s:%s for monitor failed: %w", namespace, appName, err)
 	}
 
 	return string(secret.Data[v1.ServiceAccountTokenKey]), nil

pkg/controllers/dashboard/apiservice/apiservice.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"crypto/sha256"
 	"encoding/base64"
+	"fmt"
 
 	v3 "github.com/rancher/rancher/pkg/apis/management.cattle.io/v3"
 	mgmtcontrollers "github.com/rancher/rancher/pkg/generated/controllers/management.cattle.io/v3"
@@ -16,10 +17,12 @@ import (
 	"github.com/rancher/wrangler/pkg/name"
 	"github.com/rancher/wrangler/pkg/relatedresource"
 	corev1 "k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 	apierror "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/kubernetes"
 )
 
 type handler struct {
@@ -32,6 +35,7 @@ type handler struct {
 	apiServices     mgmtcontrollers.APIServiceCache
 	services        corev1controllers.ServiceCache
 	embedded        bool
+	k8s             kubernetes.Interface
 	ctx             context.Context
 }
 
@@ -46,6 +50,7 @@ func Register(ctx context.Context, context *wrangler.Context, embedded bool) {
 		apiServices:     context.Mgmt.APIService().Cache(),
 		services:        context.Core.Service().Cache(),
 		embedded:        embedded,
+		k8s:             context.K8s,
 		ctx:             ctx,
 	}
 
@@ -142,35 +147,12 @@ func (h *handler) getToken(sa *corev1.ServiceAccount) (string, error) {
 		return "", err
 	}
 
-	// create a secret-based token for the service account
-	sName := serviceaccounttoken.ServiceAccountSecretName(sa)
-	secret, err := h.secretsCache.Get(sa.Namespace, sName)
+	// create a secret-based token for the service account if one does not exist
+	secret, err := serviceaccounttoken.EnsureSecretForServiceAccount(h.ctx, h.secretsCache.Get, h.k8s, sa)
 	if err != nil {
-		if !apierror.IsNotFound(err) {
-			return "", err
-		}
-		sc := serviceaccounttoken.SecretTemplate(sa)
-		secret, err = h.secretsClient.Create(sc)
-		if err != nil {
-			if !apierror.IsAlreadyExists(err) {
-				return "", err
-			}
-			secret, err = h.secretsCache.Get(sa.Namespace, sName)
-			if err != nil {
-				if !apierror.IsNotFound(err) {
-					return "", err
-				}
-				secret, err = h.secretsClient.Get(sa.Namespace, sName, metav1.GetOptions{})
-				if err != nil {
-					return "", err
-				}
-			}
-		}
-	}
-	token := secret.Data["token"]
-	if len(token) == 0 {
-		return "", nil
+		return "", fmt.Errorf("error ensuring secret for service account [%s:%s]: %w", sa.Namespace, sa.Name, err)
 	}
+	token := secret.Data[v1.ServiceAccountTokenKey]
 
 	hash := sha256.Sum256(token)
 	return base64.StdEncoding.EncodeToString(hash[:]), nil

pkg/controllers/provisioningv2/rke2/bootstrap/controller.go

@@ -25,6 +25,7 @@ import (
 	apierror "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/kubernetes"
 	capi "sigs.k8s.io/cluster-api/api/v1beta1"
 )
 
@@ -41,6 +42,7 @@ type handler struct {
 	deploymentCache     appcontrollers.DeploymentCache
 	rkeControlPlanes    rkecontroller.RKEControlPlaneCache
 	rkeBootstrapClient  rkecontroller.RKEBootstrapClient
+	k8s                 kubernetes.Interface
 }
 
 func Register(ctx context.Context, clients *wrangler.Context) {
@@ -53,6 +55,7 @@ func Register(ctx context.Context, clients *wrangler.Context) {
 		deploymentCache:     clients.Apps.Deployment().Cache(),
 		rkeControlPlanes:    clients.RKE.RKEControlPlane().Cache(),
 		rkeBootstrapClient:  clients.RKE.RKEBootstrap(),
+		k8s:                 clients.K8s,
 	}
 	rkecontroller.RegisterRKEBootstrapGeneratingHandler(ctx,
 		clients.RKE.RKEBootstrap(),
@@ -102,23 +105,9 @@ func (h *handler) getBootstrapSecret(namespace, name string, envVars []corev1.En
 		return nil, err
 
 	}
-	sName := serviceaccounttoken.ServiceAccountSecretName(sa)
-	secret, err := h.secretCache.Get(sa.Namespace, sName)
+	secret, err := serviceaccounttoken.EnsureSecretForServiceAccount(context.Background(), h.secretCache.Get, h.k8s, sa)
 	if err != nil {
-		if !apierror.IsNotFound(err) {
-			return nil, err
-		}
-		sc := serviceaccounttoken.SecretTemplate(sa)
-		secret, err = h.secretClient.Create(sc)
-		if err != nil {
-			if !apierror.IsAlreadyExists(err) {
-				return nil, err
-			}
-			secret, err = h.secretClient.Get(sa.Namespace, sName, metav1.GetOptions{})
-			if err != nil {
-				return nil, err
-			}
-		}
+		return nil, err
 	}
 	hash := sha256.Sum256(secret.Data["token"])
 

pkg/controllers/provisioningv2/rke2/bootstrap/controller_test.go

@@ -19,6 +19,7 @@ import (
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/client-go/kubernetes/fake"
 	"sigs.k8s.io/cluster-api/api/v1beta1"
 )
 
@@ -135,6 +136,7 @@ func Test_getBootstrapSecret(t *testing.T) {
 				secretCache:         getSecretCacheMock(tt.args.namespaceName, tt.args.secretName),
 				deploymentCache:     getDeploymentCacheMock(),
 				machineCache:        getMachineCacheMock(tt.args.namespaceName, tt.args.os),
+				k8s:                 fake.NewSimpleClientset(),
 			}
 
 			//act
@@ -248,7 +250,7 @@ func getDeploymentCacheMock() *deploymentCacheMock {
 
 func getSecretCacheMock(namespace, secretName string) *secretCacheMock {
 	mockSecretCache := new(secretCacheMock)
-	mockSecretCache.On("Get", namespace, secretName+"-token").Return(&v1.Secret{
+	mockSecretCache.On("Get", namespace, secretName).Return(&v1.Secret{
 		TypeMeta: metav1.TypeMeta{},
 		ObjectMeta: metav1.ObjectMeta{
 			Name: secretName,

pkg/controllers/provisioningv2/rke2/common.go

@@ -22,9 +22,9 @@ import (
 	"github.com/rancher/wrangler/pkg/generic"
 	"github.com/rancher/wrangler/pkg/name"
 	corev1 "k8s.io/api/core/v1"
-	apierror "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/client-go/kubernetes"
 	capi "sigs.k8s.io/cluster-api/api/v1beta1"
 	capierrors "sigs.k8s.io/cluster-api/errors"
 )
@@ -231,52 +231,17 @@ func GetPlanSecretName(planSA *corev1.ServiceAccount) (string, error) {
 
 // GetPlanServiceAccountTokenSecret retrieves the secret that corresponds to the plan service account that is passed in. It will create a secret if one does not
 // already exist for the plan service account.
-func GetPlanServiceAccountTokenSecret(secretClient corecontrollers.SecretController, planSA *corev1.ServiceAccount) (*corev1.Secret, bool, error) {
+func GetPlanServiceAccountTokenSecret(secretClient corecontrollers.SecretController, k8s kubernetes.Interface, planSA *corev1.ServiceAccount) (*corev1.Secret, bool, error) {
 	if planSA == nil {
 		return nil, false, fmt.Errorf("planSA was nil")
 	}
-	sName := serviceaccounttoken.ServiceAccountSecretName(planSA)
-	secret, err := secretClient.Cache().Get(planSA.Namespace, sName)
+	secret, err := serviceaccounttoken.EnsureSecretForServiceAccount(context.Background(), secretClient.Cache().Get, k8s, planSA)
 	if err != nil {
-		if !apierror.IsNotFound(err) {
-			return nil, false, err
-		}
-		sc := serviceaccounttoken.SecretTemplate(planSA)
-		secret, err = secretClient.Create(sc)
-		if err != nil {
-			if !apierror.IsAlreadyExists(err) {
-				return nil, false, err
-			}
-			secret, err = secretClient.Cache().Get(planSA.Namespace, sName)
-			if err != nil {
-				return nil, false, err
-			}
-		}
-	}
-	// wait for token to be populated
-	if !PlanServiceAccountTokenReady(planSA, secret) {
-		return secret, true, fmt.Errorf("planSA %s/%s token secret %s/%s was not ready for consumption yet", planSA.Namespace, planSA.Name, secret.Namespace, secret.Name)
+		return nil, false, fmt.Errorf("error ensuring secret for service account [%s:%s]: %w", planSA.Namespace, planSA.Name, err)
 	}
 	return secret, true, nil
 }
 
-func PlanServiceAccountTokenReady(planSA *corev1.ServiceAccount, tokenSecret *corev1.Secret) bool {
-	if planSA == nil || tokenSecret == nil {
-		return false
-	}
-	if tokenSecret.Name != serviceaccounttoken.ServiceAccountSecretName(planSA) {
-		return false
-	}
-	if v, ok := tokenSecret.Data[corev1.ServiceAccountTokenKey]; ok {
-		if len(v) == 0 {
-			return false
-		}
-	} else {
-		return false
-	}
-	return true
-}
-
 func PlanSecretFromBootstrapName(bootstrapName string) string {
 	return name.SafeConcatName(bootstrapName, "machine", "plan")
 }

pkg/impersonation/impersonation.go

@@ -2,6 +2,7 @@
 package impersonation
 
 import (
+	"context"
 	"fmt"
 	"reflect"
 	"sort"
@@ -100,28 +101,9 @@ func (i *Impersonator) SetUpImpersonation() (*corev1.ServiceAccount, error) {
 
 // GetToken accepts a service account and returns the service account's token.
 func (i *Impersonator) GetToken(sa *corev1.ServiceAccount) (string, error) {
-	name := serviceaccounttoken.ServiceAccountSecretName(sa)
-	secret, err := i.clusterContext.Core.Secrets(ImpersonationNamespace).Get(name, metav1.GetOptions{})
+	secret, err := serviceaccounttoken.EnsureSecretForServiceAccount(context.Background(), i.clusterContext.Core.Secrets("").Controller().Lister().Get, i.clusterContext.K8sClient, sa)
 	if err != nil {
-		if !apierrors.IsNotFound(err) {
-			logrus.Tracef("impersonation: using context for cluster %s", i.clusterContext.ClusterName)
-			return "", fmt.Errorf("error getting secret: %w", err)
-		}
-		// create the secret
-		sc := serviceaccounttoken.SecretTemplate(sa)
-		secret, err = i.clusterContext.Core.Secrets(ImpersonationNamespace).Create(sc)
-		if err != nil {
-			if !apierrors.IsAlreadyExists(err) {
-				logrus.Tracef("impersonation: using context for cluster %s", i.clusterContext.ClusterName)
-				return "", fmt.Errorf("error creating secret: %w", err)
-			}
-			secret, err = i.clusterContext.Core.Secrets(ImpersonationNamespace).Get(name, metav1.GetOptions{})
-			if err != nil {
-				logrus.Tracef("impersonation: using context for cluster %s", i.clusterContext.ClusterName)
-				logrus.Tracef("impersonation: secrret already exists")
-				return "", fmt.Errorf("error getting secret: %w", err)
-			}
-		}
+		return "", fmt.Errorf("error getting secret: %w", err)
 	}
 	token, ok := secret.Data["token"]
 	if !ok {
@@ -156,6 +138,9 @@ func (i *Impersonator) getServiceAccount() (*corev1.ServiceAccount, error) {
 func (i *Impersonator) createServiceAccount(role *rbacv1.ClusterRole) (*corev1.ServiceAccount, error) {
 	name := ImpersonationPrefix + i.user.GetUID()
 	sa, err := i.clusterContext.Core.ServiceAccounts("").Controller().Lister().Get(ImpersonationNamespace, name)
+	if err != nil && !apierrors.IsNotFound(err) {
+		return nil, fmt.Errorf("impersonation: error getting service account [%s:%s]: %w", ImpersonationNamespace, name, err)
+	}
 	if apierrors.IsNotFound(err) {
 		logrus.Debugf("impersonation: creating service account %s", name)
 		sa, err = i.clusterContext.Core.ServiceAccounts(ImpersonationNamespace).Create(&corev1.ServiceAccount{
@@ -177,16 +162,16 @@ func (i *Impersonator) createServiceAccount(role *rbacv1.ClusterRole) (*corev1.S
 			// in case cache isn't synced yet, use raw client
 			sa, err = i.clusterContext.Core.ServiceAccounts(ImpersonationNamespace).Get(name, metav1.GetOptions{})
 		}
-	}
-	if sa != nil {
-		// create secret for service account
-		sc := serviceaccounttoken.SecretTemplate(sa)
-		_, secretErr := i.clusterContext.Core.Secrets(ImpersonationNamespace).Create(sc)
-		if secretErr != nil && !apierrors.IsAlreadyExists(secretErr) {
-			return nil, fmt.Errorf("impersonation: error creating secret for service account %s", name)
+		if err != nil {
+			return nil, fmt.Errorf("impersonation: error getting service account [%s:%s]: %w", ImpersonationNamespace, name, err)
 		}
 	}
-	return sa, err
+	// create secret for service account if it was not automatically generated
+	_, err = serviceaccounttoken.EnsureSecretForServiceAccount(context.Background(), i.clusterContext.Core.Secrets("").Controller().Lister().Get, i.clusterContext.K8sClient, sa)
+	if err != nil {
+		return nil, fmt.Errorf("impersonation: error ensuring secret for service account %s: %w", name, err)
+	}
+	return sa, nil
 }
 
 func (i *Impersonator) createNamespace() error {

pkg/kontainer-engine/drivers/util/utils.go

@@ -3,9 +3,7 @@ package util
 import (
 	"context"
 	"fmt"
-	"time"
 
-	errs "github.com/pkg/errors"
 	"github.com/rancher/rancher/pkg/serviceaccounttoken"
 	v3 "github.com/rancher/rke/types"
 	"gopkg.in/yaml.v2"
@@ -94,23 +92,14 @@ func GenerateServiceAccountToken(clientset kubernetes.Interface) (string, error)
 		return "", fmt.Errorf("error creating role bindings: %v", err)
 	}
 
-	start := time.Millisecond * 250
-	for i := 0; i < 5; i++ {
-		time.Sleep(start)
-		if serviceAccount, err = clientset.CoreV1().ServiceAccounts(cattleNamespace).Get(context.TODO(), serviceAccount.Name, metav1.GetOptions{}); err != nil {
-			return "", fmt.Errorf("error getting service account: %v", err)
-		}
-		secret, err := serviceaccounttoken.CreateSecretForServiceAccount(context.TODO(), clientset, serviceAccount)
-		if err != nil {
-			return "", fmt.Errorf("error creating secret for service account: %v", err)
-		}
-		if token, ok := secret.Data["token"]; ok {
-			return string(token), nil
-		}
-		start = start * 2
+	if serviceAccount, err = clientset.CoreV1().ServiceAccounts(cattleNamespace).Get(context.Background(), serviceAccount.Name, metav1.GetOptions{}); err != nil {
+		return "", fmt.Errorf("error getting service account: %w", err)
 	}
-
-	return "", errs.New("failed to fetch serviceAccountToken")
+	secret, err := serviceaccounttoken.EnsureSecretForServiceAccount(context.Background(), nil, clientset, serviceAccount)
+	if err != nil {
+		return "", fmt.Errorf("error ensuring secret for service account: %w", err)
+	}
+	return string(secret.Data["token"]), nil
 }
 
 func DeleteLegacyServiceAccountAndRoleBinding(clientset kubernetes.Interface) error {