Move cluster credentials to a dedicated namespace

charts/fleet-agent/templates/configmap.yaml

@@ -5,6 +5,8 @@ metadata:
 data:
   config: |-
     {
-      "labels":{{toJson .Values.labels}}
+      {{ if .Values.labels }}
+      "labels":{{toJson .Values.labels}},
+      {{ end }}
       "clientID":"{{.Values.clientID}}"
     }

charts/fleet-agent/templates/secret.yaml

@@ -1,5 +1,6 @@
 apiVersion: v1
 data:
+  systemRegistrationNamespace: "{{b64enc .Values.systemRegistrationNamespace}}"
   clusterNamespace: "{{b64enc .Values.clusterNamespace}}"
   token: "{{b64enc .Values.token}}"
   apiServerURL: "{{b64enc .Values.apiServerURL}}"

charts/fleet-agent/values.yaml

@@ -16,13 +16,18 @@ token: ""
 # Labels to add to the cluster upon registration only. They are not added after the fact.
 #labels:
 #  foo: bar
+
 # The client ID of the cluster to associate with
 clientID: ""
 
+# The namespace of the cluster we are register with
+clusterNamespace: ""
+
+# The namespace containing the clusters registration secrets
+systemRegistrationNamespace: fleet-clusters-system
+
 # Please do not change the below setting unless you really know what you are doing
 internal:
   systemNamespace: fleet-system
   managedReleaseName: fleet-agent
 
-# The namespace of the cluster we are register with
-clusterNamespace: ""

modules/agent/pkg/register/register.go

@@ -160,20 +160,21 @@ func createClusterSecret(ctx context.Context, clusterID string, k8s corecontroll
 	}
 
 	secretName := registration.SecretName(request.Spec.ClientID, request.Spec.ClientRandom)
+	secretNamespace := string(secret.Data["systemRegistrationNamespace"])
 	timeout := time.After(30 * time.Minute)
 
 	for {
 		select {
 		case <-timeout:
-			return nil, fmt.Errorf("timeout waiting for secret %s/%s", ns, secretName)
+			return nil, fmt.Errorf("timeout waiting for secret %s/%s", secretNamespace, secretName)
 		case <-ctx.Done():
 			return nil, ctx.Err()
 		case <-time.After(2 * time.Second):
 		}
 
-		newSecret, err := fleetK8s.CoreV1().Secrets(ns).Get(ctx, secretName, metav1.GetOptions{})
+		newSecret, err := fleetK8s.CoreV1().Secrets(secretNamespace).Get(ctx, secretName, metav1.GetOptions{})
 		if err != nil {
-			logrus.Infof("Waiting for secret %s/%s for %s: %v", ns, secretName, request.Name, err)
+			logrus.Infof("Waiting for secret %s/%s for %s: %v", secretNamespace, secretName, request.Name, err)
 			continue
 		}
 

pkg/controllers/cluster/import.go

@@ -52,6 +52,7 @@ func RegisterImport(
 	}
 
 	clusters.OnChange(ctx, "import-cluster", h.OnChange)
+	fleetcontrollers.RegisterClusterStatusHandler(ctx, clusters, "Imported", "import-cluster", h.importCluster)
 }
 
 func (i *importHandler) OnChange(key string, cluster *fleet.Cluster) (_ *fleet.Cluster, err error) {
@@ -73,9 +74,19 @@ func (i *importHandler) OnChange(key string, cluster *fleet.Cluster) (_ *fleet.C
 		return i.clusters.Update(cluster)
 	}
 
+	return cluster, nil
+}
+
+func (i *importHandler) importCluster(cluster *fleet.Cluster, status fleet.ClusterStatus) (_ fleet.ClusterStatus, err error) {
+	if cluster.Spec.KubeConfigSecret == "" ||
+		(cluster.Status.AgentDeployed != nil && *cluster.Status.AgentDeployed) ||
+		cluster.Spec.ClientID == "" {
+		return status, nil
+	}
+
 	secret, err := i.secrets.Get(cluster.Namespace, cluster.Spec.KubeConfigSecret)
 	if err != nil {
-		return nil, err
+		return status, err
 	}
 
 	var (
@@ -87,7 +98,7 @@ func (i *importHandler) OnChange(key string, cluster *fleet.Cluster) (_ *fleet.C
 
 	if apiServerURL == "" {
 		if len(cfg.APIServerURL) == 0 {
-			return nil, fmt.Errorf("missing apiServerURL in fleet config for cluster auto registration")
+			return status, fmt.Errorf("missing apiServerURL in fleet config for cluster auto registration")
 		}
 		apiServerURL = cfg.APIServerURL
 	}
@@ -98,21 +109,21 @@ func (i *importHandler) OnChange(key string, cluster *fleet.Cluster) (_ *fleet.C
 
 	restConfig, err := clientcmd.RESTConfigFromKubeConfig(secret.Data["value"])
 	if err != nil {
-		return nil, err
+		return status, err
 	}
 
 	kc, err := kubernetes.NewForConfig(restConfig)
 	if err != nil {
-		return nil, err
+		return status, err
 	}
 
 	if _, err = kc.Discovery().ServerVersion(); err != nil {
-		return nil, err
+		return status, err
 	}
 
 	apply, err := apply.NewForConfig(restConfig)
 	if err != nil {
-		return nil, err
+		return status, err
 	}
 	apply = apply.WithDynamicLookup().WithSetID("fleet-agent-bootstrap")
 
@@ -128,7 +139,7 @@ func (i *importHandler) OnChange(key string, cluster *fleet.Cluster) (_ *fleet.C
 				TTLSeconds: ImportTokenTTL,
 			},
 		})
-		return nil, err
+		return status, err
 	}
 
 	output := &bytes.Buffer{}
@@ -139,19 +150,18 @@ func (i *importHandler) OnChange(key string, cluster *fleet.Cluster) (_ *fleet.C
 		NoCheck:  noCheck,
 	})
 	if err != nil {
-		return nil, err
+		return status, err
 	}
 
 	obj, err := yaml.ToObjects(output)
 	if err != nil {
-		return nil, err
+		return status, err
 	}
 
 	if err := apply.ApplyObjects(obj...); err != nil {
-		return nil, err
+		return status, err
 	}
 
-	cluster = cluster.DeepCopy()
-	cluster.Status.AgentDeployed = &t
-	return i.clusters.UpdateStatus(cluster)
+	status.AgentDeployed = &t
+	return status, nil
 }

pkg/controllers/clusterregistration/controller.go

@@ -7,12 +7,11 @@ import (
 	"fmt"
 	"time"
 
-	"github.com/rancher/fleet/pkg/registration"
-
 	fleetgroup "github.com/rancher/fleet/pkg/apis/fleet.cattle.io"
 	fleet "github.com/rancher/fleet/pkg/apis/fleet.cattle.io/v1alpha1"
 	"github.com/rancher/fleet/pkg/config"
 	fleetcontrollers "github.com/rancher/fleet/pkg/generated/controllers/fleet.cattle.io/v1alpha1"
+	"github.com/rancher/fleet/pkg/registration"
 	"github.com/rancher/wrangler/pkg/apply"
 	corecontrollers "github.com/rancher/wrangler/pkg/generated/controllers/core/v1"
 	rbaccontrollers "github.com/rancher/wrangler/pkg/generated/controllers/rbac/v1"
@@ -32,18 +31,20 @@ const (
 )
 
 type handler struct {
-	systemNamespace     string
-	clusterRegistration fleetcontrollers.ClusterRegistrationController
-	clusterCache        fleetcontrollers.ClusterCache
-	clusters            fleetcontrollers.ClusterClient
-	serviceAccountCache corecontrollers.ServiceAccountCache
-	secretsCache        corecontrollers.SecretCache
-	secrets             corecontrollers.SecretClient
+	systemNamespace             string
+	systemRegistrationNamespace string
+	clusterRegistration         fleetcontrollers.ClusterRegistrationController
+	clusterCache                fleetcontrollers.ClusterCache
+	clusters                    fleetcontrollers.ClusterClient
+	serviceAccountCache         corecontrollers.ServiceAccountCache
+	secretsCache                corecontrollers.SecretCache
+	secrets                     corecontrollers.SecretClient
 }
 
 func Register(ctx context.Context,
 	apply apply.Apply,
 	systemNamespace string,
+	systemRegistrationNamespace string,
 	serviceAccount corecontrollers.ServiceAccountController,
 	secret corecontrollers.SecretController,
 	role rbaccontrollers.RoleController,
@@ -54,13 +55,14 @@ func Register(ctx context.Context,
 	clusterCache fleetcontrollers.ClusterCache,
 	clusters fleetcontrollers.ClusterClient) {
 	h := &handler{
-		systemNamespace:     systemNamespace,
-		clusterRegistration: clusterRegistration,
-		clusterCache:        clusterCache,
-		clusters:            clusters,
-		serviceAccountCache: serviceAccount.Cache(),
-		secrets:             secret,
-		secretsCache:        secret.Cache(),
+		systemNamespace:             systemNamespace,
+		systemRegistrationNamespace: systemRegistrationNamespace,
+		clusterRegistration:         clusterRegistration,
+		clusterCache:                clusterCache,
+		clusters:                    clusters,
+		serviceAccountCache:         serviceAccount.Cache(),
+		secrets:                     secret,
+		secretsCache:                secret.Cache(),
 	}
 
 	fleetcontrollers.RegisterClusterRegistrationGeneratingHandler(ctx,
@@ -102,7 +104,7 @@ func (h *handler) authorizeCluster(sa *v1.ServiceAccount, cluster *fleet.Cluster
 	return &v1.Secret{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      registration.SecretName(req.Spec.ClientID, req.Spec.ClientRandom),
-			Namespace: cluster.Namespace,
+			Namespace: h.systemRegistrationNamespace,
 			Labels: map[string]string{
 				fleet.ClusterAnnotation: cluster.Name,
 			},
@@ -134,6 +136,11 @@ func (h *handler) OnChange(request *fleet.ClusterRegistration, status fleet.Clus
 		objects []runtime.Object
 	)
 
+	if status.Granted {
+		// only create the cluster for the request once
+		return nil, status, generic.ErrSkip
+	}
+
 	cluster, err := h.createOrGetCluster(request)
 	if err != nil || cluster == nil {
 		return nil, status, err

pkg/controllers/clusterregistrationtoken/handler.go

@@ -23,24 +23,27 @@ import (
 )
 
 type handler struct {
-	systemNamespace           string
-	clusterRegistrationTokens fleetcontrollers.ClusterRegistrationTokenClient
-	serviceAccountCache       corecontrollers.ServiceAccountCache
-	secretsCache              corecontrollers.SecretCache
+	systemNamespace             string
+	systemRegistrationNamespace string
+	clusterRegistrationTokens   fleetcontrollers.ClusterRegistrationTokenClient
+	serviceAccountCache         corecontrollers.ServiceAccountCache
+	secretsCache                corecontrollers.SecretCache
 }
 
 func Register(ctx context.Context,
 	systemNamespace string,
+	systemRegistrationNamespace string,
 	apply apply.Apply,
 	clusterGroupToken fleetcontrollers.ClusterRegistrationTokenController,
 	serviceAccounts corecontrollers.ServiceAccountController,
 	secretsCache corecontrollers.SecretCache,
 ) {
 	h := &handler{
-		systemNamespace:           systemNamespace,
-		clusterRegistrationTokens: clusterGroupToken,
-		serviceAccountCache:       serviceAccounts.Cache(),
-		secretsCache:              secretsCache,
+		systemNamespace:             systemNamespace,
+		systemRegistrationNamespace: systemRegistrationNamespace,
+		clusterRegistrationTokens:   clusterGroupToken,
+		serviceAccountCache:         serviceAccounts.Cache(),
+		secretsCache:                secretsCache,
 	}
 
 	fleetcontrollers.RegisterClusterRegistrationTokenGeneratingHandler(ctx,
@@ -106,11 +109,6 @@ func (h *handler) OnChange(token *fleet.ClusterRegistrationToken, status fleet.C
 					APIGroups: []string{fleetgroup.GroupName},
 					Resources: []string{fleet.ClusterRegistrationResourceName},
 				},
-				{
-					Verbs:     []string{"get"},
-					APIGroups: []string{""},
-					Resources: []string{"secrets"},
-				},
 			},
 		},
 		&rbacv1.RoleBinding{
@@ -148,10 +146,11 @@ func (h *handler) getValuesYAMLSecret(token *fleet.ClusterRegistrationToken, sec
 	}
 
 	values := map[string]interface{}{
-		"clusterNamespace": token.Namespace,
-		"apiServerURL":     config.Get().APIServerURL,
-		"apiServerCA":      string(config.Get().APIServerCA),
-		"token":            string(secret.Data["token"]),
+		"clusterNamespace":            token.Namespace,
+		"apiServerURL":                config.Get().APIServerURL,
+		"apiServerCA":                 string(config.Get().APIServerCA),
+		"token":                       string(secret.Data["token"]),
+		"systemRegistrationNamespace": h.systemRegistrationNamespace,
 	}
 
 	if h.systemNamespace != config.DefaultNamespace {

pkg/controllers/controllers.go

@@ -2,7 +2,7 @@ package controllers
 
 import (
 	"context"
-	"fmt"
+	"strings"
 
 	"github.com/rancher/fleet/pkg/controllers/bootstrap"
 	"github.com/rancher/fleet/pkg/controllers/bundle"
@@ -32,9 +32,6 @@ import (
 	"github.com/rancher/wrangler/pkg/ratelimit"
 	"github.com/rancher/wrangler/pkg/start"
 	"github.com/sirupsen/logrus"
-	corev1 "k8s.io/api/core/v1"
-	apierrors "k8s.io/apimachinery/pkg/api/errors"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/tools/clientcmd"
 )
@@ -57,23 +54,24 @@ func (a *appContext) start(ctx context.Context) error {
 	return start.All(ctx, 50, a.starters...)
 }
 
+func registrationNamespace(systemNamespace string) string {
+	systemRegistrationNamespace := strings.ReplaceAll(systemNamespace, "-system", "-clusters-system")
+	if systemRegistrationNamespace == systemNamespace {
+		return systemNamespace + "-clusters-system"
+	}
+	return systemRegistrationNamespace
+}
+
 func Register(ctx context.Context, systemNamespace string, cfg clientcmd.ClientConfig) error {
 	appCtx, err := newContext(cfg)
 	if err != nil {
 		return err
 	}
 
-	if _, err := appCtx.K8s.CoreV1().Namespaces().Get(ctx, systemNamespace, metav1.GetOptions{}); apierrors.IsNotFound(err) {
-		_, err := appCtx.K8s.CoreV1().Namespaces().Create(ctx, &corev1.Namespace{
-			ObjectMeta: metav1.ObjectMeta{
-				Name: systemNamespace,
-			},
-		}, metav1.CreateOptions{})
-		if err != nil {
-			return fmt.Errorf("failed to create namespace %s: %w", systemNamespace, err)
-		}
-	} else if err != nil {
-		return fmt.Errorf("failed to get namespace %s: %w", systemNamespace, err)
+	systemRegistrationNamespace := registrationNamespace(systemNamespace)
+
+	if err := addData(systemNamespace, systemRegistrationNamespace, appCtx); err != nil {
+		return err
 	}
 
 	// config should be registered first to ensure the global
@@ -95,6 +93,7 @@ func Register(ctx context.Context, systemNamespace string, cfg clientcmd.ClientC
 			appCtx.ClusterRegistration(),
 			appCtx.Cluster()),
 		systemNamespace,
+		systemRegistrationNamespace,
 		appCtx.Core.ServiceAccount(),
 		appCtx.Core.Secret(),
 		appCtx.RBAC.Role(),
@@ -132,6 +131,7 @@ func Register(ctx context.Context, systemNamespace string, cfg clientcmd.ClientC
 
 	clusterregistrationtoken.Register(ctx,
 		systemNamespace,
+		systemRegistrationNamespace,
 		appCtx.Apply.WithCacheTypes(
 			appCtx.Core.Secret(),
 			appCtx.Core.ServiceAccount(),