feat: fleet agent deployment configures tolerations from cluster CR

charts/fleet-crd/templates/crds.yaml

@@ -1779,6 +1779,27 @@ spec:
               agentNamespace:
                 nullable: true
                 type: string
+              agentTolerations:
+                items:
+                  properties:
+                    effect:
+                      nullable: true
+                      type: string
+                    key:
+                      nullable: true
+                      type: string
+                    operator:
+                      nullable: true
+                      type: string
+                    tolerationSeconds:
+                      nullable: true
+                      type: integer
+                    value:
+                      nullable: true
+                      type: string
+                  type: object
+                nullable: true
+                type: array
               clientID:
                 nullable: true
                 type: string

pkg/agent/manifest.go

@@ -31,6 +31,7 @@ type ManifestOptions struct {
 	AgentEnvVars          []corev1.EnvVar
 	AgentImage            string
 	AgentImagePullPolicy  string
+	AgentTolerations      []corev1.Toleration
 	CheckinInterval       string
 	Generation            string
 	PrivateRepoURL        string
@@ -93,6 +94,10 @@ func Manifest(namespace string, agentScope string, opts ManifestOptions) []runti
 	propagateDebug, _ := strconv.ParseBool(os.Getenv("FLEET_PROPAGATE_DEBUG_SETTINGS_TO_AGENTS"))
 	debug := logrus.IsLevelEnabled(logrus.DebugLevel) && propagateDebug
 	dep := agentDeployment(namespace, DefaultName, image, opts.AgentImagePullPolicy, DefaultName, false, debug)
+
+	// additional tolerations
+	dep.Spec.Template.Spec.Tolerations = append(dep.Spec.Template.Spec.Tolerations, opts.AgentTolerations...)
+
 	dep.Spec.Template.Spec.Containers[0].Env = append(dep.Spec.Template.Spec.Containers[0].Env,
 		corev1.EnvVar{
 			Name:  "AGENT_SCOPE",
@@ -242,6 +247,7 @@ func agentDeployment(namespace, name, image, imagePullPolicy, serviceAccount str
 		Value:    "linux",
 		Effect:   corev1.TaintEffectNoSchedule,
 	})
+
 	return deployment
 }
 

pkg/agent/manifest_test.go

@@ -1,6 +1,14 @@
 package agent
 
-import "testing"
+import (
+	"testing"
+
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+)
 
 func TestImageResolve(t *testing.T) {
 	tests := []struct {
@@ -21,3 +29,82 @@ func TestImageResolve(t *testing.T) {
 		}
 	}
 }
+
+func getDeploymentFromManifests(namespace string, scope string, opts ManifestOptions) *appsv1.Deployment {
+	objects := Manifest(namespace, scope, opts)
+	for _, obj := range objects {
+		dep, ok := obj.(*appsv1.Deployment)
+		if ok {
+			return dep
+		}
+	}
+	return nil
+}
+
+func TestManifestAdditionalTolerations(t *testing.T) {
+	const namespace = "fleet-system"
+	const scope = "test-scope"
+	baseOpts := ManifestOptions{
+		AgentEnvVars:          []corev1.EnvVar{},
+		AgentImage:            "rancher/fleet:1.2.3",
+		AgentImagePullPolicy:  "Always",
+		AgentTolerations:      []corev1.Toleration{},
+		CheckinInterval:       "1s",
+		Generation:            "100",
+		PrivateRepoURL:        "private.rancher.com:5000",
+		SystemDefaultRegistry: "default.rancher.com",
+	}
+
+	// these tolerations should exist regardless of what user sent
+	baseTolerations := []corev1.Toleration{
+		{Key: "cattle.io/os", Operator: "Equal", Value: "linux", Effect: "NoSchedule"},
+		{Key: "node.cloudprovider.kubernetes.io/uninitialized", Operator: "Equal", Value: "true", Effect: "NoSchedule"},
+	}
+
+	less := func(a, b corev1.Toleration) bool { return a.Key < b.Key }
+	cmpOpt := cmpopts.SortSlices(less)
+
+	for _, testCase := range []struct {
+		name                string
+		getOpts             func() ManifestOptions
+		expectedTolerations []corev1.Toleration
+	}{
+		{
+			name: "BaseOpts",
+			getOpts: func() ManifestOptions {
+				return baseOpts
+			},
+			expectedTolerations: baseTolerations,
+		},
+		{
+			name: "ExtraToleration",
+			getOpts: func() ManifestOptions {
+				withTolerationsOpts := baseOpts
+				withTolerationsOpts.AgentTolerations = []corev1.Toleration{
+					{
+						Key:      "fleet-agent",
+						Operator: "Equals",
+						Value:    "true",
+						Effect:   "NoSchedule",
+					},
+				}
+				return withTolerationsOpts
+			},
+			expectedTolerations: append(baseTolerations,
+				corev1.Toleration{Key: "fleet-agent", Operator: "Equals", Value: "true", Effect: "NoSchedule"},
+			),
+		},
+	} {
+		t.Run(testCase.name, func(t *testing.T) {
+
+			agentDeployment := getDeploymentFromManifests(namespace, scope, testCase.getOpts())
+			if agentDeployment == nil {
+				t.Fatal("there were no deployments returned from the manifests")
+			}
+
+			if !cmp.Equal(agentDeployment.Spec.Template.Spec.Tolerations, testCase.expectedTolerations, cmpOpt) {
+				t.Fatalf("tolerations were not as expected: %v", agentDeployment.Spec.Template.Spec.Tolerations)
+			}
+		})
+	}
+}

pkg/apis/fleet.cattle.io/v1alpha1/target.go

@@ -73,6 +73,9 @@ type ClusterSpec struct {
 
 	// TemplateValues defines a cluster specific mapping of values to be sent to fleet.yaml values templating
 	TemplateValues *GenericMap `json:"templateValues,omitempty"`
+
+	// AgentTolerations defines an extra set of Tolerations to be added to the Agent deployment
+	AgentTolerations []v1.Toleration `json:"agentTolerations,omitempty"`
 }
 
 type ClusterStatus struct {

pkg/controllers/cluster/import.go

@@ -265,10 +265,11 @@ func (i *importHandler) importCluster(cluster *fleet.Cluster, status fleet.Clust
 				Labels:   cluster.Labels,
 			},
 			ManifestOptions: agent.ManifestOptions{
-				AgentEnvVars:    cluster.Spec.AgentEnvVars,
-				CheckinInterval: cfg.AgentCheckinInterval.Duration.String(),
-				Generation:      string(cluster.UID) + "-" + strconv.FormatInt(cluster.Generation, 10),
-				PrivateRepoURL:  cluster.Spec.PrivateRepoURL,
+				AgentEnvVars:     cluster.Spec.AgentEnvVars,
+				AgentTolerations: cluster.Spec.AgentTolerations,
+				CheckinInterval:  cfg.AgentCheckinInterval.Duration.String(),
+				Generation:       string(cluster.UID) + "-" + strconv.FormatInt(cluster.Generation, 10),
+				PrivateRepoURL:   cluster.Spec.PrivateRepoURL,
 			},
 		})
 	if err != nil {

pkg/controllers/manageagent/manageagent.go

@@ -182,6 +182,7 @@ func (h *handler) newAgentBundle(ns string, cluster *fleet.Cluster) (runtime.Obj
 			AgentEnvVars:          cluster.Spec.AgentEnvVars,
 			AgentImage:            cfg.AgentImage,
 			AgentImagePullPolicy:  cfg.AgentImagePullPolicy,
+			AgentTolerations:      cluster.Spec.AgentTolerations,
 			CheckinInterval:       cfg.AgentCheckinInterval.Duration.String(),
 			Generation:            "bundle",
 			PrivateRepoURL:        cluster.Spec.PrivateRepoURL,