From 3875bd678b88ab623efd9b3dc29f056ebba7495b Mon Sep 17 00:00:00 2001 From: Alexey Preobrazhenskiy Date: Tue, 12 Sep 2023 14:11:09 +0200 Subject: [PATCH] Add support for setting tolerations for pods created by the daemonset --- README.md | 3 ++ cfg/config.go | 2 + cfg/config_test.go | 10 +++++ cfg/envvars.go | 11 +++++ cfg/envvars_test.go | 61 ++++++++++++++++++++++++++++ deploy/helm/templates/configmap.yaml | 1 + deploy/helm/values.yaml | 1 + deploy/openshift/configmap.yaml | 3 ++ utils/clusterutils.go | 1 + 9 files changed, 93 insertions(+) diff --git a/README.md b/README.md index 6641667..f14e45e 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,7 @@ The config values to be set are: | `IMAGE_PULL_SECRETS` | List of image pull secrets, in this format `pullsecret1;...` to add to pods created by the DaemonSet. Those secrets need to be in the image puller's namespace and a cluster administrator must create them. | `""` | | `AFFINITY` | Affinity applied to pods created by the daemonset, in this format `'{"nodeAffinity":{ ... }}'` | `'{}'` | | `KIP_IMAGE` | The image puller image to copy the `sleep` binary from | `quay.io/eclipse/kubernetes-image-puller:next` | +| `TOLERATIONS` | Tolerations applied to pods created by the daemonset, provided in this format `'[{"operator":"Exists"}]'` | `'[]'` | ### Configuration - Helm @@ -49,6 +50,7 @@ The following values can be set: | `configMap.nodeSelector` | The value of `NODE_SELECTOR` to be set in the ConfigMap | `"{}"` | | `configMap.imagePullSecrets` | The value of `IMAGE_PULL_SECRETS` | `""` | | `configMap.affinity` | The value of `AFFINITY` to be set in the ConfigMap | `"{}"` | +| `configMap.tolerations` | The value of `TOLERATIONS` to be set in the ConfigMap | `"[]"` | ### Configuration - OpenShift @@ -70,6 +72,7 @@ The following values can be set: | `NODE_SELECTOR` | The value of `NODE_SELECTOR` to be set in the ConfigMap | `"{}"` | | `IMAGE_PULL_SECRETS` | The value of `IMAGE_PULL_SECRETS` | `""` | | `AFFINITY` | The value of `AFFINITY` to be set in the ConfigMap | `"{}"` | +| `TOLERATIONS` | The value of `TOLERATIONS` to be set in the ConfigMap | `"[]"` | ### Installation - Helm diff --git a/cfg/config.go b/cfg/config.go index b1942d2..3981c56 100644 --- a/cfg/config.go +++ b/cfg/config.go @@ -27,6 +27,7 @@ type Config struct { ImagePullSecrets []string Affinity *corev1.Affinity ImagePullerImage string + Tolerations []corev1.Toleration } func GetConfig() Config { @@ -43,5 +44,6 @@ func GetConfig() Config { ImagePullSecrets: processImagePullSecretsEnvVar(), Affinity: processAffinityEnvVar(), ImagePullerImage: getEnvVarOrDefault(kipImageEnvVar, defaultImage), + Tolerations: processTolerationsEnvVar(), } } diff --git a/cfg/config_test.go b/cfg/config_test.go index cb2f8e7..758d1dc 100644 --- a/cfg/config_test.go +++ b/cfg/config_test.go @@ -40,6 +40,7 @@ func TestEnvVars(t *testing.T) { ImagePullSecrets: []string{}, Affinity: &v1.Affinity{}, ImagePullerImage: "quay.io/eclipse/kubernetes-image-puller:next", + Tolerations: []v1.Toleration{}, }, }, { @@ -52,6 +53,7 @@ func TestEnvVars(t *testing.T) { "IMAGE_PULL_SECRETS": "secret1; secret2", "AFFINITY": `{"nodeAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":{"nodeSelectorTerms":[{"matchExpressions":[{"key":"kubernetes.io/e2e-az-name","operator":"In","values":["e2e-az1","e2e-az2"]}]}]}}}`, "KIP_IMAGE": "quay.io/my-repo/kubernetes-image-puller:next", + "TOLERATIONS": `[{"effect":"NoSchedule","key":"app","operator":"Equal","value": "prod"}]`, }, want: Config{ DaemonsetName: "custom-daemonset-name", @@ -86,6 +88,14 @@ func TestEnvVars(t *testing.T) { }, }, ImagePullerImage: "quay.io/my-repo/kubernetes-image-puller:next", + Tolerations: []v1.Toleration{ + { + Key: "app", + Operator: "Equal", + Value: "prod", + Effect: "NoSchedule", + }, + }, }, }, } diff --git a/cfg/envvars.go b/cfg/envvars.go index f0d6d10..c878977 100644 --- a/cfg/envvars.go +++ b/cfg/envvars.go @@ -36,6 +36,7 @@ const ( imagePullSecretsEnvVar = "IMAGE_PULL_SECRETS" affinityEnvVar = "AFFINITY" kipImageEnvVar = "KIP_IMAGE" + tolerationsEnvVar = "TOLERATIONS" ) // Default values where applicable @@ -52,6 +53,7 @@ const ( defaultImagePullSecret = "" defaultAffinity = "{}" defaultImage = "quay.io/eclipse/kubernetes-image-puller:next" + defaultTolerations = "[]" ) func getCachingInterval() int { @@ -128,6 +130,15 @@ func processAffinityEnvVar() *corev1.Affinity { return affinity } +func processTolerationsEnvVar() []corev1.Toleration { + rawTolerations := getEnvVarOrDefault(tolerationsEnvVar, defaultTolerations) + var tolerations []corev1.Toleration + if err := json.Unmarshal([]byte(rawTolerations), &tolerations); err != nil { + log.Fatalf("Failed to unmarshal tolerations json: %s", err) + } + return tolerations +} + func getEnvVarOrExit(envVar string) string { val := os.Getenv(envVar) if val == "" { diff --git a/cfg/envvars_test.go b/cfg/envvars_test.go index 593244c..cf9ebc8 100644 --- a/cfg/envvars_test.go +++ b/cfg/envvars_test.go @@ -164,3 +164,64 @@ func Test_processAffinityEnvVar(t *testing.T) { }) } } + +func TestProcessTolerationsEnvVar(t *testing.T) { + type testcase struct { + name string + tolerations string + isTolerationsSet bool + want []v1.Toleration + } + + tests := []testcase{ + { + name: "default tolerations, TOLERATIONS set", + tolerations: "[]", + isTolerationsSet: true, + want: []v1.Toleration{}, + }, + { + name: "app prod, TOLERATIONS set", + tolerations: `[{"effect":"NoSchedule","key":"app","operator":"Equal","value": "prod"}]`, + isTolerationsSet: true, + want: []v1.Toleration{ + { + Key: "app", + Operator: "Equal", + Value: "prod", + Effect: "NoSchedule", + }, + }, + }, + { + name: "operator exists, TOLERATIONS set", + tolerations: `[{"operator":"Exists"}]`, + isTolerationsSet: true, + want: []v1.Toleration{ + { + Operator: "Exists", + }, + }, + }, + { + name: "default env var, TOLERATIONS not set", + tolerations: "[{\"this\": \"shouldn't be set\"}]", + isTolerationsSet: false, + want: []v1.Toleration{}, + }, + } + + for _, c := range tests { + t.Run(c.name, func(t *testing.T) { + defer os.Clearenv() + if c.isTolerationsSet { + os.Setenv("TOLERATIONS", c.tolerations) + } + got := processTolerationsEnvVar() + + if d := cmp.Diff(c.want, got); d != "" { + t.Errorf("(-want, +got): %s", d) + } + }) + } +} diff --git a/deploy/helm/templates/configmap.yaml b/deploy/helm/templates/configmap.yaml index 58f0ef7..90b1e0c 100644 --- a/deploy/helm/templates/configmap.yaml +++ b/deploy/helm/templates/configmap.yaml @@ -14,3 +14,4 @@ data: NODE_SELECTOR: "{{ .Values.configMap.nodeSelector }}" IMAGE_PULL_SECRETS: "{{ .Values.configMap.imagePullSecrets }}" AFFINITY: "{{ .Values.configMap.affinity }}" + TOLERATIONS: "{{ .Values.configMap.tolerations }}" diff --git a/deploy/helm/values.yaml b/deploy/helm/values.yaml index c6c908b..152a906 100644 --- a/deploy/helm/values.yaml +++ b/deploy/helm/values.yaml @@ -18,3 +18,4 @@ configMap: nodeSelector: "{}" imagePullSecrets: "" affinity: "{}" + tolerations: "[]" diff --git a/deploy/openshift/configmap.yaml b/deploy/openshift/configmap.yaml index f2834bb..439e33a 100644 --- a/deploy/openshift/configmap.yaml +++ b/deploy/openshift/configmap.yaml @@ -23,6 +23,7 @@ objects: IMAGE_PULL_SECRETS: ${IMAGE_PULL_SECRETS} AFFINITY: ${AFFINITY} KIP_IMAGE: ${KIP_IMAGE} + TOLERATIONS: ${TOLERATIONS} parameters: - name: IMAGES value: > @@ -51,3 +52,5 @@ parameters: value: "{}" - name: KIP_IMAGE value: quay.io/eclipse/kubernetes-image-puller:next +- name: TOLERATIONS + value: "[]" diff --git a/utils/clusterutils.go b/utils/clusterutils.go index 680fd4d..192bd8b 100644 --- a/utils/clusterutils.go +++ b/utils/clusterutils.go @@ -133,6 +133,7 @@ func getDaemonset(deployment *appsv1.Deployment) *appsv1.DaemonSet { ImagePullSecrets: imgPullSecrets, Affinity: cfg.Affinity, Volumes: []corev1.Volume{{Name: kipVolumeName}}, + Tolerations: cfg.Tolerations, }, }, },