From 652f92fbc8a3faf3e817f71e5f00eeb0c8adb561 Mon Sep 17 00:00:00 2001 From: tico88612 <17496418+tico88612@users.noreply.github.com> Date: Thu, 18 Apr 2024 20:25:50 +0800 Subject: [PATCH] feat: support toleration config Feat: KubeArmor chart values add kubearmor-controller toleration Signed-off-by: tico88612 <17496418+tico88612@users.noreply.github.com> Feat: KubeArmor chart values add kubearmor-relay toleration Signed-off-by: tico88612 <17496418+tico88612@users.noreply.github.com> Typo: KubeArmor chart values comment wrong fix Signed-off-by: tico88612 <17496418+tico88612@users.noreply.github.com> Feat: KubeArmorOperator chart values add deployment toleration Signed-off-by: tico88612 <17496418+tico88612@users.noreply.github.com> Chore: update KubeArmorOperator controller-gen from v0.4.1 to v0.14.0 Signed-off-by: tico88612 <17496418+tico88612@users.noreply.github.com> Feat: KubeArmorConfigSpec Relay & Controller add Toleration Signed-off-by: tico88612 <17496418+tico88612@users.noreply.github.com> Command: controller-gen update KubeArmorConfig CRD Signed-off-by: tico88612 <17496418+tico88612@users.noreply.github.com> Feat: add KubeArmorConfig toleration default value Signed-off-by: tico88612 <17496418+tico88612@users.noreply.github.com> Feat: implement tolerations in operator Signed-off-by: tico88612 <17496418+tico88612@users.noreply.github.com> --- .../helm/KubeArmor/templates/deployment.yaml | 8 ++ deployments/helm/KubeArmor/values.yaml | 10 ++- ...erator.kubearmor.com_kubearmorconfigs.yaml | 76 +++++++++++++++++++ .../templates/deployment.yaml | 4 + .../helm/KubeArmorOperator/values.yaml | 1 + deployments/operator/operator.yaml | 76 +++++++++++++++++++ pkg/KubeArmorOperator/Makefile | 2 +- .../v1/kubearmorconfig_types.go | 5 ++ pkg/KubeArmorOperator/common/defaults.go | 4 + ...erator.kubearmor.com_kubearmorconfigs.yaml | 76 +++++++++++++++++++ .../internal/controller/cluster.go | 65 ++++++++++++++++ 11 files changed, 323 insertions(+), 4 deletions(-) diff --git a/deployments/helm/KubeArmor/templates/deployment.yaml b/deployments/helm/KubeArmor/templates/deployment.yaml index b6ed02d5c1..052b0b76d6 100644 --- a/deployments/helm/KubeArmor/templates/deployment.yaml +++ b/deployments/helm/KubeArmor/templates/deployment.yaml @@ -39,6 +39,10 @@ spec: volumeMounts: {{- toYaml .Values.kubearmorRelay.tls.certVolumeMount | trim | nindent 10 }} {{- end}} + {{- if .Values.kubearmorRelay.tolerations }} + tolerations: + {{ toYaml .Values.kubearmorRelay.tolerations | indent 8 }} + {{- end }} nodeSelector: kubernetes.io/os: linux serviceAccountName: kubearmor-relay @@ -138,6 +142,10 @@ spec: runAsNonRoot: true serviceAccountName: {{ .Values.kubearmorController.name }} terminationGracePeriodSeconds: 10 + {{- if .Values.kubearmorController.tolerations }} + tolerations: + {{ toYaml .Values.kubearmorController.tolerations | indent 8 }} + {{- end }} volumes: - name: cert secret: diff --git a/deployments/helm/KubeArmor/values.yaml b/deployments/helm/KubeArmor/values.yaml index 3df4460f4f..b01c508c85 100644 --- a/deployments/helm/KubeArmor/values.yaml +++ b/deployments/helm/KubeArmor/values.yaml @@ -12,12 +12,14 @@ kubearmorRelay: # to enable/disable kubearmor-relay enabled: true image: - # kubearmor-init image repo + # kubearmor-relay image repo repository: kubearmor/kubearmor-relay-server - # kubearmor-init image tag + # kubearmor-relay image tag tag: latest - # kubearmor-init imagePullPolicy + # kubearmor-relay imagePullPolicy imagePullPolicy: Always + # kubearmor-relay tolerations + tolerations: [] # Add environment variables for STDOUT logging enableStdoutLogs: "false" enableStdoutAlerts: "false" @@ -88,6 +90,8 @@ kubearmorController: failurePolicy: Ignore # kubearmor-controller imagePullPolicy imagePullPolicy: Always + # kubearmor-controller tolerations + tolerations: [] kubearmorConfigMap: defaultFilePosture: audit diff --git a/deployments/helm/KubeArmorOperator/crds/operator.kubearmor.com_kubearmorconfigs.yaml b/deployments/helm/KubeArmorOperator/crds/operator.kubearmor.com_kubearmorconfigs.yaml index e179add2bb..523197cfc7 100644 --- a/deployments/helm/KubeArmorOperator/crds/operator.kubearmor.com_kubearmorconfigs.yaml +++ b/deployments/helm/KubeArmorOperator/crds/operator.kubearmor.com_kubearmorconfigs.yaml @@ -92,6 +92,44 @@ spec: - Never type: string type: object + kubearmorControllerToleration: + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists and Equal. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array kubearmorImage: description: ImageSpec defines the image specifications properties: @@ -131,6 +169,44 @@ spec: - Never type: string type: object + kubearmorRelayToleration: + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists and Equal. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array seccompEnabled: type: boolean tls: diff --git a/deployments/helm/KubeArmorOperator/templates/deployment.yaml b/deployments/helm/KubeArmorOperator/templates/deployment.yaml index e0a4689a55..5798f85530 100644 --- a/deployments/helm/KubeArmorOperator/templates/deployment.yaml +++ b/deployments/helm/KubeArmorOperator/templates/deployment.yaml @@ -26,4 +26,8 @@ spec: {{- end }} image: {{ include "operatorImage" . }} imagePullPolicy: {{ .Values.kubearmorOperator.imagePullPolicy }} + {{- if .Values.kubearmorOperator.tolerations }} + tolerations: + {{ toYaml .Values.kubearmorOperator.tolerations | indent 8 }} + {{- end }} serviceAccountName: {{ .Values.kubearmorOperator.name }} diff --git a/deployments/helm/KubeArmorOperator/values.yaml b/deployments/helm/KubeArmorOperator/values.yaml index f069271113..66e54cba51 100644 --- a/deployments/helm/KubeArmorOperator/values.yaml +++ b/deployments/helm/KubeArmorOperator/values.yaml @@ -36,6 +36,7 @@ kubearmorOperator: repository: kubearmor/kubearmor-operator tag: "" imagePullPolicy: IfNotPresent + tolerations: [] kubearmorConfig: defaultCapabilitiesPosture: audit diff --git a/deployments/operator/operator.yaml b/deployments/operator/operator.yaml index 09785ab19a..5c36d2a187 100644 --- a/deployments/operator/operator.yaml +++ b/deployments/operator/operator.yaml @@ -91,6 +91,44 @@ spec: - Never type: string type: object + kubearmorControllerToleration: + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists and Equal. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array kubearmorImage: description: ImageSpec defines the image specifications properties: @@ -130,6 +168,44 @@ spec: - Never type: string type: object + kubearmorRelayToleration: + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists and Equal. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array seccompEnabled: type: boolean tls: diff --git a/pkg/KubeArmorOperator/Makefile b/pkg/KubeArmorOperator/Makefile index 495437ef3c..eb9bde75fd 100644 --- a/pkg/KubeArmorOperator/Makefile +++ b/pkg/KubeArmorOperator/Makefile @@ -137,7 +137,7 @@ generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and CONTROLLER_GEN = $(GOBIN)/controller-gen .PHONY: controller-gen controller-gen: ## Download controller-gen locally if necessary. - $(call go-get-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.4.1) + $(call go-get-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.14.0) client/gen: @echo "--> Running code-generator to generate clients" diff --git a/pkg/KubeArmorOperator/api/operator.kubearmor.com/v1/kubearmorconfig_types.go b/pkg/KubeArmorOperator/api/operator.kubearmor.com/v1/kubearmorconfig_types.go index a5f276e4a4..9790e82513 100644 --- a/pkg/KubeArmorOperator/api/operator.kubearmor.com/v1/kubearmorconfig_types.go +++ b/pkg/KubeArmorOperator/api/operator.kubearmor.com/v1/kubearmorconfig_types.go @@ -4,6 +4,7 @@ package v1 import ( + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -50,8 +51,12 @@ type KubeArmorConfigSpec struct { // +kubebuilder:validation:optional KubeArmorRelayImage ImageSpec `json:"kubearmorRelayImage,omitempty"` // +kubebuilder:validation:optional + KubeArmorRelayToleration []v1.Toleration `json:"kubearmorRelayToleration,omitempty"` + // +kubebuilder:validation:optional KubeArmorControllerImage ImageSpec `json:"kubearmorControllerImage,omitempty"` // +kubebuilder:validation:optional + KubeArmorControllerToleration []v1.Toleration `json:"kubearmorControllerToleration,omitempty"` + // +kubebuilder:validation:optional KubeRbacProxyImage ImageSpec `json:"kubeRbacProxyImage,omitempty"` // +kubebuilder:validation:optional Tls Tls `json:"tls,omitempty"` diff --git a/pkg/KubeArmorOperator/common/defaults.go b/pkg/KubeArmorOperator/common/defaults.go index 9a7fe88adc..a197726c43 100644 --- a/pkg/KubeArmorOperator/common/defaults.go +++ b/pkg/KubeArmorOperator/common/defaults.go @@ -102,6 +102,10 @@ var ( SeccompProfile = "kubearmor-seccomp.json" SeccompInitProfile = "kubearmor-init-seccomp.json" + // Tolerations + KubeArmorRelayToleration []corev1.Toleration = []corev1.Toleration{} + KubeArmorControllerToleration []corev1.Toleration = []corev1.Toleration{} + // tls EnableTls bool = false ExtraDnsNames []string = []string{"localhost"} diff --git a/pkg/KubeArmorOperator/config/crd/bases/operator.kubearmor.com_kubearmorconfigs.yaml b/pkg/KubeArmorOperator/config/crd/bases/operator.kubearmor.com_kubearmorconfigs.yaml index e179add2bb..523197cfc7 100644 --- a/pkg/KubeArmorOperator/config/crd/bases/operator.kubearmor.com_kubearmorconfigs.yaml +++ b/pkg/KubeArmorOperator/config/crd/bases/operator.kubearmor.com_kubearmorconfigs.yaml @@ -92,6 +92,44 @@ spec: - Never type: string type: object + kubearmorControllerToleration: + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists and Equal. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array kubearmorImage: description: ImageSpec defines the image specifications properties: @@ -131,6 +169,44 @@ spec: - Never type: string type: object + kubearmorRelayToleration: + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists and Equal. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array seccompEnabled: type: boolean tls: diff --git a/pkg/KubeArmorOperator/internal/controller/cluster.go b/pkg/KubeArmorOperator/internal/controller/cluster.go index b0fc1945ef..b4235e9e58 100644 --- a/pkg/KubeArmorOperator/internal/controller/cluster.go +++ b/pkg/KubeArmorOperator/internal/controller/cluster.go @@ -7,6 +7,7 @@ import ( "bytes" "context" "fmt" + "reflect" "strconv" "strings" "sync" @@ -278,6 +279,7 @@ func (clusterWatcher *ClusterWatcher) WatchConfigCrd() { UpdateTlsData(&cfg.Spec) UpdateConfigMapData(&cfg.Spec) UpdateImages(&cfg.Spec) + UpdateTolerations(&cfg.Spec) UpdatedKubearmorRelayEnv(&cfg.Spec) UpdatedSeccomp(&cfg.Spec) // update status to (Installation) Created @@ -300,6 +302,7 @@ func (clusterWatcher *ClusterWatcher) WatchConfigCrd() { if common.OperatorConfigCrd != nil && cfg.Name == common.OperatorConfigCrd.Name { configChanged := UpdateConfigMapData(&cfg.Spec) imageUpdated := UpdateImages(&cfg.Spec) + tolerationUpdated := UpdateTolerations(&cfg.Spec) relayEnvUpdated := UpdatedKubearmorRelayEnv(&cfg.Spec) seccompEnabledUpdated := UpdatedSeccomp(&cfg.Spec) tlsUpdated := UpdateTlsData(&cfg.Spec) @@ -314,6 +317,9 @@ func (clusterWatcher *ClusterWatcher) WatchConfigCrd() { if len(imageUpdated) > 0 { clusterWatcher.UpdateKubeArmorImages(imageUpdated) } + if len(tolerationUpdated) > 0 { + clusterWatcher.UpdateKubeArmorTolerations(tolerationUpdated) + } if configChanged { // update status to Updating go clusterWatcher.UpdateCrdStatus(cfg.Name, common.UPDATING, common.UPDATING_MSG) @@ -420,6 +426,50 @@ func (clusterWatcher *ClusterWatcher) UpdateKubeArmorImages(images []string) err return res } +func (clusterWatcher *ClusterWatcher) UpdateKubeArmorTolerations(tolerations []string) error { + var res error + for _, toleration := range tolerations { + switch toleration { + case "relay": + relay, err := clusterWatcher.Client.AppsV1().Deployments(common.Namespace).Get(context.Background(), deployments.RelayDeploymentName, v1.GetOptions{}) + if err != nil { + clusterWatcher.Log.Warnf("Cannot get deployment=%s error=%s", deployments.RelayDeploymentName, err.Error()) + res = err + } else { + relay.Spec.Template.Spec.Tolerations = make([]corev1.Toleration, len(common.KubeArmorRelayToleration)) + copy(relay.Spec.Template.Spec.Tolerations, common.KubeArmorRelayToleration) + _, err = clusterWatcher.Client.AppsV1().Deployments(common.Namespace).Update(context.Background(), relay, v1.UpdateOptions{}) + if err != nil { + clusterWatcher.Log.Warnf("Cannot update deployment=%s error=%s", deployments.RelayDeploymentName, err.Error()) + res = err + } else { + clusterWatcher.Log.Infof("Updated Deployment=%s with tolerations", deployments.RelayDeploymentName, common.KubeArmorRelayImage) + } + } + + case "controller": + controller, err := clusterWatcher.Client.AppsV1().Deployments(common.Namespace).Get(context.Background(), deployments.KubeArmorControllerDeploymentName, v1.GetOptions{}) + if err != nil { + clusterWatcher.Log.Warnf("Cannot get deployment=%s error=%s", deployments.KubeArmorControllerDeploymentName, err.Error()) + res = err + } else { + controller.Spec.Template.Spec.Tolerations = make([]corev1.Toleration, len(common.KubeArmorControllerToleration)) + copy(controller.Spec.Template.Spec.Tolerations, common.KubeArmorControllerToleration) + _, err = clusterWatcher.Client.AppsV1().Deployments(common.Namespace).Update(context.Background(), controller, v1.UpdateOptions{}) + if err != nil { + clusterWatcher.Log.Warnf("Cannot update deployment=%s error=%s", deployments.KubeArmorControllerDeploymentName, err.Error()) + res = err + } else { + clusterWatcher.Log.Infof("Updated Deployment=%s with tolerations", deployments.KubeArmorControllerDeploymentName, common.KubeArmorControllerImage) + } + } + + } + + } + return res +} + func (clusterWatcher *ClusterWatcher) UpdateKubearmorRelayEnv(cfg *opv1.KubeArmorConfig) error { var res error relay, err := clusterWatcher.Client.AppsV1().Deployments(common.Namespace).Get(context.Background(), deployments.RelayDeploymentName, v1.GetOptions{}) @@ -529,6 +579,21 @@ func UpdateImages(config *opv1.KubeArmorConfigSpec) []string { return updatedImages } +func UpdateTolerations(config *opv1.KubeArmorConfigSpec) []string { + updatedTolerations := []string{} + if reflect.DeepEqual(common.KubeArmorRelayToleration, config.KubeArmorRelayToleration) { + common.KubeArmorRelayToleration = make([]corev1.Toleration, len(config.KubeArmorRelayToleration)) + copy(common.KubeArmorRelayToleration, config.KubeArmorRelayToleration) + updatedTolerations = append(updatedTolerations, "relay") + } + if reflect.DeepEqual(common.KubeArmorControllerToleration, config.KubeArmorControllerToleration) { + common.KubeArmorControllerToleration = make([]corev1.Toleration, len(config.KubeArmorControllerToleration)) + copy(common.KubeArmorControllerToleration, config.KubeArmorControllerToleration) + updatedTolerations = append(updatedTolerations, "controller") + } + return updatedTolerations +} + func (clusterWatcher *ClusterWatcher) UpdateCrdStatus(cfg, phase, message string) { err := wait.ExponentialBackoff(wait.Backoff{Steps: 5, Duration: 500 * time.Millisecond}, func() (bool, error) { configCrd, err := clusterWatcher.Opv1Client.OperatorV1().KubeArmorConfigs(common.Namespace).Get(context.Background(), cfg, metav1.GetOptions{})