feat: separate toleration setting for node-collector (#2006)

* feat: separate toleration setting for node-collector Signed-off-by: chenk <hen.keinan@gmail.com> * feat: separate toleration setting for node-collector Signed-off-by: chenk <hen.keinan@gmail.com> --------- Signed-off-by: chenk <hen.keinan@gmail.com>
aquasecurity · Apr 16, 2024 · 0f9e0f8 · 0f9e0f8
1 parent f8bf366
commit 0f9e0f8
Show file tree

Hide file tree

Showing 5 changed files with 27 additions and 5 deletions.
diff --git a/deploy/helm/README.md b/deploy/helm/README.md
@@ -32,6 +32,7 @@ Keeps security report resources updated
 | nodeCollector.registry | string | `"ghcr.io"` | registry of the node-collector image |
 | nodeCollector.repository | string | `"aquasecurity/node-collector"` | repository of the node-collector image |
 | nodeCollector.tag | string | `"0.1.3"` | tag version of the node-collector image |
+| nodeCollector.tolerations | list | `[]` | tolerations to be applied to the node-collector so that they can run on nodes with matching taints |
 | nodeCollector.useNodeSelector | bool | `true` | useNodeSelector determine if to use nodeSelector (by auto detecting node name) with node-collector scan job |
 | nodeCollector.volumeMounts | list | `[{"mountPath":"/var/lib/etcd","name":"var-lib-etcd","readOnly":true},{"mountPath":"/var/lib/kubelet","name":"var-lib-kubelet","readOnly":true},{"mountPath":"/var/lib/kube-scheduler","name":"var-lib-kube-scheduler","readOnly":true},{"mountPath":"/var/lib/kube-controller-manager","name":"var-lib-kube-controller-manager","readOnly":true},{"mountPath":"/etc/systemd","name":"etc-systemd","readOnly":true},{"mountPath":"/lib/systemd/","name":"lib-systemd","readOnly":true},{"mountPath":"/etc/kubernetes","name":"etc-kubernetes","readOnly":true},{"mountPath":"/etc/cni/net.d/","name":"etc-cni-netd","readOnly":true}]` | node-collector pod volume mounts definition for collecting config files information |
 | nodeCollector.volumes | list | `[{"hostPath":{"path":"/var/lib/etcd"},"name":"var-lib-etcd"},{"hostPath":{"path":"/var/lib/kubelet"},"name":"var-lib-kubelet"},{"hostPath":{"path":"/var/lib/kube-scheduler"},"name":"var-lib-kube-scheduler"},{"hostPath":{"path":"/var/lib/kube-controller-manager"},"name":"var-lib-kube-controller-manager"},{"hostPath":{"path":"/etc/systemd"},"name":"etc-systemd"},{"hostPath":{"path":"/lib/systemd"},"name":"lib-systemd"},{"hostPath":{"path":"/etc/kubernetes"},"name":"etc-kubernetes"},{"hostPath":{"path":"/etc/cni/net.d/"},"name":"etc-cni-netd"}]` | node-collector pod volumes definition for collecting config files information |
@@ -192,7 +193,7 @@ Keeps security report resources updated
 | trivyOperator.scanJobPodTemplateContainerSecurityContext | object | `{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]},"privileged":false,"readOnlyRootFilesystem":true}` | scanJobPodTemplateContainerSecurityContext SecurityContext the user wants the scanner and node collector containers (and their initContainers) to be amended with. |
 | trivyOperator.scanJobPodTemplateLabels | string | `""` | scanJobPodTemplateLabels comma-separated representation of the labels which the user wants the scanner pods to be labeled with. Example: `foo=bar,env=stage` will labeled the scanner pods with the labels `foo: bar` and `env: stage` |
 | trivyOperator.scanJobPodTemplatePodSecurityContext | object | `{}` | scanJobPodTemplatePodSecurityContext podSecurityContext the user wants the scanner and node collector pods to be amended with. Example:   RunAsUser: 10000   RunAsGroup: 10000   RunAsNonRoot: true |
-| trivyOperator.scanJobTolerations | list | `[]` | scanJobTolerations tolerations to be applied to the scanner pods and node-collector so that they can run on nodes with matching taints |
+| trivyOperator.scanJobTolerations | list | `[]` | scanJobTolerations tolerations to be applied to the scanner pods so that they can run on nodes with matching taints |
 | trivyOperator.skipInitContainers | bool | `false` | skipInitContainers when this flag is set to true, the initContainers will be skipped for the scanner and node collector pods |
 | trivyOperator.skipResourceByLabels | string | `""` | skipResourceByLabels comma-separated labels keys which trivy-operator will skip scanning on resources with matching labels |
 | trivyOperator.vulnerabilityReportsPlugin | string | `"Trivy"` | vulnerabilityReportsPlugin the name of the plugin that generates vulnerability reports `Trivy` |

diff --git a/deploy/helm/templates/configmaps/operator.yaml b/deploy/helm/templates/configmaps/operator.yaml
@@ -12,6 +12,9 @@ data:
   {{- with .Values.trivyOperator.scanJobTolerations }}
   scanJob.tolerations: {{ . | toJson | quote }}
   {{- end }}
+  {{- with .Values.nodeCollector.tolerations }}
+  nodeCollector.tolerations: {{ . | toJson | quote }}
+  {{- end }}
   {{- with .Values.nodeCollector.volumes }}
   nodeCollector.volumes: {{ . | toJson | quote }}
   {{- end }}

diff --git a/deploy/helm/values.yaml b/deploy/helm/values.yaml
@@ -227,15 +227,14 @@ trivyOperator:
   scanJobCompressLogs: true
   # -- scanJobAffinity affinity to be applied to the scanner pods and node-collector
   scanJobAffinity: []
-  # -- scanJobTolerations tolerations to be applied to the scanner pods and node-collector so that they can run on nodes with matching taints
+  # -- scanJobTolerations tolerations to be applied to the scanner pods so that they can run on nodes with matching taints
   scanJobTolerations: []
   # -- If you do want to specify tolerations, uncomment the following lines, adjust them as necessary, and remove the
   # square brackets after 'scanJobTolerations:'.
   # - key: "key1"
   #   operator: "Equal"
   #   value: "value1"
   #   effect: "NoSchedule"
-
   # -- scanJobNodeSelector nodeSelector to be applied to the scanner pods so that they can run on nodes with matching labels
   scanJobNodeSelector: {}
   # -- If you do want to specify nodeSelector, uncomment the following lines, adjust them as necessary, and remove the
@@ -662,6 +661,14 @@ nodeCollector:
   imagePullSecret: ~
   # -- excludeNodes comma-separated node labels that the node-collector job should exclude from scanning (example kubernetes.io/arch=arm64,team=dev)
   excludeNodes:
+  # -- tolerations to be applied to the node-collector so that they can run on nodes with matching taints
+  tolerations: []
+  # -- If you do want to specify tolerations, uncomment the following lines, adjust them as necessary, and remove the
+  # square brackets after 'scanJobTolerations:'.
+  # - key: "key1"
+  #   operator: "Equal"
+  #   value: "value1"
+  #   effect: "NoSchedule"
   # -- node-collector pod volume mounts definition for collecting config files information
   volumeMounts:
     - name: var-lib-etcd

diff --git a/pkg/configauditreport/controller/node.go b/pkg/configauditreport/controller/node.go
@@ -125,7 +125,7 @@ func (r *NodeReconciler) reconcileNodes() reconcile.Func {
 		if err != nil {
 			return ctrl.Result{}, fmt.Errorf("getting job affinity: %w", err)
 		}
-		jobTolerations, err := r.GetScanJobTolerations()
+		nodeTolerations, err := r.GetNodeCollectorTolerations()
 		if err != nil {
 			return ctrl.Result{}, fmt.Errorf("getting job tolerations: %w", err)
 		}
@@ -173,7 +173,7 @@ func (r *NodeReconciler) reconcileNodes() reconcile.Func {
 			j.WithServiceAccount(r.ServiceAccount),
 			j.WithCollectorTimeout(r.Config.ScanJobTimeout),
 			j.WithJobAffinity(jobAffinity),
-			j.WithJobTolerations(jobTolerations),
+			j.WithJobTolerations(nodeTolerations),
 			j.WithPodSpecSecurityContext(scanJobSecurityContext),
 			j.WithContainerSecurityContext(scanJobContainerSecurityContext),
 			j.WithPodImagePullSecrets(r.GetNodeCollectorImagePullsecret()),

diff --git a/pkg/trivyoperator/config.go b/pkg/trivyoperator/config.go
@@ -61,6 +61,7 @@ const (
 	keyConfigAuditReportsScanner         = "configAuditReports.scanner"
 	keyScanJobAffinity                   = "scanJob.affinity"
 	keyScanJobTolerations                = "scanJob.tolerations"
+	keyNodeCollectorTolerations          = "nodeCollector.tolerations"
 	KeyScanJobcompressLogs               = "scanJob.compressLogs"
 	KeyNodeCollectorVolumes              = "nodeCollector.volumes"
 	KeyNodeCollectorExcludeNodes         = "nodeCollector.excludeNodes"
@@ -193,6 +194,16 @@ func (c ConfigData) GetScanJobTolerations() ([]corev1.Toleration, error) {
 	return scanJobTolerations, err
 }
 
+func (c ConfigData) GetNodeCollectorTolerations() ([]corev1.Toleration, error) {
+	var nodeCollectorTolerations []corev1.Toleration
+	if c[keyNodeCollectorTolerations] == "" {
+		return nodeCollectorTolerations, nil
+	}
+	err := json.Unmarshal([]byte(c[keyNodeCollectorTolerations]), &nodeCollectorTolerations)
+
+	return nodeCollectorTolerations, err
+}
+
 func (c ConfigData) GetNodeCollectorImagePullsecret() []corev1.LocalObjectReference {
 	imagePullSecrets := make([]corev1.LocalObjectReference, 0)
 	imagePullSecretValue := c[KeyNodeCollectorImagePullSecret]