fix: support affinity for scan jobs (#1915)

* fix: support affinity for scan jobs

* Minor fixes

* Re-generate

* Undo static manifest changes

deploy/helm/README.md

@@ -177,6 +177,7 @@ Keeps security report resources updated
 | trivyOperator.policiesConfig | string | `""` | policiesConfig Custom Rego Policies to be used by the config audit scanner See https://github.com/aquasecurity/trivy-operator/blob/main/docs/tutorials/writing-custom-configuration-audit-policies.md for more details. |
 | trivyOperator.reportRecordFailedChecksOnly | bool | `true` | reportRecordFailedChecksOnly flag is to record only failed checks on misconfiguration reports (config-audit and rbac assessment) |
 | trivyOperator.reportResourceLabels | string | `""` | reportResourceLabels comma-separated scanned resource labels which the user wants to include in the Prometheus metrics report. Example: `owner,app` |
+| trivyOperator.scanJobAffinity | list | `[]` | scanJobAffinity affinity to be applied to the scanner pods and node-collector |
 | trivyOperator.scanJobAnnotations | string | `""` | scanJobAnnotations comma-separated representation of the annotations which the user wants the scanner pods to be annotated with. Example: `foo=bar,env=stage` will annotate the scanner pods with the annotations `foo: bar` and `env: stage` |
 | trivyOperator.scanJobAutomountServiceAccountToken | bool | `false` | scanJobAutomountServiceAccountToken the flag to enable automount for service account token on scan job |
 | trivyOperator.scanJobCompressLogs | bool | `true` | scanJobCompressLogs control whether scanjob output should be compressed or plain |

deploy/helm/templates/configmaps/operator.yaml

@@ -6,6 +6,9 @@ metadata:
   namespace: {{ include "trivy-operator.namespace" . }}
   labels: {{- include "trivy-operator.labels" . | nindent 4 }}
 data:
+  {{- with .Values.trivyOperator.scanJobAffinity }}
+  scanJob.affinity: {{ . | toJson | quote }}
+  {{- end }}
   {{- with .Values.trivyOperator.scanJobTolerations }}
   scanJob.tolerations: {{ . | toJson | quote }}
   {{- end }}
@@ -75,4 +78,4 @@ data:
   {{- with .Values.nodeCollector.imagePullSecret }}
   node.collector.imagePullSecret: "{{ . }}"
   {{- end }}
-  node.collector.nodeSelector: {{ .Values.nodeCollector.useNodeSelector | quote }}
+  node.collector.nodeSelector: {{ .Values.nodeCollector.useNodeSelector | quote }}

deploy/helm/values.yaml

@@ -216,6 +216,8 @@ trivyOperator:
   configAuditReportsPlugin: "Trivy"
   # -- scanJobCompressLogs control whether scanjob output should be compressed or plain
   scanJobCompressLogs: true
+  # -- scanJobAffinity affinity to be applied to the scanner pods and node-collector
+  scanJobAffinity: []
   # -- scanJobTolerations tolerations to be applied to the scanner pods and node-collector so that they can run on nodes with matching taints
   scanJobTolerations: []
   # -- If you do want to specify tolerations, uncomment the following lines, adjust them as necessary, and remove the

docs/getting-started/installation/configuration.md

@@ -69,6 +69,7 @@ To change the target namespace from all namespaces to the `default` namespace ed
 |---|---|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
 | `vulnerabilityReports.scanner`| `Trivy`| The name of the plugin that generates vulnerability reports. Either `Trivy` or `Aqua`.                                                                                                                                                                                                          |
 | `vulnerabilityReports.scanJobsInSameNamespace` | `"false"`| Whether to run vulnerability scan jobs in same namespace of workload. Set `"true"` to enable.                                                                                                                                                                                                   |
+| `scanJob.affinity`   | N/A| JSON representation of the [affinity] to be applied to the scanner pods and node-collector. Example: `'{"nodeAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":{"nodeSelectorTerms":[{"matchExpressions":[{"key":"kubernetes.io/os","operator":"In","values":["linux"]}]},{"matchExpressions":[{"key":"virtual-kubelet.io/provider","operator":"DoesNotExist"}]}]}}}'`
 | `scanJob.tolerations`| N/A| JSON representation of the [tolerations] to be applied to the scanner pods and node-collector so that they can run on nodes with matching taints. Example: `'[{"key":"key1", "operator":"Equal", "value":"value1", "effect":"NoSchedule"}]'`
 | `nodeCollector.volumeMounts`| see helm/values.yaml | node-collector pod volumeMounts definition for collecting config files information
 | `nodeCollector.volumes`| see helm/values.yaml | node-collector pod volumes definition for collecting config files information                                                                       |
@@ -132,6 +133,7 @@ kubectl patch cm trivy-operator-trivy-config -n trivy-system \
   -p '[{"op": "remove", "path": "/data/trivy.httpProxy"}]'
 ```
 
+[affinity]: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity
 [tolerations]: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration
 
 [prometheus]: https://github.com/prometheus

docs/settings.md

@@ -55,6 +55,7 @@ configuration settings for common use cases. For example, switch Trivy from [Sta
 | `vulnerabilityReports.scanner`                                       | `Trivy`                               | The name of the plugin that generates vulnerability reports. Either `Trivy` or `Aqua`.                                                                                                                                              |
 | `vulnerabilityReports.scanJobsInSameNamespace`                       | `"false"`                             | Whether to run vulnerability scan jobs in same namespace of workload. Set `"true"` to enable.                                                                                                                                       |
 | `configAuditReports.scanner`                                         | `Trivy`                               | The name of the plugin that generates config audit reports.                                                                                                                                                                         |
+| `scanJob.affinity`                                                   | N/A                                   | JSON representation of the [affinity] to be applied to the scanner pods and node-collector. Example: `'{"nodeAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":{"nodeSelectorTerms":[{"matchExpressions":[{"key":"kubernetes.io/os","operator":"In","values":["linux"]}]},{"matchExpressions":[{"key":"virtual-kubelet.io/provider","operator":"DoesNotExist"}]}]}}}'`           |
 | `scanJob.tolerations`                                                | N/A                                   | JSON representation of the [tolerations] to be applied to the scanner pods and node-collector so that they can run on nodes with matching taints. Example: `'[{"key":"key1", "operator":"Equal", "value":"value1", "effect":"NoSchedule"}]'`           |
 | `nodeCollector.volumeMounts`| see helm/values.yaml | node-collector pod volumeMounts definition for collecting config files information
 | `nodeCollector.volumes`| see helm/values.yaml | node-collector pod volumes definition for collecting config files information

go.mod

@@ -6,7 +6,7 @@ require (
 	github.com/CycloneDX/cyclonedx-go v0.8.0
 	github.com/aquasecurity/defsec v0.94.1
 	github.com/aquasecurity/trivy v0.49.1
-	github.com/aquasecurity/trivy-kubernetes v0.6.4-0.20240129122346-af2de58a7466
+	github.com/aquasecurity/trivy-kubernetes v0.6.4-0.20240311135805-8e927abd008d
 	github.com/bluele/gcache v0.0.2
 	github.com/caarlos0/env/v6 v6.10.1
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc

go.sum

@@ -312,8 +312,8 @@ github.com/aquasecurity/trivy-iac v0.8.0 h1:NKFhk/BTwQ0jIh4t74V8+6UIGUvPlaxO9HPl
 github.com/aquasecurity/trivy-iac v0.8.0/go.mod h1:ARiMeNqcaVWOXJmp8hmtMnNm/Jd836IOmDBUW5r4KEk=
 github.com/aquasecurity/trivy-java-db v0.0.0-20240109071736-184bd7481d48 h1:JVgBIuIYbwG+ekC5lUHUpGJboPYiCcxiz06RCtz8neI=
 github.com/aquasecurity/trivy-java-db v0.0.0-20240109071736-184bd7481d48/go.mod h1:Ldya37FLi0e/5Cjq2T5Bty7cFkzUDwTcPeQua+2M8i8=
-github.com/aquasecurity/trivy-kubernetes v0.6.4-0.20240129122346-af2de58a7466 h1:pM/cncKgNyvCzhOiFL19F4pyebktYBwl9/XyQl44OCI=
-github.com/aquasecurity/trivy-kubernetes v0.6.4-0.20240129122346-af2de58a7466/go.mod h1:pGWup/B9igRLCuJ0fmI7hft4Ni7l5zCspwtt5dvVNTY=
+github.com/aquasecurity/trivy-kubernetes v0.6.4-0.20240311135805-8e927abd008d h1:BWcT4YE8jHMWMIxZCC6sgriRP4xLTOyEe8tFoY4pnXQ=
+github.com/aquasecurity/trivy-kubernetes v0.6.4-0.20240311135805-8e927abd008d/go.mod h1:6T6aM0FnBMoNfLiPFPatPXfBpMO9Zgo/G22A5KWGijc=
 github.com/aquasecurity/trivy-policies v0.8.0 h1:LvmIdw/DfTF72Lc8L+CKLYzfb5BFYzLBGFFR95PKC74=
 github.com/aquasecurity/trivy-policies v0.8.0/go.mod h1:qF/t59pgK/0JTV6tXaeA3Iw3opzoMgzGCDcTDBmqb30=
 github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0 h1:jfIu9sQUG6Ig+0+Ap1h4unLjW6YQJpKZVmUzxsD4E/Q=

pkg/configauditreport/controller/node.go

@@ -121,6 +121,10 @@ func (r *NodeReconciler) reconcileNodes() reconcile.Func {
 		if err != nil {
 			return ctrl.Result{}, fmt.Errorf("preparing job: %w", err)
 		}
+		jobAffinity, err := r.GetScanJobAffinity()
+		if err != nil {
+			return ctrl.Result{}, fmt.Errorf("getting job affinity: %w", err)
+		}
 		jobTolerations, err := r.GetScanJobTolerations()
 		if err != nil {
 			return ctrl.Result{}, fmt.Errorf("getting job tolerations: %w", err)
@@ -168,6 +172,7 @@ func (r *NodeReconciler) reconcileNodes() reconcile.Func {
 			j.WithJobNamespace(on),
 			j.WithServiceAccount(r.ServiceAccount),
 			j.WithCollectorTimeout(r.Config.ScanJobTimeout),
+			j.WithJobAffinity(jobAffinity),
 			j.WithJobTolerations(jobTolerations),
 			j.WithPodSpecSecurityContext(scanJobSecurityContext),
 			j.WithContainerSecurityContext(scanJobContainerSecurityContext),

pkg/trivyoperator/config.go

@@ -59,6 +59,7 @@ const (
 	keyVulnerabilityReportsScanner       = "vulnerabilityReports.scanner"
 	KeyVulnerabilityScansInSameNamespace = "vulnerabilityReports.scanJobsInSameNamespace"
 	keyConfigAuditReportsScanner         = "configAuditReports.scanner"
+	keyScanJobAffinity                   = "scanJob.affinity"
 	keyScanJobTolerations                = "scanJob.tolerations"
 	KeyScanJobcompressLogs               = "scanJob.compressLogs"
 	KeyNodeCollectorVolumes              = "nodeCollector.volumes"
@@ -168,6 +169,20 @@ func (c ConfigData) GetConfigAuditReportsScanner() Scanner {
 	return Scanner(value)
 }
 
+func (c ConfigData) GetScanJobAffinity() (*corev1.Affinity, error) {
+	if c[keyScanJobAffinity] == "" {
+		return nil, nil
+	}
+
+	scanJobAffinity := &corev1.Affinity{}
+	err := json.Unmarshal([]byte(c[keyScanJobAffinity]), scanJobAffinity)
+	if err != nil {
+		return nil, fmt.Errorf("failed parsing incorrectly formatted custom scan pod template affinity: %s", c[keyScanJobAffinity])
+	}
+
+	return scanJobAffinity, nil
+}
+
 func (c ConfigData) GetScanJobTolerations() ([]corev1.Toleration, error) {
 	var scanJobTolerations []corev1.Toleration
 	if c[keyScanJobTolerations] == "" {
@@ -219,7 +234,7 @@ func (c ConfigData) GetScanJobNodeSelector() (map[string]string, error) {
 	}
 
 	if err := json.Unmarshal([]byte(c[keyScanJobNodeSelector]), &scanJobNodeSelector); err != nil {
-		return scanJobNodeSelector, fmt.Errorf("failed to parse incorrect job template nodeSelector %s: %w", c[keyScanJobNodeSelector], err)
+		return scanJobNodeSelector, fmt.Errorf("failed to parse incorrect pod template nodeSelector %s: %w", c[keyScanJobNodeSelector], err)
 	}
 
 	return scanJobNodeSelector, nil

pkg/trivyoperator/config_test.go

@@ -76,6 +76,75 @@ func TestConfigData_GetConfigAuditReportsScanner(t *testing.T) {
 	}
 }
 
+func TestConfigData_GetScanJobAffinity(t *testing.T) {
+	testCases := []struct {
+		name        string
+		config      trivyoperator.ConfigData
+		expected    *corev1.Affinity
+		expectError string
+	}{
+		{
+			name:     "no scanJob.affinity in ConfigData",
+			config:   trivyoperator.ConfigData{},
+			expected: nil,
+		},
+		{
+			name:        "scanJob.affinity value is not json",
+			config:      trivyoperator.ConfigData{"scanJob.affinity": `lolwut`},
+			expected:    nil,
+			expectError: "invalid character 'l' looking for beginning of value",
+		},
+		{
+			name:     "empty JSON array",
+			config:   trivyoperator.ConfigData{"scanJob.affinity": `{}`},
+			expected: &corev1.Affinity{},
+		},
+		{
+			name: "valid affinity",
+			config: trivyoperator.ConfigData{
+				"scanJob.affinity": `{"nodeAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":{"nodeSelectorTerms":[{"matchExpressions":[{"key":"kubernetes.io/os","operator":"In","values":["linux"]}]},{"matchExpressions":[{"key":"virtual-kubelet.io/provider","operator":"DoesNotExist"}]}]}}}`,
+			},
+			expected: &corev1.Affinity{
+				NodeAffinity: &corev1.NodeAffinity{
+					RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{
+						NodeSelectorTerms: []corev1.NodeSelectorTerm{
+							{
+								MatchExpressions: []corev1.NodeSelectorRequirement{
+									{
+										Key:      "kubernetes.io/os",
+										Operator: corev1.NodeSelectorOpIn,
+										Values:   []string{"linux"},
+									},
+								},
+							},
+							{
+								MatchExpressions: []corev1.NodeSelectorRequirement{
+									{
+										Key:      "virtual-kubelet.io/provider",
+										Operator: corev1.NodeSelectorOpDoesNotExist,
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			got, err := tc.config.GetScanJobAffinity()
+			if tc.expectError != "" {
+				assert.Error(t, err, "unexpected end of JSON input", tc.name)
+			} else {
+				assert.NoError(t, err, tc.name)
+			}
+			assert.Equal(t, tc.expected, got, tc.name)
+		})
+	}
+}
+
 func TestConfigData_GetScanJobTolerations(t *testing.T) {
 	testCases := []struct {
 		name        string
@@ -429,7 +498,7 @@ func TestConfigData_GetScanJobNodeSelector(t *testing.T) {
 				"scanJob.nodeSelector": "{dlzm",
 			},
 			expected:    map[string]string{},
-			expectError: "failed to parse incorrect job template nodeSelector {dlzm: invalid character 'd' looking for beginning of object key string",
+			expectError: "failed to parse incorrect pod template nodeSelector {dlzm: invalid character 'd' looking for beginning of object key string",
 		},
 	}
 

pkg/vulnerabilityreport/builder.go

@@ -29,6 +29,7 @@ type ScanJobBuilder struct {
 	timeout                  time.Duration
 	object                   client.Object
 	credentials              map[string]docker.Auth
+	affinity                 *corev1.Affinity
 	tolerations              []corev1.Toleration
 	nodeSelector             map[string]string
 	annotations              map[string]string
@@ -79,6 +80,11 @@ func (s *ScanJobBuilder) WithObject(object client.Object) *ScanJobBuilder {
 	return s
 }
 
+func (s *ScanJobBuilder) WithAffinity(affinity *corev1.Affinity) *ScanJobBuilder {
+	s.affinity = affinity
+	return s
+}
+
 func (s *ScanJobBuilder) WithTolerations(tolerations []corev1.Toleration) *ScanJobBuilder {
 	s.tolerations = tolerations
 	return s
@@ -135,6 +141,10 @@ func (s *ScanJobBuilder) Get() (*batchv1.Job, []*corev1.Secret, error) {
 		return nil, nil, err
 	}
 
+	if s.affinity != nil {
+		templateSpec.Affinity = s.affinity
+	}
+
 	templateSpec.Tolerations = append(templateSpec.Tolerations, s.tolerations...)
 	if s.podSecurityContext != nil {
 		templateSpec.SecurityContext = s.podSecurityContext

pkg/vulnerabilityreport/controller/workload.go

@@ -268,6 +268,11 @@ func (r *WorkloadController) SubmitScanJob(ctx context.Context, owner client.Obj
 		}
 	}
 
+	scanJobAffinity, err := r.GetScanJobAffinity()
+	if err != nil {
+		return fmt.Errorf("getting scan job affinity: %w", err)
+	}
+
 	scanJobTolerations, err := r.GetScanJobTolerations()
 	if err != nil {
 		return fmt.Errorf("getting scan job tolerations: %w", err)
@@ -310,6 +315,7 @@ func (r *WorkloadController) SubmitScanJob(ctx context.Context, owner client.Obj
 		WithTTL(r.Config.ScanJobTTL).
 		WithScanSecretTTL(r.Config.ScanSecretTTL).
 		WithObject(owner).
+		WithAffinity(scanJobAffinity).
 		WithTolerations(scanJobTolerations).
 		WithAnnotations(scanJobAnnotations).
 		WithNodeSelector(scanJobNodeSelector).