feat(operator): Integrate kube-bench (#404)

Resolves: #263

Signed-off-by: Daniel Pacak <pacak.daniel@gmail.com>
Co-authored-by: Teppei Fukuda <knqyf263@gmail.com>

CONTRIBUTING.md

@@ -182,8 +182,9 @@ started with a basic development workflow. For other install modes see [Operator
 1. Send custom resource definitions to the Kubernetes API:
 
    ```
-   $ kubectl apply -f deploy/crd/vulnerabilityreports.crd.yaml
-   $ kubectl apply -f deploy/crd/configauditreports.crd.yaml
+   $ kubectl apply -f deploy/crd/vulnerabilityreports.crd.yaml \
+       -f deploy/crd/configauditreports.crd.yaml \
+       -f deploy/crd/ciskubebenchreports.crd.yaml
    ```
 2. Send the following Kubernetes objects definitions to the Kubernetes API:
 
@@ -224,16 +225,16 @@ started with a basic development workflow. For other install modes see [Operator
 To install [Operator Lifecycle Manager][olm] (OLM) run:
 
 ```
-$ kubectl apply -f https://github.com/operator-framework/operator-lifecycle-manager/releases/download/0.15.1/crds.yaml
-$ kubectl apply -f https://github.com/operator-framework/operator-lifecycle-manager/releases/download/0.15.1/olm.yaml
+$ kubectl apply -f https://github.com/operator-framework/operator-lifecycle-manager/releases/download/0.16.1/crds.yaml
+$ kubectl apply -f https://github.com/operator-framework/operator-lifecycle-manager/releases/download/0.16.1/olm.yaml
 ```
 
 or
 
 ```
-$ curl -L https://github.com/operator-framework/operator-lifecycle-manager/releases/download/0.15.1/install.sh -o install.sh
+$ curl -L https://github.com/operator-framework/operator-lifecycle-manager/releases/download/0.16.1/install.sh -o install.sh
 $ chmod +x install.sh
-$ ./install.sh 0.15.1
+$ ./install.sh 0.16.1
 ```
 
 ### Build the Catalog Image

Makefile

@@ -88,14 +88,13 @@ itests-starboard-operator: check-env get-ginkgo
 	-coverpkg=github.com/aquasecurity/starboard/pkg/operator,\
 	github.com/aquasecurity/starboard/pkg/operator/predicate,\
 	github.com/aquasecurity/starboard/pkg/operator/controller,\
-	github.com/aquasecurity/starboard/pkg/operator/controller/job,\
-	github.com/aquasecurity/starboard/pkg/operator/controller/pod,\
 	github.com/aquasecurity/starboard/pkg/config,\
 	github.com/aquasecurity/starboard/pkg/resources,\
 	github.com/aquasecurity/starboard/pkg/trivy,\
 	github.com/aquasecurity/starboard/pkg/polaris,\
 	github.com/aquasecurity/starboard/pkg/configauditreport,\
-	github.com/aquasecurity/starboard/pkg/vulnerabilityreport \
+	github.com/aquasecurity/starboard/pkg/vulnerabilityreport,\
+	github.com/aquasecurity/starboard/pkg/kubebench \
 	./itest/starboard-operator
 
 check-env:

deploy/helm/templates/rbac.yaml

@@ -36,6 +36,14 @@ rules:
       - get
       - list
       - watch
+  - apiGroups:
+      - ""
+    resources:
+      - "nodes"
+    verbs:
+      - get
+      - list
+      - watch
   - apiGroups:
       - ""
     resources:
@@ -73,6 +81,7 @@ rules:
     resources:
       - vulnerabilityreports
       - configauditreports
+      - ciskubebenchreports
     verbs:
       - get
       - list

deploy/helm/values.yaml

@@ -50,6 +50,7 @@ envSecret:
   #       properly.
   stringData:
     OPERATOR_LOG_DEV_MODE: "false"
+    OPERATOR_CIS_KUBERNETES_BENCHMARK_ENABLED: "true"
 
 rbac:
   create: true

deploy/static/03-starboard-operator.clusterrole.yaml

@@ -14,6 +14,14 @@ rules:
       - get
       - list
       - watch
+  - apiGroups:
+      - ""
+    resources:
+      - "nodes"
+    verbs:
+      - get
+      - list
+      - watch
   - apiGroups:
       - ""
     resources:
@@ -51,6 +59,7 @@ rules:
     resources:
       - vulnerabilityreports
       - configauditreports
+      - ciskubebenchreports
     verbs:
       - get
       - list

deploy/static/06-starboard-operator.deployment.yaml

@@ -43,13 +43,15 @@ spec:
             - name: OPERATOR_LOG_DEV_MODE
               value: "false"
             - name: OPERATOR_CONCURRENT_SCAN_JOBS_LIMIT
-              value: "3"
+              value: "10"
             - name: OPERATOR_SCAN_JOB_RETRY_AFTER
               value: "30s"
             - name: OPERATOR_METRICS_BIND_ADDRESS
               value: ":8080"
             - name: OPERATOR_HEALTH_PROBE_BIND_ADDRESS
               value: ":9090"
+            - name: OPERATOR_CIS_KUBERNETES_BENCHMARK_ENABLED
+              value: "true"
           ports:
             - name: metrics
               containerPort: 8080

docs/operator/configuration.md

@@ -1,16 +1,17 @@
 Configuration of the operator's pod is done via environment variables at startup.
 
-| NAME                                  | DEFAULT                | DESCRIPTION |
-| ------------------------------------- | ---------------------- | ----------- |
-| `OPERATOR_NAMESPACE`                  | N/A                    | See [Install modes](#install-modes) |
-| `OPERATOR_TARGET_NAMESPACES`          | N/A                    | See [Install modes](#install-modes) |
-| `OPERATOR_SERVICE_ACCOUNT`            | `starboard-operator`   | The name of the service account assigned to the operator's pod |
-| `OPERATOR_LOG_DEV_MODE`               | `false`                | The flag to use (or not use) development mode (more human-readable output, extra stack traces and logging information, etc). |
-| `OPERATOR_SCAN_JOB_TIMEOUT`           | `5m`                   | The length of time to wait before giving up on a scan job |
-| `OPERATOR_CONCURRENT_SCAN_JOBS_LIMIT` | `3`                    | The maximum number of scan jobs create by the operator |
-| `OPERATOR_SCAN_JOB_RETRY_AFTER`       | `30s`                  | The duration to wait before retrying a failed scan job |
-| `OPERATOR_METRICS_BIND_ADDRESS`       | `:8080`                | The TCP address to bind to for serving [Prometheus][prometheus] metrics. It can be set to `0` to disable the metrics serving. |
-| `OPERATOR_HEALTH_PROBE_BIND_ADDRESS`  | `:9090`                | The TCP address to bind to for serving health probes, i.e. `/healthz/` and `/readyz/` endpoints. |
+| NAME                                        | DEFAULT                | DESCRIPTION |
+| ------------------------------------------- | ---------------------- | ----------- |
+| `OPERATOR_NAMESPACE`                        | N/A                    | See [Install modes](#install-modes) |
+| `OPERATOR_TARGET_NAMESPACES`                | N/A                    | See [Install modes](#install-modes) |
+| `OPERATOR_SERVICE_ACCOUNT`                  | `starboard-operator`   | The name of the service account assigned to the operator's pod |
+| `OPERATOR_LOG_DEV_MODE`                     | `false`                | The flag to use (or not use) development mode (more human-readable output, extra stack traces and logging information, etc). |
+| `OPERATOR_SCAN_JOB_TIMEOUT`                 | `5m`                   | The length of time to wait before giving up on a scan job |
+| `OPERATOR_CONCURRENT_SCAN_JOBS_LIMIT`       | `10`                   | The maximum number of scan jobs create by the operator |
+| `OPERATOR_SCAN_JOB_RETRY_AFTER`             | `30s`                  | The duration to wait before retrying a failed scan job |
+| `OPERATOR_METRICS_BIND_ADDRESS`             | `:8080`                | The TCP address to bind to for serving [Prometheus][prometheus] metrics. It can be set to `0` to disable the metrics serving. |
+| `OPERATOR_HEALTH_PROBE_BIND_ADDRESS`        | `:9090`                | The TCP address to bind to for serving health probes, i.e. `/healthz/` and `/readyz/` endpoints. |
+| `OPERATOR_CIS_KUBERNETES_BENCHMARK_ENABLED` | `true`                 | The flag to enable CIS Kubernetes Benchmark reconciler |
 
 ## Install Modes
 

go.mod

@@ -23,6 +23,6 @@ require (
 	k8s.io/code-generator v0.19.2
 	k8s.io/klog v1.0.0
 	k8s.io/utils v0.0.0-20200912215256-4140de9c8800
-	sigs.k8s.io/controller-runtime v0.7.1
+	sigs.k8s.io/controller-runtime v0.7.2
 	sigs.k8s.io/yaml v1.2.0
 )

go.sum

@@ -1224,8 +1224,8 @@ rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.9/go.mod h1:dzAXnQbTRyDlZPJX2SUPEqvnB+j7AJjtlox7PEwigU0=
-sigs.k8s.io/controller-runtime v0.7.1 h1:nqVwzVzdenfd9xIbB35pC7JJH2IXVL4hDo3MNzkyCh4=
-sigs.k8s.io/controller-runtime v0.7.1/go.mod h1:pJ3YBrJiAqMAZKi6UVGuE98ZrroV1p+pIhoHsMm9wdU=
+sigs.k8s.io/controller-runtime v0.7.2 h1:gD2JZp0bBLLuvSRYVNvox+bRCz1UUUxKDjPUCb56Ukk=
+sigs.k8s.io/controller-runtime v0.7.2/go.mod h1:pJ3YBrJiAqMAZKi6UVGuE98ZrroV1p+pIhoHsMm9wdU=
 sigs.k8s.io/kustomize v2.0.3+incompatible h1:JUufWFNlI44MdtnjUqVnvh29rR37PQFzPbLXqhyOyX0=
 sigs.k8s.io/kustomize v2.0.3+incompatible/go.mod h1:MkjgH3RdOWrievjo6c9T245dYlB5QeXV4WCbnt/PEpU=
 sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI=

itest/starboard-operator/starboard_operator_test.go

@@ -91,6 +91,17 @@ var _ = Describe("Starboard Operator", func() {
 
 	// TODO Add a scenario for rolling update, i.e. create Deployment and then update its container image.
 
+	Describe("When operator is started", func() {
+
+		It("Should scan all nodes with CIS Kubernetes Benchmark checks", func() {
+			nodeList, err := kubeClientset.CoreV1().Nodes().List(context.Background(), metav1.ListOptions{})
+			Expect(err).ToNot(HaveOccurred())
+			for _, node := range nodeList.Items {
+				Eventually(HasCISKubeBenchReportFor(node), assertionTimeout).Should(BeTrue())
+			}
+		})
+
+	})
 })
 
 func HasActiveReplicaSet(namespace, name string) func() bool {
@@ -165,3 +176,13 @@ func GetActiveReplicaSetForDeployment(namespace, name string) (*appsv1.ReplicaSe
 	}
 	return nil, nil
 }
+
+func HasCISKubeBenchReportFor(node corev1.Node) func() bool {
+	return func() bool {
+		report, err := kubeBenchReportReader.FindByOwner(context.Background(), kube.Object{Kind: kube.KindNode, Name: node.Name})
+		if err != nil {
+			return false
+		}
+		return report != nil
+	}
+}

itest/starboard-operator/suite_test.go

@@ -5,6 +5,7 @@ import (
 	"testing"
 
 	"github.com/aquasecurity/starboard/pkg/generated/clientset/versioned"
+	"github.com/aquasecurity/starboard/pkg/kubebench"
 	"github.com/aquasecurity/starboard/pkg/operator"
 	"github.com/aquasecurity/starboard/pkg/operator/etc"
 	"github.com/aquasecurity/starboard/pkg/starboard"
@@ -13,6 +14,7 @@ import (
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/utils/pointer"
 	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/envtest"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
@@ -28,9 +30,14 @@ var (
 
 var (
 	kubeClientset      kubernetes.Interface
+	kubeClient         client.Client
 	starboardClientset versioned.Interface
 )
 
+var (
+	kubeBenchReportReader kubebench.Reader
+)
+
 func TestStarboardOperator(t *testing.T) {
 	if testing.Short() {
 		t.Skip("Skipping integration test")
@@ -51,9 +58,14 @@ var _ = BeforeSuite(func(done Done) {
 	kubeClientset, err = kubernetes.NewForConfig(kubeConfig)
 	Expect(err).ToNot(HaveOccurred())
 
+	kubeClient, err = client.New(kubeConfig, client.Options{Scheme: starboard.NewScheme()})
+	Expect(err).ToNot(HaveOccurred())
+
 	starboardClientset, err = versioned.NewForConfig(kubeConfig)
 	Expect(err).ToNot(HaveOccurred())
 
+	kubeBenchReportReader = kubebench.NewReadWriter(kubeClient)
+
 	testEnv = &envtest.Environment{
 		UseExistingCluster: pointer.BoolPtr(true),
 		Config:             kubeConfig,

itest/starboard/starboard_cli_test.go

@@ -1032,7 +1032,7 @@ var _ = Describe("Starboard CLI", func() {
 
 	})
 
-	Describe("Command run kube-bench", func() {
+	Describe("Command generate ciskubebenchreports", func() {
 
 		It("should run kube-bench", func() {
 			err := cmd.Run(versionInfo, []string{
@@ -1060,10 +1060,12 @@ var _ = Describe("Starboard CLI", func() {
 							kube.LabelResourceName: Equal(node.Name),
 						}),
 						"OwnerReferences": ConsistOf(metav1.OwnerReference{
-							APIVersion: "v1",
-							Kind:       "Node",
-							Name:       node.Name,
-							UID:        node.UID,
+							APIVersion:         "v1",
+							Kind:               "Node",
+							Name:               node.Name,
+							UID:                node.UID,
+							Controller:         pointer.BoolPtr(true),
+							BlockOwnerDeletion: pointer.BoolPtr(true),
 						}),
 					}),
 					"Report": MatchFields(IgnoreExtras, Fields{

pkg/kube/starboard.go

@@ -17,6 +17,7 @@ const (
 
 	LabelConfigAuditReportScan   = "configAuditReport.scanner"
 	LabelVulnerabilityReportScan = "vulnerabilityReport.scanner"
+	LabelKubeBenchReportScan     = "kubeBenchReport.scanner"
 
 	LabelK8SAppManagedBy = "app.kubernetes.io/managed-by"
 	AppStarboardOperator = "starboard-operator"

pkg/kubebench/scanner.go

@@ -121,7 +121,7 @@ func (s *Scanner) Scan(ctx context.Context, node corev1.Node) (v1alpha1.CISKubeB
 		Report: output,
 	}
 
-	err = controllerutil.SetOwnerReference(&node, &report, s.scheme)
+	err = controllerutil.SetControllerReference(&node, &report, s.scheme)
 	if err != nil {
 		return v1alpha1.CISKubeBenchReport{}, err
 	}