feat: Integrate Conftest as ConfigAuditReports scanner (#417)

Signed-off-by: Daniel Pacak <pacak.daniel@gmail.com>

CONTRIBUTING.md

@@ -227,6 +227,8 @@ started with a basic development workflow. For other install modes see [Operator
      OPERATOR_CIS_KUBERNETES_BENCHMARK_ENABLED=true \
      OPERATOR_VULNERABILITY_SCANNER_ENABLED=true \
      OPERATOR_CONFIG_AUDIT_SCANNER_ENABLED=true \
+     OPERATOR_BATCH_DELETE_LIMIT=3 \
+     OPERATOR_BATCH_DELETE_DELAY="30s" \
      go run cmd/starboard-operator/main.go
    ```
 

Makefile

@@ -72,6 +72,7 @@ itests-starboard: check-env get-ginkgo
 	github.com/aquasecurity/starboard/pkg/kubehunter,\
 	github.com/aquasecurity/starboard/pkg/plugin/trivy,\
 	github.com/aquasecurity/starboard/pkg/plugin/polaris,\
+	github.com/aquasecurity/starboard/pkg/plugin/conftest,\
 	github.com/aquasecurity/starboard/pkg/configauditreport,\
 	github.com/aquasecurity/starboard/pkg/vulnerabilityreport \
 	./itest/starboard
@@ -89,6 +90,7 @@ itests-starboard-operator: check-env get-ginkgo
 	github.com/aquasecurity/starboard/pkg/plugin,\
 	github.com/aquasecurity/starboard/pkg/plugin/trivy,\
 	github.com/aquasecurity/starboard/pkg/plugin/polaris,\
+	github.com/aquasecurity/starboard/pkg/plugin/conftest,\
 	github.com/aquasecurity/starboard/pkg/configauditreport,\
 	github.com/aquasecurity/starboard/pkg/vulnerabilityreport,\
 	github.com/aquasecurity/starboard/pkg/kubebench \

deploy/static/03-starboard-operator.clusterrole.yaml

@@ -81,6 +81,7 @@ rules:
       - watch
       - create
       - update
+      - delete
   - apiGroups:
       - coordination.k8s.io
     resources:

deploy/static/05-starboard-operator.config.yaml

@@ -6,6 +6,7 @@ metadata:
   namespace: starboard-operator
 data:
   vulnerabilityReports.scanner: Trivy
+  configAuditReports.scanner: Polaris
   trivy.severity: UNKNOWN,LOW,MEDIUM,HIGH,CRITICAL
   trivy.imageRef: docker.io/aquasec/trivy:0.16.0
   trivy.mode: Standalone
@@ -195,6 +196,7 @@ data:
         - kube-hunter
         rules:
         - runAsRootAllowed
+  conftest.imageRef: openpolicyagent/conftest:v0.23.0
   kube-bench.imageRef: docker.io/aquasec/kube-bench:0.5.0
 ---
 apiVersion: v1

pkg/cmd/scan_vulnerabilities.go

@@ -98,11 +98,17 @@ func ScanVulnerabilityReports(buildInfo starboard.BuildInfo, cf *genericclioptio
 		if err != nil {
 			return err
 		}
-		instance, err := plugin.GetVulnerabilityReportPlugin(buildInfo, starboardConfig)
+		plugin, err := plugin.NewResolver().
+			WithBuildInfo(buildInfo).
+			WithNamespace(starboard.NamespaceName).
+			WithServiceAccountName(starboard.ServiceAccountName).
+			WithConfig(starboardConfig).
+			WithClient(kubeClient).
+			GetVulnerabilityPlugin()
 		if err != nil {
 			return err
 		}
-		scanner := vulnerabilityreport.NewScanner(kubeClientset, kubeClient, opts, instance)
+		scanner := vulnerabilityreport.NewScanner(kubeClientset, kubeClient, opts, plugin)
 		reports, err := scanner.Scan(ctx, workload)
 		if err != nil {
 			return err

pkg/configauditreport/builder.go

@@ -14,6 +14,7 @@ import (
 type Builder interface {
 	Controller(controller metav1.Object) Builder
 	PodSpecHash(hash string) Builder
+	PluginConfigHash(hash string) Builder
 	Result(result v1alpha1.ConfigAuditResult) Builder
 	Get() (v1alpha1.ConfigAuditReport, error)
 }
@@ -28,6 +29,7 @@ type builder struct {
 	scheme     *runtime.Scheme
 	controller metav1.Object
 	hash       string
+	configHash string
 	result     v1alpha1.ConfigAuditResult
 }
 
@@ -41,6 +43,11 @@ func (b *builder) PodSpecHash(hash string) Builder {
 	return b
 }
 
+func (b *builder) PluginConfigHash(hash string) Builder {
+	b.configHash = hash
+	return b
+}
+
 func (b *builder) Result(result v1alpha1.ConfigAuditResult) Builder {
 	b.result = result
 	return b
@@ -71,6 +78,10 @@ func (b *builder) Get() (v1alpha1.ConfigAuditReport, error) {
 		labels[kube.LabelPodSpecHash] = b.hash
 	}
 
+	if b.configHash != "" {
+		labels[kube.LabelPluginConfigHash] = b.configHash
+	}
+
 	reportName, err := b.reportName()
 	if err != nil {
 		return v1alpha1.ConfigAuditReport{}, err

pkg/configauditreport/builder_test.go

@@ -5,15 +5,16 @@ import (
 
 	"github.com/aquasecurity/starboard/pkg/apis/aquasecurity/v1alpha1"
 	"github.com/aquasecurity/starboard/pkg/configauditreport"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
+	"github.com/onsi/gomega"
 	appsv1 "k8s.io/api/apps/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/utils/pointer"
 )
 
 func TestBuilder(t *testing.T) {
+	g := gomega.NewGomegaWithT(t)
+
 	report, err := configauditreport.NewBuilder(scheme.Scheme).
 		Controller(&appsv1.ReplicaSet{
 			ObjectMeta: metav1.ObjectMeta{
@@ -22,10 +23,11 @@ func TestBuilder(t *testing.T) {
 			},
 		}).
 		PodSpecHash("xyz").
+		PluginConfigHash("nop").
 		Result(v1alpha1.ConfigAuditResult{}).Get()
 
-	require.NoError(t, err)
-	assert.Equal(t, v1alpha1.ConfigAuditReport{
+	g.Expect(err).ToNot(gomega.HaveOccurred())
+	g.Expect(report).To(gomega.Equal(v1alpha1.ConfigAuditReport{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      "replicaset-some-owner",
 			Namespace: "qa",
@@ -43,8 +45,9 @@ func TestBuilder(t *testing.T) {
 				"starboard.resource.name":      "some-owner",
 				"starboard.resource.namespace": "qa",
 				"pod-spec-hash":                "xyz",
+				"plugin-config-hash":           "nop",
 			},
 		},
 		Report: v1alpha1.ConfigAuditResult{},
-	}, report)
+	}))
 }

pkg/configauditreport/doc.go

@@ -1,2 +1,2 @@
-// This package provides primitives for working with Kubernetes workload configuration checkers.
+// The configauditreport package provides primitives for working with Kubernetes workload configuration checkers.
 package configauditreport

pkg/configauditreport/plugin.go

@@ -28,4 +28,8 @@ type Plugin interface {
 	// GetContainerName returns the name of the container in a pod created by a scan job
 	// to read logs from.
 	GetContainerName() string
+
+	// GetConfigHash returns hash of the plugin's configuration settings. The computed hash
+	// is used to invalidate v1alpha1.ConfigAuditReport object whenever configuration changes.
+	GetConfigHash(ctx starboard.PluginContext) (string, error)
 }

pkg/kube/starboard.go

@@ -12,8 +12,9 @@ const (
 	LabelResourceName      = "starboard.resource.name"
 	LabelResourceNamespace = "starboard.resource.namespace"
 
-	LabelContainerName = "starboard.container.name"
-	LabelPodSpecHash   = "pod-spec-hash"
+	LabelContainerName    = "starboard.container.name"
+	LabelPodSpecHash      = "pod-spec-hash"
+	LabelPluginConfigHash = "plugin-config-hash"
 
 	LabelConfigAuditReportScan   = "configAuditReport.scanner"
 	LabelVulnerabilityReportScan = "vulnerabilityReport.scanner"

pkg/operator/controller/configauditreport.go

@@ -120,11 +120,15 @@ func (r *ConfigAuditReportReconciler) reconcileWorkload(workloadKind kube.Kind)
 			return ctrl.Result{}, err
 		}
 		podSpecHash := kube.ComputeHash(podSpec)
+		pluginConfigHash, err := r.Plugin.GetConfigHash(r.PluginContext)
+		if err != nil {
+			return ctrl.Result{}, err
+		}
 
-		log = log.WithValues("podSpecHash", podSpecHash)
+		log = log.WithValues("podSpecHash", podSpecHash, "pluginConfigHash", pluginConfigHash)
 
 		log.V(1).Info("Checking whether configuration audit report exists")
-		hasReport, err := r.hasReport(ctx, workloadPartial, podSpecHash)
+		hasReport, err := r.hasReport(ctx, workloadPartial, podSpecHash, pluginConfigHash)
 		if err != nil {
 			return ctrl.Result{}, err
 		}
@@ -194,15 +198,14 @@ func (r *ConfigAuditReportReconciler) reconcileWorkload(workloadKind kube.Kind)
 	}
 }
 
-func (r *ConfigAuditReportReconciler) hasReport(ctx context.Context, owner kube.Object, hash string) (bool, error) {
+func (r *ConfigAuditReportReconciler) hasReport(ctx context.Context, owner kube.Object, podSpecHash string, pluginConfigHash string) (bool, error) {
 	report, err := r.ReadWriter.FindByOwner(ctx, owner)
 	if err != nil {
 		return false, err
 	}
 	if report != nil {
-		if report.Labels[kube.LabelPodSpecHash] == hash {
-			return true, nil
-		}
+		return report.Labels[kube.LabelPodSpecHash] == podSpecHash &&
+			report.Labels[kube.LabelPluginConfigHash] == pluginConfigHash, nil
 	}
 	return false, nil
 }
@@ -227,13 +230,18 @@ func (r *ConfigAuditReportReconciler) getScanJobName(workload kube.Object) strin
 	return fmt.Sprintf("scan-configauditreport-%s", kube.ComputeHash(workload))
 }
 
-func (r *ConfigAuditReportReconciler) getScanJob(workload kube.Object, obj client.Object, hash string) (*batchv1.Job, []*corev1.Secret, error) {
+func (r *ConfigAuditReportReconciler) getScanJob(workload kube.Object, obj client.Object, podSpecHash string) (*batchv1.Job, []*corev1.Secret, error) {
 	jobSpec, secrets, err := r.Plugin.GetScanJobSpec(r.PluginContext, obj)
 
 	if err != nil {
 		return nil, nil, err
 	}
 
+	configHash, err := r.Plugin.GetConfigHash(r.PluginContext)
+	if err != nil {
+		return nil, nil, err
+	}
+
 	return &batchv1.Job{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      r.getScanJobName(workload),
@@ -243,7 +251,8 @@ func (r *ConfigAuditReportReconciler) getScanJob(workload kube.Object, obj clien
 				kube.LabelResourceName:          workload.Name,
 				kube.LabelResourceNamespace:     workload.Namespace,
 				kube.LabelK8SAppManagedBy:       kube.AppStarboardOperator,
-				kube.LabelPodSpecHash:           hash,
+				kube.LabelPodSpecHash:           podSpecHash,
+				kube.LabelPluginConfigHash:      configHash,
 				kube.LabelConfigAuditReportScan: "true",
 			},
 		},
@@ -258,7 +267,8 @@ func (r *ConfigAuditReportReconciler) getScanJob(workload kube.Object, obj clien
 						kube.LabelResourceName:          workload.Name,
 						kube.LabelResourceNamespace:     workload.Namespace,
 						kube.LabelK8SAppManagedBy:       kube.AppStarboardOperator,
-						kube.LabelPodSpecHash:           hash,
+						kube.LabelPodSpecHash:           podSpecHash,
+						kube.LabelPluginConfigHash:      configHash,
 						kube.LabelConfigAuditReportScan: "true",
 					},
 				},
@@ -321,8 +331,12 @@ func (r *ConfigAuditReportReconciler) processCompleteScanJob(ctx context.Context
 	if !ok {
 		return fmt.Errorf("expected label %s not set", kube.LabelPodSpecHash)
 	}
+	pluginConfigHash, ok := job.Labels[kube.LabelPluginConfigHash]
+	if !ok {
+		return fmt.Errorf("expected label %s not set", kube.LabelPluginConfigHash)
+	}
 
-	hasConfigAuditReport, err := r.hasReport(ctx, owner, podSpecHash)
+	hasConfigAuditReport, err := r.hasReport(ctx, owner, podSpecHash, pluginConfigHash)
 	if err != nil {
 		return err
 	}
@@ -346,6 +360,7 @@ func (r *ConfigAuditReportReconciler) processCompleteScanJob(ctx context.Context
 	report, err := configauditreport.NewBuilder(r.Client.Scheme()).
 		Controller(ownerObj).
 		PodSpecHash(podSpecHash).
+		PluginConfigHash(pluginConfigHash).
 		Result(result).
 		Get()
 	if err != nil {

pkg/operator/controller/plugins_config.go

@@ -0,0 +1,95 @@
+package controller
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/aquasecurity/starboard/pkg/apis/aquasecurity/v1alpha1"
+	"github.com/aquasecurity/starboard/pkg/configauditreport"
+	"github.com/aquasecurity/starboard/pkg/ext"
+	"github.com/aquasecurity/starboard/pkg/kube"
+	"github.com/aquasecurity/starboard/pkg/operator/etc"
+	"github.com/aquasecurity/starboard/pkg/operator/predicate"
+	"github.com/aquasecurity/starboard/pkg/starboard"
+	"github.com/go-logr/logr"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/labels"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/builder"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+type PluginsConfigReconciler struct {
+	logr.Logger
+	etc.Config
+	client.Client
+	starboard.PluginContext
+	configauditreport.Plugin
+}
+
+func (r *PluginsConfigReconciler) SetupWithManager(mgr ctrl.Manager) error {
+	return ctrl.NewControllerManagedBy(mgr).
+		For(&corev1.ConfigMap{}, builder.WithPredicates(
+			predicate.Not(predicate.IsBeingTerminated),
+			predicate.InNamespace(r.Config.Namespace))).
+		Complete(r)
+}
+
+func (r *PluginsConfigReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	log := r.Logger.WithValues("configMap", req.NamespacedName)
+
+	// TODO Use Predicate instead
+	if req.Name != strings.ToLower("starboard-"+r.PluginContext.GetName()+"config") {
+		return ctrl.Result{}, nil
+	}
+
+	cm := &corev1.ConfigMap{}
+
+	err := r.Client.Get(ctx, req.NamespacedName, cm)
+	if err != nil {
+		if errors.IsNotFound(err) {
+			log.V(1).Info("Ignoring cached ConfigMap that must have been deleted")
+			return ctrl.Result{}, nil
+		}
+		return ctrl.Result{}, fmt.Errorf("getting ConfigMap from cache: %w", err)
+	}
+
+	configHash, err := r.Plugin.GetConfigHash(r.PluginContext)
+	if err != nil {
+		return ctrl.Result{}, err
+	}
+
+	labelSelector, err := labels.Parse(fmt.Sprintf("%s != %s", kube.LabelPluginConfigHash, configHash))
+	if err != nil {
+		return ctrl.Result{}, err
+	}
+
+	var reportList v1alpha1.ConfigAuditReportList
+	err = r.Client.List(ctx, &reportList,
+		client.Limit(r.Config.BatchDeleteLimit+1), // TODO The limit is not respected https://github.com/kubernetes-sigs/controller-runtime/issues/1422
+		client.MatchingLabelsSelector{Selector: labelSelector})
+	if err != nil {
+		return ctrl.Result{}, fmt.Errorf("listing reports: %w", err)
+	}
+
+	log.V(1).Info("Listing ConfigAuditReports",
+		"reportsCount", len(reportList.Items),
+		"batchDeleteLimit", r.Config.BatchDeleteLimit)
+
+	for i := 0; i < ext.MinInt(r.Config.BatchDeleteLimit, len(reportList.Items)); i++ {
+		report := reportList.Items[i]
+		log.V(1).Info("Deleting ConfigAuditReport", "report", report.Namespace+"/"+report.Name)
+		err := r.Client.Delete(ctx, &report)
+		if err != nil {
+			return ctrl.Result{}, err
+		}
+	}
+	if len(reportList.Items)-r.Config.BatchDeleteLimit > 0 {
+		// TODO Calculate RequeueAfter based on average scan duration?
+		return ctrl.Result{RequeueAfter: r.Config.BatchDeleteDelay}, nil
+	}
+
+	return ctrl.Result{}, nil
+}

pkg/operator/etc/config.go

@@ -23,6 +23,8 @@ type Config struct {
 	ConfigAuditScannerEnabled     bool          `env:"OPERATOR_CONFIG_AUDIT_SCANNER_ENABLED" envDefault:"true"`
 	LeaderElectionEnabled         bool          `env:"OPERATOR_LEADER_ELECTION_ENABLED" envDefault:"false"`
 	LeaderElectionID              string        `env:"OPERATOR_LEADER_ELECTION_ID" envDefault:"starboard-operator"`
+	BatchDeleteLimit              int           `env:"OPERATOR_BATCH_DELETE_LIMIT" envDefault:"10"`
+	BatchDeleteDelay              time.Duration `env:"OPERATOR_BATCH_DELETE_DELAY" envDefault:"10s"`
 }
 
 func GetOperatorConfig() (Config, error) {

pkg/operator/operator.go

@@ -180,6 +180,16 @@ func Run(buildInfo starboard.BuildInfo, operatorConfig etc.Config) error {
 		}).SetupWithManager(mgr); err != nil {
 			return fmt.Errorf("unable to setup configauditreport reconciler: %w", err)
 		}
+
+		if err = (&controller.PluginsConfigReconciler{
+			Logger:        ctrl.Log.WithName("reconciler").WithName("pluginsconfig"),
+			Config:        operatorConfig,
+			Client:        mgr.GetClient(),
+			Plugin:        plugin,
+			PluginContext: pluginContext,
+		}).SetupWithManager(mgr); err != nil {
+			return fmt.Errorf("unable to setup %T: %w", controller.PluginsConfigReconciler{}, err)
+		}
 	}
 
 	if operatorConfig.CISKubernetesBenchmarkEnabled {