diff --git a/pkg/operator/operator.go b/pkg/operator/operator.go
index 9c40215f3..3fbc321e7 100644
--- a/pkg/operator/operator.go
+++ b/pkg/operator/operator.go
@@ -207,9 +207,12 @@ func Start(ctx context.Context, buildInfo trivyoperator.BuildInfo, operatorConfi
 			return fmt.Errorf("unable to setup scan job  reconciler: %w", err)
 		}
 	}
-	policyLoader, err := buildPolicyLoader(trivyOperatorConfig)
-	if err != nil {
-		return fmt.Errorf("unable to constract policy loader: %w", err)
+	var policyLoader policy.Loader
+	if operatorConfig.ConfigAuditScannerEnabled {
+		policyLoader, err = buildPolicyLoader(trivyOperatorConfig)
+		if err != nil {
+			return fmt.Errorf("unable to constract policy loader: %w", err)
+		}
 	}
 
 	if operatorConfig.ScannerReportTTL != nil {