feat(misconf): Support custom data for rego policies for cloud

docs/docs/scanner/misconfiguration/custom/data.md

@@ -3,7 +3,7 @@
 Custom policies may require additional data in order to determine an answer.
 
 For example, an allowed list of resources that can be created. 
-Instead of hardcoding this information inside of your policy, Trivy allows passing paths to data files with the `--data` flag.
+Instead of hardcoding this information inside your policy, Trivy allows passing paths to data files with the `--data` flag.
 
 Given the following yaml file:
 

docs/docs/target/aws.md

@@ -103,5 +103,7 @@ Regardless of whether the cache is used or not, rules will be evaluated again wi
 
 You can write custom policies for Trivy to evaluate against your AWS account.
 These policies are written in [Rego](https://www.openpolicyagent.org/docs/latest/policy-language/), the same language used by [Open Policy Agent](https://www.openpolicyagent.org/).
-See the [Custom Policies](../scanner/misconfiguration/custom/index.md) page for more information.
+See the [Custom Policies](../scanner/misconfiguration/custom/index.md) page for more information on how to write custom policies.
 
+Custom policies in cloud scanning also support passing in custom data. This can be useful when you want to selectively enable/disable certain aspects of your cloud policies.
+See the [Custom Data](../scanner/misconfiguration/custom/data.md) page for more information on how to provide custom data to custom policies.

pkg/cloud/aws/commands/run_test.go

@@ -9,9 +9,9 @@ import (
 	"time"
 
 	defsecTypes "github.com/aquasecurity/defsec/pkg/types"
+	"github.com/aquasecurity/trivy/pkg/compliance/spec"
 
 	dbTypes "github.com/aquasecurity/trivy-db/pkg/types"
-	"github.com/aquasecurity/trivy/pkg/compliance/spec"
 	"github.com/aquasecurity/trivy/pkg/flag"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -287,6 +287,7 @@ const expectedS3ScanResult = `{
   ]
 }
 `
+
 const expectedCustomScanResult = `{
   "ArtifactName": "12345678",
   "ArtifactType": "aws_account",
@@ -303,13 +304,46 @@ const expectedCustomScanResult = `{
     }
   },
   "Results": [
+    {
+      "Target": "",
+      "Class": "config",
+      "Type": "cloud",
+      "MisconfSummary": {
+        "Successes": 0,
+        "Failures": 1,
+        "Exceptions": 0
+      },
+      "Misconfigurations": [
+        {
+          "Type": "AWS",
+          "Title": "Bad input data",
+          "Description": "Just failing rule with input data",
+          "Message": "Rego policy resulted in DENY",
+          "Namespace": "user.whatever",
+          "Query": "deny",
+          "Severity": "LOW",
+          "References": [
+            ""
+          ],
+          "Status": "FAIL",
+          "Layer": {},
+          "CauseMetadata": {
+            "Provider": "cloud",
+            "Service": "s3",
+            "Code": {
+              "Lines": null
+            }
+          }
+        }
+      ]
+    },
     {
       "Target": "arn:aws:s3:::examplebucket",
       "Class": "config",
       "Type": "cloud",
       "MisconfSummary": {
         "Successes": 1,
-        "Failures": 10,
+        "Failures": 9,
         "Exceptions": 0
       },
       "Misconfigurations": [
@@ -551,34 +585,13 @@ const expectedCustomScanResult = `{
               "Lines": null
             }
           }
-        },
-        {
-          "Type": "AWS",
-          "Title": "No example buckets",
-          "Description": "Buckets should not be named with \"example\" in the name",
-          "Message": "example bucket detected",
-          "Namespace": "user.whatever",
-          "Query": "deny",
-          "Severity": "LOW",
-          "References": [
-            ""
-          ],
-          "Status": "FAIL",
-          "Layer": {},
-          "CauseMetadata": {
-            "Resource": "arn:aws:s3:::examplebucket",
-            "Provider": "cloud",
-            "Service": "s3",
-            "Code": {
-              "Lines": null
-            }
-          }
         }
       ]
     }
   ]
 }
 `
+
 const expectedS3AndCloudTrailResult = `{
   "ArtifactName": "123456789",
   "ArtifactType": "aws_account",
@@ -958,8 +971,9 @@ const expectedS3AndCloudTrailResult = `{
 `
 
 func Test_Run(t *testing.T) {
-
-	regoDir := t.TempDir()
+	regoDir := filepath.Join("testdata", "Test_Run_Dir")
+	require.NoError(t, os.MkdirAll(regoDir, 0755))
+	defer os.RemoveAll(regoDir)
 
 	tests := []struct {
 		name         string
@@ -969,6 +983,7 @@ func Test_Run(t *testing.T) {
 		cacheContent string
 		regoPolicy   string
 		allServices  []string
+		inputData    string
 	}{
 		{
 			name: "fail without region",
@@ -1042,13 +1057,16 @@ func Test_Run(t *testing.T) {
 					PolicyNamespaces: []string{
 						"user",
 					},
+					DataPaths: []string{
+						filepath.Join(regoDir, "data"),
+					},
 					SkipPolicyUpdate: true,
 				},
 				MisconfOptions: flag.MisconfOptions{IncludeNonFailures: true},
 			},
 			regoPolicy: `# METADATA
-# title: No example buckets
-# description: Buckets should not be named with "example" in the name
+# title: Bad input data
+# description: Just failing rule with input data
 # scope: package
 # schemas:
 # - input: schema["input"]
@@ -1059,14 +1077,20 @@ func Test_Run(t *testing.T) {
 #     selector:
 #     - type: cloud
 package user.whatever
+import data.settings.DS123.foo
 
-deny[res] {
-	bucket := input.aws.s3.buckets[_]
-	contains(bucket.name.value, "example")
-	res := result.new("example bucket detected", bucket.name)
+deny {
+	foo == true
 }
 `,
-			cacheContent: "testdata/s3onlycache.json",
+			inputData: `{
+	"settings": {
+		"DS123": {
+			"foo": true
+		}
+	}
+}`,
+			cacheContent: filepath.Join("testdata", "s3onlycache.json"),
 			allServices:  []string{"s3"},
 			want:         expectedCustomScanResult,
 		},
@@ -1241,6 +1265,11 @@ Summary Report for compliance: my-custom-spec
 				require.NoError(t, os.WriteFile(filepath.Join(regoDir, "policies", "user.rego"), []byte(test.regoPolicy), 0644))
 			}
 
+			if test.inputData != "" {
+				require.NoError(t, os.MkdirAll(filepath.Join(regoDir, "data"), 0755))
+				require.NoError(t, os.WriteFile(filepath.Join(regoDir, "data", "data.json"), []byte(test.inputData), 0644))
+			}
+
 			if test.cacheContent != "" {
 				cacheRoot := t.TempDir()
 				test.options.CacheDir = cacheRoot
@@ -1250,7 +1279,7 @@ Summary Report for compliance: my-custom-spec
 				cacheData, err := os.ReadFile(test.cacheContent)
 				require.NoError(t, err, test.name)
 
-				require.NoError(t, os.WriteFile(cacheFile, []byte(cacheData), 0600))
+				require.NoError(t, os.WriteFile(cacheFile, cacheData, 0600))
 			}
 
 			err := Run(context.Background(), test.options)

pkg/cloud/aws/scanner/scanner.go

@@ -3,8 +3,11 @@ package scanner
 import (
 	"context"
 	"fmt"
+	"io/fs"
 	"strings"
 
+	"golang.org/x/xerrors"
+
 	"github.com/aquasecurity/defsec/pkg/framework"
 	"github.com/aquasecurity/defsec/pkg/scan"
 	"github.com/aquasecurity/defsec/pkg/scanners/cloud/aws"
@@ -14,6 +17,7 @@ import (
 	"github.com/aquasecurity/trivy/pkg/commands/operation"
 	"github.com/aquasecurity/trivy/pkg/flag"
 	"github.com/aquasecurity/trivy/pkg/log"
+	"github.com/aquasecurity/trivy/pkg/misconf"
 )
 
 type AWSScanner struct {
@@ -75,15 +79,24 @@ func (s *AWSScanner) Scan(ctx context.Context, option flag.Options) (scan.Result
 		scannerOpts = append(scannerOpts,
 			options.ScannerWithEmbeddedPolicies(false))
 	}
-	policyPaths = append(policyPaths, option.RegoOptions.PolicyPaths...)
+
+	var policyFS fs.FS
+	policyFS, policyPaths, err = misconf.CreatePolicyFS(append(policyPaths, option.RegoOptions.PolicyPaths...))
+	if err != nil {
+		return nil, false, xerrors.Errorf("unable to create policyfs: %w", err)
+	}
+
+	scannerOpts = append(scannerOpts, options.ScannerWithPolicyFilesystem(policyFS))
 	scannerOpts = append(scannerOpts, options.ScannerWithPolicyDirs(policyPaths...))
 
-	if len(option.RegoOptions.PolicyNamespaces) > 0 {
-		scannerOpts = append(
-			scannerOpts,
-			options.ScannerWithPolicyNamespaces(option.RegoOptions.PolicyNamespaces...),
-		)
+	dataFS, dataPaths, err := misconf.CreateDataFS(option.RegoOptions.DataPaths)
+	if err != nil {
+		log.Logger.Errorf("Could not load config data: %s", err)
 	}
+	scannerOpts = append(scannerOpts, options.ScannerWithDataDirs(dataPaths...))
+	scannerOpts = append(scannerOpts, options.ScannerWithDataFilesystem(dataFS))
+
+	scannerOpts = addPolicyNamespaces(option.RegoOptions.PolicyNamespaces, scannerOpts)
 
 	if option.Compliance.Spec.ID != "" {
 		scannerOpts = append(scannerOpts, options.ScannerWithSpec(option.Compliance.Spec.ID))
@@ -104,18 +117,9 @@ func (s *AWSScanner) Scan(ctx context.Context, option flag.Options) (scan.Result
 		}
 	}
 
-	var fullState *state.State
-	if previousState, err := awsCache.LoadState(); err == nil {
-		if freshState != nil {
-			fullState, err = previousState.Merge(freshState)
-			if err != nil {
-				return nil, false, err
-			}
-		} else {
-			fullState = previousState
-		}
-	} else {
-		fullState = freshState
+	fullState, err := createState(freshState, awsCache)
+	if err != nil {
+		return nil, false, err
 	}
 
 	if fullState == nil {
@@ -134,10 +138,36 @@ func (s *AWSScanner) Scan(ctx context.Context, option flag.Options) (scan.Result
 	return defsecResults, len(included) > 0, nil
 }
 
+func createState(freshState *state.State, awsCache *cache.Cache) (*state.State, error) {
+	var fullState *state.State
+	if previousState, err := awsCache.LoadState(); err == nil {
+		if freshState != nil {
+			fullState, err = previousState.Merge(freshState)
+			if err != nil {
+				return nil, err
+			}
+		} else {
+			fullState = previousState
+		}
+	} else {
+		fullState = freshState
+	}
+	return fullState, nil
+}
+
 type defsecLogger struct {
 }
 
 func (d *defsecLogger) Write(p []byte) (n int, err error) {
 	log.Logger.Debug("[defsec] " + strings.TrimSpace(string(p)))
 	return len(p), nil
 }
+func addPolicyNamespaces(namespaces []string, scannerOpts []options.ScannerOption) []options.ScannerOption {
+	if len(namespaces) > 0 {
+		scannerOpts = append(
+			scannerOpts,
+			options.ScannerWithPolicyNamespaces(namespaces...),
+		)
+	}
+	return scannerOpts
+}

pkg/commands/artifact/run.go

@@ -579,7 +579,7 @@ func initScannerConfig(opts flag.Options, cacheClient cache.Cache) (ScannerConfi
 			Trace:                   opts.Trace,
 			Namespaces:              append(opts.PolicyNamespaces, defaultPolicyNamespaces...),
 			PolicyPaths:             append(opts.PolicyPaths, downloadedPolicyPaths...),
-			DataPaths:               opts.DataPaths,
+			DataPaths:               append(opts.DataPaths, downloadedPolicyPaths...),
 			HelmValues:              opts.HelmValues,
 			HelmValueFiles:          opts.HelmValueFiles,
 			HelmFileValues:          opts.HelmFileValues,

pkg/misconf/scanner.go

@@ -200,15 +200,15 @@ func scannerOptions(t detection.FileType, opt ScannerOption) ([]options.ScannerO
 		options.ScannerWithEmbeddedPolicies(!opt.DisableEmbeddedPolicies),
 	}
 
-	policyFS, policyPaths, err := createPolicyFS(opt.PolicyPaths)
+	policyFS, policyPaths, err := CreatePolicyFS(opt.PolicyPaths)
 	if err != nil {
 		return nil, err
 	}
 	if policyFS != nil {
 		opts = append(opts, options.ScannerWithPolicyFilesystem(policyFS))
 	}
 
-	dataFS, dataPaths, err := createDataFS(opt.DataPaths, opt.K8sVersion)
+	dataFS, dataPaths, err := CreateDataFS(opt.DataPaths, opt.K8sVersion)
 	if err != nil {
 		return nil, err
 	}
@@ -284,7 +284,7 @@ func addHelmOpts(opts []options.ScannerOption, scannerOption ScannerOption) []op
 	return opts
 }
 
-func createPolicyFS(policyPaths []string) (fs.FS, []string, error) {
+func CreatePolicyFS(policyPaths []string) (fs.FS, []string, error) {
 	if len(policyPaths) == 0 {
 		return nil, nil, nil
 	}
@@ -306,11 +306,12 @@ func createPolicyFS(policyPaths []string) (fs.FS, []string, error) {
 	return mfs, policyPaths, nil
 }
 
-func createDataFS(dataPaths []string, k8sVersion string) (fs.FS, []string, error) {
+func CreateDataFS(dataPaths []string, options ...string) (fs.FS, []string, error) {
 	fsys := mapfs.New()
 
-	// Create a virtual file for Kubernetes scanning
-	if k8sVersion != "" {
+	// Check if k8sVersion is provided
+	if len(options) > 0 {
+		k8sVersion := options[0]
 		if err := fsys.MkdirAll("system", 0700); err != nil {
 			return nil, nil, err
 		}
@@ -319,13 +320,14 @@ func createDataFS(dataPaths []string, k8sVersion string) (fs.FS, []string, error
 			return nil, nil, err
 		}
 	}
+
 	for _, path := range dataPaths {
 		if err := fsys.CopyFilesUnder(path); err != nil {
 			return nil, nil, err
 		}
 	}
 
-	// data paths are no longer needed as fs.FS contains only needed files now.
+	// dataPaths are no longer needed as fs.FS contains only needed files now.
 	dataPaths = []string{"."}
 
 	return fsys, dataPaths, nil