feat(misconf): Support custom data for rego policies for cloud

docs/docs/scanner/misconfiguration/custom/data.md

@@ -3,7 +3,7 @@
 Custom policies may require additional data in order to determine an answer.
 
 For example, an allowed list of resources that can be created. 
-Instead of hardcoding this information inside of your policy, Trivy allows passing paths to data files with the `--data` flag.
+Instead of hardcoding this information inside your policy, Trivy allows passing paths to data files with the `--data` flag.
 
 Given the following yaml file:
 

docs/docs/target/aws.md

@@ -103,5 +103,7 @@ Regardless of whether the cache is used or not, rules will be evaluated again wi
 
 You can write custom policies for Trivy to evaluate against your AWS account.
 These policies are written in [Rego](https://www.openpolicyagent.org/docs/latest/policy-language/), the same language used by [Open Policy Agent](https://www.openpolicyagent.org/).
-See the [Custom Policies](../scanner/misconfiguration/custom/index.md) page for more information.
+See the [Custom Policies](../scanner/misconfiguration/custom/index.md) page for more information on how to write custom policies.
 
+Custom policies in cloud scanning also support passing in custom data. This can be useful when you want to selectively enable/disable certain aspects of your cloud policies.
+See the [Custom Data](../scanner/misconfiguration/custom/data.md) page for more information on how to provide custom data to custom policies.

go.mod

@@ -13,7 +13,7 @@ require (
 	github.com/NYTimes/gziphandler v1.1.1
 	github.com/alicebob/miniredis/v2 v2.30.3
 	github.com/aquasecurity/bolt-fixtures v0.0.0-20200903104109-d34e7f983986
-	github.com/aquasecurity/defsec v0.90.1
+	github.com/aquasecurity/defsec v0.90.2-0.20230705181508-367b32a91f64
 	github.com/aquasecurity/go-dep-parser v0.0.0-20230626110909-e7ea5097483b
 	github.com/aquasecurity/go-gem-version v0.0.0-20201115065557-8eed6fe000ce
 	github.com/aquasecurity/go-npm-version v0.0.0-20201110091526-0b796d180798

go.sum

@@ -321,8 +321,8 @@ github.com/apparentlymart/go-textseg/v13 v13.0.0 h1:Y+KvPE1NYz0xl601PVImeQfFyEy6
 github.com/apparentlymart/go-textseg/v13 v13.0.0/go.mod h1:ZK2fH7c4NqDTLtiYLvIkEghdlcqw7yxLeM89kiTRPUo=
 github.com/aquasecurity/bolt-fixtures v0.0.0-20200903104109-d34e7f983986 h1:2a30xLN2sUZcMXl50hg+PJCIDdJgIvIbVcKqLJ/ZrtM=
 github.com/aquasecurity/bolt-fixtures v0.0.0-20200903104109-d34e7f983986/go.mod h1:NT+jyeCzXk6vXR5MTkdn4z64TgGfE5HMLC8qfj5unl8=
-github.com/aquasecurity/defsec v0.90.1 h1:6c8bdv6tFnutDlY6V7uRrgZ3DqMmanPOy2VKVfmBYYM=
-github.com/aquasecurity/defsec v0.90.1/go.mod h1:ehFnrY3h2yJkd6EeHjPs2Y95431bHaFrMMurANDJumY=
+github.com/aquasecurity/defsec v0.90.2-0.20230705181508-367b32a91f64 h1:muM8+S+NM/V4OKEg+pBzOh33tAJKMz8YFGr6J/2bYxs=
+github.com/aquasecurity/defsec v0.90.2-0.20230705181508-367b32a91f64/go.mod h1:ehFnrY3h2yJkd6EeHjPs2Y95431bHaFrMMurANDJumY=
 github.com/aquasecurity/go-dep-parser v0.0.0-20230626110909-e7ea5097483b h1:9Ju7hWzTS8H9K/z1CqkJdZi+yxw1pZQZE11gVICtmTE=
 github.com/aquasecurity/go-dep-parser v0.0.0-20230626110909-e7ea5097483b/go.mod h1:VjG2wX19QDny5yKN+he0v9wuZjF0k+00173mh0FJCVU=
 github.com/aquasecurity/go-gem-version v0.0.0-20201115065557-8eed6fe000ce h1:QgBRgJvtEOBtUXilDb1MLi1p1MWoyFDXAu5DEUl5nwM=

pkg/cloud/aws/commands/run_test.go

@@ -9,9 +9,9 @@ import (
 	"time"
 
 	defsecTypes "github.com/aquasecurity/defsec/pkg/types"
+	"github.com/aquasecurity/trivy/pkg/compliance/spec"
 
 	dbTypes "github.com/aquasecurity/trivy-db/pkg/types"
-	"github.com/aquasecurity/trivy/pkg/compliance/spec"
 	"github.com/aquasecurity/trivy/pkg/flag"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -287,6 +287,7 @@ const expectedS3ScanResult = `{
   ]
 }
 `
+
 const expectedCustomScanResult = `{
   "ArtifactName": "12345678",
   "ArtifactType": "aws_account",
@@ -579,6 +580,7 @@ const expectedCustomScanResult = `{
   ]
 }
 `
+
 const expectedS3AndCloudTrailResult = `{
   "ArtifactName": "123456789",
   "ArtifactType": "aws_account",
@@ -969,6 +971,7 @@ func Test_Run(t *testing.T) {
 		cacheContent string
 		regoPolicy   string
 		allServices  []string
+		inputData    string
 	}{
 		{
 			name: "fail without region",
@@ -1059,6 +1062,7 @@ func Test_Run(t *testing.T) {
 #     selector:
 #     - type: cloud
 package user.whatever
+import data.settings.DS123.ignore_deletion_protection
 
 deny[res] {
 	bucket := input.aws.s3.buckets[_]
@@ -1250,7 +1254,7 @@ Summary Report for compliance: my-custom-spec
 				cacheData, err := os.ReadFile(test.cacheContent)
 				require.NoError(t, err, test.name)
 
-				require.NoError(t, os.WriteFile(cacheFile, []byte(cacheData), 0600))
+				require.NoError(t, os.WriteFile(cacheFile, cacheData, 0600))
 			}
 
 			err := Run(context.Background(), test.options)

pkg/cloud/aws/scanner/scanner.go

@@ -3,8 +3,11 @@ package scanner
 import (
 	"context"
 	"fmt"
+	"io/fs"
 	"strings"
 
+	"golang.org/x/xerrors"
+
 	"github.com/aquasecurity/defsec/pkg/framework"
 	"github.com/aquasecurity/defsec/pkg/scan"
 	"github.com/aquasecurity/defsec/pkg/scanners/cloud/aws"
@@ -14,6 +17,7 @@ import (
 	"github.com/aquasecurity/trivy/pkg/commands/operation"
 	"github.com/aquasecurity/trivy/pkg/flag"
 	"github.com/aquasecurity/trivy/pkg/log"
+	"github.com/aquasecurity/trivy/pkg/misconf"
 )
 
 type AWSScanner struct {
@@ -75,15 +79,24 @@ func (s *AWSScanner) Scan(ctx context.Context, option flag.Options) (scan.Result
 		scannerOpts = append(scannerOpts,
 			options.ScannerWithEmbeddedPolicies(false))
 	}
-	policyPaths = append(policyPaths, option.RegoOptions.PolicyPaths...)
+
+	var policyFS fs.FS
+	policyFS, policyPaths, err = misconf.CreatePolicyFS(append(policyPaths, option.RegoOptions.PolicyPaths...))
+	if err != nil {
+		return nil, false, xerrors.Errorf("unable to create policyfs: %w", err)
+	}
+
+	scannerOpts = append(scannerOpts, options.ScannerWithPolicyFilesystem(policyFS))
 	scannerOpts = append(scannerOpts, options.ScannerWithPolicyDirs(policyPaths...))
 
-	if len(option.RegoOptions.PolicyNamespaces) > 0 {
-		scannerOpts = append(
-			scannerOpts,
-			options.ScannerWithPolicyNamespaces(option.RegoOptions.PolicyNamespaces...),
-		)
+	dataFS, dataPaths, err := misconf.CreateDataFS(option.RegoOptions.DataPaths)
+	if err != nil {
+		log.Logger.Errorf("Could not load config data: %s", err)
 	}
+	scannerOpts = append(scannerOpts, options.ScannerWithDataDirs(dataPaths...))
+	scannerOpts = append(scannerOpts, options.ScannerWithDataFilesystem(dataFS))
+
+	scannerOpts = addPolicyNamespaces(option.RegoOptions.PolicyNamespaces, scannerOpts)
 
 	if option.Compliance.Spec.ID != "" {
 		scannerOpts = append(scannerOpts, options.ScannerWithSpec(option.Compliance.Spec.ID))
@@ -104,18 +117,9 @@ func (s *AWSScanner) Scan(ctx context.Context, option flag.Options) (scan.Result
 		}
 	}
 
-	var fullState *state.State
-	if previousState, err := awsCache.LoadState(); err == nil {
-		if freshState != nil {
-			fullState, err = previousState.Merge(freshState)
-			if err != nil {
-				return nil, false, err
-			}
-		} else {
-			fullState = previousState
-		}
-	} else {
-		fullState = freshState
+	fullState, err := createState(freshState, awsCache)
+	if err != nil {
+		return nil, false, err
 	}
 
 	if fullState == nil {
@@ -134,10 +138,36 @@ func (s *AWSScanner) Scan(ctx context.Context, option flag.Options) (scan.Result
 	return defsecResults, len(included) > 0, nil
 }
 
+func createState(freshState *state.State, awsCache *cache.Cache) (*state.State, error) {
+	var fullState *state.State
+	if previousState, err := awsCache.LoadState(); err == nil {
+		if freshState != nil {
+			fullState, err = previousState.Merge(freshState)
+			if err != nil {
+				return nil, err
+			}
+		} else {
+			fullState = previousState
+		}
+	} else {
+		fullState = freshState
+	}
+	return fullState, nil
+}
+
 type defsecLogger struct {
 }
 
 func (d *defsecLogger) Write(p []byte) (n int, err error) {
 	log.Logger.Debug("[defsec] " + strings.TrimSpace(string(p)))
 	return len(p), nil
 }
+func addPolicyNamespaces(namespaces []string, scannerOpts []options.ScannerOption) []options.ScannerOption {
+	if len(namespaces) > 0 {
+		scannerOpts = append(
+			scannerOpts,
+			options.ScannerWithPolicyNamespaces(namespaces...),
+		)
+	}
+	return scannerOpts
+}

pkg/commands/artifact/run.go

@@ -579,7 +579,7 @@ func initScannerConfig(opts flag.Options, cacheClient cache.Cache) (ScannerConfi
 			Trace:                   opts.Trace,
 			Namespaces:              append(opts.PolicyNamespaces, defaultPolicyNamespaces...),
 			PolicyPaths:             append(opts.PolicyPaths, downloadedPolicyPaths...),
-			DataPaths:               opts.DataPaths,
+			DataPaths:               append(opts.DataPaths, downloadedPolicyPaths...),
 			HelmValues:              opts.HelmValues,
 			HelmValueFiles:          opts.HelmValueFiles,
 			HelmFileValues:          opts.HelmFileValues,

pkg/misconf/scanner.go

@@ -200,15 +200,15 @@ func scannerOptions(t detection.FileType, opt ScannerOption) ([]options.ScannerO
 		options.ScannerWithEmbeddedPolicies(!opt.DisableEmbeddedPolicies),
 	}
 
-	policyFS, policyPaths, err := createPolicyFS(opt.PolicyPaths)
+	policyFS, policyPaths, err := CreatePolicyFS(opt.PolicyPaths)
 	if err != nil {
 		return nil, err
 	}
 	if policyFS != nil {
 		opts = append(opts, options.ScannerWithPolicyFilesystem(policyFS))
 	}
 
-	dataFS, dataPaths, err := createDataFS(opt.DataPaths, opt.K8sVersion)
+	dataFS, dataPaths, err := CreateDataFS(opt.DataPaths, opt.K8sVersion)
 	if err != nil {
 		return nil, err
 	}
@@ -284,7 +284,7 @@ func addHelmOpts(opts []options.ScannerOption, scannerOption ScannerOption) []op
 	return opts
 }
 
-func createPolicyFS(policyPaths []string) (fs.FS, []string, error) {
+func CreatePolicyFS(policyPaths []string) (fs.FS, []string, error) {
 	if len(policyPaths) == 0 {
 		return nil, nil, nil
 	}
@@ -306,11 +306,12 @@ func createPolicyFS(policyPaths []string) (fs.FS, []string, error) {
 	return mfs, policyPaths, nil
 }
 
-func createDataFS(dataPaths []string, k8sVersion string) (fs.FS, []string, error) {
+func CreateDataFS(dataPaths []string, options ...string) (fs.FS, []string, error) {
 	fsys := mapfs.New()
 
-	// Create a virtual file for Kubernetes scanning
-	if k8sVersion != "" {
+	// Check if k8sVersion is provided
+	if len(options) > 0 {
+		k8sVersion := options[0]
 		if err := fsys.MkdirAll("system", 0700); err != nil {
 			return nil, nil, err
 		}
@@ -319,13 +320,14 @@ func createDataFS(dataPaths []string, k8sVersion string) (fs.FS, []string, error
 			return nil, nil, err
 		}
 	}
+
 	for _, path := range dataPaths {
 		if err := fsys.CopyFilesUnder(path); err != nil {
 			return nil, nil, err
 		}
 	}
 
-	// data paths are no longer needed as fs.FS contains only needed files now.
+	// dataPaths are no longer needed as fs.FS contains only needed files now.
 	dataPaths = []string{"."}
 
 	return fsys, dataPaths, nil

pkg/misconf/scanner_test.go

@@ -2,6 +2,7 @@ package misconf
 
 import (
 	"context"
+	"io/fs"
 	"os"
 	"path/filepath"
 	"testing"
@@ -156,14 +157,29 @@ func Test_createPolicyFS(t *testing.T) {
 	t.Run("outside pwd", func(t *testing.T) {
 		tmpDir := t.TempDir()
 		require.NoError(t, os.MkdirAll(filepath.Join(tmpDir, "subdir/testdir"), 0750))
-		f, got, err := createPolicyFS([]string{filepath.Join(tmpDir, "subdir/testdir")})
-		require.NoError(t, err)
-		assert.Equal(t, []string{"."}, got)
+		f, got, err := CreatePolicyFS([]string{filepath.Join(tmpDir, "subdir/testdir")})
+		assertFS(t, tmpDir, f, got, err)
+	})
+}
 
-		d, err := f.Open(tmpDir)
-		require.NoError(t, err)
-		stat, err := d.Stat()
-		require.NoError(t, err)
-		assert.True(t, stat.IsDir())
+func Test_CreateDataFS(t *testing.T) {
+	t.Run("outside pwd", func(t *testing.T) {
+		tmpDir := t.TempDir()
+		require.NoError(t, os.MkdirAll(filepath.Join(tmpDir, "subdir/testdir"), 0750))
+		f, got, err := CreateDataFS([]string{filepath.Join(tmpDir, "subdir/testdir")})
+		assertFS(t, tmpDir, f, got, err)
 	})
 }
+
+func assertFS(t *testing.T, tmpDir string, f fs.FS, got []string, err error) {
+	t.Helper()
+
+	require.NoError(t, err)
+	assert.Equal(t, []string{"."}, got)
+
+	d, err := f.Open(tmpDir)
+	require.NoError(t, err)
+	stat, err := d.Stat()
+	require.NoError(t, err)
+	assert.True(t, stat.IsDir())
+}