Skip to content

Commit

Permalink
feat(misconf): Support custom data for rego policies for cloud (aquas…
Browse files Browse the repository at this point in the history 
…ecurity#4745)

* feat(misconf): Support custom data for cloud policies

Signed-off-by: Simar <simar@linux.com>

* use policyfs

Signed-off-by: Simar <simar@linux.com>

* refactor to reduce cyclomatic complexity

Signed-off-by: Simar <simar@linux.com>

* bump defsec

* update docs

Signed-off-by: Simar <simar@linux.com>

* update test assertion

Signed-off-by: Simar <simar@linux.com>

* update test

Need this as OPA is currently broken on Windows

open-policy-agent/opa#4521

Signed-off-by: Simar <simar@linux.com>

* fix data path

* fix(mapfs): convert volume names into dirs

* revert creating temp dirs

---------

Signed-off-by: Simar <simar@linux.com>
Co-authored-by: Teppei Fukuda <knqyf263@gmail.com>
  • Loading branch information
@simar7 @knqyf263
simar7 and knqyf263 committed Jul 17, 2023
1 parent 88243a0 commit 4bc8d29
Show file tree
Hide file tree
Showing 8 changed files with 152 additions and 74 deletions.
2 changes: 1 addition & 1 deletion docs/docs/scanner/misconfiguration/custom/data.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
Custom policies may require additional data in order to determine an answer.

For example, an allowed list of resources that can be created.
Instead of hardcoding this information inside of your policy, Trivy allows passing paths to data files with the `--data` flag.
Instead of hardcoding this information inside your policy, Trivy allows passing paths to data files with the `--data` flag.

Given the following yaml file:

Expand Down
4 changes: 3 additions & 1 deletion docs/docs/target/aws.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,5 +103,7 @@ Regardless of whether the cache is used or not, rules will be evaluated again wi

You can write custom policies for Trivy to evaluate against your AWS account.
These policies are written in [Rego](https://www.openpolicyagent.org/docs/latest/policy-language/), the same language used by [Open Policy Agent](https://www.openpolicyagent.org/).
See the [Custom Policies](../scanner/misconfiguration/custom/index.md) page for more information.
See the [Custom Policies](../scanner/misconfiguration/custom/index.md) page for more information on how to write custom policies.

Custom policies in cloud scanning also support passing in custom data. This can be useful when you want to selectively enable/disable certain aspects of your cloud policies.
See the [Custom Data](../scanner/misconfiguration/custom/data.md) page for more information on how to provide custom data to custom policies.
93 changes: 60 additions & 33 deletions pkg/cloud/aws/commands/run_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,9 @@ import (
"time"

defsecTypes "github.com/aquasecurity/defsec/pkg/types"
"github.com/aquasecurity/trivy/pkg/compliance/spec"

dbTypes "github.com/aquasecurity/trivy-db/pkg/types"
"github.com/aquasecurity/trivy/pkg/compliance/spec"
"github.com/aquasecurity/trivy/pkg/flag"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
Expand Down Expand Up @@ -287,6 +287,7 @@ const expectedS3ScanResult = `{
]
}
`

const expectedCustomScanResult = `{
"ArtifactName": "12345678",
"ArtifactType": "aws_account",
Expand All @@ -303,13 +304,46 @@ const expectedCustomScanResult = `{
}
},
"Results": [
{
"Target": "",
"Class": "config",
"Type": "cloud",
"MisconfSummary": {
"Successes": 0,
"Failures": 1,
"Exceptions": 0
},
"Misconfigurations": [
{
"Type": "AWS",
"Title": "Bad input data",
"Description": "Just failing rule with input data",
"Message": "Rego policy resulted in DENY",
"Namespace": "user.whatever",
"Query": "deny",
"Severity": "LOW",
"References": [
""
],
"Status": "FAIL",
"Layer": {},
"CauseMetadata": {
"Provider": "cloud",
"Service": "s3",
"Code": {
"Lines": null
}
}
}
]
},
{
"Target": "arn:aws:s3:::examplebucket",
"Class": "config",
"Type": "cloud",
"MisconfSummary": {
"Successes": 1,
"Failures": 10,
"Failures": 9,
"Exceptions": 0
},
"Misconfigurations": [
Expand Down Expand Up @@ -551,34 +585,13 @@ const expectedCustomScanResult = `{
"Lines": null
}
}
},
{
"Type": "AWS",
"Title": "No example buckets",
"Description": "Buckets should not be named with \"example\" in the name",
"Message": "example bucket detected",
"Namespace": "user.whatever",
"Query": "deny",
"Severity": "LOW",
"References": [
""
],
"Status": "FAIL",
"Layer": {},
"CauseMetadata": {
"Resource": "arn:aws:s3:::examplebucket",
"Provider": "cloud",
"Service": "s3",
"Code": {
"Lines": null
}
}
}
]
}
]
}
`

const expectedS3AndCloudTrailResult = `{
"ArtifactName": "123456789",
"ArtifactType": "aws_account",
Expand Down Expand Up @@ -958,7 +971,6 @@ const expectedS3AndCloudTrailResult = `{
`

func Test_Run(t *testing.T) {

regoDir := t.TempDir()

tests := []struct {
Expand All @@ -969,6 +981,7 @@ func Test_Run(t *testing.T) {
cacheContent string
regoPolicy string
allServices []string
inputData string
}{
{
name: "fail without region",
Expand Down Expand Up @@ -1042,13 +1055,16 @@ func Test_Run(t *testing.T) {
PolicyNamespaces: []string{
"user",
},
DataPaths: []string{
filepath.Join(regoDir, "data"),
},
SkipPolicyUpdate: true,
},
MisconfOptions: flag.MisconfOptions{IncludeNonFailures: true},
},
regoPolicy: `# METADATA
# title: No example buckets
# description: Buckets should not be named with "example" in the name
# title: Bad input data
# description: Just failing rule with input data
# scope: package
# schemas:
# - input: schema["input"]
Expand All @@ -1059,14 +1075,20 @@ func Test_Run(t *testing.T) {
# selector:
# - type: cloud
package user.whatever
import data.settings.DS123.foo
deny[res] {
bucket := input.aws.s3.buckets[_]
contains(bucket.name.value, "example")
res := result.new("example bucket detected", bucket.name)
deny {
foo == true
}
`,
cacheContent: "testdata/s3onlycache.json",
inputData: `{
"settings": {
"DS123": {
"foo": true
}
}
}`,
cacheContent: filepath.Join("testdata", "s3onlycache.json"),
allServices: []string{"s3"},
want: expectedCustomScanResult,
},
Expand Down Expand Up @@ -1241,6 +1263,11 @@ Summary Report for compliance: my-custom-spec
require.NoError(t, os.WriteFile(filepath.Join(regoDir, "policies", "user.rego"), []byte(test.regoPolicy), 0644))
}

if test.inputData != "" {
require.NoError(t, os.MkdirAll(filepath.Join(regoDir, "data"), 0755))
require.NoError(t, os.WriteFile(filepath.Join(regoDir, "data", "data.json"), []byte(test.inputData), 0644))
}

if test.cacheContent != "" {
cacheRoot := t.TempDir()
test.options.CacheDir = cacheRoot
Expand All @@ -1250,7 +1277,7 @@ Summary Report for compliance: my-custom-spec
cacheData, err := os.ReadFile(test.cacheContent)
require.NoError(t, err, test.name)

require.NoError(t, os.WriteFile(cacheFile, []byte(cacheData), 0600))
require.NoError(t, os.WriteFile(cacheFile, cacheData, 0600))
}

err := Run(context.Background(), test.options)
Expand Down
66 changes: 48 additions & 18 deletions pkg/cloud/aws/scanner/scanner.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,11 @@ package scanner
import (
"context"
"fmt"
"io/fs"
"strings"

"golang.org/x/xerrors"

"github.com/aquasecurity/defsec/pkg/framework"
"github.com/aquasecurity/defsec/pkg/scan"
"github.com/aquasecurity/defsec/pkg/scanners/cloud/aws"
Expand All @@ -14,6 +17,7 @@ import (
"github.com/aquasecurity/trivy/pkg/commands/operation"
"github.com/aquasecurity/trivy/pkg/flag"
"github.com/aquasecurity/trivy/pkg/log"
"github.com/aquasecurity/trivy/pkg/misconf"
)

type AWSScanner struct {
Expand Down Expand Up @@ -75,15 +79,24 @@ func (s *AWSScanner) Scan(ctx context.Context, option flag.Options) (scan.Result
scannerOpts = append(scannerOpts,
options.ScannerWithEmbeddedPolicies(false))
}
policyPaths = append(policyPaths, option.RegoOptions.PolicyPaths...)

var policyFS fs.FS
policyFS, policyPaths, err = misconf.CreatePolicyFS(append(policyPaths, option.RegoOptions.PolicyPaths...))
if err != nil {
return nil, false, xerrors.Errorf("unable to create policyfs: %w", err)
}

scannerOpts = append(scannerOpts, options.ScannerWithPolicyFilesystem(policyFS))
scannerOpts = append(scannerOpts, options.ScannerWithPolicyDirs(policyPaths...))

if len(option.RegoOptions.PolicyNamespaces) > 0 {
scannerOpts = append(
scannerOpts,
options.ScannerWithPolicyNamespaces(option.RegoOptions.PolicyNamespaces...),
)
dataFS, dataPaths, err := misconf.CreateDataFS(option.RegoOptions.DataPaths)
if err != nil {
log.Logger.Errorf("Could not load config data: %s", err)
}
scannerOpts = append(scannerOpts, options.ScannerWithDataDirs(dataPaths...))
scannerOpts = append(scannerOpts, options.ScannerWithDataFilesystem(dataFS))

scannerOpts = addPolicyNamespaces(option.RegoOptions.PolicyNamespaces, scannerOpts)

if option.Compliance.Spec.ID != "" {
scannerOpts = append(scannerOpts, options.ScannerWithSpec(option.Compliance.Spec.ID))
Expand All @@ -104,18 +117,9 @@ func (s *AWSScanner) Scan(ctx context.Context, option flag.Options) (scan.Result
}
}

var fullState *state.State
if previousState, err := awsCache.LoadState(); err == nil {
if freshState != nil {
fullState, err = previousState.Merge(freshState)
if err != nil {
return nil, false, err
}
} else {
fullState = previousState
}
} else {
fullState = freshState
fullState, err := createState(freshState, awsCache)
if err != nil {
return nil, false, err
}

if fullState == nil {
Expand All @@ -134,10 +138,36 @@ func (s *AWSScanner) Scan(ctx context.Context, option flag.Options) (scan.Result
return defsecResults, len(included) > 0, nil
}

func createState(freshState *state.State, awsCache *cache.Cache) (*state.State, error) {
var fullState *state.State
if previousState, err := awsCache.LoadState(); err == nil {
if freshState != nil {
fullState, err = previousState.Merge(freshState)
if err != nil {
return nil, err
}
} else {
fullState = previousState
}
} else {
fullState = freshState
}
return fullState, nil
}

type defsecLogger struct {
}

func (d *defsecLogger) Write(p []byte) (n int, err error) {
log.Logger.Debug("[defsec] " + strings.TrimSpace(string(p)))
return len(p), nil
}
func addPolicyNamespaces(namespaces []string, scannerOpts []options.ScannerOption) []options.ScannerOption {
if len(namespaces) > 0 {
scannerOpts = append(
scannerOpts,
options.ScannerWithPolicyNamespaces(namespaces...),
)
}
return scannerOpts
}
2 changes: 1 addition & 1 deletion pkg/commands/artifact/run.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -579,7 +579,7 @@ func initScannerConfig(opts flag.Options, cacheClient cache.Cache) (ScannerConfi
Trace: opts.Trace,
Namespaces: append(opts.PolicyNamespaces, defaultPolicyNamespaces...),
PolicyPaths: append(opts.PolicyPaths, downloadedPolicyPaths...),
DataPaths: opts.DataPaths,
DataPaths: append(opts.DataPaths),
HelmValues: opts.HelmValues,
HelmValueFiles: opts.HelmValueFiles,
HelmFileValues: opts.HelmFileValues,
Expand Down
11 changes: 6 additions & 5 deletions pkg/mapfs/fs.go
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package mapfs

import (
"fmt"
"io"
"io/fs"
"os"
Expand Down Expand Up @@ -233,11 +234,11 @@ func (m *FS) RemoveAll(path string) error {
}

func cleanPath(path string) string {
// Return if the file path is a volume name only.
// Otherwise, `filepath.Clean` changes "C:" to "C:." and
// it will no longer match the pathname held by mapfs.
if path == filepath.VolumeName(path) {
return path
// Convert the volume name like 'C:' into dir like 'C\'
if vol := filepath.VolumeName(path); len(vol) > 0 {
newVol := strings.TrimSuffix(vol, ":")
newVol = fmt.Sprintf("%s%c", newVol, filepath.Separator)
path = strings.Replace(path, vol, newVol, 1)
}
path = filepath.Clean(path)
path = filepath.ToSlash(path)
Expand Down
Loading

0 comments on commit 4bc8d29

Please sign in to comment.