Skip to content

Commit

Permalink
feat(flag): replace '--slow' with '--parallel' (#5572)
Browse files Browse the repository at this point in the history
Signed-off-by: knqyf263 <knqyf263@gmail.com>
  • Loading branch information
knqyf263 authored and pull[bot] committed Dec 4, 2023
1 parent 6604509 commit 8a25878
Show file tree
Hide file tree
Showing 31 changed files with 115 additions and 141 deletions.
2 changes: 1 addition & 1 deletion docs/docs/references/configuration/cli/trivy_filesystem.md
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ trivy filesystem [flags] PATH
--no-progress suppress progress bar
--offline-scan do not issue API requests to identify dependencies
-o, --output string output file name
--parallel int number of goroutines enabled for parallel scanning, set 0 to auto-detect parallelism (default 5)
--password strings password. Comma-separated passwords allowed. TRIVY_PASSWORD should be used for security reasons.
--policy-bundle-repository string OCI registry URL to retrieve policy bundle from (default "ghcr.io/aquasecurity/trivy-policies:0")
--policy-namespaces strings Rego namespaces
Expand All @@ -76,7 +77,6 @@ trivy filesystem [flags] PATH
--skip-files strings specify the files or glob patterns to skip
--skip-java-db-update skip updating Java index database
--skip-policy-update skip fetching rego policy updates
--slow scan over time with lower CPU and memory utilization
-t, --template string output template
--tf-exclude-downloaded-modules exclude misconfigurations for downloaded terraform modules
--tf-vars strings specify paths to override the Terraform tfvars files
Expand Down
2 changes: 1 addition & 1 deletion docs/docs/references/configuration/cli/trivy_image.md
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@ trivy image [flags] IMAGE_NAME
--no-progress suppress progress bar
--offline-scan do not issue API requests to identify dependencies
-o, --output string output file name
--parallel int number of goroutines enabled for parallel scanning, set 0 to auto-detect parallelism (default 5)
--password strings password. Comma-separated passwords allowed. TRIVY_PASSWORD should be used for security reasons.
--platform string set platform in the form os/arch if image is multi-platform capable
--policy-bundle-repository string OCI registry URL to retrieve policy bundle from (default "ghcr.io/aquasecurity/trivy-policies:0")
Expand All @@ -97,7 +98,6 @@ trivy image [flags] IMAGE_NAME
--skip-files strings specify the files or glob patterns to skip
--skip-java-db-update skip updating Java index database
--skip-policy-update skip fetching rego policy updates
--slow scan over time with lower CPU and memory utilization
-t, --template string output template
--tf-exclude-downloaded-modules exclude misconfigurations for downloaded terraform modules
--tf-vars strings specify paths to override the Terraform tfvars files
Expand Down
3 changes: 1 addition & 2 deletions docs/docs/references/configuration/cli/trivy_kubernetes.md
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ trivy kubernetes [flags] { cluster | all | specific resources like kubectl. eg:
--node-collector-namespace string specify the namespace in which the node-collector job should be deployed (default "trivy-temp")
--offline-scan do not issue API requests to identify dependencies
-o, --output string output file name
--parallel int number (between 1-20) of goroutines enabled for parallel scanning (default 5)
--parallel int number of goroutines enabled for parallel scanning, set 0 to auto-detect parallelism (default 5)
--password strings password. Comma-separated passwords allowed. TRIVY_PASSWORD should be used for security reasons.
--policy-bundle-repository string OCI registry URL to retrieve policy bundle from (default "ghcr.io/aquasecurity/trivy-policies:0")
--policy-namespaces strings Rego namespaces
Expand All @@ -87,7 +87,6 @@ trivy kubernetes [flags] { cluster | all | specific resources like kubectl. eg:
--skip-files strings specify the files or glob patterns to skip
--skip-java-db-update skip updating Java index database
--skip-policy-update skip fetching rego policy updates
--slow scan over time with lower CPU and memory utilization
-t, --template string output template
--tf-exclude-downloaded-modules exclude misconfigurations for downloaded terraform modules
--tf-vars strings specify paths to override the Terraform tfvars files
Expand Down
2 changes: 1 addition & 1 deletion docs/docs/references/configuration/cli/trivy_repository.md
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ trivy repository [flags] (REPO_PATH | REPO_URL)
--no-progress suppress progress bar
--offline-scan do not issue API requests to identify dependencies
-o, --output string output file name
--parallel int number of goroutines enabled for parallel scanning, set 0 to auto-detect parallelism (default 5)
--password strings password. Comma-separated passwords allowed. TRIVY_PASSWORD should be used for security reasons.
--policy-bundle-repository string OCI registry URL to retrieve policy bundle from (default "ghcr.io/aquasecurity/trivy-policies:0")
--policy-namespaces strings Rego namespaces
Expand All @@ -75,7 +76,6 @@ trivy repository [flags] (REPO_PATH | REPO_URL)
--skip-files strings specify the files or glob patterns to skip
--skip-java-db-update skip updating Java index database
--skip-policy-update skip fetching rego policy updates
--slow scan over time with lower CPU and memory utilization
--tag string pass the tag name to be scanned
-t, --template string output template
--tf-exclude-downloaded-modules exclude misconfigurations for downloaded terraform modules
Expand Down
2 changes: 1 addition & 1 deletion docs/docs/references/configuration/cli/trivy_rootfs.md
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ trivy rootfs [flags] ROOTDIR
--no-progress suppress progress bar
--offline-scan do not issue API requests to identify dependencies
-o, --output string output file name
--parallel int number of goroutines enabled for parallel scanning, set 0 to auto-detect parallelism (default 5)
--password strings password. Comma-separated passwords allowed. TRIVY_PASSWORD should be used for security reasons.
--policy-bundle-repository string OCI registry URL to retrieve policy bundle from (default "ghcr.io/aquasecurity/trivy-policies:0")
--policy-namespaces strings Rego namespaces
Expand All @@ -77,7 +78,6 @@ trivy rootfs [flags] ROOTDIR
--skip-files strings specify the files or glob patterns to skip
--skip-java-db-update skip updating Java index database
--skip-policy-update skip fetching rego policy updates
--slow scan over time with lower CPU and memory utilization
-t, --template string output template
--tf-exclude-downloaded-modules exclude misconfigurations for downloaded terraform modules
--tf-vars strings specify paths to override the Terraform tfvars files
Expand Down
1 change: 0 additions & 1 deletion docs/docs/references/configuration/cli/trivy_sbom.md
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,6 @@ trivy sbom [flags] SBOM_PATH
--skip-dirs strings specify the directories or glob patterns to skip
--skip-files strings specify the files or glob patterns to skip
--skip-java-db-update skip updating Java index database
--slow scan over time with lower CPU and memory utilization
-t, --template string output template
--token string for authentication in client/server mode
--token-header string specify a header name for token in client/server mode (default "Trivy-Token")
Expand Down
2 changes: 1 addition & 1 deletion docs/docs/references/configuration/cli/trivy_vm.md
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ trivy vm [flags] VM_IMAGE
--no-progress suppress progress bar
--offline-scan do not issue API requests to identify dependencies
-o, --output string output file name
--parallel int number of goroutines enabled for parallel scanning, set 0 to auto-detect parallelism (default 5)
--policy-bundle-repository string OCI registry URL to retrieve policy bundle from (default "ghcr.io/aquasecurity/trivy-policies:0")
--redis-ca string redis ca file location, if using redis as cache backend
--redis-cert string redis certificate file location, if using redis as cache backend
Expand All @@ -68,7 +69,6 @@ trivy vm [flags] VM_IMAGE
--skip-dirs strings specify the directories or glob patterns to skip
--skip-files strings specify the files or glob patterns to skip
--skip-java-db-update skip updating Java index database
--slow scan over time with lower CPU and memory utilization
-t, --template string output template
--tf-exclude-downloaded-modules exclude misconfigurations for downloaded terraform modules
--tf-vars strings specify paths to override the Terraform tfvars files
Expand Down
18 changes: 14 additions & 4 deletions integration/repo_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
package integration

import (
"fmt"
"os"
"path/filepath"
"strings"
Expand Down Expand Up @@ -36,6 +37,7 @@ func TestRepository(t *testing.T) {
command string
format types.Format
includeDevDeps bool
parallel int
}
tests := []struct {
name string
Expand Down Expand Up @@ -69,6 +71,15 @@ func TestRepository(t *testing.T) {
},
golden: "testdata/gomod-skip.json.golden",
},
{
name: "gomod in series",
args: args{
scanner: types.VulnerabilityScanner,
input: "testdata/fixtures/repo/gomod",
parallel: 1,
},
golden: "testdata/gomod.json.golden",
},
{
name: "npm",
args: args{
Expand Down Expand Up @@ -396,13 +407,12 @@ func TestRepository(t *testing.T) {

osArgs := []string{
"-q",
"--cache-dir",
cacheDir,
"--cache-dir", cacheDir,
command,
"--skip-db-update",
"--skip-policy-update",
"--format",
string(format),
"--format", string(format),
"--parallel", fmt.Sprint(tt.args.parallel),
"--offline-scan",
}

Expand Down
1 change: 1 addition & 0 deletions pkg/commands/app.go
Original file line number Diff line number Diff line change
Expand Up @@ -1098,6 +1098,7 @@ func NewSBOMCommand(globalFlags *flag.GlobalFlagGroup) *cobra.Command {
scanFlagGroup := flag.NewScanFlagGroup()
scanFlagGroup.Scanners = nil // disable '--scanners' as it always scans for vulnerabilities
scanFlagGroup.IncludeDevDeps = nil // disable '--include-dev-deps'
scanFlagGroup.Parallel = nil // disable '--parallel'

sbomFlags := &flag.Flags{
CacheFlagGroup: flag.NewCacheFlagGroup(),
Expand Down
2 changes: 1 addition & 1 deletion pkg/commands/artifact/run.go
Original file line number Diff line number Diff line change
Expand Up @@ -637,7 +637,7 @@ func initScannerConfig(opts flag.Options, cacheClient cache.Cache) (ScannerConfi
SBOMSources: opts.SBOMSources,
RekorURL: opts.RekorURL,
//Platform: opts.Platform,
Slow: opts.Slow,
Parallel: opts.Parallel,
AWSRegion: opts.Region,
AWSEndpoint: opts.Endpoint,
FileChecksum: fileChecksum,
Expand Down
2 changes: 1 addition & 1 deletion pkg/fanal/analyzer/analyzer.go
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ var (
// AnalyzerOptions is used to initialize analyzers
type AnalyzerOptions struct {
Group Group
Slow bool
Parallel int
FilePatterns []string
DisabledAnalyzers []Type
MisconfScannerOption misconf.ScannerOption
Expand Down
6 changes: 3 additions & 3 deletions pkg/fanal/analyzer/language/java/jar/jar.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,12 +33,12 @@ var requiredExtensions = []string{

// javaLibraryAnalyzer analyzes jar/war/ear/par files
type javaLibraryAnalyzer struct {
slow bool
parallel int
}

func newJavaLibraryAnalyzer(options analyzer.AnalyzerOptions) (analyzer.PostAnalyzer, error) {
return &javaLibraryAnalyzer{
slow: options.Slow,
parallel: options.Parallel,
}, nil
}

Expand Down Expand Up @@ -70,7 +70,7 @@ func (a *javaLibraryAnalyzer) PostAnalyze(ctx context.Context, input analyzer.Po
return nil
}

if err = parallel.WalkDir(ctx, input.FS, ".", a.slow, onFile, onResult); err != nil {
if err = parallel.WalkDir(ctx, input.FS, ".", a.parallel, onFile, onResult); err != nil {
return nil, xerrors.Errorf("walk dir error: %w", err)
}

Expand Down
2 changes: 1 addition & 1 deletion pkg/fanal/analyzer/language/java/jar/jar_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,7 @@ func Test_javaLibraryAnalyzer_Analyze(t *testing.T) {
// init java-trivy-db with skip update
javadb.Init("testdata", defaultJavaDBRepository, true, false, types.RegistryOptions{Insecure: false})

a := javaLibraryAnalyzer{slow: true}
a := javaLibraryAnalyzer{}
ctx := context.Background()

mfs := mapfs.New()
Expand Down
23 changes: 22 additions & 1 deletion pkg/fanal/artifact/artifact.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ type Option struct {
AppDirs []string
SBOMSources []string
RekorURL string
Slow bool // Lower CPU and memory
Parallel int
AWSRegion string
AWSEndpoint string
FileChecksum bool // For SPDX
Expand All @@ -50,6 +50,27 @@ type WalkOption struct {
ErrorCallback walker.ErrorCallback
}

func (o *Option) AnalyzerOptions() analyzer.AnalyzerOptions {
return analyzer.AnalyzerOptions{
Group: o.AnalyzerGroup,
FilePatterns: o.FilePatterns,
Parallel: o.Parallel,
DisabledAnalyzers: o.DisabledAnalyzers,
MisconfScannerOption: o.MisconfScannerOption,
SecretScannerOption: o.SecretScannerOption,
LicenseScannerOption: o.LicenseScannerOption,
}
}

func (o *Option) ConfigAnalyzerOptions() analyzer.ConfigAnalyzerOptions {
return analyzer.ConfigAnalyzerOptions{
FilePatterns: o.FilePatterns,
DisabledAnalyzers: o.DisabledAnalyzers,
MisconfScannerOption: o.MisconfScannerOption,
SecretScannerOption: o.SecretScannerOption,
}
}

func (o *Option) Sort() {
sort.Slice(o.DisabledAnalyzers, func(i, j int) bool {
return o.DisabledAnalyzers[i] < o.DisabledAnalyzers[j]
Expand Down
24 changes: 5 additions & 19 deletions pkg/fanal/artifact/image/image.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,33 +49,20 @@ func NewArtifact(img types.Image, c cache.ArtifactCache, opt artifact.Option) (a
return nil, xerrors.Errorf("handler init error: %w", err)
}

a, err := analyzer.NewAnalyzerGroup(analyzer.AnalyzerOptions{
Group: opt.AnalyzerGroup,
Slow: opt.Slow,
FilePatterns: opt.FilePatterns,
DisabledAnalyzers: opt.DisabledAnalyzers,
MisconfScannerOption: opt.MisconfScannerOption,
SecretScannerOption: opt.SecretScannerOption,
LicenseScannerOption: opt.LicenseScannerOption,
})
a, err := analyzer.NewAnalyzerGroup(opt.AnalyzerOptions())
if err != nil {
return nil, xerrors.Errorf("analyzer group error: %w", err)
}

ca, err := analyzer.NewConfigAnalyzerGroup(analyzer.ConfigAnalyzerOptions{
FilePatterns: opt.FilePatterns,
DisabledAnalyzers: opt.DisabledAnalyzers,
MisconfScannerOption: opt.MisconfScannerOption,
SecretScannerOption: opt.SecretScannerOption,
})
ca, err := analyzer.NewConfigAnalyzerGroup(opt.ConfigAnalyzerOptions())
if err != nil {
return nil, xerrors.Errorf("config analyzer group error: %w", err)
}

return Artifact{
image: img,
cache: c,
walker: walker.NewLayerTar(opt.SkipFiles, opt.SkipDirs, opt.Slow),
walker: walker.NewLayerTar(opt.SkipFiles, opt.SkipDirs),
analyzer: a,
configAnalyzer: ca,
handlerManager: handlerManager,
Expand Down Expand Up @@ -215,8 +202,7 @@ func (a Artifact) inspect(ctx context.Context, missingImage string, layerKeys, b
layerKeyMap map[string]LayerInfo, configFile *v1.ConfigFile) error {

var osFound types.OS
workers := lo.Ternary(a.artifactOption.Slow, 1, 5)
p := parallel.NewPipeline(workers, false, layerKeys, func(ctx context.Context, layerKey string) (any, error) {
p := parallel.NewPipeline(a.artifactOption.Parallel, false, layerKeys, func(ctx context.Context, layerKey string) (any, error) {
layer := layerKeyMap[layerKey]

// If it is a base layer, secret scanning should not be performed.
Expand Down Expand Up @@ -268,7 +254,7 @@ func (a Artifact) inspectLayer(ctx context.Context, layerInfo LayerInfo, disable
FileChecksum: a.artifactOption.FileChecksum,
}
result := analyzer.NewAnalysisResult()
limit := semaphore.New(a.artifactOption.Slow)
limit := semaphore.New(a.artifactOption.Parallel)

// Prepare filesystem for post analysis
composite, err := a.analyzer.PostAnalyzerFS()
Expand Down
5 changes: 1 addition & 4 deletions pkg/fanal/artifact/image/image_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2035,11 +2035,8 @@ func TestArtifact_Inspect(t *testing.T) {
wantErr: "put layer failed",
},
{
name: "sad path, PutBlob returns an error with multiple layers and Slow enabled",
name: "sad path, PutBlob returns an error with multiple layers",
imagePath: "../../test/testdata/vuln-image.tar.gz",
artifactOpt: artifact.Option{
Slow: true,
},
missingBlobsExpectation: cache.ArtifactCacheMissingBlobsExpectation{
Args: cache.ArtifactCacheMissingBlobsArgs{
ArtifactID: "sha256:33f9415ed2cd5a9cef5d5144333619745b9ec0f851f0684dd45fa79c6b26a650",
Expand Down
14 changes: 3 additions & 11 deletions pkg/fanal/artifact/local/fs.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,15 +39,7 @@ func NewArtifact(rootPath string, c cache.ArtifactCache, opt artifact.Option) (a
return nil, xerrors.Errorf("handler initialize error: %w", err)
}

a, err := analyzer.NewAnalyzerGroup(analyzer.AnalyzerOptions{
Group: opt.AnalyzerGroup,
Slow: opt.Slow,
FilePatterns: opt.FilePatterns,
DisabledAnalyzers: opt.DisabledAnalyzers,
MisconfScannerOption: opt.MisconfScannerOption,
SecretScannerOption: opt.SecretScannerOption,
LicenseScannerOption: opt.LicenseScannerOption,
})
a, err := analyzer.NewAnalyzerGroup(opt.AnalyzerOptions())
if err != nil {
return nil, xerrors.Errorf("analyzer group error: %w", err)
}
Expand All @@ -56,7 +48,7 @@ func NewArtifact(rootPath string, c cache.ArtifactCache, opt artifact.Option) (a
rootPath: filepath.ToSlash(filepath.Clean(rootPath)),
cache: c,
walker: walker.NewFS(buildPathsToSkip(rootPath, opt.SkipFiles), buildPathsToSkip(rootPath, opt.SkipDirs),
opt.Slow, opt.WalkOption.ErrorCallback),
opt.Parallel, opt.WalkOption.ErrorCallback),
analyzer: a,
handlerManager: handlerManager,

Expand Down Expand Up @@ -122,7 +114,7 @@ func buildPathsToSkip(base string, paths []string) []string {
func (a Artifact) Inspect(ctx context.Context) (types.ArtifactReference, error) {
var wg sync.WaitGroup
result := analyzer.NewAnalysisResult()
limit := semaphore.New(a.artifactOption.Slow)
limit := semaphore.New(a.artifactOption.Parallel)
opts := analyzer.AnalysisOptions{
Offline: a.artifactOption.Offline,
FileChecksum: a.artifactOption.FileChecksum,
Expand Down
2 changes: 1 addition & 1 deletion pkg/fanal/artifact/local/fs_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,7 @@ func TestArtifact_Inspect(t *testing.T) {
fields: fields{
dir: "./testdata/unknown",
},
wantErr: "walk error",
wantErr: "walk dir error",
},
{
name: "happy path with single file",
Expand Down
13 changes: 3 additions & 10 deletions pkg/fanal/artifact/vm/vm.go
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ type Storage struct {

func (a *Storage) Analyze(ctx context.Context, r *io.SectionReader) (types.BlobInfo, error) {
var wg sync.WaitGroup
limit := semaphore.New(a.artifactOption.Slow)
limit := semaphore.New(a.artifactOption.Parallel)
result := analyzer.NewAnalysisResult()

opts := analyzer.AnalysisOptions{
Expand Down Expand Up @@ -119,14 +119,7 @@ func NewArtifact(target string, c cache.ArtifactCache, opt artifact.Option) (art
if err != nil {
return nil, xerrors.Errorf("handler init error: %w", err)
}
a, err := analyzer.NewAnalyzerGroup(analyzer.AnalyzerOptions{
Group: opt.AnalyzerGroup,
FilePatterns: opt.FilePatterns,
DisabledAnalyzers: opt.DisabledAnalyzers,
MisconfScannerOption: opt.MisconfScannerOption,
SecretScannerOption: opt.SecretScannerOption,
LicenseScannerOption: opt.LicenseScannerOption,
})
a, err := analyzer.NewAnalyzerGroup(opt.AnalyzerOptions())
if err != nil {
return nil, xerrors.Errorf("analyzer group error: %w", err)
}
Expand All @@ -135,7 +128,7 @@ func NewArtifact(target string, c cache.ArtifactCache, opt artifact.Option) (art
cache: c,
analyzer: a,
handlerManager: handlerManager,
walker: walker.NewVM(opt.SkipFiles, opt.SkipDirs, opt.Slow),
walker: walker.NewVM(opt.SkipFiles, opt.SkipDirs),
artifactOption: opt,
}

Expand Down
Loading

0 comments on commit 8a25878

Please sign in to comment.