diff --git a/docs/docs/references/configuration/cli/trivy_filesystem.md b/docs/docs/references/configuration/cli/trivy_filesystem.md index e32e6af4be53..57d2414ddc23 100644 --- a/docs/docs/references/configuration/cli/trivy_filesystem.md +++ b/docs/docs/references/configuration/cli/trivy_filesystem.md @@ -54,6 +54,7 @@ trivy filesystem [flags] PATH --no-progress suppress progress bar --offline-scan do not issue API requests to identify dependencies -o, --output string output file name + --parallel int number of goroutines enabled for parallel scanning, set 0 to auto-detect parallelism (default 5) --password strings password. Comma-separated passwords allowed. TRIVY_PASSWORD should be used for security reasons. --policy-bundle-repository string OCI registry URL to retrieve policy bundle from (default "ghcr.io/aquasecurity/trivy-policies:0") --policy-namespaces strings Rego namespaces @@ -76,7 +77,6 @@ trivy filesystem [flags] PATH --skip-files strings specify the files or glob patterns to skip --skip-java-db-update skip updating Java index database --skip-policy-update skip fetching rego policy updates - --slow scan over time with lower CPU and memory utilization -t, --template string output template --tf-exclude-downloaded-modules exclude misconfigurations for downloaded terraform modules --tf-vars strings specify paths to override the Terraform tfvars files diff --git a/docs/docs/references/configuration/cli/trivy_image.md b/docs/docs/references/configuration/cli/trivy_image.md index 31daec770d61..3105bdaa3025 100644 --- a/docs/docs/references/configuration/cli/trivy_image.md +++ b/docs/docs/references/configuration/cli/trivy_image.md @@ -73,6 +73,7 @@ trivy image [flags] IMAGE_NAME --no-progress suppress progress bar --offline-scan do not issue API requests to identify dependencies -o, --output string output file name + --parallel int number of goroutines enabled for parallel scanning, set 0 to auto-detect parallelism (default 5) --password strings password. Comma-separated passwords allowed. TRIVY_PASSWORD should be used for security reasons. --platform string set platform in the form os/arch if image is multi-platform capable --policy-bundle-repository string OCI registry URL to retrieve policy bundle from (default "ghcr.io/aquasecurity/trivy-policies:0") @@ -97,7 +98,6 @@ trivy image [flags] IMAGE_NAME --skip-files strings specify the files or glob patterns to skip --skip-java-db-update skip updating Java index database --skip-policy-update skip fetching rego policy updates - --slow scan over time with lower CPU and memory utilization -t, --template string output template --tf-exclude-downloaded-modules exclude misconfigurations for downloaded terraform modules --tf-vars strings specify paths to override the Terraform tfvars files diff --git a/docs/docs/references/configuration/cli/trivy_kubernetes.md b/docs/docs/references/configuration/cli/trivy_kubernetes.md index dcc41040e7f1..a460641d2f15 100644 --- a/docs/docs/references/configuration/cli/trivy_kubernetes.md +++ b/docs/docs/references/configuration/cli/trivy_kubernetes.md @@ -65,7 +65,7 @@ trivy kubernetes [flags] { cluster | all | specific resources like kubectl. eg: --node-collector-namespace string specify the namespace in which the node-collector job should be deployed (default "trivy-temp") --offline-scan do not issue API requests to identify dependencies -o, --output string output file name - --parallel int number (between 1-20) of goroutines enabled for parallel scanning (default 5) + --parallel int number of goroutines enabled for parallel scanning, set 0 to auto-detect parallelism (default 5) --password strings password. Comma-separated passwords allowed. TRIVY_PASSWORD should be used for security reasons. --policy-bundle-repository string OCI registry URL to retrieve policy bundle from (default "ghcr.io/aquasecurity/trivy-policies:0") --policy-namespaces strings Rego namespaces @@ -87,7 +87,6 @@ trivy kubernetes [flags] { cluster | all | specific resources like kubectl. eg: --skip-files strings specify the files or glob patterns to skip --skip-java-db-update skip updating Java index database --skip-policy-update skip fetching rego policy updates - --slow scan over time with lower CPU and memory utilization -t, --template string output template --tf-exclude-downloaded-modules exclude misconfigurations for downloaded terraform modules --tf-vars strings specify paths to override the Terraform tfvars files diff --git a/docs/docs/references/configuration/cli/trivy_repository.md b/docs/docs/references/configuration/cli/trivy_repository.md index b57cfa56025c..4cf4638e99ac 100644 --- a/docs/docs/references/configuration/cli/trivy_repository.md +++ b/docs/docs/references/configuration/cli/trivy_repository.md @@ -54,6 +54,7 @@ trivy repository [flags] (REPO_PATH | REPO_URL) --no-progress suppress progress bar --offline-scan do not issue API requests to identify dependencies -o, --output string output file name + --parallel int number of goroutines enabled for parallel scanning, set 0 to auto-detect parallelism (default 5) --password strings password. Comma-separated passwords allowed. TRIVY_PASSWORD should be used for security reasons. --policy-bundle-repository string OCI registry URL to retrieve policy bundle from (default "ghcr.io/aquasecurity/trivy-policies:0") --policy-namespaces strings Rego namespaces @@ -75,7 +76,6 @@ trivy repository [flags] (REPO_PATH | REPO_URL) --skip-files strings specify the files or glob patterns to skip --skip-java-db-update skip updating Java index database --skip-policy-update skip fetching rego policy updates - --slow scan over time with lower CPU and memory utilization --tag string pass the tag name to be scanned -t, --template string output template --tf-exclude-downloaded-modules exclude misconfigurations for downloaded terraform modules diff --git a/docs/docs/references/configuration/cli/trivy_rootfs.md b/docs/docs/references/configuration/cli/trivy_rootfs.md index a5daaa47fce8..31046603d594 100644 --- a/docs/docs/references/configuration/cli/trivy_rootfs.md +++ b/docs/docs/references/configuration/cli/trivy_rootfs.md @@ -56,6 +56,7 @@ trivy rootfs [flags] ROOTDIR --no-progress suppress progress bar --offline-scan do not issue API requests to identify dependencies -o, --output string output file name + --parallel int number of goroutines enabled for parallel scanning, set 0 to auto-detect parallelism (default 5) --password strings password. Comma-separated passwords allowed. TRIVY_PASSWORD should be used for security reasons. --policy-bundle-repository string OCI registry URL to retrieve policy bundle from (default "ghcr.io/aquasecurity/trivy-policies:0") --policy-namespaces strings Rego namespaces @@ -77,7 +78,6 @@ trivy rootfs [flags] ROOTDIR --skip-files strings specify the files or glob patterns to skip --skip-java-db-update skip updating Java index database --skip-policy-update skip fetching rego policy updates - --slow scan over time with lower CPU and memory utilization -t, --template string output template --tf-exclude-downloaded-modules exclude misconfigurations for downloaded terraform modules --tf-vars strings specify paths to override the Terraform tfvars files diff --git a/docs/docs/references/configuration/cli/trivy_sbom.md b/docs/docs/references/configuration/cli/trivy_sbom.md index d031f61e4cff..9f899ac977cb 100644 --- a/docs/docs/references/configuration/cli/trivy_sbom.md +++ b/docs/docs/references/configuration/cli/trivy_sbom.md @@ -55,7 +55,6 @@ trivy sbom [flags] SBOM_PATH --skip-dirs strings specify the directories or glob patterns to skip --skip-files strings specify the files or glob patterns to skip --skip-java-db-update skip updating Java index database - --slow scan over time with lower CPU and memory utilization -t, --template string output template --token string for authentication in client/server mode --token-header string specify a header name for token in client/server mode (default "Trivy-Token") diff --git a/docs/docs/references/configuration/cli/trivy_vm.md b/docs/docs/references/configuration/cli/trivy_vm.md index fd47dfda7647..2028e3ed188d 100644 --- a/docs/docs/references/configuration/cli/trivy_vm.md +++ b/docs/docs/references/configuration/cli/trivy_vm.md @@ -51,6 +51,7 @@ trivy vm [flags] VM_IMAGE --no-progress suppress progress bar --offline-scan do not issue API requests to identify dependencies -o, --output string output file name + --parallel int number of goroutines enabled for parallel scanning, set 0 to auto-detect parallelism (default 5) --policy-bundle-repository string OCI registry URL to retrieve policy bundle from (default "ghcr.io/aquasecurity/trivy-policies:0") --redis-ca string redis ca file location, if using redis as cache backend --redis-cert string redis certificate file location, if using redis as cache backend @@ -68,7 +69,6 @@ trivy vm [flags] VM_IMAGE --skip-dirs strings specify the directories or glob patterns to skip --skip-files strings specify the files or glob patterns to skip --skip-java-db-update skip updating Java index database - --slow scan over time with lower CPU and memory utilization -t, --template string output template --tf-exclude-downloaded-modules exclude misconfigurations for downloaded terraform modules --tf-vars strings specify paths to override the Terraform tfvars files diff --git a/integration/repo_test.go b/integration/repo_test.go index e917288a0319..35aa58ac0784 100644 --- a/integration/repo_test.go +++ b/integration/repo_test.go @@ -3,6 +3,7 @@ package integration import ( + "fmt" "os" "path/filepath" "strings" @@ -36,6 +37,7 @@ func TestRepository(t *testing.T) { command string format types.Format includeDevDeps bool + parallel int } tests := []struct { name string @@ -69,6 +71,15 @@ func TestRepository(t *testing.T) { }, golden: "testdata/gomod-skip.json.golden", }, + { + name: "gomod in series", + args: args{ + scanner: types.VulnerabilityScanner, + input: "testdata/fixtures/repo/gomod", + parallel: 1, + }, + golden: "testdata/gomod.json.golden", + }, { name: "npm", args: args{ @@ -396,13 +407,12 @@ func TestRepository(t *testing.T) { osArgs := []string{ "-q", - "--cache-dir", - cacheDir, + "--cache-dir", cacheDir, command, "--skip-db-update", "--skip-policy-update", - "--format", - string(format), + "--format", string(format), + "--parallel", fmt.Sprint(tt.args.parallel), "--offline-scan", } diff --git a/pkg/commands/app.go b/pkg/commands/app.go index 4cdbc89f59a1..c6ae0ed7a212 100644 --- a/pkg/commands/app.go +++ b/pkg/commands/app.go @@ -1098,6 +1098,7 @@ func NewSBOMCommand(globalFlags *flag.GlobalFlagGroup) *cobra.Command { scanFlagGroup := flag.NewScanFlagGroup() scanFlagGroup.Scanners = nil // disable '--scanners' as it always scans for vulnerabilities scanFlagGroup.IncludeDevDeps = nil // disable '--include-dev-deps' + scanFlagGroup.Parallel = nil // disable '--parallel' sbomFlags := &flag.Flags{ CacheFlagGroup: flag.NewCacheFlagGroup(), diff --git a/pkg/commands/artifact/run.go b/pkg/commands/artifact/run.go index 5baa61787a57..b36897ffc3bb 100644 --- a/pkg/commands/artifact/run.go +++ b/pkg/commands/artifact/run.go @@ -637,7 +637,7 @@ func initScannerConfig(opts flag.Options, cacheClient cache.Cache) (ScannerConfi SBOMSources: opts.SBOMSources, RekorURL: opts.RekorURL, //Platform: opts.Platform, - Slow: opts.Slow, + Parallel: opts.Parallel, AWSRegion: opts.Region, AWSEndpoint: opts.Endpoint, FileChecksum: fileChecksum, diff --git a/pkg/fanal/analyzer/analyzer.go b/pkg/fanal/analyzer/analyzer.go index ccb578be62fa..b9f1620e07c6 100644 --- a/pkg/fanal/analyzer/analyzer.go +++ b/pkg/fanal/analyzer/analyzer.go @@ -41,7 +41,7 @@ var ( // AnalyzerOptions is used to initialize analyzers type AnalyzerOptions struct { Group Group - Slow bool + Parallel int FilePatterns []string DisabledAnalyzers []Type MisconfScannerOption misconf.ScannerOption diff --git a/pkg/fanal/analyzer/language/java/jar/jar.go b/pkg/fanal/analyzer/language/java/jar/jar.go index 0690fc0d9f20..0aeddfc66c9e 100644 --- a/pkg/fanal/analyzer/language/java/jar/jar.go +++ b/pkg/fanal/analyzer/language/java/jar/jar.go @@ -33,12 +33,12 @@ var requiredExtensions = []string{ // javaLibraryAnalyzer analyzes jar/war/ear/par files type javaLibraryAnalyzer struct { - slow bool + parallel int } func newJavaLibraryAnalyzer(options analyzer.AnalyzerOptions) (analyzer.PostAnalyzer, error) { return &javaLibraryAnalyzer{ - slow: options.Slow, + parallel: options.Parallel, }, nil } @@ -70,7 +70,7 @@ func (a *javaLibraryAnalyzer) PostAnalyze(ctx context.Context, input analyzer.Po return nil } - if err = parallel.WalkDir(ctx, input.FS, ".", a.slow, onFile, onResult); err != nil { + if err = parallel.WalkDir(ctx, input.FS, ".", a.parallel, onFile, onResult); err != nil { return nil, xerrors.Errorf("walk dir error: %w", err) } diff --git a/pkg/fanal/analyzer/language/java/jar/jar_test.go b/pkg/fanal/analyzer/language/java/jar/jar_test.go index f198c7beb041..133ead426d7a 100644 --- a/pkg/fanal/analyzer/language/java/jar/jar_test.go +++ b/pkg/fanal/analyzer/language/java/jar/jar_test.go @@ -132,7 +132,7 @@ func Test_javaLibraryAnalyzer_Analyze(t *testing.T) { // init java-trivy-db with skip update javadb.Init("testdata", defaultJavaDBRepository, true, false, types.RegistryOptions{Insecure: false}) - a := javaLibraryAnalyzer{slow: true} + a := javaLibraryAnalyzer{} ctx := context.Background() mfs := mapfs.New() diff --git a/pkg/fanal/artifact/artifact.go b/pkg/fanal/artifact/artifact.go index 2028b601c744..bcb3250f8b7a 100644 --- a/pkg/fanal/artifact/artifact.go +++ b/pkg/fanal/artifact/artifact.go @@ -23,7 +23,7 @@ type Option struct { AppDirs []string SBOMSources []string RekorURL string - Slow bool // Lower CPU and memory + Parallel int AWSRegion string AWSEndpoint string FileChecksum bool // For SPDX @@ -50,6 +50,27 @@ type WalkOption struct { ErrorCallback walker.ErrorCallback } +func (o *Option) AnalyzerOptions() analyzer.AnalyzerOptions { + return analyzer.AnalyzerOptions{ + Group: o.AnalyzerGroup, + FilePatterns: o.FilePatterns, + Parallel: o.Parallel, + DisabledAnalyzers: o.DisabledAnalyzers, + MisconfScannerOption: o.MisconfScannerOption, + SecretScannerOption: o.SecretScannerOption, + LicenseScannerOption: o.LicenseScannerOption, + } +} + +func (o *Option) ConfigAnalyzerOptions() analyzer.ConfigAnalyzerOptions { + return analyzer.ConfigAnalyzerOptions{ + FilePatterns: o.FilePatterns, + DisabledAnalyzers: o.DisabledAnalyzers, + MisconfScannerOption: o.MisconfScannerOption, + SecretScannerOption: o.SecretScannerOption, + } +} + func (o *Option) Sort() { sort.Slice(o.DisabledAnalyzers, func(i, j int) bool { return o.DisabledAnalyzers[i] < o.DisabledAnalyzers[j] diff --git a/pkg/fanal/artifact/image/image.go b/pkg/fanal/artifact/image/image.go index 87c572b32363..782d13a86097 100644 --- a/pkg/fanal/artifact/image/image.go +++ b/pkg/fanal/artifact/image/image.go @@ -49,25 +49,12 @@ func NewArtifact(img types.Image, c cache.ArtifactCache, opt artifact.Option) (a return nil, xerrors.Errorf("handler init error: %w", err) } - a, err := analyzer.NewAnalyzerGroup(analyzer.AnalyzerOptions{ - Group: opt.AnalyzerGroup, - Slow: opt.Slow, - FilePatterns: opt.FilePatterns, - DisabledAnalyzers: opt.DisabledAnalyzers, - MisconfScannerOption: opt.MisconfScannerOption, - SecretScannerOption: opt.SecretScannerOption, - LicenseScannerOption: opt.LicenseScannerOption, - }) + a, err := analyzer.NewAnalyzerGroup(opt.AnalyzerOptions()) if err != nil { return nil, xerrors.Errorf("analyzer group error: %w", err) } - ca, err := analyzer.NewConfigAnalyzerGroup(analyzer.ConfigAnalyzerOptions{ - FilePatterns: opt.FilePatterns, - DisabledAnalyzers: opt.DisabledAnalyzers, - MisconfScannerOption: opt.MisconfScannerOption, - SecretScannerOption: opt.SecretScannerOption, - }) + ca, err := analyzer.NewConfigAnalyzerGroup(opt.ConfigAnalyzerOptions()) if err != nil { return nil, xerrors.Errorf("config analyzer group error: %w", err) } @@ -75,7 +62,7 @@ func NewArtifact(img types.Image, c cache.ArtifactCache, opt artifact.Option) (a return Artifact{ image: img, cache: c, - walker: walker.NewLayerTar(opt.SkipFiles, opt.SkipDirs, opt.Slow), + walker: walker.NewLayerTar(opt.SkipFiles, opt.SkipDirs), analyzer: a, configAnalyzer: ca, handlerManager: handlerManager, @@ -215,8 +202,7 @@ func (a Artifact) inspect(ctx context.Context, missingImage string, layerKeys, b layerKeyMap map[string]LayerInfo, configFile *v1.ConfigFile) error { var osFound types.OS - workers := lo.Ternary(a.artifactOption.Slow, 1, 5) - p := parallel.NewPipeline(workers, false, layerKeys, func(ctx context.Context, layerKey string) (any, error) { + p := parallel.NewPipeline(a.artifactOption.Parallel, false, layerKeys, func(ctx context.Context, layerKey string) (any, error) { layer := layerKeyMap[layerKey] // If it is a base layer, secret scanning should not be performed. @@ -268,7 +254,7 @@ func (a Artifact) inspectLayer(ctx context.Context, layerInfo LayerInfo, disable FileChecksum: a.artifactOption.FileChecksum, } result := analyzer.NewAnalysisResult() - limit := semaphore.New(a.artifactOption.Slow) + limit := semaphore.New(a.artifactOption.Parallel) // Prepare filesystem for post analysis composite, err := a.analyzer.PostAnalyzerFS() diff --git a/pkg/fanal/artifact/image/image_test.go b/pkg/fanal/artifact/image/image_test.go index c257fccb30c2..d65df55f2fc5 100644 --- a/pkg/fanal/artifact/image/image_test.go +++ b/pkg/fanal/artifact/image/image_test.go @@ -2035,11 +2035,8 @@ func TestArtifact_Inspect(t *testing.T) { wantErr: "put layer failed", }, { - name: "sad path, PutBlob returns an error with multiple layers and Slow enabled", + name: "sad path, PutBlob returns an error with multiple layers", imagePath: "../../test/testdata/vuln-image.tar.gz", - artifactOpt: artifact.Option{ - Slow: true, - }, missingBlobsExpectation: cache.ArtifactCacheMissingBlobsExpectation{ Args: cache.ArtifactCacheMissingBlobsArgs{ ArtifactID: "sha256:33f9415ed2cd5a9cef5d5144333619745b9ec0f851f0684dd45fa79c6b26a650", diff --git a/pkg/fanal/artifact/local/fs.go b/pkg/fanal/artifact/local/fs.go index 49d20fd14f95..8d7409cdfae3 100644 --- a/pkg/fanal/artifact/local/fs.go +++ b/pkg/fanal/artifact/local/fs.go @@ -39,15 +39,7 @@ func NewArtifact(rootPath string, c cache.ArtifactCache, opt artifact.Option) (a return nil, xerrors.Errorf("handler initialize error: %w", err) } - a, err := analyzer.NewAnalyzerGroup(analyzer.AnalyzerOptions{ - Group: opt.AnalyzerGroup, - Slow: opt.Slow, - FilePatterns: opt.FilePatterns, - DisabledAnalyzers: opt.DisabledAnalyzers, - MisconfScannerOption: opt.MisconfScannerOption, - SecretScannerOption: opt.SecretScannerOption, - LicenseScannerOption: opt.LicenseScannerOption, - }) + a, err := analyzer.NewAnalyzerGroup(opt.AnalyzerOptions()) if err != nil { return nil, xerrors.Errorf("analyzer group error: %w", err) } @@ -56,7 +48,7 @@ func NewArtifact(rootPath string, c cache.ArtifactCache, opt artifact.Option) (a rootPath: filepath.ToSlash(filepath.Clean(rootPath)), cache: c, walker: walker.NewFS(buildPathsToSkip(rootPath, opt.SkipFiles), buildPathsToSkip(rootPath, opt.SkipDirs), - opt.Slow, opt.WalkOption.ErrorCallback), + opt.Parallel, opt.WalkOption.ErrorCallback), analyzer: a, handlerManager: handlerManager, @@ -122,7 +114,7 @@ func buildPathsToSkip(base string, paths []string) []string { func (a Artifact) Inspect(ctx context.Context) (types.ArtifactReference, error) { var wg sync.WaitGroup result := analyzer.NewAnalysisResult() - limit := semaphore.New(a.artifactOption.Slow) + limit := semaphore.New(a.artifactOption.Parallel) opts := analyzer.AnalysisOptions{ Offline: a.artifactOption.Offline, FileChecksum: a.artifactOption.FileChecksum, diff --git a/pkg/fanal/artifact/local/fs_test.go b/pkg/fanal/artifact/local/fs_test.go index a18288cd1ba3..e83bb0b8a7c0 100644 --- a/pkg/fanal/artifact/local/fs_test.go +++ b/pkg/fanal/artifact/local/fs_test.go @@ -166,7 +166,7 @@ func TestArtifact_Inspect(t *testing.T) { fields: fields{ dir: "./testdata/unknown", }, - wantErr: "walk error", + wantErr: "walk dir error", }, { name: "happy path with single file", diff --git a/pkg/fanal/artifact/vm/vm.go b/pkg/fanal/artifact/vm/vm.go index a9173ed00e7c..836b9dc66d3c 100644 --- a/pkg/fanal/artifact/vm/vm.go +++ b/pkg/fanal/artifact/vm/vm.go @@ -41,7 +41,7 @@ type Storage struct { func (a *Storage) Analyze(ctx context.Context, r *io.SectionReader) (types.BlobInfo, error) { var wg sync.WaitGroup - limit := semaphore.New(a.artifactOption.Slow) + limit := semaphore.New(a.artifactOption.Parallel) result := analyzer.NewAnalysisResult() opts := analyzer.AnalysisOptions{ @@ -119,14 +119,7 @@ func NewArtifact(target string, c cache.ArtifactCache, opt artifact.Option) (art if err != nil { return nil, xerrors.Errorf("handler init error: %w", err) } - a, err := analyzer.NewAnalyzerGroup(analyzer.AnalyzerOptions{ - Group: opt.AnalyzerGroup, - FilePatterns: opt.FilePatterns, - DisabledAnalyzers: opt.DisabledAnalyzers, - MisconfScannerOption: opt.MisconfScannerOption, - SecretScannerOption: opt.SecretScannerOption, - LicenseScannerOption: opt.LicenseScannerOption, - }) + a, err := analyzer.NewAnalyzerGroup(opt.AnalyzerOptions()) if err != nil { return nil, xerrors.Errorf("analyzer group error: %w", err) } @@ -135,7 +128,7 @@ func NewArtifact(target string, c cache.ArtifactCache, opt artifact.Option) (art cache: c, analyzer: a, handlerManager: handlerManager, - walker: walker.NewVM(opt.SkipFiles, opt.SkipDirs, opt.Slow), + walker: walker.NewVM(opt.SkipFiles, opt.SkipDirs), artifactOption: opt, } diff --git a/pkg/fanal/walker/fs.go b/pkg/fanal/walker/fs.go index e2e9d58c5fa4..863d3486997f 100644 --- a/pkg/fanal/walker/fs.go +++ b/pkg/fanal/walker/fs.go @@ -16,10 +16,11 @@ type ErrorCallback func(pathname string, err error) error type FS struct { walker + parallel int errCallback ErrorCallback } -func NewFS(skipFiles, skipDirs []string, slow bool, errCallback ErrorCallback) FS { +func NewFS(skipFiles, skipDirs []string, parallel int, errCallback ErrorCallback) FS { if errCallback == nil { errCallback = func(pathname string, err error) error { // ignore permission errors @@ -32,7 +33,8 @@ func NewFS(skipFiles, skipDirs []string, slow bool, errCallback ErrorCallback) F } return FS{ - walker: newWalker(skipFiles, skipDirs, slow), + walker: newWalker(skipFiles, skipDirs), + parallel: parallel, errCallback: errCallback, } } @@ -69,7 +71,7 @@ func (w FS) Walk(root string, fn WalkFunc) error { return nil } - if w.slow { + if w.parallel <= 1 { // In series: fast, with higher CPU/memory return w.walkSlow(root, walkFn) } diff --git a/pkg/fanal/walker/fs_test.go b/pkg/fanal/walker/fs_test.go index 038e15ccc7d9..aa84f6d23503 100644 --- a/pkg/fanal/walker/fs_test.go +++ b/pkg/fanal/walker/fs_test.go @@ -93,7 +93,7 @@ func TestDir_Walk(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - w := walker.NewFS(tt.fields.skipFiles, tt.fields.skipDirs, true, tt.fields.errCallback) + w := walker.NewFS(tt.fields.skipFiles, tt.fields.skipDirs, 1, tt.fields.errCallback) err := w.Walk(tt.rootDir, tt.analyzeFn) if tt.wantErr != "" { diff --git a/pkg/fanal/walker/tar.go b/pkg/fanal/walker/tar.go index ac4faaa070ac..d548d06f9c0a 100644 --- a/pkg/fanal/walker/tar.go +++ b/pkg/fanal/walker/tar.go @@ -25,14 +25,10 @@ type LayerTar struct { threshold int64 } -func NewLayerTar(skipFiles, skipDirs []string, slow bool) LayerTar { +func NewLayerTar(skipFiles, skipDirs []string) LayerTar { threshold := defaultSizeThreshold - if slow { - threshold = slowSizeThreshold - } - return LayerTar{ - walker: newWalker(skipFiles, skipDirs, slow), + walker: newWalker(skipFiles, skipDirs), threshold: threshold, } } diff --git a/pkg/fanal/walker/tar_test.go b/pkg/fanal/walker/tar_test.go index 8f1cb98a8010..fed2f9efea7d 100644 --- a/pkg/fanal/walker/tar_test.go +++ b/pkg/fanal/walker/tar_test.go @@ -81,7 +81,7 @@ func TestLayerTar_Walk(t *testing.T) { f, err := os.Open("testdata/test.tar") require.NoError(t, err) - w := walker.NewLayerTar(tt.fields.skipFiles, tt.fields.skipDirs, true) + w := walker.NewLayerTar(tt.fields.skipFiles, tt.fields.skipDirs) gotOpqDirs, gotWhFiles, err := w.Walk(f, tt.analyzeFn) if tt.wantErr != "" { diff --git a/pkg/fanal/walker/vm.go b/pkg/fanal/walker/vm.go index 52ddfe0b50ff..5d61d021388f 100644 --- a/pkg/fanal/walker/vm.go +++ b/pkg/fanal/walker/vm.go @@ -39,14 +39,10 @@ type VM struct { analyzeFn WalkFunc } -func NewVM(skipFiles, skipDirs []string, slow bool) VM { +func NewVM(skipFiles, skipDirs []string) VM { threshold := defaultSizeThreshold - if slow { - threshold = slowSizeThreshold - } - return VM{ - walker: newWalker(skipFiles, skipDirs, slow), + walker: newWalker(skipFiles, skipDirs), threshold: threshold, } } diff --git a/pkg/fanal/walker/walk.go b/pkg/fanal/walker/walk.go index 7205972b98e4..cebc86ee76d7 100644 --- a/pkg/fanal/walker/walk.go +++ b/pkg/fanal/walker/walk.go @@ -22,20 +22,16 @@ var ( } ) -const ( - defaultSizeThreshold = int64(200) << 20 // 200MB - slowSizeThreshold = int64(100) << 10 // 10KB -) +const defaultSizeThreshold = int64(200) << 20 // 200MB type WalkFunc func(filePath string, info os.FileInfo, opener analyzer.Opener) error type walker struct { skipFiles []string skipDirs []string - slow bool } -func newWalker(skipFiles, skipDirs []string, slow bool) walker { +func newWalker(skipFiles, skipDirs []string) walker { var cleanSkipFiles, cleanSkipDirs []string for _, skipFile := range skipFiles { skipFile = filepath.ToSlash(filepath.Clean(skipFile)) @@ -52,7 +48,6 @@ func newWalker(skipFiles, skipDirs []string, slow bool) walker { return walker{ skipFiles: cleanSkipFiles, skipDirs: cleanSkipDirs, - slow: slow, } } diff --git a/pkg/fanal/walker/walk_test.go b/pkg/fanal/walker/walk_test.go index 8b19a92899f1..4d52117aa1ce 100644 --- a/pkg/fanal/walker/walk_test.go +++ b/pkg/fanal/walker/walk_test.go @@ -54,7 +54,7 @@ func Test_shouldSkipFile(t *testing.T) { for i, tc := range testCases { t.Run(fmt.Sprint(i), func(t *testing.T) { - w := newWalker(tc.skipFiles, nil, false) + w := newWalker(tc.skipFiles, nil) for file, skipResult := range tc.skipMap { assert.Equal(t, skipResult, w.shouldSkipFile(filepath.ToSlash(filepath.Clean(file))), fmt.Sprintf("skipFiles: %s, file: %s", tc.skipFiles, file)) } @@ -115,7 +115,7 @@ func Test_shouldSkipDir(t *testing.T) { for i, tc := range testCases { t.Run(fmt.Sprint(i), func(t *testing.T) { - w := newWalker(nil, tc.skipDirs, false) + w := newWalker(nil, tc.skipDirs) for dir, skipResult := range tc.skipMap { assert.Equal(t, skipResult, w.shouldSkipDir(filepath.ToSlash(filepath.Clean(dir))), fmt.Sprintf("skipDirs: %s, dir: %s", tc.skipDirs, dir)) } diff --git a/pkg/flag/kubernetes_flags.go b/pkg/flag/kubernetes_flags.go index fd560e7defc4..3bd3ac25d496 100644 --- a/pkg/flag/kubernetes_flags.go +++ b/pkg/flag/kubernetes_flags.go @@ -6,7 +6,6 @@ import ( "strings" "github.com/samber/lo" - "golang.org/x/xerrors" corev1 "k8s.io/api/core/v1" ) @@ -52,12 +51,6 @@ var ( Default: "", Usage: "specify k8s version to validate outdated api by it (example: 1.21.0)", } - ParallelFlag = Flag{ - Name: "parallel", - ConfigName: "kubernetes.parallel", - Default: 5, - Usage: "number (between 1-20) of goroutines enabled for parallel scanning", - } TolerationsFlag = Flag{ Name: "tolerations", ConfigName: "kubernetes.tolerations", @@ -97,7 +90,6 @@ type K8sFlagGroup struct { KubeConfig *Flag Components *Flag K8sVersion *Flag - Parallel *Flag Tolerations *Flag AllNamespaces *Flag NodeCollectorNamespace *Flag @@ -111,7 +103,6 @@ type K8sOptions struct { KubeConfig string Components []string K8sVersion string - Parallel int Tolerations []corev1.Toleration AllNamespaces bool NodeCollectorNamespace string @@ -126,7 +117,6 @@ func NewK8sFlagGroup() *K8sFlagGroup { KubeConfig: &KubeConfigFlag, Components: &ComponentsFlag, K8sVersion: &K8sVersionFlag, - Parallel: &ParallelFlag, Tolerations: &TolerationsFlag, AllNamespaces: &AllNamespaces, NodeCollectorNamespace: &NodeCollectorNamespace, @@ -146,7 +136,6 @@ func (f *K8sFlagGroup) Flags() []*Flag { f.KubeConfig, f.Components, f.K8sVersion, - f.Parallel, f.Tolerations, f.AllNamespaces, f.NodeCollectorNamespace, @@ -160,14 +149,7 @@ func (f *K8sFlagGroup) ToOptions() (K8sOptions, error) { if err != nil { return K8sOptions{}, err } - var parallel int - if f.Parallel != nil { - parallel = getInt(f.Parallel) - // check parallel flag is a valid number between 1-20 - if parallel < 1 || parallel > 20 { - return K8sOptions{}, xerrors.Errorf("unable to parse parallel value, please ensure that the value entered is a valid number between 1-20.") - } - } + exludeNodeLabels := make(map[string]string) exludeNodes := getStringSlice(f.ExcludeNodes) for _, exludeNodeValue := range exludeNodes { @@ -184,7 +166,6 @@ func (f *K8sFlagGroup) ToOptions() (K8sOptions, error) { KubeConfig: getString(f.KubeConfig), Components: getStringSlice(f.Components), K8sVersion: getString(f.K8sVersion), - Parallel: parallel, Tolerations: tolerations, AllNamespaces: getBool(f.AllNamespaces), NodeCollectorNamespace: getString(f.NodeCollectorNamespace), diff --git a/pkg/flag/scan_flags.go b/pkg/flag/scan_flags.go index 45e29cbdcd4a..0464436a4f11 100644 --- a/pkg/flag/scan_flags.go +++ b/pkg/flag/scan_flags.go @@ -1,6 +1,8 @@ package flag import ( + "runtime" + "github.com/aquasecurity/trivy/pkg/log" "github.com/aquasecurity/trivy/pkg/types" xstrings "github.com/aquasecurity/trivy/pkg/x/strings" @@ -70,6 +72,13 @@ var ( ConfigName: "scan.slow", Default: false, Usage: "scan over time with lower CPU and memory utilization", + Deprecated: true, + } + ParallelFlag = Flag{ + Name: "parallel", + ConfigName: "scan.parallel", + Default: 5, + Usage: "number of goroutines enabled for parallel scanning, set 0 to auto-detect parallelism", } SBOMSourcesFlag = Flag{ Name: "sbom-sources", @@ -98,7 +107,8 @@ type ScanFlagGroup struct { OfflineScan *Flag Scanners *Flag FilePatterns *Flag - Slow *Flag + Slow *Flag // deprecated + Parallel *Flag SBOMSources *Flag RekorURL *Flag IncludeDevDeps *Flag @@ -111,7 +121,7 @@ type ScanOptions struct { OfflineScan bool Scanners types.Scanners FilePatterns []string - Slow bool + Parallel int SBOMSources []string RekorURL string IncludeDevDeps bool @@ -124,10 +134,11 @@ func NewScanFlagGroup() *ScanFlagGroup { OfflineScan: &OfflineScanFlag, Scanners: &ScannersFlag, FilePatterns: &FilePatternsFlag, - Slow: &SlowFlag, + Parallel: &ParallelFlag, SBOMSources: &SBOMSourcesFlag, RekorURL: &RekorURLFlag, IncludeDevDeps: &IncludeDevDepsFlag, + Slow: &SlowFlag, } } @@ -143,6 +154,7 @@ func (f *ScanFlagGroup) Flags() []*Flag { f.Scanners, f.FilePatterns, f.Slow, + f.Parallel, f.SBOMSources, f.RekorURL, f.IncludeDevDeps, @@ -155,6 +167,12 @@ func (f *ScanFlagGroup) ToOptions(args []string) (ScanOptions, error) { target = args[0] } + parallel := getInt(f.Parallel) + if f.Parallel != nil && parallel == 0 { + log.Logger.Infof("Set '--parallel' to the number of CPUs (%d)", runtime.NumCPU()) + parallel = runtime.NumCPU() + } + return ScanOptions{ Target: target, SkipDirs: getStringSlice(f.SkipDirs), @@ -162,7 +180,7 @@ func (f *ScanFlagGroup) ToOptions(args []string) (ScanOptions, error) { OfflineScan: getBool(f.OfflineScan), Scanners: getUnderlyingStringSlice[types.Scanner](f.Scanners), FilePatterns: getStringSlice(f.FilePatterns), - Slow: getBool(f.Slow), + Parallel: parallel, SBOMSources: getStringSlice(f.SBOMSources), RekorURL: getString(f.RekorURL), IncludeDevDeps: getBool(f.IncludeDevDeps), diff --git a/pkg/parallel/pipeline.go b/pkg/parallel/pipeline.go index d6879d7a1108..7dbc70f3e17c 100644 --- a/pkg/parallel/pipeline.go +++ b/pkg/parallel/pipeline.go @@ -7,6 +7,8 @@ import ( "golang.org/x/sync/errgroup" ) +const defaultWorkers = 5 + // Pipeline represents a structure for performing parallel processing. // T represents the input element type and U represents the output element type. type Pipeline[T, U any] struct { @@ -29,6 +31,9 @@ func NewPipeline[T, U any](numWorkers int, progress bool, items []T, // In case where there is no need to process the return values fn2 = func(_ U) error { return nil } } + if numWorkers == 0 { + numWorkers = defaultWorkers + } return Pipeline[T, U]{ numWorkers: numWorkers, progress: progress, diff --git a/pkg/parallel/walk.go b/pkg/parallel/walk.go index 09ba2b99ce85..a160e00acffe 100644 --- a/pkg/parallel/walk.go +++ b/pkg/parallel/walk.go @@ -12,10 +12,12 @@ import ( "github.com/aquasecurity/trivy/pkg/log" ) +const defaultParallel = 5 + type onFile[T any] func(string, fs.FileInfo, dio.ReadSeekerAt) (T, error) type onWalkResult[T any] func(T) error -func WalkDir[T any](ctx context.Context, fsys fs.FS, root string, slow bool, +func WalkDir[T any](ctx context.Context, fsys fs.FS, root string, parallel int, onFile onFile[T], onResult onWalkResult[T]) error { g, ctx := errgroup.WithContext(ctx) @@ -54,11 +56,10 @@ func WalkDir[T any](ctx context.Context, fsys fs.FS, root string, slow bool, // Start a fixed number of goroutines to read and digest files. c := make(chan T) - limit := 10 - if slow { - limit = 1 + if parallel == 0 { + parallel = defaultParallel } - for i := 0; i < limit; i++ { + for i := 0; i < parallel; i++ { g.Go(func() error { for path := range paths { if err := walk(ctx, fsys, path, c, onFile); err != nil { diff --git a/pkg/semaphore/semaphore.go b/pkg/semaphore/semaphore.go index f985cc578617..52005ab43e79 100644 --- a/pkg/semaphore/semaphore.go +++ b/pkg/semaphore/semaphore.go @@ -4,27 +4,9 @@ import "golang.org/x/sync/semaphore" const defaultSize = 5 -type options struct { - size int64 -} - -type option func(*options) - -func WithDefault(n int64) option { - return func(opts *options) { - opts.size = defaultSize - } -} - -func New(slow bool, opts ...option) *semaphore.Weighted { - o := &options{size: defaultSize} - for _, opt := range opts { - opt(o) - } - if slow { - // Process in series - return semaphore.NewWeighted(1) +func New(parallel int) *semaphore.Weighted { + if parallel == 0 { + parallel = defaultSize } - // Process in parallel - return semaphore.NewWeighted(o.size) + return semaphore.NewWeighted(int64(parallel)) }