diff --git a/pkg/fanal/analyzer/analyzer_test.go b/pkg/fanal/analyzer/analyzer_test.go index c008610024a..abc7a41f6f5 100644 --- a/pkg/fanal/analyzer/analyzer_test.go +++ b/pkg/fanal/analyzer/analyzer_test.go @@ -16,15 +16,21 @@ import ( "github.com/aquasecurity/trivy/pkg/fanal/analyzer" aos "github.com/aquasecurity/trivy/pkg/fanal/analyzer/os" "github.com/aquasecurity/trivy/pkg/fanal/types" + "github.com/aquasecurity/trivy/pkg/javadb" + "github.com/aquasecurity/trivy/pkg/mapfs" + "github.com/aquasecurity/trivy/pkg/syncx" _ "github.com/aquasecurity/trivy/pkg/fanal/analyzer/imgconf/apk" _ "github.com/aquasecurity/trivy/pkg/fanal/analyzer/language/java/jar" + _ "github.com/aquasecurity/trivy/pkg/fanal/analyzer/language/python/poetry" _ "github.com/aquasecurity/trivy/pkg/fanal/analyzer/language/ruby/bundler" _ "github.com/aquasecurity/trivy/pkg/fanal/analyzer/os/alpine" _ "github.com/aquasecurity/trivy/pkg/fanal/analyzer/os/ubuntu" _ "github.com/aquasecurity/trivy/pkg/fanal/analyzer/pkg/apk" _ "github.com/aquasecurity/trivy/pkg/fanal/analyzer/repo/apk" _ "github.com/aquasecurity/trivy/pkg/fanal/handler/all" + + _ "modernc.org/sqlite" ) func TestAnalysisResult_Merge(t *testing.T) { @@ -546,6 +552,79 @@ func TestAnalyzerGroup_AnalyzeFile(t *testing.T) { } } +func TestAnalyzerGroup_PostAnalyze(t *testing.T) { + tests := []struct { + name string + dir string + analyzerType analyzer.Type + want *analyzer.AnalysisResult + }{ + { + name: "jars with invalid jar", + dir: "testdata/post-apps/jar/", + analyzerType: analyzer.TypeJar, + want: &analyzer.AnalysisResult{ + Applications: []types.Application{ + { + Type: string(analyzer.TypeJar), + FilePath: "testdata/post-apps/jar/jackson-annotations-2.15.0-rc2.jar", + Libraries: []types.Package{ + { + Name: "com.fasterxml.jackson.core:jackson-annotations", + Version: "2.15.0-rc2", + FilePath: "testdata/post-apps/jar/jackson-annotations-2.15.0-rc2.jar", + }, + }, + }, + }, + }, + }, + { + name: "poetry files with invalid file", + dir: "testdata/post-apps/poetry/", + analyzerType: analyzer.TypePoetry, + want: &analyzer.AnalysisResult{ + Applications: []types.Application{ + { + Type: string(analyzer.TypePoetry), + FilePath: "testdata/post-apps/poetry/happy/poetry.lock", + Libraries: []types.Package{ + { + ID: "certifi@2022.12.7", + Name: "certifi", + Version: "2022.12.7", + }, + }, + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + a, err := analyzer.NewAnalyzerGroup(analyzer.AnalyzerOptions{}) + require.NoError(t, err) + + // Create a virtual filesystem + files := new(syncx.Map[analyzer.Type, *mapfs.FS]) + mfs := mapfs.New() + require.NoError(t, mfs.CopyFilesUnder(tt.dir)) + files.Store(tt.analyzerType, mfs) + + if tt.analyzerType == analyzer.TypeJar { + // init java-trivy-db with skip update + javadb.Init("./language/java/jar/testdata", "ghcr.io/aquasecurity/trivy-java-db", true, false, false) + } + + ctx := context.Background() + got := new(analyzer.AnalysisResult) + err = a.PostAnalyze(ctx, files, got, analyzer.AnalysisOptions{}) + require.NoError(t, err) + assert.Equal(t, tt.want, got) + }) + } +} + func TestAnalyzerGroup_AnalyzerVersions(t *testing.T) { tests := []struct { name string @@ -565,7 +644,8 @@ func TestAnalyzerGroup_AnalyzerVersions(t *testing.T) { "ubuntu-esm": 1, }, PostAnalyzers: map[string]int{ - "jar": 1, + "jar": 1, + "poetry": 1, }, }, }, @@ -583,7 +663,9 @@ func TestAnalyzerGroup_AnalyzerVersions(t *testing.T) { "apk": 2, "bundler": 1, }, - PostAnalyzers: map[string]int{}, + PostAnalyzers: map[string]int{ + "poetry": 1, + }, }, }, } diff --git a/pkg/fanal/analyzer/language/golang/mod/mod.go b/pkg/fanal/analyzer/language/golang/mod/mod.go index 524020c2a85..d66f22e8b57 100644 --- a/pkg/fanal/analyzer/language/golang/mod/mod.go +++ b/pkg/fanal/analyzer/language/golang/mod/mod.go @@ -65,18 +65,12 @@ func newGoModAnalyzer(opt analyzer.AnalyzerOptions) (analyzer.PostAnalyzer, erro func (a *gomodAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAnalysisInput) (*analyzer.AnalysisResult, error) { var apps []types.Application - err := fs.WalkDir(input.FS, ".", func(path string, d fs.DirEntry, err error) error { - if err != nil { - return err - } else if !d.Type().IsRegular() { - return nil - } - dir, file := filepath.Split(path) - if file != types.GoMod { - return nil - } + required := func(path string, d fs.DirEntry) bool { + return filepath.Base(path) == types.GoMod + } + err := fsutils.WalkDir(input.FS, ".", required, func(path string, d fs.DirEntry, r dio.ReadSeekerAt) error { // Parse go.mod gomod, err := parse(input.FS, path, a.modParser) if err != nil { @@ -87,7 +81,7 @@ func (a *gomodAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAnalys if lessThanGo117(gomod) { // e.g. /app/go.mod => /app/go.sum - sumPath := filepath.Join(dir, types.GoSum) + sumPath := filepath.Join(filepath.Dir(path), types.GoSum) gosum, err := parse(input.FS, sumPath, a.sumParser) if err != nil && !errors.Is(err, fs.ErrNotExist) { return xerrors.Errorf("parse error: %w", err) diff --git a/pkg/fanal/analyzer/language/golang/mod/mod_test.go b/pkg/fanal/analyzer/language/golang/mod/mod_test.go index fa23e01095b..342764d9e86 100644 --- a/pkg/fanal/analyzer/language/golang/mod/mod_test.go +++ b/pkg/fanal/analyzer/language/golang/mod/mod_test.go @@ -15,10 +15,9 @@ import ( func Test_gomodAnalyzer_Analyze(t *testing.T) { tests := []struct { - name string - dir string - want *analyzer.AnalysisResult - wantErr string + name string + dir string + want *analyzer.AnalysisResult }{ { name: "happy", @@ -121,9 +120,9 @@ func Test_gomodAnalyzer_Analyze(t *testing.T) { }, }, { - name: "sad go.mod", - dir: "testdata/sad", - wantErr: "unknown directive", + name: "sad go.mod", + dir: "testdata/sad", + want: &analyzer.AnalysisResult{}, }, } for _, tt := range tests { @@ -136,12 +135,9 @@ func Test_gomodAnalyzer_Analyze(t *testing.T) { got, err := a.PostAnalyze(ctx, analyzer.PostAnalysisInput{ FS: os.DirFS(tt.dir), }) + assert.NoError(t, err) - if tt.wantErr != "" { - require.ErrorContains(t, err, tt.wantErr) - return - } - if got != nil { + if len(got.Applications) > 0 { slices.SortFunc(got.Applications[0].Libraries, func(a, b types.Package) bool { return a.Name < b.Name }) diff --git a/pkg/fanal/analyzer/language/java/jar/jar_test.go b/pkg/fanal/analyzer/language/java/jar/jar_test.go index c4219875ece..185cfbc6133 100644 --- a/pkg/fanal/analyzer/language/java/jar/jar_test.go +++ b/pkg/fanal/analyzer/language/java/jar/jar_test.go @@ -6,13 +6,11 @@ import ( "path/filepath" "testing" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/aquasecurity/trivy/pkg/fanal/analyzer" "github.com/aquasecurity/trivy/pkg/fanal/types" "github.com/aquasecurity/trivy/pkg/javadb" "github.com/aquasecurity/trivy/pkg/mapfs" + "github.com/stretchr/testify/assert" _ "modernc.org/sqlite" ) @@ -27,7 +25,6 @@ func Test_javaLibraryAnalyzer_Analyze(t *testing.T) { inputFile string includeChecksum bool want *analyzer.AnalysisResult - wantErr string }{ { name: "happy path (WAR file)", @@ -126,7 +123,7 @@ func Test_javaLibraryAnalyzer_Analyze(t *testing.T) { { name: "sad path", inputFile: "testdata/test.txt", - wantErr: "not a valid zip file", + want: &analyzer.AnalysisResult{}, }, } for _, tt := range tests { @@ -148,11 +145,6 @@ func Test_javaLibraryAnalyzer_Analyze(t *testing.T) { Options: analyzer.AnalysisOptions{FileChecksum: tt.includeChecksum}, }) - if tt.wantErr != "" { - require.NotNil(t, err) - assert.Contains(t, err.Error(), tt.wantErr) - return - } assert.NoError(t, err) assert.Equal(t, tt.want, got) }) diff --git a/pkg/fanal/analyzer/language/nodejs/npm/npm_test.go b/pkg/fanal/analyzer/language/nodejs/npm/npm_test.go index 91a36409be0..f5e4e1a5844 100644 --- a/pkg/fanal/analyzer/language/nodejs/npm/npm_test.go +++ b/pkg/fanal/analyzer/language/nodejs/npm/npm_test.go @@ -22,10 +22,9 @@ func TestMain(m *testing.M) { func Test_npmLibraryAnalyzer_Analyze(t *testing.T) { tests := []struct { - name string - dir string - want *analyzer.AnalysisResult - wantErr string + name string + dir string + want *analyzer.AnalysisResult }{ { name: "with node_modules", @@ -156,9 +155,9 @@ func Test_npmLibraryAnalyzer_Analyze(t *testing.T) { }, }, { - name: "sad path", - dir: "testdata/sad", - wantErr: "failed to parse", + name: "sad path", + dir: "testdata/sad", + want: &analyzer.AnalysisResult{}, }, } for _, tt := range tests { @@ -170,13 +169,10 @@ func Test_npmLibraryAnalyzer_Analyze(t *testing.T) { FS: os.DirFS(tt.dir), }) - if tt.wantErr != "" { - assert.ErrorContains(t, err, tt.wantErr) - return - } - assert.NoError(t, err) - sortPkgs(got.Applications[0].Libraries) + if len(got.Applications) > 0 { + sortPkgs(got.Applications[0].Libraries) + } assert.Equal(t, tt.want, got) }) } diff --git a/pkg/fanal/analyzer/language/nodejs/yarn/yarn_test.go b/pkg/fanal/analyzer/language/nodejs/yarn/yarn_test.go index 45482ecb7b7..d108b3feb30 100644 --- a/pkg/fanal/analyzer/language/nodejs/yarn/yarn_test.go +++ b/pkg/fanal/analyzer/language/nodejs/yarn/yarn_test.go @@ -14,10 +14,9 @@ import ( func Test_yarnLibraryAnalyzer_Analyze(t *testing.T) { tests := []struct { - name string - dir string - want *analyzer.AnalysisResult - wantErr string + name string + dir string + want *analyzer.AnalysisResult }{ { name: "happy path", @@ -241,11 +240,6 @@ func Test_yarnLibraryAnalyzer_Analyze(t *testing.T) { FS: os.DirFS(tt.dir), }) - if tt.wantErr != "" { - assert.ErrorContains(t, err, tt.wantErr) - return - } - assert.NoError(t, err) assert.Equal(t, tt.want, got) }) diff --git a/pkg/fanal/analyzer/language/php/composer/composer_test.go b/pkg/fanal/analyzer/language/php/composer/composer_test.go index 12f890bb810..d2d4bb36f03 100644 --- a/pkg/fanal/analyzer/language/php/composer/composer_test.go +++ b/pkg/fanal/analyzer/language/php/composer/composer_test.go @@ -12,10 +12,9 @@ import ( func Test_composerAnalyzer_PostAnalyze(t *testing.T) { tests := []struct { - name string - dir string - want *analyzer.AnalysisResult - wantErr string + name string + dir string + want *analyzer.AnalysisResult }{ { name: "happy path", @@ -141,9 +140,9 @@ func Test_composerAnalyzer_PostAnalyze(t *testing.T) { }, }, { - name: "broken composer.lock", - dir: "testdata/sad", - wantErr: "failed to parse composer.lock", + name: "broken composer.lock", + dir: "testdata/sad", + want: &analyzer.AnalysisResult{}, }, } @@ -156,11 +155,6 @@ func Test_composerAnalyzer_PostAnalyze(t *testing.T) { FS: os.DirFS(tt.dir), }) - if tt.wantErr != "" { - assert.ErrorContains(t, err, tt.wantErr) - return - } - assert.NoError(t, err) assert.Equal(t, tt.want, got) }) diff --git a/pkg/fanal/analyzer/language/python/poetry/poetry_test.go b/pkg/fanal/analyzer/language/python/poetry/poetry_test.go index 2bebe9493be..5dfefa6423c 100644 --- a/pkg/fanal/analyzer/language/python/poetry/poetry_test.go +++ b/pkg/fanal/analyzer/language/python/poetry/poetry_test.go @@ -14,10 +14,9 @@ import ( func Test_poetryLibraryAnalyzer_Analyze(t *testing.T) { tests := []struct { - name string - dir string - want *analyzer.AnalysisResult - wantErr string + name string + dir string + want *analyzer.AnalysisResult }{ { name: "happy path", @@ -167,9 +166,9 @@ func Test_poetryLibraryAnalyzer_Analyze(t *testing.T) { }, }, { - name: "broken poetry.lock", - dir: "testdata/sad", - wantErr: "failed to parse poetry.lock", + name: "broken poetry.lock", + dir: "testdata/sad", + want: &analyzer.AnalysisResult{}, }, } @@ -182,11 +181,6 @@ func Test_poetryLibraryAnalyzer_Analyze(t *testing.T) { FS: os.DirFS(tt.dir), }) - if tt.wantErr != "" { - assert.ErrorContains(t, err, tt.wantErr) - return - } - assert.NoError(t, err) assert.Equal(t, tt.want, got) }) diff --git a/pkg/fanal/analyzer/language/rust/cargo/cargo_test.go b/pkg/fanal/analyzer/language/rust/cargo/cargo_test.go index 5766196ad26..7b49da976dd 100644 --- a/pkg/fanal/analyzer/language/rust/cargo/cargo_test.go +++ b/pkg/fanal/analyzer/language/rust/cargo/cargo_test.go @@ -13,10 +13,9 @@ import ( func Test_cargoAnalyzer_Analyze(t *testing.T) { tests := []struct { - name string - dir string - want *analyzer.AnalysisResult - wantErr string + name string + dir string + want *analyzer.AnalysisResult }{ { name: "happy path", @@ -247,9 +246,9 @@ func Test_cargoAnalyzer_Analyze(t *testing.T) { }, }, { - name: "broken Cargo.lock", - dir: "testdata/sad", - wantErr: "failed to parse Cargo.lock", + name: "broken Cargo.lock", + dir: "testdata/sad", + want: &analyzer.AnalysisResult{}, }, } @@ -262,11 +261,6 @@ func Test_cargoAnalyzer_Analyze(t *testing.T) { FS: os.DirFS(tt.dir), }) - if tt.wantErr != "" { - assert.ErrorContains(t, err, tt.wantErr) - return - } - assert.NoError(t, err) assert.Equal(t, tt.want, got) }) diff --git a/pkg/fanal/analyzer/testdata/post-apps/jar/invalid.jar b/pkg/fanal/analyzer/testdata/post-apps/jar/invalid.jar new file mode 100644 index 00000000000..d800886d9c8 --- /dev/null +++ b/pkg/fanal/analyzer/testdata/post-apps/jar/invalid.jar @@ -0,0 +1 @@ +123 \ No newline at end of file diff --git a/pkg/fanal/analyzer/testdata/post-apps/jar/jackson-annotations-2.15.0-rc2.jar b/pkg/fanal/analyzer/testdata/post-apps/jar/jackson-annotations-2.15.0-rc2.jar new file mode 100644 index 00000000000..b56a46744a6 Binary files /dev/null and b/pkg/fanal/analyzer/testdata/post-apps/jar/jackson-annotations-2.15.0-rc2.jar differ diff --git a/pkg/fanal/analyzer/testdata/post-apps/poetry/happy/poetry.lock b/pkg/fanal/analyzer/testdata/post-apps/poetry/happy/poetry.lock new file mode 100644 index 00000000000..221c62aef3e --- /dev/null +++ b/pkg/fanal/analyzer/testdata/post-apps/poetry/happy/poetry.lock @@ -0,0 +1,18 @@ +# This file is automatically @generated by Poetry and should not be changed by hand. + +[[package]] +name = "certifi" +version = "2022.12.7" +description = "Python package for providing Mozilla's CA Bundle." +category = "main" +optional = true +python-versions = ">=3.6" +files = [ + {file = "certifi-2022.12.7-py3-none-any.whl", hash = "sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18"}, + {file = "certifi-2022.12.7.tar.gz", hash = "sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3"}, +] + +[metadata] +lock-version = "2.0" +python-versions = "^3.9" +content-hash = "0ee6cb4bc2d84091d5dcb0a0110a65f244987ed427933b2f49949195e3ef69c7" diff --git a/pkg/fanal/analyzer/testdata/post-apps/poetry/sad/poetry.lock b/pkg/fanal/analyzer/testdata/post-apps/poetry/sad/poetry.lock new file mode 100644 index 00000000000..8e2f0bef135 --- /dev/null +++ b/pkg/fanal/analyzer/testdata/post-apps/poetry/sad/poetry.lock @@ -0,0 +1 @@ +[ \ No newline at end of file diff --git a/pkg/parallel/walk.go b/pkg/parallel/walk.go index 4cf832381d3..09ba2b99ce8 100644 --- a/pkg/parallel/walk.go +++ b/pkg/parallel/walk.go @@ -4,6 +4,7 @@ import ( "context" "io/fs" + "go.uber.org/zap" "golang.org/x/sync/errgroup" "golang.org/x/xerrors" @@ -104,7 +105,8 @@ func walk[T any](ctx context.Context, fsys fs.FS, path string, c chan T, onFile } res, err := onFile(path, info, rsa) if err != nil { - return xerrors.Errorf("on file: %w", err) + log.Logger.Debugw("Walk error", zap.String("file_path", path), zap.Error(err)) + return nil } select { diff --git a/pkg/utils/fsutils/fs.go b/pkg/utils/fsutils/fs.go index 6bbee5beddf..9e66ae08c1e 100644 --- a/pkg/utils/fsutils/fs.go +++ b/pkg/utils/fsutils/fs.go @@ -7,9 +7,11 @@ import ( "os" "path/filepath" + "go.uber.org/zap" "golang.org/x/xerrors" dio "github.com/aquasecurity/go-dep-parser/pkg/io" + "github.com/aquasecurity/trivy/pkg/log" ) const ( @@ -106,6 +108,9 @@ func WalkDir(fsys fs.FS, root string, required WalkDirRequiredFunc, fn WalkDirFu } defer f.Close() - return fn(path, d, file) + if err = fn(path, d, file); err != nil { + log.Logger.Debugw("Walk error", zap.String("file_path", path), zap.Error(err)) + } + return nil }) }