From 9634b42746d1b88b25c7e25c125800a72c54308f Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Wed, 2 Nov 2022 11:31:57 -0400 Subject: [PATCH] port javascript cataloger to new generic cataloger pattern (#1308) Signed-off-by: Alex Goodman Signed-off-by: Alex Goodman --- syft/pkg/catalog.go | 20 +- .../internal/pkgtest/test_generic_parser.go | 13 + syft/pkg/cataloger/javascript/cataloger.go | 80 +----- .../cataloger/javascript/cataloger_test.go | 174 ++++++----- syft/pkg/cataloger/javascript/package.go | 176 ++++++++++++ syft/pkg/cataloger/javascript/package_test.go | 58 ++++ .../javascript/parse_package_json.go | 80 ++---- .../javascript/parse_package_json_test.go | 48 +--- .../javascript/parse_package_lock.go | 46 ++- .../javascript/parse_package_lock_test.go | 137 ++++----- .../cataloger/javascript/parse_pnpm_lock.go | 18 +- .../javascript/parse_pnpm_lock_test.go | 69 ++--- .../cataloger/javascript/parse_yarn_lock.go | 31 +- .../javascript/parse_yarn_lock_test.go | 270 +++++++++--------- syft/pkg/npm_package_json_metadata.go | 30 -- syft/pkg/npm_package_json_metadata_test.go | 63 ---- syft/pkg/package.go | 22 ++ 17 files changed, 685 insertions(+), 650 deletions(-) create mode 100644 syft/pkg/cataloger/javascript/package.go create mode 100644 syft/pkg/cataloger/javascript/package_test.go delete mode 100644 syft/pkg/npm_package_json_metadata_test.go diff --git a/syft/pkg/catalog.go b/syft/pkg/catalog.go index f9069ec7b94..6445f98a84e 100644 --- a/syft/pkg/catalog.go +++ b/syft/pkg/catalog.go @@ -1,7 +1,6 @@ package pkg import ( - "sort" "sync" "github.com/jinzhu/copier" @@ -199,24 +198,7 @@ func (c *Catalog) Sorted(types ...Type) (pkgs []Package) { pkgs = append(pkgs, p) } - sort.SliceStable(pkgs, func(i, j int) bool { - if pkgs[i].Name == pkgs[j].Name { - if pkgs[i].Version == pkgs[j].Version { - iLocations := pkgs[i].Locations.ToSlice() - jLocations := pkgs[j].Locations.ToSlice() - if pkgs[i].Type == pkgs[j].Type && len(iLocations) > 0 && len(jLocations) > 0 { - if iLocations[0].String() == jLocations[0].String() { - // compare IDs as a final fallback - return pkgs[i].ID() < pkgs[j].ID() - } - return iLocations[0].String() < jLocations[0].String() - } - return pkgs[i].Type < pkgs[j].Type - } - return pkgs[i].Version < pkgs[j].Version - } - return pkgs[i].Name < pkgs[j].Name - }) + Sort(pkgs) return pkgs } diff --git a/syft/pkg/cataloger/internal/pkgtest/test_generic_parser.go b/syft/pkg/cataloger/internal/pkgtest/test_generic_parser.go index cba41e1b78f..25f8edad816 100644 --- a/syft/pkg/cataloger/internal/pkgtest/test_generic_parser.go +++ b/syft/pkg/cataloger/internal/pkgtest/test_generic_parser.go @@ -40,6 +40,19 @@ func NewCatalogTester() *CatalogTester { } } +func (p *CatalogTester) FromDirectory(t *testing.T, path string) *CatalogTester { + t.Helper() + + s, err := source.NewFromDirectory(path) + require.NoError(t, err) + + resolver, err := s.FileResolver(source.AllLayersScope) + require.NoError(t, err) + + p.resolver = resolver + return p +} + func (p *CatalogTester) FromFile(t *testing.T, path string) *CatalogTester { t.Helper() diff --git a/syft/pkg/cataloger/javascript/cataloger.go b/syft/pkg/cataloger/javascript/cataloger.go index 150a537f2e6..3e5a93685e4 100644 --- a/syft/pkg/cataloger/javascript/cataloger.go +++ b/syft/pkg/cataloger/javascript/cataloger.go @@ -4,88 +4,24 @@ Package javascript provides a concrete Cataloger implementation for JavaScript e package javascript import ( - "encoding/json" - "io" - "path" - "strings" - "github.com/anchore/syft/internal" - "github.com/anchore/syft/internal/log" - "github.com/anchore/syft/syft/pkg" - "github.com/anchore/syft/syft/pkg/cataloger/common" "github.com/anchore/syft/syft/pkg/cataloger/generic" - "github.com/anchore/syft/syft/source" ) // NewJavascriptPackageCataloger returns a new JavaScript cataloger object based on detection of npm based packages. -func NewJavascriptPackageCataloger() *common.GenericCataloger { - globParsers := map[string]common.ParserFn{ - "**/package.json": parsePackageJSON, - } - - return common.NewGenericCataloger(nil, globParsers, "javascript-package-cataloger") +func NewJavascriptPackageCataloger() *generic.Cataloger { + return generic.NewCataloger("javascript-package-cataloger"). + WithParserByGlobs(parsePackageJSON, "**/package.json") } -// NewJavascriptLockCataloger returns a new Javascript cataloger object base on package lock files. -func NewJavascriptLockCataloger() *common.GenericCataloger { - globParsers := map[string]common.ParserFn{ - "**/package-lock.json": parsePackageLock, - "**/yarn.lock": parseYarnLock, - "**/pnpm-lock.yaml": parsePnpmLock, - } - - return common.NewGenericCataloger(nil, globParsers, "javascript-lock-cataloger", addLicenses) +func NewJavascriptLockCataloger() *generic.Cataloger { + return generic.NewCataloger("javascript-lock-cataloger"). + WithParserByGlobs(parsePackageLock, "**/package-lock.json"). + WithParserByGlobs(parseYarnLock, "**/yarn.lock"). + WithParserByGlobs(parsePnpmLock, "**/pnpm-lock.yaml") } func NewNodeBinaryCataloger() *generic.Cataloger { return generic.NewCataloger("node-binary-cataloger"). WithParserByMimeTypes(parseNodeBinary, internal.ExecutableMIMETypeSet.List()...) } - -func addLicenses(resolver source.FileResolver, location source.Location, p *pkg.Package) error { - dir := path.Dir(location.RealPath) - pkgPath := []string{dir, "node_modules"} - pkgPath = append(pkgPath, strings.Split(p.Name, "/")...) - pkgPath = append(pkgPath, "package.json") - pkgFile := path.Join(pkgPath...) - locations, err := resolver.FilesByPath(pkgFile) - if err != nil { - log.Debugf("an error occurred attempting to read: %s - %+v", pkgFile, err) - return nil - } - - if len(locations) == 0 { - return nil - } - - for _, location := range locations { - contentReader, err := resolver.FileContentsByLocation(location) - if err != nil { - log.Debugf("error getting file content reader for %s: %v", pkgFile, err) - return nil - } - - contents, err := io.ReadAll(contentReader) - if err != nil { - log.Debugf("error reading file contents for %s: %v", pkgFile, err) - return nil - } - - var pkgJSON packageJSON - err = json.Unmarshal(contents, &pkgJSON) - if err != nil { - log.Debugf("error parsing %s: %v", pkgFile, err) - return nil - } - - licenses, err := pkgJSON.licensesFromJSON() - if err != nil { - log.Debugf("error getting licenses from %s: %v", pkgFile, err) - return nil - } - - p.Licenses = append(p.Licenses, licenses...) - } - - return nil -} diff --git a/syft/pkg/cataloger/javascript/cataloger_test.go b/syft/pkg/cataloger/javascript/cataloger_test.go index 325d0b65fd6..1671781e8a6 100644 --- a/syft/pkg/cataloger/javascript/cataloger_test.go +++ b/syft/pkg/cataloger/javascript/cataloger_test.go @@ -3,100 +3,120 @@ package javascript import ( "testing" - "github.com/stretchr/testify/require" - "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/pkg/cataloger/internal/pkgtest" "github.com/anchore/syft/syft/source" ) func Test_JavascriptCataloger(t *testing.T) { - expected := map[string]pkg.Package{ - "@actions/core": { - Name: "@actions/core", - Version: "1.6.0", - Language: pkg.JavaScript, - Type: pkg.NpmPkg, - Licenses: []string{"MIT"}, + locationSet := source.NewLocationSet(source.NewLocation("package-lock.json")) + expectedPkgs := []pkg.Package{ + { + Name: "@actions/core", + Version: "1.6.0", + FoundBy: "javascript-lock-cataloger", + PURL: "pkg:npm/%40actions/core@1.6.0", + Locations: locationSet, + Language: pkg.JavaScript, + Type: pkg.NpmPkg, + Licenses: []string{"MIT"}, }, - "wordwrap": { - Name: "wordwrap", - Version: "0.0.3", - Language: pkg.JavaScript, - Type: pkg.NpmPkg, + { + Name: "ansi-regex", + Version: "3.0.0", + FoundBy: "javascript-lock-cataloger", + PURL: "pkg:npm/ansi-regex@3.0.0", + Locations: locationSet, + Language: pkg.JavaScript, + Type: pkg.NpmPkg, }, - "get-stdin": { - Name: "get-stdin", - Version: "5.0.1", - Language: pkg.JavaScript, - Type: pkg.NpmPkg, + { + Name: "cowsay", + Version: "1.4.0", + FoundBy: "javascript-lock-cataloger", + PURL: "pkg:npm/cowsay@1.4.0", + Locations: locationSet, + Language: pkg.JavaScript, + Type: pkg.NpmPkg, + Licenses: []string{"MIT"}, }, - "minimist": { - Name: "minimist", - Version: "0.0.10", - Language: pkg.JavaScript, - Type: pkg.NpmPkg, + { + Name: "get-stdin", + Version: "5.0.1", + FoundBy: "javascript-lock-cataloger", + PURL: "pkg:npm/get-stdin@5.0.1", + Locations: locationSet, + Language: pkg.JavaScript, + Type: pkg.NpmPkg, }, - "optimist": { - Name: "optimist", - Version: "0.6.1", - Language: pkg.JavaScript, - Type: pkg.NpmPkg, + { + Name: "is-fullwidth-code-point", + Version: "2.0.0", + FoundBy: "javascript-lock-cataloger", + PURL: "pkg:npm/is-fullwidth-code-point@2.0.0", + Locations: locationSet, + Language: pkg.JavaScript, + Type: pkg.NpmPkg, }, - "string-width": { - Name: "string-width", - Version: "2.1.1", - Language: pkg.JavaScript, - Type: pkg.NpmPkg, + { + Name: "minimist", + Version: "0.0.10", + FoundBy: "javascript-lock-cataloger", + PURL: "pkg:npm/minimist@0.0.10", + Locations: locationSet, + Language: pkg.JavaScript, + Type: pkg.NpmPkg, }, - "strip-ansi": { - Name: "strip-ansi", - Version: "4.0.0", - Language: pkg.JavaScript, - Type: pkg.NpmPkg, + { + Name: "optimist", + Version: "0.6.1", + FoundBy: "javascript-lock-cataloger", + PURL: "pkg:npm/optimist@0.6.1", + Locations: locationSet, + Language: pkg.JavaScript, + Type: pkg.NpmPkg, }, - "strip-eof": { - Name: "wordwrap", - Version: "1.0.0", - Language: pkg.JavaScript, - Type: pkg.NpmPkg, + { + Name: "string-width", + Version: "2.1.1", + FoundBy: "javascript-lock-cataloger", + PURL: "pkg:npm/string-width@2.1.1", + Locations: locationSet, + Language: pkg.JavaScript, + Type: pkg.NpmPkg, }, - "ansi-regex": { - Name: "ansi-regex", - Version: "3.0.0", - Language: pkg.JavaScript, - Type: pkg.NpmPkg, + { + Name: "strip-ansi", + Version: "4.0.0", + FoundBy: "javascript-lock-cataloger", + PURL: "pkg:npm/strip-ansi@4.0.0", + Locations: locationSet, + Language: pkg.JavaScript, + Type: pkg.NpmPkg, }, - "is-fullwidth-code-point": { - Name: "is-fullwidth-code-point", - Version: "2.0.0", - Language: pkg.JavaScript, - Type: pkg.NpmPkg, + { + Name: "strip-eof", + Version: "1.0.0", + FoundBy: "javascript-lock-cataloger", + PURL: "pkg:npm/strip-eof@1.0.0", + Locations: locationSet, + Language: pkg.JavaScript, + Type: pkg.NpmPkg, }, - "cowsay": { - Name: "cowsay", - Version: "1.4.0", - Language: pkg.JavaScript, - Type: pkg.NpmPkg, - Licenses: []string{"MIT"}, + { + Name: "wordwrap", + Version: "0.0.3", + FoundBy: "javascript-lock-cataloger", + PURL: "pkg:npm/wordwrap@0.0.3", + Locations: locationSet, + Language: pkg.JavaScript, + Type: pkg.NpmPkg, }, } - s, err := source.NewFromDirectory("test-fixtures/pkg-lock") - require.NoError(t, err) - - resolver, err := s.FileResolver(source.AllLayersScope) - require.NoError(t, err) - - actual, _, err := NewJavascriptLockCataloger().Catalog(resolver) - if err != nil { - t.Fatalf("failed to parse package-lock.json: %+v", err) - } - - var pkgs []*pkg.Package - for _, p := range actual { - p2 := p - pkgs = append(pkgs, &p2) - } + pkgtest.NewCatalogTester(). + FromDirectory(t, "test-fixtures/pkg-lock"). + Expects(expectedPkgs, nil). + TestCataloger(t, NewJavascriptLockCataloger()) - assertPkgsEqual(t, pkgs, expected) } diff --git a/syft/pkg/cataloger/javascript/package.go b/syft/pkg/cataloger/javascript/package.go new file mode 100644 index 00000000000..810647df4de --- /dev/null +++ b/syft/pkg/cataloger/javascript/package.go @@ -0,0 +1,176 @@ +package javascript + +import ( + "encoding/json" + "io" + "path" + "strings" + + "github.com/anchore/packageurl-go" + "github.com/anchore/syft/internal/log" + "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/source" +) + +func newPackageJSONPackage(u packageJSON, locations ...source.Location) pkg.Package { + licenses, err := u.licensesFromJSON() + if err != nil { + log.Warnf("unable to extract licenses from javascript package.json: %+v", err) + } + + p := pkg.Package{ + Name: u.Name, + Version: u.Version, + Licenses: licenses, + PURL: packageURL(u.Name, u.Version), + Locations: source.NewLocationSet(locations...), + Language: pkg.JavaScript, + Type: pkg.NpmPkg, + MetadataType: pkg.NpmPackageJSONMetadataType, + Metadata: pkg.NpmPackageJSONMetadata{ + Name: u.Name, + Version: u.Version, + Author: u.Author.AuthorString(), + Homepage: u.Homepage, + URL: u.Repository.URL, + Licenses: licenses, + Private: u.Private, + }, + } + + p.SetID() + + return p +} + +func newPackageLockPackage(resolver source.FileResolver, location source.Location, name string, u lockDependency, licenseMap map[string]string) pkg.Package { + var sb strings.Builder + sb.WriteString(u.Resolved) + sb.WriteString(u.Integrity) + var licenses []string + if l, exists := licenseMap[sb.String()]; exists { + licenses = append(licenses, l) + } + + return finalizeLockPkg( + resolver, + location, + pkg.Package{ + Name: name, + Version: u.Version, + Locations: source.NewLocationSet(location), + PURL: packageURL(name, u.Version), + Language: pkg.JavaScript, + Type: pkg.NpmPkg, + Licenses: licenses, + }, + ) +} + +func newPnpmPackage(resolver source.FileResolver, location source.Location, name, version string) pkg.Package { + return finalizeLockPkg( + resolver, + location, + pkg.Package{ + Name: name, + Version: version, + Locations: source.NewLocationSet(location), + PURL: packageURL(name, version), + Language: pkg.JavaScript, + Type: pkg.NpmPkg, + }, + ) +} + +func newYarnLockPackage(resolver source.FileResolver, location source.Location, name, version string) pkg.Package { + return finalizeLockPkg( + resolver, + location, + pkg.Package{ + Name: name, + Version: version, + Locations: source.NewLocationSet(location), + PURL: packageURL(name, version), + Language: pkg.JavaScript, + Type: pkg.NpmPkg, + }, + ) +} + +func finalizeLockPkg(resolver source.FileResolver, location source.Location, p pkg.Package) pkg.Package { + p.Licenses = append(p.Licenses, addLicenses(p.Name, resolver, location)...) + p.SetID() + return p +} + +func addLicenses(name string, resolver source.FileResolver, location source.Location) (allLicenses []string) { + if resolver == nil { + return allLicenses + } + dir := path.Dir(location.RealPath) + pkgPath := []string{dir, "node_modules"} + pkgPath = append(pkgPath, strings.Split(name, "/")...) + pkgPath = append(pkgPath, "package.json") + pkgFile := path.Join(pkgPath...) + locations, err := resolver.FilesByPath(pkgFile) + + if err != nil { + log.Debugf("an error occurred attempting to read: %s - %+v", pkgFile, err) + return allLicenses + } + + if len(locations) == 0 { + return allLicenses + } + + for _, l := range locations { + contentReader, err := resolver.FileContentsByLocation(l) + if err != nil { + log.Debugf("error getting file content reader for %s: %v", pkgFile, err) + return allLicenses + } + + contents, err := io.ReadAll(contentReader) + if err != nil { + log.Debugf("error reading file contents for %s: %v", pkgFile, err) + return allLicenses + } + + var pkgJSON packageJSON + err = json.Unmarshal(contents, &pkgJSON) + if err != nil { + log.Debugf("error parsing %s: %v", pkgFile, err) + return allLicenses + } + + licenses, err := pkgJSON.licensesFromJSON() + if err != nil { + log.Debugf("error getting licenses from %s: %v", pkgFile, err) + return allLicenses + } + + allLicenses = append(allLicenses, licenses...) + } + + return allLicenses +} + +// packageURL returns the PURL for the specific NPM package (see https://github.com/package-url/purl-spec) +func packageURL(name, version string) string { + var namespace string + + fields := strings.SplitN(name, "/", 2) + if len(fields) > 1 { + namespace = fields[0] + name = fields[1] + } + + return packageurl.NewPackageURL( + packageurl.TypeNPM, + namespace, + name, + version, + nil, + "", + ).ToString() +} diff --git a/syft/pkg/cataloger/javascript/package_test.go b/syft/pkg/cataloger/javascript/package_test.go new file mode 100644 index 00000000000..4cfad820406 --- /dev/null +++ b/syft/pkg/cataloger/javascript/package_test.go @@ -0,0 +1,58 @@ +package javascript + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anchore/packageurl-go" +) + +func Test_packageURL(t *testing.T) { + + tests := []struct { + testName string + name string + version string + expected string + namespace string + }{ + { + testName: "no namespace", + name: "arborist", + version: "2.6.2", + expected: "pkg:npm/arborist@2.6.2", + }, + { + testName: "split by namespace", + name: "npmcli/arborist", + version: "2.6.2", + expected: "pkg:npm/npmcli/arborist@2.6.2", + namespace: "npmcli", + }, + { + testName: "encoding @ symobl", + name: "@npmcli/arborist", + version: "2.6.2", + expected: "pkg:npm/%40npmcli/arborist@2.6.2", + namespace: "@npmcli", + }, + } + for _, tt := range tests { + t.Run(tt.testName, func(t *testing.T) { + actual := packageURL(tt.name, tt.version) + assert.Equal(t, tt.expected, actual) + decoded, err := packageurl.FromString(actual) + require.NoError(t, err) + assert.Equal(t, tt.namespace, decoded.Namespace) + if decoded.Namespace != "" { + assert.Equal(t, tt.name, fmt.Sprintf("%s/%s", decoded.Namespace, decoded.Name)) + } else { + assert.Equal(t, tt.name, decoded.Name) + } + assert.Equal(t, tt.version, decoded.Version) + }) + } +} diff --git a/syft/pkg/cataloger/javascript/parse_package_json.go b/syft/pkg/cataloger/javascript/parse_package_json.go index 31a9f2c667c..ce0b5c71434 100644 --- a/syft/pkg/cataloger/javascript/parse_package_json.go +++ b/syft/pkg/cataloger/javascript/parse_package_json.go @@ -13,11 +13,12 @@ import ( "github.com/anchore/syft/internal/log" "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/pkg" - "github.com/anchore/syft/syft/pkg/cataloger/common" + "github.com/anchore/syft/syft/pkg/cataloger/generic" + "github.com/anchore/syft/syft/source" ) // integrity check -var _ common.ParserFn = parsePackageJSON +var _ generic.Parser = parsePackageJSON // packageJSON represents a JavaScript package.json file type packageJSON struct { @@ -49,6 +50,32 @@ type repository struct { // ---> name: "Isaac Z. Schlueter" email: "i@izs.me" url: "http://blog.izs.me" var authorPattern = regexp.MustCompile(`^\s*(?P[^<(]*)(\s+<(?P.*)>)?(\s\((?P.*)\))?\s*$`) +// parsePackageJSON parses a package.json and returns the discovered JavaScript packages. +func parsePackageJSON(_ source.FileResolver, _ *generic.Environment, reader source.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { + var pkgs []pkg.Package + dec := json.NewDecoder(reader) + + for { + var p packageJSON + if err := dec.Decode(&p); err == io.EOF { + break + } else if err != nil { + return nil, nil, fmt.Errorf("failed to parse package.json file: %w", err) + } + + if !p.hasNameAndVersionValues() { + log.Debugf("encountered package.json file without a name and/or version field, ignoring (path=%q)", reader.AccessPath()) + return nil, nil, nil + } + + pkgs = append(pkgs, newPackageJSONPackage(p, reader.Location)) + } + + pkg.Sort(pkgs) + + return pkgs, nil, nil +} + func (a *author) UnmarshalJSON(b []byte) error { var authorStr string var fields map[string]string @@ -172,55 +199,6 @@ func licensesFromJSON(b []byte) ([]license, error) { return nil, errors.New("unmarshal failed") } -// parsePackageJSON parses a package.json and returns the discovered JavaScript packages. -func parsePackageJSON(path string, reader io.Reader) ([]*pkg.Package, []artifact.Relationship, error) { - var packages []*pkg.Package - dec := json.NewDecoder(reader) - - for { - var p packageJSON - if err := dec.Decode(&p); err == io.EOF { - break - } else if err != nil { - return nil, nil, fmt.Errorf("failed to parse package.json file: %w", err) - } - - if !p.hasNameAndVersionValues() { - log.Debugf("encountered package.json file without a name and/or version field, ignoring (path=%q)", path) - return nil, nil, nil - } - - packages = append(packages, newPackageJSONPackage(p)) - } - - return packages, nil, nil -} - -func newPackageJSONPackage(p packageJSON) *pkg.Package { - licenses, err := p.licensesFromJSON() - if err != nil { - log.Warnf("unable to extract licenses from javascript package.json: %+v", err) - } - - return &pkg.Package{ - Name: p.Name, - Version: p.Version, - Licenses: licenses, - Language: pkg.JavaScript, - Type: pkg.NpmPkg, - MetadataType: pkg.NpmPackageJSONMetadataType, - Metadata: pkg.NpmPackageJSONMetadata{ - Name: p.Name, - Version: p.Version, - Author: p.Author.AuthorString(), - Homepage: p.Homepage, - URL: p.Repository.URL, - Licenses: licenses, - Private: p.Private, - }, - } -} - func (p packageJSON) hasNameAndVersionValues() bool { return p.Name != "" && p.Version != "" } diff --git a/syft/pkg/cataloger/javascript/parse_package_json_test.go b/syft/pkg/cataloger/javascript/parse_package_json_test.go index 9225737eede..9fe3722331c 100644 --- a/syft/pkg/cataloger/javascript/parse_package_json_test.go +++ b/syft/pkg/cataloger/javascript/parse_package_json_test.go @@ -1,13 +1,13 @@ package javascript import ( - "os" "testing" - "github.com/go-test/deep" "github.com/stretchr/testify/assert" "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/pkg/cataloger/internal/pkgtest" + "github.com/anchore/syft/syft/source" ) func TestParsePackageJSON(t *testing.T) { @@ -20,6 +20,7 @@ func TestParsePackageJSON(t *testing.T) { ExpectedPkg: pkg.Package{ Name: "npm", Version: "6.14.6", + PURL: "pkg:npm/npm@6.14.6", Type: pkg.NpmPkg, Licenses: []string{"Artistic-2.0"}, Language: pkg.JavaScript, @@ -39,6 +40,7 @@ func TestParsePackageJSON(t *testing.T) { ExpectedPkg: pkg.Package{ Name: "npm", Version: "6.14.6", + PURL: "pkg:npm/npm@6.14.6", Type: pkg.NpmPkg, Licenses: []string{"ISC"}, Language: pkg.JavaScript, @@ -58,6 +60,7 @@ func TestParsePackageJSON(t *testing.T) { ExpectedPkg: pkg.Package{ Name: "npm", Version: "6.14.6", + PURL: "pkg:npm/npm@6.14.6", Type: pkg.NpmPkg, Licenses: []string{"MIT", "Apache-2.0"}, Language: pkg.JavaScript, @@ -77,6 +80,7 @@ func TestParsePackageJSON(t *testing.T) { ExpectedPkg: pkg.Package{ Name: "npm", Version: "6.14.6", + PURL: "pkg:npm/npm@6.14.6", Type: pkg.NpmPkg, Licenses: nil, Language: pkg.JavaScript, @@ -96,6 +100,7 @@ func TestParsePackageJSON(t *testing.T) { ExpectedPkg: pkg.Package{ Name: "npm", Version: "6.14.6", + PURL: "pkg:npm/npm@6.14.6", Type: pkg.NpmPkg, Licenses: []string{}, Language: pkg.JavaScript, @@ -115,6 +120,7 @@ func TestParsePackageJSON(t *testing.T) { ExpectedPkg: pkg.Package{ Name: "npm", Version: "6.14.6", + PURL: "pkg:npm/npm@6.14.6", Type: pkg.NpmPkg, Licenses: []string{"Artistic-2.0"}, Language: pkg.JavaScript, @@ -134,6 +140,7 @@ func TestParsePackageJSON(t *testing.T) { ExpectedPkg: pkg.Package{ Name: "function-bind", Version: "1.1.1", + PURL: "pkg:npm/function-bind@1.1.1", Type: pkg.NpmPkg, Licenses: []string{"MIT"}, Language: pkg.JavaScript, @@ -153,6 +160,7 @@ func TestParsePackageJSON(t *testing.T) { ExpectedPkg: pkg.Package{ Name: "npm", Version: "6.14.6", + PURL: "pkg:npm/npm@6.14.6", Type: pkg.NpmPkg, Licenses: []string{"Artistic-2.0"}, Language: pkg.JavaScript, @@ -172,46 +180,16 @@ func TestParsePackageJSON(t *testing.T) { for _, test := range tests { t.Run(test.Fixture, func(t *testing.T) { - fixture, err := os.Open(test.Fixture) - if err != nil { - t.Fatalf("failed to open fixture: %+v", err) - } - - actual, _, err := parsePackageJSON("", fixture) - if err != nil { - t.Fatalf("failed to parse package-lock.json: %+v", err) - } - if len(actual) != 1 { - for _, a := range actual { - t.Log(" ", a) - } - t.Fatalf("unexpected package count: %d!=1", len(actual)) - } - - for _, d := range deep.Equal(actual[0], &test.ExpectedPkg) { - - t.Errorf("diff: %+v", d) - } + test.ExpectedPkg.Locations.Add(source.NewLocation(test.Fixture)) + pkgtest.TestFileParser(t, test.Fixture, parsePackageJSON, []pkg.Package{test.ExpectedPkg}, nil) }) } } func TestParsePackageJSON_Partial(t *testing.T) { // see https://github.com/anchore/syft/issues/311 const fixtureFile = "test-fixtures/pkg-json/package-partial.json" - fixture, err := os.Open(fixtureFile) - if err != nil { - t.Fatalf("failed to open fixture: %+v", err) - } - // TODO: no relationships are under test yet - actual, _, err := parsePackageJSON("", fixture) - if err != nil { - t.Fatalf("failed to parse package-lock.json: %+v", err) - } - - if actualCount := len(actual); actualCount != 0 { - t.Errorf("no packages should've been returned (but got %d packages)", actualCount) - } + pkgtest.TestFileParser(t, fixtureFile, parsePackageJSON, nil, nil) } func Test_pathContainsNodeModulesDirectory(t *testing.T) { diff --git a/syft/pkg/cataloger/javascript/parse_package_lock.go b/syft/pkg/cataloger/javascript/parse_package_lock.go index 5b98135bc6a..19769b3e209 100644 --- a/syft/pkg/cataloger/javascript/parse_package_lock.go +++ b/syft/pkg/cataloger/javascript/parse_package_lock.go @@ -8,28 +8,29 @@ import ( "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/pkg" - "github.com/anchore/syft/syft/pkg/cataloger/common" + "github.com/anchore/syft/syft/pkg/cataloger/generic" + "github.com/anchore/syft/syft/source" ) // integrity check -var _ common.ParserFn = parsePackageLock +var _ generic.Parser = parsePackageLock -// PackageLock represents a JavaScript package.lock json file -type PackageLock struct { +// packageLock represents a JavaScript package.lock json file +type packageLock struct { Requires bool `json:"requires"` LockfileVersion int `json:"lockfileVersion"` - Dependencies map[string]Dependency - Packages map[string]Package + Dependencies map[string]lockDependency + Packages map[string]lockPackage } -// Dependency represents a single package dependency listed in the package.lock json file -type Dependency struct { +// lockDependency represents a single package dependency listed in the package.lock json file +type lockDependency struct { Version string `json:"version"` Resolved string `json:"resolved"` Integrity string `json:"integrity"` } -type Package struct { +type lockPackage struct { Version string `json:"version"` Resolved string `json:"resolved"` Integrity string `json:"integrity"` @@ -37,18 +38,18 @@ type Package struct { } // parsePackageLock parses a package-lock.json and returns the discovered JavaScript packages. -func parsePackageLock(path string, reader io.Reader) ([]*pkg.Package, []artifact.Relationship, error) { +func parsePackageLock(resolver source.FileResolver, _ *generic.Environment, reader source.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { // in the case we find package-lock.json files in the node_modules directories, skip those // as the whole purpose of the lock file is for the specific dependencies of the root project - if pathContainsNodeModulesDirectory(path) { + if pathContainsNodeModulesDirectory(reader.AccessPath()) { return nil, nil, nil } - var packages []*pkg.Package + var pkgs []pkg.Package dec := json.NewDecoder(reader) for { - var lock PackageLock + var lock packageLock if err := dec.Decode(&lock); err == io.EOF { break } else if err != nil { @@ -63,22 +64,11 @@ func parsePackageLock(path string, reader io.Reader) ([]*pkg.Package, []artifact } for name, pkgMeta := range lock.Dependencies { - var sb strings.Builder - sb.WriteString(pkgMeta.Resolved) - sb.WriteString(pkgMeta.Integrity) - var licenses []string - if license, exists := licenseMap[sb.String()]; exists { - licenses = append(licenses, license) - } - packages = append(packages, &pkg.Package{ - Name: name, - Version: pkgMeta.Version, - Language: pkg.JavaScript, - Type: pkg.NpmPkg, - Licenses: licenses, - }) + pkgs = append(pkgs, newPackageLockPackage(resolver, reader.Location, name, pkgMeta, licenseMap)) } } - return packages, nil, nil + pkg.Sort(pkgs) + + return pkgs, nil, nil } diff --git a/syft/pkg/cataloger/javascript/parse_package_lock_test.go b/syft/pkg/cataloger/javascript/parse_package_lock_test.go index c280a61cb51..74fc84ff16e 100644 --- a/syft/pkg/cataloger/javascript/parse_package_lock_test.go +++ b/syft/pkg/cataloger/javascript/parse_package_lock_test.go @@ -1,157 +1,142 @@ package javascript import ( - "os" "testing" - "github.com/stretchr/testify/assert" - + "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/pkg/cataloger/internal/pkgtest" + "github.com/anchore/syft/syft/source" ) -func assertPkgsEqual(t *testing.T, actual []*pkg.Package, expected map[string]pkg.Package) { - t.Helper() - if len(actual) != len(expected) { - for _, a := range actual { - t.Log(" ", a) - } - t.Fatalf("unexpected package count: %d!=%d", len(actual), len(expected)) - } - - for _, a := range actual { - expectedPkg, ok := expected[a.Name] - assert.True(t, ok) - assert.Equal(t, expectedPkg.Version, a.Version, "bad version") - assert.Equal(t, expectedPkg.Language, a.Language, "bad language") - assert.Equal(t, expectedPkg.Type, a.Type, "bad type") - assert.Equal(t, expectedPkg.Licenses, a.Licenses, "bad license count") - } -} - func TestParsePackageLock(t *testing.T) { - expected := map[string]pkg.Package{ - "@actions/core": { + var expectedRelationships []artifact.Relationship + expectedPkgs := []pkg.Package{ + { Name: "@actions/core", Version: "1.6.0", + PURL: "pkg:npm/%40actions/core@1.6.0", Language: pkg.JavaScript, Type: pkg.NpmPkg, }, - "wordwrap": { - Name: "wordwrap", - Version: "0.0.3", + { + Name: "ansi-regex", + Version: "3.0.0", + PURL: "pkg:npm/ansi-regex@3.0.0", Language: pkg.JavaScript, Type: pkg.NpmPkg, }, - "get-stdin": { + { + Name: "cowsay", + Version: "1.4.0", + PURL: "pkg:npm/cowsay@1.4.0", + Language: pkg.JavaScript, + Type: pkg.NpmPkg, + }, + { Name: "get-stdin", Version: "5.0.1", + PURL: "pkg:npm/get-stdin@5.0.1", + Language: pkg.JavaScript, + Type: pkg.NpmPkg, + }, + { + Name: "is-fullwidth-code-point", + Version: "2.0.0", + PURL: "pkg:npm/is-fullwidth-code-point@2.0.0", Language: pkg.JavaScript, Type: pkg.NpmPkg, }, - "minimist": { + { Name: "minimist", Version: "0.0.10", + PURL: "pkg:npm/minimist@0.0.10", Language: pkg.JavaScript, Type: pkg.NpmPkg, }, - "optimist": { + { Name: "optimist", Version: "0.6.1", + PURL: "pkg:npm/optimist@0.6.1", Language: pkg.JavaScript, Type: pkg.NpmPkg, }, - "string-width": { + { Name: "string-width", Version: "2.1.1", + PURL: "pkg:npm/string-width@2.1.1", Language: pkg.JavaScript, Type: pkg.NpmPkg, }, - "strip-ansi": { + { Name: "strip-ansi", Version: "4.0.0", + PURL: "pkg:npm/strip-ansi@4.0.0", Language: pkg.JavaScript, Type: pkg.NpmPkg, }, - "strip-eof": { - Name: "wordwrap", + { + Name: "strip-eof", Version: "1.0.0", + PURL: "pkg:npm/strip-eof@1.0.0", Language: pkg.JavaScript, Type: pkg.NpmPkg, }, - "ansi-regex": { - Name: "ansi-regex", - Version: "3.0.0", - Language: pkg.JavaScript, - Type: pkg.NpmPkg, - }, - "is-fullwidth-code-point": { - Name: "is-fullwidth-code-point", - Version: "2.0.0", - Language: pkg.JavaScript, - Type: pkg.NpmPkg, - }, - "cowsay": { - Name: "cowsay", - Version: "1.4.0", + { + Name: "wordwrap", + Version: "0.0.3", + PURL: "pkg:npm/wordwrap@0.0.3", Language: pkg.JavaScript, Type: pkg.NpmPkg, }, } - fixture, err := os.Open("test-fixtures/pkg-lock/package-lock.json") - if err != nil { - t.Fatalf("failed to open fixture: %+v", err) - } - - // TODO: no relationships are under test yet - actual, _, err := parsePackageLock(fixture.Name(), fixture) - if err != nil { - t.Fatalf("failed to parse package-lock.json: %+v", err) + fixture := "test-fixtures/pkg-lock/package-lock.json" + for i := range expectedPkgs { + expectedPkgs[i].Locations.Add(source.NewLocation(fixture)) } - assertPkgsEqual(t, actual, expected) - + pkgtest.TestFileParser(t, fixture, parsePackageLock, expectedPkgs, expectedRelationships) } func TestParsePackageLockV2(t *testing.T) { - expected := map[string]pkg.Package{ - "@types/prop-types": { + fixture := "test-fixtures/pkg-lock/package-lock-2.json" + var expectedRelationships []artifact.Relationship + expectedPkgs := []pkg.Package{ + { Name: "@types/prop-types", Version: "15.7.5", + PURL: "pkg:npm/%40types/prop-types@15.7.5", Language: pkg.JavaScript, Type: pkg.NpmPkg, Licenses: []string{"MIT"}, }, - "@types/react": { - Name: "@types/prop-types", + { + Name: "@types/react", Version: "18.0.17", + PURL: "pkg:npm/%40types/react@18.0.17", Language: pkg.JavaScript, Type: pkg.NpmPkg, Licenses: []string{"MIT"}, }, - "@types/scheduler": { + { Name: "@types/scheduler", Version: "0.16.2", + PURL: "pkg:npm/%40types/scheduler@0.16.2", Language: pkg.JavaScript, Type: pkg.NpmPkg, Licenses: []string{"MIT"}, }, - "csstype": { + { Name: "csstype", Version: "3.1.0", + PURL: "pkg:npm/csstype@3.1.0", Language: pkg.JavaScript, Type: pkg.NpmPkg, Licenses: []string{"MIT"}, }, } - fixture, err := os.Open("test-fixtures/pkg-lock/package-lock-2.json") - if err != nil { - t.Fatalf("failed to open fixture: %+v", err) + for i := range expectedPkgs { + expectedPkgs[i].Locations.Add(source.NewLocation(fixture)) } - - actual, _, err := parsePackageLock(fixture.Name(), fixture) - if err != nil { - t.Fatalf("failed to parse package-lock.json: %+v", err) - } - - assertPkgsEqual(t, actual, expected) + pkgtest.TestFileParser(t, fixture, parsePackageLock, expectedPkgs, expectedRelationships) } diff --git a/syft/pkg/cataloger/javascript/parse_pnpm_lock.go b/syft/pkg/cataloger/javascript/parse_pnpm_lock.go index aa1cc169b08..2ae6bce1a0a 100644 --- a/syft/pkg/cataloger/javascript/parse_pnpm_lock.go +++ b/syft/pkg/cataloger/javascript/parse_pnpm_lock.go @@ -8,23 +8,24 @@ import ( "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/pkg" - "github.com/anchore/syft/syft/pkg/cataloger/common" + "github.com/anchore/syft/syft/pkg/cataloger/generic" + "github.com/anchore/syft/syft/source" ) // integrity check -var _ common.ParserFn = parsePnpmLock +var _ generic.Parser = parsePnpmLock type pnpmLockYaml struct { Dependencies map[string]string `json:"dependencies"` } -func parsePnpmLock(path string, reader io.Reader) ([]*pkg.Package, []artifact.Relationship, error) { +func parsePnpmLock(resolver source.FileResolver, _ *generic.Environment, reader source.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { bytes, err := io.ReadAll(reader) if err != nil { return nil, nil, fmt.Errorf("failed to load pnpm-lock.yaml file: %w", err) } - var pkgs []*pkg.Package + var pkgs []pkg.Package var lockFile pnpmLockYaml if err := yaml.Unmarshal(bytes, &lockFile); err != nil { @@ -32,13 +33,10 @@ func parsePnpmLock(path string, reader io.Reader) ([]*pkg.Package, []artifact.Re } for name, version := range lockFile.Dependencies { - pkgs = append(pkgs, &pkg.Package{ - Name: name, - Version: version, - Language: pkg.JavaScript, - Type: pkg.NpmPkg, - }) + pkgs = append(pkgs, newPnpmPackage(resolver, reader.Location, name, version)) } + pkg.Sort(pkgs) + return pkgs, nil, nil } diff --git a/syft/pkg/cataloger/javascript/parse_pnpm_lock_test.go b/syft/pkg/cataloger/javascript/parse_pnpm_lock_test.go index 49118075edc..e4750f949af 100644 --- a/syft/pkg/cataloger/javascript/parse_pnpm_lock_test.go +++ b/syft/pkg/cataloger/javascript/parse_pnpm_lock_test.go @@ -1,59 +1,46 @@ package javascript import ( - "os" - "sort" "testing" - "github.com/go-test/deep" - + "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/pkg/cataloger/internal/pkgtest" + "github.com/anchore/syft/syft/source" ) -func fixtureP(str string) *string { - return &str -} - func TestParsePnpmLock(t *testing.T) { - expected := []*pkg.Package{ + var expectedRelationships []artifact.Relationship + fixture := "test-fixtures/pnpm/pnpm-lock.yaml" + + locationSet := source.NewLocationSet(source.NewLocation(fixture)) + + expectedPkgs := []pkg.Package{ { - Name: "nanoid", - Version: "3.3.4", - Language: pkg.JavaScript, - Type: pkg.NpmPkg, + Name: "nanoid", + Version: "3.3.4", + PURL: "pkg:npm/nanoid@3.3.4", + Locations: locationSet, + Language: pkg.JavaScript, + Type: pkg.NpmPkg, }, { - Name: "picocolors", - Version: "1.0.0", - Language: pkg.JavaScript, - Type: pkg.NpmPkg, + Name: "picocolors", + Version: "1.0.0", + PURL: "pkg:npm/picocolors@1.0.0", + Locations: locationSet, + Language: pkg.JavaScript, + Type: pkg.NpmPkg, }, { - Name: "source-map-js", - Version: "1.0.2", - Language: pkg.JavaScript, - Type: pkg.NpmPkg, + Name: "source-map-js", + Version: "1.0.2", + PURL: "pkg:npm/source-map-js@1.0.2", + Locations: locationSet, + Language: pkg.JavaScript, + Type: pkg.NpmPkg, }, } - fixture, err := os.Open("test-fixtures/pnpm/pnpm-lock.yaml") - if err != nil { - t.Fatalf("failed to open fixture: %+v", err) - } - - // TODO: no relationships are under test yet - actual, _, err := parsePnpmLock(fixture.Name(), fixture) - if err != nil { - t.Error(err) - } - - // we have to sort this for expected to match actual since yaml maps are unordered - sort.Slice(actual, func(p, q int) bool { - return actual[p].Name < actual[q].Name - }) - - differences := deep.Equal(expected, actual) - if differences != nil { - t.Errorf("returned package list differed from expectation: %+v", differences) - } + pkgtest.TestFileParser(t, fixture, parsePnpmLock, expectedPkgs, expectedRelationships) } diff --git a/syft/pkg/cataloger/javascript/parse_yarn_lock.go b/syft/pkg/cataloger/javascript/parse_yarn_lock.go index 91ab8043a48..048f8f05c76 100644 --- a/syft/pkg/cataloger/javascript/parse_yarn_lock.go +++ b/syft/pkg/cataloger/javascript/parse_yarn_lock.go @@ -3,17 +3,17 @@ package javascript import ( "bufio" "fmt" - "io" "regexp" "github.com/anchore/syft/internal" "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/pkg" - "github.com/anchore/syft/syft/pkg/cataloger/common" + "github.com/anchore/syft/syft/pkg/cataloger/generic" + "github.com/anchore/syft/syft/source" ) // integrity check -var _ common.ParserFn = parseYarnLock +var _ generic.Parser = parseYarnLock var ( // packageNameExp matches the name of the dependency in yarn.lock @@ -42,14 +42,14 @@ const ( noVersion = "" ) -func parseYarnLock(path string, reader io.Reader) ([]*pkg.Package, []artifact.Relationship, error) { +func parseYarnLock(resolver source.FileResolver, _ *generic.Environment, reader source.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { // in the case we find yarn.lock files in the node_modules directories, skip those // as the whole purpose of the lock file is for the specific dependencies of the project - if pathContainsNodeModulesDirectory(path) { + if pathContainsNodeModulesDirectory(reader.AccessPath()) { return nil, nil, nil } - var packages []*pkg.Package + var pkgs []pkg.Package scanner := bufio.NewScanner(reader) parsedPackages := internal.NewStringSet() currentPackage := noPackage @@ -61,7 +61,7 @@ func parseYarnLock(path string, reader io.Reader) ([]*pkg.Package, []artifact.Re if packageName := findPackageName(line); packageName != noPackage { // When we find a new package, check if we have unsaved identifiers if currentPackage != noPackage && currentVersion != noVersion && !parsedPackages.Contains(currentPackage+"@"+currentVersion) { - packages = append(packages, newYarnLockPackage(currentPackage, currentVersion)) + pkgs = append(pkgs, newYarnLockPackage(resolver, reader.Location, currentPackage, currentVersion)) parsedPackages.Add(currentPackage + "@" + currentVersion) } @@ -69,7 +69,7 @@ func parseYarnLock(path string, reader io.Reader) ([]*pkg.Package, []artifact.Re } else if version := findPackageVersion(line); version != noVersion { currentVersion = version } else if packageName, version := findPackageAndVersion(line); packageName != noPackage && version != noVersion && !parsedPackages.Contains(packageName+"@"+version) { - packages = append(packages, newYarnLockPackage(packageName, version)) + pkgs = append(pkgs, newYarnLockPackage(resolver, reader.Location, packageName, version)) parsedPackages.Add(packageName + "@" + version) // Cleanup to indicate no unsaved identifiers @@ -80,7 +80,7 @@ func parseYarnLock(path string, reader io.Reader) ([]*pkg.Package, []artifact.Re // check if we have valid unsaved data after end-of-file has reached if currentPackage != noPackage && currentVersion != noVersion && !parsedPackages.Contains(currentPackage+"@"+currentVersion) { - packages = append(packages, newYarnLockPackage(currentPackage, currentVersion)) + pkgs = append(pkgs, newYarnLockPackage(resolver, reader.Location, currentPackage, currentVersion)) parsedPackages.Add(currentPackage + "@" + currentVersion) } @@ -88,7 +88,9 @@ func parseYarnLock(path string, reader io.Reader) ([]*pkg.Package, []artifact.Re return nil, nil, fmt.Errorf("failed to parse yarn.lock file: %w", err) } - return packages, nil, nil + pkg.Sort(pkgs) + + return pkgs, nil, nil } func findPackageName(line string) string { @@ -114,12 +116,3 @@ func findPackageAndVersion(line string) (string, string) { return noPackage, noVersion } - -func newYarnLockPackage(name, version string) *pkg.Package { - return &pkg.Package{ - Name: name, - Version: version, - Language: pkg.JavaScript, - Type: pkg.NpmPkg, - } -} diff --git a/syft/pkg/cataloger/javascript/parse_yarn_lock_test.go b/syft/pkg/cataloger/javascript/parse_yarn_lock_test.go index da54f19a37b..ded8850b1f4 100644 --- a/syft/pkg/cataloger/javascript/parse_yarn_lock_test.go +++ b/syft/pkg/cataloger/javascript/parse_yarn_lock_test.go @@ -1,170 +1,184 @@ package javascript import ( - "os" "testing" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" + "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/pkg/cataloger/internal/pkgtest" + "github.com/anchore/syft/syft/source" ) func TestParseYarnBerry(t *testing.T) { - expected := map[string]pkg.Package{ - "@babel/code-frame": { - Name: "@babel/code-frame", - Version: "7.10.4", - Language: pkg.JavaScript, - Type: pkg.NpmPkg, + var expectedRelationships []artifact.Relationship + fixture := "test-fixtures/yarn-berry/yarn.lock" + locations := source.NewLocationSet(source.NewLocation(fixture)) + + expectedPkgs := []pkg.Package{ + { + Name: "@babel/code-frame", + Version: "7.10.4", + Locations: locations, + PURL: "pkg:npm/%40babel/code-frame@7.10.4", + Language: pkg.JavaScript, + Type: pkg.NpmPkg, }, - "@types/minimatch": { - Name: "@types/minimatch", - Version: "3.0.3", - Language: pkg.JavaScript, - Type: pkg.NpmPkg, + { + Name: "@types/minimatch", + Version: "3.0.3", + Locations: locations, + PURL: "pkg:npm/%40types/minimatch@3.0.3", + Language: pkg.JavaScript, + Type: pkg.NpmPkg, }, - "@types/qs": { - Name: "@types/qs", - Version: "6.9.4", - Language: pkg.JavaScript, - Type: pkg.NpmPkg, + { + Name: "@types/qs", + Version: "6.9.4", + Locations: locations, + PURL: "pkg:npm/%40types/qs@6.9.4", + Language: pkg.JavaScript, + Type: pkg.NpmPkg, }, - "ajv": { - Name: "ajv", - Version: "6.12.3", - Language: pkg.JavaScript, - Type: pkg.NpmPkg, + { + Name: "ajv", + Version: "6.12.3", + Locations: locations, + PURL: "pkg:npm/ajv@6.12.3", + Language: pkg.JavaScript, + Type: pkg.NpmPkg, }, - "atob": { - Name: "atob", - Version: "2.1.2", - Language: pkg.JavaScript, - Type: pkg.NpmPkg, + { + Name: "asn1.js", + Version: "4.10.1", + Locations: locations, + PURL: "pkg:npm/asn1.js@4.10.1", + Language: pkg.JavaScript, + Type: pkg.NpmPkg, }, - "aws-sdk": { - Name: "aws-sdk", - Version: "2.706.0", - Language: pkg.JavaScript, - Type: pkg.NpmPkg, + { + Name: "atob", + Version: "2.1.2", + Locations: locations, + PURL: "pkg:npm/atob@2.1.2", + Language: pkg.JavaScript, + Type: pkg.NpmPkg, }, - "jhipster-core": { - Name: "jhipster-core", - Version: "7.3.4", - Language: pkg.JavaScript, - Type: pkg.NpmPkg, + { + Name: "aws-sdk", + Version: "2.706.0", + PURL: "pkg:npm/aws-sdk@2.706.0", + Locations: locations, + Language: pkg.JavaScript, + Type: pkg.NpmPkg, }, - "asn1.js": { - Name: "asn1.js", - Version: "4.10.1", - Language: pkg.JavaScript, - Type: pkg.NpmPkg, + { + Name: "c0n-fab_u.laTION", + Version: "7.7.7", + Locations: locations, + PURL: "pkg:npm/c0n-fab_u.laTION@7.7.7", + Language: pkg.JavaScript, + Type: pkg.NpmPkg, }, - "c0n-fab_u.laTION": { - Name: "c0n-fab_u.laTION", - Version: "7.7.7", - Language: pkg.JavaScript, - Type: pkg.NpmPkg, + { + Name: "jhipster-core", + Version: "7.3.4", + Locations: locations, + PURL: "pkg:npm/jhipster-core@7.3.4", + Language: pkg.JavaScript, + Type: pkg.NpmPkg, }, } - testFixtures := []string{ - "test-fixtures/yarn-berry/yarn.lock", - } - - for _, file := range testFixtures { - file := file - t.Run(file, func(t *testing.T) { - t.Parallel() - fixture, err := os.Open(file) - require.NoError(t, err) + pkgtest.TestFileParser(t, fixture, parseYarnLock, expectedPkgs, expectedRelationships) - // TODO: no relationships are under test yet - actual, _, err := parseYarnLock(fixture.Name(), fixture) - require.NoError(t, err) - - assertPkgsEqual(t, actual, expected) - }) - } } func TestParseYarnLock(t *testing.T) { - expected := map[string]pkg.Package{ - "@babel/code-frame": { - Name: "@babel/code-frame", - Version: "7.10.4", - Language: pkg.JavaScript, - Type: pkg.NpmPkg, - }, - "@types/minimatch": { - Name: "@types/minimatch", - Version: "3.0.3", - Language: pkg.JavaScript, - Type: pkg.NpmPkg, + var expectedRelationships []artifact.Relationship + fixture := "test-fixtures/yarn/yarn.lock" + locations := source.NewLocationSet(source.NewLocation(fixture)) + + expectedPkgs := []pkg.Package{ + { + Name: "@babel/code-frame", + Version: "7.10.4", + Locations: locations, + PURL: "pkg:npm/%40babel/code-frame@7.10.4", + Language: pkg.JavaScript, + Type: pkg.NpmPkg, }, - "@types/qs": { - Name: "@types/qs", - Version: "6.9.4", - Language: pkg.JavaScript, - Type: pkg.NpmPkg, + { + Name: "@types/minimatch", + Version: "3.0.3", + Locations: locations, + PURL: "pkg:npm/%40types/minimatch@3.0.3", + Language: pkg.JavaScript, + Type: pkg.NpmPkg, }, - "ajv": { - Name: "ajv", - Version: "6.12.3", - Language: pkg.JavaScript, - Type: pkg.NpmPkg, + { + Name: "@types/qs", + Version: "6.9.4", + Locations: locations, + PURL: "pkg:npm/%40types/qs@6.9.4", + Language: pkg.JavaScript, + Type: pkg.NpmPkg, }, - "atob": { - Name: "atob", - Version: "2.1.2", - Language: pkg.JavaScript, - Type: pkg.NpmPkg, + { + Name: "ajv", + Version: "6.12.3", + Locations: locations, + PURL: "pkg:npm/ajv@6.12.3", + Language: pkg.JavaScript, + Type: pkg.NpmPkg, }, - "aws-sdk": { - Name: "aws-sdk", - Version: "2.706.0", - Language: pkg.JavaScript, - Type: pkg.NpmPkg, + { + Name: "asn1.js", + Version: "4.10.1", + Locations: locations, + PURL: "pkg:npm/asn1.js@4.10.1", + Language: pkg.JavaScript, + Type: pkg.NpmPkg, }, - "jhipster-core": { - Name: "jhipster-core", - Version: "7.3.4", + { + Name: "atob", + Version: "2.1.2", + Locations: locations, + + PURL: "pkg:npm/atob@2.1.2", Language: pkg.JavaScript, Type: pkg.NpmPkg, }, - "asn1.js": { - Name: "asn1.js", - Version: "4.10.1", - Language: pkg.JavaScript, - Type: pkg.NpmPkg, + { + Name: "aws-sdk", + Version: "2.706.0", + Locations: locations, + PURL: "pkg:npm/aws-sdk@2.706.0", + Language: pkg.JavaScript, + Type: pkg.NpmPkg, }, - "something-i-made-up": { - Name: "something-i-made-up", - Version: "7.7.7", - Language: pkg.JavaScript, - Type: pkg.NpmPkg, + { + Name: "jhipster-core", + Version: "7.3.4", + Locations: locations, + PURL: "pkg:npm/jhipster-core@7.3.4", + Language: pkg.JavaScript, + Type: pkg.NpmPkg, }, - } - testFixtures := []string{ - "test-fixtures/yarn/yarn.lock", + { + Name: "something-i-made-up", + Version: "7.7.7", + Locations: locations, + PURL: "pkg:npm/something-i-made-up@7.7.7", + Language: pkg.JavaScript, + Type: pkg.NpmPkg, + }, } - for _, file := range testFixtures { - file := file - t.Run(file, func(t *testing.T) { - t.Parallel() - - fixture, err := os.Open(file) - require.NoError(t, err) - - // TODO: no relationships are under test yet - actual, _, err := parseYarnLock(fixture.Name(), fixture) - require.NoError(t, err) + pkgtest.TestFileParser(t, fixture, parseYarnLock, expectedPkgs, expectedRelationships) - assertPkgsEqual(t, actual, expected) - }) - } } func TestParseYarnFindPackageNames(t *testing.T) { @@ -227,7 +241,6 @@ func TestParseYarnFindPackageNames(t *testing.T) { } for _, test := range tests { - test := test t.Run(test.expected, func(t *testing.T) { t.Parallel() actual := findPackageName(test.line) @@ -316,7 +329,6 @@ func TestParseYarnFindPackageVersions(t *testing.T) { } for _, test := range tests { - test := test t.Run(test.expected, func(t *testing.T) { t.Parallel() actual := findPackageVersion(test.line) diff --git a/syft/pkg/npm_package_json_metadata.go b/syft/pkg/npm_package_json_metadata.go index 3fe51bffec8..d426a88d853 100644 --- a/syft/pkg/npm_package_json_metadata.go +++ b/syft/pkg/npm_package_json_metadata.go @@ -1,14 +1,5 @@ package pkg -import ( - "strings" - - "github.com/anchore/packageurl-go" - "github.com/anchore/syft/syft/linux" -) - -var _ urlIdentifier = (*NpmPackageJSONMetadata)(nil) - // NpmPackageJSONMetadata holds extra information that is used in pkg.Package type NpmPackageJSONMetadata struct { Name string `mapstructure:"name" json:"name"` @@ -21,24 +12,3 @@ type NpmPackageJSONMetadata struct { URL string `mapstructure:"url" json:"url"` Private bool `mapstructure:"private" json:"private"` } - -// PackageURL returns the PURL for the specific NPM package (see https://github.com/package-url/purl-spec) -func (p NpmPackageJSONMetadata) PackageURL(_ *linux.Release) string { - var namespace string - name := p.Name - - fields := strings.SplitN(p.Name, "/", 2) - if len(fields) > 1 { - namespace = fields[0] - name = fields[1] - } - - return packageurl.NewPackageURL( - packageurl.TypeNPM, - namespace, - name, - p.Version, - nil, - "", - ).ToString() -} diff --git a/syft/pkg/npm_package_json_metadata_test.go b/syft/pkg/npm_package_json_metadata_test.go deleted file mode 100644 index 4557c29d0f5..00000000000 --- a/syft/pkg/npm_package_json_metadata_test.go +++ /dev/null @@ -1,63 +0,0 @@ -package pkg - -import ( - "fmt" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/anchore/packageurl-go" -) - -func TestNpmPackageJSONMetadata_PackageURL(t *testing.T) { - - tests := []struct { - name string - metadata NpmPackageJSONMetadata - expected string - namespace string - }{ - { - name: "no namespace", - metadata: NpmPackageJSONMetadata{ - Name: "arborist", - Version: "2.6.2", - }, - expected: "pkg:npm/arborist@2.6.2", - }, - { - name: "split by namespace", - metadata: NpmPackageJSONMetadata{ - Name: "npmcli/arborist", - Version: "2.6.2", - }, - expected: "pkg:npm/npmcli/arborist@2.6.2", - namespace: "npmcli", - }, - { - name: "encoding @ symobl", - metadata: NpmPackageJSONMetadata{ - Name: "@npmcli/arborist", - Version: "2.6.2", - }, - expected: "pkg:npm/%40npmcli/arborist@2.6.2", - namespace: "@npmcli", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - actual := tt.metadata.PackageURL(nil) - assert.Equal(t, tt.expected, actual) - decoded, err := packageurl.FromString(actual) - require.NoError(t, err) - assert.Equal(t, tt.namespace, decoded.Namespace) - if decoded.Namespace != "" { - assert.Equal(t, tt.metadata.Name, fmt.Sprintf("%s/%s", decoded.Namespace, decoded.Name)) - } else { - assert.Equal(t, tt.metadata.Name, decoded.Name) - } - assert.Equal(t, tt.metadata.Version, decoded.Version) - }) - } -} diff --git a/syft/pkg/package.go b/syft/pkg/package.go index 876b789b96b..4ee75aa6425 100644 --- a/syft/pkg/package.go +++ b/syft/pkg/package.go @@ -5,6 +5,7 @@ package pkg import ( "fmt" + "sort" "github.com/anchore/syft/internal/log" "github.com/anchore/syft/syft/artifact" @@ -80,3 +81,24 @@ func (p *Package) merge(other Package) error { func IsValid(p *Package) bool { return p != nil && p.Name != "" } + +func Sort(pkgs []Package) { + sort.SliceStable(pkgs, func(i, j int) bool { + if pkgs[i].Name == pkgs[j].Name { + if pkgs[i].Version == pkgs[j].Version { + iLocations := pkgs[i].Locations.ToSlice() + jLocations := pkgs[j].Locations.ToSlice() + if pkgs[i].Type == pkgs[j].Type && len(iLocations) > 0 && len(jLocations) > 0 { + if iLocations[0].String() == jLocations[0].String() { + // compare IDs as a final fallback + return pkgs[i].ID() < pkgs[j].ID() + } + return iLocations[0].String() < jLocations[0].String() + } + return pkgs[i].Type < pkgs[j].Type + } + return pkgs[i].Version < pkgs[j].Version + } + return pkgs[i].Name < pkgs[j].Name + }) +}