From f4e18961b979f5e6d0cc3b1e4fce608c8ceb29d8 Mon Sep 17 00:00:00 2001 From: Colm O hEigeartaigh Date: Thu, 21 Mar 2024 17:20:04 +0000 Subject: [PATCH] Adding the ability to retrieve remote licenses from package.lock (#2708) Signed-off-by: Colm O hEigeartaigh --- syft/pkg/cataloger/javascript/cataloger.go | 3 +- syft/pkg/cataloger/javascript/package.go | 35 +++++++++++++++++-- .../javascript/parse_package_lock.go | 19 ++++++---- .../javascript/parse_package_lock_test.go | 15 +++++--- 4 files changed, 57 insertions(+), 15 deletions(-) diff --git a/syft/pkg/cataloger/javascript/cataloger.go b/syft/pkg/cataloger/javascript/cataloger.go index 256c759da70..649800c6c01 100644 --- a/syft/pkg/cataloger/javascript/cataloger.go +++ b/syft/pkg/cataloger/javascript/cataloger.go @@ -17,8 +17,9 @@ func NewPackageCataloger() pkg.Cataloger { // NewLockCataloger returns a new cataloger object for NPM (and NPM-adjacent, such as yarn) lock files. func NewLockCataloger(cfg CatalogerConfig) pkg.Cataloger { yarnLockAdapter := newGenericYarnLockAdapter(cfg) + packageLockAdapter := newGenericPackageLockAdapter(cfg) return generic.NewCataloger("javascript-lock-cataloger"). - WithParserByGlobs(parsePackageLock, "**/package-lock.json"). + WithParserByGlobs(packageLockAdapter.parsePackageLock, "**/package-lock.json"). WithParserByGlobs(yarnLockAdapter.parseYarnLock, "**/yarn.lock"). WithParserByGlobs(parsePnpmLock, "**/pnpm-lock.yaml") } diff --git a/syft/pkg/cataloger/javascript/package.go b/syft/pkg/cataloger/javascript/package.go index 893de6cc0bf..5f6613727d4 100644 --- a/syft/pkg/cataloger/javascript/package.go +++ b/syft/pkg/cataloger/javascript/package.go @@ -47,7 +47,7 @@ func newPackageJSONPackage(u packageJSON, indexLocation file.Location) pkg.Packa return p } -func newPackageLockV1Package(resolver file.Resolver, location file.Location, name string, u lockDependency) pkg.Package { +func newPackageLockV1Package(cfg CatalogerConfig, resolver file.Resolver, location file.Location, name string, u lockDependency) pkg.Package { version := u.Version const aliasPrefixPackageLockV1 = "npm:" @@ -63,12 +63,26 @@ func newPackageLockV1Package(resolver file.Resolver, location file.Location, nam version = canonicalPackageAndVersion[versionSeparator+1:] } + var licenseSet pkg.LicenseSet + + if cfg.SearchRemoteLicenses { + license, err := getLicenseFromNpmRegistry(cfg.NPMBaseURL, name, version) + if err == nil && license != "" { + licenses := pkg.NewLicensesFromValues(license) + licenseSet = pkg.NewLicenseSet(licenses...) + } + if err != nil { + log.Warnf("unable to extract licenses from javascript yarn.lock for package %s:%s: %+v", name, version, err) + } + } + return finalizeLockPkg( resolver, location, pkg.Package{ Name: name, Version: version, + Licenses: licenseSet, Locations: file.NewLocationSet(location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation)), PURL: packageURL(name, version), Language: pkg.JavaScript, @@ -78,7 +92,22 @@ func newPackageLockV1Package(resolver file.Resolver, location file.Location, nam ) } -func newPackageLockV2Package(resolver file.Resolver, location file.Location, name string, u lockPackage) pkg.Package { +func newPackageLockV2Package(cfg CatalogerConfig, resolver file.Resolver, location file.Location, name string, u lockPackage) pkg.Package { + var licenseSet pkg.LicenseSet + + if u.License != nil { + licenseSet = pkg.NewLicenseSet(pkg.NewLicensesFromLocation(location, u.License...)...) + } else if cfg.SearchRemoteLicenses { + license, err := getLicenseFromNpmRegistry(cfg.NPMBaseURL, name, u.Version) + if err == nil && license != "" { + licenses := pkg.NewLicensesFromValues(license) + licenseSet = pkg.NewLicenseSet(licenses...) + } + if err != nil { + log.Warnf("unable to extract licenses from javascript yarn.lock for package %s:%s: %+v", name, u.Version, err) + } + } + return finalizeLockPkg( resolver, location, @@ -86,7 +115,7 @@ func newPackageLockV2Package(resolver file.Resolver, location file.Location, nam Name: name, Version: u.Version, Locations: file.NewLocationSet(location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation)), - Licenses: pkg.NewLicenseSet(pkg.NewLicensesFromLocation(location, u.License...)...), + Licenses: licenseSet, PURL: packageURL(name, u.Version), Language: pkg.JavaScript, Type: pkg.NpmPkg, diff --git a/syft/pkg/cataloger/javascript/parse_package_lock.go b/syft/pkg/cataloger/javascript/parse_package_lock.go index 5397f374a92..1c7564cc3fe 100644 --- a/syft/pkg/cataloger/javascript/parse_package_lock.go +++ b/syft/pkg/cataloger/javascript/parse_package_lock.go @@ -15,9 +15,6 @@ import ( "github.com/anchore/syft/syft/pkg/cataloger/generic" ) -// integrity check -var _ generic.Parser = parsePackageLock - // packageLock represents a JavaScript package.lock json file type packageLock struct { Requires bool `json:"requires"` @@ -44,8 +41,18 @@ type lockPackage struct { // packageLockLicense type packageLockLicense []string +type genericPackageLockAdapter struct { + cfg CatalogerConfig +} + +func newGenericPackageLockAdapter(cfg CatalogerConfig) genericPackageLockAdapter { + return genericPackageLockAdapter{ + cfg: cfg, + } +} + // parsePackageLock parses a package-lock.json and returns the discovered JavaScript packages. -func parsePackageLock(_ context.Context, resolver file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { +func (a genericPackageLockAdapter) parsePackageLock(_ context.Context, resolver file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { // in the case we find package-lock.json files in the node_modules directories, skip those // as the whole purpose of the lock file is for the specific dependencies of the root project if pathContainsNodeModulesDirectory(reader.Path()) { @@ -66,7 +73,7 @@ func parsePackageLock(_ context.Context, resolver file.Resolver, _ *generic.Envi if lock.LockfileVersion == 1 { for name, pkgMeta := range lock.Dependencies { - pkgs = append(pkgs, newPackageLockV1Package(resolver, reader.Location, name, pkgMeta)) + pkgs = append(pkgs, newPackageLockV1Package(a.cfg, resolver, reader.Location, name, pkgMeta)) } } @@ -86,7 +93,7 @@ func parsePackageLock(_ context.Context, resolver file.Resolver, _ *generic.Envi pkgs = append( pkgs, - newPackageLockV2Package(resolver, reader.Location, getNameFromPath(name), pkgMeta), + newPackageLockV2Package(a.cfg, resolver, reader.Location, getNameFromPath(name), pkgMeta), ) } } diff --git a/syft/pkg/cataloger/javascript/parse_package_lock_test.go b/syft/pkg/cataloger/javascript/parse_package_lock_test.go index 34a1017c22c..98448bf7b43 100644 --- a/syft/pkg/cataloger/javascript/parse_package_lock_test.go +++ b/syft/pkg/cataloger/javascript/parse_package_lock_test.go @@ -106,7 +106,8 @@ func TestParsePackageLock(t *testing.T) { expectedPkgs[i].Locations.Add(file.NewLocation(fixture)) } - pkgtest.TestFileParser(t, fixture, parsePackageLock, expectedPkgs, expectedRelationships) + adapter := newGenericPackageLockAdapter(CatalogerConfig{}) + pkgtest.TestFileParser(t, fixture, adapter.parsePackageLock, expectedPkgs, expectedRelationships) } func TestParsePackageLockV2(t *testing.T) { @@ -169,7 +170,8 @@ func TestParsePackageLockV2(t *testing.T) { for i := range expectedPkgs { expectedPkgs[i].Locations.Add(file.NewLocation(fixture)) } - pkgtest.TestFileParser(t, fixture, parsePackageLock, expectedPkgs, expectedRelationships) + adapter := newGenericPackageLockAdapter(CatalogerConfig{}) + pkgtest.TestFileParser(t, fixture, adapter.parsePackageLock, expectedPkgs, expectedRelationships) } func TestParsePackageLockV3(t *testing.T) { @@ -220,7 +222,8 @@ func TestParsePackageLockV3(t *testing.T) { for i := range expectedPkgs { expectedPkgs[i].Locations.Add(file.NewLocation(fixture)) } - pkgtest.TestFileParser(t, fixture, parsePackageLock, expectedPkgs, expectedRelationships) + adapter := newGenericPackageLockAdapter(CatalogerConfig{}) + pkgtest.TestFileParser(t, fixture, adapter.parsePackageLock, expectedPkgs, expectedRelationships) } func TestParsePackageLockAlias(t *testing.T) { @@ -279,7 +282,8 @@ func TestParsePackageLockAlias(t *testing.T) { for i := range expected { expected[i].Locations.Add(file.NewLocation(pl)) } - pkgtest.TestFileParser(t, pl, parsePackageLock, expected, expectedRelationships) + adapter := newGenericPackageLockAdapter(CatalogerConfig{}) + pkgtest.TestFileParser(t, pl, adapter.parsePackageLock, expected, expectedRelationships) } } @@ -326,5 +330,6 @@ func TestParsePackageLockLicenseWithArray(t *testing.T) { for i := range expectedPkgs { expectedPkgs[i].Locations.Add(file.NewLocation(fixture)) } - pkgtest.TestFileParser(t, fixture, parsePackageLock, expectedPkgs, expectedRelationships) + adapter := newGenericPackageLockAdapter(CatalogerConfig{}) + pkgtest.TestFileParser(t, fixture, adapter.parsePackageLock, expectedPkgs, expectedRelationships) }