Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Adding the ability to retrieve remote licenses from package-lock.json #2708

Merged
merged 1 commit into from
Mar 21, 2024
Merged

Adding the ability to retrieve remote licenses from package-lock.json #2708

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
3 changes: 2 additions & 1 deletion syft/pkg/cataloger/javascript/cataloger.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,9 @@ func NewPackageCataloger() pkg.Cataloger {
// NewLockCataloger returns a new cataloger object for NPM (and NPM-adjacent, such as yarn) lock files.
func NewLockCataloger(cfg CatalogerConfig) pkg.Cataloger {
yarnLockAdapter := newGenericYarnLockAdapter(cfg)
packageLockAdapter := newGenericPackageLockAdapter(cfg)
return generic.NewCataloger("javascript-lock-cataloger").
WithParserByGlobs(parsePackageLock, "**/package-lock.json").
WithParserByGlobs(packageLockAdapter.parsePackageLock, "**/package-lock.json").
WithParserByGlobs(yarnLockAdapter.parseYarnLock, "**/yarn.lock").
WithParserByGlobs(parsePnpmLock, "**/pnpm-lock.yaml")
}
35 changes: 32 additions & 3 deletions syft/pkg/cataloger/javascript/package.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ func newPackageJSONPackage(u packageJSON, indexLocation file.Location) pkg.Packa
return p
}

func newPackageLockV1Package(resolver file.Resolver, location file.Location, name string, u lockDependency) pkg.Package {
func newPackageLockV1Package(cfg CatalogerConfig, resolver file.Resolver, location file.Location, name string, u lockDependency) pkg.Package {
version := u.Version

const aliasPrefixPackageLockV1 = "npm:"
Expand All @@ -63,12 +63,26 @@ func newPackageLockV1Package(resolver file.Resolver, location file.Location, nam
version = canonicalPackageAndVersion[versionSeparator+1:]
}

var licenseSet pkg.LicenseSet

if cfg.SearchRemoteLicenses {
license, err := getLicenseFromNpmRegistry(cfg.NPMBaseURL, name, version)
if err == nil && license != "" {
licenses := pkg.NewLicensesFromValues(license)
licenseSet = pkg.NewLicenseSet(licenses...)
}
if err != nil {
log.Warnf("unable to extract licenses from javascript yarn.lock for package %s:%s: %+v", name, version, err)
}
}

return finalizeLockPkg(
resolver,
location,
pkg.Package{
Name: name,
Version: version,
Licenses: licenseSet,
Locations: file.NewLocationSet(location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation)),
PURL: packageURL(name, version),
Language: pkg.JavaScript,
Expand All @@ -78,15 +92,30 @@ func newPackageLockV1Package(resolver file.Resolver, location file.Location, nam
)
}

func newPackageLockV2Package(resolver file.Resolver, location file.Location, name string, u lockPackage) pkg.Package {
func newPackageLockV2Package(cfg CatalogerConfig, resolver file.Resolver, location file.Location, name string, u lockPackage) pkg.Package {
var licenseSet pkg.LicenseSet

if u.License != nil {
licenseSet = pkg.NewLicenseSet(pkg.NewLicensesFromLocation(location, u.License...)...)
} else if cfg.SearchRemoteLicenses {
license, err := getLicenseFromNpmRegistry(cfg.NPMBaseURL, name, u.Version)
if err == nil && license != "" {
licenses := pkg.NewLicensesFromValues(license)
licenseSet = pkg.NewLicenseSet(licenses...)
}
if err != nil {
log.Warnf("unable to extract licenses from javascript yarn.lock for package %s:%s: %+v", name, u.Version, err)
}
}

return finalizeLockPkg(
resolver,
location,
pkg.Package{
Name: name,
Version: u.Version,
Locations: file.NewLocationSet(location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation)),
Licenses: pkg.NewLicenseSet(pkg.NewLicensesFromLocation(location, u.License...)...),
Licenses: licenseSet,
PURL: packageURL(name, u.Version),
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
Expand Down
19 changes: 13 additions & 6 deletions syft/pkg/cataloger/javascript/parse_package_lock.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,6 @@ import (
"github.com/anchore/syft/syft/pkg/cataloger/generic"
)

// integrity check
var _ generic.Parser = parsePackageLock

// packageLock represents a JavaScript package.lock json file
type packageLock struct {
Requires bool `json:"requires"`
Expand All @@ -44,8 +41,18 @@ type lockPackage struct {
// packageLockLicense
type packageLockLicense []string

type genericPackageLockAdapter struct {
cfg CatalogerConfig
}

func newGenericPackageLockAdapter(cfg CatalogerConfig) genericPackageLockAdapter {
return genericPackageLockAdapter{
cfg: cfg,
}
}

// parsePackageLock parses a package-lock.json and returns the discovered JavaScript packages.
func parsePackageLock(_ context.Context, resolver file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
func (a genericPackageLockAdapter) parsePackageLock(_ context.Context, resolver file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
// in the case we find package-lock.json files in the node_modules directories, skip those
// as the whole purpose of the lock file is for the specific dependencies of the root project
if pathContainsNodeModulesDirectory(reader.Path()) {
Expand All @@ -66,7 +73,7 @@ func parsePackageLock(_ context.Context, resolver file.Resolver, _ *generic.Envi

if lock.LockfileVersion == 1 {
for name, pkgMeta := range lock.Dependencies {
pkgs = append(pkgs, newPackageLockV1Package(resolver, reader.Location, name, pkgMeta))
pkgs = append(pkgs, newPackageLockV1Package(a.cfg, resolver, reader.Location, name, pkgMeta))
}
}

Expand All @@ -86,7 +93,7 @@ func parsePackageLock(_ context.Context, resolver file.Resolver, _ *generic.Envi

pkgs = append(
pkgs,
newPackageLockV2Package(resolver, reader.Location, getNameFromPath(name), pkgMeta),
newPackageLockV2Package(a.cfg, resolver, reader.Location, getNameFromPath(name), pkgMeta),
)
}
}
Expand Down
15 changes: 10 additions & 5 deletions syft/pkg/cataloger/javascript/parse_package_lock_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,8 @@ func TestParsePackageLock(t *testing.T) {
expectedPkgs[i].Locations.Add(file.NewLocation(fixture))
}

pkgtest.TestFileParser(t, fixture, parsePackageLock, expectedPkgs, expectedRelationships)
adapter := newGenericPackageLockAdapter(CatalogerConfig{})
pkgtest.TestFileParser(t, fixture, adapter.parsePackageLock, expectedPkgs, expectedRelationships)
}

func TestParsePackageLockV2(t *testing.T) {
Expand Down Expand Up @@ -169,7 +170,8 @@ func TestParsePackageLockV2(t *testing.T) {
for i := range expectedPkgs {
expectedPkgs[i].Locations.Add(file.NewLocation(fixture))
}
pkgtest.TestFileParser(t, fixture, parsePackageLock, expectedPkgs, expectedRelationships)
adapter := newGenericPackageLockAdapter(CatalogerConfig{})
pkgtest.TestFileParser(t, fixture, adapter.parsePackageLock, expectedPkgs, expectedRelationships)
}

func TestParsePackageLockV3(t *testing.T) {
Expand Down Expand Up @@ -220,7 +222,8 @@ func TestParsePackageLockV3(t *testing.T) {
for i := range expectedPkgs {
expectedPkgs[i].Locations.Add(file.NewLocation(fixture))
}
pkgtest.TestFileParser(t, fixture, parsePackageLock, expectedPkgs, expectedRelationships)
adapter := newGenericPackageLockAdapter(CatalogerConfig{})
pkgtest.TestFileParser(t, fixture, adapter.parsePackageLock, expectedPkgs, expectedRelationships)
}

func TestParsePackageLockAlias(t *testing.T) {
Expand Down Expand Up @@ -279,7 +282,8 @@ func TestParsePackageLockAlias(t *testing.T) {
for i := range expected {
expected[i].Locations.Add(file.NewLocation(pl))
}
pkgtest.TestFileParser(t, pl, parsePackageLock, expected, expectedRelationships)
adapter := newGenericPackageLockAdapter(CatalogerConfig{})
pkgtest.TestFileParser(t, pl, adapter.parsePackageLock, expected, expectedRelationships)
}
}

Expand Down Expand Up @@ -326,5 +330,6 @@ func TestParsePackageLockLicenseWithArray(t *testing.T) {
for i := range expectedPkgs {
expectedPkgs[i].Locations.Add(file.NewLocation(fixture))
}
pkgtest.TestFileParser(t, fixture, parsePackageLock, expectedPkgs, expectedRelationships)
adapter := newGenericPackageLockAdapter(CatalogerConfig{})
pkgtest.TestFileParser(t, fixture, adapter.parsePackageLock, expectedPkgs, expectedRelationships)
}