Skip to content

Commit

Permalink
feat: Add support for npm lockfile version 3 (#1206)
Browse files Browse the repository at this point in the history 
This PR adds support for npm lockfile version 3, which drops the
"dependencies" key and uses "packages" instead. I've refactored the
lockfile parser to make the distinction between the versions explicit
rather than the implicit behaviour before. It _might_ be worth splitting
into separate files at some point, but the logic is so minimal that I
haven't done it.

Fixes #1203
Signed-off-by: Rob Cresswell <robcresswell@users.noreply.github.com>
  • Loading branch information
@robcresswell
robcresswell committed Nov 18, 2022
1 parent 67888ee commit 9d8244b
Show file tree
Hide file tree
Showing 4 changed files with 137 additions and 16 deletions.
25 changes: 19 additions & 6 deletions syft/pkg/cataloger/javascript/package.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,13 +43,26 @@ func newPackageJSONPackage(u packageJSON, locations ...source.Location) pkg.Pack
return p
}

func newPackageLockPackage(resolver source.FileResolver, location source.Location, name string, u lockDependency, licenseMap map[string]string) pkg.Package {
var sb strings.Builder
sb.WriteString(u.Resolved)
sb.WriteString(u.Integrity)
func newPackageLockV1Package(resolver source.FileResolver, location source.Location, name string, u lockDependency) pkg.Package {
return finalizeLockPkg(
resolver,
location,
pkg.Package{
Name: name,
Version: u.Version,
Locations: source.NewLocationSet(location),
PURL: packageURL(name, u.Version),
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
},
)
}

func newPackageLockV2Package(resolver source.FileResolver, location source.Location, name string, u lockPackage) pkg.Package {

This comment has been minimized.

Sign in to view

Copy link
@Mikcl

Mikcl Nov 18, 2022

Contributor

iiuc, this function handles both V2 and V3, do we want to name it accordingly?
var licenses []string
if l, exists := licenseMap[sb.String()]; exists {
licenses = append(licenses, l)

if u.License != "" {
licenses = append(licenses, u.License)
}

return finalizeLockPkg(
Expand Down
35 changes: 25 additions & 10 deletions syft/pkg/cataloger/javascript/parse_package_lock.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,10 +32,11 @@ type lockDependency struct {
}

type lockPackage struct {
Name string `json:"name"` // only present in the root package entry (named "")
Version string `json:"version"`
Resolved string `json:"resolved"`
Integrity string `json:"integrity"`
License string `json:""`
License string `json:"license"`
}

// parsePackageLock parses a package-lock.json and returns the discovered JavaScript packages.
Expand All @@ -49,27 +50,41 @@ func parsePackageLock(resolver source.FileResolver, _ *generic.Environment, read
var pkgs []pkg.Package
dec := json.NewDecoder(reader)

var lock packageLock
for {
var lock packageLock
if err := dec.Decode(&lock); errors.Is(err, io.EOF) {
break
} else if err != nil {
return nil, nil, fmt.Errorf("failed to parse package-lock.json file: %w", err)
}
licenseMap := make(map[string]string)
for _, pkgMeta := range lock.Packages {
var sb strings.Builder
sb.WriteString(pkgMeta.Resolved)
sb.WriteString(pkgMeta.Integrity)
licenseMap[sb.String()] = pkgMeta.License
}
}

if lock.LockfileVersion == 1 {
for name, pkgMeta := range lock.Dependencies {
pkgs = append(pkgs, newPackageLockPackage(resolver, reader.Location, name, pkgMeta, licenseMap))
pkgs = append(pkgs, newPackageLockV1Package(resolver, reader.Location, name, pkgMeta))
}
}

if lock.LockfileVersion == 2 || lock.LockfileVersion == 3 {
for name, pkgMeta := range lock.Packages {
if name == "" {
if pkgMeta.Name == "" {
continue
} else {
name = pkgMeta.Name
}
}

This comment has been minimized.

Sign in to view

Copy link
@Mikcl

Mikcl Nov 18, 2022 •
edited
Loading

Contributor

There is an assumption being made here that the pkgMeta.name only exists when name == ""

This is no necessarily the case, for example with alias packages.

Thus i think we should have the following addtional check here:

if pkgMeta.name != "" {
     name = pkgMeta.name
}

Example V2 package-lock.json: https://gist.github.com/Mikcl/aef4d180f4275b66252dca32b6668375

Note this still means that alias name,version are not resolved for v1 package-lock.json, which should be a separate PR.

however, i think these few line changes are worthwhile to make here.

edit: same is true for V3

This comment has been minimized.

Sign in to view

Copy link
@Mikcl

Mikcl Nov 18, 2022

Contributor

following up with own PR #1349
pkgs = append(pkgs, newPackageLockV2Package(resolver, reader.Location, getNameFromPath(name), pkgMeta))
}
}

pkg.Sort(pkgs)

return pkgs, nil, nil
}

func getNameFromPath(path string) string {
parts := strings.Split(path, "node_modules/")
return parts[len(parts)-1]
}
53 changes: 53 additions & 0 deletions syft/pkg/cataloger/javascript/parse_package_lock_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,13 @@ func TestParsePackageLockV2(t *testing.T) {
fixture := "test-fixtures/pkg-lock/package-lock-2.json"
var expectedRelationships []artifact.Relationship
expectedPkgs := []pkg.Package{
{
Name: "npm",
Version: "6.14.6",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
PURL: "pkg:npm/npm@6.14.6",
},
{
Name: "@types/prop-types",
Version: "15.7.5",
Expand Down Expand Up @@ -140,3 +147,49 @@ func TestParsePackageLockV2(t *testing.T) {
}
pkgtest.TestFileParser(t, fixture, parsePackageLock, expectedPkgs, expectedRelationships)
}

func TestParsePackageLockV3(t *testing.T) {
fixture := "test-fixtures/pkg-lock/package-lock-3.json"
var expectedRelationships []artifact.Relationship
expectedPkgs := []pkg.Package{
{
Name: "lock-v3-fixture",
Version: "1.0.0",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
PURL: "pkg:npm/lock-v3-fixture@1.0.0",
},
{
Name: "@types/prop-types",
Version: "15.7.5",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
PURL: "pkg:npm/%40types/prop-types@15.7.5",
},
{
Name: "@types/react",
Version: "18.0.20",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
PURL: "pkg:npm/%40types/react@18.0.20",
},
{
Name: "@types/scheduler",
Version: "0.16.2",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
PURL: "pkg:npm/%40types/scheduler@0.16.2",
},
{
Name: "csstype",
Version: "3.1.1",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
PURL: "pkg:npm/csstype@3.1.1",
},
}
for i := range expectedPkgs {
expectedPkgs[i].Locations.Add(source.NewLocation(fixture))
}
pkgtest.TestFileParser(t, fixture, parsePackageLock, expectedPkgs, expectedRelationships)
}
40 changes: 40 additions & 0 deletions syft/pkg/cataloger/javascript/test-fixtures/pkg-lock/package-lock-3.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
{
"name": "lock-v3-fixture",
"version": "1.0.0",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "lock-v3-fixture",
"version": "1.0.0",
"dependencies": {
"@types/react": "^18.0.9"
}
},
"node_modules/@types/prop-types": {
"version": "15.7.5",
"resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.5.tgz",
"integrity": "sha512-JCB8C6SnDoQf0cNycqd/35A7MjcnK+ZTqE7judS6o7utxUCg6imJg3QK2qzHKszlTjcj2cn+NwMB2i96ubpj7w=="
},
"node_modules/@types/react": {
"version": "18.0.20",
"resolved": "https://registry.npmjs.org/@types/react/-/react-18.0.20.tgz",
"integrity": "sha512-MWul1teSPxujEHVwZl4a5HxQ9vVNsjTchVA+xRqv/VYGCuKGAU6UhfrTdF5aBefwD1BHUD8i/zq+O/vyCm/FrA==",
"dependencies": {
"@types/prop-types": "*",
"@types/scheduler": "*",
"csstype": "^3.0.2"
}
},
"node_modules/@types/scheduler": {
"version": "0.16.2",
"resolved": "https://registry.npmjs.org/@types/scheduler/-/scheduler-0.16.2.tgz",
"integrity": "sha512-hppQEBDmlwhFAXKJX2KnWLYu5yMfi91yazPb2l+lbJiwW+wdo1gNeRA+3RgNSO39WYX2euey41KEwnqesU2Jew=="
},
"node_modules/csstype": {
"version": "3.1.1",
"resolved": "https://registry.npmjs.org/csstype/-/csstype-3.1.1.tgz",
"integrity": "sha512-DJR/VvkAvSZW9bTouZue2sSxDwdTN92uHjqeKVm+0dAqdfNykRzQ95tay8aXMBAAPpUiq4Qcug2L7neoRh2Egw=="
}
}
}

0 comments on commit 9d8244b

Please sign in to comment.