Updates parsing of yarn.lock to use resolved URLs that are pulled…

… from yarn and npm registries (anchore#926)

Co-authored-by: Christopher Phillips <christopher.phillips@anchore.com>

syft/pkg/cataloger/javascript/parse_yarn_lock.go

@@ -25,6 +25,16 @@ var (
 	// versionExp matches the "version" line of a yarn.lock entry and captures the version value.
 	// For example: version "4.10.1" (...and the value "4.10.1" is captured)
 	versionExp = regexp.MustCompile(`^\W+version(?:\W+"|:\W+)([\w-_.]+)"?`)
+
+	// packageURLExp matches the name and version of the dependency in yarn.lock
+	// from the resolved URL, including scope/namespace prefix if any.
+	// For example:
+	//		`resolved "https://registry.yarnpkg.com/async/-/async-3.2.3.tgz#ac53dafd3f4720ee9e8a160628f18ea91df196c9"`
+	//			would return "async" and "3.2.3"
+	//
+	//		`resolved "https://registry.yarnpkg.com/@4lolo/resize-observer-polyfill/-/resize-observer-polyfill-1.5.2.tgz#58868fc7224506236b5550d0c68357f0a874b84b"`
+	//			would return "@4lolo/resize-observer-polyfill" and "1.5.2"
+	packageURLExp = regexp.MustCompile(`^\s+resolved\s+"https://registry\.(?:yarnpkg\.com|npmjs\.org)/(.+?)/-/(?:.+?)-(\d+\..+?)\.tgz`)
 )
 
 const (
@@ -43,39 +53,37 @@ func parseYarnLock(path string, reader io.Reader) ([]*pkg.Package, []artifact.Re
 	scanner := bufio.NewScanner(reader)
 	parsedPackages := internal.NewStringSet()
 	currentPackage := noPackage
+	currentVersion := noVersion
 
 	for scanner.Scan() {
 		line := scanner.Text()
 
-		if currentPackage == noPackage {
-			// Scan until we find the next package
-
-			packageName := findPackageName(line)
-			if packageName == noPackage {
-				continue
-			}
-
-			if parsedPackages.Contains(packageName) {
-				// We don't parse repeated package declarations.
-				continue
+		if packageName := findPackageName(line); packageName != noPackage {
+			// When we find a new package, check if we have unsaved identifiers
+			if currentPackage != noPackage && currentVersion != noVersion && !parsedPackages.Contains(currentPackage+"@"+currentVersion) {
+				packages = append(packages, newYarnLockPackage(currentPackage, currentVersion))
+				parsedPackages.Add(currentPackage + "@" + currentVersion)
 			}
 
 			currentPackage = packageName
-			parsedPackages.Add(currentPackage)
-
-			continue
-		}
-
-		// We've found the package entry, now we just need the version
+		} else if version := findPackageVersion(line); version != noVersion {
+			currentVersion = version
+		} else if packageName, version := findPackageAndVersion(line); packageName != noPackage && version != noVersion && !parsedPackages.Contains(packageName+"@"+version) {
+			packages = append(packages, newYarnLockPackage(packageName, version))
+			parsedPackages.Add(packageName + "@" + version)
 
-		if version := findPackageVersion(line); version != noVersion {
-			packages = append(packages, newYarnLockPackage(currentPackage, version))
+			// Cleanup to indicate no unsaved identifiers
 			currentPackage = noPackage
-
-			continue
+			currentVersion = noVersion
 		}
 	}
 
+	// check if we have valid unsaved data after end-of-file has reached
+	if currentPackage != noPackage && currentVersion != noVersion && !parsedPackages.Contains(currentPackage+"@"+currentVersion) {
+		packages = append(packages, newYarnLockPackage(currentPackage, currentVersion))
+		parsedPackages.Add(currentPackage + "@" + currentVersion)
+	}
+
 	if err := scanner.Err(); err != nil {
 		return nil, nil, fmt.Errorf("failed to parse yarn.lock file: %w", err)
 	}
@@ -99,6 +107,14 @@ func findPackageVersion(line string) string {
 	return noVersion
 }
 
+func findPackageAndVersion(line string) (string, string) {
+	if matches := packageURLExp.FindStringSubmatch(line); len(matches) >= 2 {
+		return matches[1], matches[2]
+	}
+
+	return noPackage, noVersion
+}
+
 func newYarnLockPackage(name, version string) *pkg.Package {
 	return &pkg.Package{
 		Name:     name,

syft/pkg/cataloger/javascript/parse_yarn_lock_test.go

@@ -9,7 +9,7 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-func TestParseYarnLock(t *testing.T) {
+func TestParseYarnBerry(t *testing.T) {
 	expected := map[string]pkg.Package{
 		"@babel/code-frame": {
 			Name:     "@babel/code-frame",
@@ -66,10 +66,87 @@ func TestParseYarnLock(t *testing.T) {
 			Type:     pkg.NpmPkg,
 		},
 	}
+	testFixtures := []string{
+		"test-fixtures/yarn-berry/yarn.lock",
+	}
+
+	for _, file := range testFixtures {
+		file := file
+		t.Run(file, func(t *testing.T) {
+			t.Parallel()
+
+			fixture, err := os.Open(file)
+			require.NoError(t, err)
+
+			// TODO: no relationships are under test yet
+			actual, _, err := parseYarnLock(fixture.Name(), fixture)
+			require.NoError(t, err)
+
+			assertPkgsEqual(t, actual, expected)
+		})
+	}
+}
+
+func TestParseYarnLock(t *testing.T) {
+	expected := map[string]pkg.Package{
+		"@babel/code-frame": {
+			Name:     "@babel/code-frame",
+			Version:  "7.10.4",
+			Language: pkg.JavaScript,
+			Type:     pkg.NpmPkg,
+		},
+		"@types/minimatch": {
+			Name:     "@types/minimatch",
+			Version:  "3.0.3",
+			Language: pkg.JavaScript,
+			Type:     pkg.NpmPkg,
+		},
+		"@types/qs": {
+			Name:     "@types/qs",
+			Version:  "6.9.4",
+			Language: pkg.JavaScript,
+			Type:     pkg.NpmPkg,
+		},
+		"ajv": {
+			Name:     "ajv",
+			Version:  "6.12.3",
+			Language: pkg.JavaScript,
+			Type:     pkg.NpmPkg,
+		},
+		"atob": {
+			Name:     "atob",
+			Version:  "2.1.2",
+			Language: pkg.JavaScript,
+			Type:     pkg.NpmPkg,
+		},
+		"aws-sdk": {
+			Name:     "aws-sdk",
+			Version:  "2.706.0",
+			Language: pkg.JavaScript,
+			Type:     pkg.NpmPkg,
+		},
+		"jhipster-core": {
+			Name:     "jhipster-core",
+			Version:  "7.3.4",
+			Language: pkg.JavaScript,
+			Type:     pkg.NpmPkg,
+		},
+		"asn1.js": {
+			Name:     "asn1.js",
+			Version:  "4.10.1",
+			Language: pkg.JavaScript,
+			Type:     pkg.NpmPkg,
+		},
+		"something-i-made-up": {
+			Name:     "something-i-made-up",
+			Version:  "7.7.7",
+			Language: pkg.JavaScript,
+			Type:     pkg.NpmPkg,
+		},
+	}
 
 	testFixtures := []string{
 		"test-fixtures/yarn/yarn.lock",
-		"test-fixtures/yarn-berry/yarn.lock",
 	}
 
 	for _, file := range testFixtures {

test/integration/node_packages_test.go

@@ -1,6 +1,7 @@
 package integration
 
 import (
+	"reflect"
 	"strings"
 	"testing"
 
@@ -33,19 +34,21 @@ func TestYarnPackageLockDirectory(t *testing.T) {
 	sbom, _ := catalogDirectory(t, "test-fixtures/yarn-lock")
 
 	foundPackages := internal.NewStringSet()
+	expectedPackages := internal.NewStringSet("async@0.9.2", "async@3.2.3", "merge-objects@1.0.5", "should-type@1.3.0", "@4lolo/resize-observer-polyfill@1.5.2")
 
 	for actualPkg := range sbom.Artifacts.PackageCatalog.Enumerate(pkg.NpmPkg) {
 		for _, actualLocation := range actualPkg.Locations.ToSlice() {
 			if strings.Contains(actualLocation.RealPath, "node_modules") {
 				t.Errorf("found packages from yarn.lock in node_modules: %s", actualLocation)
 			}
 		}
-		foundPackages.Add(actualPkg.Name)
+		foundPackages.Add(actualPkg.Name + "@" + actualPkg.Version)
 	}
 
 	// ensure that integration test commonTestCases stay in sync with the available catalogers
-	const expectedPackageCount = 5
-	if len(foundPackages) != expectedPackageCount {
-		t.Errorf("found the wrong set of yarn.lock packages (expected: %d, actual: %d)", expectedPackageCount, len(foundPackages))
+	if len(foundPackages) != len(expectedPackages) {
+		t.Errorf("found the wrong set of yarn.lock packages (expected: %d, actual: %d)", len(expectedPackages), len(foundPackages))
+	} else if !reflect.DeepEqual(foundPackages, expectedPackages) {
+		t.Errorf("found the wrong set of yarn.lock packages (expected: %+q, actual: %+q)", expectedPackages.ToSlice(), foundPackages.ToSlice())
 	}
 }