Support package-lock.json v2.

core/src/main/java/org/eclipse/dash/licenses/cli/PackageLockFileReader.java

@@ -1,10 +1,20 @@
+/*************************************************************************
+ * Copyright (c) 2020,2021 The Eclipse Foundation and others.
+ *
+ * This program and the accompanying materials are made available under
+ * the terms of the Eclipse Public License 2.0 which accompanies this
+ * distribution, and is available at https://www.eclipse.org/legal/epl-2.0
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ *************************************************************************/
 package org.eclipse.dash.licenses.cli;
 
 import java.io.InputStream;
 import java.util.Collection;
-import java.util.HashSet;
-import java.util.Set;
-import java.util.function.Consumer;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import org.eclipse.dash.licenses.ContentId;
 import org.eclipse.dash.licenses.IContentId;
@@ -15,49 +25,110 @@
 
 public class PackageLockFileReader implements IDependencyListReader {
 
-	private InputStream input;
+	private final InputStream input;
 
 	public PackageLockFileReader(InputStream input) {
 		this.input = input;
 	}
 
-	@Override
-	public Collection<IContentId> getContentIds() {
-		Set<IContentId> content = new HashSet<>();
-		JsonObject json = JsonUtils.readJson(input);
-		dependenciesDo(json.getJsonObject("dependencies"), id -> content.add(id));
-		return content;
+	/**
+	 * Version 2 of the package-lock.json format includes a flat associative array
+	 * of "packages"; each entry contains information about one of the dependencies.
+	 * Each instance of this class represents one package entry.
+	 */
+	class Package {
+		final String key;
+		final JsonObject value;
+
+		Package(String key, JsonObject value) {
+			this.key = key;
+			this.value = value;
+		}
+
+		/**
+		 * The content id needs to be extracted from a combination of the key and values
+		 * from the associated associative array.
+		 * <p>
+		 * For version 1, the key contains the name, and--when one is defined--the
+		 * scope:
+		 * <ul>
+		 * <li>junk</li>
+		 * <li>@babel/junk</li>
+		 * </ul>
+		 * 
+		 * <p>
+		 * For version 2, the key also includes the path:
+		 * 
+		 * <ul>
+		 * <li>node_modules/@types/babel__core</li>
+		 * <li>node_modules/@types/babel__generator</li>
+		 * <li>node_modules/@jest/transform/node_modules/slash</li>
+		 * <li>node_modules/acorn-globals</li>
+		 * </ul>
+		 * 
+		 * <p>
+		 * It's relatively easy to handle all cases with a regular expression.
+		 */
+		IContentId getContentId() {
+			Pattern pattern = Pattern.compile("(?:(?<scope>@[^\\/]+)\\/)?(?<name>[^\\/]+)$");
+			Matcher matcher = pattern.matcher(key);
+			if (matcher.find()) {
+				var namespace = matcher.group("scope");
+				if (namespace == null)
+					namespace = "-";
+				var name = matcher.group("name");
+				var version = value.asJsonObject().getString("version");
+
+				return ContentId.getContentId("npm", "npmjs", namespace, name, version);
+			}
+			return new InvalidContentId(key);
+		}
 	}
 
 	/**
-	 * Walk through the structure to identify the dependencies, and (recursively)
-	 * the dependencies of the dependencies.
-	 *
-	 * @param json     A "dependency" list from a package-lock.json file
-	 * @param consumer A single argument consumer of each listed item.
+	 * Version 1 of the package-lock.json format has a notion of dependencies. Each
+	 * dependency record contains information about a single resolve dependency.
+	 * Every dependency can have its own dependencies (recursive).
 	 */
-	private void dependenciesDo(JsonObject json, Consumer<IContentId> consumer) {
-		if (json == null)
-			return;
-		json.forEach((key, value) -> {
-			String[] parts = key.split("/");
-			String namespace;
-			String name;
-			if (parts.length == 2) {
-				namespace = parts[0];
-				name = parts[1];
-			} else {
-				namespace = "-";
-				name = key;
-			}
-			String version = value.asJsonObject().getString("version");
-			IContentId contentId = ContentId.getContentId("npm", "npmjs", namespace, name, version);
-			if (contentId == null)
-				contentId = new InvalidContentId("" + namespace + "/" + name + "@" + version);
-			consumer.accept(contentId);
-
-			dependenciesDo(json.getJsonObject("dependencies"), consumer);
-		});
+	class Dependency extends Package {
+
+		Dependency(String key, JsonObject value) {
+			super(key, value);
+		}
+
+		Stream<Dependency> getDependencies() {
+			JsonObject dependencies = value.getJsonObject("dependencies");
+			if (dependencies == null)
+				return Stream.empty();
+
+			return dependencies.entrySet().stream()
+					.map(each -> new Dependency(each.getKey(), each.getValue().asJsonObject()));
+		}
+
+		/**
+		 * Flatten the arbitrary hierarchy (tree) of dependencies into a single stream.
+		 */
+		Stream<Dependency> stream() {
+			return Stream.concat(Stream.of(this), getDependencies().flatMap(Dependency::stream));
+		}
+	}
+
+	@Override
+	public Collection<IContentId> getContentIds() {
+		JsonObject json = JsonUtils.readJson(input);
+
+		switch (json.getJsonNumber("lockfileVersion").intValue()) {
+		case 1:
+			return new Dependency("", json).stream().filter(each -> !each.key.isEmpty())
+					.map(dependency -> dependency.getContentId()).collect(Collectors.toList());
+		case 2:
+		case 3:
+			return json.getJsonObject("packages").entrySet().stream().filter(entry -> !entry.getKey().isEmpty())
+					.map(entry -> new Package(entry.getKey(), entry.getValue().asJsonObject()))
+					.map(dependency -> dependency.getContentId()).distinct().collect(Collectors.toList());
+		}
+
+		return null;
 	}
 
 }

core/src/test/java/org/eclipse/dash/licenses/tests/PackageLockFileReaderTests.java

@@ -1,5 +1,5 @@
 /*************************************************************************
- * Copyright (c) 2020, The Eclipse Foundation and others.
+ * Copyright (c) 2020,2021 The Eclipse Foundation and others.
  *
  * This program and the accompanying materials are made available under
  * the terms of the Eclipse Public License 2.0 which accompanies this
@@ -21,7 +21,7 @@
 class PackageLockFileReaderTests {
 
 	@Test
-	void testMultipleBatches() throws IOException {
+	void testV1Format() throws IOException {
 		try (InputStream input = this.getClass().getResourceAsStream("/package-lock.json")) {
 			PackageLockFileReader reader = new PackageLockFileReader(input);
 			String[] expected = { "npm/npmjs/-/loglevel/1.6.1", "npm/npmjs/-/sax/1.2.4", "npm/npmjs/-/saxes/3.1.9",
@@ -31,4 +31,15 @@ void testMultipleBatches() throws IOException {
 			assertArrayEquals(expected, found);
 		}
 	}
+
+	@Test
+	void testV2Format() throws IOException {
+		try (InputStream input = this.getClass().getResourceAsStream("/package-lock-v2.json")) {
+			PackageLockFileReader reader = new PackageLockFileReader(input);
+			String[] expected = { "npm/npmjs/-/delayed-stream/1.0.0", "npm/npmjs/-/is-descriptor/1.0.2",
+					"npm/npmjs/@babel/code-frame/7.12.13", };
+			String[] found = reader.getContentIds().stream().map(IContentId::toString).sorted().toArray(String[]::new);
+			assertArrayEquals(expected, found);
+		}
+	}
 }

core/src/test/java/package-lock-v2.json

@@ -0,0 +1,48 @@
+{
+	"name": "bumlux",
+	"version": "1.0.0",
+	"lockfileVersion": 2,
+	"requires": true,
+	"packages": {
+		"": {
+			"name": "bumlux",
+			"version": "2.0.0",
+			"license": "SEE LICENSE IN LICENSE",
+			"dependencies": {},
+			"devDependencies": {
+				"@babel/code-frame": "7.5.5"
+			}
+		},
+		"node_modules/@babel/code-frame": {
+			"version": "7.12.13",
+			"resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.12.13.tgz",
+			"integrity": "sha512-HV1Cm0Q3ZrpCR93tkWOYiuYIgLxZXZFVG2VgK+MBWjUqZTundupbfx2aXarXuw5Ko5aMcjtJgbSs4vUGBS5v6g==",
+			"dependencies": {
+				"@babel/highlight": "^7.12.13"
+			}
+		},
+		"node_modules/define-property/node_modules/is-descriptor": {
+			"version": "1.0.2",
+			"resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-1.0.2.tgz",
+			"integrity": "sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg==",
+			"dev": true,
+			"dependencies": {
+				"is-accessor-descriptor": "^1.0.0",
+				"is-data-descriptor": "^1.0.0",
+				"kind-of": "^6.0.2"
+			},
+			"engines": {
+				"node": ">=0.10.0"
+			}
+		},
+		"node_modules/delayed-stream": {
+			"version": "1.0.0",
+			"resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
+			"integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=",
+			"dev": true,
+			"engines": {
+				"node": ">=0.4.0"
+			}
+		}
+	}
+}