Adding feature to scan pnpm-lock.yaml (PNPM)

core/pom.xml

@@ -106,6 +106,12 @@
 			<artifactId>org.eclipse.jgit</artifactId>
 			<version>6.8.0.202311291450-r</version>
 		</dependency>
+		<dependency>
+			<groupId>org.yaml</groupId>
+			<artifactId>snakeyaml</artifactId>
+			<version>2.2</version>
+		</dependency>
+
 		<dependency>
 			<groupId>org.junit.jupiter</groupId>
 			<artifactId>junit-jupiter-engine</artifactId>

core/src/main/java/org/eclipse/dash/licenses/cli/Main.java

@@ -159,13 +159,15 @@ private IDependencyListReader getReader(String name) throws FileNotFoundExceptio
 		} else {
 			File input = new File(name);
 			if (input.exists()) {
-				if ("package-lock.json".equals(input.getName())) {
-					return new PackageLockFileReader(new FileInputStream(input));
-				}
-				if ("yarn.lock".equals(input.getName())) {
-					return new YarnLockFileReader(new FileReader(input));
-				}
-				return new FlatFileReader(new FileReader(input));
+                switch (input.getName()) {
+                    case "pnpm-lock.yaml":
+                        return new PnpmPackageLockFileReader(new FileInputStream(input));
+                    case "package-lock.json":
+                        return new PackageLockFileReader(new FileInputStream(input));
+                    case "yarn.lock":
+                        return new YarnLockFileReader(new FileReader(input));
+                }
+                return new FlatFileReader(new FileReader(input));
 			} else {
 				throw new FileNotFoundException(name);
 			}

core/src/main/java/org/eclipse/dash/licenses/cli/PnpmPackageLockFileReader.java

@@ -0,0 +1,123 @@
+/*************************************************************************
+ * Copyright (c) 2021 The Eclipse Foundation and others.
+ *
+ * This program and the accompanying materials are made available under
+ * the terms of the Eclipse Public License 2.0 which accompanies this
+ * distribution, and is available at https://www.eclipse.org/legal/epl-2.0
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ *************************************************************************/
+
+package org.eclipse.dash.licenses.cli;
+
+import org.eclipse.dash.licenses.ContentId;
+import org.eclipse.dash.licenses.IContentId;
+import org.eclipse.dash.licenses.InvalidContentId;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.yaml.snakeyaml.DumperOptions;
+import org.yaml.snakeyaml.LoaderOptions;
+import org.yaml.snakeyaml.Yaml;
+import org.yaml.snakeyaml.constructor.SafeConstructor;
+import org.yaml.snakeyaml.representer.Representer;
+
+import java.io.InputStream;
+import java.util.Collection;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Optional;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+/**
+ * This class is responsible for reading a PNPM package-lock file and extracting content IDs.
+ * A content ID represents a unique identifier for a package or dependency.
+ *
+ * The class implements the IDependencyListReader interface.
+ *
+ * The class uses the SnakeYAML library to parse the package-lock file in YAML format.
+ * Content ID is extracted only from the keys of the packages section of the package-lock file.
+ * The main magic is done by the regex: KEY_PATTERN.
+ *
+ **/
+public class PnpmPackageLockFileReader implements IDependencyListReader {
+    final Logger logger = LoggerFactory.getLogger(PnpmPackageLockFileReader.class);
+    private static final Pattern KEY_PATTERN = Pattern.compile("^'?(\\/?(?<namespace>@[^\\/]+)\\/)?\\/?(?<name>[^\\/@]+)[@\\/](?<version>[^(@\\/'\\n]+)(?=\\()?");
+    private final InputStream input;
+
+    /**
+     * Constructs a new PnpmPackageLockFileReader with the specified input stream.
+     *
+     * @param input the input stream of the PNPM package-lock file
+     */
+    public PnpmPackageLockFileReader(InputStream input) {
+        this.input = input;
+    }
+
+    /**
+     * Returns a collection of unique content IDs extracted from the PNPM package-lock file.
+     *
+     * @return a collection of content IDs
+     */
+    @Override
+    public Collection<IContentId> getContentIds() {
+        return contentIds().distinct().collect(Collectors.toList());
+    }
+
+    /**
+     * Parses the specified key and returns the corresponding content ID.
+     *
+     * @param key the key to parse
+     * @return the content ID extracted from the key
+     */
+    public IContentId getId(String key) {
+        var matcher = KEY_PATTERN.matcher(key);
+        if (matcher.find()) {
+            var namespace = Optional.ofNullable(matcher.group("namespace")).orElse("-");
+            var name = matcher.group("name");
+            var version = matcher.group("version");
+            return ContentId.getContentId("npm", "npmjs", namespace, name, version);
+        }
+
+        logger.debug("Invalid content id: {}", key);
+        return new InvalidContentId(key);
+    }
+
+    /**
+     * Returns a stream of content IDs extracted from the PNPM package-lock file.
+	 * We only read the keys of the packages.
+	 *
+	 * packages:
+	 *
+	 *   /@babel/preset-modules@0.1.6-no-external-plugins(@babel/core@7.23.2):
+     *
+     * @return a stream of content IDs
+     */
+	@SuppressWarnings("unchecked")
+    public Stream<IContentId> contentIds() {
+        Yaml yaml = getYamlParser();
+        Map<String, Object> load;
+        try {
+            load = yaml.load(input);
+            Map<String, Object> packages = (Map<String, Object>) load.getOrDefault("packages", new LinkedHashMap<>());
+            return packages.keySet().stream().map(this::getId);
+        } catch (Exception e) {
+            logger.error("Error reading content of package-lock.yaml file", e);
+            throw new RuntimeException("Error reading content of package-lock.yaml file");
+        }
+    }
+
+    /**
+     * Returns a YAML parser with custom options.
+     *
+     * @return a YAML parser
+     */
+    private static Yaml getYamlParser() {
+        Representer representer = new Representer(new DumperOptions());
+        representer.getPropertyUtils().setSkipMissingProperties(true);
+        LoaderOptions loaderOptions = new LoaderOptions();
+        SafeConstructor constructor = new SafeConstructor(loaderOptions);
+        return new Yaml(constructor, representer);
+    }
+}

core/src/test/java/org/eclipse/dash/licenses/tests/PnpmPackageLockFileReaderTests.java

@@ -0,0 +1,150 @@
+/*************************************************************************
+ * Copyright (c) 2020,2021 The Eclipse Foundation and others.
+ *
+ * This program and the accompanying materials are made available under
+ * the terms of the Eclipse Public License 2.0 which accompanies this
+ * distribution, and is available at https://www.eclipse.org/legal/epl-2.0
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ *************************************************************************/
+package org.eclipse.dash.licenses.tests;
+
+import org.eclipse.dash.licenses.ContentId;
+import org.eclipse.dash.licenses.IContentId;
+import org.eclipse.dash.licenses.cli.PnpmPackageLockFileReader;
+import org.junit.jupiter.api.Test;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.Charset;
+import java.util.Arrays;
+
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+class PnpmPackageLockFileReaderTests {
+
+	@Test
+	void testNoPackages() throws IOException {
+		try (InputStream input = this.getClass().getResourceAsStream("fixtures/pnpm/pnpm-lock-no-packages.yaml")) {
+			PnpmPackageLockFileReader reader = new PnpmPackageLockFileReader(input);
+			var ids = reader.getContentIds();
+			assertTrue(ids.isEmpty());
+		}
+	}
+
+	@Test
+	void testDuplicates() throws IOException {
+		try (InputStream input = this.getClass().getResourceAsStream("fixtures/pnpm/pnpm-lock-duplicate.yaml")) {
+			PnpmPackageLockFileReader reader = new PnpmPackageLockFileReader(input);
+			var ids = reader.getContentIds();
+			assertEquals(1, ids.size());
+			assertEquals("npm/npmjs/@babel/preset-modules/0.1.6-no-external-plugins", ids.iterator().next().toString());
+		}
+	}
+
+	@Test
+	void testV5Format() throws IOException {
+		try (InputStream input = this.getClass().getResourceAsStream("fixtures/pnpm/pnpm-lock-v5.yaml")) {
+			PnpmPackageLockFileReader reader = new PnpmPackageLockFileReader(input);
+			var ids = reader.getContentIds();
+
+			assertEquals(12, ids.size());
+
+			// Test that a handful of content ids are detected as expected.
+			var includes = new IContentId[] { ContentId.getContentId("npm", "npmjs", "-", "graceful-fs", "4.2.2"),
+					ContentId.getContentId("npm", "npmjs", "-", "pify", "3.0.0"),
+					ContentId.getContentId("npm", "npmjs", "-", "write-json-file", "2.3.0") };
+
+			for (IContentId id : includes) {
+				assertTrue(ids.contains(id), "Should include: " + id);
+			}
+		}
+	}
+
+	@Test
+	void testV6Format() throws IOException {
+		try (InputStream input = this.getClass().getResourceAsStream("fixtures/pnpm/pnpm-lock-v6.yaml")) {
+			PnpmPackageLockFileReader reader = new PnpmPackageLockFileReader(input);
+			var ids = reader.getContentIds();
+
+			assertEquals(579, ids.size());
+
+			// Test that a handful of content ids are detected as expected.
+			var includes = new IContentId[] { ContentId.getContentId("npm", "npmjs", "@babel", "code-frame", "7.18.6"),
+					ContentId.getContentId("npm", "npmjs", "-", "git-semver-tags", "4.1.1"),
+					ContentId.getContentId("npm", "npmjs", "-", "yargs", "17.6.2") };
+
+            for (IContentId id : includes) {
+                assertTrue(ids.contains(id), "Should include: " + id);
+            }
+		}
+	}
+
+	@Test
+	void testV9Format() throws IOException {
+		try (InputStream input = this.getClass().getResourceAsStream("fixtures/pnpm/pnpm-lock-v9.yaml")) {
+			PnpmPackageLockFileReader reader = new PnpmPackageLockFileReader(input);
+			var ids = reader.getContentIds();
+
+			assertEquals(12, ids.size());
+
+			// Test that a handful of content ids are detected as expected.
+			var includes = new IContentId[] { ContentId.getContentId("npm", "npmjs", "-", "graceful-fs", "4.2.2"),
+					ContentId.getContentId("npm", "npmjs", "-", "pify", "3.0.0"),
+					ContentId.getContentId("npm", "npmjs", "-", "write-json-file", "2.3.0") };
+
+			for (IContentId id : includes) {
+				assertTrue(ids.contains(id), "Should include: " + id);
+			}
+		}
+	}
+
+	@Test
+	void testFormat() throws IOException {
+		//try (InputStream input = this.getClass().getResourceAsStream("/pnpm-lock.yaml")) {
+		//try (InputStream input = this.getClass().getResourceAsStream("/pnpm-lock2.yaml")) {
+		try (InputStream input = this.getClass().getResourceAsStream("fixtures/pnpm/pnpm-lock-v6.yaml")) {
+			PnpmPackageLockFileReader reader = new PnpmPackageLockFileReader(input);
+			//var ids = reader.contentIds().collect(Collectors.toList());
+			var ids = reader.getContentIds();
+            assertFalse(ids.isEmpty(), "Should have some content ids");
+
+		}
+	}
+
+	@Test
+	void testAllRecordsDetected() throws IOException {
+		try (InputStream input = this.getClass().getResourceAsStream("fixtures/pnpm/pnpm-lock-v6-small.yaml")) {
+			PnpmPackageLockFileReader reader = new PnpmPackageLockFileReader(input);
+
+			String[] expected = { "npm/npmjs/-/git-semver-tags/4.1.1", "npm/npmjs/@babel/code-frame/7.18.6", "npm/npmjs/@babel/preset-modules/0.1.6-no-external-plugins"};
+			Arrays.sort(expected);
+			String[] found = reader.contentIds().map(IContentId::toString).sorted().toArray(String[]::new);
+			assertArrayEquals(expected, found);
+		}
+	}
+
+	@Test
+	void shouldReturnErrorForInvalidYamlfile() {
+		InputStream input = new ByteArrayInputStream("invalid".getBytes(Charset.defaultCharset()));
+		PnpmPackageLockFileReader reader = new PnpmPackageLockFileReader(input);
+
+		Exception exception = assertThrows(RuntimeException.class, reader::getContentIds);
+		assertEquals("Error reading content of package-lock.yaml file", exception.getMessage());
+	}
+
+	@Test
+	void shouldReturnErrorForEmptyfile() {
+		InputStream input = new ByteArrayInputStream("".getBytes(Charset.defaultCharset()));
+		PnpmPackageLockFileReader reader = new PnpmPackageLockFileReader(input);
+
+		Exception exception = assertThrows(RuntimeException.class, reader::getContentIds);
+		assertEquals("Error reading content of package-lock.yaml file", exception.getMessage());
+	}
+
+}

core/src/test/java/org/eclipse/dash/licenses/tests/fixtures/pnpm/pnpm-lock-duplicate.yaml

@@ -0,0 +1,41 @@
+lockfileVersion: 6.0
+
+specifiers:
+  braces: 1.8.5
+  dns-sync: ^0.1.3
+  fake_submodule: file:fake_submodule
+  lighter-run: ^1.2.1
+  pause-stream: 0.0.11
+  react-dom: npm:@hot-loader/react-dom
+
+dependencies:
+  braces: 1.8.5
+  dns-sync: 0.1.3
+  fake_submodule: link:fake_submodule
+  lighter-run: 1.2.1
+  pause-stream: 0.0.11
+  react-dom: /@hot-loader/react-dom/17.0.1
+
+packages:
+
+  /@babel/preset-modules@0.1.6-no-external-plugins(@babel/core@7.23.2):
+    resolution: {integrity: sha512-HrcgcIESLm9aIR842yhJ5RWan/gebQUJ6E/E5+rf0y9o6oj7w0Br+sWuL6kEQ/o/AdfvR1Je9jG18/gnpwjEyA==}
+    peerDependencies:
+      '@babel/core': ^7.0.0-0 || ^8.0.0-0 <8.0.0
+    dependencies:
+      '@babel/core': 7.23.2
+      '@babel/helper-plugin-utils': 7.22.5
+      '@babel/types': 7.23.5
+      esutils: 2.0.3
+    dev: true
+
+  /@babel/preset-modules@0.1.6-no-external-plugins(@babel/core@7.23.5):
+    resolution: {integrity: sha512-HrcgcIESLm9aIR842yhJ5RWan/gebQUJ6E/E5+rf0y9o6oj7w0Br+sWuL6kEQ/o/AdfvR1Je9jG18/gnpwjEyA==}
+    peerDependencies:
+      '@babel/core': ^7.0.0-0 || ^8.0.0-0 <8.0.0
+    dependencies:
+      '@babel/core': 7.23.5
+      '@babel/helper-plugin-utils': 7.22.5
+      '@babel/types': 7.23.5
+      esutils: 2.0.3
+    dev: true

core/src/test/java/org/eclipse/dash/licenses/tests/fixtures/pnpm/pnpm-lock-no-packages.yaml

@@ -0,0 +1,17 @@
+lockfileVersion: 5.3
+
+specifiers:
+  braces: 1.8.5
+  dns-sync: ^0.1.3
+  fake_submodule: file:fake_submodule
+  lighter-run: ^1.2.1
+  pause-stream: 0.0.11
+  react-dom: npm:@hot-loader/react-dom
+
+dependencies:
+  braces: 1.8.5
+  dns-sync: 0.1.3
+  fake_submodule: link:fake_submodule
+  lighter-run: 1.2.1
+  pause-stream: 0.0.11
+  react-dom: /@hot-loader/react-dom/17.0.1