Skip to content

Commit

Permalink
Make the PackageLockFileReader more resilient to workspaces.
Browse files Browse the repository at this point in the history
  • Loading branch information
@waynebeaton
waynebeaton committed Apr 22, 2024
1 parent 78142e4 commit 0ab33f5
Show file tree
Hide file tree
Showing 3 changed files with 10,255 additions and 45 deletions.
116 changes: 98 additions & 18 deletions core/src/main/java/org/eclipse/dash/licenses/cli/PackageLockFileReader.java
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*************************************************************************
* Copyright (c) 2020,2021 The Eclipse Foundation and others.
* Copyright (c) 2020 The Eclipse Foundation and others.
*
* This program and the accompanying materials are made available under
* the terms of the Eclipse Public License 2.0 which accompanies this
Expand All @@ -11,6 +11,7 @@

import java.io.InputStream;
import java.util.Collection;
import java.util.Collections;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
Expand All @@ -24,12 +25,15 @@
import org.slf4j.LoggerFactory;

import jakarta.json.JsonObject;
import jakarta.json.JsonString;
import jakarta.json.JsonValue;

public class PackageLockFileReader implements IDependencyListReader {
final Logger logger = LoggerFactory.getLogger(PackageLockFileReader.class);
private static final Pattern Name_Pattern = Pattern.compile("(?:(?<scope>@[^\\/]+)\\/)?(?<name>[^\\/]+)$");

private final InputStream input;
private JsonObject json;

public PackageLockFileReader(InputStream input) {
this.input = input;
Expand All @@ -49,6 +53,16 @@ class Package {
this.value = value;
}

Collection<Package> getPackages() {
/*
* More research is needed. AFAICT, it's possible to specify
* a relative directory as the key and, my observation is that,
* the directory may itself have a package-lock.json file which
* would describe additional packages.
*/
return Collections.singleton(this);
}

/**
* The content id needs to be extracted from a combination of the key and values
* from the associated associative array.
Expand Down Expand Up @@ -89,8 +103,8 @@ String getSource() {
if (isResolvedLocally())
return "local";

var resolved = value.asJsonObject().getString("resolved", "");
if (resolved.contains("registry.npmjs.org")) {
var resolved = getResolved();
if (resolved != null && resolved.contains("registry.npmjs.org")) {
return "npmjs";
} else {
logger.debug("Unknown resolved source: {}", resolved);
Expand Down Expand Up @@ -120,22 +134,36 @@ String getName() {
return null;
}

public boolean isResolvedLocally() {
var resolved = value.asJsonObject().getString("resolved", null);
boolean isResolvedLocally() {
var resolved = getResolved();
if (resolved == null)
return true;
if (resolved.startsWith("file:"))
return true;
return false;

if (key.startsWith("packages/"))
if (resolved.startsWith("file:"))
return true;

if (value.asJsonObject().getBoolean("link", false))
return true;
if (value.asJsonObject().getString("version", "").startsWith("file:"))
return true;

return false;
}

private String getResolved() {
return value.asJsonObject().getString("resolved", null);
}

private boolean isLink() {
return value.asJsonObject().getBoolean("link", false);
}

public boolean isProjectContent() {
return isLink() && isInWorkspace(getResolved());
}

@Override
public String toString() {
return key + " : " + getResolved();
}
}

/**
Expand All @@ -154,9 +182,7 @@ Stream<Dependency> getDependencies() {
if (dependencies == null)
return Stream.empty();

return dependencies
.entrySet()
.stream()
return dependencies.entrySet().stream()
.map(each -> new Dependency(each.getKey(), each.getValue().asJsonObject()));
}

Expand All @@ -174,25 +200,79 @@ public Collection<IContentId> getContentIds() {
}

public Stream<IContentId> contentIds() {
JsonObject json = JsonUtils.readJson(input);
json = JsonUtils.readJson(input);

switch (json.getJsonNumber("lockfileVersion").intValue()) {
case 1:
return new Dependency("", json)
.stream()
.filter(each -> !each.key.isEmpty())
return new Dependency("", json).stream().filter(each -> !each.key.isEmpty())
.map(dependency -> dependency.getContentId());
case 2:
case 3:
// @formatter:off
return json.getJsonObject("packages").entrySet().stream()
.filter(entry -> !entry.getKey().isEmpty())
.map(entry -> new Package(entry.getKey(), entry.getValue().asJsonObject()))
.flatMap(item -> item.getPackages().stream())
.filter(item -> !item.isProjectContent())
.map(dependency -> dependency.getContentId());
// @formatter:on
}

return Stream.empty();
}

/**
* Answer whether or not a resolved link points to a workspace. This is true
* when the link is <code>true</code> and the resolved path matches a workspace
* specified in the header.
*
* <pre>
* ...
* "node_modules/vscode-js-profile-core": {
* "resolved": "packages/vscode-js-profile-core",
* "link": true
* },
* ...
* </pre>
*
* @param value
* @return
*/
boolean isInWorkspace(String value) {
if (value == null) return false;

return getWorkspaces().anyMatch(each -> glob(each, value));
}

private Stream<String> getWorkspaces() {
return getRootPackage()
.getOrDefault("workspaces", JsonValue.EMPTY_JSON_ARRAY).asJsonArray()
.getValuesAs(JsonString.class).stream().map(JsonString::getString);
}

/**
* The root package is the one that has an empty string as the key. It's not at
* all clear to me whether package-lock.json file have more than one package;
* more research is required.
*/
private JsonObject getRootPackage() {
return getPackages().entrySet().stream()
.filter(entry -> entry.getKey().isEmpty())
.map(entry -> entry.getValue().asJsonObject())
.findFirst().orElse(JsonValue.EMPTY_JSON_OBJECT);
}

private JsonObject getPackages() {
return json.getOrDefault("packages", JsonValue.EMPTY_JSON_OBJECT).asJsonObject();
}

/**
* Do a glob match. The build-in function is a little too tightly coupled with
* the file system for my liking. This implements a simple translation from glob
* to regex that should hopefully suit most of our requirements.
*/
boolean glob(String pattern, String value) {
var regex = pattern.replace("/", "\\/").replace(".", "/.").replace("*", ".*");
return Pattern.matches(regex, value);
}
}
81 changes: 54 additions & 27 deletions core/src/test/java/org/eclipse/dash/licenses/tests/PackageLockFileReaderTests.java
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*************************************************************************
* Copyright (c) 2020,2021 The Eclipse Foundation and others.
* Copyright (c) 2020 The Eclipse Foundation and others.
*
* This program and the accompanying materials are made available under
* the terms of the Eclipse Public License 2.0 which accompanies this
Expand All @@ -12,11 +12,11 @@
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.jupiter.api.Assertions.assertEquals;

import java.io.IOException;
import java.io.InputStream;
import java.util.Arrays;
import java.util.Collection;
import java.util.stream.Collectors;

import org.eclipse.dash.licenses.ContentId;
Expand All @@ -33,31 +33,58 @@ class PackageLockFileReaderTests {
void testV1Format() throws IOException {
try (InputStream input = this.getClass().getResourceAsStream(PACKAGE_LOCK_JSON)) {
PackageLockFileReader reader = new PackageLockFileReader(input);
String[] expected = { "npm/npmjs/-/loglevel/1.6.1", "npm/npmjs/-/sax/1.2.4", "npm/npmjs/-/saxes/3.1.9",
"npm/npmjs/-/slimdom-sax-parser/1.1.3", "npm/npmjs/-/slimdom/2.2.1", "npm/npmjs/-/xml-js/1.6.11",
"npm/npmjs/-/xmlchars/1.3.1", "npm/npmjs/@namespace/fontoxpath/3.3.0" };
String[] found = reader.getContentIds().stream().map(IContentId::toString).sorted().toArray(String[]::new);
assertArrayEquals(expected, found);
Collection<IContentId> ids = reader.getContentIds();

IContentId[] includes = {
ContentId.getContentId("npm/npmjs/-/loglevel/1.6.1"),
ContentId.getContentId("npm/npmjs/-/sax/1.2.4"),
ContentId.getContentId("npm/npmjs/-/saxes/3.1.9"),
ContentId.getContentId("npm/npmjs/-/slimdom-sax-parser/1.1.3"),
ContentId.getContentId("npm/npmjs/-/slimdom/2.2.1"),
ContentId.getContentId("npm/npmjs/-/xml-js/1.6.11"),
ContentId.getContentId("npm/npmjs/-/xmlchars/1.3.1"),
ContentId.getContentId("npm/npmjs/@namespace/fontoxpath/3.3.0")
};

assertTrue(Arrays.stream(includes).allMatch(each -> ids.contains(each)));
}
}

@Test
void testV2Format() throws IOException {
try (InputStream input = this.getClass().getResourceAsStream(PACKAGE_LOCK_V2_JSON)) {
PackageLockFileReader reader = new PackageLockFileReader(input);
Collection<IContentId> ids = reader.getContentIds();

assertTrue(ids.stream().allMatch(each -> each.isValid()));

// This "test" is a little... abridged. At least this test proves
// that we're getting something in the right format from the reader
// without having to enumerate all 574 (I think) records).
String[] expected = { "npm/npmjs/@babel/code-frame/7.12.13", "npm/npmjs/@babel/compat-data/7.13.15",
"npm/npmjs/@babel/core/7.13.15" };
String[] found = reader
.getContentIds()
.stream()
.limit(3)
.map(IContentId::toString)
.sorted()
.toArray(String[]::new);
assertArrayEquals(expected, found);
IContentId[] includes = {
ContentId.getContentId("npm/npmjs/@babel/code-frame/7.12.13"),
ContentId.getContentId("npm/npmjs/@babel/compat-data/7.13.15"),
ContentId.getContentId("npm/npmjs/@babel/core/7.13.15" )
};

assertTrue(Arrays.stream(includes).allMatch(each -> ids.contains(each)));
}
}

@Test
void testV2FormatWithWorkspaces() throws IOException {
try (InputStream input = this.getClass().getResourceAsStream("/test_data_package-lock-v2-2.json")) {
PackageLockFileReader reader = new PackageLockFileReader(input);
var ids = reader.getContentIds();

assertTrue(ids.stream().allMatch(each -> each.isValid()));

IContentId[] includes = {
ContentId.getContentId("npm/npmjs/@esbuild/linux-ia32/0.20.2"),
ContentId.getContentId("npm/npmjs/@rollup/rollup-linux-powerpc64le-gnu/4.14.0")
};

assertTrue(Arrays.stream(includes).allMatch(each -> ids.contains(each)));
}
}

Expand All @@ -67,7 +94,7 @@ void testV3Format() throws IOException {
PackageLockFileReader reader = new PackageLockFileReader(input);
var ids = reader.contentIds().collect(Collectors.toList());

assertEquals(769, ids.size());
assertTrue(ids.stream().allMatch(each -> each.isValid()));

// Issue #285 Component name is remapped. Make sure that we don't see the key
// in the results. This record should manifest as langium-statemachine-dsl (see
Expand All @@ -78,14 +105,12 @@ void testV3Format() throws IOException {
var includes = new IContentId[] { ContentId.getContentId("npm", "npmjs", "-", "ansi-styles", "3.2.1"),
ContentId.getContentId("npm", "npmjs", "@typescript-eslint", "eslint-plugin", "6.4.1"),
ContentId.getContentId("npm", "npmjs", "@types", "minimatch", "3.0.5"),
ContentId.getContentId("npm", "local", "-", "langium-requirements-dsl", "2.1.0"),
ContentId.getContentId("npm", "local", "-", "langium-domainmodel-dsl", "2.1.0"),
ContentId.getContentId("npm", "local", "-", "langium-statemachine-dsl", "2.1.0") };

for (int index = 0; index < includes.length; index++) {
var id = includes[index];
assertTrue("Should include: " + id.toString(), ids.contains(id));
}
ContentId.getContentId("npm", "npmjs", "-", "langium-requirements-dsl", "2.1.0"),
ContentId.getContentId("npm", "npmjs", "-", "langium-domainmodel-dsl", "2.1.0"),
ContentId.getContentId("npm", "npmjs", "-", "langium-statemachine-dsl", "2.1.0")
};

assertTrue(Arrays.stream(includes).allMatch(each -> ids.contains(each)));
}
}

Expand All @@ -94,7 +119,9 @@ void testAllRecordsDetected() throws IOException {
try (InputStream input = this.getClass().getResourceAsStream("/differentResolved.json")) {
PackageLockFileReader reader = new PackageLockFileReader(input);

String[] expected = { "npm/npmjs/@babel/code-frame/7.12.13", "npm/local/-/some_local_package/1.2.3", };
String[] expected = {
"npm/npmjs/@babel/code-frame/7.12.13",
"npm/local/-/some_local_package/1.2.3", };
Arrays.sort(expected);
String[] found = reader.contentIds().map(IContentId::toString).sorted().toArray(String[]::new);
assertArrayEquals(expected, found);
Expand Down

0 comments on commit 0ab33f5

Please sign in to comment.