BI-529 - Add scopes to dependencies tree nodes (#371)

yahavi · web-flow · commit 4bf8444c710b · 2020-08-27T08:29:21.000+03:00
diff --git a/build-info-extractor-npm/src/main/java/org/jfrog/build/extractor/npm/NpmDriver.java b/build-info-extractor-npm/src/main/java/org/jfrog/build/extractor/npm/NpmDriver.java
@@ -3,6 +3,7 @@
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.ObjectReader;
+import com.fasterxml.jackson.databind.node.ObjectNode;
 import org.apache.commons.lang.StringUtils;
 import org.jfrog.build.api.util.Log;
 import org.jfrog.build.extractor.executor.CommandExecutor;
@@ -58,11 +59,16 @@ public JsonNode list(File workingDirectory, List<String> extraArgs) throws IOExc
         List<String> args = new ArrayList<>();
         args.add("ls");
         args.add("--json");
+        args.add("--long");
         args.addAll(extraArgs);
         try {
             CommandResults npmCommandRes = commandExecutor.exeCommand(workingDirectory, args, null);
             String res = StringUtils.isBlank(npmCommandRes.getRes()) ? "{}" : npmCommandRes.getRes();
-            return jsonReader.readTree(res);
+            JsonNode npmLsResults = jsonReader.readTree(res);
+            if (!npmCommandRes.isOk() && !npmLsResults.has("problems")) {
+                ((ObjectNode) npmLsResults).put("problems", npmCommandRes.getErr());
+            }
+            return npmLsResults;
         } catch (IOException | InterruptedException e) {
             throw new IOException("npm ls failed", e);
         }
diff --git a/build-info-extractor-npm/src/main/java/org/jfrog/build/extractor/npm/extractor/NpmBuildInfoExtractor.java b/build-info-extractor-npm/src/main/java/org/jfrog/build/extractor/npm/extractor/NpmBuildInfoExtractor.java
@@ -69,7 +69,7 @@ public Build extract(NpmProject npmProject) throws Exception {
 
         List<Dependency> dependencies = collectDependencies(workingDir);
         String moduleId = StringUtils.isNotBlank(module) ? module : npmPackageInfo.toString();
-        return createBuild(dependencies,moduleId);
+        return createBuild(dependencies, moduleId);
     }
 
     private void preparePrerequisites(String resolutionRepository, Path workingDir) throws IOException {
@@ -167,13 +167,13 @@ private void createTempNpmrc(Path workingDir, List<String> installationArgs) thr
 
     /**
      * Boolean argument can be provided in one of the following ways:
-     *  1. --arg - which infers true
-     *  2. --arg=value (true/false)
-     *  3. --arg value (true/false)
+     * 1. --arg - which infers true
+     * 2. --arg=value (true/false)
+     * 3. --arg value (true/false)
      */
-    static boolean isJsonOutputRequired(List <String> installationArgs) {
+    static boolean isJsonOutputRequired(List<String> installationArgs) {
         int jsonIndex = installationArgs.indexOf("--json");
-        if (jsonIndex > -1 ) {
+        if (jsonIndex > -1) {
             return jsonIndex == installationArgs.size() - 1 || !installationArgs.get(jsonIndex + 1).equals("false");
         }
         return installationArgs.contains("--json=true");
@@ -220,12 +220,15 @@ private void restoreNpmrc(Path workingDir) throws IOException {
     private List<Dependency> collectDependencies(Path workingDir) throws Exception {
         Map<String, Dependency> dependencies = new ConcurrentHashMap<>();
         List<NpmScope> scopes = getNpmScopes();
-        for (NpmScope scope : scopes) {
-            List<String> extraListArgs = new ArrayList<>();
-            extraListArgs.add("--only=" + scope);
-            JsonNode jsonNode = npmDriver.list(workingDir.toFile(), extraListArgs);
-            populateDependenciesMap(dependencies, scope, jsonNode);
+        if (scopes.isEmpty()) {
+            return new ArrayList<>();
         }
+        List<String> extraListArgs = new ArrayList<>();
+        if (scopes.size() == 1) {
+            extraListArgs.add("--only=" + scopes.get(0));
+        }
+        JsonNode jsonNode = npmDriver.list(workingDir.toFile(), extraListArgs);
+        populateDependenciesMap(dependencies, jsonNode);
 
         return new ArrayList<>(dependencies.values());
     }
@@ -268,10 +271,10 @@ private Build createBuild(List<Dependency> dependencies, String moduleId) {
      * 1. Create npm dependencies tree from root node of 'npm ls' command tree. Populate each node with name, version and scope.
      * 2. For each dependency, retrieve sha1 and md5 from Artifactory. Use the producer-consumer mechanism to parallelize it.
      */
-    private void populateDependenciesMap(Map<String, Dependency> dependencies, NpmScope scope, JsonNode npmDependenciesTree) throws Exception {
+    private void populateDependenciesMap(Map<String, Dependency> dependencies, JsonNode npmDependenciesTree) throws Exception {
         // Set of packages that could not be found in Artifactory
         Set<NpmPackageInfo> badPackages = Collections.synchronizedSet(new HashSet<>());
-        DefaultMutableTreeNode rootNode = NpmDependencyTree.createDependenciesTree(scope, npmDependenciesTree);
+        DefaultMutableTreeNode rootNode = NpmDependencyTree.createDependenciesTree(npmDependenciesTree);
         try (ArtifactoryDependenciesClient client = dependenciesClientBuilder.build()) {
             // Create producer Runnable
             ProducerRunnableBase[] producerRunnable = new ProducerRunnableBase[]{new NpmExtractorProducer(rootNode)};
diff --git a/build-info-extractor-npm/src/main/java/org/jfrog/build/extractor/npm/extractor/NpmDependencyTree.java b/build-info-extractor-npm/src/main/java/org/jfrog/build/extractor/npm/extractor/NpmDependencyTree.java
@@ -1,12 +1,15 @@
 package org.jfrog.build.extractor.npm.extractor;
 
 import com.fasterxml.jackson.databind.JsonNode;
-import org.apache.commons.lang3.ObjectUtils;
+import org.apache.commons.lang.StringUtils;
 import org.jfrog.build.extractor.npm.types.NpmPackageInfo;
 import org.jfrog.build.extractor.npm.types.NpmScope;
 import org.jfrog.build.extractor.scan.DependenciesTree;
+import org.jfrog.build.extractor.scan.Scope;
 
+import java.util.HashSet;
 import java.util.Map;
+import java.util.Set;
 
 /**
  * @author Yahav Itzhak
@@ -17,18 +20,21 @@ public class NpmDependencyTree {
     /**
      * Create a npm dependencies tree from the results of 'npm ls' command.
      *
-     * @param scope   - 'production' or 'development'.
      * @param npmList - Results of 'npm ls' command.
      * @return Tree of npm PackageInfos.
      * @see NpmPackageInfo
      */
-    public static DependenciesTree createDependenciesTree(NpmScope scope, JsonNode npmList) {
+    public static DependenciesTree createDependenciesTree(JsonNode npmList) {
         DependenciesTree rootNode = new DependenciesTree();
-        populateDependenciesTree(rootNode, scope, npmList.get("dependencies"));
+        populateDependenciesTree(rootNode, npmList.get("dependencies"));
+        for (DependenciesTree child : rootNode.getChildren()) {
+            NpmPackageInfo packageInfo = (NpmPackageInfo) child.getUserObject();
+            child.setScopes(getScopes(packageInfo.getName(), packageInfo.getScope()));
+        }
         return rootNode;
     }
 
-    private static void populateDependenciesTree(DependenciesTree scanTreeNode, NpmScope scope, JsonNode dependencies) {
+    private static void populateDependenciesTree(DependenciesTree scanTreeNode, JsonNode dependencies) {
         if (dependencies == null) {
             return;
         }
@@ -37,16 +43,47 @@ private static void populateDependenciesTree(DependenciesTree scanTreeNode, NpmS
             String name = stringJsonNodeEntry.getKey();
             JsonNode versionNode = stringJsonNodeEntry.getValue().get("version");
             if (versionNode != null) {
-                addSubtree(stringJsonNodeEntry, scanTreeNode, name, versionNode.asText(), scope); // Mutual recursive call
+                addSubtree(stringJsonNodeEntry, scanTreeNode, name, versionNode.asText()); // Mutual recursive call
             }
         });
     }
 
-    private static void addSubtree(Map.Entry<String, JsonNode> stringJsonNodeEntry, DependenciesTree node, String name, String version, NpmScope scope) {
-        NpmPackageInfo npmPackageInfo = new NpmPackageInfo(name, version, ObjectUtils.defaultIfNull(scope, "").toString());
-        JsonNode childDependencies = stringJsonNodeEntry.getValue().get("dependencies");
+    private static void addSubtree(Map.Entry<String, JsonNode> stringJsonNodeEntry, DependenciesTree node, String name, String version) {
+        JsonNode jsonNode = stringJsonNodeEntry.getValue();
+        String devScope = (isDev(jsonNode) ? NpmScope.DEVELOPMENT : NpmScope.PRODUCTION).toString();
+        NpmPackageInfo npmPackageInfo = new NpmPackageInfo(name, version, devScope);
+        JsonNode childDependencies = jsonNode.get("dependencies");
         DependenciesTree childTreeNode = new DependenciesTree(npmPackageInfo);
-        populateDependenciesTree(childTreeNode, scope, childDependencies); // Mutual recursive call
+        populateDependenciesTree(childTreeNode, childDependencies); // Mutual recursive call
         node.add(childTreeNode);
     }
+
+    /**
+     * Return true if the input dependency is a dev dependency.
+     *
+     * @param jsonNode - The dependency node in the 'npm ls' results
+     * @return true if the input dependency is a dev dependency
+     */
+    private static boolean isDev(JsonNode jsonNode) {
+        JsonNode development = jsonNode.get("_development");
+        return development != null && development.asBoolean(false);
+    }
+
+    /**
+     * Return a set of the relevant scopes. The set contains 'development' or 'production'. If the dependency has a
+     * custom scope, add it too.
+     *
+     * @param name     - The name of the dependency
+     * @param devScope - 'development' or 'production'
+     * @return set of the relevant scopes
+     */
+    private static Set<Scope> getScopes(String name, String devScope) {
+        Set<Scope> scopes = new HashSet<>();
+        scopes.add(new Scope(devScope));
+        String customScope = StringUtils.substringBetween(name, "@", "/");
+        if (customScope != null) {
+            scopes.add(new Scope(customScope));
+        }
+        return scopes;
+    }
 }
diff --git a/build-info-extractor/src/main/java/org/jfrog/build/extractor/scan/DependenciesTree.java b/build-info-extractor/src/main/java/org/jfrog/build/extractor/scan/DependenciesTree.java
@@ -15,6 +15,7 @@ public class DependenciesTree extends DefaultMutableTreeNode {
 
     private Set<Issue> issues = new HashSet<>();
     private Set<License> licenses = new HashSet<>();
+    private Set<Scope> scopes = new HashSet<>();
     private GeneralInfo generalInfo;
     private Issue topIssue = new Issue();
 
@@ -34,6 +35,10 @@ public void setLicenses(Set<License> licenses) {
         this.licenses = licenses;
     }
 
+    public void setScopes(Set<Scope> scopes) {
+        this.scopes = scopes;
+    }
+
     @SuppressWarnings("unused")
     public void setGeneralInfo(GeneralInfo generalInfo) {
         this.generalInfo = generalInfo;
@@ -52,6 +57,10 @@ public Set<License> getLicenses() {
         return licenses;
     }
 
+    public Set<Scope> getScopes() {
+        return scopes;
+    }
+
     /**
      * @return top severity issue of the current node and its ancestors
      */
diff --git a/build-info-extractor/src/main/java/org/jfrog/build/extractor/scan/Scope.java b/build-info-extractor/src/main/java/org/jfrog/build/extractor/scan/Scope.java
@@ -0,0 +1,56 @@
+package org.jfrog.build.extractor.scan;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import org.apache.commons.lang.StringUtils;
+
+import java.util.Objects;
+
+/**
+ * @author yahavi
+ */
+public class Scope {
+    private static final String NONE_SCOPE = "None";
+
+    private final String name;
+
+    @SuppressWarnings("unused")
+    public Scope() {
+        name = NONE_SCOPE;
+    }
+
+    public Scope(String name) {
+        this.name = StringUtils.capitalize(StringUtils.trim(name));
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    @JsonIgnore
+    @SuppressWarnings("unused")
+    public boolean isEmpty() {
+        return StringUtils.isBlank(name) || name.equals(Scope.NONE_SCOPE);
+    }
+
+    @Override
+    public boolean equals(Object other) {
+        if (this == other) {
+            return true;
+        }
+        if (other == null || getClass() != other.getClass()) {
+            return false;
+        }
+        return StringUtils.equals(toString(), other.toString());
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(name);
+    }
+
+    @Override
+    public String toString() {
+        return this.name;
+    }
+
+}
