Add hierarchical dependencies for npm (#446)

* Bugfix - build-info-extractor-npm generates wrong dependency scopes

Author: Or-Geva

Diff for: build-info-extractor-npm/src/main/java/org/jfrog/build/extractor/npm/extractor/NpmBuildInfoExtractor.java

@@ -238,12 +238,12 @@ private List<Dependency> collectDependencies(Path workingDir) throws Exception {
         if (scopes.isEmpty()) {
             return new ArrayList<>();
         }
-        List<String> extraListArgs = new ArrayList<>();
-        if (scopes.size() == 1) {
-            extraListArgs.add("--only=" + scopes.get(0));
+        for (NpmScope scope : scopes) {
+            List<String> extraListArgs = new ArrayList<>();
+            extraListArgs.add("--only=" + scope);
+            JsonNode jsonNode = npmDriver.list(workingDir.toFile(), extraListArgs);
+            populateDependenciesMap(dependencies, getDependenciesMapFromLatestBuild(), jsonNode, scope);
         }
-        JsonNode jsonNode = npmDriver.list(workingDir.toFile(), extraListArgs);
-        populateDependenciesMap(dependencies, getDependenciesMapFromLatestBuild(), jsonNode);
 
         return new ArrayList<>(dependencies.values());
     }
@@ -286,10 +286,10 @@ private Build createBuild(List<Dependency> dependencies, String moduleId) {
      * 1. Create npm dependencies tree from root node of 'npm ls' command tree. Populate each node with name, version and scope.
      * 2. For each dependency, retrieve sha1 and md5 from Artifactory. Use the producer-consumer mechanism to parallelize it.
      */
-    private void populateDependenciesMap(Map<String, Dependency> dependencies, Map<String, Dependency> previousBuildDependencies, JsonNode npmDependenciesTree) throws Exception {
+    private void populateDependenciesMap(Map<String, Dependency> dependencies, Map<String, Dependency> previousBuildDependencies, JsonNode npmDependenciesTree, NpmScope scope) throws Exception {
         // Set of packages that could not be found in Artifactory.
         Set<NpmPackageInfo> badPackages = Collections.synchronizedSet(new HashSet<>());
-        DefaultMutableTreeNode rootNode = NpmDependencyTree.createDependenciesTree(npmDependenciesTree);
+        DefaultMutableTreeNode rootNode = NpmDependencyTree.createDependenciesTree(npmDependenciesTree, scope);
         try (ArtifactoryDependenciesClient dependenciesClient = dependenciesClientBuilder.build()) {
             // Create producer Runnable.
             ProducerRunnableBase[] producerRunnable = new ProducerRunnableBase[]{new NpmExtractorProducer(rootNode)};

Diff for: build-info-extractor-npm/src/main/java/org/jfrog/build/extractor/npm/extractor/NpmDependencyTree.java

@@ -2,6 +2,7 @@
 
 import com.fasterxml.jackson.databind.JsonNode;
 import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.ArrayUtils;
 import org.jfrog.build.extractor.npm.types.NpmPackageInfo;
 import org.jfrog.build.extractor.npm.types.NpmScope;
 import org.jfrog.build.extractor.scan.DependenciesTree;
@@ -24,51 +25,47 @@ public class NpmDependencyTree {
      * @return Tree of npm PackageInfos.
      * @see NpmPackageInfo
      */
-    public static DependenciesTree createDependenciesTree(JsonNode npmList) {
+    public static DependenciesTree createDependenciesTree(JsonNode npmList, NpmScope scope) {
         DependenciesTree rootNode = new DependenciesTree();
-        populateDependenciesTree(rootNode, npmList.get("dependencies"));
+        populateDependenciesTree(rootNode, npmList.get("dependencies"), new String[]{npmList.get("name").asText() + ":" + npmList.get("version").asText()}, scope);
         for (DependenciesTree child : rootNode.getChildren()) {
             NpmPackageInfo packageInfo = (NpmPackageInfo) child.getUserObject();
             child.setScopes(getScopes(packageInfo.getName(), packageInfo.getScope()));
         }
         return rootNode;
     }
 
-    private static void populateDependenciesTree(DependenciesTree scanTreeNode, JsonNode dependencies) {
-        if (dependencies == null) {
+    /**
+     * Parses npm dependencies recursively and adds the collected dependencies to scanTreeNode.
+     *
+     * @param scanTreeNode - Output - The DependenciesTree to populate.
+     * @param dependencies - The dependencies json object generated by npm ls.
+     * @param pathToRoot   - A path-to-root dependency list. The structure of each dependency in the list is 'dependency-name:dependency-version'.
+     */
+    private static void populateDependenciesTree(DependenciesTree scanTreeNode, JsonNode dependencies, String[] pathToRoot, NpmScope scope) {
+        if (dependencies == null || pathToRoot == null) {
             return;
         }
 
         dependencies.fields().forEachRemaining(stringJsonNodeEntry -> {
             String name = stringJsonNodeEntry.getKey();
             JsonNode versionNode = stringJsonNodeEntry.getValue().get("version");
             if (versionNode != null) {
-                addSubtree(stringJsonNodeEntry, scanTreeNode, name, versionNode.asText()); // Mutual recursive call
+                addSubtree(stringJsonNodeEntry, scanTreeNode, name, versionNode.asText(), pathToRoot, scope); // Mutual recursive call
             }
         });
     }
 
-    private static void addSubtree(Map.Entry<String, JsonNode> stringJsonNodeEntry, DependenciesTree node, String name, String version) {
+    private static void addSubtree(Map.Entry<String, JsonNode> stringJsonNodeEntry, DependenciesTree node, String name, String version, String[] pathToRoot, NpmScope scope) {
         JsonNode jsonNode = stringJsonNodeEntry.getValue();
-        String devScope = (isDev(jsonNode) ? NpmScope.DEVELOPMENT : NpmScope.PRODUCTION).toString();
-        NpmPackageInfo npmPackageInfo = new NpmPackageInfo(name, version, devScope);
+        String devScope = scope.toString();
+        NpmPackageInfo npmPackageInfo = new NpmPackageInfo(name, version, devScope, pathToRoot);
         JsonNode childDependencies = jsonNode.get("dependencies");
         DependenciesTree childTreeNode = new DependenciesTree(npmPackageInfo);
-        populateDependenciesTree(childTreeNode, childDependencies); // Mutual recursive call
+        populateDependenciesTree(childTreeNode, childDependencies, ArrayUtils.insert(0, pathToRoot, npmPackageInfo.toString()), scope); // Mutual recursive call
         node.add(childTreeNode);
     }
 
-    /**
-     * Return true if the input dependency is a dev dependency.
-     *
-     * @param jsonNode - The dependency node in the 'npm ls' results
-     * @return true if the input dependency is a dev dependency
-     */
-    private static boolean isDev(JsonNode jsonNode) {
-        JsonNode development = jsonNode.get("_development");
-        return development != null && development.asBoolean(false);
-    }
-
     /**
      * Return a set of the relevant scopes. The set contains 'development' or 'production'. If the dependency has a
      * custom scope, add it too.

Diff for: build-info-extractor-npm/src/main/java/org/jfrog/build/extractor/npm/extractor/NpmExtractorConsumer.java

@@ -82,6 +82,7 @@ private boolean appendDependency(NpmPackageInfo npmPackageInfo) {
         } else {
             dependency.getScopes().add(npmPackageInfo.getScope());
         }
+        dependency.addRequiredBy(npmPackageInfo.getPathToRoot());
         return true;
     }
 

Diff for: build-info-extractor-npm/src/main/java/org/jfrog/build/extractor/npm/types/NpmPackageInfo.java

@@ -16,15 +16,22 @@ public class NpmPackageInfo implements Serializable, ProducerConsumerItem {
     private String name;
     private String version;
     private String scope;
+    /**
+     * A path-to-root dependency list that directly depends on this dependency.
+     * The structure of each dependency in the list is 'dependency-name:dependency-version'
+     * Used for 'RequiredBy' in {@link org.jfrog.build.api.Dependency}.
+     */
+    private String[] pathToRoot;
 
     @SuppressWarnings("unused")
     public NpmPackageInfo() {
     }
 
-    public NpmPackageInfo(String name, String version, String scope) {
+    public NpmPackageInfo(String name, String version, String scope, String[] pathToRoot) {
         this.name = name;
         this.version = version;
         this.scope = scope;
+        this.pathToRoot = pathToRoot;
     }
 
     public String getName() {
@@ -75,6 +82,7 @@ public void readPackageInfo(InputStream inputStream) throws IOException {
 
         setName(npmPackageInfo.getName());
         splitScopeFromName();
+        setPathToRoot(npmPackageInfo.getPathToRoot());
     }
 
     public String getModuleId() {
@@ -119,4 +127,12 @@ public int hashCode() {
     public String toString() {
         return name + ":" + version;
     }
+
+    public String[] getPathToRoot() {
+        return pathToRoot;
+    }
+
+    public void setPathToRoot(String[] pathToRoot) {
+        this.pathToRoot = pathToRoot;
+    }
 }