Dependency tree changes for CI builds scan (#459)

Author: yahavi

build-info-extractor-npm/src/main/java/org/jfrog/build/extractor/npm/extractor/NpmBuildInfoExtractor.java

@@ -283,13 +283,13 @@ private Build createBuild(List<Dependency> dependencies, String moduleId) {
 
     /**
      * Populate the dependencies map for the specified scope by:
-     * 1. Create npm dependencies tree from root node of 'npm ls' command tree. Populate each node with name, version and scope.
+     * 1. Create npm dependency tree from root node of 'npm ls' command tree. Populate each node with name, version and scope.
      * 2. For each dependency, retrieve sha1 and md5 from Artifactory. Use the producer-consumer mechanism to parallelize it.
      */
-    private void populateDependenciesMap(Map<String, Dependency> dependencies, Map<String, Dependency> previousBuildDependencies, JsonNode npmDependenciesTree, NpmScope scope) throws Exception {
+    private void populateDependenciesMap(Map<String, Dependency> dependencies, Map<String, Dependency> previousBuildDependencies, JsonNode npmDependencyTree, NpmScope scope) throws Exception {
         // Set of packages that could not be found in Artifactory.
         Set<NpmPackageInfo> badPackages = Collections.synchronizedSet(new HashSet<>());
-        DefaultMutableTreeNode rootNode = NpmDependencyTree.createDependenciesTree(npmDependenciesTree, scope);
+        DefaultMutableTreeNode rootNode = NpmDependencyTree.createDependencyTree(npmDependencyTree, scope);
         try (ArtifactoryDependenciesClient dependenciesClient = dependenciesClientBuilder.build()) {
             // Create producer Runnable.
             ProducerRunnableBase[] producerRunnable = new ProducerRunnableBase[]{new NpmExtractorProducer(rootNode)};

build-info-extractor-npm/src/main/java/org/jfrog/build/extractor/npm/extractor/NpmDependencyTree.java

@@ -5,7 +5,7 @@
 import org.apache.commons.lang3.ArrayUtils;
 import org.jfrog.build.extractor.npm.types.NpmPackageInfo;
 import org.jfrog.build.extractor.npm.types.NpmScope;
-import org.jfrog.build.extractor.scan.DependenciesTree;
+import org.jfrog.build.extractor.scan.DependencyTree;
 import org.jfrog.build.extractor.scan.Scope;
 
 import java.util.HashSet;
@@ -19,16 +19,16 @@
 public class NpmDependencyTree {
 
     /**
-     * Create a npm dependencies tree from the results of 'npm ls' command.
+     * Create a npm dependency tree from the results of 'npm ls' command.
      *
      * @param npmList - Results of 'npm ls' command.
      * @return Tree of npm PackageInfos.
      * @see NpmPackageInfo
      */
-    public static DependenciesTree createDependenciesTree(JsonNode npmList, NpmScope scope) {
-        DependenciesTree rootNode = new DependenciesTree();
+    public static DependencyTree createDependencyTree(JsonNode npmList, NpmScope scope) {
+        DependencyTree rootNode = new DependencyTree();
         populateDependenciesTree(rootNode, npmList.get("dependencies"), new String[]{npmList.get("name").asText() + ":" + npmList.get("version").asText()}, scope);
-        for (DependenciesTree child : rootNode.getChildren()) {
+        for (DependencyTree child : rootNode.getChildren()) {
             NpmPackageInfo packageInfo = (NpmPackageInfo) child.getUserObject();
             child.setScopes(getScopes(packageInfo.getName(), packageInfo.getScope()));
         }
@@ -42,7 +42,7 @@ public static DependenciesTree createDependenciesTree(JsonNode npmList, NpmScope
      * @param dependencies - The dependencies json object generated by npm ls.
      * @param pathToRoot   - A path-to-root dependency list. The structure of each dependency in the list is 'dependency-name:dependency-version'.
      */
-    private static void populateDependenciesTree(DependenciesTree scanTreeNode, JsonNode dependencies, String[] pathToRoot, NpmScope scope) {
+    private static void populateDependenciesTree(DependencyTree scanTreeNode, JsonNode dependencies, String[] pathToRoot, NpmScope scope) {
         if (dependencies == null || pathToRoot == null) {
             return;
         }
@@ -56,12 +56,12 @@ private static void populateDependenciesTree(DependenciesTree scanTreeNode, Json
         });
     }
 
-    private static void addSubtree(Map.Entry<String, JsonNode> stringJsonNodeEntry, DependenciesTree node, String name, String version, String[] pathToRoot, NpmScope scope) {
+    private static void addSubtree(Map.Entry<String, JsonNode> stringJsonNodeEntry, DependencyTree node, String name, String version, String[] pathToRoot, NpmScope scope) {
         JsonNode jsonNode = stringJsonNodeEntry.getValue();
         String devScope = scope.toString();
         NpmPackageInfo npmPackageInfo = new NpmPackageInfo(name, version, devScope, pathToRoot);
         JsonNode childDependencies = jsonNode.get("dependencies");
-        DependenciesTree childTreeNode = new DependenciesTree(npmPackageInfo);
+        DependencyTree childTreeNode = new DependencyTree(npmPackageInfo);
         populateDependenciesTree(childTreeNode, childDependencies, ArrayUtils.insert(0, pathToRoot, npmPackageInfo.toString()), scope); // Mutual recursive call
         node.add(childTreeNode);
     }

build-info-extractor-npm/src/main/java/org/jfrog/build/extractor/npm/extractor/NpmExtractorProducer.java

@@ -8,7 +8,7 @@
 import java.util.Enumeration;
 
 /**
- * Traverse over the dependencies tree of 'NpmPackageInfo's. If a node is legal - Produce it.
+ * Traverse over the dependency tree of 'NpmPackageInfo's. If a node is legal - Produce it.
  *
  * @author Yahav Itzhak
  */

build-info-extractor/src/main/java/org/jfrog/build/extractor/scan/DependenciesTree.java build-info-extractor/src/main/java/org/jfrog/build/extractor/scan/DependencyTree.java

@@ -1,40 +1,37 @@
 package org.jfrog.build.extractor.scan;
 
 import javax.swing.tree.DefaultMutableTreeNode;
-import java.util.Comparator;
-import java.util.HashSet;
-import java.util.Set;
-import java.util.Vector;
+import java.util.*;
 
 /**
- * Dependencies tree for Xray scan. Used in 'Eclipse' and 'Idea' Xray plugins.
+ * Dependency tree for Xray scan. Used in 'Eclipse' and 'Idea' Xray plugins.
  *
  * @author yahavi
  */
-public class DependenciesTree extends DefaultMutableTreeNode {
+public class DependencyTree extends DefaultMutableTreeNode {
 
-    private Set<Issue> issues = new HashSet<>();
     private Set<License> licenses = new HashSet<>();
+    private Set<Issue> issues = new HashSet<>();
     private Set<Scope> scopes = new HashSet<>();
-    private GeneralInfo generalInfo;
     private Issue topIssue = new Issue();
+    private GeneralInfo generalInfo;
 
-    public DependenciesTree() {
+    public DependencyTree() {
         super();
     }
 
-    public DependenciesTree(Object userObject) {
+    public DependencyTree(Object userObject) {
         super(userObject);
     }
 
-    public void setIssues(Set<Issue> issues) {
-        this.issues = issues;
-    }
-
     public void setLicenses(Set<License> licenses) {
         this.licenses = licenses;
     }
 
+    public void setIssues(Set<Issue> issues) {
+        this.issues = issues;
+    }
+
     public void setScopes(Set<Scope> scopes) {
         this.scopes = scopes;
     }
@@ -44,23 +41,23 @@ public void setGeneralInfo(GeneralInfo generalInfo) {
         this.generalInfo = generalInfo;
     }
 
-    @SuppressWarnings("unused")
-    public GeneralInfo getGeneralInfo() {
-        return generalInfo;
+    public Set<License> getLicenses() {
+        return licenses;
     }
 
     public Set<Issue> getIssues() {
         return issues;
     }
 
-    public Set<License> getLicenses() {
-        return licenses;
-    }
-
     public Set<Scope> getScopes() {
         return scopes;
     }
 
+    @SuppressWarnings("unused")
+    public GeneralInfo getGeneralInfo() {
+        return generalInfo;
+    }
+
     /**
      * @return top severity issue of the current node and its ancestors
      */
@@ -81,7 +78,7 @@ public int getIssueCount() {
      * @return Node's children
      */
     @SuppressWarnings({"WeakerAccess", "unchecked"})
-    public Vector<DependenciesTree> getChildren() {
+    public Vector<DependencyTree> getChildren() {
         return children != null ? children : new Vector<>();
     }
 
@@ -111,11 +108,11 @@ private void setIssuesComponent() {
 
     private void sortChildren() {
         getChildren().sort(Comparator
-                .comparing(DependenciesTree::getTopIssue, Comparator.comparing(Issue::getSeverity))
-                .thenComparing(DependenciesTree::getIssueCount)
-                .thenComparing(DependenciesTree::getChildCount)
+                .comparing(DependencyTree::getTopIssue, Comparator.comparing(Issue::getSeverity))
+                .thenComparing(DependencyTree::getIssueCount)
+                .thenComparing(DependencyTree::getChildCount)
                 .reversed()
-                .thenComparing(DependenciesTree::toString));
+                .thenComparing(DependencyTree::toString));
     }
 
     private void setTopIssue() {
@@ -128,4 +125,20 @@ private void setTopIssue() {
             }
         });
     }
+
+    /**
+     * Recursively, collect all scopes and licenses.
+     *
+     * @param allScopes   - Out - All dependency tree scopes
+     * @param allLicenses - Out - All dependency tree licenses
+     */
+    @SuppressWarnings("unused")
+    public void collectAllScopesAndLicenses(Set<Scope> allScopes, Set<License> allLicenses) {
+        Enumeration<?> enumeration = breadthFirstEnumeration();
+        while (enumeration.hasMoreElements()) {
+            DependencyTree child = (DependencyTree) enumeration.nextElement();
+            allScopes.addAll(child.getScopes());
+            allLicenses.addAll(child.getLicenses());
+        }
+    }
 }

build-info-extractor/src/main/java/org/jfrog/build/extractor/scan/GeneralInfo.java

@@ -18,6 +18,7 @@ public class GeneralInfo implements Serializable {
     private String groupId = "";
     private String artifactId = "";
     private String version = "";
+    private String sha1 = "";
 
     @SuppressWarnings("WeakerAccess")
     public GeneralInfo() {
@@ -77,6 +78,10 @@ public String getPkgType() {
         return pkgType;
     }
 
+    public String getSha1() {
+        return sha1;
+    }
+
     @SuppressWarnings("unused")
     public GeneralInfo componentId(String componentId) {
         this.componentId = componentId;
@@ -114,6 +119,11 @@ public GeneralInfo pkgType(String pkgType) {
         return this;
     }
 
+    public GeneralInfo sha1(String sha1) {
+        this.sha1 = sha1;
+        return this;
+    }
+
     private boolean isValid() {
         int colonCount = StringUtils.countMatches(componentId, ":");
         return colonCount == 1 || colonCount == 2;

build-info-extractor/src/test/java/org/jfrog/build/extractor/scan/DependenciesTreeTest.java build-info-extractor/src/test/java/org/jfrog/build/extractor/scan/DependencyTreeTest.java

@@ -16,21 +16,21 @@
 /**
  * @author yahavi
  */
-public class DependenciesTreeTest {
+public class DependencyTreeTest {
 
-    private DependenciesTree root, one, two, three, four, five;
+    private DependencyTree root, one, two, three, four, five;
 
     /**
      * Build an empty tree with 5 nodes
      */
     @BeforeClass
     public void init() {
-        root = new DependenciesTree("0");
-        one = new DependenciesTree("1");
-        two = new DependenciesTree("2");
-        three = new DependenciesTree("3");
-        four = new DependenciesTree("4");
-        five = new DependenciesTree("5");
+        root = new DependencyTree("0");
+        one = new DependencyTree("1");
+        two = new DependencyTree("2");
+        three = new DependencyTree("3");
+        four = new DependencyTree("4");
+        five = new DependencyTree("5");
         root.add(one); // 0 -> 1
         root.add(two); // 0 -> 2
         two.add(three); // 2 -> 3