GoModAnalyzer - include transitive dependencies (#2828)

* implement #2680

core/src/main/java/org/owasp/dependencycheck/analyzer/GolangModAnalyzer.java

@@ -144,6 +144,34 @@ private String getGo() {
         return "go";
     }
 
+    /**
+     * Launches `go mod help` to test if go is installed.
+     *
+     * @return a reference to the launched process
+     * @throws AnalysisException thrown if there is an issue launching `go mod`
+     */
+    private Process testGoMod(File folder) throws AnalysisException {
+        if (!folder.isDirectory()) {
+            throw new AnalysisException(String.format("%s should have been a directory.", folder.getAbsolutePath()));
+        }
+
+        final List<String> args = new ArrayList<>();
+        args.add(getGo());
+        args.add("mod");
+        args.add("edit");
+        args.add("-json");
+
+        final ProcessBuilder builder = new ProcessBuilder(args);
+        builder.directory(folder);
+        try {
+            LOGGER.info("Launching: {} from {}", args, folder);
+            return builder.start();
+        } catch (IOException ioe) {
+            throw new AnalysisException("go initialization failure; this error can be ignored if you are not analyzing Go. "
+                    + "Otherwise ensure that go is installed and the path to go is correctly specified", ioe);
+        }
+    }
+
     /**
      * Launches `go mod` in the given folder.
      *
@@ -158,9 +186,10 @@ private Process launchGoMod(File folder) throws AnalysisException {
 
         final List<String> args = new ArrayList<>();
         args.add(getGo());
-        args.add("mod");
-        args.add("edit");
+        args.add("list");
         args.add("-json");
+        args.add("-m");
+        args.add("all");
 
         final ProcessBuilder builder = new ProcessBuilder(args);
         builder.directory(folder);
@@ -186,7 +215,7 @@ protected void prepareFileTypeAnalyzer(Engine engine) throws InitializationExcep
         setEnabled(false);
         final Process process;
         try {
-            process = launchGoMod(getSettings().getTempDirectory());
+            process = testGoMod(getSettings().getTempDirectory());
         } catch (AnalysisException ae) {
             final String msg = String.format("Exception from go process: %s. Disabling %s", ae.getCause(), ANALYZER_NAME);
             throw new InitializationException(msg, ae);
@@ -252,6 +281,7 @@ protected void prepareFileTypeAnalyzer(Engine engine) throws InitializationExcep
      */
     @Override
     protected void analyzeDependency(Dependency dependency, Engine engine) throws AnalysisException {
+        //engine.removeDependency(dependency);
         final File parentFile = dependency.getActualFile().getParentFile();
         final Process process = launchGoMod(parentFile);
 
@@ -274,12 +304,10 @@ protected void analyzeDependency(Dependency dependency, Engine engine) throws An
                 }
             }
             if (!error.toString().equals("")) {
-                LOGGER.warn(error.toString());
-                throw new AnalysisException(error.toString());
+                LOGGER.warn("Warnings from go {}", error.toString());
+                //throw new AnalysisException(error.toString());
             }
-            final GoModJsonParser parser = new GoModJsonParser(process.getInputStream());
-            parser.process();
-            parser.getDependencies().forEach(goDep
+            GoModJsonParser.process(process.getInputStream()).forEach(goDep
                     -> engine.addDependency(goDep.toDependency(dependency))
             );
         } catch (IOException ioe) {

core/src/main/java/org/owasp/dependencycheck/data/golang/GoModDependency.java

@@ -115,13 +115,16 @@ private Dependency createDependency(Dependency parentDependency, String name, St
 
         packageURLBuilder.withName(moduleName);
         packageURLBuilder.withNamespace(packageNamespace);
-        packageURLBuilder.withVersion(version);
-
+        if (StringUtils.isNotBlank(version)) {
+            packageURLBuilder.withVersion(version);
+        }
         dep.setEcosystem(DEPENDENCY_ECOSYSTEM);
         dep.setDisplayFileName(name + ":" + version);
         dep.setName(moduleName);
-        dep.setVersion(version);
-        dep.setPackagePath(String.format("%s:%s", name, version));
+        if (StringUtils.isNotBlank(version)) {
+            dep.setVersion(version);
+            dep.setPackagePath(String.format("%s:%s", name, version));
+        }
         dep.setFilePath(filePath);
         dep.setSha1sum(Checksum.getSHA1Checksum(filePath));
         dep.setSha256sum(Checksum.getSHA256Checksum(filePath));
@@ -136,8 +139,9 @@ private Dependency createDependency(Dependency parentDependency, String name, St
         }
         dep.addEvidence(EvidenceType.PRODUCT, GO_MOD, "name", moduleName, Confidence.HIGHEST);
         dep.addEvidence(EvidenceType.VENDOR, GO_MOD, "name", moduleName, Confidence.HIGH);
-        dep.addEvidence(EvidenceType.VERSION, GO_MOD, "version", version, Confidence.HIGHEST);
-
+        if (StringUtils.isNotBlank(version)) {
+            dep.addEvidence(EvidenceType.VERSION, GO_MOD, "version", version, Confidence.HIGHEST);
+        }
         Identifier id;
         try {
             id = new PurlIdentifier(packageURLBuilder.build(), Confidence.HIGHEST);

core/src/main/java/org/owasp/dependencycheck/data/golang/GoModJsonParser.java

@@ -17,11 +17,12 @@
  */
 package org.owasp.dependencycheck.data.golang;
 
+import java.io.IOException;
 import org.owasp.dependencycheck.analyzer.exception.AnalysisException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import javax.annotation.concurrent.NotThreadSafe;
+import javax.annotation.concurrent.ThreadSafe;
 import javax.json.Json;
 import javax.json.JsonArray;
 import javax.json.JsonException;
@@ -32,58 +33,47 @@
 import java.io.InputStream;
 import java.util.ArrayList;
 import java.util.List;
+import org.owasp.dependencycheck.utils.JsonArrayFixingInputStream;
 
 /**
- * Parses json output from `go mod edit -json`.
+ * Parses json output from `go list -json -m all`.
  *
  * @author Matthijs van den Bos
  */
-@NotThreadSafe
-public class GoModJsonParser {
-
-    /**
-     * The JsonReader for parsing JSON.
-     */
-    private final JsonReader jsonReader;
-
-    /**
-     * The List of ComposerDependencies found.
-     */
-    private final List<GoModDependency> goModDependencies;
+@ThreadSafe
+public final class GoModJsonParser {
 
     /**
      * The LOGGER
      */
     private static final Logger LOGGER = LoggerFactory.getLogger(GoModJsonParser.class);
 
-    /**
-     * Creates a ComposerLockParser from a JsonReader and an InputStream.
-     *
-     * @param inputStream the InputStream to parse
-     */
-    public GoModJsonParser(InputStream inputStream) {
-        LOGGER.debug("Creating a ComposerLockParser");
-        this.jsonReader = Json.createReader(inputStream);
-        this.goModDependencies = new ArrayList<>();
+    private GoModJsonParser() {
     }
 
     /**
      * Process the input stream to create the list of dependencies.
      *
+     * @param inputStream the InputStream to parse
+     * @return the list of dependencies
      * @throws AnalysisException thrown when there is an error parsing the
      * results of `go mod`
      */
-    public void process() throws AnalysisException {
+    public static List<GoModDependency> process(InputStream inputStream) throws AnalysisException {
         LOGGER.debug("Beginning go.mod processing");
-        try {
-            final JsonObject composer = jsonReader.readObject();
-            if (composer.containsKey("Require") && !composer.isNull("Require")) {
-                LOGGER.debug("Found modules");
-                final JsonArray modules = composer.getJsonArray("Require");
+        List<GoModDependency> goModDependencies = new ArrayList<>();
+        try (JsonArrayFixingInputStream jsonStream = new JsonArrayFixingInputStream(inputStream)) {
+//            String array = IOUtils.toString(inputStream, UTF_8);
+//            array = array.trim().replace("}", "},");
+//            array = "[" + array.substring(0, array.length() - 1) + "]";
+//
+//            JsonReader reader = Json.createReader(new StringReader(array));
+            try (JsonReader reader = Json.createReader(jsonStream)) {
+                final JsonArray modules = reader.readArray();
                 for (JsonObject module : modules.getValuesAs(JsonObject.class)) {
                     final String path = module.getString("Path");
-                    String version = module.getString("Version");
-                    if (version.startsWith("v")) {
+                    String version = module.getString("Version", null);
+                    if (version != null && version.startsWith("v")) {
                         version = version.substring(1);
                     }
                     goModDependencies.add(new GoModDependency(path, version));
@@ -96,16 +86,10 @@ public void process() throws AnalysisException {
         } catch (IllegalStateException ise) {
             throw new AnalysisException("Illegal state in go mod stream", ise);
         } catch (ClassCastException cce) {
-            throw new AnalysisException("JSON not exactly matching output of `go mod edit -json`", cce);
+            throw new AnalysisException("JSON not exactly matching output of `go list -json -m all`", cce);
+        } catch (IOException ex) {
+            throw new AnalysisException("Error reading output of `go list -json -m all`", ex);
         }
-    }
-
-    /**
-     * Gets the list of dependencies.
-     *
-     * @return the list of dependencies
-     */
-    public List<GoModDependency> getDependencies() {
         return goModDependencies;
     }
 }

core/src/test/java/org/owasp/dependencycheck/analyzer/GolangModAnalyzerTest.java

@@ -53,6 +53,13 @@ public void setUp() throws Exception {
         getSettings().setBoolean(Settings.KEYS.ANALYZER_CENTRAL_ENABLED, false);
         getSettings().setBoolean(Settings.KEYS.ANALYZER_NODE_PACKAGE_ENABLED, false);
         getSettings().setBoolean(Settings.KEYS.ANALYZER_NODE_AUDIT_ENABLED, false);
+        //hack fix because IDE is not correctly pulling in path
+        if (getSettings().getString(Settings.KEYS.ANALYZER_GOLANG_PATH) == null) {
+            File go = new File("/usr/local/bin/go");
+            if (go.isFile() && go.canExecute()) {
+                getSettings().setString(Settings.KEYS.ANALYZER_GOLANG_PATH, "/usr/local/bin/go");
+            }
+        }
         analyzer = new GolangModAnalyzer();
         engine = new Engine(this.getSettings());
         analyzer.initialize(getSettings());
@@ -91,7 +98,7 @@ public void testGoMod() throws AnalysisException, InitializationException {
         final Dependency result = new Dependency(BaseTest.getResourceAsFile(this, "golang/go.mod"));
         analyzer.analyze(result, engine);
 
-        assertEquals(3, engine.getDependencies().length);
+        assertEquals(7, engine.getDependencies().length);
 
         boolean found = false;
         for (Dependency d : engine.getDependencies()) {

core/src/test/resources/golang/go.mod

@@ -1,7 +1,10 @@
 module my/thing
 
 require github.com/ethereum/go-ethereum v1.8.17
-require golang.org/x/crypto/salsa20 v0.0.1
-require  github.com/go-gitea/gitea v1.5.0
 
-replace bad/thing v1.4.5 => good/thing v1.4.5
+require (
+	github.com/go-gitea/gitea v1.5.0
+	golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a // indirect
+)
+
+replace bad/thing v1.4.5 => good/thing v1.4.5

core/src/test/resources/golang/go.sum

@@ -0,0 +1,8 @@
+github.com/ethereum/go-ethereum v1.8.17/go.mod h1:PwpWDrCLZrV+tfrhqqF6kPknbISMHaJv9Ln3kPCZLwY=
+github.com/go-gitea/gitea v1.5.0/go.mod h1:g8iUbfFNyuJp8u7GsSggxI8NQyuxeGTyqxogl3imbQM=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

utils/pom.xml

@@ -36,6 +36,11 @@ Copyright (c) 2014 - Jeremy Long. All Rights Reserved.
         <findbugs.onlyAnalyze>org.owasp.dependencycheck.utils.*</findbugs.onlyAnalyze>
     </properties>
     <dependencies>
+        <dependency>
+            <groupId>org.glassfish</groupId>
+            <artifactId>javax.json</artifactId>
+            <scope>test</scope>
+        </dependency>
         <dependency>
             <groupId>commons-io</groupId>
             <artifactId>commons-io</artifactId>