Merge commit '8a6725accf827ca4c184ff45ce5470e9ac649374' into sb_bazel…

…CqueryAdditionalOptions

build.gradle

@@ -8,11 +8,15 @@ import com.synopsys.integration.blackduck.service.ProjectService
 import com.synopsys.integration.blackduck.service.model.ProjectVersionWrapper
 import com.synopsys.integration.log.PrintStreamIntLogger
 
+import java.nio.file.Files
+import java.nio.file.Paths
+
 buildscript {
     ext {
         springBootVersion = '2.1.5.RELEASE'
         blackDuckCommonVersion = '44.2.14'
         polarisCommonVersion = '0.13.2'
+        junitPlatformDefaultTestTags = "integration, performance, battery"
     }
     repositories {
         mavenCentral()
@@ -43,6 +47,10 @@ def createArtifactName() {
     return "${buildDir}/libs/${project.name}-${version}.jar"
 }
 
+def createBatteryPath() {
+    return "${buildDir}/battery/"
+}
+
 allprojects {
     apply plugin: 'org.jetbrains.kotlin.jvm'
     apply plugin: 'io.spring.dependency-management'
@@ -142,6 +150,14 @@ if ("true" == project.findProperty('refresh.cache')) {
     }
 }
 
+tasks['testBattery'].doFirst {
+    def batteryPath = new File(createBatteryPath())
+    batteryPath.mkdirs()
+    environment 'BATTERY_TESTS_DETECT_JAR_PATH', createArtifactName()
+    environment 'BATTERY_TESTS_PATH', batteryPath.getCanonicalPath()
+    environment 'BATTERY_TESTS_JAVA_PATH', org.gradle.internal.jvm.Jvm.current().javaExecutable.getCanonicalPath()
+}
+
 task runDetect(type: JavaExec) {
     dependsOn build
 

detect-configuration/src/main/java/com/synopsys/integration/detect/configuration/DetectProperty.java

@@ -777,10 +777,6 @@ public enum DetectProperty {
     @AcceptableValues(value = { "BAZEL", "DETECTOR", "DOCKER", "SIGNATURE_SCAN", "BINARY_SCAN", "POLARIS", "NONE", "ALL" }, caseSensitive = true, strict = false, isCommaSeparatedList = true)
     DETECT_TOOLS_EXCLUDED("detect.tools.excluded", "Detect Tools Excluded", "5.0.0", PropertyType.STRING, PropertyAuthority.NONE),
 
-    @HelpDescription("The path to the Yarn executable.")
-    @HelpGroup(primary = GROUP_YARN, additional = { SEARCH_GROUP_GLOBAL })
-    DETECT_YARN_PATH("detect.yarn.path", "Yarn Executable", "4.0.0", PropertyType.STRING, PropertyAuthority.NONE),
-
     @HelpDescription("Set this to true to only scan production dependencies.")
     @HelpGroup(primary = GROUP_YARN, additional = { SEARCH_GROUP_GLOBAL, GROUP_SOURCE_SCAN })
     DETECT_YARN_PROD_ONLY("detect.yarn.prod.only", "Include Yarn Production Dependencies Only", "4.0.0", PropertyType.BOOLEAN, PropertyAuthority.NONE, "false"),

detectable/src/main/java/com/synopsys/integration/detectable/Detectable.java

@@ -22,11 +22,16 @@
  */
 package com.synopsys.integration.detectable;
 
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
+
 import com.synopsys.integration.detectable.detectable.exception.DetectableException;
 import com.synopsys.integration.detectable.detectable.result.DetectableResult;
 
 public abstract class Detectable {
     protected DetectableEnvironment environment;
+    protected List<File> relevantFiles = new ArrayList<>();
     private final String name;
     private final String group;
 
@@ -74,4 +79,8 @@ public String getGroupName() {
     public String getDescriptiveName() {
         return String.format("%s - %s", getGroupName(), getName());
     }
+
+    public List<File> getFoundRelevantFiles() {
+        return relevantFiles;
+    }
 }

detectable/src/main/java/com/synopsys/integration/detectable/Extraction.java

@@ -22,7 +22,9 @@
  */
 package com.synopsys.integration.detectable;
 
+import java.io.File;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -33,6 +35,7 @@
 
 public class Extraction {
     private final List<CodeLocation> codeLocations;
+    private final List<File> relevantFiles;
     private final ExtractionResultType result;
     private final Exception error;
     private final String description;
@@ -50,6 +53,7 @@ private Extraction(final Builder builder) {
         this.projectVersion = builder.projectVersion;
         this.projectName = builder.projectName;
         this.metaData = builder.metaData;
+        this.relevantFiles = builder.relevantFiles;
 
         if (result == null) {
             throw new IllegalArgumentException("An extraction requires a result type.");
@@ -58,8 +62,8 @@ private Extraction(final Builder builder) {
 
     public <T> Optional<T> getMetaData(final ExtractionMetadata<T> extractionMetadata) {
         if (metaData.containsKey(extractionMetadata)) {
-            Class<T> clazz = extractionMetadata.getMetadataClass();
-            Object value = metaData.get(extractionMetadata);
+            final Class<T> clazz = extractionMetadata.getMetadataClass();
+            final Object value = metaData.get(extractionMetadata);
             if (value != null && clazz.isAssignableFrom(value.getClass())) {
                 return Optional.of(clazz.cast(value));
             }
@@ -97,8 +101,13 @@ public ExtractionResultType getResult() {
         return result;
     }
 
+    public List<File> getRelevantFiles() {
+        return relevantFiles;
+    }
+
     public static class Builder {
         private final List<CodeLocation> codeLocations = new ArrayList<>();
+        private final List<File> relevantFiles = new ArrayList<>();
         private ExtractionResultType result;
         private Exception error;
         private String description;
@@ -169,6 +178,11 @@ public <T> Builder metaData(final ExtractionMetadata<T> key, final T value) {
             return this;
         }
 
+        public Builder relevantFiles(final File... files) {
+            this.relevantFiles.addAll(Arrays.asList(files));
+            return this;
+        }
+
         public Extraction build() {
             return new Extraction(this);
         }

detectable/src/main/java/com/synopsys/integration/detectable/detectable/executable/ExecutableOutput.java

@@ -29,17 +29,15 @@ public class ExecutableOutput {
     private int returnCode = 0;
     private final String standardOutput;
     private final String errorOutput;
+    private final String commandDescription;
 
-    public ExecutableOutput(final int returnCode, final String standardOutput, final String errorOutput) {
+    public ExecutableOutput(final String commandDescription, final int returnCode, final String standardOutput, final String errorOutput) {
+        this.commandDescription = commandDescription;
         this.returnCode = returnCode;
         this.standardOutput = standardOutput;
         this.errorOutput = errorOutput;
     }
 
-    public ExecutableOutput(final String standardOutput, final String errorOutput) {
-        this(0, standardOutput, errorOutput);
-    }
-
     public List<String> getStandardOutputAsList() {
         return Arrays.asList(standardOutput.split(System.lineSeparator()));
     }
@@ -59,4 +57,8 @@ public String getErrorOutput() {
     public int getReturnCode() {
         return returnCode;
     }
+
+    public String getCommandDescription() {
+        return commandDescription;
+    }
 }

detectable/src/main/java/com/synopsys/integration/detectable/detectable/executable/impl/SimpleExecutableResolver.java

@@ -45,13 +45,12 @@
 import com.synopsys.integration.detectable.detectable.executable.resolver.PythonResolver;
 import com.synopsys.integration.detectable.detectable.executable.resolver.Rebar3Resolver;
 import com.synopsys.integration.detectable.detectable.executable.resolver.SwiftResolver;
-import com.synopsys.integration.detectable.detectable.executable.resolver.YarnResolver;
 import com.synopsys.integration.detectable.detectable.inspector.go.GoResolver;
 
 //this will cache the find result.
 public class SimpleExecutableResolver
-    implements GradleResolver, BashResolver, CondaResolver, CpanmResolver, CpanResolver, PearResolver, Rebar3Resolver, YarnResolver, PythonResolver, PipResolver, PipenvResolver, MavenResolver, NpmResolver, BazelResolver, JavaResolver,
-                   DotNetResolver, DockerResolver, GitResolver, SwiftResolver, GoResolver {
+    implements GradleResolver, BashResolver, CondaResolver, CpanmResolver, CpanResolver, PearResolver, Rebar3Resolver, PythonResolver, PipResolver, PipenvResolver, MavenResolver, NpmResolver, BazelResolver, JavaResolver, DotNetResolver,
+                   DockerResolver, GitResolver, SwiftResolver, GoResolver {
 
     private final CachedExecutableResolverOptions executableResolverOptions;
     private final SimpleLocalExecutableFinder localExecutableFinder;
@@ -118,11 +117,6 @@ public File resolveRebar3() {
         return findCachedSystem("rebar3");
     }
 
-    @Override
-    public File resolveYarn() {
-        return findCachedSystem("yarn");
-    }
-
     @Override
     public File resolvePip() {
         final String suffix = executableResolverOptions.isPython3() ? "3" : "";

detectable/src/main/java/com/synopsys/integration/detectable/detectable/executable/impl/SimpleExecutableRunner.java

@@ -82,7 +82,7 @@ public ExecutableOutput execute(final Executable executable) throws ExecutableRu
                 final String standardOutput = standardOutputThread.getExecutableOutput().trim();
                 final String errorOutput = errorOutputThread.getExecutableOutput().trim();
 
-                final ExecutableOutput output = new ExecutableOutput(returnCode, standardOutput, errorOutput);
+                final ExecutableOutput output = new ExecutableOutput(executable.getMaskedExecutableDescription(), returnCode, standardOutput, errorOutput);
                 return output;
             }
         } catch (final Exception e) {

detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/parse/BitbakeGraphTransformer.java

@@ -88,12 +88,33 @@ public DependencyGraph transform(final BitbakeGraph bitbakeGraph, final Map<Stri
         return dependencyGraph;
     }
 
+    // Temporarily we must strip the epoch and the build revision from the version string. TODO: Remove this when the KB is ready. KBENG-961
+    // 1:1.2.3-r5 -> 1.2.3
+    public String cleanVersion(final String version) {
+        String cleanedVersion = version;
+        final int lastHyphenIndex = cleanedVersion.lastIndexOf('-');
+        if (lastHyphenIndex != -1 && lastHyphenIndex < cleanedVersion.length()) {
+            final String suffix = cleanedVersion.substring(lastHyphenIndex);
+            if (suffix.startsWith("-r")) {
+                cleanedVersion = cleanedVersion.substring(0, lastHyphenIndex);
+            }
+        }
+
+        final int epochSeparatorIndex = cleanedVersion.indexOf(':');
+        if (epochSeparatorIndex != -1 && epochSeparatorIndex < cleanedVersion.length()) {
+            cleanedVersion = cleanedVersion.substring(epochSeparatorIndex + 1);
+        }
+
+        return cleanedVersion;
+    }
+
     private Optional<ExternalId> generateExternalId(final String dependencyName, final String dependencyVersion, final Map<String, String> recipeLayerMap) {
         final String priorityLayerName = recipeLayerMap.get(dependencyName);
         ExternalId externalId = null;
 
         if (priorityLayerName != null) {
-            externalId = externalIdFactory.createYoctoExternalId(priorityLayerName, dependencyName, dependencyVersion);
+            final String version = cleanVersion(dependencyVersion);
+            externalId = externalIdFactory.createYoctoExternalId(priorityLayerName, dependencyName, version);
         } else {
             logger.debug(String.format("Failed to find component '%s' in component layer map.", dependencyName));
 

detectable/src/main/java/com/synopsys/integration/detectable/detectables/git/cli/GitCliExtractor.java

@@ -26,6 +26,7 @@
 import java.net.MalformedURLException;
 import java.util.List;
 
+import org.apache.commons.lang3.StringUtils;
 import org.slf4j.LoggerFactory;
 
 import com.synopsys.integration.detectable.Extraction;
@@ -50,7 +51,12 @@ public GitCliExtractor(final ExecutableRunner executableRunner, final GitUrlPars
     public Extraction extract(final File gitExecutable, final File directory) {
         try {
             final String repoName = getRepoName(gitExecutable, directory);
-            final String branch = getRepoBranch(gitExecutable, directory);
+            String branch = getRepoBranch(gitExecutable, directory);
+
+            if ("HEAD".equals(branch)) {
+                logger.info("HEAD is detached for this repo, using heuristics to find Git branch.");
+                branch = getRepoBranchBackup(gitExecutable, directory);
+            }
 
             return new Extraction.Builder()
                        .success()
@@ -74,6 +80,19 @@ private String getRepoBranch(final File gitExecutable, final File directory) thr
         return runGitSingleLinesResponse(gitExecutable, directory, "rev-parse", "--abbrev-ref", "HEAD").trim();
     }
 
+    private String getRepoBranchBackup(final File gitExecutable, final File directory) throws ExecutableRunnerException, IntegrationException {
+        String output = runGitSingleLinesResponse(gitExecutable, directory, "log", "-n", "1", "--pretty=%d", "HEAD").trim();
+        output = StringUtils.removeStart(output, "(");
+        output = StringUtils.removeEnd(output, ")");
+        final String[] pieces = output.split(", ");
+
+        if (pieces.length != 2 || pieces[1].startsWith("tag: ")) {
+            throw new IntegrationException("Failed to extract branch on second attempt.");
+        }
+
+        return pieces[1];
+    }
+
     private String runGitSingleLinesResponse(final File gitExecutable, final File directory, final String... commands) throws ExecutableRunnerException, IntegrationException {
         final ExecutableOutput gitOutput = executableRunner.execute(directory, gitExecutable, commands);
 

detectable/src/main/java/com/synopsys/integration/detectable/detectables/git/cli/GitUrlParser.java

@@ -23,37 +23,20 @@
 package com.synopsys.integration.detectable.detectables.git.cli;
 
 import java.net.MalformedURLException;
-import java.net.URL;
 
 import org.apache.commons.lang3.StringUtils;
 
-import com.synopsys.integration.exception.IntegrationException;
-
 public class GitUrlParser {
-
-    public String getRepoName(final String remoteUrlString) throws IntegrationException, MalformedURLException {
-        final String remoteUrlPath;
-        if (remoteUrlString.startsWith("ssh://") || remoteUrlString.startsWith("git://")) {
-            // Parses urls such as: ssh://user@synopsys.com:12345/blackducksoftware/synopsys-detect
-            final int lastIndexOfSlash = remoteUrlString.lastIndexOf('/');
-            final String projectName = remoteUrlString.substring(lastIndexOfSlash);
-            final String remainder = remoteUrlString.substring(0, lastIndexOfSlash);
-            final int remainderLastIndexOfSlash = remainder.lastIndexOf('/');
-            final String organization = remainder.substring(remainderLastIndexOfSlash);
-            remoteUrlPath = organization + projectName;
-        } else if (remoteUrlString.contains("@")) {
-            // Parses urls such as: git@github.com:blackducksoftware/synopsys-detect.git
-            final String[] tokens = remoteUrlString.split(":");
-            if (tokens.length != 2) {
-                throw new IntegrationException(String.format("Failed to extract project name from: %s", remoteUrlString));
-            }
-            remoteUrlPath = tokens[1].trim();
+    // Parses urls such as: https://github.com/blackducksoftware/synopsys-detect
+    public String getRepoName(final String remoteUrlString) throws MalformedURLException {
+        final String[] pieces = remoteUrlString.split("[/:]");
+        if (pieces.length >= 2) {
+            final String organization = pieces[pieces.length - 2];
+            final String repo = pieces[pieces.length - 1];
+            final String name = String.format("%s/%s", organization, repo);
+            return StringUtils.removeEnd(StringUtils.removeStart(name, "/"), ".git");
         } else {
-            // Parses urls such as: https://github.com/blackducksoftware/synopsys-detect
-            final URL remoteURL = new URL(remoteUrlString);
-            remoteUrlPath = remoteURL.getPath().trim();
+            throw new MalformedURLException("Failed to extract repository name from url. Not logging url for security.");
         }
-
-        return StringUtils.removeEnd(StringUtils.removeStart(remoteUrlPath, "/"), ".git");
     }
 }

detectable/src/main/java/com/synopsys/integration/detectable/detectables/git/parsing/parse/GitFileTransformer.java

@@ -23,17 +23,21 @@
 package com.synopsys.integration.detectable.detectables.git.parsing.parse;
 
 import java.net.MalformedURLException;
-import java.net.URL;
 import java.util.List;
 import java.util.Optional;
 
-import org.apache.commons.lang3.StringUtils;
-
+import com.synopsys.integration.detectable.detectables.git.cli.GitUrlParser;
 import com.synopsys.integration.detectable.detectables.git.parsing.model.GitConfigElement;
 import com.synopsys.integration.exception.IntegrationException;
 import com.synopsys.integration.util.NameVersion;
 
 public class GitFileTransformer {
+    private final GitUrlParser gitUrlParser;
+
+    public GitFileTransformer(final GitUrlParser gitUrlParser) {
+        this.gitUrlParser = gitUrlParser;
+    }
+
     public NameVersion transformGitConfigElements(final List<GitConfigElement> gitConfigElements, final String gitHead) throws IntegrationException, MalformedURLException {
         final Optional<GitConfigElement> currentBranch = gitConfigElements.stream()
                                                              .filter(gitConfigElement -> gitConfigElement.getElementType().equals("branch"))
@@ -61,9 +65,7 @@ public NameVersion transformGitConfigElements(final List<GitConfigElement> gitCo
             throw new IntegrationException("Failed to find a remote url.");
         }
 
-        final URL remoteURL = new URL(remoteUrlOptional.get());
-        final String path = remoteURL.getPath();
-        final String projectName = StringUtils.removeEnd(StringUtils.removeStart(path, "/"), ".git");
+        final String projectName = gitUrlParser.getRepoName(remoteUrlOptional.get());
         final String projectVersionName = currentBranch.get().getName().orElse(null);
 
         return new NameVersion(projectName, projectVersionName);

detectable/src/main/java/com/synopsys/integration/detectable/detectables/npm/cli/NpmCliDetectable.java

@@ -70,9 +70,10 @@ public DetectableResult applicable() {
 
         if (packageJson == null) {
             return new FileNotFoundDetectableResult(PACKAGE_JSON);
+        } else {
+            relevantFiles.add(packageJson);
         }
 
-        // addRelevantDiagnosticFile(packageJson); // TODO: Jordan fix me
         return new PassedDetectableResult();
     }
 

detectable/src/main/java/com/synopsys/integration/detectable/detectables/npm/lockfile/NpmPackageLockDetectable.java

@@ -60,13 +60,15 @@ public DetectableResult applicable() {
         lockfile = fileFinder.findFile(environment.getDirectory(), PACKAGE_LOCK_JSON);
         if (lockfile == null) {
             return new FileNotFoundDetectableResult(PACKAGE_LOCK_JSON);
+        } else {
+            relevantFiles.add(lockfile);
         }
 
-        final File foundPackageJson = fileFinder.findFile(environment.getDirectory(), PACKAGE_JSON);
-        if (foundPackageJson == null) {
+        packageJson = fileFinder.findFile(environment.getDirectory(), PACKAGE_JSON);
+        if (packageJson == null) {
             logger.warn("Npm applied but it could not find a package.json so dependencies may not be entirely accurate.");
         } else {
-            packageJson = foundPackageJson;
+            relevantFiles.add(packageJson);
         }
 
         return new PassedDetectableResult();