Merge branch 'master' of https://github.com/blackducksoftware/hub-det…

…ect.git

build.gradle

@@ -94,7 +94,7 @@ artifactory {
 }
 
 dependencies {
-    compile 'com.blackducksoftware.integration:hub-common:19.0.0'
+    compile 'com.blackducksoftware.integration:hub-common:20.0.0'
     compile 'org.springframework.boot:spring-boot-starter'
     compile 'org.apache.maven.shared:maven-invoker:3.0.0'
     compile 'com.esotericsoftware.yamlbeans:yamlbeans:1.11'

src/main/groovy/com/blackducksoftware/integration/hub/detect/Application.groovy

@@ -114,6 +114,9 @@ class Application {
     @Autowired
     List<ExitCodeReporter> exitCodeReporters;
 
+    @Autowired
+    DetectPhoneHomeManager detectPhoneHomeManager
+
     private ExitCodeType exitCodeType = ExitCodeType.SUCCESS;
     private String exitMessage = "";
 
@@ -165,6 +168,7 @@ class Application {
             if (!detectConfiguration.hubOfflineMode) {
                 hubServiceWrapper.init()
             }
+
             DetectProject detectProject = detectProjectManager.createDetectProject()
             List<File> createdBdioFiles = detectProjectManager.createBdioFiles(detectProject)
             if (!detectConfiguration.hubOfflineMode) {
@@ -180,6 +184,12 @@ class Application {
         } catch (Exception e) {
             populateExitCodeFromExceptionDetails(e)
         } finally {
+            try {
+                detectPhoneHomeManager.endPhoneHome();
+            } catch (Exception e) {
+                logger.debug(String.format('Error trying to end the phone home task: %s', e.getMessage()));
+            }
+
             if (!detectConfiguration.suppressResultsOutput) {
                 detectSummary.logResults(new Slf4jIntLogger(logger), exitCodeType, exitMessage);
             }

src/main/groovy/com/blackducksoftware/integration/hub/detect/DetectConfiguration.groovy

@@ -165,6 +165,16 @@ class DetectConfiguration {
             }
         }
 
+        if (hubSignatureScannerHostUrl) {
+            logger.info('A hub signature scanner url was provided, which requires hub offline mode. Setting hub offline mode to true.');
+            hubOfflineMode = true
+        }
+
+        if (hubSignatureScannerOfflineLocalPath) {
+            logger.info('A local hub signature scanner path was provided, which requires hub offline mode. Setting hub offline mode to true.');
+            hubOfflineMode = true
+        }
+
         if (gradleInspectorVersion.equals("latest") && gradleBomTool.isBomToolApplicable()) {
             gradleInspectorVersion = gradleBomTool.getInspectorVersion()
             logger.info("Resolved gradle inspector version from latest to: ${gradleInspectorVersion}")
@@ -179,8 +189,6 @@ class DetectConfiguration {
         }
     }
 
-
-
     /**
      * If the default source path is being used AND docker is configured, don't run unless the tool is docker
      */

src/main/groovy/com/blackducksoftware/integration/hub/detect/DetectPhoneHomeManager.java

@@ -0,0 +1,49 @@
+/**
+ * Copyright (C) 2017 Black Duck Software, Inc.
+ * http://www.blackducksoftware.com/
+ *
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package com.blackducksoftware.integration.hub.detect;
+
+import org.springframework.stereotype.Component;
+
+import com.blackducksoftware.integration.hub.dataservice.phonehome.PhoneHomeDataService;
+import com.blackducksoftware.integration.hub.dataservice.phonehome.PhoneHomeResponse;
+import com.blackducksoftware.integration.phonehome.PhoneHomeRequestBody;
+
+@Component
+public class DetectPhoneHomeManager {
+    private PhoneHomeResponse phoneHomeResponse;
+
+    public void startPhoneHome(final PhoneHomeDataService phoneHomeDataService, final PhoneHomeRequestBody phoneHomeRequestBody) {
+        phoneHomeResponse = phoneHomeDataService.startPhoneHome(phoneHomeRequestBody);
+    }
+
+    public void endPhoneHome() {
+        if (phoneHomeResponse != null) {
+            phoneHomeResponse.endPhoneHome();
+        }
+    }
+
+    public PhoneHomeResponse getPhoneHomeResponse() {
+        return phoneHomeResponse;
+    }
+
+}

src/main/groovy/com/blackducksoftware/integration/hub/detect/bomtool/docker/DockerProperties.groovy

@@ -43,6 +43,7 @@ class DockerProperties {
         dockerProperties.setProperty('output.path', bomToolOutputDirectory.getAbsolutePath())
         dockerProperties.setProperty('output.include.containerfilesystem', 'true')
         dockerProperties.setProperty('logging.level.com.blackducksoftware', detectConfiguration.getLoggingLevel())
+        dockerProperties.setProperty('phone.home', 'false')
 
         detectConfiguration.additionalDockerPropertyNames.each { propertyName ->
             String dockerKey = propertyName[DetectConfiguration.DOCKER_PROPERTY_PREFIX.length()..-1]

src/main/groovy/com/blackducksoftware/integration/hub/detect/bomtool/packagist/PackagistParser.groovy

@@ -22,6 +22,8 @@
  */
 package com.blackducksoftware.integration.hub.detect.bomtool.packagist
 
+import org.slf4j.Logger
+import org.slf4j.LoggerFactory
 import org.springframework.beans.factory.annotation.Autowired
 import org.springframework.stereotype.Component
 
@@ -43,6 +45,8 @@ import groovy.transform.TypeChecked
 @Component
 @TypeChecked
 class PackagistParser {
+    private final Logger logger = LoggerFactory.getLogger(PackagistParser.class)
+
     @Autowired
     DetectConfiguration detectConfiguration
 
@@ -68,8 +72,24 @@ class PackagistParser {
         }
         convertFromJsonToDependency(graph, null, startingPackages, packagistPackages, true)
 
+        packagistPackages.each {
+            String packageName = it.getAt('name').toString().replace('"', '')
+            String packageVersion = it.getAt('version').toString().replace('"', '')
+            ExternalId id = externalIdFactory.createNameVersionExternalId(Forge.PACKAGIST, packageName, packageVersion);
+            if (graph.getDependency(id) == null) {
+                logger.warn("A discrepency exists between the composer.json and the composer.lock - the package '${packageName}' was in the lock but was not included in the json dependency tree.");
+            }
+        }
+
+        List<String> allPackageNames =  packagistPackages.collect{it.getAt('name').toString().replace('"', '')}
+        startingPackages.each {
+            if (!allPackageNames.contains(it)) {
+                logger.warn("A discrepency exists between the composer.json and the composer.lock - the package '${it}' was in the json but not the lock.");
+            }
+        }
+
         ExternalId projectExternalId;
-        if (projectName == null || projectVersion == null){
+        if (projectName == null || projectVersion == null) {
             projectExternalId = externalIdFactory.createPathExternalId(Forge.PACKAGIST, sourcePath);
         }else{
             projectExternalId = externalIdFactory.createNameVersionExternalId(Forge.PACKAGIST, projectName, projectVersion);
@@ -92,7 +112,7 @@ class PackagistParser {
                 Dependency child = new Dependency(currentRowPackageName, currentRowPackageVersion, externalIdFactory.createNameVersionExternalId(Forge.PACKAGIST, currentRowPackageName, currentRowPackageVersion))
 
                 convertFromJsonToDependency(graph, child, getStartingPackages(it.getAsJsonObject(), false), jsonArray, false)
-                if (root){
+                if (root) {
                     graph.addChildToRoot(child)
                 }else{
                     graph.addParentWithChild(parent, child)

src/main/groovy/com/blackducksoftware/integration/hub/detect/hub/BdioUploader.groovy

@@ -22,6 +22,7 @@
  */
 package com.blackducksoftware.integration.hub.detect.hub
 
+import org.apache.commons.lang3.StringUtils
 import org.slf4j.Logger
 import org.slf4j.LoggerFactory
 import org.springframework.beans.factory.annotation.Autowired
@@ -31,8 +32,12 @@ import com.blackducksoftware.integration.hub.api.bom.BomImportRequestService
 import com.blackducksoftware.integration.hub.dataservice.phonehome.PhoneHomeDataService
 import com.blackducksoftware.integration.hub.detect.DetectConfiguration
 import com.blackducksoftware.integration.hub.detect.DetectInfo
+import com.blackducksoftware.integration.hub.detect.DetectPhoneHomeManager
+import com.blackducksoftware.integration.hub.detect.model.BomToolType
+import com.blackducksoftware.integration.hub.detect.model.DetectProject
 import com.blackducksoftware.integration.hub.global.HubServerConfig
 import com.blackducksoftware.integration.phonehome.PhoneHomeRequestBody
+import com.blackducksoftware.integration.phonehome.PhoneHomeRequestBodyBuilder
 import com.blackducksoftware.integration.phonehome.enums.ThirdPartyName
 
 import groovy.transform.TypeChecked
@@ -48,7 +53,10 @@ class BdioUploader {
     @Autowired
     DetectConfiguration detectConfiguration
 
-    void uploadBdioFiles(HubServerConfig hubServerConfig, BomImportRequestService bomImportRequestService, PhoneHomeDataService phoneHomeDataService, List<File> createdBdioFiles) {
+    @Autowired
+    DetectPhoneHomeManager detectPhoneHomeManager
+
+    void uploadBdioFiles(HubServerConfig hubServerConfig, BomImportRequestService bomImportRequestService, PhoneHomeDataService phoneHomeDataService, DetectProject detectProject, List<File> createdBdioFiles) {
         createdBdioFiles.each { file ->
             logger.info("uploading ${file.name} to ${detectConfiguration.getHubUrl()}")
             bomImportRequestService.importBomFile(file)
@@ -58,7 +66,13 @@ class BdioUploader {
         }
 
         String hubDetectVersion = detectInfo.detectVersion
-        PhoneHomeRequestBody phoneHomeRequestBody = phoneHomeDataService.createInitialPhoneHomeRequestBodyBuilder(ThirdPartyName.DETECT, hubDetectVersion, hubDetectVersion).build()
-        phoneHomeDataService.phoneHome(phoneHomeRequestBody)
+        Set<BomToolType> applicableBomTools = detectProject.getApplicableBomTools();
+        String applicableBomToolsString = StringUtils.join(applicableBomTools, ", ");
+
+        PhoneHomeRequestBodyBuilder phoneHomeRequestBodyBuilder = phoneHomeDataService.createInitialPhoneHomeRequestBodyBuilder(ThirdPartyName.DETECT, hubDetectVersion, hubDetectVersion);
+        phoneHomeRequestBodyBuilder.addToMetaDataMap('bomToolTypes', applicableBomToolsString);
+
+        PhoneHomeRequestBody phoneHomeRequestBody = phoneHomeRequestBodyBuilder.build();
+        detectPhoneHomeManager.startPhoneHome(phoneHomeDataService, phoneHomeRequestBody);
     }
 }

src/main/groovy/com/blackducksoftware/integration/hub/detect/hub/HubManager.groovy

@@ -89,7 +89,7 @@ class HubManager implements ExitCodeReporter {
             HubServerConfig hubServerConfig = hubServiceWrapper.hubServerConfig
             BomImportRequestService bomImportRequestService = hubServiceWrapper.createBomImportRequestService()
             PhoneHomeDataService phoneHomeDataService = hubServiceWrapper.createPhoneHomeDataService()
-            bdioUploader.uploadBdioFiles(hubServerConfig, bomImportRequestService, phoneHomeDataService, createdBdioFiles)
+            bdioUploader.uploadBdioFiles(hubServerConfig, bomImportRequestService, phoneHomeDataService, detectProject, createdBdioFiles)
         } else {
             logger.debug('Did not create any bdio files.')
         }

src/main/groovy/com/blackducksoftware/integration/hub/detect/model/DetectProject.groovy

@@ -71,6 +71,14 @@ class DetectProject {
         detectCodeLocations.add(detectCodeLocation)
     }
 
+    public Set<BomToolType> getApplicableBomTools() {
+        Set<BomToolType> applicableBomTools = new HashSet<>();
+        for (DetectCodeLocation detectCodeLocation : detectCodeLocations) {
+            applicableBomTools.add(detectCodeLocation.getBomToolType());
+        }
+        return applicableBomTools;
+    }
+
     public List<DetectCodeLocation> getDetectCodeLocations() {
         detectCodeLocations
     }

src/main/groovy/com/blackducksoftware/integration/hub/detect/util/FileFinder.groovy

@@ -126,7 +126,6 @@ class FileFinder {
     }
 
     private File[] findDirectoriesContainingDirectoriesToDepthRecursive(final File sourceDirectory, final String directoryPattern, int currentDepth, int maxDepth) {
-
         def files = []
         if (currentDepth > maxDepth || !sourceDirectory.isDirectory()) {
             return files as File[]