Bdio 2

build.gradle

@@ -11,7 +11,7 @@ import com.synopsys.integration.log.PrintStreamIntLogger
 buildscript {
     ext {
         springBootVersion = '2.1.5.RELEASE'
-        blackDuckCommonVersion = '44.4.5'
+        blackDuckCommonVersion = '44.5.0'
         polarisCommonVersion = '0.13.2'
         junitPlatformDefaultTestTags = "integration, performance, battery"
     }

detect-configuration/src/main/java/com/synopsys/integration/detect/configuration/DetectProperty.java

@@ -167,6 +167,11 @@ public enum DetectProperty {
     @HelpDetailed("If not set, the BDIO files are placed in a 'BDIO' subdirectory of the output directory.")
     DETECT_BDIO_OUTPUT_PATH("detect.bdio.output.path", "BDIO Output Directory", "3.0.0", PropertyType.STRING, PropertyAuthority.DIRECTORY_MANAGER),
 
+    @HelpGroup(primary = GROUP_PATHS, additional = { SEARCH_GROUP_GLOBAL })
+    @HelpDescription("The version of BDIO files to generate.")
+    @HelpDetailed("If set to false, BDIO version 1 will be generated an uploaded. If set to true, BDIO version 2 will be generated but NOT uploaded. The flag can only be set to true if Detect is in offline mode.")
+    DETECT_BDIO2_ENABLED("detect.bdio2.enabled", "BDIO 2 Enabled", "6.1.0", PropertyType.BOOLEAN, PropertyAuthority.NONE, "false"),
+
     @HelpGroup(primary = GROUP_SIGNATURE_SCANNER, additional = { GROUP_SOURCE_PATH })
     @HelpDescription("If specified, this file and this file only will be uploaded for binary scan analysis. This property takes precedence over detect.binary.scan.file.name.patterns.")
     DETECT_BINARY_SCAN_FILE("detect.binary.scan.file.path", "Binary Scan Target", "4.2.0", PropertyType.STRING, PropertyAuthority.NONE),

detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/parse/BitbakeGraphTransformer.java

@@ -106,8 +106,8 @@ private Optional<ExternalId> generateExternalId(final String dependencyName, fin
             }
         }
 
-        if (externalId != null && externalId.version.contains("AUTOINC")) {
-            externalId.version = externalId.version.replaceFirst("AUTOINC\\+[\\w|\\d]*", "X");
+        if (externalId != null && externalId.getVersion().contains("AUTOINC")) {
+            externalId.setVersion(externalId.getVersion().replaceFirst("AUTOINC\\+[\\w|\\d]*", "X"));
         }
 
         return Optional.ofNullable(externalId);

detectable/src/main/java/com/synopsys/integration/detectable/detectables/clang/dependencyfile/ClangPackageDetailsTransformer.java

@@ -68,7 +68,7 @@ private List<Dependency> toDependency(final List<Forge> forges, final PackageDet
         for (final Forge forge : forges) {
             final ExternalId extId = externalIdFactory.createArchitectureExternalId(forge, name, version, arch);
             final Dependency dep = new Dependency(name, version, extId);
-            logger.debug(String.format("forge: %s: adding %s version %s as child to dependency node tree; externalId: %s", forge.getName(), dep.name, dep.version, dep.externalId.createBdioId()));
+            logger.debug(String.format("forge: %s: adding %s version %s as child to dependency node tree; externalId: %s", forge.getName(), dep.getName(), dep.getVersion(), dep.getExternalId().createBdioId()));
             dependencies.add(dep);
         }
         return dependencies;

detectable/src/main/java/com/synopsys/integration/detectable/detectables/cocoapods/PodlockExtractor.java

@@ -31,6 +31,7 @@
 import org.slf4j.LoggerFactory;
 
 import com.synopsys.integration.bdio.graph.DependencyGraph;
+import com.synopsys.integration.bdio.graph.builder.MissingExternalIdException;
 import com.synopsys.integration.detectable.Extraction;
 import com.synopsys.integration.detectable.detectable.codelocation.CodeLocation;
 import com.synopsys.integration.detectable.detectables.cocoapods.parser.PodlockParser;
@@ -59,7 +60,7 @@ public Extraction extract(final File podlock) {
             logger.trace("Attempting to create the dependency graph from the pod lock file.");
             dependencyGraph = podlockParser.extractDependencyGraph(podLockText);
             logger.trace("Finished creating the dependency graph from the pod lock file.");
-        } catch (final IOException e) {
+        } catch (final IOException | MissingExternalIdException e) {
             return new Extraction.Builder().exception(e).build();
         }
 

detectable/src/main/java/com/synopsys/integration/detectable/detectables/cocoapods/parser/PodlockParser.java

@@ -37,6 +37,7 @@
 import com.fasterxml.jackson.dataformat.yaml.YAMLMapper;
 import com.synopsys.integration.bdio.graph.DependencyGraph;
 import com.synopsys.integration.bdio.graph.builder.LazyExternalIdDependencyGraphBuilder;
+import com.synopsys.integration.bdio.graph.builder.MissingExternalIdException;
 import com.synopsys.integration.bdio.model.Forge;
 import com.synopsys.integration.bdio.model.dependencyid.DependencyId;
 import com.synopsys.integration.bdio.model.dependencyid.NameDependencyId;
@@ -57,7 +58,7 @@ public PodlockParser(final ExternalIdFactory externalIdFactory) {
         this.externalIdFactory = externalIdFactory;
     }
 
-    public DependencyGraph extractDependencyGraph(final String podLockText) throws IOException {
+    public DependencyGraph extractDependencyGraph(final String podLockText) throws IOException, MissingExternalIdException {
         final LazyExternalIdDependencyGraphBuilder lazyBuilder = new LazyExternalIdDependencyGraphBuilder();
         final YAMLMapper mapper = new YAMLMapper();
         final PodfileLock podfileLock = mapper.readValue(podLockText, PodfileLock.class);

detectable/src/main/java/com/synopsys/integration/detectable/detectables/cran/parse/PackratLockFileParser.java

@@ -30,6 +30,7 @@
 
 import com.synopsys.integration.bdio.graph.DependencyGraph;
 import com.synopsys.integration.bdio.graph.builder.LazyExternalIdDependencyGraphBuilder;
+import com.synopsys.integration.bdio.graph.builder.MissingExternalIdException;
 import com.synopsys.integration.bdio.model.Forge;
 import com.synopsys.integration.bdio.model.dependencyid.DependencyId;
 import com.synopsys.integration.bdio.model.dependencyid.NameDependencyId;
@@ -49,7 +50,7 @@ public PackratLockFileParser(final ExternalIdFactory externalIdFactory) {
         this.externalIdFactory = externalIdFactory;
     }
 
-    public DependencyGraph parseProjectDependencies(final List<String> packratLockContents) {
+    public DependencyGraph parseProjectDependencies(final List<String> packratLockContents) throws MissingExternalIdException {
         final LazyExternalIdDependencyGraphBuilder graphBuilder = new LazyExternalIdDependencyGraphBuilder();
 
         DependencyId currentParent = null;

detectable/src/main/java/com/synopsys/integration/detectable/detectables/docker/DockerExtractor.java

@@ -174,14 +174,14 @@ private Extraction.Builder findCodeLocations(final File directoryToSearch, final
             }
 
             if (simpleBdioDocument != null) {
-                final DependencyGraph dependencyGraph = bdioTransformer.transformToDependencyGraph(simpleBdioDocument.project, simpleBdioDocument.components);
+                final DependencyGraph dependencyGraph = bdioTransformer.transformToDependencyGraph(simpleBdioDocument.getProject(), simpleBdioDocument.getComponents());
 
-                final String projectName = simpleBdioDocument.project.name;
-                final String projectVersionName = simpleBdioDocument.project.version;
+                final String projectName = simpleBdioDocument.getProject().name;
+                final String projectVersionName = simpleBdioDocument.getProject().version;
 
                 // TODO ejk - update this when project external id is not req'd anymore
-                final Forge dockerForge = new Forge(BdioId.BDIO_ID_SEPARATOR, simpleBdioDocument.project.bdioExternalIdentifier.forge);
-                final String externalIdPath = simpleBdioDocument.project.bdioExternalIdentifier.externalId;
+                final Forge dockerForge = new Forge(BdioId.BDIO_ID_SEPARATOR, simpleBdioDocument.getProject().bdioExternalIdentifier.forge);
+                final String externalIdPath = simpleBdioDocument.getProject().bdioExternalIdentifier.externalId;
                 final ExternalId projectExternalId = externalIdFactory.createPathExternalId(dockerForge, externalIdPath);
 
                 final CodeLocation detectCodeLocation = new CodeLocation(dependencyGraph, projectExternalId);

detectable/src/main/java/com/synopsys/integration/detectable/detectables/go/vendor/parse/GoVendorJsonParser.java

@@ -39,21 +39,21 @@
 
 public class GoVendorJsonParser {
     private final Logger logger = LoggerFactory.getLogger(this.getClass());
-    private ExternalIdFactory externalIdFactory;
+    private final ExternalIdFactory externalIdFactory;
 
     public GoVendorJsonParser(final ExternalIdFactory externalIdFactory) {
         this.externalIdFactory = externalIdFactory;
     }
 
     public DependencyGraph parseVendorJson(final Gson gson, final String vendorJsonContents) {
         final MutableDependencyGraph graph = new MutableMapDependencyGraph();
-        VendorJson vendorJsonData = gson.fromJson(vendorJsonContents, VendorJson.class);
+        final VendorJson vendorJsonData = gson.fromJson(vendorJsonContents, VendorJson.class);
         logger.trace(String.format("vendorJsonData: %s", vendorJsonData));
-        for (PackageData pkg : vendorJsonData.getPackages()) {
+        for (final PackageData pkg : vendorJsonData.getPackages()) {
             if (StringUtils.isNotBlank(pkg.getPath()) && StringUtils.isNotBlank(pkg.getRevision())) {
                 final ExternalId dependencyExternalId = externalIdFactory.createNameVersionExternalId(Forge.GOLANG, pkg.getPath(), pkg.getRevision());
                 final Dependency dependency = new Dependency(pkg.getPath(), pkg.getRevision(), dependencyExternalId);
-                logger.trace(String.format("dependency: %s", dependency.externalId.toString()));
+                logger.trace(String.format("dependency: %s", dependency.getExternalId().toString()));
                 graph.addChildToRoot(dependency);
             } else {
                 logger.debug(String.format("Omitting package path:'%s', revision:'%s' (one or both of path, revision is/are missing)", pkg.getPath(), pkg.getRevision()));

detectable/src/main/java/com/synopsys/integration/detectable/detectables/hex/parse/Rebar3TreeParser.java

@@ -91,8 +91,8 @@ public RebarParseResult parseRebarTreeOutput(final List<String> dependencyTreeOu
             final CodeLocation codeLocation = new CodeLocation(graph);
             return new RebarParseResult(codeLocation);
         } else {
-            final CodeLocation codeLocation = new CodeLocation(graph, project.externalId);
-            return new RebarParseResult(new NameVersion(project.name, project.version), codeLocation);
+            final CodeLocation codeLocation = new CodeLocation(graph, project.getExternalId());
+            return new RebarParseResult(new NameVersion(project.getName(), project.getVersion()), codeLocation);
         }
     }
 

detectable/src/main/java/com/synopsys/integration/detectable/detectables/maven/cli/MavenCodeLocationPackager.java

@@ -137,11 +137,14 @@ public List<MavenParseResult> extractCodeLocations(final String sourcePath, fina
                 if (level == 1) {
                     // a direct dependency, clear the stack and add this as a potential parent for the next line
                     if (scopeFilter.shouldInclude(dependency.scope)) {
-                        logger.trace(String.format("Level 1 component %s:%s:%s:%s is in scope; adding it to hierarchy root", dependency.externalId.group, dependency.externalId.name, dependency.externalId.version, dependency.scope));
+                        logger.trace(String
+                                         .format("Level 1 component %s:%s:%s:%s is in scope; adding it to hierarchy root", dependency.getExternalId().getGroup(), dependency.getExternalId().getName(), dependency.getExternalId().getVersion(),
+                                             dependency.scope));
                         currentGraph.addChildToRoot(dependency);
                         inOutOfScopeTree = false;
                     } else {
-                        logger.trace(String.format("Level 1 component %s:%s:%s:%s is a top-level out-of-scope component; entering non-scoped tree", dependency.externalId.group, dependency.externalId.name, dependency.externalId.version,
+                        logger.trace(String.format("Level 1 component %s:%s:%s:%s is a top-level out-of-scope component; entering non-scoped tree", dependency.getExternalId().getGroup(), dependency.getExternalId().getName(),
+                            dependency.getExternalId().getVersion(),
                             dependency.scope));
                         inOutOfScopeTree = true;
                     }
@@ -178,10 +181,10 @@ private void addOrphansToGraph(final MutableDependencyGraph graph, final List<De
         logger.trace(String.format("# orphans: %d", orphans.size()));
         if (orphans.size() > 0) {
             final Dependency orphanListParent = createOrphanListParentDependency();
-            logger.trace(String.format("adding orphan list parent dependency: %s", orphanListParent.externalId.toString()));
+            logger.trace(String.format("adding orphan list parent dependency: %s", orphanListParent.getExternalId().toString()));
             graph.addChildToRoot(orphanListParent);
             for (final Dependency dependency : orphans) {
-                logger.trace(String.format("adding orphan: %s", dependency.externalId.toString()));
+                logger.trace(String.format("adding orphan: %s", dependency.getExternalId().toString()));
                 graph.addParentWithChild(orphanListParent, dependency);
             }
         }
@@ -192,10 +195,12 @@ private void addDependencyIfInScope(final MutableDependencyGraph currentGraph, f
         if (scopeFilter.shouldInclude(dependency.scope)) {
             if (inOutOfScopeTree) {
                 logger.trace(
-                    String.format("component %s:%s:%s:%s is in scope but in a nonScope tree; adding it to orphans", dependency.externalId.group, dependency.externalId.name, dependency.externalId.version, dependency.scope));
+                    String.format("component %s:%s:%s:%s is in scope but in a nonScope tree; adding it to orphans", dependency.getExternalId().getGroup(), dependency.getExternalId().getName(), dependency.getExternalId().getVersion(),
+                        dependency.scope));
                 orphans.add(dependency);
             } else {
-                logger.trace(String.format("component %s:%s:%s:%s is in scope and in an in-scope tree; adding it to hierarchy", dependency.externalId.group, dependency.externalId.name, dependency.externalId.version, dependency.scope));
+                logger.trace(String.format("component %s:%s:%s:%s is in scope and in an in-scope tree; adding it to hierarchy", dependency.getExternalId().getGroup(), dependency.getExternalId().getName(),
+                    dependency.getExternalId().getVersion(), dependency.scope));
                 currentGraph.addParentWithChild(parent, dependency);
             }
         }
@@ -205,11 +210,11 @@ private MavenParseResult createMavenParseResult(final String sourcePath, final S
         final Dependency dependency = textToProject(line);
         if (null != dependency) {
             String codeLocationSourcePath = sourcePath;
-            if (!sourcePath.endsWith(dependency.name)) {
-                codeLocationSourcePath += "/" + dependency.name;
+            if (!sourcePath.endsWith(dependency.getName())) {
+                codeLocationSourcePath += "/" + dependency.getName();
             }
-            final CodeLocation codeLocation = new CodeLocation(graph, dependency.externalId, new File(codeLocationSourcePath));
-            return new MavenParseResult(dependency.name, dependency.version, codeLocation);
+            final CodeLocation codeLocation = new CodeLocation(graph, dependency.getExternalId(), new File(codeLocationSourcePath));
+            return new MavenParseResult(dependency.getName(), dependency.getVersion(), codeLocation);
         }
         return null;
     }

detectable/src/main/java/com/synopsys/integration/detectable/detectables/maven/cli/ScopedDependency.java

@@ -35,7 +35,7 @@ public class ScopedDependency extends Dependency {
     public ScopedDependency(final String name, final String version, final ExternalId externalId, final String scope) {
         super(name, version, externalId);
         if (scope == null) {
-            logger.warn(String.format("The scope for component %s:%s:%s is missing, which might produce inaccurate results", externalId.group, externalId.name, externalId.version));
+            logger.warn(String.format("The scope for component %s:%s:%s is missing, which might produce inaccurate results", externalId.getGroup(), externalId.getName(), externalId.getVersion()));
             this.scope = "";
         } else {
             this.scope = scope;

detectable/src/main/java/com/synopsys/integration/detectable/detectables/packagist/parse/PackagistParser.java

@@ -34,6 +34,7 @@
 import com.google.gson.JsonParser;
 import com.synopsys.integration.bdio.graph.DependencyGraph;
 import com.synopsys.integration.bdio.graph.builder.LazyExternalIdDependencyGraphBuilder;
+import com.synopsys.integration.bdio.graph.builder.MissingExternalIdException;
 import com.synopsys.integration.bdio.model.Forge;
 import com.synopsys.integration.bdio.model.dependencyid.NameDependencyId;
 import com.synopsys.integration.bdio.model.externalid.ExternalId;
@@ -55,7 +56,7 @@ public PackagistParser(final ExternalIdFactory externalIdFactory, final Composer
         this.composerLockDetectableOptions = composerLockDetectableOptions;
     }
 
-    public PackagistParseResult getDependencyGraphFromProject(final String composerJsonText, final String composerLockText) {
+    public PackagistParseResult getDependencyGraphFromProject(final String composerJsonText, final String composerLockText) throws MissingExternalIdException {
         final LazyExternalIdDependencyGraphBuilder builder = new LazyExternalIdDependencyGraphBuilder();
 
         final JsonObject composerJsonObject = new JsonParser().parse(composerJsonText).getAsJsonObject();

detectable/src/main/java/com/synopsys/integration/detectable/detectables/pip/parser/PipInspectorTreeParser.java

@@ -56,7 +56,6 @@ public PipInspectorTreeParser(final ExternalIdFactory externalIdFactory) {
         this.externalIdFactory = externalIdFactory;
     }
 
-
     public Optional<PipenvResult> parse(final List<String> pipInspectorOutputAsList, final String sourcePath) {
         PipenvResult parseResult = null;
 
@@ -94,8 +93,8 @@ public Optional<PipenvResult> parse(final List<String> pipInspectorOutputAsList,
         }
 
         if (project != null) {
-            final CodeLocation codeLocation = new CodeLocation(graph, project.externalId);
-            parseResult = new PipenvResult(project.name, project.version, codeLocation);
+            final CodeLocation codeLocation = new CodeLocation(graph, project.getExternalId());
+            parseResult = new PipenvResult(project.getName(), project.getVersion(), codeLocation);
         }
 
         return Optional.ofNullable(parseResult);
@@ -122,7 +121,8 @@ private Dependency parseDependencyFromLine(final String line, final String sourc
         String version = segments[1].trim();
         ExternalId externalId = externalIdFactory.createNameVersionExternalId(Forge.PYPI, name, version);
 
-        if (name.equals(UNKNOWN_PROJECT_NAME) || version.equals(UNKNOWN_PROJECT_VERSION)) {    //TODO: Pip needs some love. It shouldn't have to do this here but the change seems non-trivial. A code location with no external id should be created as such.
+        if (name.equals(UNKNOWN_PROJECT_NAME) || version.equals(
+            UNKNOWN_PROJECT_VERSION)) {    //TODO: Pip needs some love. It shouldn't have to do this here but the change seems non-trivial. A code location with no external id should be created as such.
             externalId = externalIdFactory.createPathExternalId(Forge.PYPI, sourcePath);
         }