Merge pull request #71 from blackducksoftware/sb_bazelPipeline

Sb bazel pipeline

detect-configuration/src/main/java/com/synopsys/integration/detect/configuration/DetectProperty.java

@@ -150,9 +150,9 @@ public enum DetectProperty {
     DETECT_BAZEL_TARGET("detect.bazel.target", "Bazel Target", "5.2.0", PropertyType.STRING, PropertyAuthority.NONE),
 
     @HelpGroup(primary = GROUP_BAZEL, additional = GROUP_SOURCE_SCAN)
-    @HelpDescription("The path to a file containing a list of BazelExternalIdExtractionXPathRule objects in json for overriding the default behavior).")
-    @HelpDetailed("This property is normally not set, but could potentially be used to customize the Bazel detector.")
-    DETECT_BAZEL_ADVANCED_RULES_PATH("detect.bazel.advanced.rules.path", "Bazel Advanced Rules File", "5.2.0", PropertyType.STRING, PropertyAuthority.NONE),
+    @HelpDescription("The Bazel workspace rule used to pull in external dependencies. If not set, Detect will attempt to determine the rule from the contents of the WORKSPACE file.")
+    @AcceptableValues(value = { "maven_jar", "maven_install", "UNSPECIFIED" }, caseSensitive = true, strict = true)
+    DETECT_BAZEL_DEPENDENCY_RULE("detect.bazel.dependency.type", "Bazel workspace external dependency rule", "6.0.0", PropertyType.STRING, PropertyAuthority.NONE, "UNSPECIFIED"),
 
     @HelpGroup(primary = GROUP_PATHS, additional = { SEARCH_GROUP_GLOBAL })
     @HelpDescription("The path to the output directory for all BDIO files.")

detectable/src/main/java/com/synopsys/integration/detectable/detectables/bazel/BazelDependencyParser.java

@@ -0,0 +1,51 @@
+/**
+ * detectable
+ *
+ * Copyright (c) 2019 Synopsys, Inc.
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package com.synopsys.integration.detectable.detectables.bazel;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.synopsys.integration.bdio.model.dependency.Dependency;
+import com.synopsys.integration.bdio.model.externalid.ExternalId;
+import com.synopsys.integration.bdio.model.externalid.ExternalIdFactory;
+
+public class BazelDependencyParser {
+    private final Logger logger = LoggerFactory.getLogger(this.getClass());
+    private final ExternalIdFactory externalIdFactory;
+
+    public BazelDependencyParser(final ExternalIdFactory externalIdFactory) {
+        this.externalIdFactory = externalIdFactory;
+    }
+
+    public Dependency gavStringToDependency(final String artifactString, final String separatorRegex) {
+        final String[] gavParts = artifactString.split(separatorRegex);
+        final String group = gavParts[0];
+        final String artifact = gavParts[1];
+        final String version = gavParts[2];
+
+        logger.debug(String.format("Adding dependency from external id: %s:%s:%s", group, artifact, version));
+        final ExternalId externalId = externalIdFactory.createMavenExternalId(group, artifact, version);
+        final Dependency artifactDependency = new Dependency(artifact, version, externalId);
+        return artifactDependency;
+    }
+}

detectable/src/main/java/com/synopsys/integration/detectable/detectables/bazel/BazelDetectable.java

@@ -25,30 +25,33 @@
 import java.io.File;
 
 import org.apache.commons.lang3.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 import com.synopsys.integration.detectable.Detectable;
 import com.synopsys.integration.detectable.DetectableEnvironment;
 import com.synopsys.integration.detectable.Extraction;
 import com.synopsys.integration.detectable.ExtractionEnvironment;
 import com.synopsys.integration.detectable.detectable.exception.DetectableException;
 import com.synopsys.integration.detectable.detectable.executable.resolver.BazelResolver;
+import com.synopsys.integration.detectable.detectable.file.FileFinder;
 import com.synopsys.integration.detectable.detectable.result.DetectableResult;
 import com.synopsys.integration.detectable.detectable.result.ExecutableNotFoundDetectableResult;
+import com.synopsys.integration.detectable.detectable.result.FilesNotFoundDetectableResult;
 import com.synopsys.integration.detectable.detectable.result.PassedDetectableResult;
 import com.synopsys.integration.detectable.detectable.result.PropertyInsufficientDetectableResult;
 
 public class BazelDetectable extends Detectable {
-    private final Logger logger = LoggerFactory.getLogger(this.getClass());
+    public static final String WORKSPACE_FILENAME = "WORKSPACE";
+    private final FileFinder fileFinder;
     private final BazelExtractor bazelExtractor;
     private final BazelResolver bazelResolver;
     private final BazelDetectableOptions bazelDetectableOptions;
     private File bazelExe;
+    private BazelWorkspace bazelWorkspace;
 
-    public BazelDetectable(final DetectableEnvironment environment, final BazelExtractor bazelExtractor,
+    public BazelDetectable(final DetectableEnvironment environment, final FileFinder fileFinder, final BazelExtractor bazelExtractor,
         final BazelResolver bazelResolver, final BazelDetectableOptions bazelDetectableOptions) {
         super(environment, "bazel", "bazel");
+        this.fileFinder = fileFinder;
         this.bazelExtractor = bazelExtractor;
         this.bazelResolver = bazelResolver;
         this.bazelDetectableOptions = bazelDetectableOptions;
@@ -64,6 +67,11 @@ public DetectableResult applicable() {
 
     @Override
     public DetectableResult extractable() throws DetectableException {
+        final File workspaceFile = fileFinder.findFile(environment.getDirectory(), WORKSPACE_FILENAME);
+        if (workspaceFile == null) {
+            return new FilesNotFoundDetectableResult(WORKSPACE_FILENAME);
+        }
+        bazelWorkspace = new BazelWorkspace(workspaceFile);
         bazelExe = bazelResolver.resolveBazel();
         if (bazelExe == null) {
             return new ExecutableNotFoundDetectableResult("bazel");
@@ -73,7 +81,8 @@ public DetectableResult extractable() throws DetectableException {
 
     @Override
     public Extraction extract(ExtractionEnvironment extractionEnvironment) {
-        Extraction extractResult = bazelExtractor.extract(bazelExe, environment.getDirectory(), bazelDetectableOptions.getTargetName(), bazelDetectableOptions.getFullRulesPath());
+        final BazelProjectNameGenerator projectNameGenerator = new BazelProjectNameGenerator();
+        Extraction extractResult = bazelExtractor.extract(bazelExe, environment.getDirectory(), bazelWorkspace, bazelDetectableOptions.getTargetName(), projectNameGenerator, bazelDetectableOptions.getBazelDependencyRule());
         return extractResult;
     }
 }

detectable/src/main/java/com/synopsys/integration/detectable/detectables/bazel/BazelDetectableOptions.java

@@ -24,17 +24,17 @@
 
 public class BazelDetectableOptions {
     private final String targetName;
-    private final String fullRulesPath;
-    public BazelDetectableOptions(final String targetName, final String fullRulesPath) {
+    private final String bazelDependencyRule;
+    public BazelDetectableOptions(final String targetName, final String bazelDependencyRule) {
         this.targetName = targetName;
-        this.fullRulesPath = fullRulesPath;
+        this.bazelDependencyRule = bazelDependencyRule;
     }
 
     public String getTargetName() {
         return targetName;
     }
 
-    public String getFullRulesPath() {
-        return fullRulesPath;
+    public String getBazelDependencyRule() {
+        return bazelDependencyRule;
     }
 }

detectable/src/main/java/com/synopsys/integration/detectable/detectables/bazel/BazelExtractor.java

@@ -23,68 +23,60 @@
 package com.synopsys.integration.detectable.detectables.bazel;
 
 import java.io.File;
-import java.io.IOException;
-import java.util.Collection;
+import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
-import java.util.stream.Collectors;
 
-import org.apache.commons.lang3.StringUtils;
+import org.jetbrains.annotations.NotNull;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.synopsys.integration.bdio.graph.MutableDependencyGraph;
+import com.synopsys.integration.bdio.graph.MutableMapDependencyGraph;
+import com.synopsys.integration.bdio.model.dependency.Dependency;
 import com.synopsys.integration.detectable.Extraction;
 import com.synopsys.integration.detectable.detectable.codelocation.CodeLocation;
 import com.synopsys.integration.detectable.detectable.executable.ExecutableRunner;
-import com.synopsys.integration.detectable.detectables.bazel.model.BazelExternalIdExtractionFullRule;
-import com.synopsys.integration.detectable.detectables.bazel.model.BazelExternalIdExtractionFullRuleJsonProcessor;
-import com.synopsys.integration.detectable.detectables.bazel.model.BazelExternalIdExtractionSimpleRules;
-import com.synopsys.integration.detectable.detectables.bazel.parse.BazelCodeLocationBuilder;
-import com.synopsys.integration.detectable.detectables.bazel.parse.BazelExternalIdGenerator;
-import com.synopsys.integration.detectable.detectables.bazel.parse.BazelQueryXmlOutputParser;
-import com.synopsys.integration.detectable.detectables.bazel.parse.RuleConverter;
+import com.synopsys.integration.detectable.detectables.bazel.pipeline.WorkspaceRuleChooser;
+import com.synopsys.integration.detectable.detectables.bazel.pipeline.Pipelines;
+import com.synopsys.integration.detectable.detectables.bazel.pipeline.stepexecutor.BazelCommandExecutor;
+import com.synopsys.integration.detectable.detectables.bazel.pipeline.stepexecutor.StepExecutor;
+import com.synopsys.integration.detectable.detectables.bazel.pipeline.stepexecutor.BazelVariableSubstitutor;
 
 public class BazelExtractor {
     private final Logger logger = LoggerFactory.getLogger(this.getClass());
     private final ExecutableRunner executableRunner;
-    private final BazelQueryXmlOutputParser parser;
-    private final BazelCodeLocationBuilder codeLocationGenerator;
-    private final BazelExternalIdExtractionFullRuleJsonProcessor bazelExternalIdExtractionFullRuleJsonProcessor;
+    private final BazelDependencyParser bazelDependencyParser;
+    private final WorkspaceRuleChooser workspaceRuleChooser;
 
-    public BazelExtractor(final ExecutableRunner executableRunner, BazelQueryXmlOutputParser parser,
-        final BazelCodeLocationBuilder codeLocationGenerator, final BazelExternalIdExtractionFullRuleJsonProcessor bazelExternalIdExtractionFullRuleJsonProcessor) {
+    public BazelExtractor(final ExecutableRunner executableRunner,
+        final BazelDependencyParser bazelDependencyParser, final WorkspaceRuleChooser workspaceRuleChooser) {
         this.executableRunner = executableRunner;
-        this.parser = parser;
-        this.codeLocationGenerator = codeLocationGenerator;
-        this.bazelExternalIdExtractionFullRuleJsonProcessor = bazelExternalIdExtractionFullRuleJsonProcessor;
+        this.bazelDependencyParser = bazelDependencyParser;
+        this.workspaceRuleChooser = workspaceRuleChooser;
     }
 
-    //TODO: Limit 'extractors' to 'execute' and 'read', delegate all other work.
-    public Extraction extract(final File bazelExe, final File workspaceDir, String bazelTarget, String fullRulesPath) {
-        logger.debug("Bazel extractAndPublishResults()");
+    public Extraction extract(final File bazelExe, final File workspaceDir, final BazelWorkspace bazelWorkspace, final String bazelTarget,
+        final BazelProjectNameGenerator bazelProjectNameGenerator, final String providedBazelDependencyType) {
+        logger.debug("Bazel extraction:");
         try {
-            codeLocationGenerator.setWorkspaceDir(workspaceDir);
-            List<BazelExternalIdExtractionFullRule> fullRules;
-            if (StringUtils.isNotBlank(fullRulesPath)) {
-                fullRules = loadXPathRulesFromFile(fullRulesPath);
-                logger.debug(String.format("Read %d rule(s) from %s", fullRules.size(), fullRulesPath));
-            } else {
-                BazelExternalIdExtractionSimpleRules simpleRules = new BazelExternalIdExtractionSimpleRules(fullRulesPath);
-                fullRules = simpleRules.getRules().stream()
-                                      .map(RuleConverter::simpleToFull).collect(Collectors.toList());
-                if (logger.isDebugEnabled()) {
-                    logger.debug(String.format("Using default rules:\n%s", bazelExternalIdExtractionFullRuleJsonProcessor.toJson(fullRules)));
-                }
-            }
-            BazelExternalIdGenerator externalIdGenerator = new BazelExternalIdGenerator(executableRunner, bazelExe.toString(), parser, workspaceDir, bazelTarget);
-            fullRules.stream()
-                .map(externalIdGenerator::generate)
-                .flatMap(Collection::stream)
-                .forEach(codeLocationGenerator::addDependency);
-            if (externalIdGenerator.isErrors()) {
-                return new Extraction.Builder().failure(externalIdGenerator.getErrorMessage()).build();
+            final WorkspaceRule ruleFromWorkspaceFile = bazelWorkspace.getDependencyRule();
+            final BazelCommandExecutor bazelCommandExecutor = new BazelCommandExecutor(executableRunner, workspaceDir, bazelExe);
+            final BazelVariableSubstitutor bazelVariableSubstitutor = new BazelVariableSubstitutor(bazelTarget);
+            final Pipelines pipelines = new Pipelines(bazelCommandExecutor, bazelVariableSubstitutor);
+            final WorkspaceRule workspaceRule = workspaceRuleChooser.choose(ruleFromWorkspaceFile, providedBazelDependencyType);
+            final List<StepExecutor> pipeline = pipelines.get(workspaceRule);
+
+            // Execute pipeline steps (like linux cmd piping with '|'); each step processes the output of the previous step
+            List<String> pipelineData = new ArrayList<>();
+            for (final StepExecutor pipelineStep : pipeline) {
+                pipelineData = pipelineStep.process(pipelineData);
             }
-            final List<CodeLocation> codeLocations = codeLocationGenerator.build();
-            final String projectName = cleanProjectName(bazelTarget);
+            // final pipelineData is a list of group:artifact:version strings
+            final MutableDependencyGraph dependencyGraph = gavStringsToDependencyGraph(pipelineData);
+            final CodeLocation codeLocation = new CodeLocation(dependencyGraph);
+            final List<CodeLocation> codeLocations = Arrays.asList(codeLocation);
+            final String projectName = bazelProjectNameGenerator.generateFromBazelTarget(bazelTarget);
             final Extraction.Builder builder = new Extraction.Builder()
                                                    .success(codeLocations)
                                                    .projectName(projectName);
@@ -96,18 +88,17 @@ public Extraction extract(final File bazelExe, final File workspaceDir, String b
         }
     }
 
-    private String cleanProjectName(final String bazelTarget) {
-        String projectName = bazelTarget
-                                 .replaceAll("^//", "")
-                   .replaceAll("^:", "")
-                   .replaceAll("/", "_")
-                   .replaceAll(":", "_");
-        return projectName;
-    }
-
-    private List<BazelExternalIdExtractionFullRule> loadXPathRulesFromFile(final String xPathRulesJsonFilePath) throws IOException {
-        final File jsonFile = new File(xPathRulesJsonFilePath);
-        List<BazelExternalIdExtractionFullRule> loadedRules = bazelExternalIdExtractionFullRuleJsonProcessor.load(jsonFile);
-        return loadedRules;
+    @NotNull
+    private MutableDependencyGraph gavStringsToDependencyGraph(final List<String> gavStrings) {
+        final MutableDependencyGraph dependencyGraph  = new MutableMapDependencyGraph();
+        for (String gavString : gavStrings) {
+            final Dependency artifactDependency = bazelDependencyParser.gavStringToDependency(gavString, ":");
+            try {
+                dependencyGraph.addChildToRoot(artifactDependency);
+            } catch (final Exception e) {
+                logger.error(String.format("Unable to create dependency from %s", gavString));
+            }
+        }
+        return dependencyGraph;
     }
 }

detectable/src/main/java/com/synopsys/integration/detectable/detectables/bazel/model/SearchReplacePattern.java → detectable/src/main/java/com/synopsys/integration/detectable/detectables/bazel/BazelProjectNameGenerator.java

@@ -20,22 +20,16 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package com.synopsys.integration.detectable.detectables.bazel.model;
+package com.synopsys.integration.detectable.detectables.bazel;
 
-public class SearchReplacePattern {
-    private final String searchRegex;
-    private final String replacementString;
+public class BazelProjectNameGenerator {
 
-    public SearchReplacePattern(final String searchRegex, final String replacementString) {
-        this.searchRegex = searchRegex;
-        this.replacementString = replacementString;
-    }
-
-    public String getSearchRegex() {
-        return searchRegex;
-    }
-
-    public String getReplacementString() {
-        return replacementString;
+    public String generateFromBazelTarget(final String bazelTarget) {
+        String projectName = bazelTarget
+                                 .replaceAll("^//", "")
+                                 .replaceAll("^:", "")
+                                 .replaceAll("/", "_")
+                                 .replaceAll(":", "_");
+        return projectName;
     }
 }