Skip to content

Commit

Permalink
Merge 4231bdb into 055a88a
Browse files Browse the repository at this point in the history
  • Loading branch information
@stevebillings
stevebillings committed Jul 13, 2020
2 parents 055a88a + 4231bdb commit aa20476
Show file tree
Hide file tree
Showing 44 changed files with 23,143 additions and 754 deletions.
2 changes: 1 addition & 1 deletion ...src/main/java/com/synopsys/integration/configuration/config/InvalidPropertyException.java
View file
Expand Up @@ -24,7 +24,7 @@

import com.synopsys.integration.configuration.parse.ValueParseException;

public class InvalidPropertyException extends Exception {
public class InvalidPropertyException extends RuntimeException {
public InvalidPropertyException(String propertyKey, String propertySourceName, ValueParseException innerException) {
super(String.format("The key %s in property source '%s' contained a value that could not be reasonably converted to the properties type. The exception was %s", propertyKey, propertySourceName,
innerException.getLocalizedMessage() == null ? innerException.getLocalizedMessage() : "Unknown"), innerException);
Expand Down
18 changes: 9 additions & 9 deletions .../src/main/java/com/synopsys/integration/detectable/detectables/bazel/BazelDetectable.java
View file
Expand Up @@ -48,8 +48,8 @@ public class BazelDetectable extends Detectable {
private File bazelExe;
private BazelWorkspace bazelWorkspace;

public BazelDetectable(final DetectableEnvironment environment, final FileFinder fileFinder, final BazelExtractor bazelExtractor,
final BazelResolver bazelResolver, final BazelDetectableOptions bazelDetectableOptions) {
public BazelDetectable(DetectableEnvironment environment, FileFinder fileFinder, BazelExtractor bazelExtractor,
BazelResolver bazelResolver, BazelDetectableOptions bazelDetectableOptions) {
super(environment);
this.fileFinder = fileFinder;
this.bazelExtractor = bazelExtractor;
Expand All @@ -67,7 +67,7 @@ public DetectableResult applicable() {

@Override
public DetectableResult extractable() throws DetectableException {
final File workspaceFile = fileFinder.findFile(environment.getDirectory(), WORKSPACE_FILENAME);
File workspaceFile = fileFinder.findFile(environment.getDirectory(), WORKSPACE_FILENAME);
if (workspaceFile == null) {
return new FilesNotFoundDetectableResult(WORKSPACE_FILENAME);
}
Expand All @@ -80,11 +80,11 @@ public DetectableResult extractable() throws DetectableException {
}

@Override
public Extraction extract(final ExtractionEnvironment extractionEnvironment) {
final BazelProjectNameGenerator projectNameGenerator = new BazelProjectNameGenerator();
@SuppressWarnings("OptionalGetWithoutIsPresent") // Checked in applicable.
final Extraction extractResult = bazelExtractor.extract(bazelExe, environment.getDirectory(), bazelWorkspace, bazelDetectableOptions.getTargetName().get(), projectNameGenerator, bazelDetectableOptions.getBazelDependencyRule(),
bazelDetectableOptions.getBazelCqueryAdditionalOptions());
return extractResult;
public Extraction extract(ExtractionEnvironment extractionEnvironment) {
BazelProjectNameGenerator projectNameGenerator = new BazelProjectNameGenerator();
// Checked in applicable.
return bazelExtractor
.extract(bazelExe, environment.getDirectory(), bazelWorkspace, bazelDetectableOptions.getTargetName().get(), projectNameGenerator, bazelDetectableOptions.getBazelDependencyRules(),
bazelDetectableOptions.getBazelCqueryAdditionalOptions());
}
}
17 changes: 9 additions & 8 deletions ...in/java/com/synopsys/integration/detectable/detectables/bazel/BazelDetectableOptions.java
View file
Expand Up @@ -24,28 +24,29 @@

import java.util.List;
import java.util.Optional;
import java.util.Set;

public class BazelDetectableOptions {
private final String targetName;
private final WorkspaceRule bazelDependencyRule;
private final Set<WorkspaceRule> bazelDependencyRules;
private final List<String> bazelCqueryAdditionalOptions;

public BazelDetectableOptions(final String targetName, final WorkspaceRule bazelDependencyRule,
final List<String> bazelCqueryAdditionalOptions) {
public BazelDetectableOptions(String targetName, Set<WorkspaceRule> bazelDependencyRules,
List<String> bazelCqueryAdditionalOptions) {
this.targetName = targetName;
this.bazelDependencyRule = bazelDependencyRule;
this.bazelDependencyRules = bazelDependencyRules;
this.bazelCqueryAdditionalOptions = bazelCqueryAdditionalOptions;
}

public Optional<String> getTargetName() {
return Optional.ofNullable(targetName);
}

public WorkspaceRule getBazelDependencyRule() {
return bazelDependencyRule;
}

public List<String> getBazelCqueryAdditionalOptions() {
return bazelCqueryAdditionalOptions;
}

public Set<WorkspaceRule> getBazelDependencyRules() {
return bazelDependencyRules;
}
}
82 changes: 60 additions & 22 deletions ...e/src/main/java/com/synopsys/integration/detectable/detectables/bazel/BazelExtractor.java
View file
Expand Up @@ -23,60 +23,98 @@
package com.synopsys.integration.detectable.detectables.bazel;

import java.io.File;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;

import org.jetbrains.annotations.NotNull;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.synopsys.integration.bdio.graph.MutableDependencyGraph;
import com.synopsys.integration.bdio.graph.MutableMapDependencyGraph;
import com.synopsys.integration.bdio.model.dependency.Dependency;
import com.synopsys.integration.bdio.model.externalid.ExternalIdFactory;
import com.synopsys.integration.detectable.Extraction;
import com.synopsys.integration.detectable.detectable.codelocation.CodeLocation;
import com.synopsys.integration.detectable.detectable.executable.ExecutableRunner;
import com.synopsys.integration.detectable.detectables.bazel.pipeline.Pipeline;
import com.synopsys.integration.detectable.detectables.bazel.pipeline.WorkspaceRuleChooser;
import com.synopsys.integration.detectable.detectables.bazel.pipeline.Pipelines;
import com.synopsys.integration.detectable.detectables.bazel.pipeline.WorkspaceRuleChooser;
import com.synopsys.integration.detectable.detectables.bazel.pipeline.step.BazelCommandExecutor;
import com.synopsys.integration.detectable.detectables.bazel.pipeline.step.BazelVariableSubstitutor;
import com.synopsys.integration.exception.IntegrationException;

public class BazelExtractor {
private final Logger logger = LoggerFactory.getLogger(this.getClass());
private final ExecutableRunner executableRunner;
private final ExternalIdFactory externalIdFactory;
private final WorkspaceRuleChooser workspaceRuleChooser;

public BazelExtractor(final ExecutableRunner executableRunner,
final ExternalIdFactory externalIdFactory,
final WorkspaceRuleChooser workspaceRuleChooser) {
public BazelExtractor(ExecutableRunner executableRunner,
ExternalIdFactory externalIdFactory,
WorkspaceRuleChooser workspaceRuleChooser) {
this.executableRunner = executableRunner;
this.externalIdFactory = externalIdFactory;
this.workspaceRuleChooser = workspaceRuleChooser;
}

public Extraction extract(final File bazelExe, final File workspaceDir, final BazelWorkspace bazelWorkspace, final String bazelTarget,
final BazelProjectNameGenerator bazelProjectNameGenerator, final WorkspaceRule providedBazelDependencyType,
final List<String> providedCqueryAdditionalOptions) {
public Extraction extract(File bazelExe, File workspaceDir, BazelWorkspace bazelWorkspace, String bazelTarget,
BazelProjectNameGenerator bazelProjectNameGenerator, Set<WorkspaceRule> providedDependencyRuleTypes,
List<String> providedCqueryAdditionalOptions) {
logger.debug("Bazel extraction:");
try {
final WorkspaceRule ruleFromWorkspaceFile = bazelWorkspace.getDependencyRule();
final BazelCommandExecutor bazelCommandExecutor = new BazelCommandExecutor(executableRunner, workspaceDir, bazelExe);
final BazelVariableSubstitutor bazelVariableSubstitutor = new BazelVariableSubstitutor(bazelTarget, providedCqueryAdditionalOptions);
final Pipelines pipelines = new Pipelines(bazelCommandExecutor, bazelVariableSubstitutor, externalIdFactory);
final WorkspaceRule workspaceRule = workspaceRuleChooser.choose(ruleFromWorkspaceFile, providedBazelDependencyType);
final Pipeline pipeline = pipelines.get(workspaceRule);
final MutableDependencyGraph dependencyGraph = pipeline.run();
final CodeLocation codeLocation = new CodeLocation(dependencyGraph);
final List<CodeLocation> codeLocations = Arrays.asList(codeLocation);
final String projectName = bazelProjectNameGenerator.generateFromBazelTarget(bazelTarget);
final Extraction.Builder builder = new Extraction.Builder()
.success(codeLocations)
.projectName(projectName);
return builder.build();
BazelCommandExecutor bazelCommandExecutor = new BazelCommandExecutor(executableRunner, workspaceDir, bazelExe);
BazelVariableSubstitutor bazelVariableSubstitutor = new BazelVariableSubstitutor(bazelTarget, providedCqueryAdditionalOptions);
Pipelines pipelines = new Pipelines(bazelCommandExecutor, bazelVariableSubstitutor, externalIdFactory);
Set<WorkspaceRule> workspaceRulesToQuery = workspaceRuleChooser.choose(bazelWorkspace.getDependencyRuleTypes(), providedDependencyRuleTypes);
List<Dependency> aggregatedDependencies = collectDependencies(pipelines, workspaceRulesToQuery);
return buildResults(aggregatedDependencies, bazelProjectNameGenerator.generateFromBazelTarget(bazelTarget));
} catch (Exception e) {
final String msg = String.format("Bazel processing exception: %s", e.getMessage());
String msg = String.format("Bazel processing exception: %s", e.getMessage());
logger.debug(msg, e);
return new Extraction.Builder().failure(msg).build();
}
}

private Extraction buildResults(List<Dependency> aggregatedDependencies, String projectName) {
MutableDependencyGraph dependencyGraph = createDependencyGraph(aggregatedDependencies);
CodeLocation codeLocation = new CodeLocation(dependencyGraph);
List<CodeLocation> codeLocations = Arrays.asList(codeLocation);
Extraction.Builder builder = new Extraction.Builder()
.success(codeLocations)
.projectName(projectName);
return builder.build();
}

@NotNull
private List<Dependency> collectDependencies(Pipelines pipelines, Set<WorkspaceRule> workspaceRules) throws IntegrationException {
List<Dependency> aggregatedDependencies = new ArrayList<>();
// Make sure the order of processing deterministic
List<WorkspaceRule> sortedWorkspaceRules = workspaceRules.stream()
.sorted(Comparator.naturalOrder())
.collect(Collectors.toList());

for (WorkspaceRule workspaceRule : sortedWorkspaceRules) {
logger.info(String.format("Running processing pipeline for rule %s", workspaceRule));
Pipeline pipeline = pipelines.get(workspaceRule);
List<Dependency> ruleDependencies = pipeline.run();
logger.info(String.format("Number of dependencies discovered for rule %s: %d", workspaceRule, ruleDependencies.size()));
logger.debug(String.format("Dependencies discovered for rule %s: %s", workspaceRule, ruleDependencies));
aggregatedDependencies.addAll(ruleDependencies);
}
return aggregatedDependencies;
}

@NotNull
private MutableDependencyGraph createDependencyGraph(List<Dependency> aggregatedDependencies) {
MutableDependencyGraph dependencyGraph = new MutableMapDependencyGraph();
for (Dependency dependency : aggregatedDependencies) {
dependencyGraph.addChildToRoot(dependency);
}
return dependencyGraph;
}
}
38 changes: 18 additions & 20 deletions ...e/src/main/java/com/synopsys/integration/detectable/detectables/bazel/BazelWorkspace.java
View file
Expand Up @@ -25,46 +25,44 @@
import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;

import org.apache.commons.io.FileUtils;
import org.jetbrains.annotations.Nullable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class BazelWorkspace {
private final Logger logger = LoggerFactory.getLogger(this.getClass());
private final File workspaceFile;

public BazelWorkspace(final File workspaceFile) {
public BazelWorkspace(File workspaceFile) {
this.workspaceFile = workspaceFile;
}

public WorkspaceRule getDependencyRule() {
final List<String> workspaceFileLines;
public Set<WorkspaceRule> getDependencyRuleTypes() {
List<String> workspaceFileLines;
try {
// Assumes ascii or UTF-8, like other detectors
workspaceFileLines = FileUtils.readLines(workspaceFile, StandardCharsets.UTF_8);
} catch (IOException e) {
logger.debug(String.format("Unable to parse dependency rule from %s: %s", workspaceFile.getAbsolutePath(), e.getMessage()));
return WorkspaceRule.UNSPECIFIED;
return new HashSet<>(0);
}
final WorkspaceRule dependencyRule = parseDependencyRuleFromWorkspaceFileLines(workspaceFileLines);
return dependencyRule;

return workspaceFileLines.stream()
.flatMap(this::parseDependencyRulesFromWorkspaceFileLine)
.collect(Collectors.toSet());
}

@Nullable
public WorkspaceRule parseDependencyRuleFromWorkspaceFileLines(final List<String> workspaceFileLines) {
for (final String workspaceFileLine : workspaceFileLines) {
for (final WorkspaceRule workspaceRuleCandidate : WorkspaceRule.values()) {
if (workspaceFileLine.matches(String.format("^\\s*%s\\s*\\(", workspaceRuleCandidate.getName()))) {
final WorkspaceRule parsedDependencyRule = workspaceRuleCandidate;
logger.debug(String.format("Found workspace dependency rule: %s", parsedDependencyRule.getName()));
return parsedDependencyRule;
}
}
}
logger.debug("Unable to derive dependency rule from WORKSPACE file");
return WorkspaceRule.UNSPECIFIED;
@SuppressWarnings("java:S3864") // Sonar deems peek useful for debugging.
public Stream<WorkspaceRule> parseDependencyRulesFromWorkspaceFileLine(String workspaceFileLine) {
return Arrays.stream(WorkspaceRule.values())
.filter(workspaceRule -> workspaceFileLine.matches(String.format("^\\s*%s\\s*\\(", workspaceRule.getName())))
.peek(workspaceRule -> logger.debug(String.format("Found workspace dependency rule: %s", workspaceRule.getName())));
}
}
9 changes: 3 additions & 6 deletions ...le/src/main/java/com/synopsys/integration/detectable/detectables/bazel/WorkspaceRule.java
View file
Expand Up @@ -22,17 +22,14 @@
*/
package com.synopsys.integration.detectable.detectables.bazel;

import com.synopsys.integration.exception.IntegrationException;

public enum WorkspaceRule {
MAVEN_JAR("maven_jar"),
MAVEN_INSTALL("maven_install"),
HASKELL_CABAL_LIBRARY("haskell_cabal_library"),
UNSPECIFIED(null); //changed to UNSPECIFIED so property is simpler, if this causes confusion willing to discuss - jp
HASKELL_CABAL_LIBRARY("haskell_cabal_library");

private String name;
private final String name;

WorkspaceRule(final String name) {
WorkspaceRule(String name) {
this.name = name;
}

Expand Down
8 changes: 4 additions & 4 deletions ...rc/main/java/com/synopsys/integration/detectable/detectables/bazel/pipeline/Pipeline.java
View file
Expand Up @@ -25,7 +25,7 @@
import java.util.ArrayList;
import java.util.List;

import com.synopsys.integration.bdio.graph.MutableDependencyGraph;
import com.synopsys.integration.bdio.model.dependency.Dependency;
import com.synopsys.integration.detectable.detectables.bazel.pipeline.step.FinalStep;
import com.synopsys.integration.detectable.detectables.bazel.pipeline.step.IntermediateStep;
import com.synopsys.integration.exception.IntegrationException;
Expand All @@ -34,15 +34,15 @@ public class Pipeline {
private final List<IntermediateStep> intermediateSteps;
private final FinalStep finalStep;

public Pipeline(final List<IntermediateStep> intermediateSteps, final FinalStep finalStep) {
public Pipeline(List<IntermediateStep> intermediateSteps, FinalStep finalStep) {
this.intermediateSteps = intermediateSteps;
this.finalStep = finalStep;
}

public MutableDependencyGraph run() throws IntegrationException {
public List<Dependency> run() throws IntegrationException {
// Execute pipeline steps (like linux cmd piping with '|'); each step processes the output of the previous step
List<String> pipelineData = new ArrayList<>();
for (final IntermediateStep pipelineStep : intermediateSteps) {
for (IntermediateStep pipelineStep : intermediateSteps) {
pipelineData = pipelineStep.process(pipelineData);
}
return finalStep.finish(pipelineData);
Expand Down

0 comments on commit aa20476

Please sign in to comment.