Skip to content

Support Universe Query in Bazel-Diff command #67

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Jun 2, 2021
Merged

Support Universe Query in Bazel-Diff command #67

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c2ea71f
Support Universe Query in Bazel-Diff command
tinder-maxwellelliott Jun 2, 2021
5985c16
revert change
tinder-maxwellelliott Jun 2, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 0 additions & 29 deletions integration/WORKSPACE
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,32 +19,3 @@ maven_install(
"https://jcenter.bintray.com/",
]
)

http_archive(
name = "io_bazel_rules_docker",
sha256 = "1698624e878b0607052ae6131aa216d45ebb63871ec497f26c67455b34119c80",
strip_prefix = "rules_docker-0.15.0",
urls = ["https://github.com/bazelbuild/rules_docker/releases/download/v0.15.0/rules_docker-v0.15.0.tar.gz"],
)

load(
"@io_bazel_rules_docker//repositories:repositories.bzl",
container_repositories = "repositories",
)
container_repositories()

load("@io_bazel_rules_docker//repositories:deps.bzl", container_deps = "deps")

container_deps()

load(
"@io_bazel_rules_docker//container:container.bzl",
"container_pull",
)

container_pull(
name = "openjdk_11_slim",
digest = "sha256:28b59dc9a129c75349418e3b75508fd8eef3b33dd7d796079d1d19445d907776",
registry = "index.docker.io",
repository = "library/openjdk",
)
9 changes: 4 additions & 5 deletions integration/integration_test.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,7 @@ modified_filepaths_output="$workspace_path/modified_filepaths.txt"
starting_hashes_json="$output_dir/starting_hashes.json"
final_hashes_json="$output_dir/final_hashes.json"
impacted_targets_path="$output_dir/impacted_targets.txt"
shared_flags="--config=verbose"
command_options="--incompatible_restrict_string_escapes=false"
shared_flags=""

export USE_BAZEL_VERSION=last_downstream_green

Expand All @@ -22,13 +21,13 @@ containsElement () {
return 1
}

$bazel_path run :bazel-diff $shared_flags -- generate-hashes -w $workspace_path -b $bazel_path $starting_hashes_json -co $command_options
$bazel_path run :bazel-diff $shared_flags -- generate-hashes -w $workspace_path -b $bazel_path $starting_hashes_json

$bazel_path run :bazel-diff $shared_flags -- generate-hashes -w $workspace_path -b $bazel_path -m $modified_filepaths_output $final_hashes_json -co $command_options
$bazel_path run :bazel-diff $shared_flags -- generate-hashes -w $workspace_path -b $bazel_path -m $modified_filepaths_output $final_hashes_json

awk '{gsub(/:StringGenerator.java": \"\w+\"/,"modifiedhash");print}' $final_hashes_json > /dev/null

$bazel_path run :bazel-diff $shared_flags -- -sh $starting_hashes_json -fh $final_hashes_json -w $workspace_path -b $bazel_path -o $impacted_targets_path -aq "attr('tags', 'manual', //...)" -co $command_options
$bazel_path run :bazel-diff $shared_flags -- -sh $starting_hashes_json -fh $final_hashes_json -w $workspace_path -b $bazel_path -o $impacted_targets_path -aq "attr('tags', 'manual', //...)"

IFS=$'\n' read -d '' -r -a impacted_targets < $impacted_targets_path
target1="//test/java/com/integration:bazel-diff-integration-test-lib"
Expand Down
6 changes: 3 additions & 3 deletions src/main/java/com/bazel_diff/BazelClient.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@

interface BazelClient {
List<BazelTarget> queryAllTargets() throws IOException;
Set<String> queryForImpactedTargets(Set<String> impactedTargets, String avoidQuery) throws IOException;
Set<String> queryForImpactedTargets(Set<String> impactedTargets, String avoidQuery, String universeQuery) throws IOException;
Set<BazelSourceFileTarget> convertFilepathsToSourceTargets(Set<Path> filepaths) throws IOException, NoSuchAlgorithmException;
Set<BazelSourceFileTarget> queryAllSourcefileTargets() throws IOException, NoSuchAlgorithmException;
}
Expand All @@ -47,12 +47,12 @@ public List<BazelTarget> queryAllTargets() throws IOException {
}

@Override
public Set<String> queryForImpactedTargets(Set<String> impactedTargets, String avoidQuery) throws IOException {
public Set<String> queryForImpactedTargets(Set<String> impactedTargets, String avoidQuery, String universeQuery) throws IOException {
Set<String> impactedTargetNames = new HashSet<>();
String targetQuery = impactedTargets.stream()
.map(target -> String.format("'%s'", target))
.collect(Collectors.joining(" + "));
String query = query = String.format("rdeps(//... except '//external:all-targets', %s)", targetQuery);
String query = query = String.format("rdeps(%s except '//external:all-targets', %s)", universeQuery, targetQuery);
if (avoidQuery != null) {
query = String.format("(%s) except (%s)", query, avoidQuery);
}
Expand Down
5 changes: 3 additions & 2 deletions src/main/java/com/bazel_diff/TargetHashingClient.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
interface TargetHashingClient {
Map<String, String> hashAllBazelTargets(Set<Path> modifiedFilepaths, Set<Path> seedFilepaths) throws IOException, NoSuchAlgorithmException;
Map<String, String> hashAllBazelTargetsAndSourcefiles(Set<Path> seedFilepaths) throws IOException, NoSuchAlgorithmException;
Set<String> getImpactedTargets(Map<String, String> startHashes, Map<String, String> endHashes, String avoidQuery, Boolean hashAllTargets) throws IOException;
Set<String> getImpactedTargets(Map<String, String> startHashes, Map<String, String> endHashes, String avoidQuery, String universeQuery, Boolean hashAllTargets) throws IOException;
}

class TargetHashingClientImpl implements TargetHashingClient {
Expand Down Expand Up @@ -41,6 +41,7 @@ public Set<String> getImpactedTargets(
Map<String, String> startHashes,
Map<String, String> endHashes,
String avoidQuery,
String universeQuery,
Boolean hashAllTargets)
throws IOException {
Set<String> impactedTargets = new HashSet<>();
Expand All @@ -53,7 +54,7 @@ public Set<String> getImpactedTargets(
if (hashAllTargets != null && hashAllTargets && avoidQuery == null) {
return impactedTargets;
}
return bazelClient.queryForImpactedTargets(impactedTargets, avoidQuery);
return bazelClient.queryForImpactedTargets(impactedTargets, avoidQuery, universeQuery);
}

private byte[] createDigestForTarget(
Expand Down
23 changes: 19 additions & 4 deletions src/main/java/com/bazel_diff/main.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -153,6 +153,15 @@ public Integer call() {
versionProvider = VersionProvider.class
)
class BazelDiff implements Callable<Integer> {
@ArgGroup(exclusive = true)
Exclusive exclusive;
static class Exclusive {
@Option(names = {"-a", "--all-sourcefiles"}, description = "Experimental: Hash all sourcefile targets (instead of relying on --modifiedFilepaths), Warning: Performance may degrade from reading all source files")
Boolean hashAllSourcefiles;

@Option(names = {"-u", "--universeQuery"}, description = "The universe query to use when executing `rdeps()` queries, use this to limit the search scope of impacted targets i.e. ignore external targets and such")
String universeRdepsQuery;
}

@Option(names = {"-w", "--workspacePath"}, description = "Path to Bazel workspace directory.", scope = ScopeType.INHERIT, required = true)
Path workspacePath;
Expand All @@ -169,9 +178,6 @@ class BazelDiff implements Callable<Integer> {
@Option(names = {"-o", "--output"}, scope = ScopeType.LOCAL, description = "Filepath to write the impacted Bazel targets to, newline separated")
File outputPath;

@Option(names = {"-a", "--all-sourcefiles"}, description = "Experimental: Hash all sourcefile targets (instead of relying on --modifiedFilepaths), Warning: Performance may degrade from reading all source files")
Boolean hashAllSourcefiles;

@Option(names = {"-aq", "--avoid-query"}, scope = ScopeType.LOCAL, description = "A Bazel query string, any targets that pass this query will be removed from the returned set of targets")
String avoidQuery;

Expand Down Expand Up @@ -221,7 +227,16 @@ public Integer call() throws IOException {
Map<String, String > gsonHash = new HashMap<>();
Map<String, String> startingHashes = gson.fromJson(startingFileReader, gsonHash.getClass());
Map<String, String> finalHashes = gson.fromJson(finalFileReader, gsonHash.getClass());
Set<String> impactedTargets = hashingClient.getImpactedTargets(startingHashes, finalHashes, avoidQuery, hashAllSourcefiles);
Boolean shouldHashAllSourceFiles = false;
String universeQuery = "//...";
if (exclusive != null) {
if (exclusive.hashAllSourcefiles) {
shouldHashAllSourceFiles = exclusive.hashAllSourcefiles;
} else {
universeQuery = exclusive.universeRdepsQuery;
}
}
Set<String> impactedTargets = hashingClient.getImpactedTargets(startingHashes, finalHashes, avoidQuery, universeQuery, shouldHashAllSourceFiles);
try {
FileWriter myWriter = new FileWriter(outputPath);
myWriter.write(impactedTargets.stream().collect(Collectors.joining(System.lineSeparator())));
Expand Down
16 changes: 8 additions & 8 deletions test/java/com/bazel_diff/TargetHashingClientImplTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -163,7 +163,7 @@ public void hashAllBazelTargets_sourceTargets_modifiedSources_seedFilepaths() th

@Test
public void getImpactedTargets() throws IOException {
when(bazelClientMock.queryForImpactedTargets(anySet(), anyObject())).thenReturn(
when(bazelClientMock.queryForImpactedTargets(anySet(), anyObject(), anyObject())).thenReturn(
new HashSet<>(Arrays.asList("rule1", "rule3"))
);
TargetHashingClientImpl client = new TargetHashingClientImpl(bazelClientMock, filesClientMock);
Expand All @@ -174,7 +174,7 @@ public void getImpactedTargets() throws IOException {
hash2.put("rule1", "differentrule1hash");
hash2.put("rule2", "rule2hash");
hash2.put("rule3", "rule3hash");
Set<String> impactedTargets = client.getImpactedTargets(hash1, hash2, null, false);
Set<String> impactedTargets = client.getImpactedTargets(hash1, hash2, null, null, false);
Set<String> expectedSet = new HashSet<>();
expectedSet.add("rule1");
expectedSet.add("rule3");
Expand All @@ -183,7 +183,7 @@ public void getImpactedTargets() throws IOException {

@Test
public void getImpactedTargets_withAvoidQuery() throws IOException {
when(bazelClientMock.queryForImpactedTargets(anySet(), eq("some_query"))).thenReturn(
when(bazelClientMock.queryForImpactedTargets(anySet(), eq("some_query"), eq("universe_query"))).thenReturn(
new HashSet<>(Arrays.asList("rule1"))
);
TargetHashingClientImpl client = new TargetHashingClientImpl(bazelClientMock, filesClientMock);
Expand All @@ -194,15 +194,15 @@ public void getImpactedTargets_withAvoidQuery() throws IOException {
hash2.put("rule1", "differentrule1hash");
hash2.put("rule2", "rule2hash");
hash2.put("rule3", "rule3hash");
Set<String> impactedTargets = client.getImpactedTargets(hash1, hash2, "some_query", false);
Set<String> impactedTargets = client.getImpactedTargets(hash1, hash2, "some_query", "universe_query", false);
Set<String> expectedSet = new HashSet<>();
expectedSet.add("rule1");
assertEquals(expectedSet, impactedTargets);
}

@Test
public void getImpactedTargets_withHashAllTargets() throws IOException {
when(bazelClientMock.queryForImpactedTargets(anySet(), anyObject())).thenReturn(
when(bazelClientMock.queryForImpactedTargets(anySet(), anyObject(), anyObject())).thenReturn(
new HashSet<>(Arrays.asList("rule1"))
);
TargetHashingClientImpl client = new TargetHashingClientImpl(bazelClientMock, filesClientMock);
Expand All @@ -213,7 +213,7 @@ public void getImpactedTargets_withHashAllTargets() throws IOException {
hash2.put("rule1", "differentrule1hash");
hash2.put("rule2", "rule2hash");
hash2.put("rule3", "rule3hash");
Set<String> impactedTargets = client.getImpactedTargets(hash1, hash2, null, true);
Set<String> impactedTargets = client.getImpactedTargets(hash1, hash2, null, null, true);
Set<String> expectedSet = new HashSet<>();
expectedSet.add("rule1");
expectedSet.add("rule3");
Expand All @@ -222,7 +222,7 @@ public void getImpactedTargets_withHashAllTargets() throws IOException {

@Test
public void getImpactedTargets_withHashAllTargets_withAvoidQuery() throws IOException {
when(bazelClientMock.queryForImpactedTargets(anySet(), eq("some_query"))).thenReturn(
when(bazelClientMock.queryForImpactedTargets(anySet(), eq("some_query"), anyObject())).thenReturn(
new HashSet<>(Arrays.asList("rule1"))
);
TargetHashingClientImpl client = new TargetHashingClientImpl(bazelClientMock, filesClientMock);
Expand All @@ -233,7 +233,7 @@ public void getImpactedTargets_withHashAllTargets_withAvoidQuery() throws IOExce
hash2.put("rule1", "differentrule1hash");
hash2.put("rule2", "rule2hash");
hash2.put("rule3", "rule3hash");
Set<String> impactedTargets = client.getImpactedTargets(hash1, hash2, "some_query", true);
Set<String> impactedTargets = client.getImpactedTargets(hash1, hash2, "some_query", null, true);
Set<String> expectedSet = new HashSet<>();
expectedSet.add("rule1");
assertEquals(expectedSet, impactedTargets);
Expand Down