Skip to content

Seed Hashfiles #66

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
May 26, 2021
Merged

Seed Hashfiles #66

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
180f8f3
Seed Hashfiles
tinder-maxwellelliott May 13, 2021
ae82083
update readme
tinder-maxwellelliott May 25, 2021
d849d30
new path syntax in tests
tinder-maxwellelliott May 25, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 24 additions & 16 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -134,31 +134,39 @@ Writes to the file the modified filepaths between two revisions.
`generate-hashes` Command

```terminal
Usage: bazel-diff generate-hashes [-hV] -b=<bazelPath>
[-co=<bazelCommandOptions>]
[-m=<modifiedFilepaths>]
Usage: bazel-diff generate-hashes [-m=<modifiedFilepaths> | -a] [-hV]
-b=<bazelPath> [-co=<bazelCommandOptions>]
[-s=<seedFilepaths>]
[-so=<bazelStartupOptions>]
-w=<workspacePath> <outputPath>
Writes to a file the SHA256 hashes for each Bazel Target in the provided
workspace.
<outputPath> The filepath to write the resulting JSON of dictionary
target => SHA-256 values
<outputPath> The filepath to write the resulting JSON of
dictionary target => SHA-256 values
-a, --all-sourcefiles Experimental: Hash all sourcefile targets (instead of
relying on --modifiedFilepaths), Warning:
Performance may degrade from reading all source
files
-b, --bazelPath=<bazelPath>
Path to Bazel binary
Path to Bazel binary
-co, --bazelCommandOptions=<bazelCommandOptions>
Additional space separated Bazel command options used when
invoking Bazel
-h, --help Show this help message and exit.
Additional space separated Bazel command options used
when invoking Bazel
-h, --help Show this help message and exit.
-m, --modifiedFilepaths=<modifiedFilepaths>
The path to a file containing the list of modified
filepaths in the workspace, you can use the
'modified-filepaths' command to get this list
The path to a file containing the list of modified
filepaths in the workspace, you can use the
'modified-filepaths' command to get this list
-s, --seed-filepaths=<seedFilepaths>
A text file containing a newline separated list of
filepaths, each of these filepaths will be read and
used as a seed for all targets.
-so, --bazelStartupOptions=<bazelStartupOptions>
Additional space separated Bazel client startup options
used when invoking Bazel
-V, --version Print version information and exit.
Additional space separated Bazel client startup
options used when invoking Bazel
-V, --version Print version information and exit.
-w, --workspacePath=<workspacePath>
Path to Bazel workspace directory.
Path to Bazel workspace directory.
```

## Installing
Expand Down
21 changes: 21 additions & 0 deletions src/main/java/com/bazel_diff/Files.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
package com.bazel_diff;

import java.nio.file.Path;
import java.nio.file.Files;
import java.io.IOException;

interface FilesClient {
byte[] readFile(Path path) throws IOException;
}

class FilesClientImp implements FilesClient {
FilesClientImp() {}

@Override
public byte[] readFile(Path path) throws IOException {
if (path.toFile().exists() && path.toFile().canRead()) {
return Files.readAllBytes(path);
}
return new byte[0];
}
}
55 changes: 41 additions & 14 deletions src/main/java/com/bazel_diff/TargetHashingClient.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,35 +1,39 @@
package com.bazel_diff;

import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.nio.file.Path;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.*;
import java.util.stream.Collectors;
import com.google.common.primitives.Bytes;

interface TargetHashingClient {
Map<String, String> hashAllBazelTargets(Set<Path> modifiedFilepaths) throws IOException, NoSuchAlgorithmException;
Map<String, String> hashAllBazelTargetsAndSourcefiles() throws IOException, NoSuchAlgorithmException;
Map<String, String> hashAllBazelTargets(Set<Path> modifiedFilepaths, Set<Path> seedFilepaths) throws IOException, NoSuchAlgorithmException;
Map<String, String> hashAllBazelTargetsAndSourcefiles(Set<Path> seedFilepaths) throws IOException, NoSuchAlgorithmException;
Set<String> getImpactedTargets(Map<String, String> startHashes, Map<String, String> endHashes, String avoidQuery, Boolean hashAllTargets) throws IOException;
}

class TargetHashingClientImpl implements TargetHashingClient {
private BazelClient bazelClient;
private FilesClient files;

TargetHashingClientImpl(BazelClient bazelClient) {
TargetHashingClientImpl(BazelClient bazelClient, FilesClient files) {
this.bazelClient = bazelClient;
this.files = files;
}

@Override
public Map<String, String> hashAllBazelTargets(Set<Path> modifiedFilepaths) throws IOException, NoSuchAlgorithmException {
public Map<String, String> hashAllBazelTargets(Set<Path> modifiedFilepaths, Set<Path> seedFilepaths) throws IOException, NoSuchAlgorithmException {
Set<BazelSourceFileTarget> bazelSourcefileTargets = bazelClient.convertFilepathsToSourceTargets(modifiedFilepaths);
return hashAllTargets(bazelSourcefileTargets);
return hashAllTargets(createSeedForFilepaths(seedFilepaths), bazelSourcefileTargets);
}

@Override
public Map<String, String> hashAllBazelTargetsAndSourcefiles() throws IOException, NoSuchAlgorithmException {
public Map<String, String> hashAllBazelTargetsAndSourcefiles(Set<Path> seedFilepaths) throws IOException, NoSuchAlgorithmException {
Set<BazelSourceFileTarget> bazelSourcefileTargets = bazelClient.queryAllSourcefileTargets();
return hashAllTargets(bazelSourcefileTargets);
return hashAllTargets(createSeedForFilepaths(seedFilepaths), bazelSourcefileTargets);
}

@Override
Expand All @@ -56,7 +60,8 @@ private byte[] createDigestForTarget(
BazelTarget target,
Map<String, BazelRule> allRulesMap,
Set<BazelSourceFileTarget> bazelSourcefileTargets,
Map<String, byte[]> ruleHashes
Map<String, byte[]> ruleHashes,
byte[] seedHash
) throws NoSuchAlgorithmException {
BazelRule targetRule = target.getRule();
if (target.hasSourceFile()) {
Expand All @@ -67,24 +72,31 @@ private byte[] createDigestForTarget(
if (sourceTargetDigestBytes != null) {
digest.update(sourceTargetDigestBytes);
}
if (seedHash != null) {
digest.update(seedHash);
}
return digest.digest().clone();
}
}
return createDigestForRule(targetRule, allRulesMap, ruleHashes, bazelSourcefileTargets);
return createDigestForRule(targetRule, allRulesMap, ruleHashes, bazelSourcefileTargets, seedHash);
}

private byte[] createDigestForRule(
BazelRule rule,
Map<String, BazelRule> allRulesMap,
Map<String, byte[]> ruleHashes,
Set<BazelSourceFileTarget> bazelSourcefileTargets
Set<BazelSourceFileTarget> bazelSourcefileTargets,
byte[] seedHash
) throws NoSuchAlgorithmException {
byte[] existingByteArray = ruleHashes.get(rule.getName());
if (existingByteArray != null) {
return existingByteArray;
}
MessageDigest digest = MessageDigest.getInstance("SHA-256");
digest.update(rule.getDigest());
if (seedHash != null) {
digest.update(seedHash);
}
for (String ruleInput : rule.getRuleInputList()) {
digest.update(ruleInput.getBytes());
BazelRule inputRule = allRulesMap.get(ruleInput);
Expand All @@ -94,7 +106,8 @@ private byte[] createDigestForRule(
inputRule,
allRulesMap,
ruleHashes,
bazelSourcefileTargets
bazelSourcefileTargets,
seedHash
);
if (ruleInputHash != null) {
digest.update(ruleInputHash);
Expand All @@ -108,6 +121,17 @@ private byte[] createDigestForRule(
return finalHashValue;
}

private byte[] createSeedForFilepaths(Set<Path> seedFilepaths) throws IOException, NoSuchAlgorithmException {
if (seedFilepaths == null || seedFilepaths.size() == 0) {
return new byte[0];
}
MessageDigest digest = MessageDigest.getInstance("SHA-256");
for (Path path: seedFilepaths) {
digest.update(this.files.readFile(path));
}
return digest.digest().clone();
}

private byte[] getDigestForSourceTargetName(
String sourceTargetName,
Set<BazelSourceFileTarget> bazelSourcefileTargets
Expand Down Expand Up @@ -138,7 +162,7 @@ private String getNameForTarget(BazelTarget target) {
return null;
}

private Map<String, String> hashAllTargets(Set<BazelSourceFileTarget> bazelSourcefileTargets) throws IOException, NoSuchAlgorithmException {
private Map<String, String> hashAllTargets(byte[] seedHash, Set<BazelSourceFileTarget> bazelSourcefileTargets) throws IOException, NoSuchAlgorithmException {
List<BazelTarget> allTargets = bazelClient.queryAllTargets();
Map<String, String> targetHashes = new HashMap<>();
Map<String, byte[]> ruleHashes = new HashMap<>();
Expand All @@ -159,10 +183,13 @@ private Map<String, String> hashAllTargets(Set<BazelSourceFileTarget> bazelSourc
target,
allRulesMap,
bazelSourcefileTargets,
ruleHashes
ruleHashes,
seedHash
);
if (targetDigest != null) {
targetHashes.put(targetName, convertByteArrayToString(targetDigest));
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
outputStream.write(targetDigest);
targetHashes.put(targetName, convertByteArrayToString(outputStream.toByteArray()));
}
}
return targetHashes;
Expand Down
21 changes: 16 additions & 5 deletions src/main/java/com/bazel_diff/main.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,17 +92,28 @@ static class Exclusive {
Boolean hashAllSourcefiles;
}

@Option(names = {"-s", "--seed-filepaths"}, description = "A text file containing a newline separated list of filepaths, each of these filepaths will be read and used as a seed for all targets.")
File seedFilepaths;

@Parameters(index = "0", description = "The filepath to write the resulting JSON of dictionary target => SHA-256 values")
File outputPath;

@Override
public Integer call() {
GitClient gitClient = new GitClientImpl(parent.workspacePath);
BazelClient bazelClient = new BazelClientImpl(parent.workspacePath, parent.bazelPath, parent.bazelStartupOptions, parent.bazelCommandOptions, BazelDiff.isVerbose());
TargetHashingClient hashingClient = new TargetHashingClientImpl(bazelClient);
TargetHashingClient hashingClient = new TargetHashingClientImpl(bazelClient, new FilesClientImp());
try {
gitClient.ensureAllChangesAreCommitted();
Set<Path> modifiedFilepathsSet = new HashSet<>();
Set<Path> seedFilepathsSet = new HashSet<>();
if (seedFilepaths != null) {
FileReader fileReader = new FileReader(seedFilepaths);
BufferedReader bufferedReader = new BufferedReader(fileReader);
seedFilepathsSet = bufferedReader.lines()
.map( line -> new File(line).toPath())
.collect(Collectors.toSet());
}
Map<String, String> hashes = new HashMap<>();
if (exclusive != null) {
if (exclusive.modifiedFilepaths != null) {
Expand All @@ -112,12 +123,12 @@ public Integer call() {
.lines()
.map( line -> new File(line).toPath())
.collect(Collectors.toSet());
hashes = hashingClient.hashAllBazelTargets(modifiedFilepathsSet);
hashes = hashingClient.hashAllBazelTargets(modifiedFilepathsSet, seedFilepathsSet);
} else if (exclusive.hashAllSourcefiles) {
hashes = hashingClient.hashAllBazelTargetsAndSourcefiles();
hashes = hashingClient.hashAllBazelTargetsAndSourcefiles(seedFilepathsSet);
}
} else {
hashes = hashingClient.hashAllBazelTargets(modifiedFilepathsSet);
hashes = hashingClient.hashAllBazelTargets(modifiedFilepathsSet, seedFilepathsSet);
}
Gson gson = new GsonBuilder().setPrettyPrinting().create();
FileWriter myWriter = new FileWriter(outputPath);
Expand Down Expand Up @@ -182,7 +193,7 @@ public Integer call() throws IOException {
}
GitClient gitClient = new GitClientImpl(workspacePath);
BazelClient bazelClient = new BazelClientImpl(workspacePath, bazelPath, bazelStartupOptions, bazelCommandOptions, BazelDiff.isVerbose());
TargetHashingClient hashingClient = new TargetHashingClientImpl(bazelClient);
TargetHashingClient hashingClient = new TargetHashingClientImpl(bazelClient, new FilesClientImp());
try {
gitClient.ensureAllChangesAreCommitted();
} catch (IOException | InterruptedException e) {
Expand Down
Loading