diff --git a/README.md b/README.md index 9b136d2..61f6b56 100644 --- a/README.md +++ b/README.md @@ -134,31 +134,39 @@ Writes to the file the modified filepaths between two revisions. `generate-hashes` Command ```terminal -Usage: bazel-diff generate-hashes [-hV] -b= - [-co=] - [-m=] +Usage: bazel-diff generate-hashes [-m= | -a] [-hV] + -b= [-co=] + [-s=] [-so=] -w= Writes to a file the SHA256 hashes for each Bazel Target in the provided workspace. - The filepath to write the resulting JSON of dictionary - target => SHA-256 values + The filepath to write the resulting JSON of + dictionary target => SHA-256 values + -a, --all-sourcefiles Experimental: Hash all sourcefile targets (instead of + relying on --modifiedFilepaths), Warning: + Performance may degrade from reading all source + files -b, --bazelPath= - Path to Bazel binary + Path to Bazel binary -co, --bazelCommandOptions= - Additional space separated Bazel command options used when - invoking Bazel - -h, --help Show this help message and exit. + Additional space separated Bazel command options used + when invoking Bazel + -h, --help Show this help message and exit. -m, --modifiedFilepaths= - The path to a file containing the list of modified - filepaths in the workspace, you can use the - 'modified-filepaths' command to get this list + The path to a file containing the list of modified + filepaths in the workspace, you can use the + 'modified-filepaths' command to get this list + -s, --seed-filepaths= + A text file containing a newline separated list of + filepaths, each of these filepaths will be read and + used as a seed for all targets. -so, --bazelStartupOptions= - Additional space separated Bazel client startup options - used when invoking Bazel - -V, --version Print version information and exit. + Additional space separated Bazel client startup + options used when invoking Bazel + -V, --version Print version information and exit. -w, --workspacePath= - Path to Bazel workspace directory. + Path to Bazel workspace directory. ``` ## Installing diff --git a/src/main/java/com/bazel_diff/Files.java b/src/main/java/com/bazel_diff/Files.java new file mode 100644 index 0000000..dfeaab0 --- /dev/null +++ b/src/main/java/com/bazel_diff/Files.java @@ -0,0 +1,21 @@ +package com.bazel_diff; + +import java.nio.file.Path; +import java.nio.file.Files; +import java.io.IOException; + +interface FilesClient { + byte[] readFile(Path path) throws IOException; +} + +class FilesClientImp implements FilesClient { + FilesClientImp() {} + + @Override + public byte[] readFile(Path path) throws IOException { + if (path.toFile().exists() && path.toFile().canRead()) { + return Files.readAllBytes(path); + } + return new byte[0]; + } +} diff --git a/src/main/java/com/bazel_diff/TargetHashingClient.java b/src/main/java/com/bazel_diff/TargetHashingClient.java index 33d708f..3372ad9 100644 --- a/src/main/java/com/bazel_diff/TargetHashingClient.java +++ b/src/main/java/com/bazel_diff/TargetHashingClient.java @@ -1,35 +1,39 @@ package com.bazel_diff; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.nio.file.Path; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.*; import java.util.stream.Collectors; +import com.google.common.primitives.Bytes; interface TargetHashingClient { - Map hashAllBazelTargets(Set modifiedFilepaths) throws IOException, NoSuchAlgorithmException; - Map hashAllBazelTargetsAndSourcefiles() throws IOException, NoSuchAlgorithmException; + Map hashAllBazelTargets(Set modifiedFilepaths, Set seedFilepaths) throws IOException, NoSuchAlgorithmException; + Map hashAllBazelTargetsAndSourcefiles(Set seedFilepaths) throws IOException, NoSuchAlgorithmException; Set getImpactedTargets(Map startHashes, Map endHashes, String avoidQuery, Boolean hashAllTargets) throws IOException; } class TargetHashingClientImpl implements TargetHashingClient { private BazelClient bazelClient; + private FilesClient files; - TargetHashingClientImpl(BazelClient bazelClient) { + TargetHashingClientImpl(BazelClient bazelClient, FilesClient files) { this.bazelClient = bazelClient; + this.files = files; } @Override - public Map hashAllBazelTargets(Set modifiedFilepaths) throws IOException, NoSuchAlgorithmException { + public Map hashAllBazelTargets(Set modifiedFilepaths, Set seedFilepaths) throws IOException, NoSuchAlgorithmException { Set bazelSourcefileTargets = bazelClient.convertFilepathsToSourceTargets(modifiedFilepaths); - return hashAllTargets(bazelSourcefileTargets); + return hashAllTargets(createSeedForFilepaths(seedFilepaths), bazelSourcefileTargets); } @Override - public Map hashAllBazelTargetsAndSourcefiles() throws IOException, NoSuchAlgorithmException { + public Map hashAllBazelTargetsAndSourcefiles(Set seedFilepaths) throws IOException, NoSuchAlgorithmException { Set bazelSourcefileTargets = bazelClient.queryAllSourcefileTargets(); - return hashAllTargets(bazelSourcefileTargets); + return hashAllTargets(createSeedForFilepaths(seedFilepaths), bazelSourcefileTargets); } @Override @@ -56,7 +60,8 @@ private byte[] createDigestForTarget( BazelTarget target, Map allRulesMap, Set bazelSourcefileTargets, - Map ruleHashes + Map ruleHashes, + byte[] seedHash ) throws NoSuchAlgorithmException { BazelRule targetRule = target.getRule(); if (target.hasSourceFile()) { @@ -67,17 +72,21 @@ private byte[] createDigestForTarget( if (sourceTargetDigestBytes != null) { digest.update(sourceTargetDigestBytes); } + if (seedHash != null) { + digest.update(seedHash); + } return digest.digest().clone(); } } - return createDigestForRule(targetRule, allRulesMap, ruleHashes, bazelSourcefileTargets); + return createDigestForRule(targetRule, allRulesMap, ruleHashes, bazelSourcefileTargets, seedHash); } private byte[] createDigestForRule( BazelRule rule, Map allRulesMap, Map ruleHashes, - Set bazelSourcefileTargets + Set bazelSourcefileTargets, + byte[] seedHash ) throws NoSuchAlgorithmException { byte[] existingByteArray = ruleHashes.get(rule.getName()); if (existingByteArray != null) { @@ -85,6 +94,9 @@ private byte[] createDigestForRule( } MessageDigest digest = MessageDigest.getInstance("SHA-256"); digest.update(rule.getDigest()); + if (seedHash != null) { + digest.update(seedHash); + } for (String ruleInput : rule.getRuleInputList()) { digest.update(ruleInput.getBytes()); BazelRule inputRule = allRulesMap.get(ruleInput); @@ -94,7 +106,8 @@ private byte[] createDigestForRule( inputRule, allRulesMap, ruleHashes, - bazelSourcefileTargets + bazelSourcefileTargets, + seedHash ); if (ruleInputHash != null) { digest.update(ruleInputHash); @@ -108,6 +121,17 @@ private byte[] createDigestForRule( return finalHashValue; } + private byte[] createSeedForFilepaths(Set seedFilepaths) throws IOException, NoSuchAlgorithmException { + if (seedFilepaths == null || seedFilepaths.size() == 0) { + return new byte[0]; + } + MessageDigest digest = MessageDigest.getInstance("SHA-256"); + for (Path path: seedFilepaths) { + digest.update(this.files.readFile(path)); + } + return digest.digest().clone(); + } + private byte[] getDigestForSourceTargetName( String sourceTargetName, Set bazelSourcefileTargets @@ -138,7 +162,7 @@ private String getNameForTarget(BazelTarget target) { return null; } - private Map hashAllTargets(Set bazelSourcefileTargets) throws IOException, NoSuchAlgorithmException { + private Map hashAllTargets(byte[] seedHash, Set bazelSourcefileTargets) throws IOException, NoSuchAlgorithmException { List allTargets = bazelClient.queryAllTargets(); Map targetHashes = new HashMap<>(); Map ruleHashes = new HashMap<>(); @@ -159,10 +183,13 @@ private Map hashAllTargets(Set bazelSourc target, allRulesMap, bazelSourcefileTargets, - ruleHashes + ruleHashes, + seedHash ); if (targetDigest != null) { - targetHashes.put(targetName, convertByteArrayToString(targetDigest)); + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + outputStream.write(targetDigest); + targetHashes.put(targetName, convertByteArrayToString(outputStream.toByteArray())); } } return targetHashes; diff --git a/src/main/java/com/bazel_diff/main.java b/src/main/java/com/bazel_diff/main.java index bd22e90..5f64acc 100644 --- a/src/main/java/com/bazel_diff/main.java +++ b/src/main/java/com/bazel_diff/main.java @@ -92,6 +92,9 @@ static class Exclusive { Boolean hashAllSourcefiles; } + @Option(names = {"-s", "--seed-filepaths"}, description = "A text file containing a newline separated list of filepaths, each of these filepaths will be read and used as a seed for all targets.") + File seedFilepaths; + @Parameters(index = "0", description = "The filepath to write the resulting JSON of dictionary target => SHA-256 values") File outputPath; @@ -99,10 +102,18 @@ static class Exclusive { public Integer call() { GitClient gitClient = new GitClientImpl(parent.workspacePath); BazelClient bazelClient = new BazelClientImpl(parent.workspacePath, parent.bazelPath, parent.bazelStartupOptions, parent.bazelCommandOptions, BazelDiff.isVerbose()); - TargetHashingClient hashingClient = new TargetHashingClientImpl(bazelClient); + TargetHashingClient hashingClient = new TargetHashingClientImpl(bazelClient, new FilesClientImp()); try { gitClient.ensureAllChangesAreCommitted(); Set modifiedFilepathsSet = new HashSet<>(); + Set seedFilepathsSet = new HashSet<>(); + if (seedFilepaths != null) { + FileReader fileReader = new FileReader(seedFilepaths); + BufferedReader bufferedReader = new BufferedReader(fileReader); + seedFilepathsSet = bufferedReader.lines() + .map( line -> new File(line).toPath()) + .collect(Collectors.toSet()); + } Map hashes = new HashMap<>(); if (exclusive != null) { if (exclusive.modifiedFilepaths != null) { @@ -112,12 +123,12 @@ public Integer call() { .lines() .map( line -> new File(line).toPath()) .collect(Collectors.toSet()); - hashes = hashingClient.hashAllBazelTargets(modifiedFilepathsSet); + hashes = hashingClient.hashAllBazelTargets(modifiedFilepathsSet, seedFilepathsSet); } else if (exclusive.hashAllSourcefiles) { - hashes = hashingClient.hashAllBazelTargetsAndSourcefiles(); + hashes = hashingClient.hashAllBazelTargetsAndSourcefiles(seedFilepathsSet); } } else { - hashes = hashingClient.hashAllBazelTargets(modifiedFilepathsSet); + hashes = hashingClient.hashAllBazelTargets(modifiedFilepathsSet, seedFilepathsSet); } Gson gson = new GsonBuilder().setPrettyPrinting().create(); FileWriter myWriter = new FileWriter(outputPath); @@ -182,7 +193,7 @@ public Integer call() throws IOException { } GitClient gitClient = new GitClientImpl(workspacePath); BazelClient bazelClient = new BazelClientImpl(workspacePath, bazelPath, bazelStartupOptions, bazelCommandOptions, BazelDiff.isVerbose()); - TargetHashingClient hashingClient = new TargetHashingClientImpl(bazelClient); + TargetHashingClient hashingClient = new TargetHashingClientImpl(bazelClient, new FilesClientImp()); try { gitClient.ensureAllChangesAreCommitted(); } catch (IOException | InterruptedException e) { diff --git a/test/java/com/bazel_diff/TargetHashingClientImplTests.java b/test/java/com/bazel_diff/TargetHashingClientImplTests.java index 19ad28c..8031f3e 100644 --- a/test/java/com/bazel_diff/TargetHashingClientImplTests.java +++ b/test/java/com/bazel_diff/TargetHashingClientImplTests.java @@ -7,6 +7,8 @@ import org.mockito.junit.*; import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; import java.security.NoSuchAlgorithmException; import java.util.*; @@ -18,6 +20,9 @@ public class TargetHashingClientImplTests { @Mock BazelClient bazelClientMock; + @Mock + FilesClient filesClientMock; + List defaultTargets; @Rule @@ -33,9 +38,9 @@ public void setUp() throws Exception { @Test public void hashAllBazelTargets_noTargets() throws IOException { when(bazelClientMock.queryAllTargets()).thenReturn(new ArrayList<>()); - TargetHashingClientImpl client = new TargetHashingClientImpl(bazelClientMock); + TargetHashingClientImpl client = new TargetHashingClientImpl(bazelClientMock, filesClientMock); try { - Map hash = client.hashAllBazelTargets(new HashSet<>()); + Map hash = client.hashAllBazelTargets(new HashSet<>(), new HashSet<>()); assertEquals(hash.size(), 0); } catch (IOException e) { assertTrue(false); @@ -47,9 +52,9 @@ public void hashAllBazelTargets_noTargets() throws IOException { @Test public void hashAllBazelTargets_ruleTargets() throws IOException { when(bazelClientMock.queryAllTargets()).thenReturn(defaultTargets); - TargetHashingClientImpl client = new TargetHashingClientImpl(bazelClientMock); + TargetHashingClientImpl client = new TargetHashingClientImpl(bazelClientMock, filesClientMock); try { - Map hash = client.hashAllBazelTargets(new HashSet<>()); + Map hash = client.hashAllBazelTargets(new HashSet<>(), new HashSet<>()); assertEquals(2, hash.size()); assertEquals("2c963f7c06bc1cead7e3b4759e1472383d4469fc3238dc42f8848190887b4775", hash.get("rule1")); assertEquals("bdc1abd0a07103cea34199a9c0d1020619136ff90fb88dcc3a8f873c811c1fe9", hash.get("rule2")); @@ -58,6 +63,23 @@ public void hashAllBazelTargets_ruleTargets() throws IOException { } } + @Test + public void hashAllBazelTargets_ruleTargets_seedFilepaths() throws IOException { + Set seedFilepaths = new HashSet<>(); + seedFilepaths.add(Paths.get("somefile.txt")); + when(filesClientMock.readFile(anyObject())).thenReturn("somecontent".getBytes()); + when(bazelClientMock.queryAllTargets()).thenReturn(defaultTargets); + TargetHashingClientImpl client = new TargetHashingClientImpl(bazelClientMock, filesClientMock); + try { + Map hash = client.hashAllBazelTargets(new HashSet<>(), seedFilepaths); + assertEquals(2, hash.size()); + assertEquals("0404d80eadcc2dbfe9f0d7935086e1115344a06bd76d4e16af0dfd7b4913ee60", hash.get("rule1")); + assertEquals("6fe63fa16340d18176e6d6021972c65413441b72135247179362763508ebddfe", hash.get("rule2")); + } catch (IOException | NoSuchAlgorithmException e) { + fail(e.getMessage()); + } + } + @Test public void hashAllBazelTargets_ruleTargets_ruleInputs() throws IOException, NoSuchAlgorithmException { List ruleInputs = new ArrayList<>(); @@ -67,9 +89,9 @@ public void hashAllBazelTargets_ruleTargets_ruleInputs() throws IOException, NoS BazelTarget rule4 = createRuleTarget("rule4", ruleInputs, "digest2"); defaultTargets.add(rule4); when(bazelClientMock.queryAllTargets()).thenReturn(defaultTargets); - TargetHashingClientImpl client = new TargetHashingClientImpl(bazelClientMock); + TargetHashingClientImpl client = new TargetHashingClientImpl(bazelClientMock, filesClientMock); try { - Map hash = client.hashAllBazelTargets(new HashSet<>()); + Map hash = client.hashAllBazelTargets(new HashSet<>(), new HashSet<>()); assertEquals(4, hash.size()); assertEquals("2c963f7c06bc1cead7e3b4759e1472383d4469fc3238dc42f8848190887b4775", hash.get("rule1")); assertEquals("bdc1abd0a07103cea34199a9c0d1020619136ff90fb88dcc3a8f873c811c1fe9", hash.get("rule2")); @@ -84,9 +106,9 @@ public void hashAllBazelTargets_ruleTargets_ruleInputs() throws IOException, NoS public void hashAllBazelTargets_sourceTargets_unmodifiedSources() throws IOException { defaultTargets.add(createSourceTarget("sourceFile1")); when(bazelClientMock.queryAllTargets()).thenReturn(defaultTargets); - TargetHashingClientImpl client = new TargetHashingClientImpl(bazelClientMock); + TargetHashingClientImpl client = new TargetHashingClientImpl(bazelClientMock, filesClientMock); try { - Map hash = client.hashAllBazelTargets(new HashSet<>()); + Map hash = client.hashAllBazelTargets(new HashSet<>(), new HashSet()); assertEquals(3, hash.size()); assertEquals("2c963f7c06bc1cead7e3b4759e1472383d4469fc3238dc42f8848190887b4775", hash.get("rule1")); assertEquals("bdc1abd0a07103cea34199a9c0d1020619136ff90fb88dcc3a8f873c811c1fe9", hash.get("rule2")); @@ -104,9 +126,9 @@ public void hashAllBazelTargets_sourceTargets_modifiedSources() throws IOExcepti modifiedFileTargets.add(createSourceFileTarget("sourceFile1", "digest")); when(bazelClientMock.queryAllTargets()).thenReturn(defaultTargets); when(bazelClientMock.convertFilepathsToSourceTargets(anySet())).thenReturn(modifiedFileTargets); - TargetHashingClientImpl client = new TargetHashingClientImpl(bazelClientMock); + TargetHashingClientImpl client = new TargetHashingClientImpl(bazelClientMock, filesClientMock); try { - Map hash = client.hashAllBazelTargets(new HashSet<>()); + Map hash = client.hashAllBazelTargets(new HashSet<>(), new HashSet<>()); assertEquals(3, hash.size()); assertEquals("2c963f7c06bc1cead7e3b4759e1472383d4469fc3238dc42f8848190887b4775", hash.get("rule1")); assertEquals("bdc1abd0a07103cea34199a9c0d1020619136ff90fb88dcc3a8f873c811c1fe9", hash.get("rule2")); @@ -116,12 +138,35 @@ public void hashAllBazelTargets_sourceTargets_modifiedSources() throws IOExcepti } } + @Test + public void hashAllBazelTargets_sourceTargets_modifiedSources_seedFilepaths() throws IOException, NoSuchAlgorithmException { + Set seedFilepaths = new HashSet<>(); + seedFilepaths.add(Paths.get("somefile.txt")); + when(filesClientMock.readFile(anyObject())).thenReturn("somecontent".getBytes()); + createSourceTarget("sourceFile1"); + defaultTargets.add(createSourceTarget("sourceFile1")); + Set modifiedFileTargets = new HashSet<>(); + modifiedFileTargets.add(createSourceFileTarget("sourceFile1", "digest")); + when(bazelClientMock.queryAllTargets()).thenReturn(defaultTargets); + when(bazelClientMock.convertFilepathsToSourceTargets(anySet())).thenReturn(modifiedFileTargets); + TargetHashingClientImpl client = new TargetHashingClientImpl(bazelClientMock, filesClientMock); + try { + Map hash = client.hashAllBazelTargets(new HashSet<>(), seedFilepaths); + assertEquals(3, hash.size()); + assertEquals("0404d80eadcc2dbfe9f0d7935086e1115344a06bd76d4e16af0dfd7b4913ee60", hash.get("rule1")); + assertEquals("6fe63fa16340d18176e6d6021972c65413441b72135247179362763508ebddfe", hash.get("rule2")); + assertEquals("e0c3b2abd374fa00c23696c561b3797038fd8cf05de09377aef6a5dcd7529b13", hash.get("sourceFile1")); + } catch (IOException | NoSuchAlgorithmException e) { + fail(e.getMessage()); + } + } + @Test public void getImpactedTargets() throws IOException { when(bazelClientMock.queryForImpactedTargets(anySet(), anyObject())).thenReturn( new HashSet<>(Arrays.asList("rule1", "rule3")) ); - TargetHashingClientImpl client = new TargetHashingClientImpl(bazelClientMock); + TargetHashingClientImpl client = new TargetHashingClientImpl(bazelClientMock, filesClientMock); Map hash1 = new HashMap<>(); hash1.put("rule1", "rule1hash"); hash1.put("rule2", "rule2hash"); @@ -141,7 +186,7 @@ public void getImpactedTargets_withAvoidQuery() throws IOException { when(bazelClientMock.queryForImpactedTargets(anySet(), eq("some_query"))).thenReturn( new HashSet<>(Arrays.asList("rule1")) ); - TargetHashingClientImpl client = new TargetHashingClientImpl(bazelClientMock); + TargetHashingClientImpl client = new TargetHashingClientImpl(bazelClientMock, filesClientMock); Map hash1 = new HashMap<>(); hash1.put("rule1", "rule1hash"); hash1.put("rule2", "rule2hash"); @@ -160,7 +205,7 @@ public void getImpactedTargets_withHashAllTargets() throws IOException { when(bazelClientMock.queryForImpactedTargets(anySet(), anyObject())).thenReturn( new HashSet<>(Arrays.asList("rule1")) ); - TargetHashingClientImpl client = new TargetHashingClientImpl(bazelClientMock); + TargetHashingClientImpl client = new TargetHashingClientImpl(bazelClientMock, filesClientMock); Map hash1 = new HashMap<>(); hash1.put("rule1", "rule1hash"); hash1.put("rule2", "rule2hash"); @@ -180,7 +225,7 @@ public void getImpactedTargets_withHashAllTargets_withAvoidQuery() throws IOExce when(bazelClientMock.queryForImpactedTargets(anySet(), eq("some_query"))).thenReturn( new HashSet<>(Arrays.asList("rule1")) ); - TargetHashingClientImpl client = new TargetHashingClientImpl(bazelClientMock); + TargetHashingClientImpl client = new TargetHashingClientImpl(bazelClientMock, filesClientMock); Map hash1 = new HashMap<>(); hash1.put("rule1", "rule1hash"); hash1.put("rule2", "rule2hash");