Fix computing external repo fine-grained hashes

cli/src/main/kotlin/com/bazel_diff/bazel/BazelClient.kt

@@ -38,7 +38,8 @@ class BazelClient(private val fineGrainedHashExternalRepos: Set<String>) : KoinC
 
     suspend fun queryAllSourcefileTargets(): List<Build.Target> {
         val queryEpoch = Calendar.getInstance().getTimeInMillis()
-        val targets = queryService.query("kind('source file', //...:all-targets)")
+        val allReposToQuery = listOf("@") + fineGrainedHashExternalRepos.map { "@$it" }
+        val targets = queryService.query("kind('source file', ${allReposToQuery.joinToString(" + ") { "'$it//...:all-targets'" }})")
         val queryDuration = Calendar.getInstance().getTimeInMillis() - queryEpoch
         logger.i { "All source files queried in $queryDuration" }
 

cli/src/main/kotlin/com/bazel_diff/di/Modules.kt

@@ -9,13 +9,19 @@ import com.bazel_diff.hash.TargetHasher
 import com.bazel_diff.io.ContentHashProvider
 import com.bazel_diff.log.Logger
 import com.bazel_diff.log.StderrLogger
+import com.bazel_diff.process.Redirect
+import com.bazel_diff.process.process
 import com.google.gson.GsonBuilder
+import kotlinx.coroutines.ExperimentalCoroutinesApi
+import kotlinx.coroutines.runBlocking
 import org.koin.core.module.Module
 import org.koin.core.qualifier.named
 import org.koin.dsl.module
 import java.io.File
 import java.nio.file.Path
+import java.nio.file.Paths
 
+@OptIn(ExperimentalCoroutinesApi::class)
 fun hasherModule(
     workingDirectory: Path,
     bazelPath: Path,
@@ -40,8 +46,20 @@ fun hasherModule(
     single { BuildGraphHasher(get()) }
     single { TargetHasher() }
     single { RuleHasher(fineGrainedHashExternalRepos) }
-    single { SourceFileHasher() }
+    single { SourceFileHasher(fineGrainedHashExternalRepos) }
     single(named("working-directory")) { workingDirectory }
+    single(named("output-base")) {
+        val result = runBlocking {
+            process(
+                bazelPath.toString(), "info", "output_base",
+                stdout = Redirect.CAPTURE,
+                workingDirectory = workingDirectory.toFile(),
+                stderr = Redirect.PRINT,
+                destroyForcibly = true,
+            )
+        }
+        Paths.get(result.output.single())
+    }
     single { ContentHashProvider(contentHashPath) }
 }
 

cli/src/main/kotlin/com/bazel_diff/hash/SourceFileHasher.kt

@@ -13,49 +13,69 @@ class SourceFileHasher : KoinComponent {
     private val workingDirectory: Path
     private val logger: Logger
     private val relativeFilenameToContentHash: Map<String, String>?
+    private val outputBase: Path
+    private val fineGrainedHashExternalRepos: Set<String>
+
     init {
         val logger: Logger by inject()
         this.logger = logger
     }
 
-    constructor() {
+    constructor(fineGrainedHashExternalRepos: Set<String> = emptySet()) {
         val workingDirectory: Path by inject(qualifier = named("working-directory"))
         this.workingDirectory = workingDirectory
         val contentHashProvider: ContentHashProvider by inject()
         relativeFilenameToContentHash = contentHashProvider.filenameToHash
+        val outputBase: Path by inject(qualifier = named("output-base"))
+        this.outputBase = outputBase
+        this.fineGrainedHashExternalRepos = fineGrainedHashExternalRepos
     }
 
-    constructor(workingDirectory: Path, relativeFilenameToContentHash: Map<String, String>?) {
+    constructor(workingDirectory: Path, outputBase: Path, relativeFilenameToContentHash: Map<String, String>?, fineGrainedHashExternalRepos: Set<String> = emptySet()) {
         this.workingDirectory = workingDirectory
+        this.outputBase = outputBase
         this.relativeFilenameToContentHash = relativeFilenameToContentHash
+        this.fineGrainedHashExternalRepos = fineGrainedHashExternalRepos
     }
 
     fun digest(sourceFileTarget: BazelSourceFileTarget): ByteArray {
         return sha256 {
             val name = sourceFileTarget.name
-            if (name.startsWith("//")) {
+            val filenamePath = if (name.startsWith("//")) {
                 val filenameSubstring = name.substring(2)
-                val filenamePath = filenameSubstring.replaceFirst(
-                        ":".toRegex(),
-                        if (filenameSubstring.startsWith(":")) "" else "/"
-                )
-                if (relativeFilenameToContentHash?.contains(filenamePath) == true) {
-                    val contentHash = relativeFilenameToContentHash.getValue(filenamePath)
-                    safePutBytes(contentHash.toByteArray())
-                } else {
-                    val absoluteFilePath = Paths.get(workingDirectory.toString(), filenamePath)
-                    val file = absoluteFilePath.toFile()
-                    if (file.exists()) {
-                        if (file.isFile) {
-                            putFile(file)
-                        }
-                    } else {
-                        logger.w { "File $absoluteFilePath not found" }
+                Paths.get(filenameSubstring.removePrefix(":").replace(':', '/'))
+            } else if (name.startsWith("@")) {
+                val parts = name.substring(1).split("//")
+                if (parts.size != 2) {
+                    logger.w { "Invalid source label $name" }
+                    return@sha256
+                }
+                val repoName = parts[0]
+                if (repoName !in fineGrainedHashExternalRepos) {
+                    return@sha256
+                }
+                val relativePath = Paths.get(parts[1].removePrefix(":").replace(':', '/'))
+                outputBase.resolve("external/$repoName").resolve(relativePath)
+            } else {
+                return@sha256
+            }
+            val filenamePathString = filenamePath.toString()
+            if (relativeFilenameToContentHash?.contains(filenamePathString) == true) {
+                val contentHash = relativeFilenameToContentHash.getValue(filenamePathString)
+                safePutBytes(contentHash.toByteArray())
+            } else {
+                val absoluteFilePath = workingDirectory.resolve(filenamePath)
+                val file = absoluteFilePath.toFile()
+                if (file.exists()) {
+                    if (file.isFile) {
+                        putFile(file)
                     }
+                } else {
+                    logger.w { "File $absoluteFilePath not found" }
                 }
-                safePutBytes(sourceFileTarget.seed)
-                safePutBytes(name.toByteArray())
             }
+            safePutBytes(sourceFileTarget.seed)
+            safePutBytes(name.toByteArray())
         }
     }
 

cli/src/test/kotlin/com/bazel_diff/Modules.kt

@@ -1,6 +1,7 @@
 package com.bazel_diff
 
 import com.bazel_diff.bazel.BazelClient
+import com.bazel_diff.bazel.BazelQueryService
 import com.bazel_diff.hash.BuildGraphHasher
 import com.bazel_diff.hash.RuleHasher
 import com.bazel_diff.hash.SourceFileHasher
@@ -22,6 +23,7 @@ fun testModule(): Module = module {
     single { SourceFileHasher() }
     single { GsonBuilder().setPrettyPrinting().create() }
     single(named("working-directory")) { Paths.get("working-directory") }
+    single(named("output-base")) { Paths.get("output-base") }
     single { ContentHashProvider(null) }
 }
 

cli/src/test/kotlin/com/bazel_diff/hash/SourceFileHasherTest.kt

@@ -17,6 +17,7 @@ import java.nio.file.Paths
 
 internal class SourceFileHasherTest: KoinTest {
     private val repoAbsolutePath = Paths.get("").toAbsolutePath()
+    private val outputBasePath = Files.createTempDirectory("SourceFileHasherTest")
     private val fixtureFileTarget = "//cli/src/test/kotlin/com/bazel_diff/hash/fixture:foo.ts"
     private val fixtureFileContent: ByteArray
     private val seed = "seed".toByteArray()
@@ -34,7 +35,7 @@ internal class SourceFileHasherTest: KoinTest {
 
     @Test
     fun testHashConcreteFile() = runBlocking {
-        val hasher = SourceFileHasher(repoAbsolutePath, null)
+        val hasher = SourceFileHasher(repoAbsolutePath, outputBasePath, null)
         val bazelSourceFileTarget = BazelSourceFileTarget(fixtureFileTarget, seed)
         val actual = hasher.digest(bazelSourceFileTarget).toHexString()
         val expected = sha256 {
@@ -45,9 +46,27 @@ internal class SourceFileHasherTest: KoinTest {
         assertThat(actual).isEqualTo(expected)
     }
 
+    @Test
+    fun testHashConcreteFileInExternalRepo() = runBlocking {
+        val hasher = SourceFileHasher(repoAbsolutePath, outputBasePath, null, setOf("external_repo"))
+        val externalRepoFilePath = outputBasePath.resolve("external/external_repo/path/to/my_file.txt")
+        Files.createDirectories(externalRepoFilePath.parent)
+        val externalRepoFileTarget = "@external_repo//path/to:my_file.txt"
+        val externalRepoFileContent = "hello world"
+        externalRepoFilePath.toFile().writeText(externalRepoFileContent)
+        val bazelSourceFileTarget = BazelSourceFileTarget(externalRepoFileTarget, seed)
+        val actual = hasher.digest(bazelSourceFileTarget).toHexString()
+        val expected = sha256 {
+            safePutBytes(externalRepoFileContent.toByteArray())
+            safePutBytes(seed)
+            safePutBytes(externalRepoFileTarget.toByteArray())
+        }.toHexString()
+        assertThat(actual).isEqualTo(expected)
+    }
+
     @Test
     fun testSoftHashConcreteFile() = runBlocking {
-        val hasher = SourceFileHasher(repoAbsolutePath, null)
+        val hasher = SourceFileHasher(repoAbsolutePath, outputBasePath, null)
         val bazelSourceFileTarget = BazelSourceFileTarget(fixtureFileTarget, seed)
         val actual = hasher.softDigest(bazelSourceFileTarget)?.toHexString()
         val expected = sha256 {
@@ -60,7 +79,7 @@ internal class SourceFileHasherTest: KoinTest {
 
     @Test
     fun testSoftHashNonExistedFile() = runBlocking {
-        val hasher = SourceFileHasher(repoAbsolutePath, null)
+        val hasher = SourceFileHasher(repoAbsolutePath, outputBasePath, null)
         val bazelSourceFileTarget = BazelSourceFileTarget("//i/do/not/exist", seed)
         val actual = hasher.softDigest(bazelSourceFileTarget)
         assertThat(actual).isNull()
@@ -69,7 +88,7 @@ internal class SourceFileHasherTest: KoinTest {
     @Test
     fun testSoftHashExternalTarget() = runBlocking {
         val target = "@bazel-diff//some:file"
-        val hasher = SourceFileHasher(repoAbsolutePath, null)
+        val hasher = SourceFileHasher(repoAbsolutePath, outputBasePath, null)
         val bazelSourceFileTarget = BazelSourceFileTarget(target, seed)
         val actual = hasher.softDigest(bazelSourceFileTarget)
         assertThat(actual).isNull()
@@ -78,7 +97,7 @@ internal class SourceFileHasherTest: KoinTest {
     @Test
     fun testHashNonExistedFile() = runBlocking {
         val target = "//i/do/not/exist"
-        val hasher = SourceFileHasher(repoAbsolutePath, null)
+        val hasher = SourceFileHasher(repoAbsolutePath, outputBasePath, null)
         val bazelSourceFileTarget = BazelSourceFileTarget(target, seed)
         val actual = hasher.digest(bazelSourceFileTarget).toHexString()
         val expected = sha256 {
@@ -91,7 +110,7 @@ internal class SourceFileHasherTest: KoinTest {
     @Test
     fun testHashExternalTarget() = runBlocking {
         val target = "@bazel-diff//some:file"
-        val hasher = SourceFileHasher(repoAbsolutePath, null)
+        val hasher = SourceFileHasher(repoAbsolutePath, outputBasePath, null)
         val bazelSourceFileTarget = BazelSourceFileTarget(target, seed)
         val actual = hasher.digest(bazelSourceFileTarget).toHexString()
         val expected = sha256 {}.toHexString()
@@ -101,7 +120,7 @@ internal class SourceFileHasherTest: KoinTest {
     @Test
     fun testHashWithProvidedContentHash() = runBlocking {
         val filenameToContentHash = hashMapOf("cli/src/test/kotlin/com/bazel_diff/hash/fixture/foo.ts" to "foo-content-hash")
-        val hasher = SourceFileHasher(repoAbsolutePath, filenameToContentHash)
+        val hasher = SourceFileHasher(repoAbsolutePath, outputBasePath, filenameToContentHash)
         val bazelSourceFileTarget = BazelSourceFileTarget(fixtureFileTarget, seed)
         val actual = hasher.digest(bazelSourceFileTarget).toHexString()
         val expected = sha256 {
@@ -115,7 +134,7 @@ internal class SourceFileHasherTest: KoinTest {
     @Test
     fun testHashWithProvidedContentHashButNotInKey() = runBlocking {
         val filenameToContentHash = hashMapOf("cli/src/test/kotlin/com/bazel_diff/hash/fixture/bar.ts" to "foo-content-hash")
-        val hasher = SourceFileHasher(repoAbsolutePath, filenameToContentHash)
+        val hasher = SourceFileHasher(repoAbsolutePath, outputBasePath, filenameToContentHash)
         val bazelSourceFileTarget = BazelSourceFileTarget(fixtureFileTarget, seed)
         val actual = hasher.digest(bazelSourceFileTarget).toHexString()
         val expected = sha256 {
@@ -130,7 +149,7 @@ internal class SourceFileHasherTest: KoinTest {
     fun testHashWithProvidedContentHashWithLeadingColon() = runBlocking {
         val targetName = "//:cli/src/test/kotlin/com/bazel_diff/hash/fixture/bar.ts"
         val filenameToContentHash = hashMapOf("cli/src/test/kotlin/com/bazel_diff/hash/fixture/bar.ts" to "foo-content-hash")
-        val hasher = SourceFileHasher(repoAbsolutePath, filenameToContentHash)
+        val hasher = SourceFileHasher(repoAbsolutePath, outputBasePath, filenameToContentHash)
         val bazelSourceFileTarget = BazelSourceFileTarget(targetName, seed)
         val actual = hasher.digest(bazelSourceFileTarget).toHexString()
         val expected = sha256 {

cli/src/test/resources/fixture/fine-grained-hash-external-repo-test-impacted-targets.txt

@@ -2,9 +2,12 @@
 //src/main/java/com/integration:guava-user
 //src/main/java/com/integration:libguava-user-src.jar
 //src/main/java/com/integration:libguava-user.jar
+@bazel_diff_maven//:BUILD
 @bazel_diff_maven//:com_google_errorprone_error_prone_annotations
 @bazel_diff_maven//:com_google_errorprone_error_prone_annotations_2_11_0
 @bazel_diff_maven//:com_google_guava_guava
 @bazel_diff_maven//:com_google_guava_guava_31_1_jre
+@bazel_diff_maven//:pin
+@bazel_diff_maven//:pin.sh
 @bazel_diff_maven//:v1/https/jcenter.bintray.com/com/google/errorprone/error_prone_annotations/2.11.0/error_prone_annotations-2.11.0.jar
 @bazel_diff_maven//:v1/https/jcenter.bintray.com/com/google/guava/guava/31.1-jre/guava-31.1-jre.jar