Add --fineGrainedHashExternalRepos

cli/src/main/kotlin/com/bazel_diff/bazel/BazelClient.kt

@@ -4,16 +4,17 @@ import com.bazel_diff.log.Logger
 import com.google.devtools.build.lib.query2.proto.proto2api.Build
 import org.koin.core.component.KoinComponent
 import org.koin.core.component.inject
-import java.util.concurrent.ConcurrentMap
-import java.util.Calendar
+import java.util.*
 
-class BazelClient : KoinComponent {
+class BazelClient(private val fineGrainedHashExternalRepos: Set<String>) : KoinComponent {
     private val logger: Logger by inject()
     private val queryService: BazelQueryService by inject()
 
     suspend fun queryAllTargets(): List<BazelTarget> {
         val queryEpoch = Calendar.getInstance().getTimeInMillis()
-        val targets = queryService.query("'//external:all-targets' + '//...:all-targets'")
+
+        val query = listOf("//external:all-targets", "//...:all-targets") + fineGrainedHashExternalRepos.map { "@$it//...:all-targets" }
+        val targets = queryService.query(query.joinToString(" + ") { "'$it'" })
         val queryDuration = Calendar.getInstance().getTimeInMillis() - queryEpoch
         logger.i { "All targets queried in $queryDuration" }
         return targets.mapNotNull { target: Build.Target ->
@@ -22,9 +23,11 @@ class BazelClient : KoinComponent {
                 Build.Target.Discriminator.SOURCE_FILE -> BazelTarget.SourceFile(
                     target
                 )
+
                 Build.Target.Discriminator.GENERATED_FILE -> BazelTarget.GeneratedFile(
                     target
                 )
+
                 else -> {
                     logger.w { "Unsupported target type in the build graph: ${target.type.name}" }
                     null

cli/src/main/kotlin/com/bazel_diff/bazel/BazelRule.kt

@@ -21,13 +21,15 @@ class BazelRule(private val rule: Build.Rule) {
             }
         }
     }
-    val ruleInputList: List<String>
-        get() = rule.ruleInputList.map { ruleInput: String -> transformRuleInput(ruleInput) }
+
+    fun ruleInputList(fineGrainedHashExternalRepos: Set<String>): List<String> {
+        return rule.ruleInputList.map { ruleInput: String -> transformRuleInput(fineGrainedHashExternalRepos, ruleInput) }
+    }
 
     val name: String = rule.name
 
-    private fun transformRuleInput(ruleInput: String): String {
-        if (ruleInput.startsWith("@")) {
+    private fun transformRuleInput(fineGrainedHashExternalRepos: Set<String>, ruleInput: String): String {
+        if (ruleInput.startsWith("@") && fineGrainedHashExternalRepos.none { ruleInput.startsWith("@$it") }) {
             val splitRule = ruleInput.split("//".toRegex()).dropLastWhile { it.isEmpty() }.toTypedArray()
             if (splitRule.size == 2) {
                 var externalRule = splitRule[0]

cli/src/main/kotlin/com/bazel_diff/cli/GenerateHashesCommand.kt

@@ -1,5 +1,6 @@
 package com.bazel_diff.cli
 
+import com.bazel_diff.cli.converter.CommaSeparatedValueConverter
 import com.bazel_diff.cli.converter.NormalisingPathConverter
 import com.bazel_diff.cli.converter.OptionsConverter
 import com.bazel_diff.di.hasherModule
@@ -64,6 +65,14 @@ class GenerateHashesCommand : Callable<Int> {
     )
     var bazelCommandOptions: List<String> = emptyList()
 
+    @CommandLine.Option(
+        names = ["--fineGrainedHashExternalRepos"],
+        description = ["Comma separate list of external repos in which fine-grained hashes are computed for the targets. By default, external repos are treated as an opaque blob. If an external repo is specified here, bazel-diff instead computes the hash for individual targets. For example, one wants to specify `maven` here if they user rules_jvm_external so that individual third party dependency change won't invalidate all targets in the mono repo."],
+        scope = CommandLine.ScopeType.INHERIT,
+        converter = [CommaSeparatedValueConverter::class],
+    )
+    var fineGrainedHashExternalRepos: Set<String> = emptySet()
+
     @CommandLine.Option(
         names = ["-k", "--keep_going"],
         negatable = true,
@@ -89,7 +98,7 @@ class GenerateHashesCommand : Callable<Int> {
     lateinit var spec: CommandLine.Model.CommandSpec
 
     override fun call(): Int {
-        validate(contentHashPath=contentHashPath)
+        validate(contentHashPath = contentHashPath)
 
         startKoin {
             modules(
@@ -100,6 +109,7 @@ class GenerateHashesCommand : Callable<Int> {
                     bazelStartupOptions,
                     bazelCommandOptions,
                     keepGoing,
+                    fineGrainedHashExternalRepos,
                 ),
                 loggingModule(parent.verbose),
                 serialisationModule(),

cli/src/main/kotlin/com/bazel_diff/cli/converter/CommaSeparatedValueConverter.kt

@@ -0,0 +1,9 @@
+package com.bazel_diff.cli.converter
+
+import picocli.CommandLine.ITypeConverter
+
+class CommaSeparatedValueConverter : ITypeConverter<Set<String>> {
+    override fun convert(value: String): Set<String> {
+        return value.split(",").dropLastWhile { it.isEmpty() }.toSet()
+    }
+}

cli/src/main/kotlin/com/bazel_diff/di/Modules.kt

@@ -23,6 +23,7 @@ fun hasherModule(
     startupOptions: List<String>,
     commandOptions: List<String>,
     keepGoing: Boolean?,
+    fineGrainedHashExternalRepos: Set<String>,
 ): Module = module {
     val debug = System.getProperty("DEBUG", "false").equals("true")
     single {
@@ -35,10 +36,10 @@ fun hasherModule(
             debug
         )
     }
-    single { BazelClient() }
+    single { BazelClient(fineGrainedHashExternalRepos) }
     single { BuildGraphHasher(get()) }
     single { TargetHasher() }
-    single { RuleHasher() }
+    single { RuleHasher(fineGrainedHashExternalRepos) }
     single { SourceFileHasher() }
     single(named("working-directory")) { workingDirectory }
     single { ContentHashProvider(contentHashPath) }

cli/src/main/kotlin/com/bazel_diff/hash/RuleHasher.kt

@@ -8,9 +8,10 @@ import org.koin.core.component.KoinComponent
 import org.koin.core.component.inject
 import java.util.concurrent.ConcurrentMap
 
-class RuleHasher : KoinComponent {
+class RuleHasher(private val fineGrainedHashExternalRepos: Set<String>) : KoinComponent {
     private val logger: Logger by inject()
     private val sourceFileHasher: SourceFileHasher by inject()
+
     @VisibleForTesting
     class CircularDependencyException(message: String) : Exception(message)
 
@@ -42,14 +43,15 @@ class RuleHasher : KoinComponent {
             safePutBytes(rule.digest)
             safePutBytes(seedHash)
 
-            for (ruleInput in rule.ruleInputList) {
+            for (ruleInput in rule.ruleInputList(fineGrainedHashExternalRepos)) {
                 safePutBytes(ruleInput.toByteArray())
 
                 val inputRule = allRulesMap[ruleInput]
                 when {
                     inputRule == null && sourceDigests.containsKey(ruleInput) -> {
                         safePutBytes(sourceDigests[ruleInput])
                     }
+
                     inputRule?.name != null && inputRule.name != rule.name -> {
                         val ruleInputHash = digest(
                             inputRule,
@@ -61,6 +63,7 @@ class RuleHasher : KoinComponent {
                         )
                         safePutBytes(ruleInputHash)
                     }
+
                     else -> {
                         val heuristicDigest = sourceFileHasher.softDigest(BazelSourceFileTarget(ruleInput, ByteArray(0)))
                         when {
@@ -69,6 +72,7 @@ class RuleHasher : KoinComponent {
                                 sourceDigests[ruleInput] = heuristicDigest
                                 safePutBytes(heuristicDigest)
                             }
+
                             else -> logger.w { "Unable to calculate digest for input $ruleInput for rule ${rule.name}" }
                         }
                     }

cli/src/test/kotlin/com/bazel_diff/Modules.kt

@@ -15,10 +15,10 @@ import java.nio.file.Paths
 
 fun testModule(): Module = module {
     single<Logger> { SilentLogger }
-    single { BazelClient() }
+    single { BazelClient(emptySet()) }
     single { BuildGraphHasher(get()) }
     single { TargetHasher() }
-    single { RuleHasher() }
+    single { RuleHasher(emptySet()) }
     single { SourceFileHasher() }
     single { GsonBuilder().setPrettyPrinting().create() }
     single(named("working-directory")) { Paths.get("working-directory") }

cli/src/test/kotlin/com/bazel_diff/cli/converter/CommaSeparatedValueConverterTest.kt

@@ -0,0 +1,13 @@
+package com.bazel_diff.cli.converter
+
+import assertk.assertThat
+import assertk.assertions.isEqualTo
+import org.junit.Test
+
+class CommaSeparatedValueConverterTest {
+    @Test
+    fun testConverter() {
+        val converter = CommaSeparatedValueConverter()
+        assertThat(converter.convert("a,b,c,d")).isEqualTo(listOf("a", "b", "c", "d"))
+    }
+}

cli/src/test/kotlin/com/bazel_diff/e2e/E2ETest.kt

@@ -1,28 +1,16 @@
 package com.bazel_diff.e2e
 
 import assertk.assertThat
-import assertk.assertions.contains
 import assertk.assertions.containsExactlyInAnyOrder
-import assertk.assertions.containsOnly
 import assertk.assertions.isEqualTo
-import com.bazel_diff.Main
 import com.bazel_diff.cli.BazelDiff
-import com.bazel_diff.interactor.DeserialiseHashesInteractor
-import com.bazel_diff.testModule
-import com.google.gson.Gson
 import org.junit.Rule
 import org.junit.Test
 import org.junit.rules.TemporaryFolder
-import org.koin.core.context.startKoin
 import picocli.CommandLine
 import java.io.File
-import java.io.FileInputStream
-import java.io.IOException
-import java.nio.file.Path
 import java.nio.file.Paths
-import java.nio.file.StandardCopyOption
 import java.util.zip.ZipFile
-import java.util.zip.ZipInputStream
 
 
 class E2ETest {
@@ -63,6 +51,62 @@ class E2ETest {
         assertThat(actual).isEqualTo(expected)
     }
 
+    @Test
+    fun testFineGrainedHashExternalRepo() {
+        // The difference between these two snapshot is simply upgrading Guava version. Following
+        // is the diff.
+        //
+        //   diff --git a/integration/WORKSPACE b/integration/WORKSPACE
+        //   index 617a8d6..2cb3c7d 100644
+        //   --- a/integration/WORKSPACE
+        //   +++ b/integration/WORKSPACE
+        //   @@ -15,7 +15,7 @@ maven_install(
+        //        name = "bazel_diff_maven",
+        //        artifacts = [
+        //          "junit:junit:4.12",
+        //   -      "com.google.guava:guava:31.0-jre",
+        //   +      "com.google.guava:guava:31.1-jre",
+        //        ],
+        //        repositories = [
+        //            "http://uk.maven.org/maven2",
+        //
+        // The project contains a single target that depends on Guava:
+        // //src/main/java/com/integration:guava-user
+        //
+        // So this target, its derived targets, and all other changed external targets should be
+        // the only impacted targets.
+        val projectA = extractFixtureProject("/fixture/fine-grained-hash-external-repo-test-1.zip")
+        val projectB = extractFixtureProject("/fixture/fine-grained-hash-external-repo-test-2.zip")
+
+        val workingDirectoryA = projectA
+        val workingDirectoryB = projectB
+        val bazelPath = "bazel"
+        val outputDir = temp.newFolder()
+        val from = File(outputDir, "starting_hashes.json")
+        val to = File(outputDir, "final_hashes.json")
+        val impactedTargetsOutput = File(outputDir, "impacted_targets.txt")
+
+        val cli = CommandLine(BazelDiff())
+        //From
+        cli.execute(
+            "generate-hashes", "-w", workingDirectoryA.absolutePath, "-b", bazelPath, "--fineGrainedHashExternalRepos", "bazel_diff_maven", from.absolutePath
+        )
+        //To
+        cli.execute(
+            "generate-hashes", "-w", workingDirectoryB.absolutePath, "-b", bazelPath, "--fineGrainedHashExternalRepos", "bazel_diff_maven", to.absolutePath
+        )
+        //Impacted targets
+        cli.execute(
+            "get-impacted-targets", "-sh", from.absolutePath, "-fh", to.absolutePath, "-o", impactedTargetsOutput.absolutePath
+        )
+
+        val actual: Set<String> = impactedTargetsOutput.readLines().filter { it.isNotBlank() }.toSet()
+        val expected: Set<String> =
+            javaClass.getResourceAsStream("/fixture/fine-grained-hash-external-repo-test-impacted-targets.txt").use { it.bufferedReader().readLines().filter { it.isNotBlank() }.toSet() }
+
+        assertThat(actual).isEqualTo(expected)
+    }
+
     private fun extractFixtureProject(path: String): File {
         val testProject = temp.newFolder()
         val fixtureCopy = temp.newFile()

cli/src/test/kotlin/com/bazel_diff/hash/BuildGraphHasherTest.kt

@@ -1,10 +1,8 @@
 package com.bazel_diff.hash
 
 import assertk.all
-import assertk.assertAll
 import assertk.assertThat
 import assertk.assertions.*
-import assertk.assertions.any
 import com.bazel_diff.bazel.BazelClient
 import com.bazel_diff.bazel.BazelRule
 import com.bazel_diff.bazel.BazelTarget
@@ -21,10 +19,8 @@ import org.koin.test.mock.MockProviderRule
 import org.koin.test.mock.declareMock
 import org.mockito.Mockito
 import org.mockito.junit.MockitoJUnit
-import org.mockito.kotlin.any
 import org.mockito.kotlin.mock
 import org.mockito.kotlin.whenever
-import java.util.*
 
 
 class BuildGraphHasherTest : KoinTest {
@@ -148,7 +144,7 @@ class BuildGraphHasherTest : KoinTest {
             // they are run in parallel, so we don't know whether rule3 or rule4 will be processed first
             message().matchesPredicate {
                 it!!.contains("\\brule3 -> rule4 -> rule3\\b".toRegex()) ||
-                it.contains("\\brule4 -> rule3 -> rule4\\b".toRegex())
+                    it.contains("\\brule4 -> rule3 -> rule4\\b".toRegex())
             }
         }
     }
@@ -178,7 +174,7 @@ class BuildGraphHasherTest : KoinTest {
         val target = mock<BazelTarget.Rule>()
         val rule = mock<BazelRule>()
         whenever(rule.name).thenReturn(name)
-        whenever(rule.ruleInputList).thenReturn(inputs)
+        whenever(rule.ruleInputList(emptySet())).thenReturn(inputs)
         whenever(rule.digest).thenReturn(digest.toByteArray())
         whenever(target.rule).thenReturn(rule)
         whenever(target.name).thenReturn(name)

cli/src/test/resources/fixture/fine-grained-hash-external-repo-test-impacted-targets.txt

@@ -0,0 +1,10 @@
+//external:bazel_diff_maven
+//src/main/java/com/integration:guava-user
+//src/main/java/com/integration:libguava-user-src.jar
+//src/main/java/com/integration:libguava-user.jar
+@bazel_diff_maven//:com_google_errorprone_error_prone_annotations
+@bazel_diff_maven//:com_google_errorprone_error_prone_annotations_2_11_0
+@bazel_diff_maven//:com_google_guava_guava
+@bazel_diff_maven//:com_google_guava_guava_31_1_jre
+@bazel_diff_maven//:v1/https/jcenter.bintray.com/com/google/errorprone/error_prone_annotations/2.11.0/error_prone_annotations-2.11.0.jar
+@bazel_diff_maven//:v1/https/jcenter.bintray.com/com/google/guava/guava/31.1-jre/guava-31.1-jre.jar