Part 1: Implement per repository maven_install for content filtering (

#44)

* Group dependencies by variants

* Handle configuration splitting to consider buildType as well

* Calculate exclude rules

* Add extension to merge list of maps and simplify variant configuration map calculation

* Cleanup filter dependencies calculation

* Apply formatting and refactor names

* Update bazel scripts

* Address review comments

WORKSPACE

@@ -85,7 +85,7 @@ maven_install(
                 "androidx.core:core",
             ],
             group = "androidx.constraintlayout",
-            version = "1.1.2",
+            version = "2.1.1",
         ),
         "androidx.core:core:1.5.0",
         "androidx.databinding:databinding-adapters:7.1.2",
@@ -95,9 +95,10 @@ maven_install(
         "androidx.databinding:viewbinding:7.1.2",
         "androidx.test.espresso:espresso-core:3.4.0",
         "androidx.test.ext:junit:1.1.3",
+        "com.squareup.leakcanary:leakcanary-android:2.10",
         "junit:junit:4.13.2",
         "org.jacoco:org.jacoco.ant:0.8.3",
-        "org.jetbrains.kotlin:kotlin-annotation-processing-gradle:1.6.21",
+        "org.jetbrains.kotlin:kotlin-annotation-processing-gradle:1.6.10",
         "org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.4.32",
         "org.jetbrains.kotlin:kotlin-stdlib:1.4.32",
     ],
@@ -117,6 +118,7 @@ maven_install(
         "androidx.test.espresso:espresso-core",
         "androidx.test.ext:junit",
         "com.android.support:cardview-v7",
+        "com.squareup.leakcanary:leakcanary-android",
         "junit:junit",
         "org.jacoco:org.jacoco.ant",
         "org.jetbrains.kotlin:kotlin-annotation-processing-gradle",
@@ -130,8 +132,6 @@ maven_install(
     repositories = DAGGER_REPOSITORIES + [
         "https://dl.google.com/dl/android/maven2/",
         "https://repo.maven.apache.org/maven2/",
-        "https://maven.google.com",
-        "https://repo1.maven.org/maven2",
     ],
     resolve_timeout = 1000,
     version_conflict_policy = "pinned",

build.gradle

@@ -87,6 +87,7 @@ grazel {
         variantFilter { variant ->
             if (
             variant.name != "flavor1Debug" &&
+                variant.name != "flavor2Debug" &&
                 variant.name != "flavor1DebugAndroidTest" &&
                 variant.name != "debug" &&
                 variant.name != "debugUnitTest"

grazel-gradle-plugin/src/main/kotlin/com/grab/grazel/di/GrazelComponent.kt

@@ -34,6 +34,7 @@ import com.grab.grazel.gradle.dependencies.DependenciesDataSource
 import com.grab.grazel.gradle.dependencies.DependenciesGraphsBuilder
 import com.grab.grazel.gradle.dependencies.DependenciesModule
 import com.grab.grazel.gradle.dependencies.DependencyGraphs
+import com.grab.grazel.gradle.dependencies.MavenInstallArtifactsCalculator
 import com.grab.grazel.migrate.builder.AndroidBinaryTargetBuilderModule
 import com.grab.grazel.migrate.builder.AndroidInstrumentationBinaryTargetBuilderModule
 import com.grab.grazel.migrate.builder.AndroidLibTargetBuilderModule
@@ -84,6 +85,7 @@ internal interface GrazelComponent {
     fun rootBazelFileBuilder(): Lazy<RootBazelFileBuilder>
     fun artifactsPinner(): Lazy<ArtifactsPinner>
     fun dependenciesDataSource(): Lazy<DependenciesDataSource>
+    fun mavenInstallArtifactsCalculator(): Lazy<MavenInstallArtifactsCalculator>
 }
 
 @Module(

grazel-gradle-plugin/src/main/kotlin/com/grab/grazel/gradle/BuildVariant.kt

@@ -31,6 +31,21 @@ import org.gradle.kotlin.dsl.the
 import javax.inject.Inject
 import javax.inject.Singleton
 
+
+sealed class VariantInfo {
+    object Default : VariantInfo() {
+        override fun toString() = "default"
+    }
+
+    data class AndroidFlavor(val flavorName: String) : VariantInfo() {
+        override fun toString() = flavorName
+    }
+
+    data class AndroidVariant(val baseVariant: BaseVariant) : VariantInfo() {
+        override fun toString(): String = baseVariant.name
+    }
+}
+
 internal interface AndroidVariantDataSource {
     /**
      * Variant filter instance to filter out unsupported variants

grazel-gradle-plugin/src/main/kotlin/com/grab/grazel/gradle/Configuration.kt

@@ -18,6 +18,8 @@ package com.grab.grazel.gradle
 
 import com.android.build.gradle.api.BaseVariant
 import com.grab.grazel.GrazelExtension
+import com.grab.grazel.gradle.VariantInfo.AndroidFlavor
+import com.grab.grazel.gradle.VariantInfo.AndroidVariant
 import com.grab.grazel.gradle.dependencies.BuildGraphType
 import org.gradle.api.Project
 import org.gradle.api.artifacts.Configuration
@@ -26,7 +28,9 @@ import javax.inject.Inject
 import javax.inject.Singleton
 
 enum class ConfigurationScope(val scopeName: String) {
-    BUILD(""), TEST("UnitTest"), ANDROID_TEST("AndroidTest");
+    BUILD(""),
+    TEST("UnitTest"),
+    ANDROID_TEST("AndroidTest");
 }
 
 internal fun GrazelExtension.configurationScopes(): Array<ConfigurationScope> {
@@ -57,14 +61,19 @@ internal interface ConfigurationDataSource {
      */
     fun configurations(
         project: Project,
-        vararg scope: ConfigurationScope
+        vararg scopes: ConfigurationScope
     ): Sequence<Configuration>
 
     fun isThisConfigurationBelongsToThisVariants(
         project: Project,
         vararg variants: BaseVariant?,
         configuration: Configuration
     ): Boolean
+
+    fun configurationByVariant(
+        project: Project,
+        vararg scopes: ConfigurationScope
+    ): Map<VariantInfo, List<Configuration>>
 }
 
 @Singleton
@@ -78,11 +87,31 @@ internal class DefaultConfigurationDataSource @Inject constructor(
     ): Sequence<Configuration> {
         val ignoreFlavors = androidVariantDataSource.getIgnoredFlavors(project)
         val ignoreVariants = androidVariantDataSource.getIgnoredVariants(project)
+        return filterConfigurations(project, scopes)
+            .filter { config ->
+                !config.name.let { configurationName ->
+                    ignoreFlavors.any { configurationName.contains(it.name, true) }
+                        || ignoreVariants.any { configurationName.contains(it.name, true) }
+                }
+            }
+    }
+
+    /**
+     * Only allow relevant configurations required for build. Ideally for android projects we could
+     * rely on [BaseVariant.getCompileConfiguration] etc but other configurations added by plugins won't
+     * be present there. Hence we get all configurations in the project and filter by name with some
+     * assumptions
+     */
+    private fun filterConfigurations(
+        project: Project,
+        scopes: Array<out ConfigurationScope> = ConfigurationScope.values(),
+        variantNameFilter: String? = null
+    ): Sequence<Configuration> {
         return project.configurations
             .asSequence()
             .filter { !it.name.contains("classpath", true) && !it.name.contains("lint") }
             .filter { !it.name.contains("coreLibraryDesugaring") }
-            .filter { !it.name.contains("_internal_aapt2_binary") }
+            .filter { !it.name.startsWith("_") }
             .filter { !it.name.contains("archives") }
             .filter { !it.isDynamicConfiguration() } // Remove when Grazel support dynamic-feature plugin
             .filter { configuration ->
@@ -97,13 +126,9 @@ internal class DefaultConfigurationDataSource @Inject constructor(
                     }
                 }
             }
-            .distinct()
-            .filter { config ->
-                !config.name.let { configurationName ->
-                    ignoreFlavors.any { configurationName.contains(it.name, true) }
-                        || ignoreVariants.any { configurationName.contains(it.name, true) }
-                }
-            }
+            .filter {
+                if (variantNameFilter != null) it.name.startsWith(variantNameFilter) else true
+            }.distinct()
     }
 
     override fun isThisConfigurationBelongsToThisVariants(
@@ -130,6 +155,44 @@ internal class DefaultConfigurationDataSource @Inject constructor(
             *buildGraphTypes.map { it.configurationScope }.toTypedArray()
         ).filter { it.isCanBeResolved }
     }
+
+    override fun configurationByVariant(
+        project: Project,
+        vararg scopes: ConfigurationScope
+    ): Map<VariantInfo, List<Configuration>> {
+        return if (project.isAndroid) {
+            val availableConfigurations = configurations(project)
+            val availableVariants = androidVariantDataSource.getMigratableVariants(project)
+            return availableConfigurations
+                .groupBy { configuration -> calculateVariantKey(availableVariants, configuration) }
+        } else {
+            mapOf(VariantInfo.Default to configurations(project).toList())
+        }
+    }
+
+    /** Return the grouping key that will be used to split the configurations,
+     *  first checks a matching variant, then flavor, then build type and fallback to `VariantInfo.Default`.
+     *  We could ideally just group by variant name but that is not sufficient since
+     *  there can be configurations like `debugImplementation` etc that might not
+     *  have been caught by variant name check but will be caught by buildType check.
+     */
+    private fun calculateVariantKey(
+        availableVariants: List<BaseVariant>,
+        configuration: Configuration
+    ): VariantInfo {
+        val matchingVariant = availableVariants
+            .firstOrNull() { variant -> configuration.name.startsWith(variant.name) }
+        val matchingFlavor = availableVariants
+            .firstOrNull() { variant -> configuration.name.startsWith(variant.flavorName) }
+        val matchingBuildType = availableVariants
+            .firstOrNull() { variant -> configuration.name.startsWith(variant.buildType.name) }
+        return when {
+            matchingVariant != null -> AndroidVariant(matchingVariant)
+            matchingFlavor?.flavorName?.isNotEmpty() == true -> AndroidFlavor(matchingFlavor.flavorName)
+            matchingBuildType != null -> AndroidFlavor(matchingBuildType.name)
+            else -> VariantInfo.Default
+        }
+    }
 }
 
 internal fun Configuration.isUnitTest() = name.contains("UnitTest", true) || name.startsWith("test")

grazel-gradle-plugin/src/main/kotlin/com/grab/grazel/gradle/Repository.kt

@@ -31,6 +31,11 @@ internal interface RepositoryDataSource {
      */
     val allRepositories: List<DefaultMavenArtifactRepository>
 
+    /**
+     * All configured Maven repositories in the project, key by their Gradle name.
+     */
+    val allRepositoriesByName: Map<String, DefaultMavenArtifactRepository>
+
     /**
      * The Maven repositories among `allRepositories` that can be migrated. This is usually list of Maven repositories
      * without any auth or only Basic Auth.
@@ -62,6 +67,12 @@ internal class DefaultRepositoryDataSource @Inject constructor(
             .filter { it !is DefaultMavenLocalArtifactRepository }
             .toList()
     }
+    override val allRepositoriesByName: Map<String, DefaultMavenArtifactRepository> by lazy {
+        allRepositories
+            .map { it.name to it }
+            .distinctBy { it.first }
+            .toMap()
+    }
 
     override val supportedRepositories: List<DefaultMavenArtifactRepository> by lazy {
         allRepositories

grazel-gradle-plugin/src/main/kotlin/com/grab/grazel/gradle/dependencies/Dependencies.kt

@@ -38,15 +38,6 @@ import java.io.File
 import javax.inject.Inject
 import javax.inject.Singleton
 
-/**
- * TODO To remove this once test rules support is added
- */
-private val DEFAULT_MAVEN_ARTIFACTS: List<MavenArtifact> = listOf(
-    MavenArtifact("junit", "junit", "4.12"),
-    MavenArtifact("org.mockito", "mockito-core", "3.4.6"),
-    MavenArtifact("com.nhaarman", "mockito-kotlin", "1.6.0")
-)
-
 /**
  * Maven group names for artifacts that should be excluded from dependencies calculation everywhere.
  */
@@ -55,7 +46,11 @@ internal val DEP_GROUP_EMBEDDED_BY_RULES = listOf(
     "org.jetbrains.kotlin"
 )
 
-data class ExcludeRule(
+@Deprecated(
+    "No longer used, use com.grab.grazel.gradle.dependencies.ExcludeRule instead.",
+    replaceWith = ReplaceWith("ExcludeRule", "com.grab.grazel.gradle.dependencies.ExcludeRule")
+)
+data class ExcludeRuleOld(
     val group: String,
     val artifact: String
 ) {
@@ -69,7 +64,7 @@ internal data class MavenArtifact(
     val group: String?,
     val name: String?,
     val version: String? = null,
-    val excludeRules: Set<ExcludeRule> = emptySet()
+    val excludeRuleOlds: Set<ExcludeRuleOld> = emptySet()
 ) {
     val id get() = "$group:$name"
     override fun toString() = "$group:$name:$version"
@@ -263,15 +258,15 @@ internal class DefaultDependenciesDataSource @Inject constructor(
             }
             .let(::collectForcedVersions)
 
-        return (DEFAULT_MAVEN_ARTIFACTS + externalArtifacts + forcedVersions)
+        return (externalArtifacts + forcedVersions)
             .groupBy { it.id }
             .map { (_, mavenArtifacts) ->
                 // Merge all exclude rules so that we have a cumulative set
                 mavenArtifacts
                     .first()
                     .copy(
-                        excludeRules = mavenArtifacts
-                            .flatMap(MavenArtifact::excludeRules)
+                        excludeRuleOlds = mavenArtifacts
+                            .flatMap(MavenArtifact::excludeRuleOlds)
                             .toSet()
                     )
             }.asSequence()
@@ -456,11 +451,11 @@ internal class DefaultDependenciesDataSource @Inject constructor(
             group = group,
             name = name,
             version = version,
-            excludeRules = excludeRules
+            excludeRuleOlds = excludeRules
                 .asSequence()
                 .map {
                     @Suppress("USELESS_ELVIS") // Gradle lying, module can be null
-                    ExcludeRule(it.group, it.module ?: "")
+                    ExcludeRuleOld(it.group, it.module ?: "")
                 }
                 .filterNot { it.artifact.isNullOrBlank() }
                 .filterNot { excludeArtifactsDenyList.contains(it.toString()) }
@@ -477,12 +472,12 @@ internal class DefaultDependenciesDataSource @Inject constructor(
 
 internal fun MavenArtifact.toMavenInstallArtifact(): MavenInstallArtifact {
     return when {
-        excludeRules.isEmpty() -> MavenInstallArtifact.SimpleArtifact(toString())
+        excludeRuleOlds.isEmpty() -> MavenInstallArtifact.SimpleArtifact(toString())
         else -> MavenInstallArtifact.DetailedArtifact(
             group = group!!,
             artifact = name!!,
             version = version!!,
-            exclusions = excludeRules.map { SimpleExclusion("${it.group}:${it.artifact}") }
+            exclusions = excludeRuleOlds.map { SimpleExclusion("${it.group}:${it.artifact}") }
         )
     }
 }

grazel-gradle-plugin/src/main/kotlin/com/grab/grazel/gradle/dependencies/DependenciesGraphsBuilder.kt

@@ -61,22 +61,21 @@ internal class DependenciesGraphsBuilder @Inject constructor(
                         androidVariantDataSource.getMigratableVariants(
                             sourceProject,
                             configurationScope
-                        )
-                            .forEach { variant ->
-                                if (configurationDataSource.isThisConfigurationBelongsToThisVariants(
-                                        sourceProject,
-                                        variant,
-                                        configuration = configuration
-                                    )
-                                ) {
-                                    buildGraphs.putEdgeValue(
-                                        BuildGraphType(configurationScope, variant),
-                                        sourceProject,
-                                        projectDependency.dependencyProject,
-                                        configuration
-                                    )
-                                }
+                        ).forEach { variant ->
+                            if (configurationDataSource.isThisConfigurationBelongsToThisVariants(
+                                    sourceProject,
+                                    variant,
+                                    configuration = configuration
+                                )
+                            ) {
+                                buildGraphs.putEdgeValue(
+                                    BuildGraphType(configurationScope, variant),
+                                    sourceProject,
+                                    projectDependency.dependencyProject,
+                                    configuration
+                                )
                             }
+                        }
                     }
             }
         }
@@ -97,8 +96,10 @@ internal class DependenciesGraphsBuilder @Inject constructor(
     ) {
         dependenciesDataSource.projectDependencies(project, configurationScope)
             .forEach { (configuration, projectDependency) ->
-                val variants =
-                    androidVariantDataSource.getMigratableVariants(project, configurationScope)
+                val variants = androidVariantDataSource.getMigratableVariants(
+                    project,
+                    configurationScope
+                )
                 if (variants.isNotEmpty()) {
                     variants.forEach { variant ->
                         if (variant.compileConfiguration.hierarchy.contains(configuration)) {
@@ -137,7 +138,7 @@ internal class DependenciesGraphsBuilder @Inject constructor(
         ) {
             buildGraphs.addNode(BuildGraphType(configurationScope, null), sourceProject)
         } else {
-            rootProject.logger.warn("${sourceProject.name} is a simple directory")
+            rootProject.logger.info("${sourceProject.name} is a simple directory")
         }
     }
 }