Add dependency reporting (#371)

* Direct dependencies events added * No Configure on Demand to make sure dependencies are resolved * Add traversing dependencies * Try running statistics report on CI * Remove Module Statistics report from check workflow * AAdded proper classification of dependencies
jraska · Dec 12, 2020 · 5f58340 · 5f58340
1 parent f8bfa05
commit 5f58340
Show file tree

Hide file tree

Showing 5 changed files with 110 additions and 3 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -34,7 +34,7 @@ jobs:
           key: cache-{{ checksum "build.gradle" }}-{{ checksum  "dependencies.gradle" }}-{{ checksum  "app/build.gradle" }}
       - run:
           name: Execute Data Processing
-          command: ./gradlew reportModuleStatistics --max-workers=2 --stacktrace
+          command: ./gradlew reportModuleStatistics --max-workers=2 --stacktrace --no-configure-on-demand
       - save_cache:
           paths:
             - ~/.gradle

diff --git a/plugins/src/main/java/com/jraska/gradle/buildtime/BuildReporter.kt b/plugins/src/main/java/com/jraska/gradle/buildtime/BuildReporter.kt
@@ -26,8 +26,6 @@ class BuildReporter(
   }
 
   private fun reportInternal(buildData: BuildData) {
-    val delivery = ClientDelivery()
-
     val properties = convertBuildData(buildData)
     val event = AnalyticsEvent("Android Build", properties)
     analyticsReporter.report(event)

diff --git a/plugins/src/main/java/com/jraska/module/ModuleStatistics.kt b/plugins/src/main/java/com/jraska/module/ModuleStatistics.kt
@@ -9,6 +9,7 @@ data class ProjectStatistics(
   val prodJavaTotalLines: Int,
   val prodXmlTotalLines: Int,
   val moduleStatistics: List<ModuleStatistics>,
+  val externalDependencies: List<ModuleArtifactDependency>
 )
 
 data class ModuleStatistics(
@@ -25,6 +26,23 @@ data class FileTypeStatistics(
   val type: FileType
 )
 
+data class ModuleArtifactDependency(
+  val moduleName: String,
+  val type: ModuleType,
+  val fullName: String,
+  val group: String,
+  val artifact: String,
+  val version: String?,
+  val dependencyType: ArtifactDependencyType
+)
+
+enum class ArtifactDependencyType {
+  Compile,
+  Test,
+  AndroidTest,
+  Kapt
+}
+
 enum class ModuleType {
   Api,
   Implementation,

diff --git a/plugins/src/main/java/com/jraska/module/ModuleStatsReporter.kt b/plugins/src/main/java/com/jraska/module/ModuleStatsReporter.kt
@@ -17,6 +17,11 @@ class ModuleStatsReporter(
       events.add(AnalyticsEvent("Android Module Detail", properties))
     }
 
+    stats.externalDependencies.forEach {
+      val properties = convertModuleDependency(it)
+      events.add(AnalyticsEvent("Android Module External Dependency", properties))
+    }
+
     analyticsReporter.report(*events.toTypedArray())
 
     println("$GRAPH_ICON Module stats reported to Mixpanel $GRAPH_ICON")
@@ -63,6 +68,18 @@ class ModuleStatsReporter(
     }
   }
 
+  private fun convertModuleDependency(artifactDependency: ModuleArtifactDependency): Map<String, Any?> {
+    return mapOf<String, Any?>(
+      "moduleName" to artifactDependency.moduleName,
+      "moduleType" to artifactDependency.type.name,
+      "groupId" to artifactDependency.group,
+      "artifactId" to artifactDependency.artifact,
+      "version" to artifactDependency.version,
+      "fullComponentName" to artifactDependency.fullName,
+      "dependencyType" to artifactDependency.dependencyType
+    )
+  }
+
   companion object {
     private val GRAPH_ICON = "\uD83D\uDCC9"
   }

diff --git a/plugins/src/main/java/com/jraska/module/extract/StatisticsGradleExtractor.kt b/plugins/src/main/java/com/jraska/module/extract/StatisticsGradleExtractor.kt
@@ -1,14 +1,20 @@
 package com.jraska.module.extract
 
+import com.jraska.module.ArtifactDependencyType
 import com.jraska.module.FileType
 import com.jraska.module.FileTypeStatistics
+import com.jraska.module.ModuleArtifactDependency
 import com.jraska.module.ModuleStatistics
 import com.jraska.module.ModuleType
 import com.jraska.module.ProjectStatistics
 import org.gradle.api.Project
+import org.gradle.api.artifacts.ProjectDependency
+import org.gradle.api.artifacts.ResolvedDependency
 import java.io.BufferedReader
 import java.io.File
 import java.io.FileReader
+import java.util.LinkedList
+import java.util.Queue
 
 class StatisticsGradleExtractor() {
   private val typesToLookFor = arrayOf(FileType.JAVA, FileType.KOTLIN, FileType.XML)
@@ -19,12 +25,18 @@ class StatisticsGradleExtractor() {
       .filter { it.childProjects.isEmpty() }
       .map { extractFromModule(it) }
 
+    val externalDependencies = project.rootProject
+      .subprojects
+      .filter { it.childProjects.isEmpty() }
+      .flatMap { extractDependencies(it) }
+
     return ProjectStatistics(
       modulesCount = moduleStats.size,
       apiModules = moduleStats.count { it.type == ModuleType.Api },
       appModules = moduleStats.count { it.type == ModuleType.App },
       implementationModules = moduleStats.count { it.type == ModuleType.Implementation },
       moduleStatistics = moduleStats,
+      externalDependencies = externalDependencies,
       prodKotlinTotalLines = moduleStats
         .map { module -> module.containedProdFiles.single { it.type == FileType.KOTLIN }.lineCount }
         .sum(),
@@ -39,6 +51,56 @@ class StatisticsGradleExtractor() {
     )
   }
 
+  private fun extractDependencies(module: Project): List<ModuleArtifactDependency> {
+    val moduleType = moduleType(module)
+    val moduleName = module.name
+
+    val dependencies = module.configurations
+      .filter { CONFIGURATION_TO_LOOK.containsKey(it.name) }
+      .flatMap { configuration ->
+        val projectDependencies = configuration.allDependencies.filterIsInstance(ProjectDependency::class.java)
+        configuration.resolvedConfiguration.firstLevelModuleDependencies
+          .filter { isExternalDependency(it, projectDependencies) }
+          .flatMap { traverseAndAddChildren(it) }
+          .map {
+            ModuleArtifactDependency(
+              moduleName = moduleName,
+              type = moduleType,
+              group = it.moduleGroup,
+              dependencyType = CONFIGURATION_TO_LOOK.getValue(configuration.name),
+              artifact = it.moduleName,
+              version = it.moduleVersion,
+              fullName = it.name
+            )
+          }
+      }.distinct()
+
+    return dependencies
+  }
+
+  private fun traverseAndAddChildren(firstLevelDependency: ResolvedDependency): Set<ResolvedDependency> {
+    val queue: Queue<ResolvedDependency> = LinkedList()
+
+    val dependencies = mutableSetOf(firstLevelDependency)
+    queue.offer(firstLevelDependency)
+
+    while (queue.isNotEmpty()) {
+      val element = queue.poll()
+      element.children.forEach { child ->
+        dependencies.add(child)
+        queue.offer(child)
+      }
+    }
+
+    return dependencies
+  }
+
+  private fun isExternalDependency(dependency: ResolvedDependency, projectDependencies: List<ProjectDependency>): Boolean {
+    return projectDependencies.none {
+      it.group == dependency.moduleGroup && it.name == dependency.moduleName
+    }
+  }
+
   private fun extractFromModule(module: Project): ModuleStatistics {
     val prodFileStats = typesToLookFor.map { getFileTypeStatistics(it, File(module.projectDir, "src/main")) }
     val unitTestFileStats = typesToLookFor.map { getFileTypeStatistics(it, File(module.projectDir, "src/test")) }
@@ -77,4 +139,16 @@ class StatisticsGradleExtractor() {
     }
     return lines
   }
+
+  companion object {
+    val CONFIGURATION_TO_LOOK = mapOf(
+      "debugAndroidTestCompileClasspath" to ArtifactDependencyType.AndroidTest,
+      "debugCompileClasspath" to ArtifactDependencyType.Compile,
+      "releaseCompileClasspath" to ArtifactDependencyType.Compile,
+      "debugUnitTestCompileClasspath" to ArtifactDependencyType.Test,
+      "compileClasspath" to ArtifactDependencyType.Compile,
+      "testCompileClasspath" to ArtifactDependencyType.Test,
+      "kapt" to ArtifactDependencyType.Kapt
+    )
+  }
 }