From 5f583401383848e7318fce434eeaf106924c551c Mon Sep 17 00:00:00 2001 From: Josef Raska <6277721+jraska@users.noreply.github.com> Date: Sat, 12 Dec 2020 21:35:12 +0100 Subject: [PATCH] Add dependency reporting (#371) * Direct dependencies events added * No Configure on Demand to make sure dependencies are resolved * Add traversing dependencies * Try running statistics report on CI * Remove Module Statistics report from check workflow * AAdded proper classification of dependencies --- .circleci/config.yml | 2 +- .../jraska/gradle/buildtime/BuildReporter.kt | 2 - .../com/jraska/module/ModuleStatistics.kt | 18 +++++ .../com/jraska/module/ModuleStatsReporter.kt | 17 +++++ .../extract/StatisticsGradleExtractor.kt | 74 +++++++++++++++++++ 5 files changed, 110 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f2341cfd..85c93102 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -34,7 +34,7 @@ jobs: key: cache-{{ checksum "build.gradle" }}-{{ checksum "dependencies.gradle" }}-{{ checksum "app/build.gradle" }} - run: name: Execute Data Processing - command: ./gradlew reportModuleStatistics --max-workers=2 --stacktrace + command: ./gradlew reportModuleStatistics --max-workers=2 --stacktrace --no-configure-on-demand - save_cache: paths: - ~/.gradle diff --git a/plugins/src/main/java/com/jraska/gradle/buildtime/BuildReporter.kt b/plugins/src/main/java/com/jraska/gradle/buildtime/BuildReporter.kt index 13ccd20c..134b511e 100644 --- a/plugins/src/main/java/com/jraska/gradle/buildtime/BuildReporter.kt +++ b/plugins/src/main/java/com/jraska/gradle/buildtime/BuildReporter.kt @@ -26,8 +26,6 @@ class BuildReporter( } private fun reportInternal(buildData: BuildData) { - val delivery = ClientDelivery() - val properties = convertBuildData(buildData) val event = AnalyticsEvent("Android Build", properties) analyticsReporter.report(event) diff --git a/plugins/src/main/java/com/jraska/module/ModuleStatistics.kt b/plugins/src/main/java/com/jraska/module/ModuleStatistics.kt index 501ccd90..dc297236 100644 --- a/plugins/src/main/java/com/jraska/module/ModuleStatistics.kt +++ b/plugins/src/main/java/com/jraska/module/ModuleStatistics.kt @@ -9,6 +9,7 @@ data class ProjectStatistics( val prodJavaTotalLines: Int, val prodXmlTotalLines: Int, val moduleStatistics: List, + val externalDependencies: List ) data class ModuleStatistics( @@ -25,6 +26,23 @@ data class FileTypeStatistics( val type: FileType ) +data class ModuleArtifactDependency( + val moduleName: String, + val type: ModuleType, + val fullName: String, + val group: String, + val artifact: String, + val version: String?, + val dependencyType: ArtifactDependencyType +) + +enum class ArtifactDependencyType { + Compile, + Test, + AndroidTest, + Kapt +} + enum class ModuleType { Api, Implementation, diff --git a/plugins/src/main/java/com/jraska/module/ModuleStatsReporter.kt b/plugins/src/main/java/com/jraska/module/ModuleStatsReporter.kt index 07298685..4eea87d9 100644 --- a/plugins/src/main/java/com/jraska/module/ModuleStatsReporter.kt +++ b/plugins/src/main/java/com/jraska/module/ModuleStatsReporter.kt @@ -17,6 +17,11 @@ class ModuleStatsReporter( events.add(AnalyticsEvent("Android Module Detail", properties)) } + stats.externalDependencies.forEach { + val properties = convertModuleDependency(it) + events.add(AnalyticsEvent("Android Module External Dependency", properties)) + } + analyticsReporter.report(*events.toTypedArray()) println("$GRAPH_ICON Module stats reported to Mixpanel $GRAPH_ICON") @@ -63,6 +68,18 @@ class ModuleStatsReporter( } } + private fun convertModuleDependency(artifactDependency: ModuleArtifactDependency): Map { + return mapOf( + "moduleName" to artifactDependency.moduleName, + "moduleType" to artifactDependency.type.name, + "groupId" to artifactDependency.group, + "artifactId" to artifactDependency.artifact, + "version" to artifactDependency.version, + "fullComponentName" to artifactDependency.fullName, + "dependencyType" to artifactDependency.dependencyType + ) + } + companion object { private val GRAPH_ICON = "\uD83D\uDCC9" } diff --git a/plugins/src/main/java/com/jraska/module/extract/StatisticsGradleExtractor.kt b/plugins/src/main/java/com/jraska/module/extract/StatisticsGradleExtractor.kt index f7d4efd7..5a6edbcf 100644 --- a/plugins/src/main/java/com/jraska/module/extract/StatisticsGradleExtractor.kt +++ b/plugins/src/main/java/com/jraska/module/extract/StatisticsGradleExtractor.kt @@ -1,14 +1,20 @@ package com.jraska.module.extract +import com.jraska.module.ArtifactDependencyType import com.jraska.module.FileType import com.jraska.module.FileTypeStatistics +import com.jraska.module.ModuleArtifactDependency import com.jraska.module.ModuleStatistics import com.jraska.module.ModuleType import com.jraska.module.ProjectStatistics import org.gradle.api.Project +import org.gradle.api.artifacts.ProjectDependency +import org.gradle.api.artifacts.ResolvedDependency import java.io.BufferedReader import java.io.File import java.io.FileReader +import java.util.LinkedList +import java.util.Queue class StatisticsGradleExtractor() { private val typesToLookFor = arrayOf(FileType.JAVA, FileType.KOTLIN, FileType.XML) @@ -19,12 +25,18 @@ class StatisticsGradleExtractor() { .filter { it.childProjects.isEmpty() } .map { extractFromModule(it) } + val externalDependencies = project.rootProject + .subprojects + .filter { it.childProjects.isEmpty() } + .flatMap { extractDependencies(it) } + return ProjectStatistics( modulesCount = moduleStats.size, apiModules = moduleStats.count { it.type == ModuleType.Api }, appModules = moduleStats.count { it.type == ModuleType.App }, implementationModules = moduleStats.count { it.type == ModuleType.Implementation }, moduleStatistics = moduleStats, + externalDependencies = externalDependencies, prodKotlinTotalLines = moduleStats .map { module -> module.containedProdFiles.single { it.type == FileType.KOTLIN }.lineCount } .sum(), @@ -39,6 +51,56 @@ class StatisticsGradleExtractor() { ) } + private fun extractDependencies(module: Project): List { + val moduleType = moduleType(module) + val moduleName = module.name + + val dependencies = module.configurations + .filter { CONFIGURATION_TO_LOOK.containsKey(it.name) } + .flatMap { configuration -> + val projectDependencies = configuration.allDependencies.filterIsInstance(ProjectDependency::class.java) + configuration.resolvedConfiguration.firstLevelModuleDependencies + .filter { isExternalDependency(it, projectDependencies) } + .flatMap { traverseAndAddChildren(it) } + .map { + ModuleArtifactDependency( + moduleName = moduleName, + type = moduleType, + group = it.moduleGroup, + dependencyType = CONFIGURATION_TO_LOOK.getValue(configuration.name), + artifact = it.moduleName, + version = it.moduleVersion, + fullName = it.name + ) + } + }.distinct() + + return dependencies + } + + private fun traverseAndAddChildren(firstLevelDependency: ResolvedDependency): Set { + val queue: Queue = LinkedList() + + val dependencies = mutableSetOf(firstLevelDependency) + queue.offer(firstLevelDependency) + + while (queue.isNotEmpty()) { + val element = queue.poll() + element.children.forEach { child -> + dependencies.add(child) + queue.offer(child) + } + } + + return dependencies + } + + private fun isExternalDependency(dependency: ResolvedDependency, projectDependencies: List): Boolean { + return projectDependencies.none { + it.group == dependency.moduleGroup && it.name == dependency.moduleName + } + } + private fun extractFromModule(module: Project): ModuleStatistics { val prodFileStats = typesToLookFor.map { getFileTypeStatistics(it, File(module.projectDir, "src/main")) } val unitTestFileStats = typesToLookFor.map { getFileTypeStatistics(it, File(module.projectDir, "src/test")) } @@ -77,4 +139,16 @@ class StatisticsGradleExtractor() { } return lines } + + companion object { + val CONFIGURATION_TO_LOOK = mapOf( + "debugAndroidTestCompileClasspath" to ArtifactDependencyType.AndroidTest, + "debugCompileClasspath" to ArtifactDependencyType.Compile, + "releaseCompileClasspath" to ArtifactDependencyType.Compile, + "debugUnitTestCompileClasspath" to ArtifactDependencyType.Test, + "compileClasspath" to ArtifactDependencyType.Compile, + "testCompileClasspath" to ArtifactDependencyType.Test, + "kapt" to ArtifactDependencyType.Kapt + ) + } }