From c9b6b9fc09d11cd7b0b09216f2cc1ddcb167a97d Mon Sep 17 00:00:00 2001 From: Josef Raska <6277721+jraska@users.noreply.github.com> Date: Sun, 15 Nov 2020 00:47:52 +0100 Subject: [PATCH] Add test stats reporting (#342) * Add test reporting * Simplify Fiebase Test Lab plugin, proper computing of passed property * Bette formatting into method * Error to make tests fail * Fixed failed test * Add passsed and tests count * Asserting on exit code * Intoduce test failure * Print exit code * Extract the Firebase link * Fir error test * Try to fix the TeeOutputStream * Set error streamas well * Setting only error stream --- .../client/firebase/FirebaseTestLabPlugin.kt | 73 ++++++----- .../client/firebase/TestResultReporter.kt | 5 + .../github/client/firebase/TestSuiteResult.kt | 36 ++++++ .../report/ConsoleTestResultReporter.kt | 10 ++ .../report/FirebaseResultExtractor.kt | 90 +++++++++++++ .../firebase/report/FirebaseUrlParser.kt | 12 ++ .../report/MixpanelTestResultsReporter.kt | 69 ++++++++++ .../buildtime/report/MixpanelReporter.kt | 9 +- .../java/com/jraska/gradle/git/GitInfo.kt | 11 +- .../report/FirebaseResultExtractorTest.kt | 119 ++++++++++++++++++ .../firebase/report/FirebaseUrlParserTest.kt | 64 ++++++++++ 11 files changed, 464 insertions(+), 34 deletions(-) create mode 100644 plugins/src/main/java/com/jraska/github/client/firebase/TestResultReporter.kt create mode 100644 plugins/src/main/java/com/jraska/github/client/firebase/TestSuiteResult.kt create mode 100644 plugins/src/main/java/com/jraska/github/client/firebase/report/ConsoleTestResultReporter.kt create mode 100644 plugins/src/main/java/com/jraska/github/client/firebase/report/FirebaseResultExtractor.kt create mode 100644 plugins/src/main/java/com/jraska/github/client/firebase/report/FirebaseUrlParser.kt create mode 100644 plugins/src/main/java/com/jraska/github/client/firebase/report/MixpanelTestResultsReporter.kt create mode 100644 plugins/src/test/kotlin/com/jraska/github/client/firebase/report/FirebaseResultExtractorTest.kt create mode 100644 plugins/src/test/kotlin/com/jraska/github/client/firebase/report/FirebaseUrlParserTest.kt diff --git a/plugins/src/main/java/com/jraska/github/client/firebase/FirebaseTestLabPlugin.kt b/plugins/src/main/java/com/jraska/github/client/firebase/FirebaseTestLabPlugin.kt index 59cacb3a..e267c595 100644 --- a/plugins/src/main/java/com/jraska/github/client/firebase/FirebaseTestLabPlugin.kt +++ b/plugins/src/main/java/com/jraska/github/client/firebase/FirebaseTestLabPlugin.kt @@ -1,8 +1,17 @@ package com.jraska.github.client.firebase +import com.jraska.github.client.firebase.report.ConsoleTestResultReporter +import com.jraska.github.client.firebase.report.FirebaseResultExtractor +import com.jraska.github.client.firebase.report.FirebaseUrlParser +import com.jraska.github.client.firebase.report.MixpanelTestResultsReporter +import com.jraska.gradle.git.GitInfoProvider +import com.mixpanel.mixpanelapi.MixpanelAPI +import org.apache.tools.ant.util.TeeOutputStream import org.gradle.api.Plugin import org.gradle.api.Project import org.gradle.api.tasks.Exec +import org.gradle.process.ExecResult +import java.io.ByteArrayOutputStream import java.io.File import java.time.LocalDateTime import java.time.format.DateTimeFormatter @@ -10,21 +19,14 @@ import java.time.format.DateTimeFormatter class FirebaseTestLabPlugin : Plugin { override fun apply(theProject: Project) { theProject.afterEvaluate { project -> - val setupGCloudProject = project.tasks.register("setupGCloudProject", Exec::class.java) { - it.commandLine = "gcloud config set project github-client-25b47".split(' ') - it.dependsOn(project.tasks.named("assembleDebugAndroidTest")) - } - - val setupGCloudAccount = project.tasks.register("setupGCloudAccount", Exec::class.java) { - val credentialsPath = project.createCredentialsFile() - it.commandLine = "gcloud auth activate-service-account --key-file $credentialsPath".split(' ') - - it.dependsOn(setupGCloudProject) - } - var resultsFileToPull: String? = null + project.tasks.register("runInstrumentedTestsOnFirebase", Exec::class.java) { firebaseTask -> + firebaseTask.doFirst { + project.exec("gcloud config set project github-client-25b47") + val credentialsPath = project.createCredentialsFile() + project.exec("gcloud auth activate-service-account --key-file $credentialsPath") + } - val executeTestsInTestLab = project.tasks.register("executeInstrumentedTestsOnFirebase", Exec::class.java) { val appApk = "${project.buildDir}/outputs/apk/debug/app-debug.apk" val testApk = "${project.buildDir}/outputs/apk/androidTest/debug/app-debug-androidTest.apk" val deviceName = "flame" @@ -34,9 +36,9 @@ class FirebaseTestLabPlugin : Plugin { val fcmKey = System.getenv("FCM_API_KEY") - resultsFileToPull = "gs://test-lab-twsawhz0hy5am-h35y3vymzadax/$resultDir/$deviceName-$androidVersion-en-portrait/test_result_1.xml" + val resultsFileToPull = "gs://test-lab-twsawhz0hy5am-h35y3vymzadax/$resultDir/$deviceName-$androidVersion-en-portrait/test_result_1.xml" - it.commandLine = + firebaseTask.commandLine = ("gcloud " + "firebase test android run " + "--app $appApk " + @@ -46,28 +48,43 @@ class FirebaseTestLabPlugin : Plugin { "--no-performance-metrics " + "--environment-variables FCM_API_KEY=$fcmKey") .split(' ') + firebaseTask.isIgnoreExitValue = true - it.dependsOn(project.tasks.named("assembleDebugAndroidTest")) - it.dependsOn(project.tasks.named("assembleDebug")) - it.dependsOn(setupGCloudAccount) - } + val decorativeStream = ByteArrayOutputStream() + firebaseTask.errorOutput = TeeOutputStream(decorativeStream, System.err) - val pullResults = project.tasks.register("pullFirebaseXmlResults", Exec::class.java) { task -> - task.dependsOn(executeTestsInTestLab) + firebaseTask.doLast { + val outputFile = "${project.buildDir}/test-results/firebase-tests-results.xml" + project.exec("gsutil cp $resultsFileToPull $outputFile") - task.doFirst { - task.commandLine = "gsutil cp $resultsFileToPull ${project.buildDir}/test-results/firebase-tests-results.xml".split(' ') + val firebaseUrl = FirebaseUrlParser.parse(decorativeStream.toString()) + + val result = FirebaseResultExtractor(firebaseUrl, GitInfoProvider.gitInfo(project), device).extract(File(outputFile).readText()) + val mixpanelToken: String? = System.getenv("GITHUB_CLIENT_MIXPANEL_API_KEY") + val reporter = if (mixpanelToken == null) { + println("'GITHUB_CLIENT_MIXPANEL_API_KEY' not set, data will be reported to console only") + ConsoleTestResultReporter() + } else { + MixpanelTestResultsReporter(mixpanelToken, MixpanelAPI()) + } + + reporter.report(result) + firebaseTask.execResult!!.assertNormalExitValue() } - } - project.tasks.register("runInstrumentedTestsOnFirebase") { - it.dependsOn(executeTestsInTestLab) - it.dependsOn(pullResults) + firebaseTask.dependsOn(project.tasks.named("assembleDebugAndroidTest")) + firebaseTask.dependsOn(project.tasks.named("assembleDebug")) } } } - fun Project.createCredentialsFile(): String { + private fun Project.exec(command: String): ExecResult { + return exec { + it.commandLine(command.split(" ")) + } + } + + private fun Project.createCredentialsFile(): String { val credentialsPath = "$buildDir/credentials.json" val credentials = System.getenv("GCLOUD_CREDENTIALS") if (credentials != null) { diff --git a/plugins/src/main/java/com/jraska/github/client/firebase/TestResultReporter.kt b/plugins/src/main/java/com/jraska/github/client/firebase/TestResultReporter.kt new file mode 100644 index 00000000..cdb306b4 --- /dev/null +++ b/plugins/src/main/java/com/jraska/github/client/firebase/TestResultReporter.kt @@ -0,0 +1,5 @@ +package com.jraska.github.client.firebase + +interface TestResultReporter { + fun report(results: TestSuiteResult) +} diff --git a/plugins/src/main/java/com/jraska/github/client/firebase/TestSuiteResult.kt b/plugins/src/main/java/com/jraska/github/client/firebase/TestSuiteResult.kt new file mode 100644 index 00000000..7b151c31 --- /dev/null +++ b/plugins/src/main/java/com/jraska/github/client/firebase/TestSuiteResult.kt @@ -0,0 +1,36 @@ +package com.jraska.github.client.firebase + +import com.jraska.gradle.git.GitInfo + +data class TestSuiteResult( + val testResults: List, + val time: Double, + val suitePassed: Boolean, + val testsCount: Int, + val failedCount: Int, + val errorsCount: Int, + val passedCount: Int, + val ignoredCount: Int, + val flakyCount: Int, + val firebaseUrl: String, + val gitInfo: GitInfo, + val device: String +) + +data class TestResult( + val outcome: TestOutcome, + val className: String, + val methodName: String, + val time: Double, + val fullName: String, + val gitInfo: GitInfo, + val firebaseUrl: String, + val failure: String?, + val device: String +) + +enum class TestOutcome { + PASSED, + FAILED, + FLAKY +} diff --git a/plugins/src/main/java/com/jraska/github/client/firebase/report/ConsoleTestResultReporter.kt b/plugins/src/main/java/com/jraska/github/client/firebase/report/ConsoleTestResultReporter.kt new file mode 100644 index 00000000..dd7db0c9 --- /dev/null +++ b/plugins/src/main/java/com/jraska/github/client/firebase/report/ConsoleTestResultReporter.kt @@ -0,0 +1,10 @@ +package com.jraska.github.client.firebase.report + +import com.jraska.github.client.firebase.TestResultReporter +import com.jraska.github.client.firebase.TestSuiteResult + +class ConsoleTestResultReporter : TestResultReporter { + override fun report(results: TestSuiteResult) { + println(results) + } +} diff --git a/plugins/src/main/java/com/jraska/github/client/firebase/report/FirebaseResultExtractor.kt b/plugins/src/main/java/com/jraska/github/client/firebase/report/FirebaseResultExtractor.kt new file mode 100644 index 00000000..1c025e5a --- /dev/null +++ b/plugins/src/main/java/com/jraska/github/client/firebase/report/FirebaseResultExtractor.kt @@ -0,0 +1,90 @@ +package com.jraska.github.client.firebase.report + +import com.jraska.github.client.firebase.TestOutcome +import com.jraska.github.client.firebase.TestResult +import com.jraska.github.client.firebase.TestSuiteResult +import com.jraska.gradle.git.GitInfo +import groovy.util.Node +import groovy.util.NodeList +import groovy.util.XmlParser + +class FirebaseResultExtractor( + val firebaseUrl: String, + val gitInfo: GitInfo, + val device: String +) { + fun extract(xml: String): TestSuiteResult { + val testSuiteNode = XmlParser().parseText(xml) + + val testsCount = testSuiteNode.attributeInt("tests") + val flakyTests = testSuiteNode.attributeInt("flakes") + val ignoredCount = testSuiteNode.attributeInt("skipped") + val failedCount = testSuiteNode.attributeInt("failures") + val errorsCount = testSuiteNode.attributeInt("errors") + val time = testSuiteNode.attributeDouble("time") + val passedCount = testsCount - ignoredCount - failedCount - errorsCount + + val tests = (testSuiteNode.get("testcase") as NodeList) + .map { it as Node } + .filter { it.attributeString("name") != "null" } + .map { parseTestResult(it) } + + val suitePassed = errorsCount == 0 && failedCount == 0 + + return TestSuiteResult( + testResults = tests, + time = time, + testsCount = testsCount, + device = device, + gitInfo = gitInfo, + firebaseUrl = firebaseUrl, + errorsCount = errorsCount, + passedCount = passedCount, + failedCount = failedCount, + flakyCount = flakyTests, + ignoredCount = ignoredCount, + suitePassed = suitePassed + ) + } + + private fun parseTestResult(testNode: Node): TestResult { + val flaky = testNode.attributeBoolean("flaky") + val failure = ((testNode.get("failure") as NodeList?)?.firstOrNull() as Node?)?.text() ?: "" + + val outcome = when { + flaky -> TestOutcome.FLAKY + failure.isNotEmpty() -> TestOutcome.FAILED + else -> TestOutcome.PASSED + } + + val methodName = testNode.attributeString("name") + val className = testNode.attributeString("classname") + return TestResult( + methodName = methodName, + className = className, + time = testNode.attributeDouble("time"), + failure = failure, + outcome = outcome, + firebaseUrl = firebaseUrl, + gitInfo = gitInfo, + device = device, + fullName = "$className#$methodName" + ) + } + + private fun Node.attributeInt(name: String): Int { + return attribute(name)?.toString()?.toInt() ?: 0 + } + + private fun Node.attributeDouble(name: String): Double { + return attribute(name).toString().toDouble() + } + + private fun Node.attributeString(name: String): String { + return attribute(name).toString() + } + + private fun Node.attributeBoolean(name: String): Boolean { + return attribute(name)?.toString()?.toBoolean() ?: false + } +} diff --git a/plugins/src/main/java/com/jraska/github/client/firebase/report/FirebaseUrlParser.kt b/plugins/src/main/java/com/jraska/github/client/firebase/report/FirebaseUrlParser.kt new file mode 100644 index 00000000..184999b0 --- /dev/null +++ b/plugins/src/main/java/com/jraska/github/client/firebase/report/FirebaseUrlParser.kt @@ -0,0 +1,12 @@ +package com.jraska.github.client.firebase.report + +object FirebaseUrlParser { + private val urlPattern = """Test results will be streamed to \[(\S*)\]""".toPattern() + + fun parse(output: String): String { + val matcher = urlPattern.matcher(output) + + matcher.find() + return matcher.group(1) + } +} diff --git a/plugins/src/main/java/com/jraska/github/client/firebase/report/MixpanelTestResultsReporter.kt b/plugins/src/main/java/com/jraska/github/client/firebase/report/MixpanelTestResultsReporter.kt new file mode 100644 index 00000000..900cc25b --- /dev/null +++ b/plugins/src/main/java/com/jraska/github/client/firebase/report/MixpanelTestResultsReporter.kt @@ -0,0 +1,69 @@ +package com.jraska.github.client.firebase.report + +import com.jraska.github.client.firebase.TestResult +import com.jraska.github.client.firebase.TestResultReporter +import com.jraska.github.client.firebase.TestSuiteResult +import com.mixpanel.mixpanelapi.ClientDelivery +import com.mixpanel.mixpanelapi.MessageBuilder +import com.mixpanel.mixpanelapi.MixpanelAPI +import org.json.JSONObject + +class MixpanelTestResultsReporter( + private val apiKey: String, + private val api: MixpanelAPI +) : TestResultReporter { + override fun report(results: TestSuiteResult) { + val delivery = ClientDelivery() + + val properties = convertTestSuite(results) + val messageBuilder = MessageBuilder(apiKey) + val moduleEvent = messageBuilder + .event(SINGLE_NAME_FOR_TEST_REPORTS_USER, "Android Test Suite Firebase", JSONObject(properties)) + + delivery.addMessage(moduleEvent) + + results.testResults.forEach { + val testProperties = convertSingleTest(it) + + val estateEvent = messageBuilder.event(SINGLE_NAME_FOR_TEST_REPORTS_USER, "Android Test Firebase", JSONObject(testProperties)) + delivery.addMessage(estateEvent) + } + + api.deliver(delivery) + + println("$FLAG_ICON Test result reported to Mixpanel $FLAG_ICON") + } + + private fun convertSingleTest(testResult: TestResult): Map { + return mutableMapOf( + "className" to testResult.className, + "methodName" to testResult.methodName, + "device" to testResult.device, + "firebaseUrl" to testResult.firebaseUrl, + "fullName" to testResult.fullName, + "failure" to testResult.failure, + "outcome" to testResult.outcome, + "testTime" to testResult.time + ).apply { putAll(testResult.gitInfo.asAnalyticsProperties()) } + } + + private fun convertTestSuite(results: TestSuiteResult): Map { + return mutableMapOf( + "passed" to results.suitePassed, + "suiteTime" to results.time, + "device" to results.device, + "firebaseUrl" to results.firebaseUrl, + "passedCount" to results.passedCount, + "testsCount" to results.testsCount, + "ignoredCount" to results.ignoredCount, + "flakyCount" to results.flakyCount, + "failedCount" to results.failedCount, + "errorsCount" to results.errorsCount + ).apply { putAll(results.gitInfo.asAnalyticsProperties()) } + } + + companion object { + private val FLAG_ICON = "\uD83C\uDFC1" + private val SINGLE_NAME_FOR_TEST_REPORTS_USER = "Test Reporter" + } +} diff --git a/plugins/src/main/java/com/jraska/gradle/buildtime/report/MixpanelReporter.kt b/plugins/src/main/java/com/jraska/gradle/buildtime/report/MixpanelReporter.kt index ed5904bd..78fc9350 100644 --- a/plugins/src/main/java/com/jraska/gradle/buildtime/report/MixpanelReporter.kt +++ b/plugins/src/main/java/com/jraska/gradle/buildtime/report/MixpanelReporter.kt @@ -60,12 +60,11 @@ class MixpanelReporter( "tasksUpToDate" to buildData.taskStatistics.upToDate, "tasksFromCache" to buildData.taskStatistics.fromCache, "tasksExecuted" to buildData.taskStatistics.executed, - "gitBranch" to buildData.gitInfo.branchName, - "gitCommit" to buildData.gitInfo.commitId, - "gitDirty" to buildData.gitInfo.dirty, - "gitStatus" to buildData.gitInfo.status, "buildDataCollectionOverhead" to buildData.buildDataCollectionOverhead - ).apply { putAll(buildData.parameters) } + ).apply { + putAll(buildData.parameters) + putAll(buildData.gitInfo.asAnalyticsProperties()) + } } companion object { diff --git a/plugins/src/main/java/com/jraska/gradle/git/GitInfo.kt b/plugins/src/main/java/com/jraska/gradle/git/GitInfo.kt index 87bb9b5a..ad54f703 100644 --- a/plugins/src/main/java/com/jraska/gradle/git/GitInfo.kt +++ b/plugins/src/main/java/com/jraska/gradle/git/GitInfo.kt @@ -5,4 +5,13 @@ class GitInfo( val commitId: String, val dirty: Boolean, val status: String -) +) { + fun asAnalyticsProperties(): Map { + return mapOf( + "gitBranch" to branchName, + "gitCommit" to commitId, + "gitDirty" to dirty, + "gitStatus" to status, + ) + } +} diff --git a/plugins/src/test/kotlin/com/jraska/github/client/firebase/report/FirebaseResultExtractorTest.kt b/plugins/src/test/kotlin/com/jraska/github/client/firebase/report/FirebaseResultExtractorTest.kt new file mode 100644 index 00000000..4c13456f --- /dev/null +++ b/plugins/src/test/kotlin/com/jraska/github/client/firebase/report/FirebaseResultExtractorTest.kt @@ -0,0 +1,119 @@ +package com.jraska.github.client.firebase.report + +import com.jraska.github.client.firebase.TestOutcome +import com.jraska.gradle.git.GitInfo +import org.assertj.core.api.Assertions.assertThat +import org.junit.Before +import org.junit.Test + +class FirebaseResultExtractorTest { + + lateinit var extractor: FirebaseResultExtractor + + @Before + fun setUp() { + extractor = FirebaseResultExtractor("someUrl", GitInfo("exampleBrach", "123", false, ""), "someDevice") + } + + @Test + fun whenSuccessfulResult_thenParsesCorrectly() { + val suiteResult = extractor.extract(SUCCESS_RESULT) + + assertThat(suiteResult.failedCount).isZero() + assertThat(suiteResult.ignoredCount).isZero() + assertThat(suiteResult.testsCount).isEqualTo(14) + assertThat(suiteResult.flakyCount).isZero() + assertThat(suiteResult.suitePassed).isTrue() + assertThat(suiteResult.time).isEqualTo(15.511) + assertThat(suiteResult.testResults).hasSize(14) + + val firstTest = suiteResult.testResults[0] + assertThat(firstTest.time).isEqualTo(0.026) + assertThat(firstTest.outcome).isEqualTo(TestOutcome.PASSED) + assertThat(firstTest.className).isEqualTo("com.jraska.github.client.AppSetupTest") + assertThat(firstTest.methodName).isEqualTo("appCreateEventFired") + assertThat(firstTest.fullName).isEqualTo("com.jraska.github.client.AppSetupTest#appCreateEventFired") + + val ninthTest = suiteResult.testResults[8] + assertThat(ninthTest.time).isEqualTo(1.182) + assertThat(ninthTest.outcome).isEqualTo(TestOutcome.PASSED) + assertThat(ninthTest.className).isEqualTo("com.jraska.github.client.users.UsersActivityFlowTest") + assertThat(ninthTest.methodName).isEqualTo("whenStarts_thenDisplaysUsers") + assertThat(ninthTest.fullName).isEqualTo("com.jraska.github.client.users.UsersActivityFlowTest#whenStarts_thenDisplaysUsers") + } + + @Test + fun whenErrorResult_thenParsesCorrectly() { + val suiteResult = extractor.extract(ERROR_RESULT) + + assertThat(suiteResult.failedCount).isOne() + assertThat(suiteResult.ignoredCount).isZero() + assertThat(suiteResult.testsCount).isEqualTo(14) + assertThat(suiteResult.flakyCount).isZero() + assertThat(suiteResult.suitePassed).isFalse() + assertThat(suiteResult.time).isEqualTo(13.846) + assertThat(suiteResult.testResults).hasSize(14) + + val firstTest = suiteResult.testResults[0] + assertThat(firstTest.time).isEqualTo(0.0) + assertThat(firstTest.outcome).isEqualTo(TestOutcome.PASSED) + assertThat(firstTest.className).isEqualTo("com.jraska.github.client.AppSetupTest") + assertThat(firstTest.methodName).isEqualTo("appCreateEventFired") + assertThat(firstTest.failure).isEmpty() + assertThat(firstTest.fullName).isEqualTo("com.jraska.github.client.AppSetupTest#appCreateEventFired") + + + val failedTest = suiteResult.testResults[7] + assertThat(failedTest.time).isEqualTo(0.853) + assertThat(failedTest.outcome).isEqualTo(TestOutcome.FAILED) + assertThat(failedTest.className).isEqualTo("com.jraska.github.client.settings.SettingsTest") + assertThat(failedTest.methodName).isEqualTo("whenConsoleClicked_thenConsoleOpens") + assertThat(failedTest.fullName).isEqualTo("com.jraska.github.client.settings.SettingsTest#whenConsoleClicked_thenConsoleOpens") + assertThat(failedTest.failure).startsWith("androidx.test.espresso.NoMatchingViewException: No views in hierarchy found matching: with") + } + + companion object { + val SUCCESS_RESULT = + """ + + + + + + + + + + + + + + + + + + """.trimIndent() + + val ERROR_RESULT = """ + + + + + + + + + + + androidx.test.espresso.NoMatchingViewException: No views in hierarchy found matching: with id is.runner.TestExecutor.execute(TestExecutor.java:56) at androidx.test.runner.AndroidJUnitRunner.onStart(AndroidJUnitRunner.java:395) at android.app.Instrumentation .run(Instrumentation.java:2196) + + + + + + + + + """.trimIndent() + } +} diff --git a/plugins/src/test/kotlin/com/jraska/github/client/firebase/report/FirebaseUrlParserTest.kt b/plugins/src/test/kotlin/com/jraska/github/client/firebase/report/FirebaseUrlParserTest.kt new file mode 100644 index 00000000..23efae1f --- /dev/null +++ b/plugins/src/test/kotlin/com/jraska/github/client/firebase/report/FirebaseUrlParserTest.kt @@ -0,0 +1,64 @@ +package com.jraska.github.client.firebase.report + +import org.assertj.core.api.Assertions.assertThat +import org.junit.Test + +class FirebaseUrlParserTest { + @Test + fun findsUrlProperly() { + val url = FirebaseUrlParser.parse(EXAMPLE_OUTPUT) + + assertThat(url).isEqualTo("https://console.firebase.google.com/project/github-client-25b47/testlab/histories/bh.45e06288a93d3fad/matrices/4937539158600939569") + } + + companion object { + val EXAMPLE_OUTPUT = """ + Have questions, feedback, or issues? Get support by visiting: + https://firebase.google.com/support/ + + Uploading [/home/circleci/code/app/build/outputs/apk/debug/app-debug.apk] to Firebase Test Lab... + Uploading [/home/circleci/code/app/build/outputs/apk/androidTest/debug/app-debug-androidTest.apk] to Firebase Test Lab... + Raw results will be stored in your GCS bucket at [https://console.developers.google.com/storage/browser/test-lab-twsawhz0hy5am-h35y3vymzadax/2020-11-14T23:02:48.776162/] + + Test [matrix-1vohggy3nox8r] has been created in the Google Cloud. + Firebase Test Lab will execute your instrumentation test on 1 device(s). + Creating individual test executions... + ..............................................done. + + Test results will be streamed to [https://console.firebase.google.com/project/github-client-25b47/testlab/histories/bh.45e06288a93d3fad/matrices/4937539158600939569]. + 23:03:51 Test is Pending + 23:04:27 Starting attempt 1. + 23:04:27 Started logcat recording. + 23:04:27 Started crash monitoring. + 23:04:27 Preparing device. + 23:04:27 Test is Running + 23:04:39 Logging in to Google account on device. + 23:04:39 Installing apps. + 23:04:39 Retrieving Pre-Test Package Stats information from the device. + 23:04:39 Retrieving Performance Environment information from the device. + 23:04:39 Started crash detection. + 23:04:39 Started Out of memory detection + 23:04:39 Started video recording. + 23:04:39 Starting instrumentation test. + 23:05:04 Completed instrumentation test. + 23:05:04 Stopped video recording. + 23:05:04 Retrieving Post-test Package Stats information from the device. + 23:05:04 Logging out of Google account on device. + 23:05:04 Stopped crash monitoring. + 23:05:04 Stopped logcat recording. + 23:05:04 Done. Test time = 20 (secs) + 23:05:04 Starting results processing. Attempt: 1 + 23:05:16 Completed results processing. Time taken = 6 (secs) + 23:05:16 Test is Finished + + Instrumentation testing complete. + + More details are available at [https://console.firebase.google.com/project/github-client-25b47/testlab/histories/bh.45e06288a93d3fad/matrices/4937539158600939569]. + ┌─────────┬──────────────────────┬────────────────────────────────┐ + │ OUTCOME │ TEST_AXIS_VALUE │ TEST_DETAILS │ + ├─────────┼──────────────────────┼────────────────────────────────┤ + │ Failed │ flame-29-en-portrait │ 1 test cases failed, 13 passed │ + └─────────┴──────────────────────┴────────────────────────────────┘ + """.trimIndent() + } +}