From 476af3f01187b621cd164c448b82ede085f829d6 Mon Sep 17 00:00:00 2001 From: stefanosiano Date: Wed, 20 Apr 2022 10:51:55 +0200 Subject: [PATCH 01/22] added benchmark ui tests added end2end ui tests module with base test class (early version) --- buildSrc/src/main/java/Config.kt | 12 +- sentry-android-ui-tests/benchmark/.gitignore | 1 + .../benchmark/benchmark-proguard-rules.pro | 37 +++++ .../benchmark/build.gradle.kts | 82 +++++++++++ .../src/androidTest/AndroidManifest.xml | 20 +++ .../uitests/benchmark/SentryBenchmarkTest.kt | 119 +++++++++++++++ .../benchmark/util/BenchmarkOperation.kt | 81 ++++++++++ .../util/BenchmarkOperationResult.kt | 85 +++++++++++ .../benchmark/src/main/AndroidManifest.xml | 18 +++ .../uitests/benchmark/BenchmarkActivity.kt | 139 ++++++++++++++++++ .../main/res/layout/activity_benchmark.xml | 29 ++++ .../main/res/layout/benchmark_item_list.xml | 31 ++++ sentry-android-ui-tests/end2end/.gitignore | 1 + .../end2end/build.gradle.kts | 75 ++++++++++ .../end2end/consumer-rules.pro | 0 .../end2end/proguard-rules.pro | 21 +++ .../android/uitests/end2end/BaseUiTest.kt | 61 ++++++++ .../end2end/ExampleInstrumentedTest.kt | 35 +++++ .../end2end/mockservers/TestMockWebServers.kt | 22 +++ .../end2end/src/main/AndroidManifest.xml | 18 +++ .../android/uitests/end2end/EmptyActivity.kt | 26 ++++ settings.gradle.kts | 2 + 22 files changed, 913 insertions(+), 2 deletions(-) create mode 100644 sentry-android-ui-tests/benchmark/.gitignore create mode 100644 sentry-android-ui-tests/benchmark/benchmark-proguard-rules.pro create mode 100644 sentry-android-ui-tests/benchmark/build.gradle.kts create mode 100644 sentry-android-ui-tests/benchmark/src/androidTest/AndroidManifest.xml create mode 100644 sentry-android-ui-tests/benchmark/src/androidTest/java/io/sentry/android/uitests/benchmark/SentryBenchmarkTest.kt create mode 100644 sentry-android-ui-tests/benchmark/src/androidTest/java/io/sentry/android/uitests/benchmark/util/BenchmarkOperation.kt create mode 100644 sentry-android-ui-tests/benchmark/src/androidTest/java/io/sentry/android/uitests/benchmark/util/BenchmarkOperationResult.kt create mode 100644 sentry-android-ui-tests/benchmark/src/main/AndroidManifest.xml create mode 100644 sentry-android-ui-tests/benchmark/src/main/java/io/sentry/android/uitests/benchmark/BenchmarkActivity.kt create mode 100644 sentry-android-ui-tests/benchmark/src/main/res/layout/activity_benchmark.xml create mode 100644 sentry-android-ui-tests/benchmark/src/main/res/layout/benchmark_item_list.xml create mode 100644 sentry-android-ui-tests/end2end/.gitignore create mode 100644 sentry-android-ui-tests/end2end/build.gradle.kts create mode 100644 sentry-android-ui-tests/end2end/consumer-rules.pro create mode 100644 sentry-android-ui-tests/end2end/proguard-rules.pro create mode 100644 sentry-android-ui-tests/end2end/src/androidTest/java/io/sentry/android/uitests/end2end/BaseUiTest.kt create mode 100644 sentry-android-ui-tests/end2end/src/androidTest/java/io/sentry/android/uitests/end2end/ExampleInstrumentedTest.kt create mode 100644 sentry-android-ui-tests/end2end/src/androidTest/java/io/sentry/android/uitests/end2end/mockservers/TestMockWebServers.kt create mode 100644 sentry-android-ui-tests/end2end/src/main/AndroidManifest.xml create mode 100644 sentry-android-ui-tests/end2end/src/main/java/io/sentry/android/uitests/end2end/EmptyActivity.kt diff --git a/buildSrc/src/main/java/Config.kt b/buildSrc/src/main/java/Config.kt index 10f5a605008..96debddaecc 100644 --- a/buildSrc/src/main/java/Config.kt +++ b/buildSrc/src/main/java/Config.kt @@ -46,6 +46,7 @@ object Config { val okhttpBom = "com.squareup.okhttp3:okhttp-bom:$okHttpVersion" val okhttp = "com.squareup.okhttp3:okhttp" val leakCanary = "com.squareup.leakcanary:leakcanary-android:2.8.1" + val constraintLayout = "androidx.constraintlayout:constraintlayout:2.1.3" private val lifecycleVersion = "2.2.0" val lifecycleProcess = "androidx.lifecycle:lifecycle-process:$lifecycleVersion" @@ -111,12 +112,19 @@ object Config { } object TestLibs { - private val androidxTestVersion = "1.4.0-rc01" + private val androidxTestVersion = "1.4.0" + private val espressoVersion = "3.4.0" val kotlinTestJunit = "org.jetbrains.kotlin:kotlin-test-junit:$kotlinVersion" val androidxCore = "androidx.test:core:$androidxTestVersion" val androidxRunner = "androidx.test:runner:$androidxTestVersion" - val androidxJunit = "androidx.test.ext:junit:1.1.3-rc01" + val androidxTestCoreKtx = "androidx.test:core-ktx:$androidxTestVersion" + val androidxTestRules = "androidx.test:rules:$androidxTestVersion" + val androidxBenchmarkJunit = "androidx.benchmark:benchmark-junit4:1.0.0" + val espressoCore = "androidx.test.espresso:espresso-core:$espressoVersion" + val espressoIdlingResource = "androidx.test.espresso:espresso-idling-resource:$espressoVersion" + val androidxTestOrchestrator = "androidx.test:orchestrator:1.4.1" + val androidxJunit = "androidx.test.ext:junit:1.1.3" val androidxCoreKtx = "androidx.core:core-ktx:1.7.0" val robolectric = "org.robolectric:robolectric:4.7.3" val mockitoKotlin = "com.nhaarman.mockitokotlin2:mockito-kotlin:2.2.0" diff --git a/sentry-android-ui-tests/benchmark/.gitignore b/sentry-android-ui-tests/benchmark/.gitignore new file mode 100644 index 00000000000..796b96d1c40 --- /dev/null +++ b/sentry-android-ui-tests/benchmark/.gitignore @@ -0,0 +1 @@ +/build diff --git a/sentry-android-ui-tests/benchmark/benchmark-proguard-rules.pro b/sentry-android-ui-tests/benchmark/benchmark-proguard-rules.pro new file mode 100644 index 00000000000..75bf7d64dff --- /dev/null +++ b/sentry-android-ui-tests/benchmark/benchmark-proguard-rules.pro @@ -0,0 +1,37 @@ +# Add project specific ProGuard rules here. +# You can control the set of applied configuration files using the +# proguardFiles setting in build.gradle.kts. +# +# For more details, see +# http://developer.android.com/guide/developing/tools/proguard.html + +# If your project uses WebView with JS, uncomment the following +# and specify the fully qualified class name to the JavaScript interface +# class: +#-keepclassmembers class fqcn.of.javascript.interface.for.webview { +# public *; +#} + +# Uncomment this to preserve the line number information for +# debugging stack traces. +#-keepattributes SourceFile,LineNumberTable + +# If you keep the line number information, uncomment this to +# hide the original source file name. +#-renamesourcefileattribute SourceFile + +-dontobfuscate + +-ignorewarnings + +-keepattributes *Annotation* + +-dontnote junit.framework.** +-dontnote junit.runner.** + +-dontwarn androidx.test.** +-dontwarn org.junit.** +-dontwarn org.hamcrest.** +-dontwarn com.squareup.javawriter.JavaWriter + +-keepclasseswithmembers @org.junit.runner.RunWith public class * diff --git a/sentry-android-ui-tests/benchmark/build.gradle.kts b/sentry-android-ui-tests/benchmark/build.gradle.kts new file mode 100644 index 00000000000..a05f9fba484 --- /dev/null +++ b/sentry-android-ui-tests/benchmark/build.gradle.kts @@ -0,0 +1,82 @@ +plugins { + id("com.android.library") + kotlin("android") +} + +android { + compileSdk = Config.Android.compileSdkVersion + + defaultConfig { + minSdk = Config.Android.minSdkVersionOkHttp + targetSdk = Config.Android.targetSdkVersion + + testInstrumentationRunner = "androidx.benchmark.junit4.AndroidBenchmarkRunner" + // Runs each test in its own instance of Instrumentation. This way they are isolated from + // one another and get their own Application instance. + // https://developer.android.com/training/testing/instrumented-tests/androidx-test-libraries/runner#enable-gradle + testInstrumentationRunnerArguments["clearPackageData"] = "true" + } + +// testOptions { +// execution = "ANDROIDX_TEST_ORCHESTRATOR" it doesn't work with Android 11... +// } + + testBuildType = "release" + + signingConfigs { + getByName("debug") { + storeFile = rootProject.file("debug.keystore") + storePassword = "android" + keyAlias = "androiddebugkey" + keyPassword = "android" + } + } + + buildTypes { + named("debug") { + // Since debuggable can"t be modified by gradle for library modules, + // it must be done in a manifest - see src/androidTest/AndroidManifest.xml + isMinifyEnabled = true + proguardFiles(getDefaultProguardFile("proguard-android-optimize.txt"), "benchmark-proguard-rules.pro") + } + named("release") { + isDefault = true + isMinifyEnabled = true + isShrinkResources = false + signingConfig = signingConfigs.getByName("debug") // to be able to run release mode + proguardFiles(getDefaultProguardFile("proguard-android-optimize.txt"), "benchmark-proguard-rules.pro") + + addManifestPlaceholders( + mapOf( + "sentryDebug" to false, + "sentryEnvironment" to "release" + ) + ) + } + } + + kotlinOptions { + jvmTarget = "1.8" + } +} + +dependencies { + + implementation(kotlin(Config.kotlinStdLib, org.jetbrains.kotlin.config.KotlinCompilerVersion.VERSION)) + + implementation(projects.sentryAndroid) + implementation(Config.Libs.appCompat) + implementation(Config.Libs.androidxCore) + implementation(Config.Libs.androidxRecylerView) + implementation(Config.Libs.constraintLayout) + implementation(Config.TestLibs.espressoIdlingResource) + + androidTestImplementation(Config.TestLibs.kotlinTestJunit) + androidTestImplementation(Config.TestLibs.androidxBenchmarkJunit) + androidTestImplementation(Config.TestLibs.espressoCore) + androidTestImplementation(Config.TestLibs.androidxTestCoreKtx) + androidTestImplementation(Config.TestLibs.androidxRunner) + androidTestImplementation(Config.TestLibs.androidxTestRules) + androidTestImplementation(Config.TestLibs.androidxJunit) + androidTestUtil(Config.TestLibs.androidxTestOrchestrator) +} diff --git a/sentry-android-ui-tests/benchmark/src/androidTest/AndroidManifest.xml b/sentry-android-ui-tests/benchmark/src/androidTest/AndroidManifest.xml new file mode 100644 index 00000000000..73bef08f171 --- /dev/null +++ b/sentry-android-ui-tests/benchmark/src/androidTest/AndroidManifest.xml @@ -0,0 +1,20 @@ + + + + + + + + + diff --git a/sentry-android-ui-tests/benchmark/src/androidTest/java/io/sentry/android/uitests/benchmark/SentryBenchmarkTest.kt b/sentry-android-ui-tests/benchmark/src/androidTest/java/io/sentry/android/uitests/benchmark/SentryBenchmarkTest.kt new file mode 100644 index 00000000000..9d45c7cbf5d --- /dev/null +++ b/sentry-android-ui-tests/benchmark/src/androidTest/java/io/sentry/android/uitests/benchmark/SentryBenchmarkTest.kt @@ -0,0 +1,119 @@ +package io.sentry.android.uitests.benchmark + +import android.content.Context +import androidx.lifecycle.Lifecycle +import androidx.test.core.app.ApplicationProvider +import androidx.test.core.app.launchActivity +import androidx.test.espresso.Espresso +import androidx.test.espresso.Espresso.onView +import androidx.test.espresso.action.ViewActions.swipeUp +import androidx.test.espresso.matcher.ViewMatchers.withId +import androidx.test.ext.junit.runners.AndroidJUnit4 +import androidx.test.platform.app.InstrumentationRegistry +import androidx.test.runner.AndroidJUnitRunner +import io.sentry.ITransaction +import io.sentry.Sentry +import io.sentry.android.uitests.benchmark.util.BenchmarkOperation +import org.junit.Test +import org.junit.runner.RunWith +import kotlin.test.BeforeTest +import kotlin.test.assertTrue + +/** + * ## On Comparing Operations + * + * Originally [androidx.benchmark.junit4.BenchmarkRule] was used along with two tests: one to + * benchmark an operation without Specto tracing, and another to measure the same operation with + * Specto tracing. However when running this setup to compare the exact same operation showed that + * the one that ran first always had significantly more frames dropped, so comparing two different + * operations on equal terms was not possible. + * + * The current approach interweaves the two operations, running them in an alternating sequence. + * When the two operations are the same, we get (nearly) identical results, as expected. + */ +@RunWith(AndroidJUnit4::class) +class SentryBenchmarkTest { + + private lateinit var runner: AndroidJUnitRunner + private lateinit var context: Context + + @BeforeTest + fun baseSetUp() { + runner = InstrumentationRegistry.getInstrumentation() as AndroidJUnitRunner + context = ApplicationProvider.getApplicationContext() + } + + @Test + fun benchmarkProfiledTransaction() { + // runOnMainSync ensure that we're using Sentry on a thread that belongs to the actual + // application, rather than the test application. + // todo Right now we cannot enable profiling via code, but only via manifest. + // We should fix it and initialize via code, so that different tests can use different options. + runner.runOnMainSync { +// SentryAndroid.init(context) { options: SentryOptions -> +// options.dsn = "https://12345678901234567890123456789012@fakeuri.ingest.sentry.io/1234567" +// options.isEnableAutoSessionTracking = false +// options.tracesSampleRate = 1.0 +// options.isTraceSampling = true +// options.isProfilingEnabled = true +// } + } + + val benchmarkOperationNoTransaction = BenchmarkOperation(runner, getOperation(runner) { null }) + val benchmarkOperationProfiled = BenchmarkOperation(runner, getOperation(runner) { + Sentry.startTransaction("Benchmark", "ProfiledTransaction") + }) + repeat(2) { + benchmarkOperationNoTransaction.warmup() + benchmarkOperationProfiled.warmup() + } + repeat(10) { + benchmarkOperationNoTransaction.iterate() + benchmarkOperationProfiled.iterate() + } + + val noTransactionResult = benchmarkOperationNoTransaction.getResult("NoTransaction") + val profiledResult = benchmarkOperationProfiled.getResult("ProfiledTransaction") + + println("=====================================") + println(noTransactionResult) + println(profiledResult) + println("=====================================") + + // We expect the profiled operation to be slower than the no transaction one, but not slower than 5% + val comparisonResult = profiledResult.compare(noTransactionResult) + assertTrue { comparisonResult.durationIncrease >= 0 } + assertTrue { comparisonResult.durationIncrease < 5.0 } + assertTrue { comparisonResult.cpuTimeIncrease >= 0 } + assertTrue { comparisonResult.cpuTimeIncrease < 5.0 } + assertTrue { comparisonResult.fpsDecrease >= 0 } + assertTrue { comparisonResult.fpsDecrease < 5.0 } + assertTrue { comparisonResult.droppedFramesIncrease >= 0 } + assertTrue { comparisonResult.droppedFramesIncrease < 5.0 } + } + + // Operation that will be compared + private fun getOperation(runner: AndroidJUnitRunner, transactionBuilder: () -> ITransaction?): () -> Unit = { + var transaction: ITransaction? = null + // Launch the benchmark activity + val benchmarkScenario = launchActivity() + // Starts a transaction (it can be null, but we still runOnMainSync to make operations as similar as possible) + runner.runOnMainSync { + transaction = transactionBuilder() + } + // Just swipe the list some times: this is the benchmarked operation + repeat(2) { + onView(withId(R.id.benchmark_transaction_list)).perform(swipeUp()) + Espresso.onIdle() + } + // We finish the transaction + runner.runOnMainSync { + transaction?.finish() + } + // We swipe a last time to measure how finishing the transaction may affect other operations + onView(withId(R.id.benchmark_transaction_list)).perform(swipeUp()) + Espresso.onIdle() + + benchmarkScenario.moveToState(Lifecycle.State.DESTROYED) + } +} diff --git a/sentry-android-ui-tests/benchmark/src/androidTest/java/io/sentry/android/uitests/benchmark/util/BenchmarkOperation.kt b/sentry-android-ui-tests/benchmark/src/androidTest/java/io/sentry/android/uitests/benchmark/util/BenchmarkOperation.kt new file mode 100644 index 00000000000..5b1f92afa22 --- /dev/null +++ b/sentry-android-ui-tests/benchmark/src/androidTest/java/io/sentry/android/uitests/benchmark/util/BenchmarkOperation.kt @@ -0,0 +1,81 @@ +package io.sentry.android.uitests.benchmark.util + +import android.os.Process +import android.os.SystemClock +import android.view.Choreographer +import androidx.test.runner.AndroidJUnitRunner + +// 60 FPS is the recommended target: https://www.youtube.com/watch?v=CaMTIgxCSqU +private const val FRAME_DURATION_60FPS_NS: Double = 1_000_000_000 / 60.0 + +internal class BenchmarkOperation(runner: AndroidJUnitRunner, private val op: () -> Unit) { + + private lateinit var choreographer: Choreographer + private var iterations: Int = 0 + private var durationNanos: Long = 0 + private var cpuDurationMillis: Long = 0 + private var frames: Int = 0 + private var droppedFrames: Double = 0.0 + private var lastFrameTimeNanos: Long = 0 + + init { + // Must run on the main thread to get the main thread choreographer. + runner.runOnMainSync { + choreographer = Choreographer.getInstance() + } + } + + fun warmup() { + op() + isolate() + } + + fun iterate() { + val startRealtimeNs = SystemClock.elapsedRealtimeNanos() + val startCpuTimeMs = Process.getElapsedCpuTime() + + lastFrameTimeNanos = startRealtimeNs + iterations++ + choreographer.postFrameCallback(frameCallback) + + op() + + choreographer.removeFrameCallback(frameCallback) + cpuDurationMillis += Process.getElapsedCpuTime() - startCpuTimeMs + durationNanos += SystemClock.elapsedRealtimeNanos() - startRealtimeNs + + isolate() + } + + fun getResult(operationName: String): BenchmarkOperationResult = BenchmarkOperationResult( + cpuDurationMillis / iterations, + droppedFrames / iterations, + durationNanos / iterations, + (frames * 1_000_000_000L / durationNanos).toInt(), + operationName + ) + + /** + * Helps ensure that operations don't impact one another. Doesn't appear to currently have an + * impact on the benchmark, but it might later on. + */ + private fun isolate() { + Thread.sleep(500) + Runtime.getRuntime().gc() + } + + private val frameCallback = object : Choreographer.FrameCallback { + override fun doFrame(frameTimeNanos: Long) { + frames++ + val timeSinceLastFrameNanos = frameTimeNanos - lastFrameTimeNanos + if (timeSinceLastFrameNanos > FRAME_DURATION_60FPS_NS) { + // Fractions of frames dropped are weighted to improve the accuracy of the results. + // For example, 31ms between frames is much worse than 17ms, even though both + // durations are within the "1 frame dropped" range. + droppedFrames += timeSinceLastFrameNanos / FRAME_DURATION_60FPS_NS - 1 + } + lastFrameTimeNanos = frameTimeNanos + choreographer.postFrameCallback(this) + } + } +} diff --git a/sentry-android-ui-tests/benchmark/src/androidTest/java/io/sentry/android/uitests/benchmark/util/BenchmarkOperationResult.kt b/sentry-android-ui-tests/benchmark/src/androidTest/java/io/sentry/android/uitests/benchmark/util/BenchmarkOperationResult.kt new file mode 100644 index 00000000000..db8965724e1 --- /dev/null +++ b/sentry-android-ui-tests/benchmark/src/androidTest/java/io/sentry/android/uitests/benchmark/util/BenchmarkOperationResult.kt @@ -0,0 +1,85 @@ +package io.sentry.android.uitests.benchmark.util + +import java.util.concurrent.TimeUnit + +internal data class BenchmarkOperationResult( + val avgCpuTimeMillis: Long, + val avgDroppedFrames: Double, + val avgDurationNanos: Long, + val avgFramesPerSecond: Int, + val operationName: String +) { + fun compare(other: BenchmarkOperationResult): BenchmarkOperationComparison { + val durationDiffNanos = avgDurationNanos - other.avgDurationNanos + val durationIncreasePercentage = durationDiffNanos * 100.0 / other.avgDurationNanos + println("Measuring the duration increase of the operation in the transaction. If it's low enough, " + + "no end user will ever realize it") + println("[${other.operationName}] Average duration: ${other.avgDurationNanos} ns") + println("[${operationName}] Average duration: $avgDurationNanos ns") + if (durationIncreasePercentage > 0) { + println("Duration increase: %.2f%%".format(durationIncreasePercentage)) + } else { + println("No measurable duration increase detected.") + } + + println("--------------------") + + val cpuTimeDiff = (avgCpuTimeMillis - other.avgCpuTimeMillis) / Runtime.getRuntime().availableProcessors() + val cpuTimeOverheadPercentage = cpuTimeDiff * 100.0 / other.avgCpuTimeMillis + // Cpu time spent profiling is weighted based on available threads, as profiling runs on 1 thread only. + println("Measuring the increased cpu time. It has no direct impact on performance of the app, " + + "but it has on battery usage, as the cpu is 'awaken' longer.") + println("The weighted difference of cpu times is $cpuTimeDiff ms.") + println("[${other.operationName}] Cpu time: ${other.avgCpuTimeMillis} ms") + println("[${operationName}] Cpu time: $avgCpuTimeMillis ms") + if (cpuTimeOverheadPercentage > 0) { + println("CPU time overhead: %.2f%%".format(cpuTimeOverheadPercentage)) + } else { + println("No measurable CPU time overhead detected.") + } + + println("--------------------") + + val fpsDiff = other.avgFramesPerSecond - avgFramesPerSecond + val fpsDecreasePercentage = fpsDiff * 100.0 / other.avgFramesPerSecond + println("Measuring the decreased fps. Not really important, as even if fps are the same, the cpu could be " + + "doing more work in the frame window, and it could be hidden by checking average fps only.") + println("[${other.operationName}] Average FPS: ${other.avgFramesPerSecond}") + println("[${operationName}] Average FPS: $avgFramesPerSecond") + if (fpsDecreasePercentage > 0) { + println("FPS decrease: %.2f%%".format(fpsDecreasePercentage)) + } else { + println("No measurable FPS decrease detected.") + } + + println("--------------------") + + val droppedFramesDiff = avgDroppedFrames - other.avgDroppedFrames + val totalExpectedFrames = TimeUnit.NANOSECONDS.toMillis(other.avgDurationNanos) * 60 / 1000 + val droppedFramesIncreasePercentage = droppedFramesDiff * 100 / (totalExpectedFrames-other.avgDroppedFrames) + println("Measuring the increased fps drop rate. Very important, as it weights dropped frames based on the " + + "time passed between each frame. This is the metric end users can perceive as 'performance' in app usage.") + println("Dropped frames are calculated based on a target of 60 frames per second ($totalExpectedFrames total frames).") + println("[${other.operationName}] Average dropped frames: ${other.avgDroppedFrames}") + println("[${operationName}] Average dropped frames: $avgDroppedFrames") + if (droppedFramesIncreasePercentage > 0) { + println("Frame drop increase: %.2f%%".format(droppedFramesIncreasePercentage)) + } else { + println("No measurable frame drop increase detected.") + } + + return BenchmarkOperationComparison( + cpuTimeOverheadPercentage, + droppedFramesIncreasePercentage, + durationIncreasePercentage, + fpsDecreasePercentage + ) + } +} + +internal data class BenchmarkOperationComparison( + val cpuTimeIncrease: Double, + val droppedFramesIncrease: Double, + val durationIncrease: Double, + val fpsDecrease: Double +) diff --git a/sentry-android-ui-tests/benchmark/src/main/AndroidManifest.xml b/sentry-android-ui-tests/benchmark/src/main/AndroidManifest.xml new file mode 100644 index 00000000000..caaa237f1ad --- /dev/null +++ b/sentry-android-ui-tests/benchmark/src/main/AndroidManifest.xml @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + + diff --git a/sentry-android-ui-tests/benchmark/src/main/java/io/sentry/android/uitests/benchmark/BenchmarkActivity.kt b/sentry-android-ui-tests/benchmark/src/main/java/io/sentry/android/uitests/benchmark/BenchmarkActivity.kt new file mode 100644 index 00000000000..1155645702a --- /dev/null +++ b/sentry-android-ui-tests/benchmark/src/main/java/io/sentry/android/uitests/benchmark/BenchmarkActivity.kt @@ -0,0 +1,139 @@ +package io.sentry.android.uitests.benchmark + +import android.annotation.SuppressLint +import android.graphics.Bitmap +import android.graphics.Color +import android.os.Bundle +import android.view.LayoutInflater +import android.view.View +import android.view.ViewGroup +import android.widget.ImageView +import android.widget.TextView +import androidx.appcompat.app.AppCompatActivity +import androidx.recyclerview.widget.LinearLayoutManager +import androidx.recyclerview.widget.RecyclerView +import androidx.test.espresso.IdlingRegistry +import androidx.test.espresso.IdlingResource +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicBoolean +import kotlin.random.Random + +/** Idling resource based on a boolean flag. */ +class BooleanIdlingResource(private val name: String) : IdlingResource { + + private val isIdle = AtomicBoolean(true) + + private val isIdleLock = Object() + + private var callback: IdlingResource.ResourceCallback? = null + + fun setIdle(idling: Boolean) { + if (!isIdle.getAndSet(idling) && idling) { + callback?.onTransitionToIdle() + } + } + + override fun getName(): String = name + + override fun isIdleNow(): Boolean = synchronized(isIdleLock) { isIdle.get() } + + override fun registerIdleTransitionCallback(callback: IdlingResource.ResourceCallback?) { + this.callback = callback + } + +} + +/** A simple list of items, using [RecyclerView]. */ +class BenchmarkActivity : AppCompatActivity() { + + companion object { + + /** The activity will set this manage when scrolling. */ + val scrollingIdlingResource = BooleanIdlingResource("benchmark-activity") + + init { + IdlingRegistry.getInstance().register(scrollingIdlingResource) + } + } + + internal class ViewHolder(view: View) : RecyclerView.ViewHolder(view) { + val imageView = view.findViewById(R.id.benchmark_item_list_image) + val textView = view.findViewById(R.id.benchmark_item_list_text) + } + + internal inner class Adapter : RecyclerView.Adapter() { + override fun onCreateViewHolder(parent: ViewGroup, viewType: Int): ViewHolder { + val view = LayoutInflater.from(parent.context).inflate(R.layout.benchmark_item_list, parent, false) + return ViewHolder(view) + } + + @Suppress("MagicNumber") + override fun onBindViewHolder(holder: ViewHolder, position: Int) { + holder.imageView.setImageBitmap(generateBitmap()) + + @SuppressLint("SetTextI18n") + holder.textView.text = "Item $position ${"sentry ".repeat(position)}" + } + + // Disables view recycling. + override fun getItemViewType(position: Int): Int = position + + override fun getItemCount(): Int = 200 + } + + /** + * Each background thread will run non-stop calculations during the benchmark. One such thread seems + * enough to represent a busy application. This number can be increased to mimic busier applications. + */ + private val backgroundThreadPoolSize = 1 + private val executor: ExecutorService = Executors.newFixedThreadPool(backgroundThreadPoolSize) + private var resumed = false + + override fun onCreate(savedInstanceState: Bundle?) { + super.onCreate(savedInstanceState) + + setContentView(R.layout.activity_benchmark) + + findViewById(R.id.benchmark_transaction_list).apply { + layoutManager = LinearLayoutManager(this@BenchmarkActivity) + adapter = Adapter() + addOnScrollListener(object : RecyclerView.OnScrollListener() { + override fun onScrollStateChanged(recyclerView: RecyclerView, newState: Int) { + super.onScrollStateChanged(recyclerView, newState) + scrollingIdlingResource.setIdle(newState == RecyclerView.SCROLL_STATE_IDLE) + } + }) + } + } + + @Suppress("MagicNumber") + internal fun generateBitmap(): Bitmap { + val bitmapSize = 100 + val colors = (0 until (bitmapSize * bitmapSize)).map { + Color.rgb(Random.nextInt(256), Random.nextInt(256), Random.nextInt(256)) + }.toIntArray() + return Bitmap.createBitmap(colors, bitmapSize, bitmapSize, Bitmap.Config.ARGB_8888) + } + + @Suppress("MagicNumber") + override fun onResume() { + super.onResume() + resumed = true + + repeat(backgroundThreadPoolSize) { + executor.execute { + var x = 0 + for (i in 0..1_000_000_000) { + x += i * i + if (!resumed) break + } + } + } + } + + override fun onPause() { + super.onPause() + resumed = false + } +} diff --git a/sentry-android-ui-tests/benchmark/src/main/res/layout/activity_benchmark.xml b/sentry-android-ui-tests/benchmark/src/main/res/layout/activity_benchmark.xml new file mode 100644 index 00000000000..f70a1c2449f --- /dev/null +++ b/sentry-android-ui-tests/benchmark/src/main/res/layout/activity_benchmark.xml @@ -0,0 +1,29 @@ + + + + + + + + diff --git a/sentry-android-ui-tests/benchmark/src/main/res/layout/benchmark_item_list.xml b/sentry-android-ui-tests/benchmark/src/main/res/layout/benchmark_item_list.xml new file mode 100644 index 00000000000..a3bc14e3a78 --- /dev/null +++ b/sentry-android-ui-tests/benchmark/src/main/res/layout/benchmark_item_list.xml @@ -0,0 +1,31 @@ + + + + + + + + diff --git a/sentry-android-ui-tests/end2end/.gitignore b/sentry-android-ui-tests/end2end/.gitignore new file mode 100644 index 00000000000..796b96d1c40 --- /dev/null +++ b/sentry-android-ui-tests/end2end/.gitignore @@ -0,0 +1 @@ +/build diff --git a/sentry-android-ui-tests/end2end/build.gradle.kts b/sentry-android-ui-tests/end2end/build.gradle.kts new file mode 100644 index 00000000000..c22c066fb6a --- /dev/null +++ b/sentry-android-ui-tests/end2end/build.gradle.kts @@ -0,0 +1,75 @@ +plugins { + id("com.android.library") + kotlin("android") +} + +android { + compileSdk = Config.Android.compileSdkVersion + + defaultConfig { + minSdk = Config.Android.minSdkVersionOkHttp + targetSdk = Config.Android.targetSdkVersion + + testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner" + // Runs each test in its own instance of Instrumentation. This way they are isolated from + // one another and get their own Application instance. + // https://developer.android.com/training/testing/instrumented-tests/androidx-test-libraries/runner#enable-gradle + testInstrumentationRunnerArguments["clearPackageData"] = "true" + consumerProguardFiles("consumer-rules.pro") + } + +// testOptions { +// execution = "ANDROIDX_TEST_ORCHESTRATOR" it doesn't work with Android 11... +// } + + testBuildType = "debug" + + signingConfigs { + getByName("debug") { + storeFile = rootProject.file("debug.keystore") + storePassword = "android" + keyAlias = "androiddebugkey" + keyPassword = "android" + } + } + + buildTypes { + named("debug") { + isMinifyEnabled = true + signingConfig = signingConfigs.getByName("debug") + proguardFiles(getDefaultProguardFile("proguard-android-optimize.txt")) + + addManifestPlaceholders( + mapOf( + "sentryDebug" to true, + "sentryEnvironment" to "release" + ) + ) + } + } + + kotlinOptions { + jvmTarget = "1.8" + } +} + +dependencies { + + implementation(kotlin(Config.kotlinStdLib, org.jetbrains.kotlin.config.KotlinCompilerVersion.VERSION)) + + implementation(projects.sentryAndroid) + implementation(Config.Libs.appCompat) + implementation(Config.Libs.androidxCore) + implementation(Config.Libs.androidxRecylerView) + implementation(Config.Libs.constraintLayout) + implementation(Config.TestLibs.espressoIdlingResource) + + androidTestImplementation(Config.TestLibs.kotlinTestJunit) + androidTestImplementation(Config.TestLibs.mockWebserver) + androidTestImplementation(Config.TestLibs.espressoCore) + androidTestImplementation(Config.TestLibs.androidxTestCoreKtx) + androidTestImplementation(Config.TestLibs.androidxRunner) + androidTestImplementation(Config.TestLibs.androidxTestRules) + androidTestImplementation(Config.TestLibs.androidxJunit) + androidTestUtil(Config.TestLibs.androidxTestOrchestrator) +} diff --git a/sentry-android-ui-tests/end2end/consumer-rules.pro b/sentry-android-ui-tests/end2end/consumer-rules.pro new file mode 100644 index 00000000000..e69de29bb2d diff --git a/sentry-android-ui-tests/end2end/proguard-rules.pro b/sentry-android-ui-tests/end2end/proguard-rules.pro new file mode 100644 index 00000000000..f1b424510da --- /dev/null +++ b/sentry-android-ui-tests/end2end/proguard-rules.pro @@ -0,0 +1,21 @@ +# Add project specific ProGuard rules here. +# You can control the set of applied configuration files using the +# proguardFiles setting in build.gradle. +# +# For more details, see +# http://developer.android.com/guide/developing/tools/proguard.html + +# If your project uses WebView with JS, uncomment the following +# and specify the fully qualified class name to the JavaScript interface +# class: +#-keepclassmembers class fqcn.of.javascript.interface.for.webview { +# public *; +#} + +# Uncomment this to preserve the line number information for +# debugging stack traces. +#-keepattributes SourceFile,LineNumberTable + +# If you keep the line number information, uncomment this to +# hide the original source file name. +#-renamesourcefileattribute SourceFile diff --git a/sentry-android-ui-tests/end2end/src/androidTest/java/io/sentry/android/uitests/end2end/BaseUiTest.kt b/sentry-android-ui-tests/end2end/src/androidTest/java/io/sentry/android/uitests/end2end/BaseUiTest.kt new file mode 100644 index 00000000000..a14f5b665bb --- /dev/null +++ b/sentry-android-ui-tests/end2end/src/androidTest/java/io/sentry/android/uitests/end2end/BaseUiTest.kt @@ -0,0 +1,61 @@ +package io.sentry.android.uitests.end2end + +import android.app.Application +import android.content.Context +import android.content.pm.ApplicationInfo +import androidx.test.core.app.ApplicationProvider +import androidx.test.espresso.Espresso +import androidx.test.ext.junit.runners.AndroidJUnit4 +import androidx.test.platform.app.InstrumentationRegistry +import androidx.test.runner.AndroidJUnitRunner +import io.sentry.android.uitests.end2end.mockservers.TestMockWebServers +import org.junit.runner.RunWith +import java.net.InetAddress +import kotlin.test.AfterTest +import kotlin.test.BeforeTest + +@RunWith(AndroidJUnit4::class) +abstract class BaseUiTest { + + protected lateinit var runner: AndroidJUnitRunner + protected lateinit var context: Context + protected val servers = TestMockWebServers() + + protected val mainThreadId: Long + get() { + var id = -1L + runner.runOnMainSync { id = android.os.Process.myTid().toLong() } + check(id != -1L) + return id + } + + protected val applicationId: String + get() = InstrumentationRegistry.getInstrumentation().context + .packageName.removeSuffix(".test") + + protected val isAppDebuggable = + ( + ApplicationProvider.getApplicationContext().applicationInfo.flags and + ApplicationInfo.FLAG_DEBUGGABLE + ) != 0 + + /** + * Waits until the Specto SDK is idle. + */ + fun waitUntilIdle() { + // We rely on Espresso's idling resources. + Espresso.onIdle() + } + + @BeforeTest + fun baseSetUp() { + runner = InstrumentationRegistry.getInstrumentation() as AndroidJUnitRunner + context = ApplicationProvider.getApplicationContext() + servers.relay.start() + } + + @AfterTest + fun baseFinish() { + servers.relay.shutdown() + } +} diff --git a/sentry-android-ui-tests/end2end/src/androidTest/java/io/sentry/android/uitests/end2end/ExampleInstrumentedTest.kt b/sentry-android-ui-tests/end2end/src/androidTest/java/io/sentry/android/uitests/end2end/ExampleInstrumentedTest.kt new file mode 100644 index 00000000000..aaa55e32ad7 --- /dev/null +++ b/sentry-android-ui-tests/end2end/src/androidTest/java/io/sentry/android/uitests/end2end/ExampleInstrumentedTest.kt @@ -0,0 +1,35 @@ +package io.sentry.android.uitests.end2end + +import androidx.lifecycle.Lifecycle +import androidx.test.core.app.launchActivity +import androidx.test.ext.junit.runners.AndroidJUnit4 +import io.sentry.Sentry + +import org.junit.Test +import org.junit.runner.RunWith + +import org.junit.Assert.* + +/** + * Instrumented test, which will execute on an Android device. + * + * See [testing documentation](http://d.android.com/tools/testing). + */ +@RunWith(AndroidJUnit4::class) +class ExampleInstrumentedTest : BaseUiTest() { + + @Test + fun useAppContext() { + + Sentry.init { + it.dsn = "http://12345678901234567890123456789012@${servers.relay.hostName}:${servers.relay.port}/1234567" + } + + val emptyActivity = launchActivity() + Sentry.captureMessage("aaaaa") + Thread.sleep(10000) + emptyActivity.moveToState(Lifecycle.State.DESTROYED) + // Context of the app under test. + assertEquals("io.sentry.android.uitests.end2end.test", context.packageName) + } +} diff --git a/sentry-android-ui-tests/end2end/src/androidTest/java/io/sentry/android/uitests/end2end/mockservers/TestMockWebServers.kt b/sentry-android-ui-tests/end2end/src/androidTest/java/io/sentry/android/uitests/end2end/mockservers/TestMockWebServers.kt new file mode 100644 index 00000000000..728127e124c --- /dev/null +++ b/sentry-android-ui-tests/end2end/src/androidTest/java/io/sentry/android/uitests/end2end/mockservers/TestMockWebServers.kt @@ -0,0 +1,22 @@ +package io.sentry.android.uitests.end2end.mockservers + +import okhttp3.mockwebserver.Dispatcher +import okhttp3.mockwebserver.MockResponse +import okhttp3.mockwebserver.MockWebServer +import okhttp3.mockwebserver.RecordedRequest + +/** Interface exposing mock webservers */ +class TestMockWebServers { + + /** Mocks a relay server. */ + val relay = MockWebServer() + + init { + relay.dispatcher = object : Dispatcher() { + override fun dispatch(request: RecordedRequest): MockResponse { + request.path + return MockResponse() + } + } + } +} diff --git a/sentry-android-ui-tests/end2end/src/main/AndroidManifest.xml b/sentry-android-ui-tests/end2end/src/main/AndroidManifest.xml new file mode 100644 index 00000000000..7ae0b9662a1 --- /dev/null +++ b/sentry-android-ui-tests/end2end/src/main/AndroidManifest.xml @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + diff --git a/sentry-android-ui-tests/end2end/src/main/java/io/sentry/android/uitests/end2end/EmptyActivity.kt b/sentry-android-ui-tests/end2end/src/main/java/io/sentry/android/uitests/end2end/EmptyActivity.kt new file mode 100644 index 00000000000..6d033fa8f26 --- /dev/null +++ b/sentry-android-ui-tests/end2end/src/main/java/io/sentry/android/uitests/end2end/EmptyActivity.kt @@ -0,0 +1,26 @@ +package io.sentry.android.uitests.end2end + +import android.os.Bundle +import androidx.appcompat.app.AppCompatActivity +import io.sentry.Sentry + +class EmptyActivity : AppCompatActivity() { + + override fun onCreate(savedInstanceState: Bundle?) { + super.onCreate(savedInstanceState) + + Thread { + val t = Sentry.startTransaction("e2e tests", "empty onCreate") + try { + throw Exception("This is a test.") + } catch (e: Exception) { + Sentry.captureException(e) + } + + Thread.sleep(1000) + t.finish() + }.start() + Thread.sleep(5000) + + } +} diff --git a/settings.gradle.kts b/settings.gradle.kts index 45e3483ea13..139e37db922 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -12,6 +12,8 @@ include( "sentry-android-timber", "sentry-android-okhttp", "sentry-android-fragment", + "sentry-android-ui-tests:benchmark", + "sentry-android-ui-tests:end2end", "sentry-apollo", "sentry-test-support", "sentry-log4j2", From d62a03c04e68411cabbdeed34f8665ee2dbc2fa7 Mon Sep 17 00:00:00 2001 From: stefanosiano Date: Tue, 26 Apr 2022 12:33:03 +0200 Subject: [PATCH 02/22] big cleanup of BaseUiTest added mock relay server with envelope and items assertions --- .../benchmark/build.gradle.kts | 1 + .../benchmark/src/main/AndroidManifest.xml | 4 +- .../uitests/benchmark/BenchmarkActivity.kt | 32 +----- .../android/uitests/end2end/BaseUiTest.kt | 61 ++++++----- .../end2end/ExampleInstrumentedTest.kt | 22 ++-- .../end2end/mockservers/EnvelopeAsserter.kt | 25 +++++ .../uitests/end2end/mockservers/MockRelay.kt | 101 ++++++++++++++++++ .../end2end/mockservers/RelayAsserter.kt | 45 ++++++++ .../end2end/mockservers/TestMockWebServers.kt | 22 ---- .../android/uitests/end2end/EmptyActivity.kt | 14 --- .../end2end/utils/BooleanIdlingResource.kt | 29 +++++ 11 files changed, 252 insertions(+), 104 deletions(-) create mode 100644 sentry-android-ui-tests/end2end/src/androidTest/java/io/sentry/android/uitests/end2end/mockservers/EnvelopeAsserter.kt create mode 100644 sentry-android-ui-tests/end2end/src/androidTest/java/io/sentry/android/uitests/end2end/mockservers/MockRelay.kt create mode 100644 sentry-android-ui-tests/end2end/src/androidTest/java/io/sentry/android/uitests/end2end/mockservers/RelayAsserter.kt delete mode 100644 sentry-android-ui-tests/end2end/src/androidTest/java/io/sentry/android/uitests/end2end/mockservers/TestMockWebServers.kt create mode 100644 sentry-android-ui-tests/end2end/src/main/java/io/sentry/android/uitests/end2end/utils/BooleanIdlingResource.kt diff --git a/sentry-android-ui-tests/benchmark/build.gradle.kts b/sentry-android-ui-tests/benchmark/build.gradle.kts index a05f9fba484..87bb8fe85ec 100644 --- a/sentry-android-ui-tests/benchmark/build.gradle.kts +++ b/sentry-android-ui-tests/benchmark/build.gradle.kts @@ -65,6 +65,7 @@ dependencies { implementation(kotlin(Config.kotlinStdLib, org.jetbrains.kotlin.config.KotlinCompilerVersion.VERSION)) implementation(projects.sentryAndroid) + implementation(projects.sentryAndroidUiTests.end2end) implementation(Config.Libs.appCompat) implementation(Config.Libs.androidxCore) implementation(Config.Libs.androidxRecylerView) diff --git a/sentry-android-ui-tests/benchmark/src/main/AndroidManifest.xml b/sentry-android-ui-tests/benchmark/src/main/AndroidManifest.xml index caaa237f1ad..734a753cf23 100644 --- a/sentry-android-ui-tests/benchmark/src/main/AndroidManifest.xml +++ b/sentry-android-ui-tests/benchmark/src/main/AndroidManifest.xml @@ -1,10 +1,12 @@ + android:label="Sentry Benchmark" + tools:replace="android:label"> diff --git a/sentry-android-ui-tests/benchmark/src/main/java/io/sentry/android/uitests/benchmark/BenchmarkActivity.kt b/sentry-android-ui-tests/benchmark/src/main/java/io/sentry/android/uitests/benchmark/BenchmarkActivity.kt index 1155645702a..a85e2d4e8c3 100644 --- a/sentry-android-ui-tests/benchmark/src/main/java/io/sentry/android/uitests/benchmark/BenchmarkActivity.kt +++ b/sentry-android-ui-tests/benchmark/src/main/java/io/sentry/android/uitests/benchmark/BenchmarkActivity.kt @@ -13,37 +13,11 @@ import androidx.appcompat.app.AppCompatActivity import androidx.recyclerview.widget.LinearLayoutManager import androidx.recyclerview.widget.RecyclerView import androidx.test.espresso.IdlingRegistry -import androidx.test.espresso.IdlingResource +import io.sentry.android.uitests.end2end.utils.BooleanIdlingResource import java.util.concurrent.ExecutorService import java.util.concurrent.Executors -import java.util.concurrent.atomic.AtomicBoolean import kotlin.random.Random -/** Idling resource based on a boolean flag. */ -class BooleanIdlingResource(private val name: String) : IdlingResource { - - private val isIdle = AtomicBoolean(true) - - private val isIdleLock = Object() - - private var callback: IdlingResource.ResourceCallback? = null - - fun setIdle(idling: Boolean) { - if (!isIdle.getAndSet(idling) && idling) { - callback?.onTransitionToIdle() - } - } - - override fun getName(): String = name - - override fun isIdleNow(): Boolean = synchronized(isIdleLock) { isIdle.get() } - - override fun registerIdleTransitionCallback(callback: IdlingResource.ResourceCallback?) { - this.callback = callback - } - -} - /** A simple list of items, using [RecyclerView]. */ class BenchmarkActivity : AppCompatActivity() { @@ -58,8 +32,8 @@ class BenchmarkActivity : AppCompatActivity() { } internal class ViewHolder(view: View) : RecyclerView.ViewHolder(view) { - val imageView = view.findViewById(R.id.benchmark_item_list_image) - val textView = view.findViewById(R.id.benchmark_item_list_text) + val imageView: ImageView = view.findViewById(R.id.benchmark_item_list_image) + val textView: TextView = view.findViewById(R.id.benchmark_item_list_text) } internal inner class Adapter : RecyclerView.Adapter() { diff --git a/sentry-android-ui-tests/end2end/src/androidTest/java/io/sentry/android/uitests/end2end/BaseUiTest.kt b/sentry-android-ui-tests/end2end/src/androidTest/java/io/sentry/android/uitests/end2end/BaseUiTest.kt index a14f5b665bb..490fea7f081 100644 --- a/sentry-android-ui-tests/end2end/src/androidTest/java/io/sentry/android/uitests/end2end/BaseUiTest.kt +++ b/sentry-android-ui-tests/end2end/src/androidTest/java/io/sentry/android/uitests/end2end/BaseUiTest.kt @@ -5,57 +5,60 @@ import android.content.Context import android.content.pm.ApplicationInfo import androidx.test.core.app.ApplicationProvider import androidx.test.espresso.Espresso +import androidx.test.espresso.IdlingRegistry +import androidx.test.espresso.idling.CountingIdlingResource import androidx.test.ext.junit.runners.AndroidJUnit4 import androidx.test.platform.app.InstrumentationRegistry import androidx.test.runner.AndroidJUnitRunner -import io.sentry.android.uitests.end2end.mockservers.TestMockWebServers +import io.sentry.Sentry +import io.sentry.SentryOptions +import io.sentry.android.uitests.end2end.mockservers.MockRelay import org.junit.runner.RunWith -import java.net.InetAddress import kotlin.test.AfterTest import kotlin.test.BeforeTest @RunWith(AndroidJUnit4::class) abstract class BaseUiTest { + protected val relayIdlingResource = CountingIdlingResource("relay-requests") protected lateinit var runner: AndroidJUnitRunner protected lateinit var context: Context - protected val servers = TestMockWebServers() - - protected val mainThreadId: Long - get() { - var id = -1L - runner.runOnMainSync { id = android.os.Process.myTid().toLong() } - check(id != -1L) - return id - } + protected lateinit var dsn: String + protected val relay = MockRelay(false, relayIdlingResource) - protected val applicationId: String - get() = InstrumentationRegistry.getInstrumentation().context - .packageName.removeSuffix(".test") - - protected val isAppDebuggable = - ( - ApplicationProvider.getApplicationContext().applicationInfo.flags and - ApplicationInfo.FLAG_DEBUGGABLE - ) != 0 - - /** - * Waits until the Specto SDK is idle. - */ - fun waitUntilIdle() { - // We rely on Espresso's idling resources. - Espresso.onIdle() + init { + IdlingRegistry.getInstance().register(relayIdlingResource) } @BeforeTest fun baseSetUp() { runner = InstrumentationRegistry.getInstrumentation() as AndroidJUnitRunner context = ApplicationProvider.getApplicationContext() - servers.relay.start() + relay.start() + dsn = "http://key@${relay.hostName}:${relay.port}/${relay.dsnProject}" } @AfterTest fun baseFinish() { - servers.relay.shutdown() + relay.shutdown() + } + + protected fun initSentry( + relayCheckIdlingResources: Boolean = true, + optionsConfiguration: ((options: SentryOptions) -> Unit)? = null + ) { + relay.checkIdlingResources = relayCheckIdlingResources + Sentry.init { + it.dsn = dsn + optionsConfiguration?.invoke(it) + } } } + +/** + * Waits until the Specto SDK is idle. + */ +fun waitUntilIdle() { + // We rely on Espresso's idling resources. + Espresso.onIdle() +} diff --git a/sentry-android-ui-tests/end2end/src/androidTest/java/io/sentry/android/uitests/end2end/ExampleInstrumentedTest.kt b/sentry-android-ui-tests/end2end/src/androidTest/java/io/sentry/android/uitests/end2end/ExampleInstrumentedTest.kt index aaa55e32ad7..6261cf38cc7 100644 --- a/sentry-android-ui-tests/end2end/src/androidTest/java/io/sentry/android/uitests/end2end/ExampleInstrumentedTest.kt +++ b/sentry-android-ui-tests/end2end/src/androidTest/java/io/sentry/android/uitests/end2end/ExampleInstrumentedTest.kt @@ -4,6 +4,7 @@ import androidx.lifecycle.Lifecycle import androidx.test.core.app.launchActivity import androidx.test.ext.junit.runners.AndroidJUnit4 import io.sentry.Sentry +import io.sentry.SentryEvent import org.junit.Test import org.junit.runner.RunWith @@ -21,15 +22,18 @@ class ExampleInstrumentedTest : BaseUiTest() { @Test fun useAppContext() { - Sentry.init { - it.dsn = "http://12345678901234567890123456789012@${servers.relay.hostName}:${servers.relay.port}/1234567" - } + initSentry(true) + relayIdlingResource.increment() + Sentry.captureMessage("Message captured during test") - val emptyActivity = launchActivity() - Sentry.captureMessage("aaaaa") - Thread.sleep(10000) - emptyActivity.moveToState(Lifecycle.State.DESTROYED) - // Context of the app under test. - assertEquals("io.sentry.android.uitests.end2end.test", context.packageName) + relay.assert { + assertEnvelope { + val event = it.assertItem(SentryEvent::class.java) + it.assertNoOtherItems() + assertTrue(event.message?.formatted == "Message captured during test") + } + assertNoOtherEnvelopes() + assertNoOtherRequests() + } } } diff --git a/sentry-android-ui-tests/end2end/src/androidTest/java/io/sentry/android/uitests/end2end/mockservers/EnvelopeAsserter.kt b/sentry-android-ui-tests/end2end/src/androidTest/java/io/sentry/android/uitests/end2end/mockservers/EnvelopeAsserter.kt new file mode 100644 index 00000000000..355b8dd2ef6 --- /dev/null +++ b/sentry-android-ui-tests/end2end/src/androidTest/java/io/sentry/android/uitests/end2end/mockservers/EnvelopeAsserter.kt @@ -0,0 +1,25 @@ +package io.sentry.android.uitests.end2end.mockservers + +import io.sentry.Sentry +import io.sentry.SentryEnvelope +import okhttp3.mockwebserver.MockResponse + +class EnvelopeAsserter(val envelope: SentryEnvelope, val response: MockResponse) { + private val unassertedItems = envelope.items.toMutableList() + + fun assertItem(clazz: Class): T { + val item = unassertedItems.mapIndexed { index, it -> + val deserialized = Sentry.getCurrentHub().options.serializer.deserialize(it.data.inputStream().reader(), clazz) + deserialized?.let { Pair(index, it) } + }.filterNotNull().firstOrNull() + ?: throw AssertionError("No item found of type: ${clazz.name}") + unassertedItems.removeAt(item.first) + return item.second + } + + fun assertNoOtherItems() { + if (unassertedItems.isNotEmpty()) { + throw AssertionError("There were other items: $unassertedItems") + } + } +} diff --git a/sentry-android-ui-tests/end2end/src/androidTest/java/io/sentry/android/uitests/end2end/mockservers/MockRelay.kt b/sentry-android-ui-tests/end2end/src/androidTest/java/io/sentry/android/uitests/end2end/mockservers/MockRelay.kt new file mode 100644 index 00000000000..1180e674c89 --- /dev/null +++ b/sentry-android-ui-tests/end2end/src/androidTest/java/io/sentry/android/uitests/end2end/mockservers/MockRelay.kt @@ -0,0 +1,101 @@ +package io.sentry.android.uitests.end2end.mockservers + +import androidx.test.espresso.idling.CountingIdlingResource +import io.sentry.android.uitests.end2end.BaseUiTest +import io.sentry.android.uitests.end2end.utils.BooleanIdlingResource +import io.sentry.android.uitests.end2end.waitUntilIdle +import okhttp3.mockwebserver.Dispatcher +import okhttp3.mockwebserver.MockResponse +import okhttp3.mockwebserver.MockWebServer +import okhttp3.mockwebserver.RecordedRequest + +/** Interface exposing mock webservers */ +class MockRelay ( + var checkIdlingResources: Boolean, + private val relayIdlingResource: CountingIdlingResource +) { + + /** Mocks a relay server. */ + private val relay = MockWebServer() + + val hostName + get() = relay.hostName + val port + get() = relay.port + + val dsnProject = "1234" + private val envelopePath = "/api/$dsnProject/envelope/" + + private val unassertedEnvelopes = mutableListOf() + private val unassertedRequests = mutableListOf() + + private val responses = mutableListOf<(RecordedRequest) -> MockResponse?>() + + init { + relay.dispatcher = object : Dispatcher() { + override fun dispatch(request: RecordedRequest): MockResponse { + val response = responses.asSequence() + .mapNotNull { it(request) } + .firstOrNull() + ?: MockResponse().setResponseCode(404) + dispatchSentRequest(RelayAsserter.RelayResponse(request, response)) + + if (checkIdlingResources) { + relayIdlingResource.decrement() + } + return response + } + } + } + + fun start() = relay.start() + fun shutdown() { + clearResponses() + relay.shutdown() + } + + fun addResponse(response: (RecordedRequest) -> MockResponse?) { + // Responses are added to the beginning of the list so they'll take precedence over + // previously added ones. + responses.add(0, response) + } + + fun addResponse( + filter: (RecordedRequest) -> Boolean, + responseBuilder: ((request: RecordedRequest, response: MockResponse) -> Unit)? = null + ) { + addResponse { request -> + if (filter(request)) { + MockResponse().also { response -> + responseBuilder?.invoke(request, response) + } + } else { + null + } + } + } + + fun clearResponses() { + responses.clear() + } + + fun assert(assertion: RelayAsserter.() -> Unit) { + if (checkIdlingResources) { + waitUntilIdle() + } + assertion(RelayAsserter(unassertedEnvelopes, unassertedRequests)) + } + + private fun dispatchSentRequest(relayResponse: RelayAsserter.RelayResponse) { + when { + relayResponse.request.path == envelopePath -> { + unassertedEnvelopes.add(relayResponse) + } + else -> { + unassertedRequests.add(relayResponse) + } + } + } + + fun MockResponse.hasSucceed() = status.matches(".*2[\\d]{2} OK".toRegex()) +} diff --git a/sentry-android-ui-tests/end2end/src/androidTest/java/io/sentry/android/uitests/end2end/mockservers/RelayAsserter.kt b/sentry-android-ui-tests/end2end/src/androidTest/java/io/sentry/android/uitests/end2end/mockservers/RelayAsserter.kt new file mode 100644 index 00000000000..de9358fd005 --- /dev/null +++ b/sentry-android-ui-tests/end2end/src/androidTest/java/io/sentry/android/uitests/end2end/mockservers/RelayAsserter.kt @@ -0,0 +1,45 @@ +package io.sentry.android.uitests.end2end.mockservers + +import io.sentry.EnvelopeReader +import io.sentry.Sentry +import okhttp3.mockwebserver.MockResponse +import okhttp3.mockwebserver.RecordedRequest +import java.util.zip.GZIPInputStream + +class RelayAsserter( + private val unassertedEnvelopes: MutableList, + private val unassertedRequests: MutableList +) { + + fun assertRawRequest(assertion: ((request: RecordedRequest, response: MockResponse) -> Unit)? = null) { + val relayResponse = unassertedEnvelopes.removeFirstOrNull() + ?: throw AssertionError("No envelope request found") + assertion?.let { + it(relayResponse.request, relayResponse.response) + } + } + + fun assertEnvelope(assertion: (asserter: EnvelopeAsserter) -> Unit) { + assertRawRequest { request, response -> + val envelope = EnvelopeReader(Sentry.getCurrentHub().options.serializer) + .read(GZIPInputStream(request.body.inputStream())) + ?: throw AssertionError("Was unable to parse the request as an envelope: $request") + assertion(EnvelopeAsserter(envelope, response)) + } + } + + fun assertNoOtherEnvelopes() { + if (unassertedEnvelopes.isNotEmpty()) { + throw AssertionError("There were other ${unassertedEnvelopes.size} envelope requests: $unassertedEnvelopes") + } + } + + fun assertNoOtherRequests() { + assertNoOtherEnvelopes() + if (unassertedRequests.isNotEmpty()) { + throw AssertionError("There were other ${unassertedRequests.size} requests: $unassertedRequests") + } + } + + data class RelayResponse (val request: RecordedRequest, val response: MockResponse) +} diff --git a/sentry-android-ui-tests/end2end/src/androidTest/java/io/sentry/android/uitests/end2end/mockservers/TestMockWebServers.kt b/sentry-android-ui-tests/end2end/src/androidTest/java/io/sentry/android/uitests/end2end/mockservers/TestMockWebServers.kt deleted file mode 100644 index 728127e124c..00000000000 --- a/sentry-android-ui-tests/end2end/src/androidTest/java/io/sentry/android/uitests/end2end/mockservers/TestMockWebServers.kt +++ /dev/null @@ -1,22 +0,0 @@ -package io.sentry.android.uitests.end2end.mockservers - -import okhttp3.mockwebserver.Dispatcher -import okhttp3.mockwebserver.MockResponse -import okhttp3.mockwebserver.MockWebServer -import okhttp3.mockwebserver.RecordedRequest - -/** Interface exposing mock webservers */ -class TestMockWebServers { - - /** Mocks a relay server. */ - val relay = MockWebServer() - - init { - relay.dispatcher = object : Dispatcher() { - override fun dispatch(request: RecordedRequest): MockResponse { - request.path - return MockResponse() - } - } - } -} diff --git a/sentry-android-ui-tests/end2end/src/main/java/io/sentry/android/uitests/end2end/EmptyActivity.kt b/sentry-android-ui-tests/end2end/src/main/java/io/sentry/android/uitests/end2end/EmptyActivity.kt index 6d033fa8f26..ec10b4160a3 100644 --- a/sentry-android-ui-tests/end2end/src/main/java/io/sentry/android/uitests/end2end/EmptyActivity.kt +++ b/sentry-android-ui-tests/end2end/src/main/java/io/sentry/android/uitests/end2end/EmptyActivity.kt @@ -8,19 +8,5 @@ class EmptyActivity : AppCompatActivity() { override fun onCreate(savedInstanceState: Bundle?) { super.onCreate(savedInstanceState) - - Thread { - val t = Sentry.startTransaction("e2e tests", "empty onCreate") - try { - throw Exception("This is a test.") - } catch (e: Exception) { - Sentry.captureException(e) - } - - Thread.sleep(1000) - t.finish() - }.start() - Thread.sleep(5000) - } } diff --git a/sentry-android-ui-tests/end2end/src/main/java/io/sentry/android/uitests/end2end/utils/BooleanIdlingResource.kt b/sentry-android-ui-tests/end2end/src/main/java/io/sentry/android/uitests/end2end/utils/BooleanIdlingResource.kt new file mode 100644 index 00000000000..62be29514fa --- /dev/null +++ b/sentry-android-ui-tests/end2end/src/main/java/io/sentry/android/uitests/end2end/utils/BooleanIdlingResource.kt @@ -0,0 +1,29 @@ +package io.sentry.android.uitests.end2end.utils + +import androidx.test.espresso.IdlingResource +import java.util.concurrent.atomic.AtomicBoolean + +/** Idling resource based on a boolean flag. */ +class BooleanIdlingResource(private val name: String) : IdlingResource { + + private val isIdle = AtomicBoolean(true) + + private val isIdleLock = Object() + + private var callback: IdlingResource.ResourceCallback? = null + + fun setIdle(idling: Boolean) { + if (!isIdle.getAndSet(idling) && idling) { + callback?.onTransitionToIdle() + } + } + + override fun getName(): String = name + + override fun isIdleNow(): Boolean = synchronized(isIdleLock) { isIdle.get() } + + override fun registerIdleTransitionCallback(callback: IdlingResource.ResourceCallback?) { + this.callback = callback + } + +} From c6dc27f15abc7cd31bd53bb5fb56d874861a6241 Mon Sep 17 00:00:00 2001 From: stefanosiano Date: Tue, 26 Apr 2022 15:59:50 +0200 Subject: [PATCH 03/22] changed benchmark to manual init app cache is deleted on test start MockRelay defaults to success response --- .../uitests/benchmark/SentryBenchmarkTest.kt | 19 ++++++------ .../benchmark/src/main/AndroidManifest.xml | 5 +-- .../android/uitests/end2end/BaseUiTest.kt | 8 +++-- .../end2end/ExampleInstrumentedTest.kt | 31 +++++++++++++++++++ .../uitests/end2end/mockservers/MockRelay.kt | 3 +- 5 files changed, 50 insertions(+), 16 deletions(-) diff --git a/sentry-android-ui-tests/benchmark/src/androidTest/java/io/sentry/android/uitests/benchmark/SentryBenchmarkTest.kt b/sentry-android-ui-tests/benchmark/src/androidTest/java/io/sentry/android/uitests/benchmark/SentryBenchmarkTest.kt index 9d45c7cbf5d..bf9a8231ef1 100644 --- a/sentry-android-ui-tests/benchmark/src/androidTest/java/io/sentry/android/uitests/benchmark/SentryBenchmarkTest.kt +++ b/sentry-android-ui-tests/benchmark/src/androidTest/java/io/sentry/android/uitests/benchmark/SentryBenchmarkTest.kt @@ -13,9 +13,12 @@ import androidx.test.platform.app.InstrumentationRegistry import androidx.test.runner.AndroidJUnitRunner import io.sentry.ITransaction import io.sentry.Sentry +import io.sentry.SentryOptions +import io.sentry.android.core.SentryAndroid import io.sentry.android.uitests.benchmark.util.BenchmarkOperation import org.junit.Test import org.junit.runner.RunWith +import java.io.File import kotlin.test.BeforeTest import kotlin.test.assertTrue @@ -47,16 +50,14 @@ class SentryBenchmarkTest { fun benchmarkProfiledTransaction() { // runOnMainSync ensure that we're using Sentry on a thread that belongs to the actual // application, rather than the test application. - // todo Right now we cannot enable profiling via code, but only via manifest. - // We should fix it and initialize via code, so that different tests can use different options. runner.runOnMainSync { -// SentryAndroid.init(context) { options: SentryOptions -> -// options.dsn = "https://12345678901234567890123456789012@fakeuri.ingest.sentry.io/1234567" -// options.isEnableAutoSessionTracking = false -// options.tracesSampleRate = 1.0 -// options.isTraceSampling = true -// options.isProfilingEnabled = true -// } + SentryAndroid.init(context) { options: SentryOptions -> + options.dsn = "https://fakesecret@fakeuri.ingest.sentry.io/1234567" + options.isEnableAutoSessionTracking = false + options.tracesSampleRate = 1.0 + options.isTraceSampling = true + options.isProfilingEnabled = true + } } val benchmarkOperationNoTransaction = BenchmarkOperation(runner, getOperation(runner) { null }) diff --git a/sentry-android-ui-tests/benchmark/src/main/AndroidManifest.xml b/sentry-android-ui-tests/benchmark/src/main/AndroidManifest.xml index 734a753cf23..7f53b1fdf06 100644 --- a/sentry-android-ui-tests/benchmark/src/main/AndroidManifest.xml +++ b/sentry-android-ui-tests/benchmark/src/main/AndroidManifest.xml @@ -10,10 +10,7 @@ - - - - + diff --git a/sentry-android-ui-tests/end2end/src/androidTest/java/io/sentry/android/uitests/end2end/BaseUiTest.kt b/sentry-android-ui-tests/end2end/src/androidTest/java/io/sentry/android/uitests/end2end/BaseUiTest.kt index 490fea7f081..a234a409d2f 100644 --- a/sentry-android-ui-tests/end2end/src/androidTest/java/io/sentry/android/uitests/end2end/BaseUiTest.kt +++ b/sentry-android-ui-tests/end2end/src/androidTest/java/io/sentry/android/uitests/end2end/BaseUiTest.kt @@ -12,6 +12,7 @@ import androidx.test.platform.app.InstrumentationRegistry import androidx.test.runner.AndroidJUnitRunner import io.sentry.Sentry import io.sentry.SentryOptions +import io.sentry.android.core.SentryAndroid import io.sentry.android.uitests.end2end.mockservers.MockRelay import org.junit.runner.RunWith import kotlin.test.AfterTest @@ -27,20 +28,23 @@ abstract class BaseUiTest { protected val relay = MockRelay(false, relayIdlingResource) init { - IdlingRegistry.getInstance().register(relayIdlingResource) } @BeforeTest fun baseSetUp() { runner = InstrumentationRegistry.getInstrumentation() as AndroidJUnitRunner context = ApplicationProvider.getApplicationContext() + context.cacheDir.deleteRecursively() + IdlingRegistry.getInstance().register(relayIdlingResource) relay.start() dsn = "http://key@${relay.hostName}:${relay.port}/${relay.dsnProject}" } @AfterTest fun baseFinish() { + IdlingRegistry.getInstance().unregister(relayIdlingResource) relay.shutdown() + Sentry.close() } protected fun initSentry( @@ -48,7 +52,7 @@ abstract class BaseUiTest { optionsConfiguration: ((options: SentryOptions) -> Unit)? = null ) { relay.checkIdlingResources = relayCheckIdlingResources - Sentry.init { + SentryAndroid.init(context) { it.dsn = dsn optionsConfiguration?.invoke(it) } diff --git a/sentry-android-ui-tests/end2end/src/androidTest/java/io/sentry/android/uitests/end2end/ExampleInstrumentedTest.kt b/sentry-android-ui-tests/end2end/src/androidTest/java/io/sentry/android/uitests/end2end/ExampleInstrumentedTest.kt index 6261cf38cc7..7978d90fa28 100644 --- a/sentry-android-ui-tests/end2end/src/androidTest/java/io/sentry/android/uitests/end2end/ExampleInstrumentedTest.kt +++ b/sentry-android-ui-tests/end2end/src/androidTest/java/io/sentry/android/uitests/end2end/ExampleInstrumentedTest.kt @@ -3,8 +3,12 @@ package io.sentry.android.uitests.end2end import androidx.lifecycle.Lifecycle import androidx.test.core.app.launchActivity import androidx.test.ext.junit.runners.AndroidJUnit4 +import io.sentry.ProfilingTraceData import io.sentry.Sentry import io.sentry.SentryEvent +import io.sentry.SentryOptions +import io.sentry.protocol.SentryTransaction +import okhttp3.mockwebserver.MockResponse import org.junit.Test import org.junit.runner.RunWith @@ -36,4 +40,31 @@ class ExampleInstrumentedTest : BaseUiTest() { assertNoOtherRequests() } } + + @Test + fun useAppContex2t() { + + initSentry(true) { options: SentryOptions -> + options.isEnableAutoSessionTracking = false + options.tracesSampleRate = 1.0 + options.isTraceSampling = true + options.isProfilingEnabled = true + } + relayIdlingResource.increment() + val transaction = Sentry.startTransaction("e2etests", "test1") + + transaction.finish() + relay.assert { + assertEnvelope { + val transactionItem = it.assertItem(SentryTransaction::class.java) + val profilingTraceData = it.assertItem(ProfilingTraceData::class.java) + it.assertNoOtherItems() + assertTrue(transactionItem.transaction == "e2etests") + assertTrue(profilingTraceData.transactionId == transactionItem.eventId.toString()) + assertTrue(profilingTraceData.transactionName == "e2etests") + } + assertNoOtherEnvelopes() + assertNoOtherRequests() + } + } } diff --git a/sentry-android-ui-tests/end2end/src/androidTest/java/io/sentry/android/uitests/end2end/mockservers/MockRelay.kt b/sentry-android-ui-tests/end2end/src/androidTest/java/io/sentry/android/uitests/end2end/mockservers/MockRelay.kt index 1180e674c89..669d406d437 100644 --- a/sentry-android-ui-tests/end2end/src/androidTest/java/io/sentry/android/uitests/end2end/mockservers/MockRelay.kt +++ b/sentry-android-ui-tests/end2end/src/androidTest/java/io/sentry/android/uitests/end2end/mockservers/MockRelay.kt @@ -1,5 +1,6 @@ package io.sentry.android.uitests.end2end.mockservers +import android.util.Log import androidx.test.espresso.idling.CountingIdlingResource import io.sentry.android.uitests.end2end.BaseUiTest import io.sentry.android.uitests.end2end.utils.BooleanIdlingResource @@ -37,7 +38,7 @@ class MockRelay ( val response = responses.asSequence() .mapNotNull { it(request) } .firstOrNull() - ?: MockResponse().setResponseCode(404) + ?: MockResponse() dispatchSentRequest(RelayAsserter.RelayResponse(request, response)) if (checkIdlingResources) { From a580cc32ed9b493aafbb317a7037b1aec623b975 Mon Sep 17 00:00:00 2001 From: stefanosiano Date: Thu, 28 Apr 2022 10:38:45 +0200 Subject: [PATCH 04/22] started with javadoc and cleanup of benchmark test --- .../uitests/benchmark/BenchmarkActivity.kt | 44 +++--------------- .../BenchmarkTransactionListAdapter.kt | 46 +++++++++++++++++++ 2 files changed, 52 insertions(+), 38 deletions(-) create mode 100644 sentry-android-ui-tests/benchmark/src/main/java/io/sentry/android/uitests/benchmark/BenchmarkTransactionListAdapter.kt diff --git a/sentry-android-ui-tests/benchmark/src/main/java/io/sentry/android/uitests/benchmark/BenchmarkActivity.kt b/sentry-android-ui-tests/benchmark/src/main/java/io/sentry/android/uitests/benchmark/BenchmarkActivity.kt index a85e2d4e8c3..5279097a97a 100644 --- a/sentry-android-ui-tests/benchmark/src/main/java/io/sentry/android/uitests/benchmark/BenchmarkActivity.kt +++ b/sentry-android-ui-tests/benchmark/src/main/java/io/sentry/android/uitests/benchmark/BenchmarkActivity.kt @@ -18,7 +18,7 @@ import java.util.concurrent.ExecutorService import java.util.concurrent.Executors import kotlin.random.Random -/** A simple list of items, using [RecyclerView]. */ +/** A simple activity with a list of bitmaps. */ class BenchmarkActivity : AppCompatActivity() { companion object { @@ -31,34 +31,10 @@ class BenchmarkActivity : AppCompatActivity() { } } - internal class ViewHolder(view: View) : RecyclerView.ViewHolder(view) { - val imageView: ImageView = view.findViewById(R.id.benchmark_item_list_image) - val textView: TextView = view.findViewById(R.id.benchmark_item_list_text) - } - - internal inner class Adapter : RecyclerView.Adapter() { - override fun onCreateViewHolder(parent: ViewGroup, viewType: Int): ViewHolder { - val view = LayoutInflater.from(parent.context).inflate(R.layout.benchmark_item_list, parent, false) - return ViewHolder(view) - } - - @Suppress("MagicNumber") - override fun onBindViewHolder(holder: ViewHolder, position: Int) { - holder.imageView.setImageBitmap(generateBitmap()) - - @SuppressLint("SetTextI18n") - holder.textView.text = "Item $position ${"sentry ".repeat(position)}" - } - - // Disables view recycling. - override fun getItemViewType(position: Int): Int = position - - override fun getItemCount(): Int = 200 - } - /** - * Each background thread will run non-stop calculations during the benchmark. One such thread seems - * enough to represent a busy application. This number can be increased to mimic busier applications. + * Each background thread will run non-stop calculations during the benchmark. + * One such thread seems enough to represent a busy application. + * This number can be increased to mimic busier applications. */ private val backgroundThreadPoolSize = 1 private val executor: ExecutorService = Executors.newFixedThreadPool(backgroundThreadPoolSize) @@ -71,7 +47,7 @@ class BenchmarkActivity : AppCompatActivity() { findViewById(R.id.benchmark_transaction_list).apply { layoutManager = LinearLayoutManager(this@BenchmarkActivity) - adapter = Adapter() + adapter = BenchmarkTransactionListAdapter() addOnScrollListener(object : RecyclerView.OnScrollListener() { override fun onScrollStateChanged(recyclerView: RecyclerView, newState: Int) { super.onScrollStateChanged(recyclerView, newState) @@ -81,20 +57,12 @@ class BenchmarkActivity : AppCompatActivity() { } } - @Suppress("MagicNumber") - internal fun generateBitmap(): Bitmap { - val bitmapSize = 100 - val colors = (0 until (bitmapSize * bitmapSize)).map { - Color.rgb(Random.nextInt(256), Random.nextInt(256), Random.nextInt(256)) - }.toIntArray() - return Bitmap.createBitmap(colors, bitmapSize, bitmapSize, Bitmap.Config.ARGB_8888) - } - @Suppress("MagicNumber") override fun onResume() { super.onResume() resumed = true + // Do operations until the activity is paused. repeat(backgroundThreadPoolSize) { executor.execute { var x = 0 diff --git a/sentry-android-ui-tests/benchmark/src/main/java/io/sentry/android/uitests/benchmark/BenchmarkTransactionListAdapter.kt b/sentry-android-ui-tests/benchmark/src/main/java/io/sentry/android/uitests/benchmark/BenchmarkTransactionListAdapter.kt new file mode 100644 index 00000000000..72754b130d9 --- /dev/null +++ b/sentry-android-ui-tests/benchmark/src/main/java/io/sentry/android/uitests/benchmark/BenchmarkTransactionListAdapter.kt @@ -0,0 +1,46 @@ +package io.sentry.android.uitests.benchmark + +import android.annotation.SuppressLint +import android.graphics.Bitmap +import android.graphics.Color +import android.view.LayoutInflater +import android.view.View +import android.view.ViewGroup +import android.widget.ImageView +import android.widget.TextView +import androidx.recyclerview.widget.RecyclerView +import kotlin.random.Random + +/** Simple [RecyclerView.Adapter] that generates a bitmap and a text to show for each item. */ +internal class BenchmarkTransactionListAdapter : RecyclerView.Adapter() { + + override fun onCreateViewHolder(parent: ViewGroup, viewType: Int): ViewHolder { + val view = LayoutInflater.from(parent.context).inflate(R.layout.benchmark_item_list, parent, false) + return ViewHolder(view) + } + + override fun onBindViewHolder(holder: ViewHolder, position: Int) { + holder.imageView.setImageBitmap(generateBitmap()) + + @SuppressLint("SetTextI18n") + holder.textView.text = "Item $position ${"sentry ".repeat(position)}" + } + + private fun generateBitmap(): Bitmap { + val bitmapSize = 100 + val colors = (0 until (bitmapSize * bitmapSize)).map { + Color.rgb(Random.nextInt(256), Random.nextInt(256), Random.nextInt(256)) + }.toIntArray() + return Bitmap.createBitmap(colors, bitmapSize, bitmapSize, Bitmap.Config.ARGB_8888) + } + + // Disables view recycling. + override fun getItemViewType(position: Int): Int = position + + override fun getItemCount(): Int = 200 +} + +internal class ViewHolder(view: View) : RecyclerView.ViewHolder(view) { + val imageView: ImageView = view.findViewById(R.id.benchmark_item_list_image) + val textView: TextView = view.findViewById(R.id.benchmark_item_list_text) +} From 985039610d0f473830eb5f5650449de5842f35e6 Mon Sep 17 00:00:00 2001 From: stefanosiano Date: Thu, 28 Apr 2022 19:57:30 +0200 Subject: [PATCH 05/22] cleaned everything added comments and javadoc to Benchmark module added a README --- .../benchmark/build.gradle.kts | 20 +++-- .../src/androidTest/AndroidManifest.xml | 20 ----- .../uitests/benchmark/SentryBenchmarkTest.kt | 85 +++++++++++-------- .../benchmark/util/BenchmarkOperation.kt | 63 ++++++++++++-- .../util/BenchmarkOperationResult.kt | 15 +++- .../benchmark/src/main/AndroidManifest.xml | 17 ++-- .../uitests/benchmark/BenchmarkActivity.kt | 17 +--- sentry-android-ui-tests/end2end/README.md | 21 +++++ .../end2end/build.gradle.kts | 10 ++- .../android/uitests/end2end/BaseUiTest.kt | 21 +++-- ...leInstrumentedTest.kt => EnvelopeTests.kt} | 14 +-- 11 files changed, 187 insertions(+), 116 deletions(-) delete mode 100644 sentry-android-ui-tests/benchmark/src/androidTest/AndroidManifest.xml create mode 100644 sentry-android-ui-tests/end2end/README.md rename sentry-android-ui-tests/end2end/src/androidTest/java/io/sentry/android/uitests/end2end/{ExampleInstrumentedTest.kt => EnvelopeTests.kt} (83%) diff --git a/sentry-android-ui-tests/benchmark/build.gradle.kts b/sentry-android-ui-tests/benchmark/build.gradle.kts index 87bb8fe85ec..c3883623307 100644 --- a/sentry-android-ui-tests/benchmark/build.gradle.kts +++ b/sentry-android-ui-tests/benchmark/build.gradle.kts @@ -14,14 +14,14 @@ android { // Runs each test in its own instance of Instrumentation. This way they are isolated from // one another and get their own Application instance. // https://developer.android.com/training/testing/instrumented-tests/androidx-test-libraries/runner#enable-gradle - testInstrumentationRunnerArguments["clearPackageData"] = "true" + // This doesn't work on some devices with Android 11+. Clearing package data resets permissions. + // Check the readme for more info. +// testInstrumentationRunnerArguments["clearPackageData"] = "true" } -// testOptions { -// execution = "ANDROIDX_TEST_ORCHESTRATOR" it doesn't work with Android 11... -// } - - testBuildType = "release" + testOptions { + execution = "ANDROIDX_TEST_ORCHESTRATOR" + } signingConfigs { getByName("debug") { @@ -38,9 +38,15 @@ android { // it must be done in a manifest - see src/androidTest/AndroidManifest.xml isMinifyEnabled = true proguardFiles(getDefaultProguardFile("proguard-android-optimize.txt"), "benchmark-proguard-rules.pro") + + addManifestPlaceholders( + mapOf( + "sentryDebug" to false, + "sentryEnvironment" to "release" + ) + ) } named("release") { - isDefault = true isMinifyEnabled = true isShrinkResources = false signingConfig = signingConfigs.getByName("debug") // to be able to run release mode diff --git a/sentry-android-ui-tests/benchmark/src/androidTest/AndroidManifest.xml b/sentry-android-ui-tests/benchmark/src/androidTest/AndroidManifest.xml deleted file mode 100644 index 73bef08f171..00000000000 --- a/sentry-android-ui-tests/benchmark/src/androidTest/AndroidManifest.xml +++ /dev/null @@ -1,20 +0,0 @@ - - - - - - - - - diff --git a/sentry-android-ui-tests/benchmark/src/androidTest/java/io/sentry/android/uitests/benchmark/SentryBenchmarkTest.kt b/sentry-android-ui-tests/benchmark/src/androidTest/java/io/sentry/android/uitests/benchmark/SentryBenchmarkTest.kt index bf9a8231ef1..ee773f4a0d4 100644 --- a/sentry-android-ui-tests/benchmark/src/androidTest/java/io/sentry/android/uitests/benchmark/SentryBenchmarkTest.kt +++ b/sentry-android-ui-tests/benchmark/src/androidTest/java/io/sentry/android/uitests/benchmark/SentryBenchmarkTest.kt @@ -6,34 +6,26 @@ import androidx.test.core.app.ApplicationProvider import androidx.test.core.app.launchActivity import androidx.test.espresso.Espresso import androidx.test.espresso.Espresso.onView +import androidx.test.espresso.IdlingRegistry import androidx.test.espresso.action.ViewActions.swipeUp import androidx.test.espresso.matcher.ViewMatchers.withId import androidx.test.ext.junit.runners.AndroidJUnit4 import androidx.test.platform.app.InstrumentationRegistry +import androidx.test.rule.GrantPermissionRule import androidx.test.runner.AndroidJUnitRunner import io.sentry.ITransaction import io.sentry.Sentry import io.sentry.SentryOptions import io.sentry.android.core.SentryAndroid import io.sentry.android.uitests.benchmark.util.BenchmarkOperation +import org.junit.Rule import org.junit.Test import org.junit.runner.RunWith import java.io.File +import kotlin.test.AfterTest import kotlin.test.BeforeTest import kotlin.test.assertTrue -/** - * ## On Comparing Operations - * - * Originally [androidx.benchmark.junit4.BenchmarkRule] was used along with two tests: one to - * benchmark an operation without Specto tracing, and another to measure the same operation with - * Specto tracing. However when running this setup to compare the exact same operation showed that - * the one that ran first always had significantly more frames dropped, so comparing two different - * operations on equal terms was not possible. - * - * The current approach interweaves the two operations, running them in an alternating sequence. - * When the two operations are the same, we get (nearly) identical results, as expected. - */ @RunWith(AndroidJUnit4::class) class SentryBenchmarkTest { @@ -41,18 +33,48 @@ class SentryBenchmarkTest { private lateinit var context: Context @BeforeTest - fun baseSetUp() { + fun setUp() { runner = InstrumentationRegistry.getInstrumentation() as AndroidJUnitRunner context = ApplicationProvider.getApplicationContext() + context.cacheDir.deleteRecursively() + IdlingRegistry.getInstance().register(BenchmarkActivity.scrollingIdlingResource) + } + + @AfterTest + fun cleanup() { + IdlingRegistry.getInstance().unregister(BenchmarkActivity.scrollingIdlingResource) + } + + @Test + fun benchmarkSameOperation() { + + // We compare two operation that are the same. We expect the increases to be negligible, as the results + // should be very similar. + val op1 = BenchmarkOperation(runner, getOperation(runner) { null }) + val op2 = BenchmarkOperation(runner, getOperation(runner) { null }) + val comparisonResult = BenchmarkOperation.compare(op1, "Op1", op2, "Op2") + + assertTrue { comparisonResult.durationIncrease >= -1 } + assertTrue { comparisonResult.durationIncrease < 1 } + assertTrue { comparisonResult.cpuTimeIncrease >= -1 } + assertTrue { comparisonResult.cpuTimeIncrease < 1 } + // The fps decrease is skipped for the moment, due to approximation: + // if an operation runs at 59.49 fps and the other at 59.51, they are considered 59 and 60 fps respectively. + // Their difference would be 1 / 60 * 100 = 1.66666% + // On slow devices, a difference of 1 fps on an average of 20 fps would account for 5% decrease. + // On even slower devices the difference would be even higher. Let's skip for now, as it's not important anyway. +// assertTrue { comparisonResult.fpsDecrease >= -2 } +// assertTrue { comparisonResult.fpsDecrease < 2 } + assertTrue { comparisonResult.droppedFramesIncrease >= -1 } + assertTrue { comparisonResult.droppedFramesIncrease < 1 } } @Test fun benchmarkProfiledTransaction() { - // runOnMainSync ensure that we're using Sentry on a thread that belongs to the actual - // application, rather than the test application. + runner.runOnMainSync { SentryAndroid.init(context) { options: SentryOptions -> - options.dsn = "https://fakesecret@fakeuri.ingest.sentry.io/1234567" + options.dsn = "https://key@uri/1234567" options.isEnableAutoSessionTracking = false options.tracesSampleRate = 1.0 options.isTraceSampling = true @@ -60,29 +82,19 @@ class SentryBenchmarkTest { } } + // We compare the same operation with and without profiled transaction. + // We expect the profiled transaction operation to be slower, but not slower than 5%. val benchmarkOperationNoTransaction = BenchmarkOperation(runner, getOperation(runner) { null }) val benchmarkOperationProfiled = BenchmarkOperation(runner, getOperation(runner) { Sentry.startTransaction("Benchmark", "ProfiledTransaction") }) - repeat(2) { - benchmarkOperationNoTransaction.warmup() - benchmarkOperationProfiled.warmup() - } - repeat(10) { - benchmarkOperationNoTransaction.iterate() - benchmarkOperationProfiled.iterate() - } - - val noTransactionResult = benchmarkOperationNoTransaction.getResult("NoTransaction") - val profiledResult = benchmarkOperationProfiled.getResult("ProfiledTransaction") - - println("=====================================") - println(noTransactionResult) - println(profiledResult) - println("=====================================") + val comparisonResult = BenchmarkOperation.compare( + benchmarkOperationNoTransaction, + "NoTransaction", + benchmarkOperationProfiled, + "ProfiledTransaction" + ) - // We expect the profiled operation to be slower than the no transaction one, but not slower than 5% - val comparisonResult = profiledResult.compare(noTransactionResult) assertTrue { comparisonResult.durationIncrease >= 0 } assertTrue { comparisonResult.durationIncrease < 5.0 } assertTrue { comparisonResult.cpuTimeIncrease >= 0 } @@ -93,7 +105,10 @@ class SentryBenchmarkTest { assertTrue { comparisonResult.droppedFramesIncrease < 5.0 } } - // Operation that will be compared + /** + * Operation that will be compared: it launches [BenchmarkActivity], swipe the list and closes it. + * The [transactionBuilder] is used to create the transaction before the swipes. + */ private fun getOperation(runner: AndroidJUnitRunner, transactionBuilder: () -> ITransaction?): () -> Unit = { var transaction: ITransaction? = null // Launch the benchmark activity diff --git a/sentry-android-ui-tests/benchmark/src/androidTest/java/io/sentry/android/uitests/benchmark/util/BenchmarkOperation.kt b/sentry-android-ui-tests/benchmark/src/androidTest/java/io/sentry/android/uitests/benchmark/util/BenchmarkOperation.kt index 5b1f92afa22..46f71ad1662 100644 --- a/sentry-android-ui-tests/benchmark/src/androidTest/java/io/sentry/android/uitests/benchmark/util/BenchmarkOperation.kt +++ b/sentry-android-ui-tests/benchmark/src/androidTest/java/io/sentry/android/uitests/benchmark/util/BenchmarkOperation.kt @@ -8,8 +8,58 @@ import androidx.test.runner.AndroidJUnitRunner // 60 FPS is the recommended target: https://www.youtube.com/watch?v=CaMTIgxCSqU private const val FRAME_DURATION_60FPS_NS: Double = 1_000_000_000 / 60.0 +/** + * Class that allows to benchmark some operations. + * Create two [BenchmarkOperation] objects and compare them using [BenchmarkOperation.compare] to get + * a [BenchmarkResult] with relative measured overheads. + */ + internal class BenchmarkOperation(runner: AndroidJUnitRunner, private val op: () -> Unit) { + companion object { + + /** + * Running two operations sequentially (running 10 times the first and then 10 times the second) results in the + * first operation to always be slower, so comparing two different operations on equal terms is not possible. + * This method runs [op1] and [op2] in an alternating sequence. + * When [op1] and [op2] are the same, we get (nearly) identical results, as expected. + * You can adjust [warmupIterations] and [measuredIterations]. The lower they are, the faster the benchmark, + * but accuracy decreases. + */ + fun compare( + op1: BenchmarkOperation, + op1Name: String, + op2: BenchmarkOperation, + op2Name: String, + warmupIterations: Int = 3, + measuredIterations: Int = 15 + ): BenchmarkResult { + // The first operations are the slowest, as the device is still doing things like filling the cache. + repeat(warmupIterations) { + op1.warmup() + op2.warmup() + } + // Now we can measure the operations (in alternating sequence). + repeat(measuredIterations) { + op1.iterate() + op2.iterate() + } + val op1Result = op1.getResult(op1Name) + val op2Result = op2.getResult(op2Name) + + // Let's print the raw results. + println("=====================================") + println(op1Name) + println(op1Result) + println("=====================================") + println(op2Name) + println(op2Result) + println("=====================================") + + return op2Result.compare(op1Result) + } + } + private lateinit var choreographer: Choreographer private var iterations: Int = 0 private var durationNanos: Long = 0 @@ -25,12 +75,14 @@ internal class BenchmarkOperation(runner: AndroidJUnitRunner, private val op: () } } - fun warmup() { + /** Run the operation without measuring it. */ + private fun warmup() { op() isolate() } - fun iterate() { + /** Run the operation and measure it, updating benchmark data. */ + private fun iterate() { val startRealtimeNs = SystemClock.elapsedRealtimeNanos() val startCpuTimeMs = Process.getElapsedCpuTime() @@ -47,7 +99,8 @@ internal class BenchmarkOperation(runner: AndroidJUnitRunner, private val op: () isolate() } - fun getResult(operationName: String): BenchmarkOperationResult = BenchmarkOperationResult( + /** Return the [BenchmarkOperationResult] for the operation. */ + private fun getResult(operationName: String): BenchmarkOperationResult = BenchmarkOperationResult( cpuDurationMillis / iterations, droppedFrames / iterations, durationNanos / iterations, @@ -56,8 +109,8 @@ internal class BenchmarkOperation(runner: AndroidJUnitRunner, private val op: () ) /** - * Helps ensure that operations don't impact one another. Doesn't appear to currently have an - * impact on the benchmark, but it might later on. + * Helps ensure that operations don't impact one another. + * Doesn't appear to currently have an impact on the benchmark. */ private fun isolate() { Thread.sleep(500) diff --git a/sentry-android-ui-tests/benchmark/src/androidTest/java/io/sentry/android/uitests/benchmark/util/BenchmarkOperationResult.kt b/sentry-android-ui-tests/benchmark/src/androidTest/java/io/sentry/android/uitests/benchmark/util/BenchmarkOperationResult.kt index db8965724e1..438714e3772 100644 --- a/sentry-android-ui-tests/benchmark/src/androidTest/java/io/sentry/android/uitests/benchmark/util/BenchmarkOperationResult.kt +++ b/sentry-android-ui-tests/benchmark/src/androidTest/java/io/sentry/android/uitests/benchmark/util/BenchmarkOperationResult.kt @@ -2,6 +2,7 @@ package io.sentry.android.uitests.benchmark.util import java.util.concurrent.TimeUnit +/** Stores the results of a [BenchmarkOperation]. */ internal data class BenchmarkOperationResult( val avgCpuTimeMillis: Long, val avgDroppedFrames: Double, @@ -9,7 +10,12 @@ internal data class BenchmarkOperationResult( val avgFramesPerSecond: Int, val operationName: String ) { - fun compare(other: BenchmarkOperationResult): BenchmarkOperationComparison { + /** + * Compare two [BenchmarkOperationResult], calculating increases of each parameter in percentage. + */ + fun compare(other: BenchmarkOperationResult): BenchmarkResult { + + // Measure average duration val durationDiffNanos = avgDurationNanos - other.avgDurationNanos val durationIncreasePercentage = durationDiffNanos * 100.0 / other.avgDurationNanos println("Measuring the duration increase of the operation in the transaction. If it's low enough, " + @@ -24,6 +30,7 @@ internal data class BenchmarkOperationResult( println("--------------------") + // Measure average cpu time val cpuTimeDiff = (avgCpuTimeMillis - other.avgCpuTimeMillis) / Runtime.getRuntime().availableProcessors() val cpuTimeOverheadPercentage = cpuTimeDiff * 100.0 / other.avgCpuTimeMillis // Cpu time spent profiling is weighted based on available threads, as profiling runs on 1 thread only. @@ -40,6 +47,7 @@ internal data class BenchmarkOperationResult( println("--------------------") + // Measure average fps val fpsDiff = other.avgFramesPerSecond - avgFramesPerSecond val fpsDecreasePercentage = fpsDiff * 100.0 / other.avgFramesPerSecond println("Measuring the decreased fps. Not really important, as even if fps are the same, the cpu could be " + @@ -54,6 +62,7 @@ internal data class BenchmarkOperationResult( println("--------------------") + // Measure average dropped frames val droppedFramesDiff = avgDroppedFrames - other.avgDroppedFrames val totalExpectedFrames = TimeUnit.NANOSECONDS.toMillis(other.avgDurationNanos) * 60 / 1000 val droppedFramesIncreasePercentage = droppedFramesDiff * 100 / (totalExpectedFrames-other.avgDroppedFrames) @@ -68,7 +77,7 @@ internal data class BenchmarkOperationResult( println("No measurable frame drop increase detected.") } - return BenchmarkOperationComparison( + return BenchmarkResult( cpuTimeOverheadPercentage, droppedFramesIncreasePercentage, durationIncreasePercentage, @@ -77,7 +86,7 @@ internal data class BenchmarkOperationResult( } } -internal data class BenchmarkOperationComparison( +internal data class BenchmarkResult( val cpuTimeIncrease: Double, val droppedFramesIncrease: Double, val durationIncrease: Double, diff --git a/sentry-android-ui-tests/benchmark/src/main/AndroidManifest.xml b/sentry-android-ui-tests/benchmark/src/main/AndroidManifest.xml index 7f53b1fdf06..58c0f6a0830 100644 --- a/sentry-android-ui-tests/benchmark/src/main/AndroidManifest.xml +++ b/sentry-android-ui-tests/benchmark/src/main/AndroidManifest.xml @@ -1,12 +1,19 @@ + xmlns:tools="http://schemas.android.com/tools"> + + android:theme="@style/Theme.AppCompat" + android:label="Sentry Benchmark" + android:debuggable="false" + tools:ignore="HardcodedDebugMode" + tools:replace="android:label,android:debuggable"> diff --git a/sentry-android-ui-tests/benchmark/src/main/java/io/sentry/android/uitests/benchmark/BenchmarkActivity.kt b/sentry-android-ui-tests/benchmark/src/main/java/io/sentry/android/uitests/benchmark/BenchmarkActivity.kt index 5279097a97a..d46e25b797c 100644 --- a/sentry-android-ui-tests/benchmark/src/main/java/io/sentry/android/uitests/benchmark/BenchmarkActivity.kt +++ b/sentry-android-ui-tests/benchmark/src/main/java/io/sentry/android/uitests/benchmark/BenchmarkActivity.kt @@ -1,14 +1,6 @@ package io.sentry.android.uitests.benchmark -import android.annotation.SuppressLint -import android.graphics.Bitmap -import android.graphics.Color import android.os.Bundle -import android.view.LayoutInflater -import android.view.View -import android.view.ViewGroup -import android.widget.ImageView -import android.widget.TextView import androidx.appcompat.app.AppCompatActivity import androidx.recyclerview.widget.LinearLayoutManager import androidx.recyclerview.widget.RecyclerView @@ -16,19 +8,14 @@ import androidx.test.espresso.IdlingRegistry import io.sentry.android.uitests.end2end.utils.BooleanIdlingResource import java.util.concurrent.ExecutorService import java.util.concurrent.Executors -import kotlin.random.Random /** A simple activity with a list of bitmaps. */ class BenchmarkActivity : AppCompatActivity() { companion object { - /** The activity will set this manage when scrolling. */ + /** The activity will set this when scrolling. */ val scrollingIdlingResource = BooleanIdlingResource("benchmark-activity") - - init { - IdlingRegistry.getInstance().register(scrollingIdlingResource) - } } /** @@ -45,6 +32,7 @@ class BenchmarkActivity : AppCompatActivity() { setContentView(R.layout.activity_benchmark) + // We show a simple list that changes the idling resource findViewById(R.id.benchmark_transaction_list).apply { layoutManager = LinearLayoutManager(this@BenchmarkActivity) adapter = BenchmarkTransactionListAdapter() @@ -57,7 +45,6 @@ class BenchmarkActivity : AppCompatActivity() { } } - @Suppress("MagicNumber") override fun onResume() { super.onResume() resumed = true diff --git a/sentry-android-ui-tests/end2end/README.md b/sentry-android-ui-tests/end2end/README.md new file mode 100644 index 00000000000..033503c5c37 --- /dev/null +++ b/sentry-android-ui-tests/end2end/README.md @@ -0,0 +1,21 @@ +Ui tests for Android +=========== +Here will be put all ui tests for Android, running through Google's Espresso. +By default the envelopes sent to relay are caught by a mock server which allows us to check the envelopes sent. + +# How to use + +Simply run `./gradlew connectedCheck` to run all ui tests of all modules (requires a connected device, either physical or an emulator). + +# Troubleshooting + +There is an issue with Android 11+ (Xiaomi only?). +In order to start an activity from the background (which the test orchestrator does internally), the app needs a special permission, which cannot be granted without user interaction. +To allow it on the device go in Settings -> apps -> manage apps -> Select the app, like `Sentry End2End Tests` -> Other permissions -> `Display pop-up windows while running in the background`. +The path may be different on other devices. +On older versions of Android there is no problem. + +For this reason we cannot use the `testInstrumentationRunnerArguments["clearPackageData"] = "true"` in the build.gradle file, as clearing package data resets permissions. +This flag is used to run each test in its own instance of Instrumentation. This way they are isolated from one another and get their own Application instance. +More on https://developer.android.com/training/testing/instrumented-tests/androidx-test-libraries/runner#enable-gradle + diff --git a/sentry-android-ui-tests/end2end/build.gradle.kts b/sentry-android-ui-tests/end2end/build.gradle.kts index c22c066fb6a..201cbe6317a 100644 --- a/sentry-android-ui-tests/end2end/build.gradle.kts +++ b/sentry-android-ui-tests/end2end/build.gradle.kts @@ -14,13 +14,15 @@ android { // Runs each test in its own instance of Instrumentation. This way they are isolated from // one another and get their own Application instance. // https://developer.android.com/training/testing/instrumented-tests/androidx-test-libraries/runner#enable-gradle - testInstrumentationRunnerArguments["clearPackageData"] = "true" + // This doesn't work on some devices with Android 11+. Clearing package data resets permissions. + // Check the readme for more info. +// testInstrumentationRunnerArguments["clearPackageData"] = "true" consumerProguardFiles("consumer-rules.pro") } -// testOptions { -// execution = "ANDROIDX_TEST_ORCHESTRATOR" it doesn't work with Android 11... -// } + testOptions { + execution = "ANDROIDX_TEST_ORCHESTRATOR" + } testBuildType = "debug" diff --git a/sentry-android-ui-tests/end2end/src/androidTest/java/io/sentry/android/uitests/end2end/BaseUiTest.kt b/sentry-android-ui-tests/end2end/src/androidTest/java/io/sentry/android/uitests/end2end/BaseUiTest.kt index a234a409d2f..b216fe0be38 100644 --- a/sentry-android-ui-tests/end2end/src/androidTest/java/io/sentry/android/uitests/end2end/BaseUiTest.kt +++ b/sentry-android-ui-tests/end2end/src/androidTest/java/io/sentry/android/uitests/end2end/BaseUiTest.kt @@ -1,8 +1,6 @@ package io.sentry.android.uitests.end2end -import android.app.Application import android.content.Context -import android.content.pm.ApplicationInfo import androidx.test.core.app.ApplicationProvider import androidx.test.espresso.Espresso import androidx.test.espresso.IdlingRegistry @@ -25,24 +23,23 @@ abstract class BaseUiTest { protected lateinit var runner: AndroidJUnitRunner protected lateinit var context: Context protected lateinit var dsn: String - protected val relay = MockRelay(false, relayIdlingResource) - - init { - } + protected var relayCheckIdlingResources: Boolean = false + protected val relay = MockRelay(relayCheckIdlingResources, relayIdlingResource) @BeforeTest fun baseSetUp() { runner = InstrumentationRegistry.getInstrumentation() as AndroidJUnitRunner context = ApplicationProvider.getApplicationContext() context.cacheDir.deleteRecursively() - IdlingRegistry.getInstance().register(relayIdlingResource) relay.start() dsn = "http://key@${relay.hostName}:${relay.port}/${relay.dsnProject}" } @AfterTest fun baseFinish() { - IdlingRegistry.getInstance().unregister(relayIdlingResource) + if (relayCheckIdlingResources) { + IdlingRegistry.getInstance().unregister(relayIdlingResource) + } relay.shutdown() Sentry.close() } @@ -51,7 +48,11 @@ abstract class BaseUiTest { relayCheckIdlingResources: Boolean = true, optionsConfiguration: ((options: SentryOptions) -> Unit)? = null ) { + this.relayCheckIdlingResources = relayCheckIdlingResources relay.checkIdlingResources = relayCheckIdlingResources + if (relayCheckIdlingResources) { + IdlingRegistry.getInstance().register(relayIdlingResource) + } SentryAndroid.init(context) { it.dsn = dsn optionsConfiguration?.invoke(it) @@ -59,9 +60,7 @@ abstract class BaseUiTest { } } -/** - * Waits until the Specto SDK is idle. - */ +/** Waits until the Sentry SDK is idle. */ fun waitUntilIdle() { // We rely on Espresso's idling resources. Espresso.onIdle() diff --git a/sentry-android-ui-tests/end2end/src/androidTest/java/io/sentry/android/uitests/end2end/ExampleInstrumentedTest.kt b/sentry-android-ui-tests/end2end/src/androidTest/java/io/sentry/android/uitests/end2end/EnvelopeTests.kt similarity index 83% rename from sentry-android-ui-tests/end2end/src/androidTest/java/io/sentry/android/uitests/end2end/ExampleInstrumentedTest.kt rename to sentry-android-ui-tests/end2end/src/androidTest/java/io/sentry/android/uitests/end2end/EnvelopeTests.kt index 7978d90fa28..d56c76d8378 100644 --- a/sentry-android-ui-tests/end2end/src/androidTest/java/io/sentry/android/uitests/end2end/ExampleInstrumentedTest.kt +++ b/sentry-android-ui-tests/end2end/src/androidTest/java/io/sentry/android/uitests/end2end/EnvelopeTests.kt @@ -1,30 +1,22 @@ package io.sentry.android.uitests.end2end -import androidx.lifecycle.Lifecycle -import androidx.test.core.app.launchActivity import androidx.test.ext.junit.runners.AndroidJUnit4 import io.sentry.ProfilingTraceData import io.sentry.Sentry import io.sentry.SentryEvent import io.sentry.SentryOptions import io.sentry.protocol.SentryTransaction -import okhttp3.mockwebserver.MockResponse import org.junit.Test import org.junit.runner.RunWith import org.junit.Assert.* -/** - * Instrumented test, which will execute on an Android device. - * - * See [testing documentation](http://d.android.com/tools/testing). - */ @RunWith(AndroidJUnit4::class) -class ExampleInstrumentedTest : BaseUiTest() { +class EnvelopeTests : BaseUiTest() { @Test - fun useAppContext() { + fun checkEnvelopeCaptureMessage() { initSentry(true) relayIdlingResource.increment() @@ -42,7 +34,7 @@ class ExampleInstrumentedTest : BaseUiTest() { } @Test - fun useAppContex2t() { + fun checkEnvelopeProfiledTransaction() { initSentry(true) { options: SentryOptions -> options.isEnableAutoSessionTracking = false From fc4ce1d0e2839dddf0073db0174185374a292a9b Mon Sep 17 00:00:00 2001 From: stefanosiano Date: Thu, 28 Apr 2022 20:46:49 +0200 Subject: [PATCH 06/22] fixed spotless issues --- CHANGELOG.md | 2 ++ build.gradle.kts | 4 ++- .../uitests/benchmark/SentryBenchmarkTest.kt | 12 +++---- .../util/BenchmarkOperationResult.kt | 34 ++++++++++++------- .../benchmark/src/main/AndroidManifest.xml | 3 +- .../uitests/benchmark/BenchmarkActivity.kt | 1 - .../android/uitests/end2end/EnvelopeTests.kt | 7 ++-- .../uitests/end2end/mockservers/MockRelay.kt | 5 +-- .../end2end/mockservers/RelayAsserter.kt | 2 +- .../android/uitests/end2end/EmptyActivity.kt | 1 - .../end2end/utils/BooleanIdlingResource.kt | 1 - 11 files changed, 39 insertions(+), 33 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 60db399290d..5cef4498756 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## Unreleased +* Tests: added Android UI tests (#2013) + * Fix: Profiling rate decreased from 300hz to 100hz (#1997) * Fix: Android profiling initializes on first profile start (#2009) * Fix: Allow disabling sending of client reports via Android Manifest and external options (#2007) diff --git a/build.gradle.kts b/build.gradle.kts index 2d210a4bc94..10a8a43c7d8 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -45,6 +45,8 @@ apiValidation { ) ignoredProjects.addAll( listOf( + "benchmark", + "end2end", "sentry-samples-android", "sentry-samples-console", "sentry-samples-jul", @@ -165,7 +167,7 @@ gradle.projectsEvaluated { "https://docs.spring.io/spring-boot/docs/current/api/" ) subprojects - .filter { !it.name.contains("sample") } + .filter { !it.name.contains("sample") && it.name != "benchmark" && it.name != "end2end" } .forEach { proj -> proj.tasks.withType().forEach { javadocTask -> source += javadocTask.source diff --git a/sentry-android-ui-tests/benchmark/src/androidTest/java/io/sentry/android/uitests/benchmark/SentryBenchmarkTest.kt b/sentry-android-ui-tests/benchmark/src/androidTest/java/io/sentry/android/uitests/benchmark/SentryBenchmarkTest.kt index ee773f4a0d4..7f344a22611 100644 --- a/sentry-android-ui-tests/benchmark/src/androidTest/java/io/sentry/android/uitests/benchmark/SentryBenchmarkTest.kt +++ b/sentry-android-ui-tests/benchmark/src/androidTest/java/io/sentry/android/uitests/benchmark/SentryBenchmarkTest.kt @@ -11,17 +11,14 @@ import androidx.test.espresso.action.ViewActions.swipeUp import androidx.test.espresso.matcher.ViewMatchers.withId import androidx.test.ext.junit.runners.AndroidJUnit4 import androidx.test.platform.app.InstrumentationRegistry -import androidx.test.rule.GrantPermissionRule import androidx.test.runner.AndroidJUnitRunner import io.sentry.ITransaction import io.sentry.Sentry import io.sentry.SentryOptions import io.sentry.android.core.SentryAndroid import io.sentry.android.uitests.benchmark.util.BenchmarkOperation -import org.junit.Rule import org.junit.Test import org.junit.runner.RunWith -import java.io.File import kotlin.test.AfterTest import kotlin.test.BeforeTest import kotlin.test.assertTrue @@ -85,9 +82,12 @@ class SentryBenchmarkTest { // We compare the same operation with and without profiled transaction. // We expect the profiled transaction operation to be slower, but not slower than 5%. val benchmarkOperationNoTransaction = BenchmarkOperation(runner, getOperation(runner) { null }) - val benchmarkOperationProfiled = BenchmarkOperation(runner, getOperation(runner) { - Sentry.startTransaction("Benchmark", "ProfiledTransaction") - }) + val benchmarkOperationProfiled = BenchmarkOperation( + runner, + getOperation(runner) { + Sentry.startTransaction("Benchmark", "ProfiledTransaction") + } + ) val comparisonResult = BenchmarkOperation.compare( benchmarkOperationNoTransaction, "NoTransaction", diff --git a/sentry-android-ui-tests/benchmark/src/androidTest/java/io/sentry/android/uitests/benchmark/util/BenchmarkOperationResult.kt b/sentry-android-ui-tests/benchmark/src/androidTest/java/io/sentry/android/uitests/benchmark/util/BenchmarkOperationResult.kt index 438714e3772..3f96fe1cdd3 100644 --- a/sentry-android-ui-tests/benchmark/src/androidTest/java/io/sentry/android/uitests/benchmark/util/BenchmarkOperationResult.kt +++ b/sentry-android-ui-tests/benchmark/src/androidTest/java/io/sentry/android/uitests/benchmark/util/BenchmarkOperationResult.kt @@ -18,10 +18,12 @@ internal data class BenchmarkOperationResult( // Measure average duration val durationDiffNanos = avgDurationNanos - other.avgDurationNanos val durationIncreasePercentage = durationDiffNanos * 100.0 / other.avgDurationNanos - println("Measuring the duration increase of the operation in the transaction. If it's low enough, " + - "no end user will ever realize it") + println( + "Measuring the duration increase of the operation in the transaction. If it's low enough, " + + "no end user will ever realize it" + ) println("[${other.operationName}] Average duration: ${other.avgDurationNanos} ns") - println("[${operationName}] Average duration: $avgDurationNanos ns") + println("[$operationName] Average duration: $avgDurationNanos ns") if (durationIncreasePercentage > 0) { println("Duration increase: %.2f%%".format(durationIncreasePercentage)) } else { @@ -34,11 +36,13 @@ internal data class BenchmarkOperationResult( val cpuTimeDiff = (avgCpuTimeMillis - other.avgCpuTimeMillis) / Runtime.getRuntime().availableProcessors() val cpuTimeOverheadPercentage = cpuTimeDiff * 100.0 / other.avgCpuTimeMillis // Cpu time spent profiling is weighted based on available threads, as profiling runs on 1 thread only. - println("Measuring the increased cpu time. It has no direct impact on performance of the app, " + - "but it has on battery usage, as the cpu is 'awaken' longer.") + println( + "Measuring the increased cpu time. It has no direct impact on performance of the app, " + + "but it has on battery usage, as the cpu is 'awaken' longer." + ) println("The weighted difference of cpu times is $cpuTimeDiff ms.") println("[${other.operationName}] Cpu time: ${other.avgCpuTimeMillis} ms") - println("[${operationName}] Cpu time: $avgCpuTimeMillis ms") + println("[$operationName] Cpu time: $avgCpuTimeMillis ms") if (cpuTimeOverheadPercentage > 0) { println("CPU time overhead: %.2f%%".format(cpuTimeOverheadPercentage)) } else { @@ -50,10 +54,12 @@ internal data class BenchmarkOperationResult( // Measure average fps val fpsDiff = other.avgFramesPerSecond - avgFramesPerSecond val fpsDecreasePercentage = fpsDiff * 100.0 / other.avgFramesPerSecond - println("Measuring the decreased fps. Not really important, as even if fps are the same, the cpu could be " + - "doing more work in the frame window, and it could be hidden by checking average fps only.") + println( + "Measuring the decreased fps. Not really important, as even if fps are the same, the cpu could be " + + "doing more work in the frame window, and it could be hidden by checking average fps only." + ) println("[${other.operationName}] Average FPS: ${other.avgFramesPerSecond}") - println("[${operationName}] Average FPS: $avgFramesPerSecond") + println("[$operationName] Average FPS: $avgFramesPerSecond") if (fpsDecreasePercentage > 0) { println("FPS decrease: %.2f%%".format(fpsDecreasePercentage)) } else { @@ -65,12 +71,14 @@ internal data class BenchmarkOperationResult( // Measure average dropped frames val droppedFramesDiff = avgDroppedFrames - other.avgDroppedFrames val totalExpectedFrames = TimeUnit.NANOSECONDS.toMillis(other.avgDurationNanos) * 60 / 1000 - val droppedFramesIncreasePercentage = droppedFramesDiff * 100 / (totalExpectedFrames-other.avgDroppedFrames) - println("Measuring the increased fps drop rate. Very important, as it weights dropped frames based on the " + - "time passed between each frame. This is the metric end users can perceive as 'performance' in app usage.") + val droppedFramesIncreasePercentage = droppedFramesDiff * 100 / (totalExpectedFrames - other.avgDroppedFrames) + println( + "Measuring the increased fps drop rate. Very important, as it weights dropped frames based on the " + + "time passed between each frame. This is the metric end users can perceive as 'performance' in app usage." + ) println("Dropped frames are calculated based on a target of 60 frames per second ($totalExpectedFrames total frames).") println("[${other.operationName}] Average dropped frames: ${other.avgDroppedFrames}") - println("[${operationName}] Average dropped frames: $avgDroppedFrames") + println("[$operationName] Average dropped frames: $avgDroppedFrames") if (droppedFramesIncreasePercentage > 0) { println("Frame drop increase: %.2f%%".format(droppedFramesIncreasePercentage)) } else { diff --git a/sentry-android-ui-tests/benchmark/src/main/AndroidManifest.xml b/sentry-android-ui-tests/benchmark/src/main/AndroidManifest.xml index 58c0f6a0830..56d93dee2a9 100644 --- a/sentry-android-ui-tests/benchmark/src/main/AndroidManifest.xml +++ b/sentry-android-ui-tests/benchmark/src/main/AndroidManifest.xml @@ -1,6 +1,7 @@ + xmlns:tools="http://schemas.android.com/tools" + package="io.sentry.android.uitests.benchmark"> - - diff --git a/sentry-android-ui-tests/end2end/src/main/java/io/sentry/android/uitests/end2end/EmptyActivity.kt b/sentry-uitest/sentry-uitest-android/src/main/java/io/sentry/uitest/android/EmptyActivity.kt similarity index 84% rename from sentry-android-ui-tests/end2end/src/main/java/io/sentry/android/uitests/end2end/EmptyActivity.kt rename to sentry-uitest/sentry-uitest-android/src/main/java/io/sentry/uitest/android/EmptyActivity.kt index ea78052c7f1..b8f7d5f373a 100644 --- a/sentry-android-ui-tests/end2end/src/main/java/io/sentry/android/uitests/end2end/EmptyActivity.kt +++ b/sentry-uitest/sentry-uitest-android/src/main/java/io/sentry/uitest/android/EmptyActivity.kt @@ -1,4 +1,4 @@ -package io.sentry.android.uitests.end2end +package io.sentry.uitest.android import android.os.Bundle import androidx.appcompat.app.AppCompatActivity diff --git a/sentry-android-ui-tests/end2end/src/main/java/io/sentry/android/uitests/end2end/utils/BooleanIdlingResource.kt b/sentry-uitest/sentry-uitest-android/src/main/java/io/sentry/uitest/android/utils/BooleanIdlingResource.kt similarity index 82% rename from sentry-android-ui-tests/end2end/src/main/java/io/sentry/android/uitests/end2end/utils/BooleanIdlingResource.kt rename to sentry-uitest/sentry-uitest-android/src/main/java/io/sentry/uitest/android/utils/BooleanIdlingResource.kt index 5477eb62887..86a5b26709a 100644 --- a/sentry-android-ui-tests/end2end/src/main/java/io/sentry/android/uitests/end2end/utils/BooleanIdlingResource.kt +++ b/sentry-uitest/sentry-uitest-android/src/main/java/io/sentry/uitest/android/utils/BooleanIdlingResource.kt @@ -1,4 +1,4 @@ -package io.sentry.android.uitests.end2end.utils +package io.sentry.uitest.android.utils import androidx.test.espresso.IdlingResource import java.util.concurrent.atomic.AtomicBoolean @@ -12,6 +12,7 @@ class BooleanIdlingResource(private val name: String) : IdlingResource { private var callback: IdlingResource.ResourceCallback? = null + /** Sets whether this resource is currently in idle state. */ fun setIdle(idling: Boolean) { if (!isIdle.getAndSet(idling) && idling) { callback?.onTransitionToIdle() @@ -20,6 +21,7 @@ class BooleanIdlingResource(private val name: String) : IdlingResource { override fun getName(): String = name + /** Check if this resource is currently in idle state. */ override fun isIdleNow(): Boolean = synchronized(isIdleLock) { isIdle.get() } override fun registerIdleTransitionCallback(callback: IdlingResource.ResourceCallback?) { diff --git a/settings.gradle.kts b/settings.gradle.kts index 6c92b877b8d..f38e71c378d 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -12,8 +12,6 @@ include( "sentry-android-timber", "sentry-android-okhttp", "sentry-android-fragment", - "sentry-android-ui-tests:benchmark", - "sentry-android-ui-tests:end2end", "sentry-apollo", "sentry-test-support", "sentry-log4j2", @@ -38,7 +36,9 @@ include( "sentry-samples:sentry-samples-spring", "sentry-samples:sentry-samples-spring-boot", "sentry-samples:sentry-samples-spring-boot-webflux", - "sentry-samples:sentry-samples-netflix-dgs" + "sentry-samples:sentry-samples-netflix-dgs", + "sentry-uitest:sentry-uitest-android-benchmark", + "sentry-uitest:sentry-uitest-android" ) gradle.beforeProject { From 7a9c217c4de2fe54172d644761c4b2d156d95aa4 Mon Sep 17 00:00:00 2001 From: stefanosiano Date: Wed, 4 May 2022 10:40:53 -0400 Subject: [PATCH 08/22] updated some gradle config to try to fix github build --- .../sentry-uitest-android-benchmark/build.gradle.kts | 4 ++-- sentry-uitest/sentry-uitest-android/build.gradle.kts | 10 +++++++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/sentry-uitest/sentry-uitest-android-benchmark/build.gradle.kts b/sentry-uitest/sentry-uitest-android-benchmark/build.gradle.kts index 12a89d63184..819564fc1ba 100644 --- a/sentry-uitest/sentry-uitest-android-benchmark/build.gradle.kts +++ b/sentry-uitest/sentry-uitest-android-benchmark/build.gradle.kts @@ -33,13 +33,13 @@ android { } buildTypes { - named("debug") { + getByName("debug") { // Since debuggable can"t be modified by gradle for library modules, // it must be done in a manifest - see src/androidTest/AndroidManifest.xml isMinifyEnabled = true proguardFiles(getDefaultProguardFile("proguard-android-optimize.txt"), "sentry-uitest-android-benchmark-proguard-rules.pro") } - named("release") { + getByName("release") { isMinifyEnabled = true isShrinkResources = false signingConfig = signingConfigs.getByName("debug") // to be able to run release mode diff --git a/sentry-uitest/sentry-uitest-android/build.gradle.kts b/sentry-uitest/sentry-uitest-android/build.gradle.kts index 3009a4e8cb4..676b01000e8 100644 --- a/sentry-uitest/sentry-uitest-android/build.gradle.kts +++ b/sentry-uitest/sentry-uitest-android/build.gradle.kts @@ -24,8 +24,6 @@ android { execution = "ANDROIDX_TEST_ORCHESTRATOR" } - testBuildType = "debug" - signingConfigs { getByName("debug") { storeFile = rootProject.file("debug.keystore") @@ -36,11 +34,17 @@ android { } buildTypes { - named("debug") { + getByName("debug") { isMinifyEnabled = true signingConfig = signingConfigs.getByName("debug") proguardFiles(getDefaultProguardFile("proguard-android-optimize.txt")) } + getByName("release") { + isMinifyEnabled = true + proguardFiles(getDefaultProguardFile("proguard-android-optimize.txt"), "proguard-rules.pro") + signingConfig = signingConfigs.getByName("debug") // to be able to run release mode + isShrinkResources = false + } } kotlinOptions { From 45b3669d516e8eb538e7a040d441dc216cedac8b Mon Sep 17 00:00:00 2001 From: stefanosiano Date: Wed, 4 May 2022 11:49:00 -0400 Subject: [PATCH 09/22] updated some gradle config to try to fix github build --- sentry-uitest/sentry-uitest-android/build.gradle.kts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/sentry-uitest/sentry-uitest-android/build.gradle.kts b/sentry-uitest/sentry-uitest-android/build.gradle.kts index 676b01000e8..aa7aa4346ec 100644 --- a/sentry-uitest/sentry-uitest-android/build.gradle.kts +++ b/sentry-uitest/sentry-uitest-android/build.gradle.kts @@ -50,6 +50,12 @@ android { kotlinOptions { jvmTarget = "1.8" } + + variantFilter { + if (Config.Android.shouldSkipDebugVariant(buildType.name)) { + ignore = true + } + } } dependencies { From 4177c939fbf395f6ffe175a7b7d70052b6d6284c Mon Sep 17 00:00:00 2001 From: stefanosiano Date: Wed, 4 May 2022 11:53:40 -0400 Subject: [PATCH 10/22] updated some gradle config to fix github build --- .../sentry-uitest-android-benchmark/build.gradle.kts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/sentry-uitest/sentry-uitest-android-benchmark/build.gradle.kts b/sentry-uitest/sentry-uitest-android-benchmark/build.gradle.kts index 819564fc1ba..f31a419e572 100644 --- a/sentry-uitest/sentry-uitest-android-benchmark/build.gradle.kts +++ b/sentry-uitest/sentry-uitest-android-benchmark/build.gradle.kts @@ -50,6 +50,12 @@ android { kotlinOptions { jvmTarget = "1.8" } + + variantFilter { + if (Config.Android.shouldSkipDebugVariant(buildType.name)) { + ignore = true + } + } } dependencies { From d477a59b87c71e5a946447834a20648346b5c74d Mon Sep 17 00:00:00 2001 From: stefanosiano Date: Wed, 4 May 2022 17:32:46 -0400 Subject: [PATCH 11/22] changed ui tests module to application modules, to have an apk to upload to saucelabs removed dependency between test modules --- CHANGELOG.md | 2 -- .../build.gradle.kts | 10 ++++--- .../src/main/AndroidManifest.xml | 3 +- .../android/benchmark/BenchmarkActivity.kt | 1 - .../benchmark/BooleanIdlingResource.kt | 30 +++++++++++++++++++ .../sentry-uitest-android/build.gradle.kts | 10 ++++--- 6 files changed, 43 insertions(+), 13 deletions(-) create mode 100644 sentry-uitest/sentry-uitest-android-benchmark/src/main/java/io/sentry/uitest/android/benchmark/BooleanIdlingResource.kt diff --git a/CHANGELOG.md b/CHANGELOG.md index 1ff34e04f93..6c63fe76731 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,8 +2,6 @@ ## Unreleased -* Tests: added Android UI tests (#2013) - ### Fix * Change order of event filtering mechanisms (#2001) diff --git a/sentry-uitest/sentry-uitest-android-benchmark/build.gradle.kts b/sentry-uitest/sentry-uitest-android-benchmark/build.gradle.kts index f31a419e572..41ff3b43c97 100644 --- a/sentry-uitest/sentry-uitest-android-benchmark/build.gradle.kts +++ b/sentry-uitest/sentry-uitest-android-benchmark/build.gradle.kts @@ -1,5 +1,5 @@ plugins { - id("com.android.library") + id("com.android.application") kotlin("android") } @@ -7,8 +7,11 @@ android { compileSdk = Config.Android.compileSdkVersion defaultConfig { - minSdk = Config.Android.minSdkVersionOkHttp + applicationId = "io.sentry.uitest.android.benchmark" + minSdk = Config.Android.minSdkVersionNdk targetSdk = Config.Android.targetSdkVersion + versionCode = 1 + versionName = "1.0.0" testInstrumentationRunner = "androidx.benchmark.junit4.AndroidBenchmarkRunner" // Runs each test in its own instance of Instrumentation. This way they are isolated from @@ -48,7 +51,7 @@ android { } kotlinOptions { - jvmTarget = "1.8" + jvmTarget = JavaVersion.VERSION_1_8.toString() } variantFilter { @@ -63,7 +66,6 @@ dependencies { implementation(kotlin(Config.kotlinStdLib, org.jetbrains.kotlin.config.KotlinCompilerVersion.VERSION)) implementation(projects.sentryAndroid) - implementation(projects.sentryUitest.sentryUitestAndroid) implementation(Config.Libs.appCompat) implementation(Config.Libs.androidxCore) implementation(Config.Libs.androidxRecylerView) diff --git a/sentry-uitest/sentry-uitest-android-benchmark/src/main/AndroidManifest.xml b/sentry-uitest/sentry-uitest-android-benchmark/src/main/AndroidManifest.xml index 6b512fee1bd..3d83913bd5b 100644 --- a/sentry-uitest/sentry-uitest-android-benchmark/src/main/AndroidManifest.xml +++ b/sentry-uitest/sentry-uitest-android-benchmark/src/main/AndroidManifest.xml @@ -13,8 +13,7 @@ android:theme="@style/Theme.AppCompat" android:label="Sentry Benchmark" android:debuggable="false" - tools:ignore="HardcodedDebugMode" - tools:replace="android:label,android:debuggable"> + tools:ignore="HardcodedDebugMode"> diff --git a/sentry-uitest/sentry-uitest-android-benchmark/src/main/java/io/sentry/uitest/android/benchmark/BenchmarkActivity.kt b/sentry-uitest/sentry-uitest-android-benchmark/src/main/java/io/sentry/uitest/android/benchmark/BenchmarkActivity.kt index 68054bff380..cf245569fd9 100644 --- a/sentry-uitest/sentry-uitest-android-benchmark/src/main/java/io/sentry/uitest/android/benchmark/BenchmarkActivity.kt +++ b/sentry-uitest/sentry-uitest-android-benchmark/src/main/java/io/sentry/uitest/android/benchmark/BenchmarkActivity.kt @@ -4,7 +4,6 @@ import android.os.Bundle import androidx.appcompat.app.AppCompatActivity import androidx.recyclerview.widget.LinearLayoutManager import androidx.recyclerview.widget.RecyclerView -import io.sentry.uitest.android.utils.BooleanIdlingResource import java.util.concurrent.ExecutorService import java.util.concurrent.Executors diff --git a/sentry-uitest/sentry-uitest-android-benchmark/src/main/java/io/sentry/uitest/android/benchmark/BooleanIdlingResource.kt b/sentry-uitest/sentry-uitest-android-benchmark/src/main/java/io/sentry/uitest/android/benchmark/BooleanIdlingResource.kt new file mode 100644 index 00000000000..edd7d78b618 --- /dev/null +++ b/sentry-uitest/sentry-uitest-android-benchmark/src/main/java/io/sentry/uitest/android/benchmark/BooleanIdlingResource.kt @@ -0,0 +1,30 @@ +package io.sentry.uitest.android.benchmark + +import androidx.test.espresso.IdlingResource +import java.util.concurrent.atomic.AtomicBoolean + +/** Idling resource based on a boolean flag. */ +class BooleanIdlingResource(private val name: String) : IdlingResource { + + private val isIdle = AtomicBoolean(true) + + private val isIdleLock = Object() + + private var callback: IdlingResource.ResourceCallback? = null + + /** Sets whether this resource is currently in idle state. */ + fun setIdle(idling: Boolean) { + if (!isIdle.getAndSet(idling) && idling) { + callback?.onTransitionToIdle() + } + } + + override fun getName(): String = name + + /** Check if this resource is currently in idle state. */ + override fun isIdleNow(): Boolean = synchronized(isIdleLock) { isIdle.get() } + + override fun registerIdleTransitionCallback(callback: IdlingResource.ResourceCallback?) { + this.callback = callback + } +} diff --git a/sentry-uitest/sentry-uitest-android/build.gradle.kts b/sentry-uitest/sentry-uitest-android/build.gradle.kts index aa7aa4346ec..db7501dab0f 100644 --- a/sentry-uitest/sentry-uitest-android/build.gradle.kts +++ b/sentry-uitest/sentry-uitest-android/build.gradle.kts @@ -1,5 +1,5 @@ plugins { - id("com.android.library") + id("com.android.application") kotlin("android") } @@ -7,8 +7,11 @@ android { compileSdk = Config.Android.compileSdkVersion defaultConfig { - minSdk = Config.Android.minSdkVersionOkHttp + applicationId = "io.sentry.uitest.android" + minSdk = Config.Android.minSdkVersionNdk targetSdk = Config.Android.targetSdkVersion + versionCode = 1 + versionName = "1.0.0" testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner" // Runs each test in its own instance of Instrumentation. This way they are isolated from @@ -17,7 +20,6 @@ android { // This doesn't work on some devices with Android 11+. Clearing package data resets permissions. // Check the readme for more info. // testInstrumentationRunnerArguments["clearPackageData"] = "true" - consumerProguardFiles("consumer-rules.pro") } testOptions { @@ -48,7 +50,7 @@ android { } kotlinOptions { - jvmTarget = "1.8" + jvmTarget = JavaVersion.VERSION_1_8.toString() } variantFilter { From 862819da8dd8ae0ad5af3b359b2e63f45280a3a2 Mon Sep 17 00:00:00 2001 From: stefanosiano Date: Thu, 5 May 2022 09:57:45 -0400 Subject: [PATCH 12/22] added detekt and errorprone to uitests modules --- .../build.gradle.kts | 46 +++++++++++++++++++ .../src/main/AndroidManifest.xml | 1 + .../android/benchmark/BenchmarkActivity.kt | 1 + .../BenchmarkTransactionListAdapter.kt | 1 + .../sentry-uitest-android/build.gradle.kts | 46 +++++++++++++++++++ .../src/main/AndroidManifest.xml | 7 ++- 6 files changed, 100 insertions(+), 2 deletions(-) diff --git a/sentry-uitest/sentry-uitest-android-benchmark/build.gradle.kts b/sentry-uitest/sentry-uitest-android-benchmark/build.gradle.kts index 41ff3b43c97..f6e23216e40 100644 --- a/sentry-uitest/sentry-uitest-android-benchmark/build.gradle.kts +++ b/sentry-uitest/sentry-uitest-android-benchmark/build.gradle.kts @@ -1,6 +1,18 @@ +import io.gitlab.arturbosch.detekt.Detekt +import io.gitlab.arturbosch.detekt.extensions.DetektExtension +import net.ltgt.gradle.errorprone.errorprone + plugins { id("com.android.application") kotlin("android") + id(Config.QualityPlugins.errorProne) + id(Config.QualityPlugins.gradleVersions) + id(Config.QualityPlugins.detektPlugin) +} + +configure { + sourceCompatibility = JavaVersion.VERSION_1_8 + targetCompatibility = JavaVersion.VERSION_1_8 } android { @@ -54,6 +66,14 @@ android { jvmTarget = JavaVersion.VERSION_1_8.toString() } + lint { + warningsAsErrors = true + checkDependencies = true + + // We run a full lint analysis as build part in CI, so skip vital checks for assemble tasks. + checkReleaseBuilds = false + } + variantFilter { if (Config.Android.shouldSkipDebugVariant(buildType.name)) { ignore = true @@ -72,6 +92,11 @@ dependencies { implementation(Config.Libs.constraintLayout) implementation(Config.TestLibs.espressoIdlingResource) + compileOnly(Config.CompileOnly.nopen) + errorprone(Config.CompileOnly.nopenChecker) + errorprone(Config.CompileOnly.errorprone) + errorprone(Config.CompileOnly.errorProneNullAway) + androidTestImplementation(Config.TestLibs.kotlinTestJunit) androidTestImplementation(Config.TestLibs.androidxBenchmarkJunit) androidTestImplementation(Config.TestLibs.espressoCore) @@ -81,3 +106,24 @@ dependencies { androidTestImplementation(Config.TestLibs.androidxJunit) androidTestUtil(Config.TestLibs.androidxTestOrchestrator) } + +tasks.withType().configureEach { + options.errorprone { + check("NullAway", net.ltgt.gradle.errorprone.CheckSeverity.ERROR) + option("NullAway:AnnotatedPackages", "io.sentry") + } +} + +tasks.withType { + // Target version of the generated JVM bytecode. It is used for type resolution. + jvmTarget = JavaVersion.VERSION_1_8.toString() +} + +configure { + buildUponDefaultConfig = true + allRules = true +} + +kotlin { + explicitApi() +} diff --git a/sentry-uitest/sentry-uitest-android-benchmark/src/main/AndroidManifest.xml b/sentry-uitest/sentry-uitest-android-benchmark/src/main/AndroidManifest.xml index 3d83913bd5b..1460b98a741 100644 --- a/sentry-uitest/sentry-uitest-android-benchmark/src/main/AndroidManifest.xml +++ b/sentry-uitest/sentry-uitest-android-benchmark/src/main/AndroidManifest.xml @@ -12,6 +12,7 @@ diff --git a/sentry-uitest/sentry-uitest-android-benchmark/src/main/java/io/sentry/uitest/android/benchmark/BenchmarkActivity.kt b/sentry-uitest/sentry-uitest-android-benchmark/src/main/java/io/sentry/uitest/android/benchmark/BenchmarkActivity.kt index cf245569fd9..054427ec1dc 100644 --- a/sentry-uitest/sentry-uitest-android-benchmark/src/main/java/io/sentry/uitest/android/benchmark/BenchmarkActivity.kt +++ b/sentry-uitest/sentry-uitest-android-benchmark/src/main/java/io/sentry/uitest/android/benchmark/BenchmarkActivity.kt @@ -43,6 +43,7 @@ class BenchmarkActivity : AppCompatActivity() { } } + @Suppress("MagicNumber") override fun onResume() { super.onResume() resumed = true diff --git a/sentry-uitest/sentry-uitest-android-benchmark/src/main/java/io/sentry/uitest/android/benchmark/BenchmarkTransactionListAdapter.kt b/sentry-uitest/sentry-uitest-android-benchmark/src/main/java/io/sentry/uitest/android/benchmark/BenchmarkTransactionListAdapter.kt index c0d05b9b8a3..685f7a45d6e 100644 --- a/sentry-uitest/sentry-uitest-android-benchmark/src/main/java/io/sentry/uitest/android/benchmark/BenchmarkTransactionListAdapter.kt +++ b/sentry-uitest/sentry-uitest-android-benchmark/src/main/java/io/sentry/uitest/android/benchmark/BenchmarkTransactionListAdapter.kt @@ -26,6 +26,7 @@ internal class BenchmarkTransactionListAdapter : RecyclerView.Adapter { + sourceCompatibility = JavaVersion.VERSION_1_8 + targetCompatibility = JavaVersion.VERSION_1_8 } android { @@ -53,6 +65,14 @@ android { jvmTarget = JavaVersion.VERSION_1_8.toString() } + lint { + warningsAsErrors = true + checkDependencies = true + + // We run a full lint analysis as build part in CI, so skip vital checks for assemble tasks. + checkReleaseBuilds = false + } + variantFilter { if (Config.Android.shouldSkipDebugVariant(buildType.name)) { ignore = true @@ -69,6 +89,11 @@ dependencies { implementation(Config.Libs.androidxCore) implementation(Config.TestLibs.espressoIdlingResource) + compileOnly(Config.CompileOnly.nopen) + errorprone(Config.CompileOnly.nopenChecker) + errorprone(Config.CompileOnly.errorprone) + errorprone(Config.CompileOnly.errorProneNullAway) + androidTestImplementation(Config.TestLibs.kotlinTestJunit) androidTestImplementation(Config.TestLibs.mockWebserver) androidTestImplementation(Config.TestLibs.espressoCore) @@ -78,3 +103,24 @@ dependencies { androidTestImplementation(Config.TestLibs.androidxJunit) androidTestUtil(Config.TestLibs.androidxTestOrchestrator) } + +tasks.withType().configureEach { + options.errorprone { + check("NullAway", net.ltgt.gradle.errorprone.CheckSeverity.ERROR) + option("NullAway:AnnotatedPackages", "io.sentry") + } +} + +tasks.withType { + // Target version of the generated JVM bytecode. It is used for type resolution. + jvmTarget = JavaVersion.VERSION_1_8.toString() +} + +configure { + buildUponDefaultConfig = true + allRules = true +} + +kotlin { + explicitApi() +} diff --git a/sentry-uitest/sentry-uitest-android/src/main/AndroidManifest.xml b/sentry-uitest/sentry-uitest-android/src/main/AndroidManifest.xml index 1d6f4aba1ff..7c09b73aa47 100644 --- a/sentry-uitest/sentry-uitest-android/src/main/AndroidManifest.xml +++ b/sentry-uitest/sentry-uitest-android/src/main/AndroidManifest.xml @@ -1,11 +1,14 @@ + xmlns:tools="http://schemas.android.com/tools" + package="io.sentry.uitest.android"> + android:icon="@android:mipmap/sym_def_app_icon" + android:usesCleartextTraffic="true" + tools:targetApi="m"> From a30ecdce30d1dae8e8d4f92f2b0dd6d8225e0a04 Mon Sep 17 00:00:00 2001 From: stefanosiano Date: Thu, 5 May 2022 12:11:49 -0400 Subject: [PATCH 13/22] added detekt and errorprone to uitests modules --- .../sentry-uitest-android-benchmark/build.gradle.kts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sentry-uitest/sentry-uitest-android-benchmark/build.gradle.kts b/sentry-uitest/sentry-uitest-android-benchmark/build.gradle.kts index f6e23216e40..a641d063ce6 100644 --- a/sentry-uitest/sentry-uitest-android-benchmark/build.gradle.kts +++ b/sentry-uitest/sentry-uitest-android-benchmark/build.gradle.kts @@ -114,12 +114,12 @@ tasks.withType().configureEach { } } -tasks.withType { +tasks.withType { // Target version of the generated JVM bytecode. It is used for type resolution. jvmTarget = JavaVersion.VERSION_1_8.toString() } -configure { +configure { buildUponDefaultConfig = true allRules = true } From f73b9403aa838bc0b0b574ebaa9222449ec3c388 Mon Sep 17 00:00:00 2001 From: stefanosiano Date: Thu, 12 May 2022 20:06:29 +0200 Subject: [PATCH 14/22] added saucelabs config files removed proguard fix typo in Hub transactionListener -> transactionProfiler --- .gitignore | 1 + .sauce/sentry-uitest-android-benchmark.yml | 29 +++++++++++++++++ .sauce/sentry-uitest-android-end2end.yml | 31 +++++++++++++++++++ buildSrc/src/main/java/Config.kt | 1 - .../build.gradle.kts | 18 ++++++----- sentry-uitest/sentry-uitest-android/README.md | 15 +++++++++ .../sentry-uitest-android/build.gradle.kts | 14 +++++---- .../sentry-uitest-android/consumer-rules.pro | 0 sentry/src/main/java/io/sentry/Hub.java | 4 +-- 9 files changed, 96 insertions(+), 17 deletions(-) create mode 100644 .sauce/sentry-uitest-android-benchmark.yml create mode 100644 .sauce/sentry-uitest-android-end2end.yml delete mode 100644 sentry-uitest/sentry-uitest-android/consumer-rules.pro diff --git a/.gitignore b/.gitignore index 03d9c5ea4bb..93b00e96cd6 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ .idea/ .gradle/ build/ +artifacts/ out/ local.properties **.iml diff --git a/.sauce/sentry-uitest-android-benchmark.yml b/.sauce/sentry-uitest-android-benchmark.yml new file mode 100644 index 00000000000..0e8115523e0 --- /dev/null +++ b/.sauce/sentry-uitest-android-benchmark.yml @@ -0,0 +1,29 @@ +apiVersion: v1alpha +kind: espresso +sauce: + region: us-west-1 + # Controls how many suites are executed at the same time (sauce test env only). + concurrency: 1 + metadata: + name: Android benchmarks with Espresso + tags: + - benchmarks + - android +espresso: + app: ./sentry-uitest/sentry-uitest-android-benchmark/build/outputs/apk/release/sentry-uitest-android-benchmark-release.apk + testApp: ./sentry-uitest/sentry-uitest-android-benchmark/build/outputs/apk/androidTest/release/sentry-uitest-android-benchmark-release-androidTest.apk +suites: + name: "Android Benchmarks" + devices: + - name: "Google Pixel 2" + platformVersion: 11 + - id: Google_Pixel_2_real_us + testOptions: + useTestOrchestrator: true +# Controls what artifacts to fetch when the suite on Sauce Cloud has finished. +artifacts: + download: + when: always + match: + - junit.xml + directory: ./artifacts/ diff --git a/.sauce/sentry-uitest-android-end2end.yml b/.sauce/sentry-uitest-android-end2end.yml new file mode 100644 index 00000000000..e510833e799 --- /dev/null +++ b/.sauce/sentry-uitest-android-end2end.yml @@ -0,0 +1,31 @@ +apiVersion: v1alpha +kind: espresso +sauce: + region: us-west-1 + # Controls how many suites are executed at the same time (sauce test env only). + concurrency: 1 + metadata: + name: Android end2end tests with Espresso + tags: + - e2e + - android +espresso: + app: ./sentry-uitest/sentry-uitest-android/build/outputs/apk/release/sentry-uitest-android-release.apk + testApp: ./sentry-uitest/sentry-uitest-android/build/outputs/apk/androidTest/release/sentry-uitest-android-release-androidTest.apk +suites: + name: "Android End2end" + emulators: + - name: "Android GoogleApi Emulator" + orientation: portrait + platformVersions: + - "11.0" + - "10.0" + testOptions: + useTestOrchestrator: true +# Controls what artifacts to fetch when the suite on Sauce Cloud has finished. +artifacts: + download: + when: always + match: + - junit.xml + directory: ./artifacts/ diff --git a/buildSrc/src/main/java/Config.kt b/buildSrc/src/main/java/Config.kt index 403928598b5..9a8769da0dc 100644 --- a/buildSrc/src/main/java/Config.kt +++ b/buildSrc/src/main/java/Config.kt @@ -119,7 +119,6 @@ object Config { val androidxRunner = "androidx.test:runner:$androidxTestVersion" val androidxTestCoreKtx = "androidx.test:core-ktx:$androidxTestVersion" val androidxTestRules = "androidx.test:rules:$androidxTestVersion" - val androidxBenchmarkJunit = "androidx.benchmark:benchmark-junit4:1.0.0" val espressoCore = "androidx.test.espresso:espresso-core:$espressoVersion" val espressoIdlingResource = "androidx.test.espresso:espresso-idling-resource:$espressoVersion" val androidxTestOrchestrator = "androidx.test:orchestrator:1.4.1" diff --git a/sentry-uitest/sentry-uitest-android-benchmark/build.gradle.kts b/sentry-uitest/sentry-uitest-android-benchmark/build.gradle.kts index a641d063ce6..1e656432f88 100644 --- a/sentry-uitest/sentry-uitest-android-benchmark/build.gradle.kts +++ b/sentry-uitest/sentry-uitest-android-benchmark/build.gradle.kts @@ -25,7 +25,7 @@ android { versionCode = 1 versionName = "1.0.0" - testInstrumentationRunner = "androidx.benchmark.junit4.AndroidBenchmarkRunner" + testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner" // Runs each test in its own instance of Instrumentation. This way they are isolated from // one another and get their own Application instance. // https://developer.android.com/training/testing/instrumented-tests/androidx-test-libraries/runner#enable-gradle @@ -47,18 +47,21 @@ android { } } + testBuildType = System.getProperty("testBuildType", "debug") + buildTypes { getByName("debug") { - // Since debuggable can"t be modified by gradle for library modules, - // it must be done in a manifest - see src/androidTest/AndroidManifest.xml - isMinifyEnabled = true - proguardFiles(getDefaultProguardFile("proguard-android-optimize.txt"), "sentry-uitest-android-benchmark-proguard-rules.pro") + isDebuggable = false + isMinifyEnabled = false + signingConfig = signingConfigs.getByName("debug") + proguardFiles(getDefaultProguardFile("proguard-android-optimize.txt"), "benchmark-proguard-rules.pro") } getByName("release") { - isMinifyEnabled = true + isDebuggable = false + isMinifyEnabled = false isShrinkResources = false signingConfig = signingConfigs.getByName("debug") // to be able to run release mode - proguardFiles(getDefaultProguardFile("proguard-android-optimize.txt"), "sentry-uitest-android-benchmark-proguard-rules.pro") + proguardFiles(getDefaultProguardFile("proguard-android-optimize.txt"), "benchmark-proguard-rules.pro") } } @@ -98,7 +101,6 @@ dependencies { errorprone(Config.CompileOnly.errorProneNullAway) androidTestImplementation(Config.TestLibs.kotlinTestJunit) - androidTestImplementation(Config.TestLibs.androidxBenchmarkJunit) androidTestImplementation(Config.TestLibs.espressoCore) androidTestImplementation(Config.TestLibs.androidxTestCoreKtx) androidTestImplementation(Config.TestLibs.androidxRunner) diff --git a/sentry-uitest/sentry-uitest-android/README.md b/sentry-uitest/sentry-uitest-android/README.md index ae62397a6b5..b73b6c1abc0 100644 --- a/sentry-uitest/sentry-uitest-android/README.md +++ b/sentry-uitest/sentry-uitest-android/README.md @@ -10,6 +10,21 @@ _Care: the benchmarks need to run the tests multiple times to get reliable resul If you don't care about benchmark tests you can run `./gradlew connectedCheck -x :sentry-uitest:sentry-uitest-android-benchmark:connectedCheck`. You can run benchmark tests only with `./gradlew :sentry-uitest:sentry-uitest-android-benchmark:connectedCheck`. +# SauceLabs +To run on saucelabs execute following commands (need also `SAUCE_USERNAME` and `SAUCE_ACCESS_KEY` environment variables): +For Benchmarks: +``` +./gradlew :sentry-uitest:sentry-uitest-android-benchmark:assembleRelease +./gradlew :sentry-uitest:sentry-uitest-android-benchmark:assembleAndroidTest -DtestBuildType=release +saucectl run -c .sauce/sentry-uitest-android-benchmark.yml +``` +For End 2 End: +``` +./gradlew :sentry-uitest:sentry-uitest-android:assembleRelease +./gradlew :sentry-uitest:sentry-uitest-android:assembleAndroidTest -DtestBuildType=release +saucectl run -c .sauce/sentry-uitest-android-end2end.yml +``` + # Troubleshooting There is an issue with Android 11+ (Xiaomi only?). diff --git a/sentry-uitest/sentry-uitest-android/build.gradle.kts b/sentry-uitest/sentry-uitest-android/build.gradle.kts index 367096a805a..d8c40685706 100644 --- a/sentry-uitest/sentry-uitest-android/build.gradle.kts +++ b/sentry-uitest/sentry-uitest-android/build.gradle.kts @@ -47,17 +47,19 @@ android { } } + testBuildType = System.getProperty("testBuildType", "debug") + buildTypes { getByName("debug") { - isMinifyEnabled = true + isMinifyEnabled = false signingConfig = signingConfigs.getByName("debug") proguardFiles(getDefaultProguardFile("proguard-android-optimize.txt")) } getByName("release") { - isMinifyEnabled = true - proguardFiles(getDefaultProguardFile("proguard-android-optimize.txt"), "proguard-rules.pro") - signingConfig = signingConfigs.getByName("debug") // to be able to run release mode + isMinifyEnabled = false isShrinkResources = false + proguardFiles(getDefaultProguardFile("proguard-android-optimize.txt")) + signingConfig = signingConfigs.getByName("debug") // to be able to run release mode } } @@ -95,11 +97,11 @@ dependencies { errorprone(Config.CompileOnly.errorProneNullAway) androidTestImplementation(Config.TestLibs.kotlinTestJunit) - androidTestImplementation(Config.TestLibs.mockWebserver) androidTestImplementation(Config.TestLibs.espressoCore) - androidTestImplementation(Config.TestLibs.androidxTestCoreKtx) androidTestImplementation(Config.TestLibs.androidxRunner) androidTestImplementation(Config.TestLibs.androidxTestRules) + androidTestImplementation(Config.TestLibs.androidxTestCoreKtx) + androidTestImplementation(Config.TestLibs.mockWebserver) androidTestImplementation(Config.TestLibs.androidxJunit) androidTestUtil(Config.TestLibs.androidxTestOrchestrator) } diff --git a/sentry-uitest/sentry-uitest-android/consumer-rules.pro b/sentry-uitest/sentry-uitest-android/consumer-rules.pro deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/sentry/src/main/java/io/sentry/Hub.java b/sentry/src/main/java/io/sentry/Hub.java index fd3dc0e18ad..e9a781324b8 100644 --- a/sentry/src/main/java/io/sentry/Hub.java +++ b/sentry/src/main/java/io/sentry/Hub.java @@ -682,8 +682,8 @@ public void flush(long timeoutMillis) { // The listener is called only if the transaction exists, as the transaction is needed to // stop it if (samplingDecision && options.isProfilingEnabled()) { - final ITransactionProfiler transactionListener = options.getTransactionProfiler(); - transactionListener.onTransactionStart(transaction); + final ITransactionProfiler transactionProfiler = options.getTransactionProfiler(); + transactionProfiler.onTransactionStart(transaction); } } if (bindToScope) { From dd38ed48a93d44dfda05ddf7c465681e71666f15 Mon Sep 17 00:00:00 2001 From: stefanosiano Date: Tue, 17 May 2022 17:46:02 +0200 Subject: [PATCH 15/22] fixed proguard rules (disabled shrinking) and enabled proguard printing the number of cpu cores in benchmark results pass a Choreographer object instead of the Runner to the BenchmarkOperation --- .../benchmark-proguard-rules.pro | 6 +- .../build.gradle.kts | 7 ++- .../android/benchmark/SentryBenchmarkTest.kt | 56 ++++++++++--------- .../benchmark/util/BenchmarkOperation.kt | 19 +++---- .../util/BenchmarkOperationResult.kt | 5 +- 5 files changed, 45 insertions(+), 48 deletions(-) diff --git a/sentry-uitest/sentry-uitest-android-benchmark/benchmark-proguard-rules.pro b/sentry-uitest/sentry-uitest-android-benchmark/benchmark-proguard-rules.pro index 75bf7d64dff..8f5e14b25e7 100644 --- a/sentry-uitest/sentry-uitest-android-benchmark/benchmark-proguard-rules.pro +++ b/sentry-uitest/sentry-uitest-android-benchmark/benchmark-proguard-rules.pro @@ -21,6 +21,8 @@ #-renamesourcefileattribute SourceFile -dontobfuscate +#Shrinking removes annotations and "unused classes" from test apk, so we don't shrink +-dontshrink -ignorewarnings @@ -31,7 +33,3 @@ -dontwarn androidx.test.** -dontwarn org.junit.** --dontwarn org.hamcrest.** --dontwarn com.squareup.javawriter.JavaWriter - --keepclasseswithmembers @org.junit.runner.RunWith public class * diff --git a/sentry-uitest/sentry-uitest-android-benchmark/build.gradle.kts b/sentry-uitest/sentry-uitest-android-benchmark/build.gradle.kts index 1e656432f88..4212b34ae9c 100644 --- a/sentry-uitest/sentry-uitest-android-benchmark/build.gradle.kts +++ b/sentry-uitest/sentry-uitest-android-benchmark/build.gradle.kts @@ -52,14 +52,15 @@ android { buildTypes { getByName("debug") { isDebuggable = false - isMinifyEnabled = false + isMinifyEnabled = true + isMinifyEnabled = true signingConfig = signingConfigs.getByName("debug") proguardFiles(getDefaultProguardFile("proguard-android-optimize.txt"), "benchmark-proguard-rules.pro") } getByName("release") { isDebuggable = false - isMinifyEnabled = false - isShrinkResources = false + isMinifyEnabled = true + isShrinkResources = true signingConfig = signingConfigs.getByName("debug") // to be able to run release mode proguardFiles(getDefaultProguardFile("proguard-android-optimize.txt"), "benchmark-proguard-rules.pro") } diff --git a/sentry-uitest/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/SentryBenchmarkTest.kt b/sentry-uitest/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/SentryBenchmarkTest.kt index 2955441b71b..8725e93b6b5 100644 --- a/sentry-uitest/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/SentryBenchmarkTest.kt +++ b/sentry-uitest/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/SentryBenchmarkTest.kt @@ -1,6 +1,7 @@ package io.sentry.uitest.android.benchmark import android.content.Context +import android.view.Choreographer import androidx.lifecycle.Lifecycle import androidx.test.core.app.ApplicationProvider import androidx.test.core.app.launchActivity @@ -28,6 +29,7 @@ class SentryBenchmarkTest { private lateinit var runner: AndroidJUnitRunner private lateinit var context: Context + private lateinit var choreographer: Choreographer @BeforeTest fun setUp() { @@ -35,6 +37,10 @@ class SentryBenchmarkTest { context = ApplicationProvider.getApplicationContext() context.cacheDir.deleteRecursively() IdlingRegistry.getInstance().register(BenchmarkActivity.scrollingIdlingResource) + // Must run on the main thread to get the main thread choreographer. + runner.runOnMainSync { + choreographer = Choreographer.getInstance() + } } @AfterTest @@ -47,23 +53,19 @@ class SentryBenchmarkTest { // We compare two operation that are the same. We expect the increases to be negligible, as the results // should be very similar. - val op1 = BenchmarkOperation(runner, getOperation(runner) { null }) - val op2 = BenchmarkOperation(runner, getOperation(runner) { null }) + val op1 = BenchmarkOperation(choreographer, getOperation(runner)) + val op2 = BenchmarkOperation(choreographer, getOperation(runner)) val comparisonResult = BenchmarkOperation.compare(op1, "Op1", op2, "Op2") - assertTrue { comparisonResult.durationIncrease >= -1 } - assertTrue { comparisonResult.durationIncrease < 1 } - assertTrue { comparisonResult.cpuTimeIncrease >= -1 } - assertTrue { comparisonResult.cpuTimeIncrease < 1 } + assertTrue { comparisonResult.durationIncrease in -1F..1F } + assertTrue { comparisonResult.cpuTimeIncrease in -1F..1F } // The fps decrease is skipped for the moment, due to approximation: // if an operation runs at 59.49 fps and the other at 59.51, they are considered 59 and 60 fps respectively. // Their difference would be 1 / 60 * 100 = 1.66666% // On slow devices, a difference of 1 fps on an average of 20 fps would account for 5% decrease. // On even slower devices the difference would be even higher. Let's skip for now, as it's not important anyway. -// assertTrue { comparisonResult.fpsDecrease >= -2 } -// assertTrue { comparisonResult.fpsDecrease < 2 } - assertTrue { comparisonResult.droppedFramesIncrease >= -1 } - assertTrue { comparisonResult.droppedFramesIncrease < 1 } +// assertTrue { comparisonResult.fpsDecrease in -2F..2F } + assertTrue { comparisonResult.droppedFramesIncrease in -1F..1F } } @Test @@ -81,9 +83,9 @@ class SentryBenchmarkTest { // We compare the same operation with and without profiled transaction. // We expect the profiled transaction operation to be slower, but not slower than 5%. - val benchmarkOperationNoTransaction = BenchmarkOperation(runner, getOperation(runner) { null }) + val benchmarkOperationNoTransaction = BenchmarkOperation(choreographer, getOperation(runner)) val benchmarkOperationProfiled = BenchmarkOperation( - runner, + choreographer, getOperation(runner) { Sentry.startTransaction("Benchmark", "ProfiledTransaction") } @@ -95,21 +97,17 @@ class SentryBenchmarkTest { "ProfiledTransaction" ) - assertTrue { comparisonResult.durationIncrease >= 0 } - assertTrue { comparisonResult.durationIncrease < 5.0 } - assertTrue { comparisonResult.cpuTimeIncrease >= 0 } - assertTrue { comparisonResult.cpuTimeIncrease < 5.0 } - assertTrue { comparisonResult.fpsDecrease >= 0 } - assertTrue { comparisonResult.fpsDecrease < 5.0 } - assertTrue { comparisonResult.droppedFramesIncrease >= 0 } - assertTrue { comparisonResult.droppedFramesIncrease < 5.0 } + assertTrue { comparisonResult.durationIncrease in -1F..5F } + assertTrue { comparisonResult.cpuTimeIncrease in -1F..5F } + assertTrue { comparisonResult.fpsDecrease in -1F..5F } + assertTrue { comparisonResult.droppedFramesIncrease in -1F..5F } } /** * Operation that will be compared: it launches [BenchmarkActivity], swipe the list and closes it. * The [transactionBuilder] is used to create the transaction before the swipes. */ - private fun getOperation(runner: AndroidJUnitRunner, transactionBuilder: () -> ITransaction?): () -> Unit = { + private fun getOperation(runner: AndroidJUnitRunner, transactionBuilder: () -> ITransaction? = { null }): () -> Unit = { var transaction: ITransaction? = null // Launch the sentry-uitest-android-benchmark activity val benchmarkScenario = launchActivity() @@ -118,18 +116,22 @@ class SentryBenchmarkTest { transaction = transactionBuilder() } // Just swipe the list some times: this is the benchmarked operation - repeat(2) { - onView(withId(R.id.benchmark_transaction_list)).perform(swipeUp()) - Espresso.onIdle() - } + swipeList(2) // We finish the transaction runner.runOnMainSync { transaction?.finish() } // We swipe a last time to measure how finishing the transaction may affect other operations - onView(withId(R.id.benchmark_transaction_list)).perform(swipeUp()) - Espresso.onIdle() + swipeList(1) benchmarkScenario.moveToState(Lifecycle.State.DESTROYED) } + + private fun swipeList(times: Int) { + repeat(times) { + Thread.sleep(100) + onView(withId(R.id.benchmark_transaction_list)).perform(swipeUp()) + Espresso.onIdle() + } + } } diff --git a/sentry-uitest/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/util/BenchmarkOperation.kt b/sentry-uitest/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/util/BenchmarkOperation.kt index f4bf631c200..0177f7ed751 100644 --- a/sentry-uitest/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/util/BenchmarkOperation.kt +++ b/sentry-uitest/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/util/BenchmarkOperation.kt @@ -3,7 +3,7 @@ package io.sentry.uitest.android.benchmark.util import android.os.Process import android.os.SystemClock import android.view.Choreographer -import androidx.test.runner.AndroidJUnitRunner +import java.util.concurrent.TimeUnit // 60 FPS is the recommended target: https://www.youtube.com/watch?v=CaMTIgxCSqU private const val FRAME_DURATION_60FPS_NS: Double = 1_000_000_000 / 60.0 @@ -14,7 +14,7 @@ private const val FRAME_DURATION_60FPS_NS: Double = 1_000_000_000 / 60.0 * a [BenchmarkResult] with relative measured overheads. */ -internal class BenchmarkOperation(runner: AndroidJUnitRunner, private val op: () -> Unit) { +internal class BenchmarkOperation(private val choreographer: Choreographer, private val op: () -> Unit) { companion object { @@ -60,7 +60,6 @@ internal class BenchmarkOperation(runner: AndroidJUnitRunner, private val op: () } } - private lateinit var choreographer: Choreographer private var iterations: Int = 0 private var durationNanos: Long = 0 private var cpuDurationMillis: Long = 0 @@ -68,13 +67,6 @@ internal class BenchmarkOperation(runner: AndroidJUnitRunner, private val op: () private var droppedFrames: Double = 0.0 private var lastFrameTimeNanos: Long = 0 - init { - // Must run on the main thread to get the main thread choreographer. - runner.runOnMainSync { - choreographer = Choreographer.getInstance() - } - } - /** Run the operation without measuring it. */ private fun warmup() { op() @@ -104,17 +96,20 @@ internal class BenchmarkOperation(runner: AndroidJUnitRunner, private val op: () cpuDurationMillis / iterations, droppedFrames / iterations, durationNanos / iterations, - (frames * 1_000_000_000L / durationNanos).toInt(), + // fps = counted frames per seconds converted into frames per nanoseconds, divided by duration in nanoseconds + // We don't convert the duration into seconds to avoid issues with rounding and possible division by 0 + (frames * TimeUnit.SECONDS.toNanos(1) / durationNanos).toInt(), operationName ) /** * Helps ensure that operations don't impact one another. - * Doesn't appear to currently have an impact on the sentry-uitest-android-benchmark. + * Doesn't appear to currently have an impact on the benchmark. */ private fun isolate() { Thread.sleep(500) Runtime.getRuntime().gc() + Thread.sleep(100) } private val frameCallback = object : Choreographer.FrameCallback { diff --git a/sentry-uitest/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/util/BenchmarkOperationResult.kt b/sentry-uitest/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/util/BenchmarkOperationResult.kt index a9fc19e96dd..764235f945b 100644 --- a/sentry-uitest/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/util/BenchmarkOperationResult.kt +++ b/sentry-uitest/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/util/BenchmarkOperationResult.kt @@ -33,14 +33,15 @@ internal data class BenchmarkOperationResult( println("--------------------") // Measure average cpu time - val cpuTimeDiff = (avgCpuTimeMillis - other.avgCpuTimeMillis) / Runtime.getRuntime().availableProcessors() + val cores = Runtime.getRuntime().availableProcessors() + val cpuTimeDiff = (avgCpuTimeMillis - other.avgCpuTimeMillis) / cores val cpuTimeOverheadPercentage = cpuTimeDiff * 100.0 / other.avgCpuTimeMillis // Cpu time spent profiling is weighted based on available threads, as profiling runs on 1 thread only. println( "Measuring the increased cpu time. It has no direct impact on performance of the app, " + "but it has on battery usage, as the cpu is 'awaken' longer." ) - println("The weighted difference of cpu times is $cpuTimeDiff ms.") + println("The weighted difference of cpu times is $cpuTimeDiff ms (over $cores available cores).") println("[${other.operationName}] Cpu time: ${other.avgCpuTimeMillis} ms") println("[$operationName] Cpu time: $avgCpuTimeMillis ms") if (cpuTimeOverheadPercentage > 0) { From 8f62a8cbf920454d6e105a0bdf1065f8c1044ca6 Mon Sep 17 00:00:00 2001 From: stefanosiano Date: Thu, 19 May 2022 10:35:22 +0200 Subject: [PATCH 16/22] reverted profiling comparison rages from -1..5 to 0..5 --- .../uitest/android/benchmark/SentryBenchmarkTest.kt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sentry-uitest/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/SentryBenchmarkTest.kt b/sentry-uitest/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/SentryBenchmarkTest.kt index 8725e93b6b5..1f7b79ddc4b 100644 --- a/sentry-uitest/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/SentryBenchmarkTest.kt +++ b/sentry-uitest/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/SentryBenchmarkTest.kt @@ -97,10 +97,10 @@ class SentryBenchmarkTest { "ProfiledTransaction" ) - assertTrue { comparisonResult.durationIncrease in -1F..5F } - assertTrue { comparisonResult.cpuTimeIncrease in -1F..5F } - assertTrue { comparisonResult.fpsDecrease in -1F..5F } - assertTrue { comparisonResult.droppedFramesIncrease in -1F..5F } + assertTrue { comparisonResult.durationIncrease in 0F..5F } + assertTrue { comparisonResult.cpuTimeIncrease in 0F..5F } + assertTrue { comparisonResult.fpsDecrease in 0F..5F } + assertTrue { comparisonResult.droppedFramesIncrease in 0F..5F } } /** From 2a9f83bd47fb7499b1329d076c98f25397b35266 Mon Sep 17 00:00:00 2001 From: stefanosiano Date: Mon, 23 May 2022 11:43:22 +0200 Subject: [PATCH 17/22] removed sentry providers through manifest instead of disabling the sdk added ViewBinding few cleanups moved "androidx.test.runner.AndroidJUnitRunner" in Config.TestLibs.androidJUnitRunner --- buildSrc/src/main/java/Config.kt | 1 + sentry-android-core/build.gradle.kts | 2 +- sentry-android-ndk/build.gradle.kts | 2 +- sentry-android-timber/build.gradle.kts | 2 +- .../build.gradle.kts | 16 +++++----- .../android/benchmark/SentryBenchmarkTest.kt | 32 +++++++++---------- .../benchmark/util/BenchmarkOperation.kt | 1 - .../src/main/AndroidManifest.xml | 14 +++++--- .../android/benchmark/BenchmarkActivity.kt | 7 ++-- .../BenchmarkTransactionListAdapter.kt | 12 +++---- .../sentry-uitest-android/build.gradle.kts | 23 +++++++------ .../io/sentry/uitest/android/BaseUiTest.kt | 12 +++---- .../io/sentry/uitest/android/EnvelopeTests.kt | 4 +-- .../src/main/AndroidManifest.xml | 10 +++++- 14 files changed, 74 insertions(+), 64 deletions(-) diff --git a/buildSrc/src/main/java/Config.kt b/buildSrc/src/main/java/Config.kt index 9a8769da0dc..fbab16d8b63 100644 --- a/buildSrc/src/main/java/Config.kt +++ b/buildSrc/src/main/java/Config.kt @@ -114,6 +114,7 @@ object Config { private val androidxTestVersion = "1.4.0" private val espressoVersion = "3.4.0" + val androidJUnitRunner = "androidx.test.runner.AndroidJUnitRunner" val kotlinTestJunit = "org.jetbrains.kotlin:kotlin-test-junit:$kotlinVersion" val androidxCore = "androidx.test:core:$androidxTestVersion" val androidxRunner = "androidx.test:runner:$androidxTestVersion" diff --git a/sentry-android-core/build.gradle.kts b/sentry-android-core/build.gradle.kts index 7937655aad1..73243b80fa9 100644 --- a/sentry-android-core/build.gradle.kts +++ b/sentry-android-core/build.gradle.kts @@ -16,7 +16,7 @@ android { targetSdk = Config.Android.targetSdkVersion minSdk = Config.Android.minSdkVersion - testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner" + testInstrumentationRunner = Config.TestLibs.androidJUnitRunner buildConfigField("String", "SENTRY_ANDROID_SDK_NAME", "\"${Config.Sentry.SENTRY_ANDROID_SDK_NAME}\"") diff --git a/sentry-android-ndk/build.gradle.kts b/sentry-android-ndk/build.gradle.kts index 84a7b0e568b..67b9dacc495 100644 --- a/sentry-android-ndk/build.gradle.kts +++ b/sentry-android-ndk/build.gradle.kts @@ -24,7 +24,7 @@ android { targetSdk = Config.Android.targetSdkVersion minSdk = Config.Android.minSdkVersionNdk // NDK requires a higher API level than core. - testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner" + testInstrumentationRunner = Config.TestLibs.androidJUnitRunner externalNativeBuild { cmake { diff --git a/sentry-android-timber/build.gradle.kts b/sentry-android-timber/build.gradle.kts index 9dd9e1155ea..16649360d2b 100644 --- a/sentry-android-timber/build.gradle.kts +++ b/sentry-android-timber/build.gradle.kts @@ -17,7 +17,7 @@ android { targetSdk = Config.Android.targetSdkVersion minSdk = Config.Android.minSdkVersion - testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner" + testInstrumentationRunner = Config.TestLibs.androidJUnitRunner // for AGP 4.1 buildConfigField("String", "VERSION_NAME", "\"${project.version}\"") diff --git a/sentry-uitest/sentry-uitest-android-benchmark/build.gradle.kts b/sentry-uitest/sentry-uitest-android-benchmark/build.gradle.kts index 4212b34ae9c..ed39396aa51 100644 --- a/sentry-uitest/sentry-uitest-android-benchmark/build.gradle.kts +++ b/sentry-uitest/sentry-uitest-android-benchmark/build.gradle.kts @@ -10,11 +10,6 @@ plugins { id(Config.QualityPlugins.detektPlugin) } -configure { - sourceCompatibility = JavaVersion.VERSION_1_8 - targetCompatibility = JavaVersion.VERSION_1_8 -} - android { compileSdk = Config.Android.compileSdkVersion @@ -25,7 +20,7 @@ android { versionCode = 1 versionName = "1.0.0" - testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner" + testInstrumentationRunner = Config.TestLibs.androidJUnitRunner // Runs each test in its own instance of Instrumentation. This way they are isolated from // one another and get their own Application instance. // https://developer.android.com/training/testing/instrumented-tests/androidx-test-libraries/runner#enable-gradle @@ -34,6 +29,12 @@ android { // testInstrumentationRunnerArguments["clearPackageData"] = "true" } + buildFeatures { + // Determines whether to support View Binding. + // Note that the viewBinding.enabled property is now deprecated. + viewBinding = true + } + testOptions { execution = "ANDROIDX_TEST_ORCHESTRATOR" } @@ -53,12 +54,10 @@ android { getByName("debug") { isDebuggable = false isMinifyEnabled = true - isMinifyEnabled = true signingConfig = signingConfigs.getByName("debug") proguardFiles(getDefaultProguardFile("proguard-android-optimize.txt"), "benchmark-proguard-rules.pro") } getByName("release") { - isDebuggable = false isMinifyEnabled = true isShrinkResources = true signingConfig = signingConfigs.getByName("debug") // to be able to run release mode @@ -114,6 +113,7 @@ tasks.withType().configureEach { options.errorprone { check("NullAway", net.ltgt.gradle.errorprone.CheckSeverity.ERROR) option("NullAway:AnnotatedPackages", "io.sentry") + option("NullAway:UnannotatedSubPackages", "io.sentry.uitest.android.benchmark.databinding") } } diff --git a/sentry-uitest/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/SentryBenchmarkTest.kt b/sentry-uitest/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/SentryBenchmarkTest.kt index 1f7b79ddc4b..a56135fdfe9 100644 --- a/sentry-uitest/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/SentryBenchmarkTest.kt +++ b/sentry-uitest/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/SentryBenchmarkTest.kt @@ -1,6 +1,7 @@ package io.sentry.uitest.android.benchmark import android.content.Context +import android.content.pm.ApplicationInfo import android.view.Choreographer import androidx.lifecycle.Lifecycle import androidx.test.core.app.ApplicationProvider @@ -18,10 +19,11 @@ import io.sentry.Sentry import io.sentry.SentryOptions import io.sentry.android.core.SentryAndroid import io.sentry.uitest.android.benchmark.util.BenchmarkOperation -import org.junit.Test import org.junit.runner.RunWith import kotlin.test.AfterTest import kotlin.test.BeforeTest +import kotlin.test.Test +import kotlin.test.assertFalse import kotlin.test.assertTrue @RunWith(AndroidJUnit4::class) @@ -57,15 +59,11 @@ class SentryBenchmarkTest { val op2 = BenchmarkOperation(choreographer, getOperation(runner)) val comparisonResult = BenchmarkOperation.compare(op1, "Op1", op2, "Op2") - assertTrue { comparisonResult.durationIncrease in -1F..1F } - assertTrue { comparisonResult.cpuTimeIncrease in -1F..1F } - // The fps decrease is skipped for the moment, due to approximation: - // if an operation runs at 59.49 fps and the other at 59.51, they are considered 59 and 60 fps respectively. - // Their difference would be 1 / 60 * 100 = 1.66666% - // On slow devices, a difference of 1 fps on an average of 20 fps would account for 5% decrease. - // On even slower devices the difference would be even higher. Let's skip for now, as it's not important anyway. -// assertTrue { comparisonResult.fpsDecrease in -2F..2F } - assertTrue { comparisonResult.droppedFramesIncrease in -1F..1F } + assertTrue(comparisonResult.durationIncrease in -1F..1F) + assertTrue(comparisonResult.cpuTimeIncrease in -1F..1F) + // The fps decrease comparison is skipped, due to approximation: 59.51 and 59.49 fps are considered 60 and 59, + // respectively. Also, if the average fps is 20 or 60, a difference of 1 fps becomes 5% or 1.66% respectively. + assertTrue(comparisonResult.droppedFramesIncrease in -1F..1F) } @Test @@ -74,9 +72,7 @@ class SentryBenchmarkTest { runner.runOnMainSync { SentryAndroid.init(context) { options: SentryOptions -> options.dsn = "https://key@uri/1234567" - options.isEnableAutoSessionTracking = false options.tracesSampleRate = 1.0 - options.isTraceSampling = true options.isProfilingEnabled = true } } @@ -97,10 +93,14 @@ class SentryBenchmarkTest { "ProfiledTransaction" ) - assertTrue { comparisonResult.durationIncrease in 0F..5F } - assertTrue { comparisonResult.cpuTimeIncrease in 0F..5F } - assertTrue { comparisonResult.fpsDecrease in 0F..5F } - assertTrue { comparisonResult.droppedFramesIncrease in 0F..5F } + runner.runOnMainSync { + Sentry.close() + } + + assertTrue(comparisonResult.durationIncrease in 0F..5F) + assertTrue(comparisonResult.cpuTimeIncrease in 0F..5F) + assertTrue(comparisonResult.fpsDecrease in 0F..5F) + assertTrue(comparisonResult.droppedFramesIncrease in 0F..5F) } /** diff --git a/sentry-uitest/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/util/BenchmarkOperation.kt b/sentry-uitest/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/util/BenchmarkOperation.kt index 0177f7ed751..44532b4ce55 100644 --- a/sentry-uitest/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/util/BenchmarkOperation.kt +++ b/sentry-uitest/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/util/BenchmarkOperation.kt @@ -13,7 +13,6 @@ private const val FRAME_DURATION_60FPS_NS: Double = 1_000_000_000 / 60.0 * Create two [BenchmarkOperation] objects and compare them using [BenchmarkOperation.compare] to get * a [BenchmarkResult] with relative measured overheads. */ - internal class BenchmarkOperation(private val choreographer: Choreographer, private val op: () -> Unit) { companion object { diff --git a/sentry-uitest/sentry-uitest-android-benchmark/src/main/AndroidManifest.xml b/sentry-uitest/sentry-uitest-android-benchmark/src/main/AndroidManifest.xml index 1460b98a741..fa2c6feb143 100644 --- a/sentry-uitest/sentry-uitest-android-benchmark/src/main/AndroidManifest.xml +++ b/sentry-uitest/sentry-uitest-android-benchmark/src/main/AndroidManifest.xml @@ -12,13 +12,19 @@ + android:icon="@android:mipmap/sym_def_app_icon"> - + + + diff --git a/sentry-uitest/sentry-uitest-android-benchmark/src/main/java/io/sentry/uitest/android/benchmark/BenchmarkActivity.kt b/sentry-uitest/sentry-uitest-android-benchmark/src/main/java/io/sentry/uitest/android/benchmark/BenchmarkActivity.kt index 054427ec1dc..56d72de097e 100644 --- a/sentry-uitest/sentry-uitest-android-benchmark/src/main/java/io/sentry/uitest/android/benchmark/BenchmarkActivity.kt +++ b/sentry-uitest/sentry-uitest-android-benchmark/src/main/java/io/sentry/uitest/android/benchmark/BenchmarkActivity.kt @@ -4,6 +4,7 @@ import android.os.Bundle import androidx.appcompat.app.AppCompatActivity import androidx.recyclerview.widget.LinearLayoutManager import androidx.recyclerview.widget.RecyclerView +import io.sentry.uitest.android.benchmark.databinding.ActivityBenchmarkBinding import java.util.concurrent.ExecutorService import java.util.concurrent.Executors @@ -24,14 +25,16 @@ class BenchmarkActivity : AppCompatActivity() { private val backgroundThreadPoolSize = 1 private val executor: ExecutorService = Executors.newFixedThreadPool(backgroundThreadPoolSize) private var resumed = false + private lateinit var binding: ActivityBenchmarkBinding override fun onCreate(savedInstanceState: Bundle?) { super.onCreate(savedInstanceState) - setContentView(R.layout.activity_benchmark) + binding = ActivityBenchmarkBinding.inflate(layoutInflater) + setContentView(binding.root) // We show a simple list that changes the idling resource - findViewById(R.id.benchmark_transaction_list).apply { + binding.benchmarkTransactionList.apply { layoutManager = LinearLayoutManager(this@BenchmarkActivity) adapter = BenchmarkTransactionListAdapter() addOnScrollListener(object : RecyclerView.OnScrollListener() { diff --git a/sentry-uitest/sentry-uitest-android-benchmark/src/main/java/io/sentry/uitest/android/benchmark/BenchmarkTransactionListAdapter.kt b/sentry-uitest/sentry-uitest-android-benchmark/src/main/java/io/sentry/uitest/android/benchmark/BenchmarkTransactionListAdapter.kt index 685f7a45d6e..86eb423c52a 100644 --- a/sentry-uitest/sentry-uitest-android-benchmark/src/main/java/io/sentry/uitest/android/benchmark/BenchmarkTransactionListAdapter.kt +++ b/sentry-uitest/sentry-uitest-android-benchmark/src/main/java/io/sentry/uitest/android/benchmark/BenchmarkTransactionListAdapter.kt @@ -4,19 +4,19 @@ import android.annotation.SuppressLint import android.graphics.Bitmap import android.graphics.Color import android.view.LayoutInflater -import android.view.View import android.view.ViewGroup import android.widget.ImageView import android.widget.TextView import androidx.recyclerview.widget.RecyclerView +import io.sentry.uitest.android.benchmark.databinding.BenchmarkItemListBinding import kotlin.random.Random /** Simple [RecyclerView.Adapter] that generates a bitmap and a text to show for each item. */ internal class BenchmarkTransactionListAdapter : RecyclerView.Adapter() { override fun onCreateViewHolder(parent: ViewGroup, viewType: Int): ViewHolder { - val view = LayoutInflater.from(parent.context).inflate(R.layout.benchmark_item_list, parent, false) - return ViewHolder(view) + val binding = BenchmarkItemListBinding.inflate(LayoutInflater.from(parent.context), parent, false) + return ViewHolder(binding) } override fun onBindViewHolder(holder: ViewHolder, position: Int) { @@ -41,7 +41,7 @@ internal class BenchmarkTransactionListAdapter : RecyclerView.Adapter { - sourceCompatibility = JavaVersion.VERSION_1_8 - targetCompatibility = JavaVersion.VERSION_1_8 -} - android { compileSdk = Config.Android.compileSdkVersion defaultConfig { - applicationId = "io.sentry.uitest.android" minSdk = Config.Android.minSdkVersionNdk targetSdk = Config.Android.targetSdkVersion - versionCode = 1 - versionName = "1.0.0" - testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner" + testInstrumentationRunner = Config.TestLibs.androidJUnitRunner // Runs each test in its own instance of Instrumentation. This way they are isolated from // one another and get their own Application instance. // https://developer.android.com/training/testing/instrumented-tests/androidx-test-libraries/runner#enable-gradle @@ -34,6 +26,12 @@ android { // testInstrumentationRunnerArguments["clearPackageData"] = "true" } + buildFeatures { + // Determines whether to support View Binding. + // Note that the viewBinding.enabled property is now deprecated. + viewBinding = true + } + testOptions { execution = "ANDROIDX_TEST_ORCHESTRATOR" } @@ -51,12 +49,12 @@ android { buildTypes { getByName("debug") { - isMinifyEnabled = false + isMinifyEnabled = true signingConfig = signingConfigs.getByName("debug") proguardFiles(getDefaultProguardFile("proguard-android-optimize.txt")) } getByName("release") { - isMinifyEnabled = false + isMinifyEnabled = true isShrinkResources = false proguardFiles(getDefaultProguardFile("proguard-android-optimize.txt")) signingConfig = signingConfigs.getByName("debug") // to be able to run release mode @@ -110,6 +108,7 @@ tasks.withType().configureEach { options.errorprone { check("NullAway", net.ltgt.gradle.errorprone.CheckSeverity.ERROR) option("NullAway:AnnotatedPackages", "io.sentry") + option("NullAway:UnannotatedSubPackages", "io.sentry.uitest.android.databinding") } } diff --git a/sentry-uitest/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/BaseUiTest.kt b/sentry-uitest/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/BaseUiTest.kt index 301061179ab..581c69b0c87 100644 --- a/sentry-uitest/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/BaseUiTest.kt +++ b/sentry-uitest/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/BaseUiTest.kt @@ -25,15 +25,14 @@ abstract class BaseUiTest { // The mockDsn cannot be changed. If a custom dsn needs to be used, it can be set in the options as usual private set /** - * Idling resource that will be checked by the relay server (if [relayWaitForRequests] is true). + * Idling resource that will be checked by the relay server (if [initSentry] param relayWaitForRequests is true). * This should be increased to match any envelope that will be sent during the test, * so that they can later be checked. */ protected val relayIdlingResource = CountingIdlingResource("relay-requests") - /** Whether relay should wait for requests when asserting envelopes. Enable usage of [relayIdlingResource]. */ - private var relayWaitForRequests: Boolean = false + /** Mock relay server that receives all envelopes sent during the test. */ - protected val relay = MockRelay(relayWaitForRequests, relayIdlingResource) + protected val relay = MockRelay(false, relayIdlingResource) @BeforeTest fun baseSetUp() { @@ -46,9 +45,7 @@ abstract class BaseUiTest { @AfterTest fun baseFinish() { - if (relayWaitForRequests) { - IdlingRegistry.getInstance().unregister(relayIdlingResource) - } + IdlingRegistry.getInstance().unregister(relayIdlingResource) relay.shutdown() Sentry.close() } @@ -63,7 +60,6 @@ abstract class BaseUiTest { relayWaitForRequests: Boolean = false, optionsConfiguration: ((options: SentryOptions) -> Unit)? = null ) { - this.relayWaitForRequests = relayWaitForRequests relay.waitForRequests = relayWaitForRequests if (relayWaitForRequests) { IdlingRegistry.getInstance().register(relayIdlingResource) diff --git a/sentry-uitest/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/EnvelopeTests.kt b/sentry-uitest/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/EnvelopeTests.kt index 360a6e6476a..54c544e260c 100644 --- a/sentry-uitest/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/EnvelopeTests.kt +++ b/sentry-uitest/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/EnvelopeTests.kt @@ -6,8 +6,8 @@ import io.sentry.Sentry import io.sentry.SentryEvent import io.sentry.SentryOptions import io.sentry.protocol.SentryTransaction -import org.junit.Test import org.junit.runner.RunWith +import kotlin.test.Test import kotlin.test.assertEquals import kotlin.test.assertTrue @@ -36,9 +36,7 @@ class EnvelopeTests : BaseUiTest() { fun checkEnvelopeProfiledTransaction() { initSentry(true) { options: SentryOptions -> - options.isEnableAutoSessionTracking = false options.tracesSampleRate = 1.0 - options.isTraceSampling = true options.isProfilingEnabled = true } relayIdlingResource.increment() diff --git a/sentry-uitest/sentry-uitest-android/src/main/AndroidManifest.xml b/sentry-uitest/sentry-uitest-android/src/main/AndroidManifest.xml index 7c09b73aa47..15e849b6f90 100644 --- a/sentry-uitest/sentry-uitest-android/src/main/AndroidManifest.xml +++ b/sentry-uitest/sentry-uitest-android/src/main/AndroidManifest.xml @@ -13,7 +13,15 @@ - + + + From a9a163fbdafc3fab00231df357d0bdce109dd026 Mon Sep 17 00:00:00 2001 From: Sentry Github Bot Date: Mon, 23 May 2022 09:47:37 +0000 Subject: [PATCH 18/22] Format code --- sentry-android-ndk/sentry-native | 2 +- .../io/sentry/uitest/android/benchmark/SentryBenchmarkTest.kt | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/sentry-android-ndk/sentry-native b/sentry-android-ndk/sentry-native index 3436a29d839..5500192dda0 160000 --- a/sentry-android-ndk/sentry-native +++ b/sentry-android-ndk/sentry-native @@ -1 +1 @@ -Subproject commit 3436a29d839aa7437548be940ab62a85ca699635 +Subproject commit 5500192dda05c82468787b2b0637e9c2688b9aed diff --git a/sentry-uitest/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/SentryBenchmarkTest.kt b/sentry-uitest/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/SentryBenchmarkTest.kt index a56135fdfe9..013fbccf659 100644 --- a/sentry-uitest/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/SentryBenchmarkTest.kt +++ b/sentry-uitest/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/SentryBenchmarkTest.kt @@ -1,7 +1,6 @@ package io.sentry.uitest.android.benchmark import android.content.Context -import android.content.pm.ApplicationInfo import android.view.Choreographer import androidx.lifecycle.Lifecycle import androidx.test.core.app.ApplicationProvider @@ -23,7 +22,6 @@ import org.junit.runner.RunWith import kotlin.test.AfterTest import kotlin.test.BeforeTest import kotlin.test.Test -import kotlin.test.assertFalse import kotlin.test.assertTrue @RunWith(AndroidJUnit4::class) From ba41b1ecab1115ad3e4fffbd2f4f6f6ed8b11903 Mon Sep 17 00:00:00 2001 From: stefanosiano Date: Mon, 23 May 2022 13:40:26 +0200 Subject: [PATCH 19/22] renamed sentry-uitest folder to sentry-android-integration-tests --- .sauce/sentry-uitest-android-benchmark.yml | 4 ++-- .sauce/sentry-uitest-android-end2end.yml | 4 ++-- build.gradle.kts | 4 ++-- .../sentry-uitest-android-benchmark/.gitignore | 0 .../benchmark-proguard-rules.pro | 0 .../sentry-uitest-android-benchmark/build.gradle.kts | 0 .../uitest/android/benchmark/SentryBenchmarkTest.kt | 2 -- .../android/benchmark/util/BenchmarkOperation.kt | 4 ++-- .../benchmark/util/BenchmarkOperationResult.kt | 0 .../src/main/AndroidManifest.xml | 0 .../uitest/android/benchmark/BenchmarkActivity.kt | 2 +- .../benchmark/BenchmarkTransactionListAdapter.kt | 0 .../android/benchmark/BooleanIdlingResource.kt | 0 .../src/main/res/layout/activity_benchmark.xml | 0 .../src/main/res/layout/benchmark_item_list.xml | 0 .../sentry-uitest-android/.gitignore | 0 .../sentry-uitest-android/README.md | 12 ++++++------ .../sentry-uitest-android/build.gradle.kts | 0 .../sentry-uitest-android/proguard-rules.pro | 0 .../java/io/sentry/uitest/android/BaseUiTest.kt | 0 .../java/io/sentry/uitest/android/EnvelopeTests.kt | 0 .../uitest/android/mockservers/EnvelopeAsserter.kt | 0 .../sentry/uitest/android/mockservers/MockRelay.kt | 0 .../uitest/android/mockservers/RelayAsserter.kt | 0 .../src/main/AndroidManifest.xml | 0 .../java/io/sentry/uitest/android/EmptyActivity.kt | 0 .../uitest/android/utils/BooleanIdlingResource.kt | 0 settings.gradle.kts | 4 ++-- 28 files changed, 17 insertions(+), 19 deletions(-) rename {sentry-uitest => sentry-android-integration-tests}/sentry-uitest-android-benchmark/.gitignore (100%) rename {sentry-uitest => sentry-android-integration-tests}/sentry-uitest-android-benchmark/benchmark-proguard-rules.pro (100%) rename {sentry-uitest => sentry-android-integration-tests}/sentry-uitest-android-benchmark/build.gradle.kts (100%) rename {sentry-uitest => sentry-android-integration-tests}/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/SentryBenchmarkTest.kt (98%) rename {sentry-uitest => sentry-android-integration-tests}/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/util/BenchmarkOperation.kt (97%) rename {sentry-uitest => sentry-android-integration-tests}/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/util/BenchmarkOperationResult.kt (100%) rename {sentry-uitest => sentry-android-integration-tests}/sentry-uitest-android-benchmark/src/main/AndroidManifest.xml (100%) rename {sentry-uitest => sentry-android-integration-tests}/sentry-uitest-android-benchmark/src/main/java/io/sentry/uitest/android/benchmark/BenchmarkActivity.kt (98%) rename {sentry-uitest => sentry-android-integration-tests}/sentry-uitest-android-benchmark/src/main/java/io/sentry/uitest/android/benchmark/BenchmarkTransactionListAdapter.kt (100%) rename {sentry-uitest => sentry-android-integration-tests}/sentry-uitest-android-benchmark/src/main/java/io/sentry/uitest/android/benchmark/BooleanIdlingResource.kt (100%) rename {sentry-uitest => sentry-android-integration-tests}/sentry-uitest-android-benchmark/src/main/res/layout/activity_benchmark.xml (100%) rename {sentry-uitest => sentry-android-integration-tests}/sentry-uitest-android-benchmark/src/main/res/layout/benchmark_item_list.xml (100%) rename {sentry-uitest => sentry-android-integration-tests}/sentry-uitest-android/.gitignore (100%) rename {sentry-uitest => sentry-android-integration-tests}/sentry-uitest-android/README.md (74%) rename {sentry-uitest => sentry-android-integration-tests}/sentry-uitest-android/build.gradle.kts (100%) rename {sentry-uitest => sentry-android-integration-tests}/sentry-uitest-android/proguard-rules.pro (100%) rename {sentry-uitest => sentry-android-integration-tests}/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/BaseUiTest.kt (100%) rename {sentry-uitest => sentry-android-integration-tests}/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/EnvelopeTests.kt (100%) rename {sentry-uitest => sentry-android-integration-tests}/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/mockservers/EnvelopeAsserter.kt (100%) rename {sentry-uitest => sentry-android-integration-tests}/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/mockservers/MockRelay.kt (100%) rename {sentry-uitest => sentry-android-integration-tests}/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/mockservers/RelayAsserter.kt (100%) rename {sentry-uitest => sentry-android-integration-tests}/sentry-uitest-android/src/main/AndroidManifest.xml (100%) rename {sentry-uitest => sentry-android-integration-tests}/sentry-uitest-android/src/main/java/io/sentry/uitest/android/EmptyActivity.kt (100%) rename {sentry-uitest => sentry-android-integration-tests}/sentry-uitest-android/src/main/java/io/sentry/uitest/android/utils/BooleanIdlingResource.kt (100%) diff --git a/.sauce/sentry-uitest-android-benchmark.yml b/.sauce/sentry-uitest-android-benchmark.yml index 0e8115523e0..35bce15f84e 100644 --- a/.sauce/sentry-uitest-android-benchmark.yml +++ b/.sauce/sentry-uitest-android-benchmark.yml @@ -10,8 +10,8 @@ sauce: - benchmarks - android espresso: - app: ./sentry-uitest/sentry-uitest-android-benchmark/build/outputs/apk/release/sentry-uitest-android-benchmark-release.apk - testApp: ./sentry-uitest/sentry-uitest-android-benchmark/build/outputs/apk/androidTest/release/sentry-uitest-android-benchmark-release-androidTest.apk + app: ./sentry-android-integration-tests/sentry-uitest-android-benchmark/build/outputs/apk/release/sentry-uitest-android-benchmark-release.apk + testApp: ./sentry-android-integration-tests/sentry-uitest-android-benchmark/build/outputs/apk/androidTest/release/sentry-uitest-android-benchmark-release-androidTest.apk suites: name: "Android Benchmarks" devices: diff --git a/.sauce/sentry-uitest-android-end2end.yml b/.sauce/sentry-uitest-android-end2end.yml index e510833e799..62178a4b0dc 100644 --- a/.sauce/sentry-uitest-android-end2end.yml +++ b/.sauce/sentry-uitest-android-end2end.yml @@ -10,8 +10,8 @@ sauce: - e2e - android espresso: - app: ./sentry-uitest/sentry-uitest-android/build/outputs/apk/release/sentry-uitest-android-release.apk - testApp: ./sentry-uitest/sentry-uitest-android/build/outputs/apk/androidTest/release/sentry-uitest-android-release-androidTest.apk + app: ./sentry-android-integration-tests/sentry-uitest-android/build/outputs/apk/release/sentry-uitest-android-release.apk + testApp: ./sentry-android-integration-tests/sentry-uitest-android/build/outputs/apk/androidTest/release/sentry-uitest-android-release-androidTest.apk suites: name: "Android End2end" emulators: diff --git a/build.gradle.kts b/build.gradle.kts index 26ae5cc6258..278b2a94226 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -88,7 +88,7 @@ allprojects { } subprojects { - if (!this.name.contains("sample") && !this.name.contains("uitest") && this.name != "sentry-test-support") { + if (!this.name.contains("sample") && !this.name.contains("integration-tests") && this.name != "sentry-test-support") { apply() val sep = File.separator @@ -167,7 +167,7 @@ gradle.projectsEvaluated { "https://docs.spring.io/spring-boot/docs/current/api/" ) subprojects - .filter { !it.name.contains("sample") && !it.name.contains("uitest") } + .filter { !it.name.contains("sample") && !it.name.contains("integration-tests") } .forEach { proj -> proj.tasks.withType().forEach { javadocTask -> source += javadocTask.source diff --git a/sentry-uitest/sentry-uitest-android-benchmark/.gitignore b/sentry-android-integration-tests/sentry-uitest-android-benchmark/.gitignore similarity index 100% rename from sentry-uitest/sentry-uitest-android-benchmark/.gitignore rename to sentry-android-integration-tests/sentry-uitest-android-benchmark/.gitignore diff --git a/sentry-uitest/sentry-uitest-android-benchmark/benchmark-proguard-rules.pro b/sentry-android-integration-tests/sentry-uitest-android-benchmark/benchmark-proguard-rules.pro similarity index 100% rename from sentry-uitest/sentry-uitest-android-benchmark/benchmark-proguard-rules.pro rename to sentry-android-integration-tests/sentry-uitest-android-benchmark/benchmark-proguard-rules.pro diff --git a/sentry-uitest/sentry-uitest-android-benchmark/build.gradle.kts b/sentry-android-integration-tests/sentry-uitest-android-benchmark/build.gradle.kts similarity index 100% rename from sentry-uitest/sentry-uitest-android-benchmark/build.gradle.kts rename to sentry-android-integration-tests/sentry-uitest-android-benchmark/build.gradle.kts diff --git a/sentry-uitest/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/SentryBenchmarkTest.kt b/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/SentryBenchmarkTest.kt similarity index 98% rename from sentry-uitest/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/SentryBenchmarkTest.kt rename to sentry-android-integration-tests/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/SentryBenchmarkTest.kt index a56135fdfe9..013fbccf659 100644 --- a/sentry-uitest/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/SentryBenchmarkTest.kt +++ b/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/SentryBenchmarkTest.kt @@ -1,7 +1,6 @@ package io.sentry.uitest.android.benchmark import android.content.Context -import android.content.pm.ApplicationInfo import android.view.Choreographer import androidx.lifecycle.Lifecycle import androidx.test.core.app.ApplicationProvider @@ -23,7 +22,6 @@ import org.junit.runner.RunWith import kotlin.test.AfterTest import kotlin.test.BeforeTest import kotlin.test.Test -import kotlin.test.assertFalse import kotlin.test.assertTrue @RunWith(AndroidJUnit4::class) diff --git a/sentry-uitest/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/util/BenchmarkOperation.kt b/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/util/BenchmarkOperation.kt similarity index 97% rename from sentry-uitest/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/util/BenchmarkOperation.kt rename to sentry-android-integration-tests/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/util/BenchmarkOperation.kt index 44532b4ce55..72a71e7cb7e 100644 --- a/sentry-uitest/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/util/BenchmarkOperation.kt +++ b/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/util/BenchmarkOperation.kt @@ -9,7 +9,7 @@ import java.util.concurrent.TimeUnit private const val FRAME_DURATION_60FPS_NS: Double = 1_000_000_000 / 60.0 /** - * Class that allows to sentry-uitest-android-benchmark some operations. + * Class that allows to benchmark some operations. * Create two [BenchmarkOperation] objects and compare them using [BenchmarkOperation.compare] to get * a [BenchmarkResult] with relative measured overheads. */ @@ -22,7 +22,7 @@ internal class BenchmarkOperation(private val choreographer: Choreographer, priv * first operation to always be slower, so comparing two different operations on equal terms is not possible. * This method runs [op1] and [op2] in an alternating sequence. * When [op1] and [op2] are the same, we get (nearly) identical results, as expected. - * You can adjust [warmupIterations] and [measuredIterations]. The lower they are, the faster the sentry-uitest-android-benchmark, + * You can adjust [warmupIterations] and [measuredIterations]. The lower they are, the faster the benchmark, * but accuracy decreases. */ fun compare( diff --git a/sentry-uitest/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/util/BenchmarkOperationResult.kt b/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/util/BenchmarkOperationResult.kt similarity index 100% rename from sentry-uitest/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/util/BenchmarkOperationResult.kt rename to sentry-android-integration-tests/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/util/BenchmarkOperationResult.kt diff --git a/sentry-uitest/sentry-uitest-android-benchmark/src/main/AndroidManifest.xml b/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/main/AndroidManifest.xml similarity index 100% rename from sentry-uitest/sentry-uitest-android-benchmark/src/main/AndroidManifest.xml rename to sentry-android-integration-tests/sentry-uitest-android-benchmark/src/main/AndroidManifest.xml diff --git a/sentry-uitest/sentry-uitest-android-benchmark/src/main/java/io/sentry/uitest/android/benchmark/BenchmarkActivity.kt b/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/main/java/io/sentry/uitest/android/benchmark/BenchmarkActivity.kt similarity index 98% rename from sentry-uitest/sentry-uitest-android-benchmark/src/main/java/io/sentry/uitest/android/benchmark/BenchmarkActivity.kt rename to sentry-android-integration-tests/sentry-uitest-android-benchmark/src/main/java/io/sentry/uitest/android/benchmark/BenchmarkActivity.kt index 56d72de097e..dcf75c0c78e 100644 --- a/sentry-uitest/sentry-uitest-android-benchmark/src/main/java/io/sentry/uitest/android/benchmark/BenchmarkActivity.kt +++ b/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/main/java/io/sentry/uitest/android/benchmark/BenchmarkActivity.kt @@ -18,7 +18,7 @@ class BenchmarkActivity : AppCompatActivity() { } /** - * Each background thread will run non-stop calculations during the sentry-uitest-android-benchmark. + * Each background thread will run non-stop calculations during the benchmark. * One such thread seems enough to represent a busy application. * This number can be increased to mimic busier applications. */ diff --git a/sentry-uitest/sentry-uitest-android-benchmark/src/main/java/io/sentry/uitest/android/benchmark/BenchmarkTransactionListAdapter.kt b/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/main/java/io/sentry/uitest/android/benchmark/BenchmarkTransactionListAdapter.kt similarity index 100% rename from sentry-uitest/sentry-uitest-android-benchmark/src/main/java/io/sentry/uitest/android/benchmark/BenchmarkTransactionListAdapter.kt rename to sentry-android-integration-tests/sentry-uitest-android-benchmark/src/main/java/io/sentry/uitest/android/benchmark/BenchmarkTransactionListAdapter.kt diff --git a/sentry-uitest/sentry-uitest-android-benchmark/src/main/java/io/sentry/uitest/android/benchmark/BooleanIdlingResource.kt b/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/main/java/io/sentry/uitest/android/benchmark/BooleanIdlingResource.kt similarity index 100% rename from sentry-uitest/sentry-uitest-android-benchmark/src/main/java/io/sentry/uitest/android/benchmark/BooleanIdlingResource.kt rename to sentry-android-integration-tests/sentry-uitest-android-benchmark/src/main/java/io/sentry/uitest/android/benchmark/BooleanIdlingResource.kt diff --git a/sentry-uitest/sentry-uitest-android-benchmark/src/main/res/layout/activity_benchmark.xml b/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/main/res/layout/activity_benchmark.xml similarity index 100% rename from sentry-uitest/sentry-uitest-android-benchmark/src/main/res/layout/activity_benchmark.xml rename to sentry-android-integration-tests/sentry-uitest-android-benchmark/src/main/res/layout/activity_benchmark.xml diff --git a/sentry-uitest/sentry-uitest-android-benchmark/src/main/res/layout/benchmark_item_list.xml b/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/main/res/layout/benchmark_item_list.xml similarity index 100% rename from sentry-uitest/sentry-uitest-android-benchmark/src/main/res/layout/benchmark_item_list.xml rename to sentry-android-integration-tests/sentry-uitest-android-benchmark/src/main/res/layout/benchmark_item_list.xml diff --git a/sentry-uitest/sentry-uitest-android/.gitignore b/sentry-android-integration-tests/sentry-uitest-android/.gitignore similarity index 100% rename from sentry-uitest/sentry-uitest-android/.gitignore rename to sentry-android-integration-tests/sentry-uitest-android/.gitignore diff --git a/sentry-uitest/sentry-uitest-android/README.md b/sentry-android-integration-tests/sentry-uitest-android/README.md similarity index 74% rename from sentry-uitest/sentry-uitest-android/README.md rename to sentry-android-integration-tests/sentry-uitest-android/README.md index b73b6c1abc0..c11397383d0 100644 --- a/sentry-uitest/sentry-uitest-android/README.md +++ b/sentry-android-integration-tests/sentry-uitest-android/README.md @@ -7,21 +7,21 @@ By default the envelopes sent to relay are caught by a mock server which allows Simply run `./gradlew connectedCheck` to run all ui tests of all modules (requires a connected device, either physical or an emulator). _Care: the benchmarks need to run the tests multiple times to get reliable results. This means they can take a long time (several minutes)._ -If you don't care about benchmark tests you can run `./gradlew connectedCheck -x :sentry-uitest:sentry-uitest-android-benchmark:connectedCheck`. -You can run benchmark tests only with `./gradlew :sentry-uitest:sentry-uitest-android-benchmark:connectedCheck`. +If you don't care about benchmark tests you can run `./gradlew connectedCheck -x :sentry-android-integration-tests:sentry-uitest-android-benchmark:connectedCheck`. +You can run benchmark tests only with `./gradlew :sentry-android-integration-tests:sentry-uitest-android-benchmark:connectedCheck`. # SauceLabs To run on saucelabs execute following commands (need also `SAUCE_USERNAME` and `SAUCE_ACCESS_KEY` environment variables): For Benchmarks: ``` -./gradlew :sentry-uitest:sentry-uitest-android-benchmark:assembleRelease -./gradlew :sentry-uitest:sentry-uitest-android-benchmark:assembleAndroidTest -DtestBuildType=release +./gradlew :sentry-android-integration-tests:sentry-uitest-android-benchmark:assembleRelease +./gradlew :sentry-android-integration-tests:sentry-uitest-android-benchmark:assembleAndroidTest -DtestBuildType=release saucectl run -c .sauce/sentry-uitest-android-benchmark.yml ``` For End 2 End: ``` -./gradlew :sentry-uitest:sentry-uitest-android:assembleRelease -./gradlew :sentry-uitest:sentry-uitest-android:assembleAndroidTest -DtestBuildType=release +./gradlew :sentry-android-integration-tests:sentry-uitest-android:assembleRelease +./gradlew :sentry-android-integration-tests:sentry-uitest-android:assembleAndroidTest -DtestBuildType=release saucectl run -c .sauce/sentry-uitest-android-end2end.yml ``` diff --git a/sentry-uitest/sentry-uitest-android/build.gradle.kts b/sentry-android-integration-tests/sentry-uitest-android/build.gradle.kts similarity index 100% rename from sentry-uitest/sentry-uitest-android/build.gradle.kts rename to sentry-android-integration-tests/sentry-uitest-android/build.gradle.kts diff --git a/sentry-uitest/sentry-uitest-android/proguard-rules.pro b/sentry-android-integration-tests/sentry-uitest-android/proguard-rules.pro similarity index 100% rename from sentry-uitest/sentry-uitest-android/proguard-rules.pro rename to sentry-android-integration-tests/sentry-uitest-android/proguard-rules.pro diff --git a/sentry-uitest/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/BaseUiTest.kt b/sentry-android-integration-tests/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/BaseUiTest.kt similarity index 100% rename from sentry-uitest/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/BaseUiTest.kt rename to sentry-android-integration-tests/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/BaseUiTest.kt diff --git a/sentry-uitest/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/EnvelopeTests.kt b/sentry-android-integration-tests/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/EnvelopeTests.kt similarity index 100% rename from sentry-uitest/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/EnvelopeTests.kt rename to sentry-android-integration-tests/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/EnvelopeTests.kt diff --git a/sentry-uitest/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/mockservers/EnvelopeAsserter.kt b/sentry-android-integration-tests/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/mockservers/EnvelopeAsserter.kt similarity index 100% rename from sentry-uitest/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/mockservers/EnvelopeAsserter.kt rename to sentry-android-integration-tests/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/mockservers/EnvelopeAsserter.kt diff --git a/sentry-uitest/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/mockservers/MockRelay.kt b/sentry-android-integration-tests/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/mockservers/MockRelay.kt similarity index 100% rename from sentry-uitest/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/mockservers/MockRelay.kt rename to sentry-android-integration-tests/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/mockservers/MockRelay.kt diff --git a/sentry-uitest/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/mockservers/RelayAsserter.kt b/sentry-android-integration-tests/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/mockservers/RelayAsserter.kt similarity index 100% rename from sentry-uitest/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/mockservers/RelayAsserter.kt rename to sentry-android-integration-tests/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/mockservers/RelayAsserter.kt diff --git a/sentry-uitest/sentry-uitest-android/src/main/AndroidManifest.xml b/sentry-android-integration-tests/sentry-uitest-android/src/main/AndroidManifest.xml similarity index 100% rename from sentry-uitest/sentry-uitest-android/src/main/AndroidManifest.xml rename to sentry-android-integration-tests/sentry-uitest-android/src/main/AndroidManifest.xml diff --git a/sentry-uitest/sentry-uitest-android/src/main/java/io/sentry/uitest/android/EmptyActivity.kt b/sentry-android-integration-tests/sentry-uitest-android/src/main/java/io/sentry/uitest/android/EmptyActivity.kt similarity index 100% rename from sentry-uitest/sentry-uitest-android/src/main/java/io/sentry/uitest/android/EmptyActivity.kt rename to sentry-android-integration-tests/sentry-uitest-android/src/main/java/io/sentry/uitest/android/EmptyActivity.kt diff --git a/sentry-uitest/sentry-uitest-android/src/main/java/io/sentry/uitest/android/utils/BooleanIdlingResource.kt b/sentry-android-integration-tests/sentry-uitest-android/src/main/java/io/sentry/uitest/android/utils/BooleanIdlingResource.kt similarity index 100% rename from sentry-uitest/sentry-uitest-android/src/main/java/io/sentry/uitest/android/utils/BooleanIdlingResource.kt rename to sentry-android-integration-tests/sentry-uitest-android/src/main/java/io/sentry/uitest/android/utils/BooleanIdlingResource.kt diff --git a/settings.gradle.kts b/settings.gradle.kts index f38e71c378d..5fe49492cae 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -37,8 +37,8 @@ include( "sentry-samples:sentry-samples-spring-boot", "sentry-samples:sentry-samples-spring-boot-webflux", "sentry-samples:sentry-samples-netflix-dgs", - "sentry-uitest:sentry-uitest-android-benchmark", - "sentry-uitest:sentry-uitest-android" + "sentry-android-integration-tests:sentry-uitest-android-benchmark", + "sentry-android-integration-tests:sentry-uitest-android" ) gradle.beforeProject { From 6af1f1c7498e2b11629ef097ef4b19a9ccadc0eb Mon Sep 17 00:00:00 2001 From: stefanosiano Date: Tue, 24 May 2022 15:01:15 +0200 Subject: [PATCH 20/22] moved EnvelopeAsserter.assertItem to assertions.kt as assertEnvelopeItem assertEnvelopeItem is now a --- .../sentry-uitest-android/build.gradle.kts | 1 + .../io/sentry/uitest/android/EnvelopeTests.kt | 6 +++--- .../android/mockservers/EnvelopeAsserter.kt | 17 ++++++----------- .../src/main/kotlin/io/sentry/assertions.kt | 13 +++++++++++++ 4 files changed, 23 insertions(+), 14 deletions(-) diff --git a/sentry-android-integration-tests/sentry-uitest-android/build.gradle.kts b/sentry-android-integration-tests/sentry-uitest-android/build.gradle.kts index ddeafeaf5ef..75d95d5bde6 100644 --- a/sentry-android-integration-tests/sentry-uitest-android/build.gradle.kts +++ b/sentry-android-integration-tests/sentry-uitest-android/build.gradle.kts @@ -94,6 +94,7 @@ dependencies { errorprone(Config.CompileOnly.errorprone) errorprone(Config.CompileOnly.errorProneNullAway) + androidTestImplementation(projects.sentryTestSupport) androidTestImplementation(Config.TestLibs.kotlinTestJunit) androidTestImplementation(Config.TestLibs.espressoCore) androidTestImplementation(Config.TestLibs.androidxRunner) diff --git a/sentry-android-integration-tests/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/EnvelopeTests.kt b/sentry-android-integration-tests/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/EnvelopeTests.kt index 54c544e260c..df4bf05c71a 100644 --- a/sentry-android-integration-tests/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/EnvelopeTests.kt +++ b/sentry-android-integration-tests/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/EnvelopeTests.kt @@ -23,7 +23,7 @@ class EnvelopeTests : BaseUiTest() { relay.assert { assertEnvelope { - val event = it.assertItem(SentryEvent::class.java) + val event: SentryEvent = it.assertItem() it.assertNoOtherItems() assertTrue(event.message?.formatted == "Message captured during test") } @@ -45,8 +45,8 @@ class EnvelopeTests : BaseUiTest() { transaction.finish() relay.assert { assertEnvelope { - val transactionItem = it.assertItem(SentryTransaction::class.java) - val profilingTraceData = it.assertItem(ProfilingTraceData::class.java) + val transactionItem: SentryTransaction = it.assertItem() + val profilingTraceData: ProfilingTraceData = it.assertItem() it.assertNoOtherItems() assertTrue(transactionItem.transaction == "e2etests") assertEquals(profilingTraceData.transactionId, transactionItem.eventId.toString()) diff --git a/sentry-android-integration-tests/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/mockservers/EnvelopeAsserter.kt b/sentry-android-integration-tests/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/mockservers/EnvelopeAsserter.kt index 5da63a806fa..7273531aeea 100644 --- a/sentry-android-integration-tests/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/mockservers/EnvelopeAsserter.kt +++ b/sentry-android-integration-tests/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/mockservers/EnvelopeAsserter.kt @@ -1,7 +1,7 @@ package io.sentry.uitest.android.mockservers -import io.sentry.Sentry import io.sentry.SentryEnvelope +import io.sentry.assertEnvelopeItem import okhttp3.mockwebserver.MockResponse /** @@ -10,20 +10,15 @@ import okhttp3.mockwebserver.MockResponse */ class EnvelopeAsserter(val envelope: SentryEnvelope, val response: MockResponse) { /** List of items to assert. */ - private val unassertedItems = envelope.items.toMutableList() + val unassertedItems = envelope.items.toMutableList() /** - * Asserts an envelope item that can be deserialized as an instance of [clazz] exists and returns the first one. + * Asserts an envelope item of [T] exists and returns the first one. * The asserted item is then removed from internal list of unasserted items. */ - fun assertItem(clazz: Class): T { - val item = unassertedItems.mapIndexed { index, it -> - val deserialized = Sentry.getCurrentHub().options.serializer.deserialize(it.data.inputStream().reader(), clazz) - deserialized?.let { Pair(index, it) } - }.filterNotNull().firstOrNull() - ?: throw AssertionError("No item found of type: ${clazz.name}") - unassertedItems.removeAt(item.first) - return item.second + inline fun assertItem(): T = assertEnvelopeItem(unassertedItems) { index, item -> + unassertedItems.removeAt(index) + return item } /** Asserts there are no other items in the envelope. */ diff --git a/sentry-test-support/src/main/kotlin/io/sentry/assertions.kt b/sentry-test-support/src/main/kotlin/io/sentry/assertions.kt index d1054fbec22..2bffce6eae6 100644 --- a/sentry-test-support/src/main/kotlin/io/sentry/assertions.kt +++ b/sentry-test-support/src/main/kotlin/io/sentry/assertions.kt @@ -29,6 +29,19 @@ fun checkTransaction(predicate: (SentryTransaction) -> Unit): SentryEnvelope { } } +/** + * Asserts an envelope item of [T] exists in [items] and returns the first one. Otherwise it throws an [AssertionError]. + */ +inline fun assertEnvelopeItem(items: List, predicate: (index: Int, item: T) -> Unit): T { + val item = items.mapIndexedNotNull { index, it -> + val deserialized = JsonSerializer(SentryOptions()).deserialize(it.data.inputStream().reader(), T::class.java) + deserialized?.let { Pair(index, it) } + }.firstOrNull() + ?: throw AssertionError("No item found of type: ${T::class.java.name}") + predicate(item.first, item.second) + return item.second +} + /** * Modified version of check from mockito-kotlin Verification.kt, that does not print errors of type `SkipError`. */ From 695eb2fa85a2913fd5f8894a881809c0db32cb80 Mon Sep 17 00:00:00 2001 From: stefanosiano Date: Wed, 25 May 2022 11:39:31 +0200 Subject: [PATCH 21/22] moved few prints to comments in BenchmarkOperationResult --- .../util/BenchmarkOperationResult.kt | 29 +++++++++---------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/util/BenchmarkOperationResult.kt b/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/util/BenchmarkOperationResult.kt index 764235f945b..47a954c49bc 100644 --- a/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/util/BenchmarkOperationResult.kt +++ b/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/util/BenchmarkOperationResult.kt @@ -18,10 +18,6 @@ internal data class BenchmarkOperationResult( // Measure average duration val durationDiffNanos = avgDurationNanos - other.avgDurationNanos val durationIncreasePercentage = durationDiffNanos * 100.0 / other.avgDurationNanos - println( - "Measuring the duration increase of the operation in the transaction. If it's low enough, " + - "no end user will ever realize it" - ) println("[${other.operationName}] Average duration: ${other.avgDurationNanos} ns") println("[$operationName] Average duration: $avgDurationNanos ns") if (durationIncreasePercentage > 0) { @@ -37,10 +33,6 @@ internal data class BenchmarkOperationResult( val cpuTimeDiff = (avgCpuTimeMillis - other.avgCpuTimeMillis) / cores val cpuTimeOverheadPercentage = cpuTimeDiff * 100.0 / other.avgCpuTimeMillis // Cpu time spent profiling is weighted based on available threads, as profiling runs on 1 thread only. - println( - "Measuring the increased cpu time. It has no direct impact on performance of the app, " + - "but it has on battery usage, as the cpu is 'awaken' longer." - ) println("The weighted difference of cpu times is $cpuTimeDiff ms (over $cores available cores).") println("[${other.operationName}] Cpu time: ${other.avgCpuTimeMillis} ms") println("[$operationName] Cpu time: $avgCpuTimeMillis ms") @@ -55,10 +47,6 @@ internal data class BenchmarkOperationResult( // Measure average fps val fpsDiff = other.avgFramesPerSecond - avgFramesPerSecond val fpsDecreasePercentage = fpsDiff * 100.0 / other.avgFramesPerSecond - println( - "Measuring the decreased fps. Not really important, as even if fps are the same, the cpu could be " + - "doing more work in the frame window, and it could be hidden by checking average fps only." - ) println("[${other.operationName}] Average FPS: ${other.avgFramesPerSecond}") println("[$operationName] Average FPS: $avgFramesPerSecond") if (fpsDecreasePercentage > 0) { @@ -73,10 +61,6 @@ internal data class BenchmarkOperationResult( val droppedFramesDiff = avgDroppedFrames - other.avgDroppedFrames val totalExpectedFrames = TimeUnit.NANOSECONDS.toMillis(other.avgDurationNanos) * 60 / 1000 val droppedFramesIncreasePercentage = droppedFramesDiff * 100 / (totalExpectedFrames - other.avgDroppedFrames) - println( - "Measuring the increased fps drop rate. Very important, as it weights dropped frames based on the " + - "time passed between each frame. This is the metric end users can perceive as 'performance' in app usage." - ) println("Dropped frames are calculated based on a target of 60 frames per second ($totalExpectedFrames total frames).") println("[${other.operationName}] Average dropped frames: ${other.avgDroppedFrames}") println("[$operationName] Average dropped frames: $avgDroppedFrames") @@ -96,8 +80,21 @@ internal data class BenchmarkOperationResult( } internal data class BenchmarkResult( + /** + * Increase of cpu time in percentage. + * It has no direct impact on performance of the app, but it has on battery usage, as the cpu is 'awaken' longer. + */ val cpuTimeIncrease: Double, + /** + * Increase of dropped frames in percentage.Very important, as it weights dropped frames based on the time + * passed between each frame. This is the metric end users can perceive as 'performance' in app usage. + */ val droppedFramesIncrease: Double, + /** Increase of duration in percentage. If it's low enough, no end user will ever realize it. */ val durationIncrease: Double, + /** + * Decrease of fps in percentage. Not really important, as even if fps are the same, the cpu could be + * doing more work in the frame window, and it could be hidden by checking average fps only. + */ val fpsDecrease: Double ) From b3ae6f69b917edefa10f6e977f2853513ce70fe1 Mon Sep 17 00:00:00 2001 From: stefanosiano Date: Thu, 26 May 2022 10:58:52 +0200 Subject: [PATCH 22/22] sentry-uitest-android-benchmark now depends on sentry-uitest-android BooleanIdlingResource is now only in sentry-uitest-android --- .../build.gradle.kts | 1 + .../src/main/AndroidManifest.xml | 3 +- .../android/benchmark/BenchmarkActivity.kt | 1 + .../benchmark/BooleanIdlingResource.kt | 30 ------------------- 4 files changed, 4 insertions(+), 31 deletions(-) delete mode 100644 sentry-android-integration-tests/sentry-uitest-android-benchmark/src/main/java/io/sentry/uitest/android/benchmark/BooleanIdlingResource.kt diff --git a/sentry-android-integration-tests/sentry-uitest-android-benchmark/build.gradle.kts b/sentry-android-integration-tests/sentry-uitest-android-benchmark/build.gradle.kts index ed39396aa51..0ee897b3b0d 100644 --- a/sentry-android-integration-tests/sentry-uitest-android-benchmark/build.gradle.kts +++ b/sentry-android-integration-tests/sentry-uitest-android-benchmark/build.gradle.kts @@ -88,6 +88,7 @@ dependencies { implementation(kotlin(Config.kotlinStdLib, org.jetbrains.kotlin.config.KotlinCompilerVersion.VERSION)) + implementation(projects.sentryAndroidIntegrationTests.sentryUitestAndroid) implementation(projects.sentryAndroid) implementation(Config.Libs.appCompat) implementation(Config.Libs.androidxCore) diff --git a/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/main/AndroidManifest.xml b/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/main/AndroidManifest.xml index fa2c6feb143..3c1d9e2de5d 100644 --- a/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/main/AndroidManifest.xml +++ b/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/main/AndroidManifest.xml @@ -12,7 +12,8 @@ + android:icon="@android:mipmap/sym_def_app_icon" + tools:replace="android:label"> diff --git a/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/main/java/io/sentry/uitest/android/benchmark/BenchmarkActivity.kt b/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/main/java/io/sentry/uitest/android/benchmark/BenchmarkActivity.kt index dcf75c0c78e..5a33232bb1f 100644 --- a/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/main/java/io/sentry/uitest/android/benchmark/BenchmarkActivity.kt +++ b/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/main/java/io/sentry/uitest/android/benchmark/BenchmarkActivity.kt @@ -5,6 +5,7 @@ import androidx.appcompat.app.AppCompatActivity import androidx.recyclerview.widget.LinearLayoutManager import androidx.recyclerview.widget.RecyclerView import io.sentry.uitest.android.benchmark.databinding.ActivityBenchmarkBinding +import io.sentry.uitest.android.utils.BooleanIdlingResource import java.util.concurrent.ExecutorService import java.util.concurrent.Executors diff --git a/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/main/java/io/sentry/uitest/android/benchmark/BooleanIdlingResource.kt b/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/main/java/io/sentry/uitest/android/benchmark/BooleanIdlingResource.kt deleted file mode 100644 index edd7d78b618..00000000000 --- a/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/main/java/io/sentry/uitest/android/benchmark/BooleanIdlingResource.kt +++ /dev/null @@ -1,30 +0,0 @@ -package io.sentry.uitest.android.benchmark - -import androidx.test.espresso.IdlingResource -import java.util.concurrent.atomic.AtomicBoolean - -/** Idling resource based on a boolean flag. */ -class BooleanIdlingResource(private val name: String) : IdlingResource { - - private val isIdle = AtomicBoolean(true) - - private val isIdleLock = Object() - - private var callback: IdlingResource.ResourceCallback? = null - - /** Sets whether this resource is currently in idle state. */ - fun setIdle(idling: Boolean) { - if (!isIdle.getAndSet(idling) && idling) { - callback?.onTransitionToIdle() - } - } - - override fun getName(): String = name - - /** Check if this resource is currently in idle state. */ - override fun isIdleNow(): Boolean = synchronized(isIdleLock) { isIdle.get() } - - override fun registerIdleTransitionCallback(callback: IdlingResource.ResourceCallback?) { - this.callback = callback - } -}