diff --git a/.gitignore b/.gitignore index 03d9c5ea4bb..93b00e96cd6 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ .idea/ .gradle/ build/ +artifacts/ out/ local.properties **.iml diff --git a/.sauce/sentry-uitest-android-benchmark.yml b/.sauce/sentry-uitest-android-benchmark.yml new file mode 100644 index 00000000000..35bce15f84e --- /dev/null +++ b/.sauce/sentry-uitest-android-benchmark.yml @@ -0,0 +1,29 @@ +apiVersion: v1alpha +kind: espresso +sauce: + region: us-west-1 + # Controls how many suites are executed at the same time (sauce test env only). + concurrency: 1 + metadata: + name: Android benchmarks with Espresso + tags: + - benchmarks + - android +espresso: + app: ./sentry-android-integration-tests/sentry-uitest-android-benchmark/build/outputs/apk/release/sentry-uitest-android-benchmark-release.apk + testApp: ./sentry-android-integration-tests/sentry-uitest-android-benchmark/build/outputs/apk/androidTest/release/sentry-uitest-android-benchmark-release-androidTest.apk +suites: + name: "Android Benchmarks" + devices: + - name: "Google Pixel 2" + platformVersion: 11 + - id: Google_Pixel_2_real_us + testOptions: + useTestOrchestrator: true +# Controls what artifacts to fetch when the suite on Sauce Cloud has finished. +artifacts: + download: + when: always + match: + - junit.xml + directory: ./artifacts/ diff --git a/.sauce/sentry-uitest-android-end2end.yml b/.sauce/sentry-uitest-android-end2end.yml new file mode 100644 index 00000000000..62178a4b0dc --- /dev/null +++ b/.sauce/sentry-uitest-android-end2end.yml @@ -0,0 +1,31 @@ +apiVersion: v1alpha +kind: espresso +sauce: + region: us-west-1 + # Controls how many suites are executed at the same time (sauce test env only). + concurrency: 1 + metadata: + name: Android end2end tests with Espresso + tags: + - e2e + - android +espresso: + app: ./sentry-android-integration-tests/sentry-uitest-android/build/outputs/apk/release/sentry-uitest-android-release.apk + testApp: ./sentry-android-integration-tests/sentry-uitest-android/build/outputs/apk/androidTest/release/sentry-uitest-android-release-androidTest.apk +suites: + name: "Android End2end" + emulators: + - name: "Android GoogleApi Emulator" + orientation: portrait + platformVersions: + - "11.0" + - "10.0" + testOptions: + useTestOrchestrator: true +# Controls what artifacts to fetch when the suite on Sauce Cloud has finished. +artifacts: + download: + when: always + match: + - junit.xml + directory: ./artifacts/ diff --git a/build.gradle.kts b/build.gradle.kts index 2d210a4bc94..278b2a94226 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -56,6 +56,8 @@ apiValidation { "sentry-samples-spring-boot", "sentry-samples-spring-boot-webflux", "sentry-samples-netflix-dgs", + "sentry-uitest-android", + "sentry-uitest-android-benchmark", ) ) } @@ -86,7 +88,7 @@ allprojects { } subprojects { - if (!this.name.contains("sample") && this.name != "sentry-test-support") { + if (!this.name.contains("sample") && !this.name.contains("integration-tests") && this.name != "sentry-test-support") { apply() val sep = File.separator @@ -165,7 +167,7 @@ gradle.projectsEvaluated { "https://docs.spring.io/spring-boot/docs/current/api/" ) subprojects - .filter { !it.name.contains("sample") } + .filter { !it.name.contains("sample") && !it.name.contains("integration-tests") } .forEach { proj -> proj.tasks.withType().forEach { javadocTask -> source += javadocTask.source diff --git a/buildSrc/src/main/java/Config.kt b/buildSrc/src/main/java/Config.kt index 29adc89d428..fbab16d8b63 100644 --- a/buildSrc/src/main/java/Config.kt +++ b/buildSrc/src/main/java/Config.kt @@ -39,11 +39,12 @@ object Config { object Libs { val okHttpVersion = "4.9.2" - val appCompat = "androidx.appcompat:appcompat:1.2.0" + val appCompat = "androidx.appcompat:appcompat:1.3.0" val timber = "com.jakewharton.timber:timber:4.7.1" val okhttpBom = "com.squareup.okhttp3:okhttp-bom:$okHttpVersion" val okhttp = "com.squareup.okhttp3:okhttp" val leakCanary = "com.squareup.leakcanary:leakcanary-android:2.8.1" + val constraintLayout = "androidx.constraintlayout:constraintlayout:2.1.3" private val lifecycleVersion = "2.2.0" val lifecycleProcess = "androidx.lifecycle:lifecycle-process:$lifecycleVersion" @@ -110,12 +111,19 @@ object Config { } object TestLibs { - private val androidxTestVersion = "1.4.0-rc01" + private val androidxTestVersion = "1.4.0" + private val espressoVersion = "3.4.0" + val androidJUnitRunner = "androidx.test.runner.AndroidJUnitRunner" val kotlinTestJunit = "org.jetbrains.kotlin:kotlin-test-junit:$kotlinVersion" val androidxCore = "androidx.test:core:$androidxTestVersion" val androidxRunner = "androidx.test:runner:$androidxTestVersion" - val androidxJunit = "androidx.test.ext:junit:1.1.3-rc01" + val androidxTestCoreKtx = "androidx.test:core-ktx:$androidxTestVersion" + val androidxTestRules = "androidx.test:rules:$androidxTestVersion" + val espressoCore = "androidx.test.espresso:espresso-core:$espressoVersion" + val espressoIdlingResource = "androidx.test.espresso:espresso-idling-resource:$espressoVersion" + val androidxTestOrchestrator = "androidx.test:orchestrator:1.4.1" + val androidxJunit = "androidx.test.ext:junit:1.1.3" val androidxCoreKtx = "androidx.core:core-ktx:1.7.0" val robolectric = "org.robolectric:robolectric:4.7.3" val mockitoKotlin = "com.nhaarman.mockitokotlin2:mockito-kotlin:2.2.0" diff --git a/sentry-android-core/build.gradle.kts b/sentry-android-core/build.gradle.kts index 7937655aad1..73243b80fa9 100644 --- a/sentry-android-core/build.gradle.kts +++ b/sentry-android-core/build.gradle.kts @@ -16,7 +16,7 @@ android { targetSdk = Config.Android.targetSdkVersion minSdk = Config.Android.minSdkVersion - testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner" + testInstrumentationRunner = Config.TestLibs.androidJUnitRunner buildConfigField("String", "SENTRY_ANDROID_SDK_NAME", "\"${Config.Sentry.SENTRY_ANDROID_SDK_NAME}\"") diff --git a/sentry-android-integration-tests/sentry-uitest-android-benchmark/.gitignore b/sentry-android-integration-tests/sentry-uitest-android-benchmark/.gitignore new file mode 100644 index 00000000000..796b96d1c40 --- /dev/null +++ b/sentry-android-integration-tests/sentry-uitest-android-benchmark/.gitignore @@ -0,0 +1 @@ +/build diff --git a/sentry-android-integration-tests/sentry-uitest-android-benchmark/benchmark-proguard-rules.pro b/sentry-android-integration-tests/sentry-uitest-android-benchmark/benchmark-proguard-rules.pro new file mode 100644 index 00000000000..8f5e14b25e7 --- /dev/null +++ b/sentry-android-integration-tests/sentry-uitest-android-benchmark/benchmark-proguard-rules.pro @@ -0,0 +1,35 @@ +# Add project specific ProGuard rules here. +# You can control the set of applied configuration files using the +# proguardFiles setting in build.gradle.kts. +# +# For more details, see +# http://developer.android.com/guide/developing/tools/proguard.html + +# If your project uses WebView with JS, uncomment the following +# and specify the fully qualified class name to the JavaScript interface +# class: +#-keepclassmembers class fqcn.of.javascript.interface.for.webview { +# public *; +#} + +# Uncomment this to preserve the line number information for +# debugging stack traces. +#-keepattributes SourceFile,LineNumberTable + +# If you keep the line number information, uncomment this to +# hide the original source file name. +#-renamesourcefileattribute SourceFile + +-dontobfuscate +#Shrinking removes annotations and "unused classes" from test apk, so we don't shrink +-dontshrink + +-ignorewarnings + +-keepattributes *Annotation* + +-dontnote junit.framework.** +-dontnote junit.runner.** + +-dontwarn androidx.test.** +-dontwarn org.junit.** diff --git a/sentry-android-integration-tests/sentry-uitest-android-benchmark/build.gradle.kts b/sentry-android-integration-tests/sentry-uitest-android-benchmark/build.gradle.kts new file mode 100644 index 00000000000..0ee897b3b0d --- /dev/null +++ b/sentry-android-integration-tests/sentry-uitest-android-benchmark/build.gradle.kts @@ -0,0 +1,133 @@ +import io.gitlab.arturbosch.detekt.Detekt +import io.gitlab.arturbosch.detekt.extensions.DetektExtension +import net.ltgt.gradle.errorprone.errorprone + +plugins { + id("com.android.application") + kotlin("android") + id(Config.QualityPlugins.errorProne) + id(Config.QualityPlugins.gradleVersions) + id(Config.QualityPlugins.detektPlugin) +} + +android { + compileSdk = Config.Android.compileSdkVersion + + defaultConfig { + applicationId = "io.sentry.uitest.android.benchmark" + minSdk = Config.Android.minSdkVersionNdk + targetSdk = Config.Android.targetSdkVersion + versionCode = 1 + versionName = "1.0.0" + + testInstrumentationRunner = Config.TestLibs.androidJUnitRunner + // Runs each test in its own instance of Instrumentation. This way they are isolated from + // one another and get their own Application instance. + // https://developer.android.com/training/testing/instrumented-tests/androidx-test-libraries/runner#enable-gradle + // This doesn't work on some devices with Android 11+. Clearing package data resets permissions. + // Check the readme for more info. +// testInstrumentationRunnerArguments["clearPackageData"] = "true" + } + + buildFeatures { + // Determines whether to support View Binding. + // Note that the viewBinding.enabled property is now deprecated. + viewBinding = true + } + + testOptions { + execution = "ANDROIDX_TEST_ORCHESTRATOR" + } + + signingConfigs { + getByName("debug") { + storeFile = rootProject.file("debug.keystore") + storePassword = "android" + keyAlias = "androiddebugkey" + keyPassword = "android" + } + } + + testBuildType = System.getProperty("testBuildType", "debug") + + buildTypes { + getByName("debug") { + isDebuggable = false + isMinifyEnabled = true + signingConfig = signingConfigs.getByName("debug") + proguardFiles(getDefaultProguardFile("proguard-android-optimize.txt"), "benchmark-proguard-rules.pro") + } + getByName("release") { + isMinifyEnabled = true + isShrinkResources = true + signingConfig = signingConfigs.getByName("debug") // to be able to run release mode + proguardFiles(getDefaultProguardFile("proguard-android-optimize.txt"), "benchmark-proguard-rules.pro") + } + } + + kotlinOptions { + jvmTarget = JavaVersion.VERSION_1_8.toString() + } + + lint { + warningsAsErrors = true + checkDependencies = true + + // We run a full lint analysis as build part in CI, so skip vital checks for assemble tasks. + checkReleaseBuilds = false + } + + variantFilter { + if (Config.Android.shouldSkipDebugVariant(buildType.name)) { + ignore = true + } + } +} + +dependencies { + + implementation(kotlin(Config.kotlinStdLib, org.jetbrains.kotlin.config.KotlinCompilerVersion.VERSION)) + + implementation(projects.sentryAndroidIntegrationTests.sentryUitestAndroid) + implementation(projects.sentryAndroid) + implementation(Config.Libs.appCompat) + implementation(Config.Libs.androidxCore) + implementation(Config.Libs.androidxRecylerView) + implementation(Config.Libs.constraintLayout) + implementation(Config.TestLibs.espressoIdlingResource) + + compileOnly(Config.CompileOnly.nopen) + errorprone(Config.CompileOnly.nopenChecker) + errorprone(Config.CompileOnly.errorprone) + errorprone(Config.CompileOnly.errorProneNullAway) + + androidTestImplementation(Config.TestLibs.kotlinTestJunit) + androidTestImplementation(Config.TestLibs.espressoCore) + androidTestImplementation(Config.TestLibs.androidxTestCoreKtx) + androidTestImplementation(Config.TestLibs.androidxRunner) + androidTestImplementation(Config.TestLibs.androidxTestRules) + androidTestImplementation(Config.TestLibs.androidxJunit) + androidTestUtil(Config.TestLibs.androidxTestOrchestrator) +} + +tasks.withType().configureEach { + options.errorprone { + check("NullAway", net.ltgt.gradle.errorprone.CheckSeverity.ERROR) + option("NullAway:AnnotatedPackages", "io.sentry") + option("NullAway:UnannotatedSubPackages", "io.sentry.uitest.android.benchmark.databinding") + } +} + +tasks.withType { + // Target version of the generated JVM bytecode. It is used for type resolution. + jvmTarget = JavaVersion.VERSION_1_8.toString() +} + +configure { + buildUponDefaultConfig = true + allRules = true +} + +kotlin { + explicitApi() +} diff --git a/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/SentryBenchmarkTest.kt b/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/SentryBenchmarkTest.kt new file mode 100644 index 00000000000..013fbccf659 --- /dev/null +++ b/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/SentryBenchmarkTest.kt @@ -0,0 +1,135 @@ +package io.sentry.uitest.android.benchmark + +import android.content.Context +import android.view.Choreographer +import androidx.lifecycle.Lifecycle +import androidx.test.core.app.ApplicationProvider +import androidx.test.core.app.launchActivity +import androidx.test.espresso.Espresso +import androidx.test.espresso.Espresso.onView +import androidx.test.espresso.IdlingRegistry +import androidx.test.espresso.action.ViewActions.swipeUp +import androidx.test.espresso.matcher.ViewMatchers.withId +import androidx.test.ext.junit.runners.AndroidJUnit4 +import androidx.test.platform.app.InstrumentationRegistry +import androidx.test.runner.AndroidJUnitRunner +import io.sentry.ITransaction +import io.sentry.Sentry +import io.sentry.SentryOptions +import io.sentry.android.core.SentryAndroid +import io.sentry.uitest.android.benchmark.util.BenchmarkOperation +import org.junit.runner.RunWith +import kotlin.test.AfterTest +import kotlin.test.BeforeTest +import kotlin.test.Test +import kotlin.test.assertTrue + +@RunWith(AndroidJUnit4::class) +class SentryBenchmarkTest { + + private lateinit var runner: AndroidJUnitRunner + private lateinit var context: Context + private lateinit var choreographer: Choreographer + + @BeforeTest + fun setUp() { + runner = InstrumentationRegistry.getInstrumentation() as AndroidJUnitRunner + context = ApplicationProvider.getApplicationContext() + context.cacheDir.deleteRecursively() + IdlingRegistry.getInstance().register(BenchmarkActivity.scrollingIdlingResource) + // Must run on the main thread to get the main thread choreographer. + runner.runOnMainSync { + choreographer = Choreographer.getInstance() + } + } + + @AfterTest + fun cleanup() { + IdlingRegistry.getInstance().unregister(BenchmarkActivity.scrollingIdlingResource) + } + + @Test + fun benchmarkSameOperation() { + + // We compare two operation that are the same. We expect the increases to be negligible, as the results + // should be very similar. + val op1 = BenchmarkOperation(choreographer, getOperation(runner)) + val op2 = BenchmarkOperation(choreographer, getOperation(runner)) + val comparisonResult = BenchmarkOperation.compare(op1, "Op1", op2, "Op2") + + assertTrue(comparisonResult.durationIncrease in -1F..1F) + assertTrue(comparisonResult.cpuTimeIncrease in -1F..1F) + // The fps decrease comparison is skipped, due to approximation: 59.51 and 59.49 fps are considered 60 and 59, + // respectively. Also, if the average fps is 20 or 60, a difference of 1 fps becomes 5% or 1.66% respectively. + assertTrue(comparisonResult.droppedFramesIncrease in -1F..1F) + } + + @Test + fun benchmarkProfiledTransaction() { + + runner.runOnMainSync { + SentryAndroid.init(context) { options: SentryOptions -> + options.dsn = "https://key@uri/1234567" + options.tracesSampleRate = 1.0 + options.isProfilingEnabled = true + } + } + + // We compare the same operation with and without profiled transaction. + // We expect the profiled transaction operation to be slower, but not slower than 5%. + val benchmarkOperationNoTransaction = BenchmarkOperation(choreographer, getOperation(runner)) + val benchmarkOperationProfiled = BenchmarkOperation( + choreographer, + getOperation(runner) { + Sentry.startTransaction("Benchmark", "ProfiledTransaction") + } + ) + val comparisonResult = BenchmarkOperation.compare( + benchmarkOperationNoTransaction, + "NoTransaction", + benchmarkOperationProfiled, + "ProfiledTransaction" + ) + + runner.runOnMainSync { + Sentry.close() + } + + assertTrue(comparisonResult.durationIncrease in 0F..5F) + assertTrue(comparisonResult.cpuTimeIncrease in 0F..5F) + assertTrue(comparisonResult.fpsDecrease in 0F..5F) + assertTrue(comparisonResult.droppedFramesIncrease in 0F..5F) + } + + /** + * Operation that will be compared: it launches [BenchmarkActivity], swipe the list and closes it. + * The [transactionBuilder] is used to create the transaction before the swipes. + */ + private fun getOperation(runner: AndroidJUnitRunner, transactionBuilder: () -> ITransaction? = { null }): () -> Unit = { + var transaction: ITransaction? = null + // Launch the sentry-uitest-android-benchmark activity + val benchmarkScenario = launchActivity() + // Starts a transaction (it can be null, but we still runOnMainSync to make operations as similar as possible) + runner.runOnMainSync { + transaction = transactionBuilder() + } + // Just swipe the list some times: this is the benchmarked operation + swipeList(2) + // We finish the transaction + runner.runOnMainSync { + transaction?.finish() + } + // We swipe a last time to measure how finishing the transaction may affect other operations + swipeList(1) + + benchmarkScenario.moveToState(Lifecycle.State.DESTROYED) + } + + private fun swipeList(times: Int) { + repeat(times) { + Thread.sleep(100) + onView(withId(R.id.benchmark_transaction_list)).perform(swipeUp()) + Espresso.onIdle() + } + } +} diff --git a/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/util/BenchmarkOperation.kt b/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/util/BenchmarkOperation.kt new file mode 100644 index 00000000000..72a71e7cb7e --- /dev/null +++ b/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/util/BenchmarkOperation.kt @@ -0,0 +1,128 @@ +package io.sentry.uitest.android.benchmark.util + +import android.os.Process +import android.os.SystemClock +import android.view.Choreographer +import java.util.concurrent.TimeUnit + +// 60 FPS is the recommended target: https://www.youtube.com/watch?v=CaMTIgxCSqU +private const val FRAME_DURATION_60FPS_NS: Double = 1_000_000_000 / 60.0 + +/** + * Class that allows to benchmark some operations. + * Create two [BenchmarkOperation] objects and compare them using [BenchmarkOperation.compare] to get + * a [BenchmarkResult] with relative measured overheads. + */ +internal class BenchmarkOperation(private val choreographer: Choreographer, private val op: () -> Unit) { + + companion object { + + /** + * Running two operations sequentially (running 10 times the first and then 10 times the second) results in the + * first operation to always be slower, so comparing two different operations on equal terms is not possible. + * This method runs [op1] and [op2] in an alternating sequence. + * When [op1] and [op2] are the same, we get (nearly) identical results, as expected. + * You can adjust [warmupIterations] and [measuredIterations]. The lower they are, the faster the benchmark, + * but accuracy decreases. + */ + fun compare( + op1: BenchmarkOperation, + op1Name: String, + op2: BenchmarkOperation, + op2Name: String, + warmupIterations: Int = 3, + measuredIterations: Int = 15 + ): BenchmarkResult { + // The first operations are the slowest, as the device is still doing things like filling the cache. + repeat(warmupIterations) { + op1.warmup() + op2.warmup() + } + // Now we can measure the operations (in alternating sequence). + repeat(measuredIterations) { + op1.iterate() + op2.iterate() + } + val op1Result = op1.getResult(op1Name) + val op2Result = op2.getResult(op2Name) + + // Let's print the raw results. + println("=====================================") + println(op1Name) + println(op1Result) + println("=====================================") + println(op2Name) + println(op2Result) + println("=====================================") + + return op2Result.compare(op1Result) + } + } + + private var iterations: Int = 0 + private var durationNanos: Long = 0 + private var cpuDurationMillis: Long = 0 + private var frames: Int = 0 + private var droppedFrames: Double = 0.0 + private var lastFrameTimeNanos: Long = 0 + + /** Run the operation without measuring it. */ + private fun warmup() { + op() + isolate() + } + + /** Run the operation and measure it, updating sentry-uitest-android-benchmark data. */ + private fun iterate() { + val startRealtimeNs = SystemClock.elapsedRealtimeNanos() + val startCpuTimeMs = Process.getElapsedCpuTime() + + lastFrameTimeNanos = startRealtimeNs + iterations++ + choreographer.postFrameCallback(frameCallback) + + op() + + choreographer.removeFrameCallback(frameCallback) + cpuDurationMillis += Process.getElapsedCpuTime() - startCpuTimeMs + durationNanos += SystemClock.elapsedRealtimeNanos() - startRealtimeNs + + isolate() + } + + /** Return the [BenchmarkOperationResult] for the operation. */ + private fun getResult(operationName: String): BenchmarkOperationResult = BenchmarkOperationResult( + cpuDurationMillis / iterations, + droppedFrames / iterations, + durationNanos / iterations, + // fps = counted frames per seconds converted into frames per nanoseconds, divided by duration in nanoseconds + // We don't convert the duration into seconds to avoid issues with rounding and possible division by 0 + (frames * TimeUnit.SECONDS.toNanos(1) / durationNanos).toInt(), + operationName + ) + + /** + * Helps ensure that operations don't impact one another. + * Doesn't appear to currently have an impact on the benchmark. + */ + private fun isolate() { + Thread.sleep(500) + Runtime.getRuntime().gc() + Thread.sleep(100) + } + + private val frameCallback = object : Choreographer.FrameCallback { + override fun doFrame(frameTimeNanos: Long) { + frames++ + val timeSinceLastFrameNanos = frameTimeNanos - lastFrameTimeNanos + if (timeSinceLastFrameNanos > FRAME_DURATION_60FPS_NS) { + // Fractions of frames dropped are weighted to improve the accuracy of the results. + // For example, 31ms between frames is much worse than 17ms, even though both + // durations are within the "1 frame dropped" range. + droppedFrames += timeSinceLastFrameNanos / FRAME_DURATION_60FPS_NS - 1 + } + lastFrameTimeNanos = frameTimeNanos + choreographer.postFrameCallback(this) + } + } +} diff --git a/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/util/BenchmarkOperationResult.kt b/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/util/BenchmarkOperationResult.kt new file mode 100644 index 00000000000..47a954c49bc --- /dev/null +++ b/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/util/BenchmarkOperationResult.kt @@ -0,0 +1,100 @@ +package io.sentry.uitest.android.benchmark.util + +import java.util.concurrent.TimeUnit + +/** Stores the results of a [BenchmarkOperation]. */ +internal data class BenchmarkOperationResult( + val avgCpuTimeMillis: Long, + val avgDroppedFrames: Double, + val avgDurationNanos: Long, + val avgFramesPerSecond: Int, + val operationName: String +) { + /** + * Compare two [BenchmarkOperationResult], calculating increases of each parameter in percentage. + */ + fun compare(other: BenchmarkOperationResult): BenchmarkResult { + + // Measure average duration + val durationDiffNanos = avgDurationNanos - other.avgDurationNanos + val durationIncreasePercentage = durationDiffNanos * 100.0 / other.avgDurationNanos + println("[${other.operationName}] Average duration: ${other.avgDurationNanos} ns") + println("[$operationName] Average duration: $avgDurationNanos ns") + if (durationIncreasePercentage > 0) { + println("Duration increase: %.2f%%".format(durationIncreasePercentage)) + } else { + println("No measurable duration increase detected.") + } + + println("--------------------") + + // Measure average cpu time + val cores = Runtime.getRuntime().availableProcessors() + val cpuTimeDiff = (avgCpuTimeMillis - other.avgCpuTimeMillis) / cores + val cpuTimeOverheadPercentage = cpuTimeDiff * 100.0 / other.avgCpuTimeMillis + // Cpu time spent profiling is weighted based on available threads, as profiling runs on 1 thread only. + println("The weighted difference of cpu times is $cpuTimeDiff ms (over $cores available cores).") + println("[${other.operationName}] Cpu time: ${other.avgCpuTimeMillis} ms") + println("[$operationName] Cpu time: $avgCpuTimeMillis ms") + if (cpuTimeOverheadPercentage > 0) { + println("CPU time overhead: %.2f%%".format(cpuTimeOverheadPercentage)) + } else { + println("No measurable CPU time overhead detected.") + } + + println("--------------------") + + // Measure average fps + val fpsDiff = other.avgFramesPerSecond - avgFramesPerSecond + val fpsDecreasePercentage = fpsDiff * 100.0 / other.avgFramesPerSecond + println("[${other.operationName}] Average FPS: ${other.avgFramesPerSecond}") + println("[$operationName] Average FPS: $avgFramesPerSecond") + if (fpsDecreasePercentage > 0) { + println("FPS decrease: %.2f%%".format(fpsDecreasePercentage)) + } else { + println("No measurable FPS decrease detected.") + } + + println("--------------------") + + // Measure average dropped frames + val droppedFramesDiff = avgDroppedFrames - other.avgDroppedFrames + val totalExpectedFrames = TimeUnit.NANOSECONDS.toMillis(other.avgDurationNanos) * 60 / 1000 + val droppedFramesIncreasePercentage = droppedFramesDiff * 100 / (totalExpectedFrames - other.avgDroppedFrames) + println("Dropped frames are calculated based on a target of 60 frames per second ($totalExpectedFrames total frames).") + println("[${other.operationName}] Average dropped frames: ${other.avgDroppedFrames}") + println("[$operationName] Average dropped frames: $avgDroppedFrames") + if (droppedFramesIncreasePercentage > 0) { + println("Frame drop increase: %.2f%%".format(droppedFramesIncreasePercentage)) + } else { + println("No measurable frame drop increase detected.") + } + + return BenchmarkResult( + cpuTimeOverheadPercentage, + droppedFramesIncreasePercentage, + durationIncreasePercentage, + fpsDecreasePercentage + ) + } +} + +internal data class BenchmarkResult( + /** + * Increase of cpu time in percentage. + * It has no direct impact on performance of the app, but it has on battery usage, as the cpu is 'awaken' longer. + */ + val cpuTimeIncrease: Double, + /** + * Increase of dropped frames in percentage.Very important, as it weights dropped frames based on the time + * passed between each frame. This is the metric end users can perceive as 'performance' in app usage. + */ + val droppedFramesIncrease: Double, + /** Increase of duration in percentage. If it's low enough, no end user will ever realize it. */ + val durationIncrease: Double, + /** + * Decrease of fps in percentage. Not really important, as even if fps are the same, the cpu could be + * doing more work in the frame window, and it could be hidden by checking average fps only. + */ + val fpsDecrease: Double +) diff --git a/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/main/AndroidManifest.xml b/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/main/AndroidManifest.xml new file mode 100644 index 00000000000..3c1d9e2de5d --- /dev/null +++ b/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/main/AndroidManifest.xml @@ -0,0 +1,32 @@ + + + + + + + + + + + + + + + diff --git a/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/main/java/io/sentry/uitest/android/benchmark/BenchmarkActivity.kt b/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/main/java/io/sentry/uitest/android/benchmark/BenchmarkActivity.kt new file mode 100644 index 00000000000..5a33232bb1f --- /dev/null +++ b/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/main/java/io/sentry/uitest/android/benchmark/BenchmarkActivity.kt @@ -0,0 +1,71 @@ +package io.sentry.uitest.android.benchmark + +import android.os.Bundle +import androidx.appcompat.app.AppCompatActivity +import androidx.recyclerview.widget.LinearLayoutManager +import androidx.recyclerview.widget.RecyclerView +import io.sentry.uitest.android.benchmark.databinding.ActivityBenchmarkBinding +import io.sentry.uitest.android.utils.BooleanIdlingResource +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +/** A simple activity with a list of bitmaps. */ +class BenchmarkActivity : AppCompatActivity() { + + companion object { + + /** The activity will set this when scrolling. */ + val scrollingIdlingResource = BooleanIdlingResource("sentry-uitest-android-benchmark-activity") + } + + /** + * Each background thread will run non-stop calculations during the benchmark. + * One such thread seems enough to represent a busy application. + * This number can be increased to mimic busier applications. + */ + private val backgroundThreadPoolSize = 1 + private val executor: ExecutorService = Executors.newFixedThreadPool(backgroundThreadPoolSize) + private var resumed = false + private lateinit var binding: ActivityBenchmarkBinding + + override fun onCreate(savedInstanceState: Bundle?) { + super.onCreate(savedInstanceState) + + binding = ActivityBenchmarkBinding.inflate(layoutInflater) + setContentView(binding.root) + + // We show a simple list that changes the idling resource + binding.benchmarkTransactionList.apply { + layoutManager = LinearLayoutManager(this@BenchmarkActivity) + adapter = BenchmarkTransactionListAdapter() + addOnScrollListener(object : RecyclerView.OnScrollListener() { + override fun onScrollStateChanged(recyclerView: RecyclerView, newState: Int) { + super.onScrollStateChanged(recyclerView, newState) + scrollingIdlingResource.setIdle(newState == RecyclerView.SCROLL_STATE_IDLE) + } + }) + } + } + + @Suppress("MagicNumber") + override fun onResume() { + super.onResume() + resumed = true + + // Do operations until the activity is paused. + repeat(backgroundThreadPoolSize) { + executor.execute { + var x = 0 + for (i in 0..1_000_000_000) { + x += i * i + if (!resumed) break + } + } + } + } + + override fun onPause() { + super.onPause() + resumed = false + } +} diff --git a/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/main/java/io/sentry/uitest/android/benchmark/BenchmarkTransactionListAdapter.kt b/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/main/java/io/sentry/uitest/android/benchmark/BenchmarkTransactionListAdapter.kt new file mode 100644 index 00000000000..86eb423c52a --- /dev/null +++ b/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/main/java/io/sentry/uitest/android/benchmark/BenchmarkTransactionListAdapter.kt @@ -0,0 +1,47 @@ +package io.sentry.uitest.android.benchmark + +import android.annotation.SuppressLint +import android.graphics.Bitmap +import android.graphics.Color +import android.view.LayoutInflater +import android.view.ViewGroup +import android.widget.ImageView +import android.widget.TextView +import androidx.recyclerview.widget.RecyclerView +import io.sentry.uitest.android.benchmark.databinding.BenchmarkItemListBinding +import kotlin.random.Random + +/** Simple [RecyclerView.Adapter] that generates a bitmap and a text to show for each item. */ +internal class BenchmarkTransactionListAdapter : RecyclerView.Adapter() { + + override fun onCreateViewHolder(parent: ViewGroup, viewType: Int): ViewHolder { + val binding = BenchmarkItemListBinding.inflate(LayoutInflater.from(parent.context), parent, false) + return ViewHolder(binding) + } + + override fun onBindViewHolder(holder: ViewHolder, position: Int) { + holder.imageView.setImageBitmap(generateBitmap()) + + @SuppressLint("SetTextI18n") + holder.textView.text = "Item $position ${"sentry ".repeat(position)}" + } + + @Suppress("MagicNumber") + private fun generateBitmap(): Bitmap { + val bitmapSize = 100 + val colors = (0 until (bitmapSize * bitmapSize)).map { + Color.rgb(Random.nextInt(256), Random.nextInt(256), Random.nextInt(256)) + }.toIntArray() + return Bitmap.createBitmap(colors, bitmapSize, bitmapSize, Bitmap.Config.ARGB_8888) + } + + // Disables view recycling. + override fun getItemViewType(position: Int): Int = position + + override fun getItemCount(): Int = 200 +} + +internal class ViewHolder(binding: BenchmarkItemListBinding) : RecyclerView.ViewHolder(binding.root) { + val imageView: ImageView = binding.benchmarkItemListImage + val textView: TextView = binding.benchmarkItemListText +} diff --git a/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/main/res/layout/activity_benchmark.xml b/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/main/res/layout/activity_benchmark.xml new file mode 100644 index 00000000000..f70a1c2449f --- /dev/null +++ b/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/main/res/layout/activity_benchmark.xml @@ -0,0 +1,29 @@ + + + + + + + + diff --git a/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/main/res/layout/benchmark_item_list.xml b/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/main/res/layout/benchmark_item_list.xml new file mode 100644 index 00000000000..a3bc14e3a78 --- /dev/null +++ b/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/main/res/layout/benchmark_item_list.xml @@ -0,0 +1,31 @@ + + + + + + + + diff --git a/sentry-android-integration-tests/sentry-uitest-android/.gitignore b/sentry-android-integration-tests/sentry-uitest-android/.gitignore new file mode 100644 index 00000000000..796b96d1c40 --- /dev/null +++ b/sentry-android-integration-tests/sentry-uitest-android/.gitignore @@ -0,0 +1 @@ +/build diff --git a/sentry-android-integration-tests/sentry-uitest-android/README.md b/sentry-android-integration-tests/sentry-uitest-android/README.md new file mode 100644 index 00000000000..c11397383d0 --- /dev/null +++ b/sentry-android-integration-tests/sentry-uitest-android/README.md @@ -0,0 +1,39 @@ +Ui tests for Android +=========== +Here will be put all ui tests for Android, running through Google's Espresso. +By default the envelopes sent to relay are caught by a mock server which allows us to check the envelopes sent. + +# How to use + +Simply run `./gradlew connectedCheck` to run all ui tests of all modules (requires a connected device, either physical or an emulator). +_Care: the benchmarks need to run the tests multiple times to get reliable results. This means they can take a long time (several minutes)._ +If you don't care about benchmark tests you can run `./gradlew connectedCheck -x :sentry-android-integration-tests:sentry-uitest-android-benchmark:connectedCheck`. +You can run benchmark tests only with `./gradlew :sentry-android-integration-tests:sentry-uitest-android-benchmark:connectedCheck`. + +# SauceLabs +To run on saucelabs execute following commands (need also `SAUCE_USERNAME` and `SAUCE_ACCESS_KEY` environment variables): +For Benchmarks: +``` +./gradlew :sentry-android-integration-tests:sentry-uitest-android-benchmark:assembleRelease +./gradlew :sentry-android-integration-tests:sentry-uitest-android-benchmark:assembleAndroidTest -DtestBuildType=release +saucectl run -c .sauce/sentry-uitest-android-benchmark.yml +``` +For End 2 End: +``` +./gradlew :sentry-android-integration-tests:sentry-uitest-android:assembleRelease +./gradlew :sentry-android-integration-tests:sentry-uitest-android:assembleAndroidTest -DtestBuildType=release +saucectl run -c .sauce/sentry-uitest-android-end2end.yml +``` + +# Troubleshooting + +There is an issue with Android 11+ (Xiaomi only?). +In order to start an activity from the background (which the test orchestrator does internally), the app needs a special permission, which cannot be granted without user interaction. +To allow it on the device go in Settings -> apps -> manage apps -> Select the app, like `Sentry End2End Tests` -> Other permissions -> `Display pop-up windows while running in the background`. +The path may be different on other devices. +On older versions of Android there is no problem. + +For this reason we cannot use the `testInstrumentationRunnerArguments["clearPackageData"] = "true"` in the build.gradle file, as clearing package data resets permissions. +This flag is used to run each test in its own instance of Instrumentation. This way they are isolated from one another and get their own Application instance. +More on https://developer.android.com/training/testing/instrumented-tests/androidx-test-libraries/runner#enable-gradle + diff --git a/sentry-android-integration-tests/sentry-uitest-android/build.gradle.kts b/sentry-android-integration-tests/sentry-uitest-android/build.gradle.kts new file mode 100644 index 00000000000..75d95d5bde6 --- /dev/null +++ b/sentry-android-integration-tests/sentry-uitest-android/build.gradle.kts @@ -0,0 +1,128 @@ +import io.gitlab.arturbosch.detekt.Detekt +import io.gitlab.arturbosch.detekt.extensions.DetektExtension +import net.ltgt.gradle.errorprone.errorprone + +plugins { + id("com.android.library") + kotlin("android") + id(Config.QualityPlugins.errorProne) + id(Config.QualityPlugins.gradleVersions) + id(Config.QualityPlugins.detektPlugin) +} + +android { + compileSdk = Config.Android.compileSdkVersion + + defaultConfig { + minSdk = Config.Android.minSdkVersionNdk + targetSdk = Config.Android.targetSdkVersion + + testInstrumentationRunner = Config.TestLibs.androidJUnitRunner + // Runs each test in its own instance of Instrumentation. This way they are isolated from + // one another and get their own Application instance. + // https://developer.android.com/training/testing/instrumented-tests/androidx-test-libraries/runner#enable-gradle + // This doesn't work on some devices with Android 11+. Clearing package data resets permissions. + // Check the readme for more info. +// testInstrumentationRunnerArguments["clearPackageData"] = "true" + } + + buildFeatures { + // Determines whether to support View Binding. + // Note that the viewBinding.enabled property is now deprecated. + viewBinding = true + } + + testOptions { + execution = "ANDROIDX_TEST_ORCHESTRATOR" + } + + signingConfigs { + getByName("debug") { + storeFile = rootProject.file("debug.keystore") + storePassword = "android" + keyAlias = "androiddebugkey" + keyPassword = "android" + } + } + + testBuildType = System.getProperty("testBuildType", "debug") + + buildTypes { + getByName("debug") { + isMinifyEnabled = true + signingConfig = signingConfigs.getByName("debug") + proguardFiles(getDefaultProguardFile("proguard-android-optimize.txt")) + } + getByName("release") { + isMinifyEnabled = true + isShrinkResources = false + proguardFiles(getDefaultProguardFile("proguard-android-optimize.txt")) + signingConfig = signingConfigs.getByName("debug") // to be able to run release mode + } + } + + kotlinOptions { + jvmTarget = JavaVersion.VERSION_1_8.toString() + } + + lint { + warningsAsErrors = true + checkDependencies = true + + // We run a full lint analysis as build part in CI, so skip vital checks for assemble tasks. + checkReleaseBuilds = false + } + + variantFilter { + if (Config.Android.shouldSkipDebugVariant(buildType.name)) { + ignore = true + } + } +} + +dependencies { + + implementation(kotlin(Config.kotlinStdLib, org.jetbrains.kotlin.config.KotlinCompilerVersion.VERSION)) + + implementation(projects.sentryAndroid) + implementation(Config.Libs.appCompat) + implementation(Config.Libs.androidxCore) + implementation(Config.TestLibs.espressoIdlingResource) + + compileOnly(Config.CompileOnly.nopen) + errorprone(Config.CompileOnly.nopenChecker) + errorprone(Config.CompileOnly.errorprone) + errorprone(Config.CompileOnly.errorProneNullAway) + + androidTestImplementation(projects.sentryTestSupport) + androidTestImplementation(Config.TestLibs.kotlinTestJunit) + androidTestImplementation(Config.TestLibs.espressoCore) + androidTestImplementation(Config.TestLibs.androidxRunner) + androidTestImplementation(Config.TestLibs.androidxTestRules) + androidTestImplementation(Config.TestLibs.androidxTestCoreKtx) + androidTestImplementation(Config.TestLibs.mockWebserver) + androidTestImplementation(Config.TestLibs.androidxJunit) + androidTestUtil(Config.TestLibs.androidxTestOrchestrator) +} + +tasks.withType().configureEach { + options.errorprone { + check("NullAway", net.ltgt.gradle.errorprone.CheckSeverity.ERROR) + option("NullAway:AnnotatedPackages", "io.sentry") + option("NullAway:UnannotatedSubPackages", "io.sentry.uitest.android.databinding") + } +} + +tasks.withType { + // Target version of the generated JVM bytecode. It is used for type resolution. + jvmTarget = JavaVersion.VERSION_1_8.toString() +} + +configure { + buildUponDefaultConfig = true + allRules = true +} + +kotlin { + explicitApi() +} diff --git a/sentry-android-integration-tests/sentry-uitest-android/proguard-rules.pro b/sentry-android-integration-tests/sentry-uitest-android/proguard-rules.pro new file mode 100644 index 00000000000..f1b424510da --- /dev/null +++ b/sentry-android-integration-tests/sentry-uitest-android/proguard-rules.pro @@ -0,0 +1,21 @@ +# Add project specific ProGuard rules here. +# You can control the set of applied configuration files using the +# proguardFiles setting in build.gradle. +# +# For more details, see +# http://developer.android.com/guide/developing/tools/proguard.html + +# If your project uses WebView with JS, uncomment the following +# and specify the fully qualified class name to the JavaScript interface +# class: +#-keepclassmembers class fqcn.of.javascript.interface.for.webview { +# public *; +#} + +# Uncomment this to preserve the line number information for +# debugging stack traces. +#-keepattributes SourceFile,LineNumberTable + +# If you keep the line number information, uncomment this to +# hide the original source file name. +#-renamesourcefileattribute SourceFile diff --git a/sentry-android-integration-tests/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/BaseUiTest.kt b/sentry-android-integration-tests/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/BaseUiTest.kt new file mode 100644 index 00000000000..581c69b0c87 --- /dev/null +++ b/sentry-android-integration-tests/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/BaseUiTest.kt @@ -0,0 +1,78 @@ +package io.sentry.uitest.android + +import android.content.Context +import androidx.test.core.app.ApplicationProvider +import androidx.test.espresso.Espresso +import androidx.test.espresso.IdlingRegistry +import androidx.test.espresso.idling.CountingIdlingResource +import androidx.test.platform.app.InstrumentationRegistry +import androidx.test.runner.AndroidJUnitRunner +import io.sentry.Sentry +import io.sentry.SentryOptions +import io.sentry.android.core.SentryAndroid +import io.sentry.uitest.android.mockservers.MockRelay +import kotlin.test.AfterTest +import kotlin.test.BeforeTest + +abstract class BaseUiTest { + + /** Runner of the test. */ + protected lateinit var runner: AndroidJUnitRunner + /** Application context for the current test. */ + protected lateinit var context: Context + /** Mock dsn used to send envelopes to our mock [relay] server. */ + protected lateinit var mockDsn: String + // The mockDsn cannot be changed. If a custom dsn needs to be used, it can be set in the options as usual + private set + /** + * Idling resource that will be checked by the relay server (if [initSentry] param relayWaitForRequests is true). + * This should be increased to match any envelope that will be sent during the test, + * so that they can later be checked. + */ + protected val relayIdlingResource = CountingIdlingResource("relay-requests") + + /** Mock relay server that receives all envelopes sent during the test. */ + protected val relay = MockRelay(false, relayIdlingResource) + + @BeforeTest + fun baseSetUp() { + runner = InstrumentationRegistry.getInstrumentation() as AndroidJUnitRunner + context = ApplicationProvider.getApplicationContext() + context.cacheDir.deleteRecursively() + relay.start() + mockDsn = relay.createMockDsn() + } + + @AfterTest + fun baseFinish() { + IdlingRegistry.getInstance().unregister(relayIdlingResource) + relay.shutdown() + Sentry.close() + } + + /** + * Initializes the Sentry sdk through [SentryAndroid.init] with a default dsn used to catch envelopes with [relay]. + * [relayWaitForRequests] sets whether [relay] should wait for all the envelopes sent when doing assertions. + * If true, [relayIdlingResource] should be increased to match any envelope that will be sent during the test. + * Sentry options can be adjusted as usual through [optionsConfiguration]. + */ + protected fun initSentry( + relayWaitForRequests: Boolean = false, + optionsConfiguration: ((options: SentryOptions) -> Unit)? = null + ) { + relay.waitForRequests = relayWaitForRequests + if (relayWaitForRequests) { + IdlingRegistry.getInstance().register(relayIdlingResource) + } + SentryAndroid.init(context) { + it.dsn = mockDsn + optionsConfiguration?.invoke(it) + } + } +} + +/** Waits until the Sentry SDK is idle. */ +fun waitUntilIdle() { + // We rely on Espresso's idling resources. + Espresso.onIdle() +} diff --git a/sentry-android-integration-tests/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/EnvelopeTests.kt b/sentry-android-integration-tests/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/EnvelopeTests.kt new file mode 100644 index 00000000000..df4bf05c71a --- /dev/null +++ b/sentry-android-integration-tests/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/EnvelopeTests.kt @@ -0,0 +1,59 @@ +package io.sentry.uitest.android + +import androidx.test.ext.junit.runners.AndroidJUnit4 +import io.sentry.ProfilingTraceData +import io.sentry.Sentry +import io.sentry.SentryEvent +import io.sentry.SentryOptions +import io.sentry.protocol.SentryTransaction +import org.junit.runner.RunWith +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertTrue + +@RunWith(AndroidJUnit4::class) +class EnvelopeTests : BaseUiTest() { + + @Test + fun checkEnvelopeCaptureMessage() { + + initSentry(true) + relayIdlingResource.increment() + Sentry.captureMessage("Message captured during test") + + relay.assert { + assertEnvelope { + val event: SentryEvent = it.assertItem() + it.assertNoOtherItems() + assertTrue(event.message?.formatted == "Message captured during test") + } + assertNoOtherEnvelopes() + assertNoOtherRequests() + } + } + + @Test + fun checkEnvelopeProfiledTransaction() { + + initSentry(true) { options: SentryOptions -> + options.tracesSampleRate = 1.0 + options.isProfilingEnabled = true + } + relayIdlingResource.increment() + val transaction = Sentry.startTransaction("e2etests", "test1") + + transaction.finish() + relay.assert { + assertEnvelope { + val transactionItem: SentryTransaction = it.assertItem() + val profilingTraceData: ProfilingTraceData = it.assertItem() + it.assertNoOtherItems() + assertTrue(transactionItem.transaction == "e2etests") + assertEquals(profilingTraceData.transactionId, transactionItem.eventId.toString()) + assertTrue(profilingTraceData.transactionName == "e2etests") + } + assertNoOtherEnvelopes() + assertNoOtherRequests() + } + } +} diff --git a/sentry-android-integration-tests/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/mockservers/EnvelopeAsserter.kt b/sentry-android-integration-tests/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/mockservers/EnvelopeAsserter.kt new file mode 100644 index 00000000000..7273531aeea --- /dev/null +++ b/sentry-android-integration-tests/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/mockservers/EnvelopeAsserter.kt @@ -0,0 +1,30 @@ +package io.sentry.uitest.android.mockservers + +import io.sentry.SentryEnvelope +import io.sentry.assertEnvelopeItem +import okhttp3.mockwebserver.MockResponse + +/** + * Class to make assertions on an envelope caught by [MockRelay]. + * It contains the sent envelope and the returned response, too. + */ +class EnvelopeAsserter(val envelope: SentryEnvelope, val response: MockResponse) { + /** List of items to assert. */ + val unassertedItems = envelope.items.toMutableList() + + /** + * Asserts an envelope item of [T] exists and returns the first one. + * The asserted item is then removed from internal list of unasserted items. + */ + inline fun assertItem(): T = assertEnvelopeItem(unassertedItems) { index, item -> + unassertedItems.removeAt(index) + return item + } + + /** Asserts there are no other items in the envelope. */ + fun assertNoOtherItems() { + if (unassertedItems.isNotEmpty()) { + throw AssertionError("There were other items: $unassertedItems") + } + } +} diff --git a/sentry-android-integration-tests/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/mockservers/MockRelay.kt b/sentry-android-integration-tests/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/mockservers/MockRelay.kt new file mode 100644 index 00000000000..ae28eb8e571 --- /dev/null +++ b/sentry-android-integration-tests/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/mockservers/MockRelay.kt @@ -0,0 +1,103 @@ +package io.sentry.uitest.android.mockservers + +import androidx.test.espresso.idling.CountingIdlingResource +import io.sentry.uitest.android.waitUntilIdle +import okhttp3.mockwebserver.Dispatcher +import okhttp3.mockwebserver.MockResponse +import okhttp3.mockwebserver.MockWebServer +import okhttp3.mockwebserver.RecordedRequest + +/** Mocks a relay server. */ +class MockRelay( + var waitForRequests: Boolean, + private val relayIdlingResource: CountingIdlingResource +) { + + /** Mocks a relay server. */ + private val relay = MockWebServer() + + private val dsnProject = "1234" + private val envelopePath = "/api/$dsnProject/envelope/" + + /** List of unasserted requests sent to the [envelopePath]. */ + private val unassertedEnvelopes = mutableListOf() + + /** List of unasserted requests not contained in [unassertedEnvelopes]. */ + private val unassertedRequests = mutableListOf() + + /** List of responses to return when a request is sent. */ + private val responses = mutableListOf<(RecordedRequest) -> MockResponse?>() + + init { + relay.dispatcher = object : Dispatcher() { + override fun dispatch(request: RecordedRequest): MockResponse { + // We check if there is any custom response previously set to return to this request, + // otherwise we return a successful MockResponse. + val response = responses.asSequence() + .mapNotNull { it(request) } + .firstOrNull() + ?: MockResponse() + + // Based on the path of the request, we populate the right list. + val relayResponse = RelayAsserter.RelayResponse(request, response) + when (request.path) { + envelopePath -> { + unassertedEnvelopes.add(relayResponse) + } + else -> { + unassertedRequests.add(relayResponse) + } + } + + // If we are waiting for requests to be received, we decrement the associated counter. + if (waitForRequests) { + relayIdlingResource.decrement() + } + return response + } + } + } + + /** Creates a dsn that will send request to this [MockRelay]. */ + fun createMockDsn() = "http://key@${relay.hostName}:${relay.port}/$dsnProject" + + /** Starts the mock relay server. */ + fun start() = relay.start() + + /** Shutdown the mock relay server and clear everything. */ + fun shutdown() { + responses.clear() + relay.shutdown() + } + + /** Add a custom response to be returned at the next request received. */ + fun addResponse(response: (RecordedRequest) -> MockResponse?) { + // Responses are added to the beginning of the list so they'll take precedence over + // previously added ones. + responses.add(0, response) + } + + /** Add a custom response to be returned at the next request received, if it satisfies the [filter]. */ + fun addResponse( + filter: (RecordedRequest) -> Boolean, + responseBuilder: ((request: RecordedRequest, response: MockResponse) -> Unit)? = null + ) { + addResponse { request -> + if (filter(request)) { + MockResponse().also { response -> + responseBuilder?.invoke(request, response) + } + } else { + null + } + } + } + + /** Wait to receive all requests (if [waitForRequests] is true) and run the [assertion]. */ + fun assert(assertion: RelayAsserter.() -> Unit) { + if (waitForRequests) { + waitUntilIdle() + } + assertion(RelayAsserter(unassertedEnvelopes, unassertedRequests)) + } +} diff --git a/sentry-android-integration-tests/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/mockservers/RelayAsserter.kt b/sentry-android-integration-tests/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/mockservers/RelayAsserter.kt new file mode 100644 index 00000000000..0ecb8acfff0 --- /dev/null +++ b/sentry-android-integration-tests/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/mockservers/RelayAsserter.kt @@ -0,0 +1,75 @@ +package io.sentry.uitest.android.mockservers + +import io.sentry.EnvelopeReader +import io.sentry.Sentry +import okhttp3.mockwebserver.MockResponse +import okhttp3.mockwebserver.RecordedRequest +import java.util.zip.GZIPInputStream + +/** Class used to assert requests sent to [MockRelay]. */ +class RelayAsserter( + private val unassertedEnvelopes: MutableList, + private val unassertedRequests: MutableList +) { + + /** + * Asserts an envelope request exists and allows to make other assertions on it and its response. + * The asserted envelope request is then removed from internal list of unasserted envelope. + */ + fun assertRawEnvelope(assertion: ((request: RecordedRequest, response: MockResponse) -> Unit)? = null) { + val relayResponse = unassertedEnvelopes.removeFirstOrNull() + ?: throw AssertionError("No envelope request found") + assertion?.let { + it(relayResponse.request, relayResponse.response) + } + } + + /** + * Asserts a request exists and makes other assertions on it and its response. + * The asserted request is then removed from internal list of unasserted requests. + */ + fun assertRawRequest(assertion: ((request: RecordedRequest, response: MockResponse) -> Unit)? = null) { + val relayResponse = unassertedRequests.removeFirstOrNull() + ?: throw AssertionError("No raw request found") + assertion?.let { + it(relayResponse.request, relayResponse.response) + } + } + + /** + * Asserts a request exists, parses it as an envelope and makes other assertions through a [EnvelopeAsserter]. + * The asserted envelope is then removed from internal list of unasserted envelopes. + */ + fun assertEnvelope(assertion: (asserter: EnvelopeAsserter) -> Unit) { + assertRawEnvelope { request, response -> + // Parse the request to rebuild the original envelope. If it fails we throw an assertion error. + val envelope = EnvelopeReader(Sentry.getCurrentHub().options.serializer) + .read(GZIPInputStream(request.body.inputStream())) + ?: throw AssertionError("Was unable to parse the request as an envelope: $request") + assertion(EnvelopeAsserter(envelope, response)) + } + } + + /** Asserts no other envelopes were sent. */ + fun assertNoOtherEnvelopes() { + if (unassertedEnvelopes.isNotEmpty()) { + throw AssertionError("There were other ${unassertedEnvelopes.size} envelope requests: $unassertedEnvelopes") + } + } + + /** Asserts no other raw requests were sent. */ + fun assertNoOtherRawRequests() { + assertNoOtherEnvelopes() + if (unassertedRequests.isNotEmpty()) { + throw AssertionError("There were other ${unassertedRequests.size} requests: $unassertedRequests") + } + } + + /** Asserts no other requests or envelopes were sent. */ + fun assertNoOtherRequests() { + assertNoOtherEnvelopes() + assertNoOtherRawRequests() + } + + data class RelayResponse(val request: RecordedRequest, val response: MockResponse) +} diff --git a/sentry-android-integration-tests/sentry-uitest-android/src/main/AndroidManifest.xml b/sentry-android-integration-tests/sentry-uitest-android/src/main/AndroidManifest.xml new file mode 100644 index 00000000000..15e849b6f90 --- /dev/null +++ b/sentry-android-integration-tests/sentry-uitest-android/src/main/AndroidManifest.xml @@ -0,0 +1,28 @@ + + + + + + + + + + + + + + + diff --git a/sentry-android-integration-tests/sentry-uitest-android/src/main/java/io/sentry/uitest/android/EmptyActivity.kt b/sentry-android-integration-tests/sentry-uitest-android/src/main/java/io/sentry/uitest/android/EmptyActivity.kt new file mode 100644 index 00000000000..b8f7d5f373a --- /dev/null +++ b/sentry-android-integration-tests/sentry-uitest-android/src/main/java/io/sentry/uitest/android/EmptyActivity.kt @@ -0,0 +1,11 @@ +package io.sentry.uitest.android + +import android.os.Bundle +import androidx.appcompat.app.AppCompatActivity + +class EmptyActivity : AppCompatActivity() { + + override fun onCreate(savedInstanceState: Bundle?) { + super.onCreate(savedInstanceState) + } +} diff --git a/sentry-android-integration-tests/sentry-uitest-android/src/main/java/io/sentry/uitest/android/utils/BooleanIdlingResource.kt b/sentry-android-integration-tests/sentry-uitest-android/src/main/java/io/sentry/uitest/android/utils/BooleanIdlingResource.kt new file mode 100644 index 00000000000..86a5b26709a --- /dev/null +++ b/sentry-android-integration-tests/sentry-uitest-android/src/main/java/io/sentry/uitest/android/utils/BooleanIdlingResource.kt @@ -0,0 +1,30 @@ +package io.sentry.uitest.android.utils + +import androidx.test.espresso.IdlingResource +import java.util.concurrent.atomic.AtomicBoolean + +/** Idling resource based on a boolean flag. */ +class BooleanIdlingResource(private val name: String) : IdlingResource { + + private val isIdle = AtomicBoolean(true) + + private val isIdleLock = Object() + + private var callback: IdlingResource.ResourceCallback? = null + + /** Sets whether this resource is currently in idle state. */ + fun setIdle(idling: Boolean) { + if (!isIdle.getAndSet(idling) && idling) { + callback?.onTransitionToIdle() + } + } + + override fun getName(): String = name + + /** Check if this resource is currently in idle state. */ + override fun isIdleNow(): Boolean = synchronized(isIdleLock) { isIdle.get() } + + override fun registerIdleTransitionCallback(callback: IdlingResource.ResourceCallback?) { + this.callback = callback + } +} diff --git a/sentry-android-ndk/build.gradle.kts b/sentry-android-ndk/build.gradle.kts index 84a7b0e568b..67b9dacc495 100644 --- a/sentry-android-ndk/build.gradle.kts +++ b/sentry-android-ndk/build.gradle.kts @@ -24,7 +24,7 @@ android { targetSdk = Config.Android.targetSdkVersion minSdk = Config.Android.minSdkVersionNdk // NDK requires a higher API level than core. - testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner" + testInstrumentationRunner = Config.TestLibs.androidJUnitRunner externalNativeBuild { cmake { diff --git a/sentry-android-ndk/sentry-native b/sentry-android-ndk/sentry-native index 3436a29d839..5500192dda0 160000 --- a/sentry-android-ndk/sentry-native +++ b/sentry-android-ndk/sentry-native @@ -1 +1 @@ -Subproject commit 3436a29d839aa7437548be940ab62a85ca699635 +Subproject commit 5500192dda05c82468787b2b0637e9c2688b9aed diff --git a/sentry-android-timber/build.gradle.kts b/sentry-android-timber/build.gradle.kts index 9dd9e1155ea..16649360d2b 100644 --- a/sentry-android-timber/build.gradle.kts +++ b/sentry-android-timber/build.gradle.kts @@ -17,7 +17,7 @@ android { targetSdk = Config.Android.targetSdkVersion minSdk = Config.Android.minSdkVersion - testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner" + testInstrumentationRunner = Config.TestLibs.androidJUnitRunner // for AGP 4.1 buildConfigField("String", "VERSION_NAME", "\"${project.version}\"") diff --git a/sentry-test-support/src/main/kotlin/io/sentry/assertions.kt b/sentry-test-support/src/main/kotlin/io/sentry/assertions.kt index d1054fbec22..2bffce6eae6 100644 --- a/sentry-test-support/src/main/kotlin/io/sentry/assertions.kt +++ b/sentry-test-support/src/main/kotlin/io/sentry/assertions.kt @@ -29,6 +29,19 @@ fun checkTransaction(predicate: (SentryTransaction) -> Unit): SentryEnvelope { } } +/** + * Asserts an envelope item of [T] exists in [items] and returns the first one. Otherwise it throws an [AssertionError]. + */ +inline fun assertEnvelopeItem(items: List, predicate: (index: Int, item: T) -> Unit): T { + val item = items.mapIndexedNotNull { index, it -> + val deserialized = JsonSerializer(SentryOptions()).deserialize(it.data.inputStream().reader(), T::class.java) + deserialized?.let { Pair(index, it) } + }.firstOrNull() + ?: throw AssertionError("No item found of type: ${T::class.java.name}") + predicate(item.first, item.second) + return item.second +} + /** * Modified version of check from mockito-kotlin Verification.kt, that does not print errors of type `SkipError`. */ diff --git a/sentry/src/main/java/io/sentry/Hub.java b/sentry/src/main/java/io/sentry/Hub.java index fd3dc0e18ad..e9a781324b8 100644 --- a/sentry/src/main/java/io/sentry/Hub.java +++ b/sentry/src/main/java/io/sentry/Hub.java @@ -682,8 +682,8 @@ public void flush(long timeoutMillis) { // The listener is called only if the transaction exists, as the transaction is needed to // stop it if (samplingDecision && options.isProfilingEnabled()) { - final ITransactionProfiler transactionListener = options.getTransactionProfiler(); - transactionListener.onTransactionStart(transaction); + final ITransactionProfiler transactionProfiler = options.getTransactionProfiler(); + transactionProfiler.onTransactionStart(transaction); } } if (bindToScope) { diff --git a/settings.gradle.kts b/settings.gradle.kts index a017e9bd692..5fe49492cae 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -36,7 +36,9 @@ include( "sentry-samples:sentry-samples-spring", "sentry-samples:sentry-samples-spring-boot", "sentry-samples:sentry-samples-spring-boot-webflux", - "sentry-samples:sentry-samples-netflix-dgs" + "sentry-samples:sentry-samples-netflix-dgs", + "sentry-android-integration-tests:sentry-uitest-android-benchmark", + "sentry-android-integration-tests:sentry-uitest-android" ) gradle.beforeProject {