diff --git a/.github/workflows/integration-tests-benchmarks.yml b/.github/workflows/integration-tests-benchmarks.yml new file mode 100644 index 0000000000..fcb15361b2 --- /dev/null +++ b/.github/workflows/integration-tests-benchmarks.yml @@ -0,0 +1,38 @@ +name: "Integration Tests - Benchmarks" +on: + push: + branches: + - main + - release/** + pull_request: + +jobs: + test: + name: Benchmarks + runs-on: ubuntu-latest + + steps: + - name: Git checkout + uses: actions/checkout@v2 + + - name: "Set up Java: 11" + uses: actions/setup-java@v2 + with: + java-version: "11" + distribution: "adopt" + + # Clean, build and release a test apk + - name: Make assembleBenchmarks + run: make assembleBenchmarks + + # We stop gradle at the end to make sure the cache folders + # don't contain any lock files and are free to be cached. + - name: Make stop + run: make stop + + - name: Run Tests in SauceLab + uses: saucelabs/saucectl-run-action@v1 + with: + sauce-username: ${{ secrets.SAUCE_USERNAME }} + sauce-access-key: ${{ secrets.SAUCE_ACCESS_KEY }} + config-file: .sauce/sentry-uitest-android-benchmark.yml diff --git a/.github/workflows/integration-tests-ui.yml b/.github/workflows/integration-tests-ui.yml new file mode 100644 index 0000000000..6d9f76fc6c --- /dev/null +++ b/.github/workflows/integration-tests-ui.yml @@ -0,0 +1,38 @@ +name: "Integration Tests - Ui tests" +on: + push: + branches: + - main + - release/** + pull_request: + +jobs: + test: + name: Ui tests + runs-on: ubuntu-latest + + steps: + - name: Git checkout + uses: actions/checkout@v2 + + - name: "Set up Java: 11" + uses: actions/setup-java@v2 + with: + java-version: "11" + distribution: "adopt" + + # Clean, build and release a test apk + - name: Make assembleUiTests + run: make assembleUiTests + + # We stop gradle at the end to make sure the cache folders + # don't contain any lock files and are free to be cached. + - name: Make stop + run: make stop + + - name: Run Tests in SauceLab + uses: saucelabs/saucectl-run-action@v1 + with: + sauce-username: ${{ secrets.SAUCE_USERNAME }} + sauce-access-key: ${{ secrets.SAUCE_ACCESS_KEY }} + config-file: .sauce/sentry-uitest-android-ui.yml diff --git a/.sauce/sentry-uitest-android-benchmark.yml b/.sauce/sentry-uitest-android-benchmark.yml index 35bce15f84..e9d8f51434 100644 --- a/.sauce/sentry-uitest-android-benchmark.yml +++ b/.sauce/sentry-uitest-android-benchmark.yml @@ -2,25 +2,59 @@ apiVersion: v1alpha kind: espresso sauce: region: us-west-1 - # Controls how many suites are executed at the same time (sauce test env only). - concurrency: 1 + concurrency: 2 metadata: name: Android benchmarks with Espresso tags: - benchmarks - android + +defaults: + timeout: 30m + espresso: app: ./sentry-android-integration-tests/sentry-uitest-android-benchmark/build/outputs/apk/release/sentry-uitest-android-benchmark-release.apk testApp: ./sentry-android-integration-tests/sentry-uitest-android-benchmark/build/outputs/apk/androidTest/release/sentry-uitest-android-benchmark-release-androidTest.apk + suites: - name: "Android Benchmarks" - devices: - - name: "Google Pixel 2" - platformVersion: 11 - - id: Google_Pixel_2_real_us - testOptions: - useTestOrchestrator: true -# Controls what artifacts to fetch when the suite on Sauce Cloud has finished. + + # Devices are chosen so that there is a high-end and a low-end device for each api level + - name: "Android 12 (api 31)" + devices: + - id: Google_Pixel_6_Pro_real_us # Google Pixel 6 Pro - api 31 (12) + - id: Google_Pixel_3_12_real_us # Google Pixel 3 - api 31 (12) + testOptions: + useTestOrchestrator: true + + - name: "Android 11 (api 30)" + devices: + - id: OnePlus_9_Pro_real_us # OnePlus 9 Pro - api 30 (11) + - id: Google_Pixel_2_real_us # Google Pixel 2 - api 30 (11) + testOptions: + useTestOrchestrator: true + +# Commenting for the moment, due to the error "Cannot install test-services-1.4.1.apk on device" on low Android versions +# - name: "Android 5 (api 22)" +# devices: +# - id: Amazon_Kindle_Fire_HD_8_real_us # Amazon Kindle Fire HD 8 - api 22 (5.1.1) +# testOptions: +# useTestOrchestrator: true + +# - id: Google_Pixel_4_XL_real_us1 # Google Pixel 4 XL - api 29 (10) +# - id: Motorola_Moto_G_Power_real_us # Motorola Moto G Power (2021) - api 29 (10) +# - id: Samsung_Galaxy_S8_plus_real_us # Samsung Galaxy S8+ - api 28 (9) +# - id: LG_G8_ThinQ_real_us # LG G8 ThinQ - api 28 (9) +# - id: OnePlus_5_real_us # OnePlus 5 - api 27 (8.1.0) +# - id: LG_K30_real_us1 # LG K30 - api 27 (8.1.0) +# - id: HTC_10_real_us # HTC 10 - api 26 (8.0.0) +# - id: Samsung_A3_real # Samsung Galaxy A3 2017 - api 26 (8.0.0) +# - id: ZTE_Axon_7_real2_us # ZTE Axon 7 - api 25 (7.1.1) +# - id: Motorola_Moto_X_Play_real # Motorola Moto X Play - api 25 (7.1.1) +# - id: Samsung_note_5_real_us # Samsung Galaxy Note 5 - api 24 (7.0) +# - id: LG_K10_real # LG K10 - api 24 (7.0) +# - id: Samsung_Galaxy_S6_Edge_Plus_real # Samsung Galaxy S6 Edge+ - api 23 (6.0.1) +# - id: Samsung_Tab_E_real_us # Samsung Tab E - api 23 (6.0.1) + artifacts: download: when: always diff --git a/.sauce/sentry-uitest-android-end2end.yml b/.sauce/sentry-uitest-android-ui.yml similarity index 77% rename from .sauce/sentry-uitest-android-end2end.yml rename to .sauce/sentry-uitest-android-ui.yml index 62178a4b0d..7a247efb15 100644 --- a/.sauce/sentry-uitest-android-end2end.yml +++ b/.sauce/sentry-uitest-android-ui.yml @@ -13,15 +13,13 @@ espresso: app: ./sentry-android-integration-tests/sentry-uitest-android/build/outputs/apk/release/sentry-uitest-android-release.apk testApp: ./sentry-android-integration-tests/sentry-uitest-android/build/outputs/apk/androidTest/release/sentry-uitest-android-release-androidTest.apk suites: - name: "Android End2end" - emulators: - - name: "Android GoogleApi Emulator" - orientation: portrait - platformVersions: - - "11.0" - - "10.0" - testOptions: - useTestOrchestrator: true + + - name: "Android 12 (api 31)" + devices: + - id: Samsung_Galaxy_S22_Ultra_5G_real_us # Samsung Galaxy S22 Ultra 5G - api 31 (12) + testOptions: + useTestOrchestrator: true + # Controls what artifacts to fetch when the suite on Sauce Cloud has finished. artifacts: download: diff --git a/Makefile b/Makefile index baece8b27b..16545dc18e 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,8 @@ -.PHONY: all clean compile dryRelease update stop checkFormat format api +.PHONY: all clean compile dryRelease update stop checkFormat format api assembleBenchmarkTestRelease assembleUiTestRelease all: stop clean checkFormat compile dryRelease +assembleBenchmarks: stop clean assembleBenchmarkTestRelease +assembleUiTests: stop clean assembleUiTestRelease # deep clean clean: @@ -35,3 +37,13 @@ format: # Binary compatibility validator api: ./gradlew apiDump + +# Assemble release and Android test apk of the uitest-android-benchmark module +assembleBenchmarkTestRelease: + ./gradlew :sentry-android-integration-tests:sentry-uitest-android-benchmark:assembleRelease + ./gradlew :sentry-android-integration-tests:sentry-uitest-android-benchmark:assembleAndroidTest -DtestBuildType=release + +# Assemble release and Android test apk of the uitest-android module +assembleUiTestRelease: + ./gradlew :sentry-android-integration-tests:sentry-uitest-android:assembleRelease + ./gradlew :sentry-android-integration-tests:sentry-uitest-android:assembleAndroidTest -DtestBuildType=release diff --git a/sentry-android-integration-tests/sentry-uitest-android-benchmark/build.gradle.kts b/sentry-android-integration-tests/sentry-uitest-android-benchmark/build.gradle.kts index 0ee897b3b0..ed39396aa5 100644 --- a/sentry-android-integration-tests/sentry-uitest-android-benchmark/build.gradle.kts +++ b/sentry-android-integration-tests/sentry-uitest-android-benchmark/build.gradle.kts @@ -88,7 +88,6 @@ dependencies { implementation(kotlin(Config.kotlinStdLib, org.jetbrains.kotlin.config.KotlinCompilerVersion.VERSION)) - implementation(projects.sentryAndroidIntegrationTests.sentryUitestAndroid) implementation(projects.sentryAndroid) implementation(Config.Libs.appCompat) implementation(Config.Libs.androidxCore) diff --git a/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/BaseBenchmarkTest.kt b/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/BaseBenchmarkTest.kt new file mode 100644 index 0000000000..01a4a050a4 --- /dev/null +++ b/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/BaseBenchmarkTest.kt @@ -0,0 +1,26 @@ +package io.sentry.uitest.android.benchmark + +import android.content.Context +import android.view.Choreographer +import androidx.test.core.app.ApplicationProvider +import androidx.test.platform.app.InstrumentationRegistry +import androidx.test.runner.AndroidJUnitRunner +import kotlin.test.BeforeTest + +abstract class BaseBenchmarkTest { + + protected lateinit var runner: AndroidJUnitRunner + protected lateinit var context: Context + protected lateinit var choreographer: Choreographer + + @BeforeTest + fun baseSetUp() { + runner = InstrumentationRegistry.getInstrumentation() as AndroidJUnitRunner + context = ApplicationProvider.getApplicationContext() + context.cacheDir.deleteRecursively() + // Must run on the main thread to get the main thread choreographer. + runner.runOnMainSync { + choreographer = Choreographer.getInstance() + } + } +} diff --git a/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/SdkBenchmarkTest.kt b/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/SdkBenchmarkTest.kt new file mode 100644 index 0000000000..7215a39b03 --- /dev/null +++ b/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/SdkBenchmarkTest.kt @@ -0,0 +1,58 @@ +package io.sentry.uitest.android.benchmark + +import androidx.lifecycle.Lifecycle +import androidx.test.core.app.launchActivity +import androidx.test.ext.junit.runners.AndroidJUnit4 +import io.sentry.Sentry +import io.sentry.android.core.SentryAndroid +import io.sentry.uitest.android.benchmark.util.BenchmarkOperation +import org.junit.runner.RunWith +import java.util.concurrent.TimeUnit +import kotlin.test.Test +import kotlin.test.assertTrue + +@RunWith(AndroidJUnit4::class) +class SdkBenchmarkTest : BaseBenchmarkTest() { + + @Test + fun benchmarkSdkInit() { + + // We compare starting an activity with and without the sdk init, to measure its impact on startup time. + val opNoSdk = getOperation() + val opSimpleSdk = getOperation { + SentryAndroid.init(context) { + it.dsn = "https://key@host/proj" + } + } + val opNoSdk2 = getOperation() + val opPerfProfilingSdk = getOperation { + SentryAndroid.init(context) { + it.dsn = "https://key@host/proj" + it.isProfilingEnabled = true + it.tracesSampleRate = 1.0 + } + } + val simpleSdkResult = BenchmarkOperation.compare(opNoSdk, "No Sdk", opSimpleSdk, "Simple Sdk") + val perfProfilingSdkResult = BenchmarkOperation.compare(opNoSdk2, "No Sdk", opPerfProfilingSdk, "Sdk with perf and profiling") + + val maxDurationThreshold = TimeUnit.MILLISECONDS.toNanos(100) + assertTrue(simpleSdkResult.durationIncreaseNanos in 0..maxDurationThreshold) + assertTrue(simpleSdkResult.cpuTimeIncreaseMillis in 0..100) + assertTrue(perfProfilingSdkResult.durationIncreaseNanos in simpleSdkResult.durationIncreaseNanos..maxDurationThreshold) + assertTrue(perfProfilingSdkResult.cpuTimeIncreaseMillis in simpleSdkResult.cpuTimeIncreaseMillis..100) + } + + private fun getOperation(init: (() -> Unit)? = null) = BenchmarkOperation( + choreographer, + op = { + runner.runOnMainSync { + init?.invoke() + } + val benchmarkScenario = launchActivity() + benchmarkScenario.moveToState(Lifecycle.State.DESTROYED) + }, + after = { + Sentry.close() + } + ) +} diff --git a/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/SentryBenchmarkTest.kt b/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/SentryBenchmarkTest.kt index 013fbccf65..f4b12f9a03 100644 --- a/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/SentryBenchmarkTest.kt +++ b/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/SentryBenchmarkTest.kt @@ -1,9 +1,6 @@ package io.sentry.uitest.android.benchmark -import android.content.Context -import android.view.Choreographer import androidx.lifecycle.Lifecycle -import androidx.test.core.app.ApplicationProvider import androidx.test.core.app.launchActivity import androidx.test.espresso.Espresso import androidx.test.espresso.Espresso.onView @@ -11,7 +8,6 @@ import androidx.test.espresso.IdlingRegistry import androidx.test.espresso.action.ViewActions.swipeUp import androidx.test.espresso.matcher.ViewMatchers.withId import androidx.test.ext.junit.runners.AndroidJUnit4 -import androidx.test.platform.app.InstrumentationRegistry import androidx.test.runner.AndroidJUnitRunner import io.sentry.ITransaction import io.sentry.Sentry @@ -25,22 +21,11 @@ import kotlin.test.Test import kotlin.test.assertTrue @RunWith(AndroidJUnit4::class) -class SentryBenchmarkTest { - - private lateinit var runner: AndroidJUnitRunner - private lateinit var context: Context - private lateinit var choreographer: Choreographer +class SentryBenchmarkTest : BaseBenchmarkTest() { @BeforeTest fun setUp() { - runner = InstrumentationRegistry.getInstrumentation() as AndroidJUnitRunner - context = ApplicationProvider.getApplicationContext() - context.cacheDir.deleteRecursively() IdlingRegistry.getInstance().register(BenchmarkActivity.scrollingIdlingResource) - // Must run on the main thread to get the main thread choreographer. - runner.runOnMainSync { - choreographer = Choreographer.getInstance() - } } @AfterTest @@ -53,35 +38,42 @@ class SentryBenchmarkTest { // We compare two operation that are the same. We expect the increases to be negligible, as the results // should be very similar. - val op1 = BenchmarkOperation(choreographer, getOperation(runner)) - val op2 = BenchmarkOperation(choreographer, getOperation(runner)) + val op1 = BenchmarkOperation(choreographer, op = getOperation(runner)) + val op2 = BenchmarkOperation(choreographer, op = getOperation(runner)) val comparisonResult = BenchmarkOperation.compare(op1, "Op1", op2, "Op2") - assertTrue(comparisonResult.durationIncrease in -1F..1F) - assertTrue(comparisonResult.cpuTimeIncrease in -1F..1F) + assertTrue(comparisonResult.durationIncreasePercentage in -1F..1F) + assertTrue(comparisonResult.cpuTimeIncreasePercentage in -1F..1F) // The fps decrease comparison is skipped, due to approximation: 59.51 and 59.49 fps are considered 60 and 59, // respectively. Also, if the average fps is 20 or 60, a difference of 1 fps becomes 5% or 1.66% respectively. - assertTrue(comparisonResult.droppedFramesIncrease in -1F..1F) + assertTrue(comparisonResult.droppedFramesIncreasePercentage in -1F..1F) } @Test fun benchmarkProfiledTransaction() { - runner.runOnMainSync { - SentryAndroid.init(context) { options: SentryOptions -> - options.dsn = "https://key@uri/1234567" - options.tracesSampleRate = 1.0 - options.isProfilingEnabled = true - } - } - // We compare the same operation with and without profiled transaction. // We expect the profiled transaction operation to be slower, but not slower than 5%. - val benchmarkOperationNoTransaction = BenchmarkOperation(choreographer, getOperation(runner)) + val benchmarkOperationNoTransaction = BenchmarkOperation(choreographer, op = getOperation(runner)) val benchmarkOperationProfiled = BenchmarkOperation( choreographer, - getOperation(runner) { + before = { + runner.runOnMainSync { + SentryAndroid.init(context) { options: SentryOptions -> + options.dsn = "https://key@uri/1234567" + options.tracesSampleRate = 1.0 + options.isProfilingEnabled = true + options.isEnableAutoSessionTracking = false + } + } + }, + op = getOperation(runner) { Sentry.startTransaction("Benchmark", "ProfiledTransaction") + }, + after = { + runner.runOnMainSync { + Sentry.close() + } } ) val comparisonResult = BenchmarkOperation.compare( @@ -91,14 +83,10 @@ class SentryBenchmarkTest { "ProfiledTransaction" ) - runner.runOnMainSync { - Sentry.close() - } - - assertTrue(comparisonResult.durationIncrease in 0F..5F) - assertTrue(comparisonResult.cpuTimeIncrease in 0F..5F) - assertTrue(comparisonResult.fpsDecrease in 0F..5F) - assertTrue(comparisonResult.droppedFramesIncrease in 0F..5F) + assertTrue(comparisonResult.durationIncreasePercentage in 0F..5F) + assertTrue(comparisonResult.cpuTimeIncreasePercentage in 0F..5F) + assertTrue(comparisonResult.fpsDecreasePercentage in 0F..5F) + assertTrue(comparisonResult.droppedFramesIncreasePercentage in 0F..5F) } /** diff --git a/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/util/BenchmarkOperation.kt b/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/util/BenchmarkOperation.kt index 72a71e7cb7..ca19ba1852 100644 --- a/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/util/BenchmarkOperation.kt +++ b/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/util/BenchmarkOperation.kt @@ -11,9 +11,14 @@ private const val FRAME_DURATION_60FPS_NS: Double = 1_000_000_000 / 60.0 /** * Class that allows to benchmark some operations. * Create two [BenchmarkOperation] objects and compare them using [BenchmarkOperation.compare] to get - * a [BenchmarkResult] with relative measured overheads. + * a [BenchmarkResult] with relative or absolute measured overheads. */ -internal class BenchmarkOperation(private val choreographer: Choreographer, private val op: () -> Unit) { +internal class BenchmarkOperation( + private val choreographer: Choreographer, + private val before: (() -> Unit)? = null, + private val after: (() -> Unit)? = null, + private val op: () -> Unit +) { companion object { @@ -33,6 +38,9 @@ internal class BenchmarkOperation(private val choreographer: Choreographer, priv warmupIterations: Int = 3, measuredIterations: Int = 15 ): BenchmarkResult { + // Android pushes the "installed app" event to other apps and the system itself. + // Let's give it time to do whatever it wants before starting measuring the operations. + Thread.sleep(2000) // The first operations are the slowest, as the device is still doing things like filling the cache. repeat(warmupIterations) { op1.warmup() @@ -68,12 +76,16 @@ internal class BenchmarkOperation(private val choreographer: Choreographer, priv /** Run the operation without measuring it. */ private fun warmup() { + before?.invoke() op() + after?.invoke() isolate() } /** Run the operation and measure it, updating sentry-uitest-android-benchmark data. */ private fun iterate() { + before?.invoke() + Thread.sleep(200) val startRealtimeNs = SystemClock.elapsedRealtimeNanos() val startCpuTimeMs = Process.getElapsedCpuTime() @@ -87,6 +99,7 @@ internal class BenchmarkOperation(private val choreographer: Choreographer, priv cpuDurationMillis += Process.getElapsedCpuTime() - startCpuTimeMs durationNanos += SystemClock.elapsedRealtimeNanos() - startRealtimeNs + after?.invoke() isolate() } diff --git a/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/util/BenchmarkOperationResult.kt b/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/util/BenchmarkOperationResult.kt index 47a954c49b..1bf765ae9f 100644 --- a/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/util/BenchmarkOperationResult.kt +++ b/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/util/BenchmarkOperationResult.kt @@ -10,19 +10,22 @@ internal data class BenchmarkOperationResult( val avgFramesPerSecond: Int, val operationName: String ) { - /** - * Compare two [BenchmarkOperationResult], calculating increases of each parameter in percentage. - */ + /** Compare two [BenchmarkOperation], calculating increases of each parameter. */ fun compare(other: BenchmarkOperationResult): BenchmarkResult { // Measure average duration - val durationDiffNanos = avgDurationNanos - other.avgDurationNanos - val durationIncreasePercentage = durationDiffNanos * 100.0 / other.avgDurationNanos + val durationIncreaseNanos = avgDurationNanos - other.avgDurationNanos + val durationIncreasePercentage = durationIncreaseNanos * 100.0 / other.avgDurationNanos println("[${other.operationName}] Average duration: ${other.avgDurationNanos} ns") println("[$operationName] Average duration: $avgDurationNanos ns") - if (durationIncreasePercentage > 0) { - println("Duration increase: %.2f%%".format(durationIncreasePercentage)) - } else { + println( + "Duration increase: %.2f%% (%d ns = %d ms)".format( + durationIncreasePercentage, + durationIncreaseNanos, + TimeUnit.NANOSECONDS.toMillis(durationIncreaseNanos) + ) + ) + if (durationIncreasePercentage <= 0) { println("No measurable duration increase detected.") } @@ -30,71 +33,81 @@ internal data class BenchmarkOperationResult( // Measure average cpu time val cores = Runtime.getRuntime().availableProcessors() - val cpuTimeDiff = (avgCpuTimeMillis - other.avgCpuTimeMillis) / cores - val cpuTimeOverheadPercentage = cpuTimeDiff * 100.0 / other.avgCpuTimeMillis + val cpuTimeIncreaseMillis = (avgCpuTimeMillis - other.avgCpuTimeMillis) / cores + val cpuTimeOverheadPercentage = cpuTimeIncreaseMillis * 100.0 / other.avgCpuTimeMillis // Cpu time spent profiling is weighted based on available threads, as profiling runs on 1 thread only. - println("The weighted difference of cpu times is $cpuTimeDiff ms (over $cores available cores).") + println("The weighted difference of cpu times is $cpuTimeIncreaseMillis ms (over $cores available cores).") println("[${other.operationName}] Cpu time: ${other.avgCpuTimeMillis} ms") println("[$operationName] Cpu time: $avgCpuTimeMillis ms") - if (cpuTimeOverheadPercentage > 0) { - println("CPU time overhead: %.2f%%".format(cpuTimeOverheadPercentage)) - } else { + println("CPU time overhead: %.2f%% (%d ms)".format(cpuTimeOverheadPercentage, cpuTimeIncreaseMillis)) + if (cpuTimeOverheadPercentage <= 0) { println("No measurable CPU time overhead detected.") } println("--------------------") // Measure average fps - val fpsDiff = other.avgFramesPerSecond - avgFramesPerSecond - val fpsDecreasePercentage = fpsDiff * 100.0 / other.avgFramesPerSecond + val fpsDecrease = other.avgFramesPerSecond - avgFramesPerSecond + val fpsDecreasePercentage = fpsDecrease * 100.0 / other.avgFramesPerSecond println("[${other.operationName}] Average FPS: ${other.avgFramesPerSecond}") println("[$operationName] Average FPS: $avgFramesPerSecond") - if (fpsDecreasePercentage > 0) { - println("FPS decrease: %.2f%%".format(fpsDecreasePercentage)) - } else { + println("FPS decrease: %.2f%% (%d fps)".format(fpsDecreasePercentage, fpsDecrease)) + if (fpsDecreasePercentage <= 0) { println("No measurable FPS decrease detected.") } println("--------------------") // Measure average dropped frames - val droppedFramesDiff = avgDroppedFrames - other.avgDroppedFrames + val droppedFramesIncrease = avgDroppedFrames - other.avgDroppedFrames val totalExpectedFrames = TimeUnit.NANOSECONDS.toMillis(other.avgDurationNanos) * 60 / 1000 - val droppedFramesIncreasePercentage = droppedFramesDiff * 100 / (totalExpectedFrames - other.avgDroppedFrames) + val droppedFramesIncreasePercentage = droppedFramesIncrease * 100 / (totalExpectedFrames - other.avgDroppedFrames) println("Dropped frames are calculated based on a target of 60 frames per second ($totalExpectedFrames total frames).") println("[${other.operationName}] Average dropped frames: ${other.avgDroppedFrames}") println("[$operationName] Average dropped frames: $avgDroppedFrames") - if (droppedFramesIncreasePercentage > 0) { - println("Frame drop increase: %.2f%%".format(droppedFramesIncreasePercentage)) - } else { + println("Frame drop increase: %.2f%% (%.2f)".format(droppedFramesIncreasePercentage, droppedFramesIncrease)) + if (droppedFramesIncreasePercentage <= 0) { println("No measurable frame drop increase detected.") } return BenchmarkResult( + cpuTimeIncreaseMillis, cpuTimeOverheadPercentage, + droppedFramesIncrease, droppedFramesIncreasePercentage, + durationIncreaseNanos, durationIncreasePercentage, + fpsDecrease, fpsDecreasePercentage ) } } +/** Result of the [BenchmarkOperation] comparison. */ internal data class BenchmarkResult( /** - * Increase of cpu time in percentage. + * Increase of cpu time in milliseconds. * It has no direct impact on performance of the app, but it has on battery usage, as the cpu is 'awaken' longer. */ - val cpuTimeIncrease: Double, + val cpuTimeIncreaseMillis: Long, + /** Increase of cpu time in percentage. */ + val cpuTimeIncreasePercentage: Double, /** - * Increase of dropped frames in percentage.Very important, as it weights dropped frames based on the time + * Increase of dropped frames.Very important, as it weights dropped frames based on the time * passed between each frame. This is the metric end users can perceive as 'performance' in app usage. */ val droppedFramesIncrease: Double, - /** Increase of duration in percentage. If it's low enough, no end user will ever realize it. */ - val durationIncrease: Double, + /** Increase of dropped frames in percentage. */ + val droppedFramesIncreasePercentage: Double, + /** Increase of duration in nanoseconds. If it's low enough, no end user will ever realize it. */ + val durationIncreaseNanos: Long, + /** Increase of duration in percentage. */ + val durationIncreasePercentage: Double, /** - * Decrease of fps in percentage. Not really important, as even if fps are the same, the cpu could be + * Decrease of fps. Not really important, as even if fps are the same, the cpu could be * doing more work in the frame window, and it could be hidden by checking average fps only. */ - val fpsDecrease: Double + val fpsDecrease: Int, + /** Decrease of fps in percentage. */ + val fpsDecreasePercentage: Double ) diff --git a/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/main/java/io/sentry/uitest/android/benchmark/BenchmarkActivity.kt b/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/main/java/io/sentry/uitest/android/benchmark/BenchmarkActivity.kt index 5a33232bb1..fcd239af83 100644 --- a/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/main/java/io/sentry/uitest/android/benchmark/BenchmarkActivity.kt +++ b/sentry-android-integration-tests/sentry-uitest-android-benchmark/src/main/java/io/sentry/uitest/android/benchmark/BenchmarkActivity.kt @@ -4,8 +4,8 @@ import android.os.Bundle import androidx.appcompat.app.AppCompatActivity import androidx.recyclerview.widget.LinearLayoutManager import androidx.recyclerview.widget.RecyclerView +import androidx.test.espresso.idling.CountingIdlingResource import io.sentry.uitest.android.benchmark.databinding.ActivityBenchmarkBinding -import io.sentry.uitest.android.utils.BooleanIdlingResource import java.util.concurrent.ExecutorService import java.util.concurrent.Executors @@ -15,7 +15,7 @@ class BenchmarkActivity : AppCompatActivity() { companion object { /** The activity will set this when scrolling. */ - val scrollingIdlingResource = BooleanIdlingResource("sentry-uitest-android-benchmark-activity") + val scrollingIdlingResource = CountingIdlingResource("sentry-uitest-android-benchmark-activityScrolling") } /** @@ -41,7 +41,10 @@ class BenchmarkActivity : AppCompatActivity() { addOnScrollListener(object : RecyclerView.OnScrollListener() { override fun onScrollStateChanged(recyclerView: RecyclerView, newState: Int) { super.onScrollStateChanged(recyclerView, newState) - scrollingIdlingResource.setIdle(newState == RecyclerView.SCROLL_STATE_IDLE) + if (newState == RecyclerView.SCROLL_STATE_DRAGGING) + scrollingIdlingResource.increment() + if (newState == RecyclerView.SCROLL_STATE_IDLE) + scrollingIdlingResource.decrement() } }) } diff --git a/sentry-android-integration-tests/sentry-uitest-android/build.gradle.kts b/sentry-android-integration-tests/sentry-uitest-android/build.gradle.kts index 75d95d5bde..f1cea81b1c 100644 --- a/sentry-android-integration-tests/sentry-uitest-android/build.gradle.kts +++ b/sentry-android-integration-tests/sentry-uitest-android/build.gradle.kts @@ -3,7 +3,7 @@ import io.gitlab.arturbosch.detekt.extensions.DetektExtension import net.ltgt.gradle.errorprone.errorprone plugins { - id("com.android.library") + id("com.android.application") kotlin("android") id(Config.QualityPlugins.errorProne) id(Config.QualityPlugins.gradleVersions) @@ -16,6 +16,8 @@ android { defaultConfig { minSdk = Config.Android.minSdkVersionNdk targetSdk = Config.Android.targetSdkVersion + versionCode = 1 + versionName = "1.0.0" testInstrumentationRunner = Config.TestLibs.androidJUnitRunner // Runs each test in its own instance of Instrumentation. This way they are isolated from