Tests/android UI tests absolute

.github/workflows/tests-benchmark.yml

@@ -0,0 +1,45 @@
+name: "Integration Tests"
+on:
+  push:
+    branches:
+      - main
+      - release/**
+  pull_request:
+
+jobs:
+  build:
+    name: Benchmarks
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        # Zulu Community distribution of OpenJDK
+        java: ["11"]
+
+    steps:
+      - name: Git checkout
+        uses: actions/checkout@v2
+
+      - name: "Set up Java: ${{ matrix.java }}"
+        uses: actions/setup-java@v2
+        with:
+          java-version: ${{ matrix.java }}
+          distribution: "adopt"
+
+      # Clean, build and release a test apk
+      - name: Make assembleBenchmarks
+        run: make assembleBenchmarks
+
+      # We stop gradle at the end to make sure the cache folders
+      # don't contain any lock files and are free to be cached.
+      - name: Make stop
+        run: make stop
+
+      - name: Install SauceLab command line
+        run: npm install -g saucectl
+
+      - name: Run Tests in SauceLab
+        env:
+          SAUCE_USERNAME: ${{ secrets.SAUCE_USERNAME }}
+          SAUCE_ACCESS_KEY: ${{ secrets.SAUCE_ACCESS_KEY }}
+        run: saucectl run -c .sauce/sentry-uitest-android-benchmark.yml

.github/workflows/tests-uitests.yml

@@ -0,0 +1,45 @@
+name: "Integration Tests"
+on:
+  push:
+    branches:
+      - main
+      - release/**
+  pull_request:
+
+jobs:
+  build:
+    name: Ui tests
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        # Zulu Community distribution of OpenJDK
+        java: ["11"]
+
+    steps:
+      - name: Git checkout
+        uses: actions/checkout@v2
+
+      - name: "Set up Java: ${{ matrix.java }}"
+        uses: actions/setup-java@v2
+        with:
+          java-version: ${{ matrix.java }}
+          distribution: "adopt"
+
+      # Clean, build and release a test apk
+      - name: Make assembleUiTests
+        run: make assembleUiTests
+
+      # We stop gradle at the end to make sure the cache folders
+      # don't contain any lock files and are free to be cached.
+      - name: Make stop
+        run: make stop
+
+      - name: Install SauceLab command line
+        run: npm install -g saucectl
+
+      - name: Run Tests in SauceLab
+        env:
+          SAUCE_USERNAME: ${{ secrets.SAUCE_USERNAME }}
+          SAUCE_ACCESS_KEY: ${{ secrets.SAUCE_ACCESS_KEY }}
+        run: saucectl run -c .sauce/sentry-uitest-android-end2end.yml

.sauce/sentry-uitest-android-benchmark.yml

@@ -2,25 +2,53 @@ apiVersion: v1alpha
 kind: espresso
 sauce:
   region: us-west-1
-  # Controls how many suites are executed at the same time (sauce test env only).
-  concurrency: 1
+  concurrency: 2
   metadata:
     name: Android benchmarks with Espresso
     tags:
       - benchmarks
       - android
+
+defaults:
+  timeout: 20m
+
 espresso:
   app: ./sentry-android-integration-tests/sentry-uitest-android-benchmark/build/outputs/apk/release/sentry-uitest-android-benchmark-release.apk
   testApp: ./sentry-android-integration-tests/sentry-uitest-android-benchmark/build/outputs/apk/androidTest/release/sentry-uitest-android-benchmark-release-androidTest.apk
+
 suites:
-  name: "Android Benchmarks"
-  devices:
-    - name: "Google Pixel 2"
-      platformVersion: 11
-    - id: Google_Pixel_2_real_us
-  testOptions:
-    useTestOrchestrator: true
-# Controls what artifacts to fetch when the suite on Sauce Cloud has finished.
+
+  # Devices are chosen so that there is a high-end and a low-end device for each api level
+  - name: "Android 12 (api 31)"
+    devices:
+      - id: Google_Pixel_6_Pro_real_us # Google Pixel 6 Pro - api 31 (12)
+      - id: Google_Pixel_3_12_real_us # Google Pixel 3 - api 31 (12)
+    testOptions:
+      useTestOrchestrator: true
+
+  - name: "Android 11 (api 30)"
+    devices:
+      - id: OnePlus_9_Pro_real_us # OnePlus 9 Pro - api 30 (11)
+      - id: Google_Pixel_2_real_us # Google Pixel 2 - api 30 (11)
+    testOptions:
+      useTestOrchestrator: true
+#      - id: Google_Pixel_4_XL_real_us1 # Google Pixel 4 XL - api 29 (10)
+#      - id: Motorola_Moto_G_Power_real_us # Motorola Moto G Power (2021) - api 29 (10)
+#      - id: Samsung_Galaxy_S8_plus_real_us # Samsung Galaxy S8+ - api 28 (9)
+#      - id: LG_G8_ThinQ_real_us # LG G8 ThinQ - api 28 (9)
+#      - id: OnePlus_5_real_us # OnePlus 5 - api 27 (8.1.0)
+#      - id: LG_K30_real_us1 # LG K30 - api 27 (8.1.0)
+#      - id: HTC_10_real_us # HTC 10 - api 26 (8.0.0)
+#      - id: Samsung_A3_real # Samsung Galaxy A3 2017 - api 26 (8.0.0)
+#      - id: ZTE_Axon_7_real2_us # ZTE Axon 7 - api 25 (7.1.1)
+#      - id: Motorola_Moto_X_Play_real # Motorola Moto X Play - api 25 (7.1.1)
+#      - id: Samsung_note_5_real_us # Samsung Galaxy Note 5 - api 24 (7.0)
+#      - id: LG_K10_real # LG K10 - api 24 (7.0)
+#      - id: Samsung_Galaxy_S6_Edge_Plus_real # Samsung Galaxy S6 Edge+ - api 23 (6.0.1)
+#      - id: Samsung_Tab_E_real_us # Samsung Tab E - api 23 (6.0.1)
+#      - id: Samsung_Galaxy_Note_4_5_real # Samsung Galaxy Note 4 - api 22 (5.1.1)
+#      - id: Sony_Xperia_Z1_real # Sony Xperia Z1 - api 22 (5.1.1)
+
 artifacts:
   download:
     when: always

Makefile

@@ -1,6 +1,8 @@
-.PHONY: all clean compile dryRelease update stop checkFormat format api
+.PHONY: all clean compile dryRelease update stop checkFormat format api assembleBenchmarkTestRelease assembleUiTestRelease
 
 all: stop clean checkFormat compile dryRelease
+assembleBenchmarks: stop clean assembleBenchmarkTestRelease
+assembleUiTests: stop clean assembleUiTestRelease
 
 # deep clean
 clean:
@@ -35,3 +37,13 @@ format:
 # Binary compatibility validator
 api:
 	./gradlew apiDump
+
+# Assemble release and Android test apk of the uitest-android-benchmark module
+assembleBenchmarkTestRelease:
+	./gradlew :sentry-android-integration-tests:sentry-uitest-android-benchmark:assembleRelease
+	./gradlew :sentry-android-integration-tests:sentry-uitest-android-benchmark:assembleAndroidTest -DtestBuildType=release
+
+# Assemble release and Android test apk of the uitest-android module
+assembleUiTestRelease:
+	./gradlew :sentry-android-integration-tests:sentry-uitest-android:assembleRelease
+	./gradlew :sentry-android-integration-tests:sentry-uitest-android:assembleAndroidTest -DtestBuildType=release

sentry-android-integration-tests/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/BaseBenchmarkTest.kt

@@ -0,0 +1,20 @@
+package io.sentry.uitest.android.benchmark
+
+import android.content.Context
+import androidx.test.core.app.ApplicationProvider
+import androidx.test.platform.app.InstrumentationRegistry
+import androidx.test.runner.AndroidJUnitRunner
+import kotlin.test.BeforeTest
+
+abstract class BaseBenchmarkTest {
+
+    protected lateinit var runner: AndroidJUnitRunner
+    protected lateinit var context: Context
+
+    @BeforeTest
+    fun baseSetUp() {
+        runner = InstrumentationRegistry.getInstrumentation() as AndroidJUnitRunner
+        context = ApplicationProvider.getApplicationContext()
+        context.cacheDir.deleteRecursively()
+    }
+}

sentry-android-integration-tests/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/SdkBenchmarkTest.kt

@@ -0,0 +1,82 @@
+package io.sentry.uitest.android.benchmark
+
+import android.view.Choreographer
+import androidx.lifecycle.Lifecycle
+import androidx.test.core.app.launchActivity
+import androidx.test.espresso.Espresso
+import androidx.test.espresso.IdlingRegistry
+import androidx.test.ext.junit.runners.AndroidJUnit4
+import io.sentry.Sentry
+import io.sentry.android.core.SentryAndroid
+import io.sentry.uitest.android.benchmark.util.BenchmarkOperation
+import org.junit.runner.RunWith
+import java.util.concurrent.TimeUnit
+import kotlin.test.AfterTest
+import kotlin.test.BeforeTest
+import kotlin.test.Test
+import kotlin.test.assertTrue
+
+@RunWith(AndroidJUnit4::class)
+class SdkBenchmarkTest : BaseBenchmarkTest() {
+
+    private lateinit var choreographer: Choreographer
+
+    @BeforeTest
+    fun setUp() {
+        IdlingRegistry.getInstance().register(BenchmarkActivity.activityStartedIdlingResource)
+        // Must run on the main thread to get the main thread choreographer.
+        runner.runOnMainSync {
+            choreographer = Choreographer.getInstance()
+        }
+    }
+
+    @AfterTest
+    fun cleanup() {
+        IdlingRegistry.getInstance().unregister(BenchmarkActivity.activityStartedIdlingResource)
+    }
+
+    @Test
+    fun benchmarkSdkInit() {
+
+        // We compare starting an activity with and without the sdk init, to measure its impact on startup time.
+        val opNoSdk = getOperation()
+        val opSimpleSdk = getOperation {
+            SentryAndroid.init(context) {
+                it.dsn = "https://key@host/proj"
+            }
+        }
+        val opNoSdk2 = getOperation()
+        val opPerfProfilingSdk = getOperation {
+            SentryAndroid.init(context) {
+                it.dsn = "https://key@host/proj"
+                it.isProfilingEnabled = true
+                it.isEnableAutoActivityLifecycleTracing = true
+                it.tracesSampleRate = 1.0
+            }
+        }
+        val simpleSdkResult = BenchmarkOperation.compare(opNoSdk, "No Sdk", opSimpleSdk, "Simple Sdk")
+        val perfProfilingSdkResult = BenchmarkOperation.compare(opNoSdk2, "No Sdk", opPerfProfilingSdk, "Sdk with perf and profiling")
+
+        val maxDurationThreshold = TimeUnit.MILLISECONDS.toNanos(100)
+        assertTrue(simpleSdkResult.durationIncreaseNanos in 0..maxDurationThreshold)
+        assertTrue(simpleSdkResult.cpuTimeIncreaseMillis in 0..100)
+        assertTrue(perfProfilingSdkResult.durationIncreaseNanos in simpleSdkResult.durationIncreaseNanos..maxDurationThreshold)
+        assertTrue(perfProfilingSdkResult.cpuTimeIncreaseMillis in simpleSdkResult.cpuTimeIncreaseMillis..100)
+    }
+
+    private fun getOperation(init: (() -> Unit)? = null) = BenchmarkOperation(
+        choreographer,
+        before = { BenchmarkActivity.activityStartedIdlingResource.setIdle(false) },
+        op = {
+            runner.runOnMainSync {
+                init?.invoke()
+            }
+            val benchmarkScenario = launchActivity<BenchmarkActivity>()
+            Espresso.onIdle()
+            benchmarkScenario.moveToState(Lifecycle.State.DESTROYED)
+        },
+        after = {
+            Sentry.close()
+        }
+    )
+}

sentry-android-integration-tests/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/SentryBenchmarkTest.kt

@@ -1,17 +1,14 @@
 package io.sentry.uitest.android.benchmark
 
-import android.content.Context
 import android.view.Choreographer
 import androidx.lifecycle.Lifecycle
-import androidx.test.core.app.ApplicationProvider
 import androidx.test.core.app.launchActivity
 import androidx.test.espresso.Espresso
 import androidx.test.espresso.Espresso.onView
 import androidx.test.espresso.IdlingRegistry
 import androidx.test.espresso.action.ViewActions.swipeUp
 import androidx.test.espresso.matcher.ViewMatchers.withId
 import androidx.test.ext.junit.runners.AndroidJUnit4
-import androidx.test.platform.app.InstrumentationRegistry
 import androidx.test.runner.AndroidJUnitRunner
 import io.sentry.ITransaction
 import io.sentry.Sentry
@@ -25,17 +22,12 @@ import kotlin.test.Test
 import kotlin.test.assertTrue
 
 @RunWith(AndroidJUnit4::class)
-class SentryBenchmarkTest {
+class SentryBenchmarkTest : BaseBenchmarkTest() {
 
-    private lateinit var runner: AndroidJUnitRunner
-    private lateinit var context: Context
     private lateinit var choreographer: Choreographer
 
     @BeforeTest
     fun setUp() {
-        runner = InstrumentationRegistry.getInstrumentation() as AndroidJUnitRunner
-        context = ApplicationProvider.getApplicationContext()
-        context.cacheDir.deleteRecursively()
         IdlingRegistry.getInstance().register(BenchmarkActivity.scrollingIdlingResource)
         // Must run on the main thread to get the main thread choreographer.
         runner.runOnMainSync {
@@ -53,35 +45,42 @@ class SentryBenchmarkTest {
 
         // We compare two operation that are the same. We expect the increases to be negligible, as the results
         // should be very similar.
-        val op1 = BenchmarkOperation(choreographer, getOperation(runner))
-        val op2 = BenchmarkOperation(choreographer, getOperation(runner))
+        val op1 = BenchmarkOperation(choreographer, op = getOperation(runner))
+        val op2 = BenchmarkOperation(choreographer, op = getOperation(runner))
         val comparisonResult = BenchmarkOperation.compare(op1, "Op1", op2, "Op2")
 
-        assertTrue(comparisonResult.durationIncrease in -1F..1F)
-        assertTrue(comparisonResult.cpuTimeIncrease in -1F..1F)
+        assertTrue(comparisonResult.durationIncreasePercentage in -1F..1F)
+        assertTrue(comparisonResult.cpuTimeIncreasePercentage in -1F..1F)
         // The fps decrease comparison is skipped, due to approximation: 59.51 and 59.49 fps are considered 60 and 59,
         // respectively. Also, if the average fps is 20 or 60, a difference of 1 fps becomes 5% or 1.66% respectively.
-        assertTrue(comparisonResult.droppedFramesIncrease in -1F..1F)
+        assertTrue(comparisonResult.droppedFramesIncreasePercentage in -1F..1F)
     }
 
     @Test
     fun benchmarkProfiledTransaction() {
 
-        runner.runOnMainSync {
-            SentryAndroid.init(context) { options: SentryOptions ->
-                options.dsn = "https://key@uri/1234567"
-                options.tracesSampleRate = 1.0
-                options.isProfilingEnabled = true
-            }
-        }
-
         // We compare the same operation with and without profiled transaction.
         // We expect the profiled transaction operation to be slower, but not slower than 5%.
-        val benchmarkOperationNoTransaction = BenchmarkOperation(choreographer, getOperation(runner))
+        val benchmarkOperationNoTransaction = BenchmarkOperation(choreographer, op = getOperation(runner))
         val benchmarkOperationProfiled = BenchmarkOperation(
             choreographer,
-            getOperation(runner) {
+            before = {
+                runner.runOnMainSync {
+                    SentryAndroid.init(context) { options: SentryOptions ->
+                        options.dsn = "https://key@uri/1234567"
+                        options.tracesSampleRate = 1.0
+                        options.isProfilingEnabled = true
+                        options.isEnableAutoSessionTracking = false
+                    }
+                }
+            },
+            op = getOperation(runner) {
                 Sentry.startTransaction("Benchmark", "ProfiledTransaction")
+            },
+            after = {
+                runner.runOnMainSync {
+                    Sentry.close()
+                }
             }
         )
         val comparisonResult = BenchmarkOperation.compare(
@@ -91,14 +90,10 @@ class SentryBenchmarkTest {
             "ProfiledTransaction"
         )
 
-        runner.runOnMainSync {
-            Sentry.close()
-        }
-
-        assertTrue(comparisonResult.durationIncrease in 0F..5F)
-        assertTrue(comparisonResult.cpuTimeIncrease in 0F..5F)
-        assertTrue(comparisonResult.fpsDecrease in 0F..5F)
-        assertTrue(comparisonResult.droppedFramesIncrease in 0F..5F)
+        assertTrue(comparisonResult.durationIncreasePercentage in 0F..5F)
+        assertTrue(comparisonResult.cpuTimeIncreasePercentage in 0F..5F)
+        assertTrue(comparisonResult.fpsDecreasePercentage in 0F..5F)
+        assertTrue(comparisonResult.droppedFramesIncreasePercentage in 0F..5F)
     }
 
     /**