Tests/android UI tests absolute

.github/workflows/integration-tests-benchmarks.yml

@@ -0,0 +1,38 @@
+name: "Integration Tests - Benchmarks"
+on:
+  push:
+    branches:
+      - main
+      - release/**
+  pull_request:
+
+jobs:
+  test:
+    name: Benchmarks
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Git checkout
+        uses: actions/checkout@v2
+
+      - name: "Set up Java: 11"
+        uses: actions/setup-java@v2
+        with:
+          java-version: "11"
+          distribution: "adopt"
+
+      # Clean, build and release a test apk
+      - name: Make assembleBenchmarks
+        run: make assembleBenchmarks
+
+      # We stop gradle at the end to make sure the cache folders
+      # don't contain any lock files and are free to be cached.
+      - name: Make stop
+        run: make stop
+
+      - name: Run Tests in SauceLab
+        uses: saucelabs/saucectl-run-action@v1
+        with:
+          sauce-username: ${{ secrets.SAUCE_USERNAME }}
+          sauce-access-key: ${{ secrets.SAUCE_ACCESS_KEY }}
+          config-file: .sauce/sentry-uitest-android-benchmark.yml

.github/workflows/integration-tests-ui.yml

@@ -0,0 +1,38 @@
+name: "Integration Tests - Ui tests"
+on:
+  push:
+    branches:
+      - main
+      - release/**
+  pull_request:
+
+jobs:
+  test:
+    name: Ui tests
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Git checkout
+        uses: actions/checkout@v2
+
+      - name: "Set up Java: 11"
+        uses: actions/setup-java@v2
+        with:
+          java-version: "11"
+          distribution: "adopt"
+
+      # Clean, build and release a test apk
+      - name: Make assembleUiTests
+        run: make assembleUiTests
+
+      # We stop gradle at the end to make sure the cache folders
+      # don't contain any lock files and are free to be cached.
+      - name: Make stop
+        run: make stop
+
+      - name: Run Tests in SauceLab
+        uses: saucelabs/saucectl-run-action@v1
+        with:
+          sauce-username: ${{ secrets.SAUCE_USERNAME }}
+          sauce-access-key: ${{ secrets.SAUCE_ACCESS_KEY }}
+          config-file: .sauce/sentry-uitest-android-ui.yml

.sauce/sentry-uitest-android-benchmark.yml

@@ -2,25 +2,59 @@ apiVersion: v1alpha
 kind: espresso
 sauce:
   region: us-west-1
-  # Controls how many suites are executed at the same time (sauce test env only).
-  concurrency: 1
+  concurrency: 2
   metadata:
     name: Android benchmarks with Espresso
     tags:
       - benchmarks
       - android
+
+defaults:
+  timeout: 30m
+
 espresso:
   app: ./sentry-android-integration-tests/sentry-uitest-android-benchmark/build/outputs/apk/release/sentry-uitest-android-benchmark-release.apk
   testApp: ./sentry-android-integration-tests/sentry-uitest-android-benchmark/build/outputs/apk/androidTest/release/sentry-uitest-android-benchmark-release-androidTest.apk
+
 suites:
-  name: "Android Benchmarks"
-  devices:
-    - name: "Google Pixel 2"
-      platformVersion: 11
-    - id: Google_Pixel_2_real_us
-  testOptions:
-    useTestOrchestrator: true
-# Controls what artifacts to fetch when the suite on Sauce Cloud has finished.
+
+  # Devices are chosen so that there is a high-end and a low-end device for each api level
+  - name: "Android 12 (api 31)"
+    devices:
+      - id: Google_Pixel_6_Pro_real_us # Google Pixel 6 Pro - api 31 (12)
+      - id: Google_Pixel_3_12_real_us # Google Pixel 3 - api 31 (12)
+    testOptions:
+      useTestOrchestrator: true
+
+  - name: "Android 11 (api 30)"
+    devices:
+      - id: OnePlus_9_Pro_real_us # OnePlus 9 Pro - api 30 (11)
+      - id: Google_Pixel_2_real_us # Google Pixel 2 - api 30 (11)
+    testOptions:
+      useTestOrchestrator: true
+
+# Commenting for the moment, due to the error "Cannot install test-services-1.4.1.apk on device" on low Android versions
+#  - name: "Android 5 (api 22)"
+#    devices:
+#      - id: Amazon_Kindle_Fire_HD_8_real_us # Amazon Kindle Fire HD 8 - api 22 (5.1.1)
+#    testOptions:
+#      useTestOrchestrator: true
+
+#      - id: Google_Pixel_4_XL_real_us1 # Google Pixel 4 XL - api 29 (10)
+#      - id: Motorola_Moto_G_Power_real_us # Motorola Moto G Power (2021) - api 29 (10)
+#      - id: Samsung_Galaxy_S8_plus_real_us # Samsung Galaxy S8+ - api 28 (9)
+#      - id: LG_G8_ThinQ_real_us # LG G8 ThinQ - api 28 (9)
+#      - id: OnePlus_5_real_us # OnePlus 5 - api 27 (8.1.0)
+#      - id: LG_K30_real_us1 # LG K30 - api 27 (8.1.0)
+#      - id: HTC_10_real_us # HTC 10 - api 26 (8.0.0)
+#      - id: Samsung_A3_real # Samsung Galaxy A3 2017 - api 26 (8.0.0)
+#      - id: ZTE_Axon_7_real2_us # ZTE Axon 7 - api 25 (7.1.1)
+#      - id: Motorola_Moto_X_Play_real # Motorola Moto X Play - api 25 (7.1.1)
+#      - id: Samsung_note_5_real_us # Samsung Galaxy Note 5 - api 24 (7.0)
+#      - id: LG_K10_real # LG K10 - api 24 (7.0)
+#      - id: Samsung_Galaxy_S6_Edge_Plus_real # Samsung Galaxy S6 Edge+ - api 23 (6.0.1)
+#      - id: Samsung_Tab_E_real_us # Samsung Tab E - api 23 (6.0.1)
+
 artifacts:
   download:
     when: always

.sauce/sentry-uitest-android-end2end.yml → .sauce/sentry-uitest-android-ui.yml

@@ -13,15 +13,13 @@ espresso:
   app: ./sentry-android-integration-tests/sentry-uitest-android/build/outputs/apk/release/sentry-uitest-android-release.apk
   testApp: ./sentry-android-integration-tests/sentry-uitest-android/build/outputs/apk/androidTest/release/sentry-uitest-android-release-androidTest.apk
 suites:
-  name: "Android End2end"
-  emulators:
-    - name: "Android GoogleApi Emulator"
-      orientation: portrait
-      platformVersions:
-        - "11.0"
-        - "10.0"
-  testOptions:
-    useTestOrchestrator: true
+
+  - name: "Android 12 (api 31)"
+    devices:
+      - id: Samsung_Galaxy_S22_Ultra_5G_real_us # Samsung Galaxy S22 Ultra 5G - api 31 (12)
+    testOptions:
+      useTestOrchestrator: true
+
 # Controls what artifacts to fetch when the suite on Sauce Cloud has finished.
 artifacts:
   download:

Makefile

@@ -1,6 +1,8 @@
-.PHONY: all clean compile dryRelease update stop checkFormat format api
+.PHONY: all clean compile dryRelease update stop checkFormat format api assembleBenchmarkTestRelease assembleUiTestRelease
 
 all: stop clean checkFormat compile dryRelease
+assembleBenchmarks: stop clean assembleBenchmarkTestRelease
+assembleUiTests: stop clean assembleUiTestRelease
 
 # deep clean
 clean:
@@ -35,3 +37,13 @@ format:
 # Binary compatibility validator
 api:
 	./gradlew apiDump
+
+# Assemble release and Android test apk of the uitest-android-benchmark module
+assembleBenchmarkTestRelease:
+	./gradlew :sentry-android-integration-tests:sentry-uitest-android-benchmark:assembleRelease
+	./gradlew :sentry-android-integration-tests:sentry-uitest-android-benchmark:assembleAndroidTest -DtestBuildType=release
+
+# Assemble release and Android test apk of the uitest-android module
+assembleUiTestRelease:
+	./gradlew :sentry-android-integration-tests:sentry-uitest-android:assembleRelease
+	./gradlew :sentry-android-integration-tests:sentry-uitest-android:assembleAndroidTest -DtestBuildType=release

sentry-android-integration-tests/sentry-uitest-android-benchmark/build.gradle.kts

@@ -88,7 +88,6 @@ dependencies {
 
     implementation(kotlin(Config.kotlinStdLib, org.jetbrains.kotlin.config.KotlinCompilerVersion.VERSION))
 
-    implementation(projects.sentryAndroidIntegrationTests.sentryUitestAndroid)
     implementation(projects.sentryAndroid)
     implementation(Config.Libs.appCompat)
     implementation(Config.Libs.androidxCore)

sentry-android-integration-tests/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/BaseBenchmarkTest.kt

@@ -0,0 +1,26 @@
+package io.sentry.uitest.android.benchmark
+
+import android.content.Context
+import android.view.Choreographer
+import androidx.test.core.app.ApplicationProvider
+import androidx.test.platform.app.InstrumentationRegistry
+import androidx.test.runner.AndroidJUnitRunner
+import kotlin.test.BeforeTest
+
+abstract class BaseBenchmarkTest {
+
+    protected lateinit var runner: AndroidJUnitRunner
+    protected lateinit var context: Context
+    protected lateinit var choreographer: Choreographer
+
+    @BeforeTest
+    fun baseSetUp() {
+        runner = InstrumentationRegistry.getInstrumentation() as AndroidJUnitRunner
+        context = ApplicationProvider.getApplicationContext()
+        context.cacheDir.deleteRecursively()
+        // Must run on the main thread to get the main thread choreographer.
+        runner.runOnMainSync {
+            choreographer = Choreographer.getInstance()
+        }
+    }
+}

sentry-android-integration-tests/sentry-uitest-android-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/SdkBenchmarkTest.kt

@@ -0,0 +1,58 @@
+package io.sentry.uitest.android.benchmark
+
+import androidx.lifecycle.Lifecycle
+import androidx.test.core.app.launchActivity
+import androidx.test.ext.junit.runners.AndroidJUnit4
+import io.sentry.Sentry
+import io.sentry.android.core.SentryAndroid
+import io.sentry.uitest.android.benchmark.util.BenchmarkOperation
+import org.junit.runner.RunWith
+import java.util.concurrent.TimeUnit
+import kotlin.test.Test
+import kotlin.test.assertTrue
+
+@RunWith(AndroidJUnit4::class)
+class SdkBenchmarkTest : BaseBenchmarkTest() {
+
+    @Test
+    fun benchmarkSdkInit() {
+
+        // We compare starting an activity with and without the sdk init, to measure its impact on startup time.
+        val opNoSdk = getOperation()
+        val opSimpleSdk = getOperation {
+            SentryAndroid.init(context) {
+                it.dsn = "https://key@host/proj"
+            }
+        }
+        val opNoSdk2 = getOperation()
+        val opPerfProfilingSdk = getOperation {
+            SentryAndroid.init(context) {
+                it.dsn = "https://key@host/proj"
+                it.isProfilingEnabled = true
+                it.tracesSampleRate = 1.0
+            }
+        }
+        val simpleSdkResult = BenchmarkOperation.compare(opNoSdk, "No Sdk", opSimpleSdk, "Simple Sdk")
+        val perfProfilingSdkResult = BenchmarkOperation.compare(opNoSdk2, "No Sdk", opPerfProfilingSdk, "Sdk with perf and profiling")
+
+        val maxDurationThreshold = TimeUnit.MILLISECONDS.toNanos(100)
+        assertTrue(simpleSdkResult.durationIncreaseNanos in 0..maxDurationThreshold)
+        assertTrue(simpleSdkResult.cpuTimeIncreaseMillis in 0..100)
+        assertTrue(perfProfilingSdkResult.durationIncreaseNanos in simpleSdkResult.durationIncreaseNanos..maxDurationThreshold)
+        assertTrue(perfProfilingSdkResult.cpuTimeIncreaseMillis in simpleSdkResult.cpuTimeIncreaseMillis..100)
+    }
+
+    private fun getOperation(init: (() -> Unit)? = null) = BenchmarkOperation(
+        choreographer,
+        op = {
+            runner.runOnMainSync {
+                init?.invoke()
+            }
+            val benchmarkScenario = launchActivity<BenchmarkActivity>()
+            benchmarkScenario.moveToState(Lifecycle.State.DESTROYED)
+        },
+        after = {
+            Sentry.close()
+        }
+    )
+}