Fixed checkEnvelopeProfiledTransaction ui test (#2752)

* increased saucelabs timeout on all tests
* increased concurrency on ui tests
* updated saucelabs build name
* reenabled and fixed checkEnvelopeProfiledTransaction test (skipped on agp-matrix workflow)

.github/workflows/integration-tests-benchmarks.yml

@@ -30,8 +30,9 @@ jobs:
           java-version: '17'
           distribution: 'temurin'
 
-      # Clean, build and release a test apk
+      # Clean, build and release a test apk, but only if we will run the benchmark
       - name: Make assembleBenchmarks
+        if: env.SAUCE_USERNAME != null
         run: make assembleBenchmarks
 
       # We stop gradle at the end to make sure the cache folders

.github/workflows/integration-tests-ui.yml

@@ -33,8 +33,9 @@ jobs:
           java-version: '17'
           distribution: 'temurin'
 
-      # Clean, build and release a test apk
+      # Clean, build and release a test apk, but only if we will run the benchmark
       - name: Make assembleUiTests
+        if: env.SAUCE_USERNAME != null
         run: make assembleUiTests
 
       # We stop gradle at the end to make sure the cache folders

.sauce/sentry-uitest-android-benchmark-lite.yml

@@ -2,15 +2,15 @@ apiVersion: v1alpha
 kind: espresso
 sauce:
   region: us-west-1
-  concurrency: 2
+  concurrency: 1
   metadata:
-    build: sentry-uitest-android-benchmark-lite-$CI_COMMIT_SHORT_SHA
+    build: sentry-uitest-android-benchmark-lite-$GITHUB_REF-$GITHUB_SHA
     tags:
       - benchmarks
       - android
 
 defaults:
-  timeout: 20m
+  timeout: 40m
 
 espresso:
   app: ./sentry-android-integration-tests/sentry-uitest-android-benchmark/build/outputs/apk/release/sentry-uitest-android-benchmark-release.apk

.sauce/sentry-uitest-android-benchmark.yml

@@ -2,15 +2,15 @@ apiVersion: v1alpha
 kind: espresso
 sauce:
   region: us-west-1
-  concurrency: 2
+  concurrency: 3
   metadata:
-    build: sentry-uitest-android-benchmark-$CI_COMMIT_SHORT_SHA
+    build: sentry-uitest-android-benchmark-$GITHUB_REF-$GITHUB_SHA
     tags:
       - benchmarks
       - android
 
 defaults:
-  timeout: 30m
+  timeout: 90m
 
 espresso:
   app: ./sentry-android-integration-tests/sentry-uitest-android-benchmark/build/outputs/apk/release/sentry-uitest-android-benchmark-release.apk

.sauce/sentry-uitest-android-ui.yml

@@ -3,15 +3,15 @@ kind: espresso
 sauce:
   region: us-west-1
   # Controls how many suites are executed at the same time (sauce test env only).
-  concurrency: 1
+  concurrency: 4
   metadata:
-    build: sentry-uitest-android-ui@$CI_COMMIT_SHORT_SHA
+    build: sentry-uitest-android-ui-$GITHUB_REF-$GITHUB_SHA
     tags:
       - e2e
       - android
 
 defaults:
-  timeout: 30m
+  timeout: 40m
 
 espresso:
   app: ./sentry-android-integration-tests/sentry-uitest-android/build/outputs/apk/release/sentry-uitest-android-release.apk

sentry-android-integration-tests/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/EnvelopeTests.kt

@@ -15,10 +15,10 @@ import io.sentry.android.core.SentryAndroidOptions
 import io.sentry.assertEnvelopeItem
 import io.sentry.profilemeasurements.ProfileMeasurement
 import io.sentry.protocol.SentryTransaction
+import org.junit.Assume.assumeNotNull
 import org.junit.runner.RunWith
 import java.io.File
 import java.util.concurrent.TimeUnit
-import kotlin.test.Ignore
 import kotlin.test.Test
 import kotlin.test.assertEquals
 import kotlin.test.assertNotNull
@@ -44,7 +44,6 @@ class EnvelopeTests : BaseUiTest() {
         }
     }
 
-    @Ignore("Something is wrong with measurements assertions, flaking almost every time")
     @Test
     fun checkEnvelopeProfiledTransaction() {
         initSentry(true) { options: SentryAndroidOptions ->
@@ -89,10 +88,14 @@ class EnvelopeTests : BaseUiTest() {
                 // We check the measurements have been collected with expected units
                 val slowFrames = profilingTraceData.measurementsMap[ProfileMeasurement.ID_SLOW_FRAME_RENDERS]
                 val frozenFrames = profilingTraceData.measurementsMap[ProfileMeasurement.ID_FROZEN_FRAME_RENDERS]
-                val frameRates = profilingTraceData.measurementsMap[ProfileMeasurement.ID_SCREEN_FRAME_RATES]!!
+                val frameRates = profilingTraceData.measurementsMap[ProfileMeasurement.ID_SCREEN_FRAME_RATES]
                 val memoryStats = profilingTraceData.measurementsMap[ProfileMeasurement.ID_MEMORY_FOOTPRINT]
                 val memoryNativeStats = profilingTraceData.measurementsMap[ProfileMeasurement.ID_MEMORY_NATIVE_FOOTPRINT]
                 val cpuStats = profilingTraceData.measurementsMap[ProfileMeasurement.ID_CPU_USAGE]
+
+                // Frame rate could be null in headless emulator tests (agp-matrix workflow)
+                assumeNotNull(frameRates)
+
                 // Slow and frozen frames can be null (in case there were none)
                 if (slowFrames != null) {
                     assertEquals(ProfileMeasurement.UNIT_NANOSECONDS, slowFrames.unit)
@@ -101,8 +104,9 @@ class EnvelopeTests : BaseUiTest() {
                     assertEquals(ProfileMeasurement.UNIT_NANOSECONDS, frozenFrames.unit)
                 }
                 // There could be no slow/frozen frames, but we expect at least one frame rate
-                assertEquals(ProfileMeasurement.UNIT_HZ, frameRates.unit)
+                assertEquals(ProfileMeasurement.UNIT_HZ, frameRates!!.unit)
                 assertTrue(frameRates.values.isNotEmpty())
+
                 memoryStats?.let {
                     assertEquals(ProfileMeasurement.UNIT_BYTES, it.unit)
                     assertEquals(true, it.values.isNotEmpty())
@@ -116,25 +120,29 @@ class EnvelopeTests : BaseUiTest() {
                     assertEquals(true, it.values.isNotEmpty())
                 }
 
-                // We allow measurements to be added since the start up to the end of the profile, with a small tolerance due to threading
-                val maxTimestampAllowed = profilingTraceData.durationNs.toLong() + TimeUnit.SECONDS.toNanos(2)
-                val allMeasurements = profilingTraceData.measurementsMap.values
+                // We allow measurements to be added since the start up to the end of the profile
+                val maxTimestampAllowed = profilingTraceData.durationNs.toLong()
 
                 profilingTraceData.measurementsMap.entries.filter {
-                    it.value.values.isNotEmpty()
+                    // We need at least two measurements, as one will be dropped by our tests
+                    it.value.values.size > 1
                 }.forEach { entry ->
                     val name = entry.key
                     val measurement = entry.value
 
-                    val values = measurement.values.sortedBy { it.relativeStartNs.toLong() }
+                    // The last frame measurement could be outside the transaction duration,
+                    //  since when the transaction finishes the frame callback is removed from the activity,
+                    //  but internally it is already cached and will be called anyway in the next frame.
+                    val values = measurement.values.sortedBy { it.relativeStartNs.toLong() }.dropLast(1)
+
                     // There should be no measurement before the profile starts
                     assertTrue(
                         values.first().relativeStartNs.toLong() > 0,
                         "First measurement value for '$name' is <=0"
                     )
                     // There should be no measurement after the profile ends
                     assertTrue(
-                        values.last().relativeStartNs.toLong() < maxTimestampAllowed,
+                        values.last().relativeStartNs.toLong() <= maxTimestampAllowed,
                         "Last measurement value for '$name' is outside bounds (was: ${values.last().relativeStartNs.toLong()}ns, max: ${maxTimestampAllowed}ns"
                     )