-
-
Notifications
You must be signed in to change notification settings - Fork 427
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Tests/android UI tests absolute (#2077)
- Loading branch information
1 parent
e4a46aa
commit a83efb8
Showing
13 changed files
with
319 additions
and
97 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,38 @@ | ||
| name: "Integration Tests - Benchmarks" | ||
| on: | ||
| push: | ||
| branches: | ||
| - main | ||
| - release/** | ||
| pull_request: | ||
|
|
||
| jobs: | ||
| test: | ||
| name: Benchmarks | ||
| runs-on: ubuntu-latest | ||
|
|
||
| steps: | ||
| - name: Git checkout | ||
| uses: actions/checkout@v2 | ||
|
|
||
| - name: "Set up Java: 11" | ||
| uses: actions/setup-java@v2 | ||
| with: | ||
| java-version: "11" | ||
| distribution: "adopt" | ||
|
|
||
| # Clean, build and release a test apk | ||
| - name: Make assembleBenchmarks | ||
| run: make assembleBenchmarks | ||
|
|
||
| # We stop gradle at the end to make sure the cache folders | ||
| # don't contain any lock files and are free to be cached. | ||
| - name: Make stop | ||
| run: make stop | ||
|
|
||
| - name: Run Tests in SauceLab | ||
| uses: saucelabs/saucectl-run-action@v1 | ||
| with: | ||
| sauce-username: ${{ secrets.SAUCE_USERNAME }} | ||
| sauce-access-key: ${{ secrets.SAUCE_ACCESS_KEY }} | ||
| config-file: .sauce/sentry-uitest-android-benchmark.yml |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,38 @@ | ||
| name: "Integration Tests - Ui tests" | ||
| on: | ||
| push: | ||
| branches: | ||
| - main | ||
| - release/** | ||
| pull_request: | ||
|
|
||
| jobs: | ||
| test: | ||
| name: Ui tests | ||
| runs-on: ubuntu-latest | ||
|
|
||
| steps: | ||
| - name: Git checkout | ||
| uses: actions/checkout@v2 | ||
|
|
||
| - name: "Set up Java: 11" | ||
| uses: actions/setup-java@v2 | ||
| with: | ||
| java-version: "11" | ||
| distribution: "adopt" | ||
|
|
||
| # Clean, build and release a test apk | ||
| - name: Make assembleUiTests | ||
| run: make assembleUiTests | ||
|
|
||
| # We stop gradle at the end to make sure the cache folders | ||
| # don't contain any lock files and are free to be cached. | ||
| - name: Make stop | ||
| run: make stop | ||
|
|
||
| - name: Run Tests in SauceLab | ||
| uses: saucelabs/saucectl-run-action@v1 | ||
| with: | ||
| sauce-username: ${{ secrets.SAUCE_USERNAME }} | ||
| sauce-access-key: ${{ secrets.SAUCE_ACCESS_KEY }} | ||
| config-file: .sauce/sentry-uitest-android-ui.yml |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
26 changes: 26 additions & 0 deletions
26
...id-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/BaseBenchmarkTest.kt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,26 @@ | ||
| package io.sentry.uitest.android.benchmark | ||
|
|
||
| import android.content.Context | ||
| import android.view.Choreographer | ||
| import androidx.test.core.app.ApplicationProvider | ||
| import androidx.test.platform.app.InstrumentationRegistry | ||
| import androidx.test.runner.AndroidJUnitRunner | ||
| import kotlin.test.BeforeTest | ||
|
|
||
| abstract class BaseBenchmarkTest { | ||
|
|
||
| protected lateinit var runner: AndroidJUnitRunner | ||
| protected lateinit var context: Context | ||
| protected lateinit var choreographer: Choreographer | ||
|
|
||
| @BeforeTest | ||
| fun baseSetUp() { | ||
| runner = InstrumentationRegistry.getInstrumentation() as AndroidJUnitRunner | ||
| context = ApplicationProvider.getApplicationContext() | ||
| context.cacheDir.deleteRecursively() | ||
| // Must run on the main thread to get the main thread choreographer. | ||
| runner.runOnMainSync { | ||
| choreographer = Choreographer.getInstance() | ||
| } | ||
| } | ||
| } |
58 changes: 58 additions & 0 deletions
58
...oid-benchmark/src/androidTest/java/io/sentry/uitest/android/benchmark/SdkBenchmarkTest.kt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,58 @@ | ||
| package io.sentry.uitest.android.benchmark | ||
|
|
||
| import androidx.lifecycle.Lifecycle | ||
| import androidx.test.core.app.launchActivity | ||
| import androidx.test.ext.junit.runners.AndroidJUnit4 | ||
| import io.sentry.Sentry | ||
| import io.sentry.android.core.SentryAndroid | ||
| import io.sentry.uitest.android.benchmark.util.BenchmarkOperation | ||
| import org.junit.runner.RunWith | ||
| import java.util.concurrent.TimeUnit | ||
| import kotlin.test.Test | ||
| import kotlin.test.assertTrue | ||
|
|
||
| @RunWith(AndroidJUnit4::class) | ||
| class SdkBenchmarkTest : BaseBenchmarkTest() { | ||
|
|
||
| @Test | ||
| fun benchmarkSdkInit() { | ||
|
|
||
| // We compare starting an activity with and without the sdk init, to measure its impact on startup time. | ||
| val opNoSdk = getOperation() | ||
| val opSimpleSdk = getOperation { | ||
| SentryAndroid.init(context) { | ||
| it.dsn = "https://key@host/proj" | ||
| } | ||
| } | ||
| val opNoSdk2 = getOperation() | ||
| val opPerfProfilingSdk = getOperation { | ||
| SentryAndroid.init(context) { | ||
| it.dsn = "https://key@host/proj" | ||
| it.isProfilingEnabled = true | ||
| it.tracesSampleRate = 1.0 | ||
| } | ||
| } | ||
| val simpleSdkResult = BenchmarkOperation.compare(opNoSdk, "No Sdk", opSimpleSdk, "Simple Sdk") | ||
| val perfProfilingSdkResult = BenchmarkOperation.compare(opNoSdk2, "No Sdk", opPerfProfilingSdk, "Sdk with perf and profiling") | ||
|
|
||
| val maxDurationThreshold = TimeUnit.MILLISECONDS.toNanos(100) | ||
| assertTrue(simpleSdkResult.durationIncreaseNanos in 0..maxDurationThreshold) | ||
| assertTrue(simpleSdkResult.cpuTimeIncreaseMillis in 0..100) | ||
| assertTrue(perfProfilingSdkResult.durationIncreaseNanos in simpleSdkResult.durationIncreaseNanos..maxDurationThreshold) | ||
| assertTrue(perfProfilingSdkResult.cpuTimeIncreaseMillis in simpleSdkResult.cpuTimeIncreaseMillis..100) | ||
| } | ||
|
|
||
| private fun getOperation(init: (() -> Unit)? = null) = BenchmarkOperation( | ||
| choreographer, | ||
| op = { | ||
| runner.runOnMainSync { | ||
| init?.invoke() | ||
| } | ||
| val benchmarkScenario = launchActivity<BenchmarkActivity>() | ||
| benchmarkScenario.moveToState(Lifecycle.State.DESTROYED) | ||
| }, | ||
| after = { | ||
| Sentry.close() | ||
| } | ||
| ) | ||
| } |
Oops, something went wrong.