For #26792 - Change active Firebase Test Lab emulator from x86 to ARM (…

…#26793) * For #26792 - Change active Firebase Test Lab emulator from x86 to ARM Update startup-test kind.yml Jobs for ARM Update ui-test kind.yml Jobs for ARM Cleans up and renames Flank configurations for ARM Updates ui-test.sh for ARM * Fix BookmarksTest.kt and HomeScreenTest.kt * Fix intent matcher for SettingsAboutTest Adds a helper function for Play Store intent * Fix SettingsSearchTest.kt * ktlint fix TestHelper.kt and SettingsRobot.kt
mozilla-mobile · Sep 14, 2022 · 4e99fe8 · 4e99fe8
1 parent 2899a66
commit 4e99fe8
Show file tree

Hide file tree

Showing 21 changed files with 255 additions and 453 deletions.
diff --git a/app/src/androidTest/java/org/mozilla/fenix/helpers/TestHelper.kt b/app/src/androidTest/java/org/mozilla/fenix/helpers/TestHelper.kt
@@ -115,6 +115,12 @@ object TestHelper {
         )
     }
 
+    fun waitUntilSnackbarGone() {
+        mDevice.findObject(
+            UiSelector().resourceId("$packageName:id/snackbar_layout"),
+        ).waitUntilGone(waitingTime)
+    }
+
     fun verifyUrl(urlSubstring: String, resourceName: String, resId: Int) {
         waitUntilObjectIsFound(resourceName)
         mDevice.findObject(UiSelector().text(urlSubstring)).waitForExists(waitingTime)
@@ -252,6 +258,18 @@ object TestHelper {
         }
     }
 
+    fun assertPlayStoreOpens() {
+        if (isPackageInstalled(Constants.PackageName.GOOGLE_PLAY_SERVICES)) {
+            try {
+                intended(toPackage(Constants.PackageName.GOOGLE_PLAY_SERVICES))
+            } catch (e: AssertionFailedError) {
+                BrowserRobot().verifyRateOnGooglePlayURL()
+            }
+        } else {
+            BrowserRobot().verifyRateOnGooglePlayURL()
+        }
+    }
+
     /**
      * Checks whether the latest activity of the application is used for custom tabs or PWAs.
      *

diff --git a/app/src/androidTest/java/org/mozilla/fenix/ui/BookmarksTest.kt b/app/src/androidTest/java/org/mozilla/fenix/ui/BookmarksTest.kt
@@ -26,6 +26,7 @@ import org.mozilla.fenix.helpers.HomeActivityTestRule
 import org.mozilla.fenix.helpers.RecyclerViewIdlingResource
 import org.mozilla.fenix.helpers.RetryTestRule
 import org.mozilla.fenix.helpers.TestAssetHelper
+import org.mozilla.fenix.helpers.TestHelper
 import org.mozilla.fenix.helpers.TestHelper.longTapSelectItem
 import org.mozilla.fenix.ui.robots.bookmarksMenu
 import org.mozilla.fenix.ui.robots.browserScreen
@@ -526,6 +527,8 @@ class BookmarksTest {
             confirmDeletion()
             verifyDeleteSnackBarText()
             verifyFolderTitle("3")
+            // On some devices we need to wait for the Snackbar to be gone before continuing
+            TestHelper.waitUntilSnackbarGone()
         }.closeMenu {
         }
 

diff --git a/app/src/androidTest/java/org/mozilla/fenix/ui/SettingsAboutTest.kt b/app/src/androidTest/java/org/mozilla/fenix/ui/SettingsAboutTest.kt
@@ -15,6 +15,7 @@ import org.junit.Rule
 import org.junit.Test
 import org.mozilla.fenix.ext.settings
 import org.mozilla.fenix.helpers.AndroidAssetDispatcher
+import org.mozilla.fenix.helpers.FeatureSettingsHelper
 import org.mozilla.fenix.helpers.HomeActivityIntentTestRule
 import org.mozilla.fenix.helpers.RetryTestRule
 import org.mozilla.fenix.helpers.TestHelper.mDevice
@@ -31,6 +32,7 @@ class SettingsAboutTest {
 
     private lateinit var mDevice: UiDevice
     private lateinit var mockWebServer: MockWebServer
+    private val featureSettingsHelper = FeatureSettingsHelper()
 
     @get:Rule
     val activityIntentTestRule = HomeActivityIntentTestRule()
@@ -51,6 +53,7 @@ class SettingsAboutTest {
     @After
     fun tearDown() {
         mockWebServer.shutdown()
+        featureSettingsHelper.resetAllFeatureFlags()
     }
 
     // Walks through settings menu and sub-menus to ensure all items are present
@@ -70,6 +73,8 @@ class SettingsAboutTest {
     // ABOUT
     @Test
     fun verifyRateOnGooglePlayRedirect() {
+        val settings = activityIntentTestRule.activity.settings()
+        settings.shouldShowTotalCookieProtectionCFR = false
         homeScreen {
         }.openThreeDotMenu {
         }.openSettings {

diff --git a/app/src/androidTest/java/org/mozilla/fenix/ui/SettingsAdvancedTest.kt b/app/src/androidTest/java/org/mozilla/fenix/ui/SettingsAdvancedTest.kt
@@ -13,11 +13,10 @@ import org.junit.Rule
 import org.junit.Test
 import org.mozilla.fenix.customannotations.SmokeTest
 import org.mozilla.fenix.helpers.AndroidAssetDispatcher
-import org.mozilla.fenix.helpers.Constants.PackageName.GOOGLE_PLAY_SERVICES
 import org.mozilla.fenix.helpers.FeatureSettingsHelper
 import org.mozilla.fenix.helpers.HomeActivityIntentTestRule
 import org.mozilla.fenix.helpers.TestAssetHelper
-import org.mozilla.fenix.helpers.TestHelper.assertNativeAppOpens
+import org.mozilla.fenix.helpers.TestHelper
 import org.mozilla.fenix.ui.robots.homeScreen
 import org.mozilla.fenix.ui.robots.navigationToolbar
 
@@ -76,7 +75,6 @@ class SettingsAdvancedTest {
     @Test
     fun openLinkInAppTest() {
         val defaultWebPage = TestAssetHelper.getGenericAsset(mockWebServer, 3)
-        val playStoreUrl = "play.google.com/store/apps/details?id=org.mozilla.fenix"
 
         homeScreen {
         }.openThreeDotMenu {
@@ -91,7 +89,7 @@ class SettingsAdvancedTest {
             mDevice.waitForIdle()
             clickLinkMatchingText("Mozilla Playstore link")
             mDevice.waitForIdle()
-            assertNativeAppOpens(GOOGLE_PLAY_SERVICES, playStoreUrl)
+            TestHelper.assertPlayStoreOpens()
         }
     }
 }
diff --git a/app/src/androidTest/java/org/mozilla/fenix/ui/SettingsSearchTest.kt b/app/src/androidTest/java/org/mozilla/fenix/ui/SettingsSearchTest.kt
@@ -35,6 +35,7 @@ class SettingsSearchTest {
         }
         featureSettingsHelper.setJumpBackCFREnabled(false)
         featureSettingsHelper.setShowWallpaperOnboarding(false)
+        featureSettingsHelper.setTCPCFREnabled(false)
     }
 
     @After

diff --git a/app/src/androidTest/java/org/mozilla/fenix/ui/robots/SettingsRobot.kt b/app/src/androidTest/java/org/mozilla/fenix/ui/robots/SettingsRobot.kt
@@ -6,6 +6,8 @@
 
 package org.mozilla.fenix.ui.robots
 
+import android.content.Intent
+import android.net.Uri
 import androidx.recyclerview.widget.RecyclerView
 import androidx.test.espresso.Espresso.onView
 import androidx.test.espresso.ViewInteraction
@@ -14,7 +16,8 @@ import androidx.test.espresso.assertion.ViewAssertions.matches
 import androidx.test.espresso.contrib.RecyclerViewActions
 import androidx.test.espresso.intent.Intents.intended
 import androidx.test.espresso.intent.matcher.IntentMatchers
-import androidx.test.espresso.intent.matcher.IntentMatchers.toPackage
+import androidx.test.espresso.intent.matcher.IntentMatchers.hasAction
+import androidx.test.espresso.intent.matcher.IntentMatchers.hasData
 import androidx.test.espresso.matcher.ViewMatchers
 import androidx.test.espresso.matcher.ViewMatchers.Visibility
 import androidx.test.espresso.matcher.ViewMatchers.hasDescendant
@@ -33,6 +36,7 @@ import androidx.test.uiautomator.UiObject
 import androidx.test.uiautomator.UiScrollable
 import androidx.test.uiautomator.UiSelector
 import androidx.test.uiautomator.Until
+import junit.framework.AssertionFailedError
 import org.hamcrest.CoreMatchers
 import org.hamcrest.Matchers.allOf
 import org.hamcrest.Matchers.endsWith
@@ -49,6 +53,7 @@ import org.mozilla.fenix.helpers.TestHelper.mDevice
 import org.mozilla.fenix.helpers.TestHelper.packageName
 import org.mozilla.fenix.helpers.TestHelper.scrollToElementByText
 import org.mozilla.fenix.helpers.click
+import org.mozilla.fenix.settings.SupportUtils
 import org.mozilla.fenix.ui.robots.SettingsRobot.Companion.DEFAULT_APPS_SETTINGS_ACTION
 
 /**
@@ -128,7 +133,6 @@ class SettingsRobot {
         fun openAboutFirefoxPreview(interact: SettingsSubMenuAboutRobot.() -> Unit):
             SettingsSubMenuAboutRobot.Transition {
             aboutFirefoxHeading().click()
-
             SettingsSubMenuAboutRobot().interact()
             return SettingsSubMenuAboutRobot.Transition()
         }
@@ -583,7 +587,16 @@ fun clickRateButtonGooglePlay() {
 
 private fun assertGooglePlayRedirect() {
     if (isPackageInstalled(GOOGLE_PLAY_SERVICES)) {
-        intended(toPackage(GOOGLE_PLAY_SERVICES))
+        try {
+            intended(
+                allOf(
+                    hasAction(Intent.ACTION_VIEW),
+                    hasData(Uri.parse(SupportUtils.RATE_APP_URL)),
+                ),
+            )
+        } catch (e: AssertionFailedError) {
+            BrowserRobot().verifyRateOnGooglePlayURL()
+        }
     } else {
         BrowserRobot().verifyRateOnGooglePlayURL()
     }

diff --git a/automation/taskcluster/androidTest/flank-arm-beta.yml b/automation/taskcluster/androidTest/flank-arm-beta.yml
@@ -0,0 +1,42 @@
+# Google Cloud Documentation: https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run
+# Flank Documentation: https://flank.github.io/flank/
+gcloud:
+  results-bucket: fenix_test_artifacts
+  record-video: true
+
+  timeout: 30m
+  async: false
+  num-flaky-test-attempts: 1
+
+  app: /app/path
+  test: /test/path
+
+  auto-google-login: false
+  use-orchestrator: true
+  environment-variables:
+    clearPackageData: true
+  directories-to-pull:
+    - /sdcard/screenshots
+  performance-metrics: true
+
+  test-targets:
+    - class org.mozilla.fenix.ui.NavigationToolbarTest#visitURLTest
+    - class org.mozilla.fenix.ui.HistoryTest#visitedUrlHistoryTest
+    - class org.mozilla.fenix.ui.SmokeTest#openMainMenuSettingsItemTest
+    - class org.mozilla.fenix.ui.SettingsSearchTest#toggleSearchSuggestionsTest
+    - class org.mozilla.fenix.ui.CollectionTest#deleteCollectionTest
+    - class org.mozilla.fenix.ui.SmokeTest#noHistoryInPrivateBrowsingTest
+    - class org.mozilla.fenix.ui.NoNetworkAccessStartupTests#noNetworkConnectionStartupTest
+
+  device:
+    - model: Pixel2.arm
+      version: 30
+      locale: en_US
+
+flank:
+  project: GOOGLE_PROJECT
+  max-test-shards: 1
+  num-test-runs: 1
+  output-style: compact
+  full-junit-result: true
+
diff --git a/automation/taskcluster/androidTest/flank-arm-legacy-api-tests.yml b/automation/taskcluster/androidTest/flank-arm-legacy-api-tests.yml
@@ -0,0 +1,47 @@
+# Google Cloud Documentation: https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run
+# Flank Documentation: https://flank.github.io/flank/
+gcloud:
+  results-bucket: fenix_test_artifacts
+  record-video: true
+
+  timeout: 30m
+  async: false
+  num-flaky-test-attempts: 2
+
+  app: /app/path
+  test: /test/path
+
+  auto-google-login: false
+  use-orchestrator: true
+  environment-variables:
+    clearPackageData: true
+  directories-to-pull:
+    - /sdcard/screenshots
+  performance-metrics: true
+
+  test-targets:
+    - class org.mozilla.fenix.ui.DownloadTest#pauseResumeCancelDownloadTest
+    - class org.mozilla.fenix.ui.SearchTest#scanButtonAllowPermissionTest
+    - class org.mozilla.fenix.ui.SearchTest#scanButtonDenyPermissionTest
+    - class org.mozilla.fenix.ui.SitePermissionsTest#audioVideoPermissionChoiceOnEachRequestTest
+    - class org.mozilla.fenix.ui.SitePermissionsTest#cameraPermissionChoiceOnEachRequestTest
+    - class org.mozilla.fenix.ui.SitePermissionsTest#microphonePermissionChoiceOnEachRequestTest
+    - class org.mozilla.fenix.ui.SitePermissionsTest#rememberAllowAudioVideoPermissionChoiceTest
+    - class org.mozilla.fenix.ui.SitePermissionsTest#rememberAllowCameraPermissionChoiceTest
+    - class org.mozilla.fenix.ui.SitePermissionsTest#rememberAllowMicrophonePermissionChoiceTest
+    - class org.mozilla.fenix.ui.SitePermissionsTest#rememberBlockAudioVideoPermissionChoiceTest
+    - class org.mozilla.fenix.ui.SitePermissionsTest#rememberBlockCameraPermissionChoiceTest
+    - class org.mozilla.fenix.ui.SitePermissionsTest#rememberBlockMicrophonePermissionChoiceTest
+    - class org.mozilla.fenix.ui.SmokeTest#redirectToAppPermissionsSystemSettingsTest
+
+  device:
+    - model: Pixel2.arm
+      version: 28
+      locale: en_US
+
+flank:
+  project: GOOGLE_PROJECT
+  max-test-shards: -1
+  num-test-runs: 1
+  output-style: compact
+  full-junit-result: true
diff --git a/automation/taskcluster/androidTest/flank-arm-screenshots-tests.yml b/automation/taskcluster/androidTest/flank-arm-screenshots-tests.yml
@@ -0,0 +1,35 @@
+# Google Cloud Documentation: https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run
+# Flank Documentation: https://flank.github.io/flank/
+gcloud:
+  results-bucket: fenix_test_artifacts
+  record-video: true
+
+  timeout: 30m
+  async: false
+  num-flaky-test-attempts: 1
+
+  app: /app/path
+  test: /test/path
+
+  auto-google-login: false
+  use-orchestrator: true
+  environment-variables:
+    clearPackageData: true
+  directories-to-pull:
+    - /sdcard/screenshots
+  performance-metrics: true
+
+  test-targets:
+   - package org.mozilla.fenix.screenshots
+
+  device:
+   - model: Pixel2.arm
+     version: 30
+     locale: en_US
+
+flank:
+  project: GOOGLE_PROJECT
+  max-test-shards: 1
+  num-test-runs: 1
+  output-style: compact
+  full-junit-result: true
diff --git a/automation/taskcluster/androidTest/flank-arm-start-test.yml b/automation/taskcluster/androidTest/flank-arm-start-test.yml
@@ -0,0 +1,41 @@
+# Google Cloud Documentation: https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run
+# Flank Documentation: https://flank.github.io/flank/
+gcloud:
+  results-bucket: fenix_test_artifacts
+  record-video: true
+
+  timeout: 30m
+  async: false
+  num-flaky-test-attempts: 1
+
+  app: /app/path
+  test: /test/path
+
+  auto-google-login: false
+  use-orchestrator: true
+  environment-variables:
+    clearPackageData: true
+  directories-to-pull:
+    - /sdcard/screenshots
+  performance-metrics: true
+
+  test-targets:
+   - class org.mozilla.fenix.ui.NavigationToolbarTest#visitURLTest
+   - class org.mozilla.fenix.ui.HistoryTest#visitedUrlHistoryTest
+   - class org.mozilla.fenix.ui.SmokeTest#openMainMenuSettingsItemTest
+   - class org.mozilla.fenix.ui.SettingsSearchTest#toggleSearchSuggestionsTest
+   - class org.mozilla.fenix.ui.CollectionTest#deleteCollectionTest
+   - class org.mozilla.fenix.ui.SmokeTest#noHistoryInPrivateBrowsingTest
+   - class org.mozilla.fenix.ui.NoNetworkAccessStartupTests#noNetworkConnectionStartupTest
+
+  device:
+   - model: Pixel2.arm
+     version: 30
+     locale: en_US
+
+flank:
+  project: GOOGLE_PROJECT
+  max-test-shards: 1
+  num-test-runs: 1
+  output-style: compact
+  full-junit-result: true