Bug 1729776 - Enable the Firefox Suggest "offline" scenario by defaul…

…t for users in the US region with en-* locales. r=mythmon,daleharvey,mak

Enable the Firefox Suggest "offline" scenario by default for users in the US
region with en-* locales.

Previously we relied on Nimbus to enable the offline scenario, and the goal here
is to make it permanent for all users in the US `home` region using en-* locales
so that we don't need Nimbus for it anymore.

With Nimbus, there were two essential mechanisms that restricted the scenario to
the desired population: the `browser.urlbar.quicksuggest.enabled` pref, which is
a global toggle for Firefox Suggest suggestions regardless of region and locale,
and a Nimbus recipe that enabled the pref for US en-* users only.

Without Nimbus, we have only the `browser.urlbar.quicksuggest.enabled` pref. We
can't rely on a server-side solution to target a specific population, so we need
to do it in the client. This patch keeps the default `false` value of
`browser.urlbar.quicksuggest.enabled` in firefox.js, and then it sets a new
default-branch value for the pref for the US en-* population on app startup.

There's actually a set of prefs related to the offline scenario that need to be
set, not only `browser.urlbar.quicksuggest.enabled`.

Depends on D124943

Differential Revision: https://phabricator.services.mozilla.com/D125024

browser/components/BrowserGlue.jsm

@@ -1646,6 +1646,8 @@ BrowserGlue.prototype = {
 
     ProcessHangMonitor.init();
 
+    UrlbarPrefs.maybeEnableOfflineQuickSuggest();
+
     // A channel for "remote troubleshooting" code...
     let channel = new WebChannel(
       "remote-troubleshooting",

browser/components/preferences/privacy.js

@@ -1925,13 +1925,15 @@ var gPrivacyPane = {
    * Initializes the address bar section.
    */
   _initAddressBar() {
-    // Update the Firefox Suggest section on Nimbus changes.
-    this._updateFirefoxSuggestSection = this._updateFirefoxSuggestSection.bind(
-      this
-    );
-    NimbusFeatures.urlbar.onUpdate(this._updateFirefoxSuggestSection);
+    // Update the Firefox Suggest section on Nimbus changes. Note that
+    // `onUpdate` passes the Nimbus feature name as its first arg but that is
+    // not the arg that `_updateFirefoxSuggestSection` expects, so we do not
+    // want to pass it on.
+    this._firefoxSuggestNimbusUpdate = () =>
+      this._updateFirefoxSuggestSection();
+    NimbusFeatures.urlbar.onUpdate(this._firefoxSuggestNimbusUpdate);
     window.addEventListener("unload", () =>
-      NimbusFeatures.urlbar.off(this._updateFirefoxSuggestSection)
+      NimbusFeatures.urlbar.off(this._firefoxSuggestNimbusUpdate)
     );
 
     // Set up the sponsored checkbox. When the main checkbox is checked, the

browser/components/preferences/tests/browser_privacy_firefoxSuggest.js

@@ -53,7 +53,7 @@ add_task(async function() {
   await doVisibilityTest({
     initialDefaultBranchValue: undefined,
     initialUserBranchValue: undefined,
-    initialExpectedVisibility: false,
+    initialExpectedVisibility: true,
     newDefaultBranchValue: false,
     newUserBranchValue: false,
     newExpectedVisibility: false,
@@ -86,7 +86,7 @@ add_task(async function() {
   await doVisibilityTest({
     initialDefaultBranchValue: undefined,
     initialUserBranchValue: undefined,
-    initialExpectedVisibility: false,
+    initialExpectedVisibility: true,
     newDefaultBranchValue: true,
     newUserBranchValue: undefined,
     newExpectedVisibility: true,
@@ -97,7 +97,7 @@ add_task(async function() {
   await doVisibilityTest({
     initialDefaultBranchValue: undefined,
     initialUserBranchValue: undefined,
-    initialExpectedVisibility: false,
+    initialExpectedVisibility: true,
     newDefaultBranchValue: undefined,
     newUserBranchValue: true,
     newExpectedVisibility: true,
@@ -133,7 +133,7 @@ add_task(async function() {
     initialExpectedVisibility: false,
     newDefaultBranchValue: true,
     newUserBranchValue: undefined,
-    newExpectedVisibility: true,
+    newExpectedVisibility: false,
   });
 });
 

browser/components/urlbar/UrlbarPrefs.jsm

@@ -19,6 +19,7 @@ const { XPCOMUtils } = ChromeUtils.import(
 
 XPCOMUtils.defineLazyModuleGetters(this, {
   NimbusFeatures: "resource://nimbus/ExperimentAPI.jsm",
+  Region: "resource://gre/modules/Region.jsm",
   UrlbarUtils: "resource:///modules/UrlbarUtils.jsm",
 });
 
@@ -525,6 +526,38 @@ class Preferences {
     );
   }
 
+  /**
+   * Depending on certain conditions [1], possibly enables on the default prefs
+   * branch the Firefox Suggest "offline" scenario, which means Firefox Suggest
+   * (quick suggest) will be fully enabled by default without showing onboarding
+   * and without sending data to Mozilla [2]. Users can opt out in the prefs UI,
+   * which will set overriding prefs on the user branch. Note that values set
+   * programatically on the default branch like this do not persist across app
+   * restarts, so this needs to be called on every startup until these pref
+   * values are codified in firefox.js.
+   *
+   * [1] Currently the conditions are: the user's home region must be US and
+   * their locale must be en-*
+   *
+   * [2] In contrast, the "online" scenario sends data to Mozilla and requires
+   * user opt-in via onboarding before Firefox Suggest is fully enabled.
+   */
+  async maybeEnableOfflineQuickSuggest() {
+    // `Region.home` is null before init finishes, so await it.
+    await Region.init();
+    if (
+      Region.home == "US" &&
+      Services.locale.appLocaleAsBCP47.substring(0, 2) == "en"
+    ) {
+      let prefs = Services.prefs.getDefaultBranch("browser.urlbar.");
+      prefs.setBoolPref("quicksuggest.enabled", true);
+      prefs.setCharPref("quicksuggest.scenario", "offline");
+      prefs.setBoolPref("quicksuggest.shouldShowOnboardingDialog", false);
+      prefs.setBoolPref("suggest.quicksuggest", true);
+      prefs.setBoolPref("suggest.quicksuggest.sponsored", true);
+    }
+  }
+
   /**
    * Adds a preference observer.  Observers are held weakly.
    *

browser/components/urlbar/UrlbarQuickSuggest.jsm

@@ -300,6 +300,33 @@ class Suggestions {
    * are saved locally.
    */
   async _ensureAttachmentsDownloaded() {
+    // Make sure we don't re-enter this method, which can happen due to a cycle
+    // created by our remote settings sync listener as follows:
+    //
+    // Pref change -> onPrefChanged -> onEnabledUpdate -> _setupRemoteSettings
+    // -> _ensureAttachmentsDownloaded -> this._rs.get -> RemoteSettingsClient
+    // calls sync on itself -> RemoteSettingsClient emits a sync event ->
+    // _onSettingsSync -> _ensureAttachmentsDownloaded
+    //
+    // Because RemoteSettingsClient awaits when it emits its sync event, we get
+    // a deadlock in that call stack. Quick suggest will not be able to complete
+    // initialization and return suggestions until something else causes it to
+    // fetch the data again. Restarting the app also fixes it because it seems
+    // RemoteSettingsClient takes a different code path on initialization after
+    // restart, presumably because the data was successfully downloaded and
+    // cached before the deadlock.
+    if (this._ensureAttachmentsDownloadedRunning) {
+      return;
+    }
+    this._ensureAttachmentsDownloadedRunning = true;
+    try {
+      await this._ensureAttachmentsDownloadedHelper();
+    } finally {
+      this._ensureAttachmentsDownloadedRunning = false;
+    }
+  }
+
+  async _ensureAttachmentsDownloadedHelper() {
     log.info("_ensureAttachmentsDownloaded started");
     let dataOpts = { useCache: true };
     let data = await this._rs.get({ filters: { type: "data" } });

browser/components/urlbar/UrlbarView.jsm

@@ -2171,7 +2171,7 @@ class UrlbarView {
       );
     }
 
-    if (UrlbarPrefs.get("quicksuggest.enabled")) {
+    if (UrlbarPrefs.get("quickSuggestEnabled")) {
       idArgs.push({ id: "urlbar-result-action-sponsored" });
     }
 

browser/components/urlbar/tests/UrlbarTestUtils.jsm

@@ -21,10 +21,12 @@ XPCOMUtils.defineLazyModuleGetters(this, {
   PrivateBrowsingUtils: "resource://gre/modules/PrivateBrowsingUtils.jsm",
   Services: "resource://gre/modules/Services.jsm",
   setTimeout: "resource://gre/modules/Timer.jsm",
+  sinon: "resource://testing-common/Sinon.jsm",
   TestUtils: "resource://testing-common/TestUtils.jsm",
   UrlbarController: "resource:///modules/UrlbarController.jsm",
   UrlbarPrefs: "resource:///modules/UrlbarPrefs.jsm",
   UrlbarProvider: "resource:///modules/UrlbarUtils.jsm",
+  UrlbarQuickSuggest: "resource:///modules/UrlbarQuickSuggest.jsm",
   UrlbarSearchUtils: "resource:///modules/UrlbarSearchUtils.jsm",
   UrlbarUtils: "resource:///modules/UrlbarUtils.jsm",
 });
@@ -843,6 +845,33 @@ var UrlbarTestUtils = {
     });
     return doExperimentCleanup;
   },
+
+  /**
+   * Waits for quick suggest initialization to finish, ensures its data will not
+   * be updated again during the test, and also optionally sets it up with mock
+   * data.
+   *
+   * @param {array} [data]
+   *   Array of quick suggest data objects. If not given, then this function
+   *   won't set up any mock data.
+   */
+  async ensureQuickSuggestInit(data = null) {
+    this._testScope?.info("Awaiting UrlbarQuickSuggest.init");
+    await UrlbarQuickSuggest.init();
+    this._testScope?.info("Done awaiting UrlbarQuickSuggest.init");
+    let sandbox = sinon.createSandbox();
+    sandbox.stub(UrlbarQuickSuggest, "_ensureAttachmentsDownloadedHelper");
+    this._testScope?.registerCleanupFunction(() => sandbox.restore());
+    if (data) {
+      this._testScope?.info(
+        "Awaiting UrlbarQuickSuggest._processSuggestionsJSON"
+      );
+      await UrlbarQuickSuggest._processSuggestionsJSON(data);
+      this._testScope?.info(
+        "Done awaiting UrlbarQuickSuggest._processSuggestionsJSON"
+      );
+    }
+  },
 };
 
 UrlbarTestUtils.formHistory = {

browser/components/urlbar/tests/browser/browser_quicksuggest.js

@@ -57,41 +57,27 @@ async function assertNoQuickSuggestResults(win = window) {
 
 add_task(async function init() {
   await PlacesUtils.history.clear();
+  await PlacesUtils.bookmarks.eraseEverything();
   await UrlbarTestUtils.formHistory.clear();
 
-  Services.prefs.clearUserPref(SEEN_DIALOG_PREF);
-  Services.prefs.clearUserPref("browser.urlbar.quicksuggest.seenRestarts");
-
-  let doExperimentCleanup = await UrlbarTestUtils.enrollExperiment({
-    valueOverrides: {
-      quickSuggestEnabled: true,
-      quickSuggestShouldShowOnboardingDialog: true,
-    },
-  });
-
-  await UrlbarQuickSuggest.init();
-  await UrlbarQuickSuggest._processSuggestionsJSON(TEST_DATA);
-  let onEnabled = UrlbarQuickSuggest.onEnabledUpdate;
-  UrlbarQuickSuggest.onEnabledUpdate = () => {};
-
-  registerCleanupFunction(async function() {
-    await doExperimentCleanup();
-    UrlbarQuickSuggest.onEnabledUpdate = onEnabled;
-
-    // The onboarding test task causes prefs to be set, so clear them when done
-    // so we leave a blank slate for other tests.
-    UrlbarPrefs.clear("suggest.quicksuggest");
-    UrlbarPrefs.clear("suggest.quicksuggest.sponsored");
-    UrlbarPrefs.clear("quicksuggest.shouldShowOnboardingDialog");
-    UrlbarPrefs.clear("quicksuggest.showedOnboardingDialog");
-    UrlbarPrefs.clear("quicksuggest.seenRestarts");
-  });
+  await UrlbarTestUtils.ensureQuickSuggestInit(TEST_DATA);
 });
 
-// Tests the onboarding dialog. This task must run first because it fully
-// enables the feature (both sponsored and non-sponsored suggestions) by virtue
-// of showing the onboarding.
 add_task(async function test_onboarding() {
+  // Set up prefs so that onboarding will be shown.
+  await SpecialPowers.pushPrefEnv({
+    set: [
+      ["browser.urlbar.quicksuggest.shouldShowOnboardingDialog", true],
+      [
+        "browser.urlbar.quicksuggest.quicksuggest.showedOnboardingDialog",
+        false,
+      ],
+      ["browser.urlbar.quicksuggest.seenRestarts", 0],
+      ["browser.urlbar.suggest.quicksuggest", false],
+      ["browser.urlbar.suggest.quicksuggest.sponsored", false],
+    ],
+  });
+
   await UrlbarTestUtils.promiseAutocompleteResultPopup({
     window,
     value: "fra",
@@ -119,6 +105,13 @@ add_task(async function test_onboarding() {
 
   info("Waiting for dialog and pref change");
   await Promise.all([dialogPromise, prefPromise]);
+
+  await SpecialPowers.popPrefEnv();
+
+  // Clear prefs that are set by virtue of showing and accepting the onboarding.
+  UrlbarPrefs.clear("quicksuggest.shouldShowOnboardingDialog");
+  UrlbarPrefs.clear("quicksuggest.showedOnboardingDialog");
+  UrlbarPrefs.clear("quicksuggest.seenRestarts");
 });
 
 // Tests a sponsored result and keyword highlighting.

browser/components/urlbar/tests/browser/browser_quicksuggest_configuration.js

@@ -16,18 +16,27 @@ const SHOWED_ONBOARDING_DIALOG_PREF =
 const SEEN_RESTART_PREF = "browser.urlbar.quicksuggest.seenRestarts";
 
 add_task(async function init() {
-  await UrlbarQuickSuggest.init();
-  let onEnabled = UrlbarQuickSuggest.onEnabledUpdate;
-  UrlbarQuickSuggest.onEnabledUpdate = () => {};
-  registerCleanupFunction(async function() {
-    UrlbarQuickSuggest.onEnabledUpdate = onEnabled;
-  });
+  await UrlbarTestUtils.ensureQuickSuggestInit();
 });
 
 // The default is to wait for no browser restarts to show the onboarding dialog
 // on the first restart. This tests that we can override it by configuring the
 // `showOnboardingDialogOnNthRestart`
 add_task(async function test_override_wait_after_n_restarts() {
+  // Set up prefs so that onboarding will be shown.
+  await SpecialPowers.pushPrefEnv({
+    set: [
+      ["browser.urlbar.quicksuggest.shouldShowOnboardingDialog", true],
+      [
+        "browser.urlbar.quicksuggest.quicksuggest.showedOnboardingDialog",
+        false,
+      ],
+      ["browser.urlbar.quicksuggest.seenRestarts", 0],
+      ["browser.urlbar.suggest.quicksuggest", false],
+      ["browser.urlbar.suggest.quicksuggest.sponsored", false],
+    ],
+  });
+
   await UrlbarTestUtils.withExperiment({
     valueOverrides: {
       quickSuggestEnabled: true,
@@ -58,6 +67,8 @@ add_task(async function test_override_wait_after_n_restarts() {
       await Promise.all([dialogPromise, prefPromise]);
     },
   });
+
+  await SpecialPowers.popPrefEnv();
   clearOnboardingPrefs();
 });
 

browser/components/urlbar/tests/browser/browser_quicksuggest_indexes.js

@@ -65,24 +65,11 @@ add_task(async function init() {
   let oldDefaultEngine = await Services.search.getDefault();
   await Services.search.setDefault(Services.search.getEngineByName("Example"));
 
-  await UrlbarQuickSuggest.init();
-  let { _createTree } = UrlbarQuickSuggest;
-  UrlbarQuickSuggest._createTree = () => {};
-  await UrlbarQuickSuggest._processSuggestionsJSON(TEST_DATA);
-
-  await SpecialPowers.pushPrefEnv({
-    set: [
-      ["browser.urlbar.quicksuggest.enabled", true],
-      ["browser.urlbar.quicksuggest.shouldShowOnboardingDialog", false],
-      ["browser.urlbar.suggest.quicksuggest", true],
-      ["browser.urlbar.suggest.quicksuggest.sponsored", true],
-    ],
-  });
+  await UrlbarTestUtils.ensureQuickSuggestInit(TEST_DATA);
 
   registerCleanupFunction(async () => {
     await PlacesUtils.history.clear();
     Services.search.setDefault(oldDefaultEngine);
-    UrlbarQuickSuggest._createTree = _createTree;
   });
 });
 

browser/components/urlbar/tests/browser/browser_urlbar_telemetry_quicksuggest.js

@@ -57,26 +57,18 @@ add_task(async function init() {
     PartnerLinkAttribution._pingCentre,
     "sendStructuredIngestionPing"
   );
-  sandbox.stub(UrlbarQuickSuggest, "_setupRemoteSettings").resolves(true);
 
   await PlacesUtils.history.clear();
+  await PlacesUtils.bookmarks.eraseEverything();
   await UrlbarTestUtils.formHistory.clear();
-  await SpecialPowers.pushPrefEnv({
-    set: [
-      [EXPERIMENT_PREF, true],
-      ["browser.urlbar." + SUGGEST_PREF, true],
-      ["browser.urlbar.suggest.quicksuggest.sponsored", true],
-    ],
-  });
 
   // Add a mock engine so we don't hit the network.
   await SearchTestUtils.installSearchExtension();
   let oldDefaultEngine = await Services.search.getDefault();
   Services.search.setDefault(Services.search.getEngineByName("Example"));
 
   // Set up Quick Suggest.
-  await UrlbarQuickSuggest.init();
-  await UrlbarQuickSuggest._processSuggestionsJSON(TEST_DATA);
+  await UrlbarTestUtils.ensureQuickSuggestInit(TEST_DATA);
   UrlbarProviderQuickSuggest._helpUrl = TEST_HELP_URL;
 
   // Enable local telemetry recording for the duration of the test.

browser/components/urlbar/tests/unit/test_quicksuggest.js

@@ -90,9 +90,7 @@ add_task(async function init() {
   // Set up the remote settings client with the test data. Need to set the
   // `suggest.quicksuggest` pref to make `init` finish.
   UrlbarPrefs.set("suggest.quicksuggest", true);
-  await UrlbarQuickSuggest.init();
-  await UrlbarQuickSuggest._processSuggestionsJSON(REMOTE_SETTINGS_DATA);
-  sinon.stub(UrlbarQuickSuggest, "onEnabledUpdate").get(() => {});
+  await UrlbarTestUtils.ensureQuickSuggestInit(REMOTE_SETTINGS_DATA);
 });
 
 // Tests with only non-sponsored suggestions enabled with a matching search

browser/components/urlbar/tests/unit/test_quicksuggest_merino.js

@@ -48,9 +48,7 @@ add_task(async function init() {
   UrlbarPrefs.set(PREF_MERINO_ENDPOINT_URL, url.toString());
 
   // Set up the remote settings client with the test data.
-  await UrlbarQuickSuggest.init();
-  await UrlbarQuickSuggest._processSuggestionsJSON(REMOTE_SETTINGS_DATA);
-  UrlbarQuickSuggest.onEnabledUpdate = () => {};
+  await UrlbarTestUtils.ensureQuickSuggestInit(REMOTE_SETTINGS_DATA);
 });
 
 // Tests with Merino enabled and remote settings disabled.