Breach Results FxA Experiment (Growth Exp 3) (#1659)

* Added experiment logic

* Added A/B test code

* Fixed lint errors from rebase

* Add experiment info to the user (dashboard) template

* Removed rebase artifacts, corrected if-logic in HTML blocks

* Removed the blanket "transport: beacon" settings and moved them to event options as it was breaking analytics reporting in Chrome environments.

* Add new FxA entry points (treatment branch) into FxA reporting

* Revised experiment campaign ID

* Revised home page render to not include experiment assignment functionality, except for when a user passes a URL param.

* Revised test for revised cohort assignment (50/50)

* Removed Experiment 2 Artifacts

* Removed language logic for cohort selection

* Revised cohort split to be 30/30/40

* Reordered view/click ping listeners to fire after the page view event fires.

* Updated tests for scan/user controllers

* Fixed lint errors

* Updated getExperimentBranch unit test comments

* Removed getExperimentBranch function from the user controller, and only set meta info on the page if the user has already been sorted into the experiment.

* Remove debug logging

* Fixed #1652

controllers/home.js

@@ -3,7 +3,7 @@
 const AppConstants = require("../app-constants");
 const DB = require("../db/DB");
 const { scanResult } = require("../scan-results");
-const { generatePageToken, getExperimentBranch } = require("./utils");
+const { generatePageToken } = require("./utils");
 
 const EXPERIMENTS_ENABLED = (AppConstants.EXPERIMENT_ACTIVE === "1");
 
@@ -17,19 +17,12 @@ async function home(req, res) {
   let featuredBreach = null;
   let scanFeaturedBreach = false;
 
-  let experimentBranch = null;
-  let isUserInExperiment = null;
-  let experimentBranchB = null;
-
-
-  if (EXPERIMENTS_ENABLED) {
-    const coinFlipNumber = Math.floor(Math.random() * 100);
-    experimentBranch = getExperimentBranch(req, coinFlipNumber);
-    // req.session.excludeFromExperiment is set to remember that the user has been excluded from the experiment.
-    if (!experimentBranch) { req.session.excludeFromExperiment = true; }
-    req.session.experimentBranch = experimentBranch;
-    isUserInExperiment = (experimentBranch === "vb");
-    experimentBranchB = (experimentBranch === "vb" && isUserInExperiment);
+  // Growth Experiment: Set experiment branch from the homepage through URL queryParams
+  // but do not render any info on  the /home template.
+  if (EXPERIMENTS_ENABLED && req.query.experimentBranch) {
+    if (["va", "vb"].includes(req.query.experimentBranch)) {
+      req.session.experimentBranch = req.query.experimentBranch;
+    }
   }
 
   if (req.session.user && !req.query.breach) {
@@ -57,8 +50,6 @@ async function home(req, res) {
       scanFeaturedBreach,
       pageToken: formTokens.pageToken,
       csrfToken: formTokens.csrfToken,
-      experimentBranch,
-      experimentBranchB,
     });
   }
 
@@ -68,9 +59,6 @@ async function home(req, res) {
     scanFeaturedBreach,
     pageToken: formTokens.pageToken,
     csrfToken: formTokens.csrfToken,
-    experimentBranch,
-    isUserInExperiment,
-    experimentBranchB,
   });
 }
 

controllers/oauth.js

@@ -115,7 +115,7 @@ async function confirmed(req, res, next, client = FxAOAuthClient) {
   }
   // Update existing user's FxA data
   await DB._updateFxAData(existingUser, fxaUser.accessToken, fxaUser.refreshToken, fxaProfileData);
-    res.redirect(returnURL.pathname + returnURL.search);
+  res.redirect(returnURL.pathname + returnURL.search);
 }
 
 module.exports = {

controllers/user.js

@@ -12,6 +12,7 @@ const HIBP = require("../hibp");
 const { resultsSummary } = require("../scan-results");
 const sha1 = require("../sha1-utils");
 
+const EXPERIMENTS_ENABLED = (AppConstants.EXPERIMENT_ACTIVE === "1");
 
 const FXA_MONITOR_SCOPE = "https://identity.mozilla.com/apps/monitor";
 
@@ -230,6 +231,18 @@ async function getDashboard(req, res) {
   const allBreaches = req.app.locals.breaches;
   const { verifiedEmails, unverifiedEmails } = await getAllEmailsAndBreaches(user, allBreaches);
 
+  let experimentBranch = null;
+  let isUserInExperiment = null;
+  let experimentBranchB = null;
+
+  if (EXPERIMENTS_ENABLED && req.session.experimentBranch) {
+    if (!req.session.excludeFromExperiment) {
+      experimentBranch = req.session.experimentBranch;
+      isUserInExperiment = (experimentBranch === "vb");
+      experimentBranchB = (experimentBranch === "vb" && isUserInExperiment);
+    }
+  }
+
   let lastAddedEmail = null;
 
   req.session.user = await DB.setBreachesLastShownNow(user);
@@ -245,6 +258,9 @@ async function getDashboard(req, res) {
     verifiedEmails,
     unverifiedEmails,
     whichPartial: "dashboards/breaches-dash",
+    experimentBranch,
+    isUserInExperiment,
+    experimentBranchB,
   });
 }
 

controllers/utils.js

@@ -40,21 +40,6 @@ function getExperimentBranch(req, sorterNum) {
     return false;
   }
 
-  // If we cannot parse req.headers["accept-language"], we should not
-  // enroll users in the experiment.
-  if (!req.headers || !req.headers["accept-language"]){
-    log.debug("No headers or accept-language information present.");
-    return false;
-  }
-
-  // If the user doesn't have an English variant langauge selected as their primary language,
-  // we do not enroll them in the experiment.
-  const lang = req.headers["accept-language"].split(",");
-  if (!lang[0].includes("en")) {
-    log.debug("Preferred language is not English variant: ", lang[0]);
-    return false;
-  }
-
   // If URL param has experimentBranch entry, use that branch;
   if (req.query.experimentBranch) {
     if (!["va", "vb"].includes(req.query.experimentBranch)) {
@@ -73,14 +58,14 @@ function getExperimentBranch(req, sorterNum) {
     return req.session.experimentBranch;
   }
 
-  // Growth Team Experiment 2 only wants to expose 58% of all site traffic to
-  // the experiment. Of the 58% percent inside the experiment, will be split
+  // Growth Team Experiment 2 only wants to expose 60% of all site traffic to
+  // the experiment. Of the 60% percent inside the experiment, will be split
   // 50/50 between treatment and control.
-  if (sorterNum < 29) {
+  if (sorterNum < 30) {
     log.debug("This session has been randomly assigned to the control group. (va)");
     req.session.experimentBranch = "va";
     return "va";
-  } else if (sorterNum > 28 && sorterNum < 58) {
+  } else if (sorterNum > 29 && sorterNum < 60) {
     log.debug("This session has been randomly assigned to the treatment group. (vb)");
     req.session.experimentBranch = "vb";
     return "vb";

public/css/forms.css

@@ -25,10 +25,6 @@ input:-webkit-autofill {
   margin: 12px auto;
 }
 
-.form-desc.experiment {
-  max-width: 290px;
-}
-
 input[type="submit"],
 input[type="email"] {
   height: 48px;

public/css/monitor.css

@@ -14,10 +14,6 @@
   margin-bottom: 0;
 }
 
-.featured-breach.experiment-subhead h2.landing-headline {
-  margin-bottom: 22px;
-}
-
 .landing-bottom-bar {
   background-image: url(/img/landing/background-noodle-middle.svg), linear-gradient(224deg, #2150c8, #712291);
   background-repeat: no-repeat;
@@ -141,7 +137,6 @@ h3.sb-callout {
     min-height: 100vh;
   }
 
-  .featured-breach.experiment-subhead h2.landing-headline,
   .featured-breach h2.landing-headline,
   h2.landing-headline,
   .landing-subhead {

public/css/scan-results.css

@@ -112,6 +112,10 @@
   transition: all 0.15s ease-in-out;
 }
 
+.temp-marketing-btn-blue.experiment {
+  margin-top: 16px;
+}
+
 .temp-marketing-btn-blue:hover {
   background-color: var(--blue4);
   transition: all 0.15s ease-in-out;