diff --git a/.eslintignore b/.eslintignore index ac0ff8ea7b..7e6bc04987 100644 --- a/.eslintignore +++ b/.eslintignore @@ -6,5 +6,6 @@ data/content/ firefox/ logs/ stats.json +addon/shield-utils/ test/test-ColorAnalyzer.js test/test-SimpleStorage.js diff --git a/.gitignore b/.gitignore index 159cf7022c..cd073e8b95 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ logs/ dist/ data/content/ activity-streams-env/ +addon/shield-utils/ firefox/ config.yml .amo_config.json diff --git a/.travis.yml b/.travis.yml index cac4099841..63c7c04267 100644 --- a/.travis.yml +++ b/.travis.yml @@ -46,6 +46,11 @@ deploy: script: bin/continuous-integration.sh prerelease on: branch: release-1.1.7 + - provider: script + skip_cleanup: true + script: bin/continuous-integration.sh shield + on: + branch: shield_study notifications: email: false diff --git a/SHIELD.md b/SHIELD.md new file mode 100644 index 0000000000..19f08883cb --- /dev/null +++ b/SHIELD.md @@ -0,0 +1,99 @@ +#Activity Stream SHIELD Study + +## How to run this SHIELD study + +tldr; Run the command ```npm run once``` and get a randomly assigned variant. + +The SHIELD study work currently lives on branch ```shield_study``` in this repo. To run the study, checkout the branch, run command ```npm start``` in one terminal window, and in another terminal window run command ```shield run . -- -b nightly```. This will randomly pick one of the two variants provided in the study and start it in Nightly. At this point you can see which variant it chose. You can click around and print the ping payloads that get sent. If you want to see one variant specifically, run either ```shield run . Tiles -- -b nightly``` or ```shield run . ActivityStream -- -b nightly``` instead to load up the variant specified. + +## How it works + +#### Specifics of this study +In this [SHIELD study](https://wiki.mozilla.org/Firefox/Shield/Shield_Studies) a user will be randomly assigned into either the control group or the variant group. If they are in the control group, Activity Stream will **not** be loaded into their browser, and instead they will have the original about:newtab page as their default. +#### About the metrics +At this point any interaction that the user makes with about:newtab will send a ping to our custom data pipeline where it will be recorded for analysis. If they are in the variant group, normal Activity Stream will be loaded and all user events will be collected and recorded as normal. The pings belonging to a SHIELD study user will contain an extra field named ```shield_variant``` which will contain either the name of the control group or the name of the variant group. This will be what we use to distinguish which group each unique user belongs to. + +**Sample pings:** +```json +{ + "event":"CLICK", + "action_position":0, + "source":"AFFILIATE", + "shield_variant":"shield-study-01-Tiles", + "action":"activity_stream_event", + "tab_id":"-3-2", + "client_id":"ee88c4ef-4f1d-8149-9582-b727dc92e89f", + "addon_version":"1.1.5", + "locale":"en-US", + "page":"NEW_TAB", + "session_id":"{ab6f3b3e-dfc6-f944-b082-6f4bbbc21e72}" +} +``` +```json +{ + "event":"SEARCH", + "action":"activity_stream_event", + "tab_id":"-3-2", + "client_id":"daf5dc62-08e4-e840-837d-c2adb3b62463", + "addon_version":"1.1.5", + "locale":"en-US", + "page":"NEW_TAB", + "session_id":"{2426a185-b9bf-a84d-b359-61eb11452382}", + "shield_variant":"shield-study-01-Tiles"} +``` +```json +{ + "tab_id":"-3-2", + "session_id":"{2426a185-b9bf-a84d-b359-61eb11452382}", + "total_history_size":100, + "total_bookmarks":10, + "load_reason":"newtab", + "url":"about:newtab", + "unload_reason":"search", + "client_id":"daf5dc62-08e4-e840-837d-c2adb3b62463", + "addon_version":"1.1.5", + "locale":"en-US", + "page":"NEW_TAB", + "shield_variant":"shield-study-01-Tiles", + "action":"activity_stream_session", + "session_duration":6242 +} +``` + +## What we hope to achieve + +The success criteria for the SHIELD study is as follows: +- Increase the average number of top sites clicks per user per day +- Increase the average number of newtab searches per user per day +- Increase the average number of newtab sessions per user per day +- Maintain search volume + + +## Studies +###shield-study-01 + +Dates of study: November xx 2016 - November xx 2016 + +Duration: 14 days + +Activity Stream addon version: 1.1.x + +Metrics measured: Clicks and blocks on top sites, search, sessions, performance related events + +Total number of users: 20,000 + +Total number of users per arm: 10,000 + +Control group: 'Tiles' + +Variant group: 'Activity Stream' + +User eligibility criteria: + +- Must have about:newtab enabled +- Must have local set to en-US +- Must not have Test Pilot add on installed + +Findings: *links to dashboards go here* + +###shield-study-02 diff --git a/addon/TabTracker.js b/addon/TabTracker.js index f46387917a..243b55ae38 100644 --- a/addon/TabTracker.js +++ b/addon/TabTracker.js @@ -30,6 +30,10 @@ function TabTracker(options) { this._addListeners(); } simplePrefs.on(TELEMETRY_PREF, this._onPrefChange); + + // shield fields + this._shieldVariant = options.shield_variant; + this._testPilotVersion = options.tp_version; } TabTracker.prototype = { @@ -84,6 +88,14 @@ TabTracker.prototype = { if (this._experimentID) { payload.experiment_id = this._experimentID; } + + // shield fields + if (this._shieldVariant) { + payload.shield_variant = this._shieldVariant; + } + if (this._testPilotVersion) { + payload.tp_version = this._testPilotVersion; + } }, uninit() { diff --git a/addon/index.js b/addon/index.js new file mode 100644 index 0000000000..9b00f8c50c --- /dev/null +++ b/addon/index.js @@ -0,0 +1,38 @@ +/* globals require, exports */ +"use strict"; +const self = require("sdk/self"); +const shield = require("./shield-utils/index"); +const {when: unload} = require("sdk/system/unload"); +const {Feature} = require("./shield"); +const {Cu} = require("chrome"); +const feature = new Feature(); + +const studyConfig = { + name: "ACTIVITY_STREAM", + days: 14, + variations: { + "ActivityStream": () => feature.loadActivityStream(), + "Tiles": () => feature.loadActivityStream() + } +}; + +class OurStudy extends shield.Study { + isEligible() { + return super.isEligible() && feature.isEligible(); + } + shutdown(reason, variant) { + feature.shutdown(reason, variant); + } + setVariant(variant) { + feature.setVariant(variant); + } + checkTestPilot() { + return new Promise(resolve => { + feature.doesHaveTestPilot().then(resolve).catch(err => Cu.reportError(err)); + }); + } +} +const thisStudy = new OurStudy(studyConfig); +thisStudy.setVariant(thisStudy.variation); +thisStudy.checkTestPilot().then(() => thisStudy.startup(self.loadReason)).catch(err => Cu.reportError(err)); +unload(reason => thisStudy.shutdown(reason, thisStudy.variation)); diff --git a/addon/shield.js b/addon/shield.js new file mode 100644 index 0000000000..f652abd86c --- /dev/null +++ b/addon/shield.js @@ -0,0 +1,210 @@ +/* globals require, exports, Locale, Task, ClientID */ +"use strict"; +const {Cu} = require("chrome"); +const {PlacesProvider} = require("addon/PlacesProvider"); +const {MetadataStore, METASTORE_NAME} = require("addon/MetadataStore"); +const {TelemetrySender} = require("addon/TelemetrySender"); +const {TabTracker} = require("addon/TabTracker"); +const {ActivityStreams} = require("addon/ActivityStreams"); +const {setTimeout, clearTimeout} = require("sdk/timers"); +const prefs = require("sdk/preferences/service"); +const {getAddonByID} = require("sdk/addon/manager"); +const {PageMod} = require("sdk/page-mod"); + +Cu.import("resource://gre/modules/ClientID.jsm"); +Cu.import("resource://gre/modules/Task.jsm"); +Cu.import("resource://gre/modules/Locale.jsm"); + +const {OS} = Cu.import("resource://gre/modules/osfile.jsm", {}); + +// The constant to set the limit of MetadataStore reconnection +// The addon will try reconnecting to the database in the next minute periodically, +// if it fails to establish the connection in the addon initialization +const kMaxConnectRetry = 120; + +function Feature() {} + +Feature.prototype = { + _options: {}, + _variant: null, + _dlp: null, + _tilesPageMod: null, + _activityStreamApp: null, + _metadataStore: null, + _tabTracker: null, + _telemetrySender: null, + _connectRetried: 0, + _reconnectTimeoutID: null, + _initializedTiles: false, + _doesHaveTestPilot: false, + _eligibilityFilters: [], + _placesQueries: null, + _originalPreloadPref: null, + + setVariant(variant) { + this._variant = `shield-study-01-${variant}`; + }, + + setUp: Task.async(function*() { + this._originalPreloadPref = prefs.get("browser.newtab.preload"); + prefs.set("browser.newtab.preload", false); + PlacesProvider.links.init(); + const clientID = yield ClientID.getClientID(); + this._options.clientID = clientID; + this._options.shield_variant = this._variant; + this._options.telemetry = false; + this._options.tp_version = "1.1.7"; + this._tabTracker = new TabTracker(this._options); + this._telemetrySender = new TelemetrySender(); + }), + + _setUpEligibilityFilters() { + this._eligibilityFilters.push(Locale.getLocale() === "en-US"); + this._eligibilityFilters.push(prefs.get("browser.newtabpage.enabled") === true); + }, + + migrateMetadataStore: Task.async(function*() { + const sourcePath = OS.Path.join(OS.Constants.Path.localProfileDir, METASTORE_NAME); + const destPath = OS.Path.join(OS.Constants.Path.profileDir, METASTORE_NAME); + + const exists = yield OS.File.exists(sourcePath); + if (exists) { + try { + yield OS.File.move(sourcePath, destPath); + } catch (e) { + Cu.reportError(`Failed to move metadata store: ${e.message}. Removing the database file`); + yield OS.File.remove(sourcePath); + } + } + }), + + reconnectMetadataStore() { + if (this._connectRetried > kMaxConnectRetry) { + throw new Error("Metadata store reconnecting has reached the maximum limit"); + } + + this._reconnectTimeoutID = setTimeout(() => { + this._metadataStore.asyncConnect().then(() => {this._connectRetried = 0;}) + .catch(error => { + // increment the connect counter to avoid the endless retry + this._connectRetried++; + this.reconnectMetadataStore(); + Cu.reportError(error); + }); + }, 500); + }, + + loadActivityStream: Task.async(function*() { + yield this.setUp(); + if (this._options.loadReason === "upgrade") { + yield this.migrateMetadataStore(); + } + this._metadataStore = new MetadataStore(); + try { + yield this._metadataStore.asyncConnect(); + } catch (e) { + this.reconnectMetadataStore(); + } + this._activityStreamApp = new ActivityStreams(this._metadataStore, this._tabTracker, this._telemetrySender, this._options); + // don't override the homepage to have activity stream + this._activityStreamApp._setHomePage = () => {}; + try { + this._activityStreamApp.init(); + } catch (e) { + Cu.reportError(e); + } + }), + + loadTiles: Task.async(function*() { + yield this.setUp(); + this.initializeNewTab(); + if (!this._placesQueries) { + this._placesQueries = { + getHistorySize() {return PlacesProvider.links.getHistorySize();}, + getBookmarksSize() {return PlacesProvider.links.getBookmarksSize();} + }; + } + this._tabTracker.init(["about:newtab"], this._placesQueries); + this._initializedTiles = true; + }), + + initializeNewTab() { + let reportEvent = (event, data = {}) => { + let payload = Object.assign({event}, data); + this._tabTracker.handleUserEvent(payload); + }; + + // Instrument tile actions, e.g., click + this._dlp = Cu.import("resource:///modules/DirectoryLinksProvider.jsm", {}).DirectoryLinksProvider; + this._dlp._reportSitesAction = this._dlp.reportSitesAction; + this._dlp.reportSitesAction = function(sites, action, action_position) { + if (action.search(/^(click|block)$/) === 0) { + reportEvent(action.toUpperCase(), { + action_position, + source: sites[action_position].link.type.toUpperCase() + }); + } + return this._reportSitesAction(sites, action, action_position); + }; + + // Instrument search actions + this._tilesPageMod = new PageMod({ + attachTo: ["existing", "top"], + contentScript: `addEventListener("ContentSearchClient", ({detail}) => { + if (detail.type === "Search") self.port.emit("search", detail.data) + });`, + include: "about:newtab", + onAttach(worker) { + worker.port.on("search", () => reportEvent("SEARCH")); + } + }); + }, + + isEligible() { + this._setUpEligibilityFilters(); + return this._eligibilityFilters.every(item => item) && !this._doesHaveTestPilot; + }, + + doesHaveTestPilot() { + const testPilotID = "@testpilot-addon"; + return getAddonByID(testPilotID).then(addon => { + this._doesHaveTestPilot = !!addon; + }); + }, + + shutdown(reason, variant) { + if (this._activityStreamApp && variant === "ActivityStream") { + this._activityStreamApp.unload(reason); + this._activityStreamApp = null; + + if (this._reconnectTimeoutID) { + clearTimeout(this._reconnectTimeoutID); + this._reconnectTimeoutID = null; + } + + if (this._metadataStore) { + if (reason === "uninstall" || reason === "disable") { + this._metadataStore.asyncTearDown(); + } else { + this._metadataStore.asyncClose(); + } + } + PlacesProvider.links.uninit(); + this._tabTracker = null; + this._telemetrySender = null; + prefs.set("browser.newtab.preload", this._originalPreloadPref); + } else if (this._initializedTiles && variant === "Tiles") { + this._dlp.reportSitesAction = this._dlp._reportSitesAction; + this._tilesPageMod.destroy(); + this._tabTracker.uninit(); + this._telemetrySender.uninit(); + this._initializedTiles = false; + PlacesProvider.links.uninit(); + this._tabTracker = null; + this._telemetrySender = null; + prefs.set("browser.newtab.preload", this._originalPreloadPref); + } + } +}; + +exports.Feature = Feature; diff --git a/data_dictionary.md b/data_dictionary.md index 46467f7220..bdd9a105cc 100644 --- a/data_dictionary.md +++ b/data_dictionary.md @@ -104,6 +104,8 @@ The Activity Stream addon sends two distinct types of pings to the backend (HTTP | `url` | [Optional] The URL of the recommendation shown in one of the highlights spots, if any. | :one: | `value` | [Required] An integer that represents the measured performance value. Can store counts, times in milliseconds, and should always be a positive integer.| :one: | `ver` | [Auto populated by Onyx] The version of the Onyx API the ping was sent to. | :one: +| `shield_variant` | [Optional] The current variant a user is in for the SHIELD study | :one: +| `tp_version` | [Optional] The current version of the test pilot Activity Stream that shield users are running at time of study release. | :one: **Where:** diff --git a/data_events.md b/data_events.md index 27efc44065..b74bae7b8a 100644 --- a/data_events.md +++ b/data_events.md @@ -48,7 +48,10 @@ A user event ping includes some basic metadata (tab id, addon version, etc.) as   "tab_id": "-5-2",   "client_id": "26288a14-5cc4-d14f-ae0a-bb01ef45be9c",   "addon_version": "1.0.12", -  "locale": "en-US" +  "locale": "en-US", + // Optional fields, only set if they are in a shield study + "shield_variant": ["shield-study-01-ActivityStream" | "shield-study-01-Tiles"], + "tp_version": "1.1.6" } ``` @@ -69,7 +72,10 @@ A user event ping includes some basic metadata (tab id, addon version, etc.) as   // Optional field, only sent if a recommendation site gets clicked "url": "https://www.example.com", // Optional field, only sent if a recommendation site gets clicked - "recommender_type": "pocket-trending" + "recommender_type": "pocket-trending", + // Optional fields, only set if they are in a shield study + "shield_variant": ["shield-study-01-ActivityStream" | "shield-study-01-Tiles"], + "tp_version": "1.1.6" } ``` @@ -107,7 +113,10 @@ A user event ping includes some basic metadata (tab id, addon version, etc.) as   // optional field, only sent if a recommendation site gets clicked "url": "https://www.example.com", // optional field, only sent if a recommendation site gets clicked - "recommender_type": "pocket-trending" + "recommender_type": "pocket-trending", + // Optional fields, only set if they are in a shield study + "shield_variant": ["shield-study-01-ActivityStream" | "shield-study-01-Tiles"], + "tp_version": "1.1.6" } ``` @@ -170,7 +179,10 @@ All `"activity_stream_session"` pings have the following basic shape. Some field "locale": "en-US", "page": "NEW_TAB", "action": "activity_stream_session", - "session_duration": 4199 + "session_duration": 4199, + // Optional fields, only set if they are in a shield study + "shield_variant": ["shield-study-01-ActivityStream" | "shield-study-01-Tiles"], + "tp_version": "1.1.6" } ``` diff --git a/fabfile.py b/fabfile.py index eb3e7807a5..3fe845535d 100755 --- a/fabfile.py +++ b/fabfile.py @@ -12,6 +12,7 @@ env.bucket_name = "moz-activity-streams" env.bucket_name_dev = "moz-activity-streams-dev" env.bucket_name_prerelease = "moz-activity-streams-prerelease" +env.bucket_name_shield = "moz-activity-streams-shield-study" env.amo_addon_name = "activity_streams_experiment" S3 = boto.connect_s3() @@ -23,6 +24,10 @@ PRERELEASE_UPDATE_LINK = "{}/dist/activity-streams-latest.xpi".format(PRERELEASE_BUCKET_URL) PRERELEASE_UPDATE_URL = "{}/dist/update.rdf".format(PRERELEASE_BUCKET_URL) +SHIELD_BUCKET_URL = "https://moz-activity-streams-shield-study.s3.amazonaws.com" +SHIELD_UPDATE_LINK = "{}/dist/activity-streams-latest.xpi".format(PRERELEASE_BUCKET_URL) +SHIELD_UPDATE_URL = "{}/dist/update.rdf".format(PRERELEASE_BUCKET_URL) + def _get_dev_version(version): """ Get dev version from package.json. It always increments the patch version by 1 @@ -67,6 +72,24 @@ def make_prerelease_manifest(fresh_manifest=True): sort_keys=True, indent=2, separators=(',', ': ')) +def make_shield_manifest(fresh_manifest=True): + if to_bool(fresh_manifest): + restore_manifest() + + with open("./package.json", "r+") as f: + current_time = int(time.time()) + manifest = json.load(f) + manifest["title"] = "{} Shield Study".format(manifest["title"]) + manifest["updateLink"] = SHIELD_UPDATE_LINK + manifest["updateURL"] = SHIELD_UPDATE_URL + manifest["version"] = "{}-shield-study-{}".format( + _get_dev_version(manifest["version"]), current_time) + f.seek(0) + f.truncate(0) + json.dump(manifest, f, + sort_keys=True, indent=2, separators=(',', ': ')) + + def restore_manifest(): local("git checkout -- ./package.json") @@ -206,7 +229,7 @@ def deploy(run_package=True, destination=None, bucket_name = env.bucket_name if destination: - assert destination in ["dev", "prerelease"], "destination should be in ['dev', 'prerelease']" + assert destination in ["dev", "prerelease", "shield"], "destination should be in ['dev', 'prerelease', 'shield']" print "Making {} deploy".format(destination) if destination == "dev": bucket_name = env.bucket_name_dev @@ -214,6 +237,9 @@ def deploy(run_package=True, destination=None, elif destination == "prerelease": bucket_name = env.bucket_name_prerelease make_prerelease_manifest() + elif destination == "shield": + bucket_name = env.bucket_name_shield + make_shield_manifest() run_package = to_bool(run_package) end_signing = None diff --git a/package.json b/package.json index 0799b96605..892a615953 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "activity-streams", "description": "A rich visual history feed and a reimagined home page make it easier than ever to find exactly what you're looking for in Firefox.\n\nLearn more about this Test Pilot experiment at https://testpilot.firefox.com/.", - "version": "1.1.6", + "version": "0.0.0", "author": "Mozilla (https://mozilla.org/)", "bugs": { "url": "https://github.com/mozilla/activity-stream/issues" @@ -23,6 +23,8 @@ "redux-thunk": "2.1.0", "redux-watch": "1.1.1", "reselect": "2.5.4", + "shield-studies-addon-utils": "^2.0.0", + "shield-study-cli": "^1.2.4", "url-parse": "1.1.7" }, "devDependencies": { @@ -95,7 +97,7 @@ "activity-stream" ], "license": "MPL-2.0", - "main": "addon/main.js", + "main": "addon/index.js", "preferences": [ { "name": "performance.log", @@ -211,12 +213,13 @@ "repository": "mozilla/activity-stream", "scripts": { "once": "npm run bundle && npm run firefox", - "clean": "rimraf data/content/* && mkdirp data/content", + "clean": "rimraf data/content/* && mkdirp data/content && rimraf addon/shield-utils/", "changelog": "conventional-changelog -i CHANGELOG.md -s", "copyTestImages": "cpx \"node_modules/tippy-top-sites/images/**/*\" data/content/favicons/images", "copyTopSitesJson": "cpx \"node_modules/tippy-top-sites/top_sites.json\" data/content/favicons", + "copyShieldUtils": "cpx \"node_modules/shield-studies-addon-utils/lib/{event-target.js,index.js}\" addon/shield-utils/", "bundle": "npm-run-all bundle:*", - "prebundle": "npm run clean && npm run copyTestImages && npm run copyTopSitesJson", + "prebundle": "npm run clean && npm run copyTestImages && npm run copyTopSitesJson && npm run copyShieldUtils", "bundle:static": "cpx \"content-src/static/**/*\" data/content", "bundle:svgo": "svgo -q -f data/content/img/", "bundle:webpack": "NODE_ENV=production webpack", @@ -225,7 +228,7 @@ "bundle:html": "node ./bin/generate-html.js > data/content/activity-streams.html", "postbundle": "du -hs ./data/content/*", "start": "npm-run-all --parallel start:*", - "prestart": "npm run clean && npm run copyTestImages && npm run copyTopSitesJson", + "prestart": "npm run clean && npm run copyTestImages && npm run copyTopSitesJson && npm run copyShieldUtils", "start:static": "npm run bundle:static -- -w", "start:webpack": "webpack -w", "start:webpackAddon": "npm run bundle:webpackAddon -- -w", @@ -257,8 +260,8 @@ "__": "# NOTE: THESE SCRIPTS ARE COMPILED!!! EDIT yamscripts.yml instead!!!" }, "title": "Activity Stream", - "updateLink": "https://testpilot.firefox.com/files/activitystream/latest", - "updateURL": "https://testpilot.firefox.com/files/activitystream/updates.json", + "updateLink": "https://addons.mozilla.org/firefox/downloads/latest/activity-stream-shield-study", + "updateURL": "https://addons.mozilla.org/firefox/downloads/latest/activity-stream-shield-study", "permissions": { "multiprocess": true, "private-browsing": true diff --git a/yamscripts.yml b/yamscripts.yml index 13d015331c..673df5e41e 100644 --- a/yamscripts.yml +++ b/yamscripts.yml @@ -8,7 +8,7 @@ scripts: # once: Build/serve the assets and run the add-on once: =>bundle && =>firefox - clean: rimraf data/content/* && mkdirp data/content + clean: rimraf data/content/* && mkdirp data/content && rimraf addon/shield-utils/ changelog: conventional-changelog -i CHANGELOG.md -s @@ -16,9 +16,11 @@ scripts: copyTopSitesJson: cpx "node_modules/tippy-top-sites/top_sites.json" data/content/favicons + copyShieldUtils: cpx "node_modules/shield-studies-addon-utils/lib/{event-target.js,index.js}" addon/shield-utils/ + # bundle: Bundle assets for production bundle: - pre: =>clean && =>copyTestImages && =>copyTopSitesJson + pre: =>clean && =>copyTestImages && =>copyTopSitesJson && =>copyShieldUtils static: cpx "content-src/static/**/*" data/content svgo: svgo -q -f data/content/img/ webpack: NODE_ENV=production webpack @@ -30,7 +32,7 @@ scripts: # start: Start watching/compiling assets, start: _parallel: true - pre: =>clean && =>copyTestImages && =>copyTopSitesJson + pre: =>clean && =>copyTestImages && =>copyTopSitesJson && =>copyShieldUtils static: =>bundle:static -- -w webpack: webpack -w webpackAddon: =>bundle:webpackAddon -- -w