Gh 1763/metrics feature/new triggers for broken page pings

app/panel/reducers/blocking.js

@@ -168,6 +168,7 @@ const _updateTrackerTrustRestrict = (state, action) => {
 	sendMessage('setPanelData', {
 		site_specific_unblocks: updated_site_specific_unblocks,
 		site_specific_blocks: updated_site_specific_blocks,
+		brokenPageMetricsTrackerTrustOrUnblock: msg.trust || (!msg.trust && !msg.restrict),
 	});
 
 	return {

app/panel/reducers/summary.js

@@ -145,6 +145,7 @@ const _updateSitePolicy = (state, action) => {
 	sendMessage('setPanelData', {
 		site_whitelist: updated_whitelist,
 		site_blacklist: updated_blacklist,
+		brokenPageMetricsWhitelistSite: updated_site_policy === 2,
 	});
 
 	return {

app/panel/utils/blocking.js

@@ -214,7 +214,10 @@ export function updateTrackerBlocked(state, action) {
 	});
 
 	// persist to background
-	sendMessage('setPanelData', { selected_app_ids: updated_app_ids });
+	sendMessage('setPanelData', {
+		selected_app_ids: updated_app_ids,
+		brokenPageMetricsTrackerTrustOrUnblock: !blocked
+	});
 
 	return {
 		categories: updated_categories,

src/background.js

@@ -1155,6 +1155,8 @@ function initializeDispatcher() {
 	}, 200));
 
 	dispatcher.on('globals.save.paused_blocking', () => {
+		// if user has paused Ghostery, suspect broken page
+		if (globals.SESSION.paused_blocking) { metrics.handleBrokenPageTrigger(globals.BROKEN_PAGE_PAUSE); }
 		// update content script state when blocking is paused/unpaused
 		cliqz.modules.core.action('refreshAppState');
 	});

src/classes/EventHandlers.js

@@ -25,6 +25,7 @@ import conf from './Conf';
 import foundBugs from './FoundBugs';
 import globals from './Globals';
 import latency from './Latency';
+import metrics from './Metrics';
 import panelData from './PanelData';
 import Policy, { BLOCK_REASON_SS_UNBLOCKED, BLOCK_REASON_C2P_ALLOWED_THROUGH } from './Policy';
 import PolicySmartBlock from './PolicySmartBlock';
@@ -110,6 +111,7 @@ class EventHandlers {
 		if (frameId === 0) {
 			// update reload info before creating/clearing tab info
 			if (transitionType === 'reload' && !transitionQualifiers.includes('forward_back')) {
+				metrics.handleBrokenPageTrigger(globals.BROKEN_PAGE_REFRESH);
 				tabInfo.setTabInfo(tabId, 'numOfReloads', tabInfo.getTabInfo(tabId, 'numOfReloads') + 1);
 			} else if (transitionType !== 'auto_subframe' && transitionType !== 'manual_subframe') {
 				tabInfo.setTabInfo(tabId, 'reloaded', false);
@@ -545,7 +547,11 @@ class EventHandlers {
 	 *
 	 * @param  {Object} tab 	 Details of the tab that was created
 	 */
-	onTabCreated() {}
+	onTabCreated(tab) {
+		const { url } = tab;
+
+		metrics.handleBrokenPageTrigger(globals.BROKEN_PAGE_NEW_TAB, url);
+	}
 
 	/**
 	 * Handler for tabs.onActivated event.

src/classes/Globals.js

@@ -70,6 +70,13 @@ class Globals {
 		this.BLACKLISTED = 1;
 		this.WHITELISTED = 2;
 
+		// Broken page metrics named constants
+		this.BROKEN_PAGE_REFRESH = 1;
+		this.BROKEN_PAGE_WHITELIST = 2;
+		this.BROKEN_PAGE_PAUSE = 3;
+		this.BROKEN_PAGE_TRACKER_TRUST_OR_UNBLOCK = 4;
+		this.BROKEN_PAGE_NEW_TAB = 5;
+
 		// data stores
 		this.REDIRECT_MAP = new Map();
 		this.BLOCKED_REDIRECT_DATA = {};

src/classes/Metrics.js

@@ -14,7 +14,7 @@
 import globals from './Globals';
 import conf from './Conf';
 import { log, prefsSet, prefsGet } from '../utils/common';
-import { processUrlQuery } from '../utils/utils';
+import { getActiveTab, processUrlQuery } from '../utils/utils';
 import rewards from './Rewards';
 
 // CONSTANTS
@@ -30,6 +30,12 @@ const FIRST_REWARD_METRICS = ['rewards_first_accept', 'rewards_first_reject', 'r
 const { METRICS_SUB_DOMAIN, EXTENSION_VERSION, BROWSER_INFO } = globals;
 const IS_EDGE = (BROWSER_INFO.name === 'edge');
 const MAX_DELAYED_PINGS = 100;
+
+// Note that this threshold is intentionally different from the 30 second threshold in PolicySmartBlock,
+// which is used to set the reloaded property on tab objects and to activate smart unblock behavior
+// see GH-1797 for more details
+const BROKEN_PAGE_METRICS_THRESHOLD = 60000; // 60 seconds
+
 /**
  * Class for handling telemetry pings.
  * @memberOf  BackgroundClasses
@@ -39,6 +45,13 @@ class Metrics {
 		this.utm_source = '';
 		this.utm_campaign = '';
 		this.ping_set = new Set();
+		this._brokenPageWatcher = {
+			on: false,
+			triggerId: '',
+			triggerTime: '',
+			timeoutId: null,
+			url: '',
+		};
 	}
 
 	/**
@@ -107,6 +120,75 @@ class Metrics {
 			});
 	}
 
+	/**
+	 * Responds to individual user actions and sequences of user actions that may indicate a broken page,
+	 * sending broken_page pings as needed
+	 * For example, sends a broken_page ping when the user whitelists a site,
+	 * then refreshes the page less than a minute later
+	 * @param 	{int} 		triggerId	'what specifically triggered this broken_page ping?' identifier sent along to the metrics server
+	 * @param 	{string} 	newTabUrl	for checking whether user has opened the same url in a new tab, which confirms a suspicion raised by certain triggers
+	 */
+	handleBrokenPageTrigger(triggerId, newTabUrl = null) {
+		if (this._brokenPageWatcher.on && triggerId === globals.BROKEN_PAGE_REFRESH) {
+			this.ping('broken-page');
+			this._unplugBrokenPageWatcher();
+			return;
+		}
+
+		if (this._brokenPageWatcher.on && triggerId === globals.BROKEN_PAGE_NEW_TAB && this._brokenPageWatcher.url === newTabUrl) {
+			this.ping('broken-page');
+			this._unplugBrokenPageWatcher();
+			return;
+		}
+
+		if (triggerId === globals.BROKEN_PAGE_NEW_TAB) { return; }
+
+		this._resetBrokenPageWatcher(triggerId);
+	}
+
+	/**
+	 * handleBrokenPageTrigger helper
+	 * starts the temporary watch for a second suspicious user action in response to a first
+	 * @param 	{int} 		triggerId	'what specifically triggered this broken_page ping?' identifier sent along to the metrics server
+	 * @private
+	 */
+	_resetBrokenPageWatcher(triggerId) {
+		this._clearBrokenPageWatcherTimeout();
+
+		getActiveTab((tab) => {
+			const tabUrl = tab && tab.url ? tab.url : '';
+
+			this._brokenPageWatcher = Object.assign({}, {
+				on: true,
+				triggerId,
+				triggerTime: Date.now(),
+				timeoutId: setTimeout(this._clearBrokenPageWatcher, BROKEN_PAGE_METRICS_THRESHOLD),
+				url: tabUrl,
+			});
+		});
+	}
+
+	/**
+	 * handleBrokenPageTrigger helper
+	 * @private
+	 */
+	_unplugBrokenPageWatcher() {
+		this._clearBrokenPageWatcherTimeout();
+
+		this._brokenPageWatcher = Object.assign({}, {
+			on: false,
+			triggerId: '',
+			triggerTime: '',
+			timeoutId: null,
+			url: '',
+		});
+	}
+
+	_clearBrokenPageWatcherTimeout() {
+		const { timeoutId } = this._brokenPageWatcher;
+		if (timeoutId) { clearTimeout(timeoutId); }
+	}
+
 	/**
 	* Prepare data and send telemetry pings.
 	* @param {string} type    type of the telemetry ping
@@ -322,6 +404,12 @@ class Metrics {
 			metrics_url +=
 				// Reward ID
 				`&rid=${encodeURIComponent(this._getRewardId().toString())}`;
+		} else if (type === 'broken_page' && this._brokenPageWatcher.on) {
+			metrics_url +=
+				// What triggered the broken page ping?
+				`&setup_path=${encodeURIComponent(this._brokenPageWatcher.triggerId.toString())}` +
+				// How much time passed between the trigger and the page refresh / open in new tab?
+				`&setup_block=${encodeURIComponent((Date.now() - this._brokenPageWatcher.triggerTime).toString())}`;
 		}
 
 		return metrics_url;

src/classes/PanelData.js

@@ -21,6 +21,7 @@ import conf from './Conf';
 import foundBugs from './FoundBugs';
 import bugDb from './BugDb';
 import globals from './Globals';
+import metrics from './Metrics';
 import Policy from './Policy';
 import tabInfo from './TabInfo';
 import rewards from './Rewards';
@@ -647,6 +648,14 @@ class PanelData {
 			tabInfo.setTabInfo(this._activeTab.id, 'needsReload', data.needsReload);
 		}
 
+		if (data.brokenPageMetricsTrackerTrustOrUnblock) {
+			metrics.handleBrokenPageTrigger(globals.BROKEN_PAGE_TRACKER_TRUST_OR_UNBLOCK);
+		}
+
+		if (data.brokenPageMetricsWhitelistSite) {
+			metrics.handleBrokenPageTrigger(globals.BROKEN_PAGE_WHITELIST);
+		}
+
 		if (syncSetDataChanged) {
 			// Push conf changes to the server
 			account.saveUserSettings().catch(err => log('PanelData saveUserSettings', err));

src/classes/PolicySmartBlock.js

@@ -251,11 +251,13 @@ class PolicySmartBlock {
 	checkReloadThreshold(tabId) {
 		if (!this.shouldCheck(tabId)) { return false; }
 
-		const THRESHHOLD = 30000; // 30 seconds
+		// Note that this threshold is different from the broken page ping threshold in Metrics, which is 60 seconds
+		// see GH-1797 for more details
+		const SMART_BLOCK_BEHAVIOR_THRESHOLD = 30000; // 30 seconds
 
 		return (
 			tabInfo.getTabInfoPersist(tabId, 'numOfReloads') > 1 &&
-			((Date.now() - tabInfo.getTabInfoPersist(tabId, 'firstLoadTimestamp')) < THRESHHOLD) || false
+			((Date.now() - tabInfo.getTabInfoPersist(tabId, 'firstLoadTimestamp')) < SMART_BLOCK_BEHAVIOR_THRESHOLD) || false
 		);
 	}
 

src/classes/TabInfo.js

@@ -70,10 +70,6 @@ class TabInfo {
 			insecureRedirects: [],
 		};
 
-		if (info.reloaded) {
-			metrics.ping('broken_page');
-		}
-
 		this._tabInfo[tab_id] = info;
 		this._updateUrl(tab_id, tab_url);
 	}