GH-1566 - Smart blocking should check for Click2Play

app/panel/components/BuildingBlocks/DonutGraph.jsx

@@ -11,7 +11,7 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0
  */
 
-import throttle from 'lodash.throttle';
+import { throttle } from 'underscore';
 import React from 'react';
 import ClassNames from 'classnames';
 import {

app/panel/components/BuildingBlocks/StatsGraph.jsx

@@ -11,9 +11,9 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0
  */
 
+import { isEqual } from 'underscore';
 import React from 'react';
 import * as D3 from 'd3';
-import isEqual from 'lodash.isequal';
 
 /**
  * Generates an animated graph displaying locally stored stats

package.json

@@ -52,8 +52,6 @@
     "foundation-sites": "^6.4.4-rc1",
     "history": "^4.7.2",
     "json-api-normalizer": "^0.4.10",
-    "lodash.isequal": "^4.5.0",
-    "lodash.throttle": "^4.1.1",
     "moment": "^2.19.1",
     "prop-types": "^15.6.2",
     "query-string": "^6.1.0",
@@ -71,6 +69,7 @@
     "spanan": "^2.0.0",
     "ua-parser-js": "^0.7.17",
     "underscore": "^1.8.3",
+    "underscore-template-loader": "^1.0.0",
     "url-search-params": "^0.10.2",
     "whatwg-fetch": "^3.0.0"
   },
@@ -96,7 +95,6 @@
     "eslint-plugin-react": "^7.6.1",
     "fs-extra": "^4.0.3",
     "glob": "^7.1.2",
-    "html-loader": "^0.5.1",
     "jest": "^23.6.0",
     "jsdoc": "^3.5.5",
     "jsonfile": "^4.0.0",

src/background.js

@@ -18,7 +18,7 @@
 /**
  * @namespace Background
  */
-import _ from 'underscore';
+import { debounce, every, size } from 'underscore';
 import moment from 'moment/min/moment-with-locales.min';
 import cliqz, { prefs } from './classes/Cliqz';
 // object class
@@ -1058,11 +1058,11 @@ function onMessageHandler(request, sender, callback) {
  */
 function initializeDispatcher() {
 	dispatcher.on('conf.save.selected_app_ids', (appIds) => {
-		const num_selected = _.size(appIds);
+		const num_selected = size(appIds);
 		const { db } = bugDb;
 		db.noneSelected = (num_selected === 0);
-		// can't simply compare num_selected and _.size(db.apps) since apps get removed sometimes
-		db.allSelected = (!!num_selected && _.every(db.apps, (app, app_id) => appIds.hasOwnProperty(app_id)));
+		// can't simply compare num_selected and size(db.apps) since apps get removed sometimes
+		db.allSelected = (!!num_selected && every(db.apps, (app, app_id) => appIds.hasOwnProperty(app_id)));
 	});
 	dispatcher.on('conf.save.site_whitelist', () => {
 		// TODO debounce with below
@@ -1118,7 +1118,7 @@ function initializeDispatcher() {
 		}
 	});
 
-	dispatcher.on('conf.changed.settings', _.debounce((key) => {
+	dispatcher.on('conf.changed.settings', debounce((key) => {
 		log('Conf value changed for a watched user setting:', key);
 	}, 200));
 

src/classes/Account.js

@@ -13,7 +13,7 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0
  */
 
-import _ from 'underscore';
+import { isEqual } from 'underscore';
 import normalize from 'json-api-normalizer';
 import build from 'redux-object';
 import RSVP from 'rsvp';
@@ -542,7 +542,7 @@ class Account {
 		}
 		SYNC_SET.forEach((key) => {
 			if (settings[key] !== undefined &&
-				!_.isEqual(conf[key], settings[key])) {
+				!isEqual(conf[key], settings[key])) {
 				conf[key] = settings[key];
 			}
 		});

src/classes/BugDb.js

@@ -14,7 +14,7 @@
 /* eslint no-param-reassign: 0 */
 /* eslint no-shadow: 0 */
 
-import _ from 'underscore';
+import { difference, each, every, keys, reduce, size } from 'underscore';
 import conf from './Conf';
 import Updatable from './Updatable';
 import { defineLazyProperty, flushChromeMemoryCache } from '../utils/utils';
@@ -38,9 +38,9 @@ class BugDb extends Updatable {
 	updateNewAppIds(new_apps, old_apps) {
 		log('updating newAppIds...');
 
-		const new_app_ids = _.difference(
-			_.keys(new_apps),
-			_.keys(old_apps)
+		const new_app_ids = difference(
+			keys(new_apps),
+			keys(old_apps)
 		).map(Number);
 
 		conf.new_app_ids = new_app_ids;
@@ -55,7 +55,7 @@ class BugDb extends Updatable {
 		if (conf.block_by_default) {
 			log('applying block-by-default...');
 			const { selected_app_ids } = conf;
-			_.each(new_app_ids, (app_id) => {
+			each(new_app_ids, (app_id) => {
 				selected_app_ids[app_id] = 1;
 			});
 			conf.selected_app_ids = selected_app_ids;
@@ -184,13 +184,13 @@ class BugDb extends Updatable {
 
 		log('setting bugdb noneSelected/allSelected...');
 
-		const num_selected = _.size(conf.selected_app_ids);
+		const num_selected = size(conf.selected_app_ids);
 		db.noneSelected = (num_selected === 0);
 
 		// since allSelected is slow to eval, make it lazy
 		defineLazyProperty(db, 'allSelected', () => {
-			const num_selected = _.size(conf.selected_app_ids);
-			return (!!num_selected && _.every(db.apps, (app, app_id) => conf.selected_app_ids.hasOwnProperty(app_id)));
+			const num_selected = size(conf.selected_app_ids);
+			return (!!num_selected && every(db.apps, (app, app_id) => conf.selected_app_ids.hasOwnProperty(app_id)));
 		});
 
 		log('processed bugdb...');
@@ -211,7 +211,7 @@ class BugDb extends Updatable {
 
 				// pre-trie/legacy db
 				} else if (old_bugs.hasOwnProperty('bugsVersion') && bugs.version !== old_bugs.bugsVersion) {
-					const old_apps = _.reduce(old_bugs.bugs, (memo, bug) => {
+					const old_apps = reduce(old_bugs.bugs, (memo, bug) => {
 						memo[bug.aid] = true;
 						return memo;
 					}, {});

src/classes/Click2PlayDb.js

@@ -66,18 +66,35 @@ class Click2PlayDb extends Updatable {
 
 	// TODO memory leak when you close tabs before reset() can run?
 	reset(tab_id) {
-		delete this.allowOnceList[tab_id];
+		if (!this.allowOnceList.hasOwnProperty(tab_id)) { return; }
+
+		const entries = Object.entries(this.allowOnceList[tab_id]);
+		let keep = false;
+		for (const [appID, count] of entries) {
+			const newCount = count - 1;
+			this.allowOnceList[tab_id][appID] = newCount;
+			if (newCount > 0) {
+				keep = true;
+			}
+		}
+		if (!keep) {
+			delete this.allowOnceList[tab_id];
+		}
 	}
 
 	allowedOnce(tab_id, aid) {
-		return this.allowOnceList.hasOwnProperty(tab_id) && this.allowOnceList[tab_id].hasOwnProperty(aid);
+		return (
+			this.allowOnceList.hasOwnProperty(tab_id) &&
+			this.allowOnceList[tab_id].hasOwnProperty(aid) &&
+			this.allowOnceList[tab_id][aid] > 0
+		);
 	}
 
 	allowOnce(app_ids, tab_id) {
 		this.allowOnceList[tab_id] = {};
 
 		app_ids.forEach((app_id) => {
-			this.allowOnceList[tab_id][app_id] = 1;
+			this.allowOnceList[tab_id][app_id] = 2;
 		});
 	}
 

src/classes/EventHandlers.js

@@ -14,7 +14,7 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0
  */
 
-import _ from 'underscore';
+import { map, object, reduce, throttle } from 'underscore';
 import bugDb from './BugDb';
 import button from './BrowserButton';
 import c2pDb from './Click2PlayDb';
@@ -35,7 +35,6 @@ import { isBug } from '../utils/matcher';
 import * as utils from '../utils/utils';
 
 const IS_EDGE = (globals.BROWSER_INFO.name === 'edge');
-const RequestsMap = new Map();
 /**
  * This class is a collection of handlers for
  * webNavigation, webRequest and tabs events.
@@ -52,7 +51,7 @@ class EventHandlers {
 		// Use a 1sec interval to limit calls on pages with a large number of requests.
 		// Don't use tabId with button.update for cases where tab is switched before throttle delay is reached.
 		// ToDo: Remove this function when there is an event for AdBlocker:foundAd.
-		this._throttleButtonUpdate = _.throttle((tabId) => {
+		this._throttleButtonUpdate = throttle((tabId) => {
 			button.update(tabId);
 		}, 1000, { leading: false });
 	}
@@ -66,11 +65,10 @@ class EventHandlers {
 	onBeforeNavigate(details) {
 		const { tabId, frameId, url } = details;
 
-		// frameId === 0 indicates the navigation event ocurred in the content window, not a subframe
+		// frameId === 0 indicates the navigation event occurred in the content window, not a subframe
 		if (frameId === 0) {
 			log(`❤ ❤ ❤ Tab ${tabId} navigating to ${url} ❤ ❤ ❤`);
 
-			RequestsMap.clear();
 			this._clearTabData(tabId);
 			this._resetNotifications();
 			// TODO understand why this does not work when placed in the 'reload' branch in onCommitted
@@ -80,6 +78,7 @@ class EventHandlers {
 			tabInfo.create(tabId, url);
 			foundBugs.update(tabId);
 			button.update(tabId);
+			this._eventReset(details.tabId);
 
 			// Workaround for foundBugs/tabInfo memory leak when the user triggers
 			// prefetching/prerendering but never loads the page. Wait two minutes
@@ -105,7 +104,7 @@ class EventHandlers {
 			tabId, frameId, transitionType, transitionQualifiers
 		} = details;
 
-		// frameId === 0 indicates the navigation event ocurred in the content window, not a subframe
+		// frameId === 0 indicates the navigation event occurred in the content window, not a subframe
 		if (frameId === 0) {
 			// update reload info before creating/clearing tab info
 			if (transitionType === 'reload' && !transitionQualifiers.includes('forward_back')) {
@@ -239,7 +238,7 @@ class EventHandlers {
 					if (result) {
 						utils.sendMessage(
 							tab_id, 'showUpgradeAlert', {
-								translations: _.object(_.map(alert_messages, key => [key, chrome.i18n.getMessage(key)])),
+								translations: object(map(alert_messages, key => [key, chrome.i18n.getMessage(key)])),
 								language: conf.language,
 								major_upgrade: globals.JUST_UPGRADED_FROM_7
 							},
@@ -257,7 +256,7 @@ class EventHandlers {
 						if (result) {
 							utils.sendMessage(
 								tab_id, 'showLibraryUpdateAlert', {
-									translations: _.object(_.map(alert_messages, key => [key, chrome.i18n.getMessage(key)])),
+									translations: object(map(alert_messages, key => [key, chrome.i18n.getMessage(key)])),
 									language: conf.language
 								},
 								() => {
@@ -285,8 +284,6 @@ class EventHandlers {
 			return;
 		}
 
-		RequestsMap.clear();
-
 		// Code below executes for top level frame only
 		log(`foundBugs: ${foundBugs.getAppsCount(details.tabId)}, tab_id: ${details.tabId}`);
 
@@ -296,11 +293,6 @@ class EventHandlers {
 				log('onNavigationCompleted injectScript error', err);
 			});
 		}
-		// The problem is that requests may continue well after onNavigationCompleted
-		// This breaks allow once for C2P, as it clears too early
-		setTimeout(() => {
-			this._eventReset(details.tabId);
-		}, 2000);
 	}
 
 	/**
@@ -427,6 +419,9 @@ class EventHandlers {
 			};
 		}
 
+		const smartBlocked = !block ? this.policySmartBlock.shouldBlock(app_id, cat_id, tab_id, page_url, details.type, details.timeStamp) : false;
+		const smartUnblocked = block ? this.policySmartBlock.shouldUnblock(app_id, cat_id, tab_id, page_url, details.type) : false;
+
 		// process the tracker asynchronously
 		// very important to block request processing as little as necessary
 		setTimeout(() => {
@@ -436,20 +431,16 @@ class EventHandlers {
 				type: details.type,
 				url: details.url,
 				block,
+				smartBlocked,
 				tab_id,
 				from_frame: details.parentFrameId !== -1
 			});
 		}, 1);
 
-		if (block) {
-			if (this.policySmartBlock.shouldUnblock(app_id, cat_id, tab_id, page_url, details.type)) {
-				return { cancel: false };
-			}
+		if ((block && !smartUnblocked) || smartBlocked) {
 			return this._blockHelper(details, tab_id, app_id, bug_id, request_id, fromRedirect);
 		}
-		if (this.policySmartBlock.shouldBlock(app_id, cat_id, tab_id, page_url, details.type, details.timeStamp)) {
-			return this._blockHelper(details, tab_id, app_id, bug_id, request_id);
-		}
+
 		return { cancel: false };
 	}
 
@@ -600,13 +591,14 @@ class EventHandlers {
 	 */
 	_processBug(details) {
 		const {
-			bug_id, app_id, type, url, block, tab_id
+			bug_id, app_id, type, url, block, smartBlocked, tab_id
 		} = details;
 		const tab = tabInfo.getTabInfo(tab_id);
+		const allowedOnce = c2pDb.allowedOnce(details.tab_id, app_id);
 
 		let num_apps_old;
 
-		log((block ? 'Blocked' : 'Found'), type, url);
+		log((block || smartBlocked ? 'Blocked' : 'Found'), type, url);
 		log(`^^^ Pattern ID ${bug_id} on tab ID ${tab_id}`);
 
 		if (conf.show_alert) {
@@ -620,13 +612,13 @@ class EventHandlers {
 		// throttled in PanelData
 		panelData.updatePanelUI();
 
-		if (block && (conf.enable_click2play || conf.enable_click2playSocial)) {
+		if ((block || smartBlocked) && (conf.enable_click2play || conf.enable_click2playSocial) && !allowedOnce) {
 			buildC2P(details, app_id);
 		}
 
 		// Note: tab.purplebox handles a race condition where this function is sometimes called before onNavigation()
 		if (conf.show_alert && tab && !tab.prefetched && tab.purplebox) {
-			if (foundBugs.getAppsCount(details.tab_id) > num_apps_old || c2pDb.allowedOnce(details.tab_id, app_id)) {
+			if (foundBugs.getAppsCount(details.tab_id) > num_apps_old || allowedOnce) {
 				this.purplebox.updateBox(details.tab_id, app_id);
 			}
 		}
@@ -678,7 +670,7 @@ class EventHandlers {
 				const surrogates = surrogatedb.getForTracker(details.url, appId, bugId, ti.host);
 
 				if (surrogates.length > 0) {
-					code = _.reduce(surrogates, (memo, s) => {
+					code = reduce(surrogates, (memo, s) => {
 						memo += s.code; // eslint-disable-line no-param-reassign
 						return memo;
 					}, '');