Merge pull request #185 from duckduckgo/develop

release version 2018.3.29

browsers/chrome/manifest.json

@@ -2,7 +2,7 @@
     "name": "__MSG_appName__",
     "description": "__MSG_appDesc__",
     "default_locale": "en",
-    "version": "2018.3.23",
+    "version": "2018.3.29",
     "icons": {
         "16": "img/icon_16.png",
         "48": "img/icon_48.png",

browsers/firefox/manifest.json

@@ -6,7 +6,7 @@
               "strict_min_version": "57.0"
         }
     },
-    "version": "2018.3.23",
+    "version": "2018.3.29",
     "description": "Privacy, simplified. Protect your data as you search and browse: tracker blocking, smarter encryption, private search, and more.",
     "icons": {
         "16": "img/icon_16.png",

scripts/buildEntityMap.js

@@ -1,23 +1,28 @@
 var request = require('request'),
     fs = require('fs');
 
-request('https://raw.githubusercontent.com/mozilla-services/shavar-prod-lists/master/disconnect-entitylist.json', function (err, res, body) {
+let requestData = {
+    method: 'get',
+    uri: 'https://duckduckgo.com/contentblocking.js?l=entitylist2',
+    gzip: true
+}
 
-      if (err) {
-          return console.log(err);
-      }
+request(requestData, (err, res, body) => {
+    if (err) {
+        return console.log(err);
+    }
+
+    let json = JSON.parse(body);
+    let out = {};
 
-      let json = JSON.parse(body);
-      let out = {};
+    for(let parent in json) {
+        json[parent].properties.map(url => {
+            out[url] = parent;
+        });
+        json[parent].resources.map(url => {
+            out[url] = parent;
+        });
+    }
 
-      for(let parent in json) {
-          json[parent].properties.map(url => {
-              out[url] = parent;
-          });
-          json[parent].resources.map(url => {
-              out[url] = parent;
-          });
-      }
-
-      fs.writeFile('shared/data/tracker_lists/entityMap.json', JSON.stringify(out), (err) => { if(err) console.log(err)} );
+    fs.writeFile('shared/data/tracker_lists/entityMap.json', JSON.stringify(out), (err) => { if(err) console.log(err)} );
 });

shared/data/constants.js

@@ -3,7 +3,7 @@ module.exports = {
     "blockLists": [
         "trackersWithParentCompany.json"
     ],
-    "entityList": "https://duckduckgo.com/contentblocking.js?l=entitylist",
+    "entityList": "https://duckduckgo.com/contentblocking.js?l=entitylist2",
     "entityMap": "data/tracker_lists/entityMap.json",
     "easylists": ["privacy", "general"],
     "blocking": ["Advertising", "Analytics"],
@@ -43,7 +43,7 @@ module.exports = {
         "appnexus": 10,
         "oracle": 10,
         "mediamath": 9,
-        "yahoo": 9,
+        "oath": 9,
         "maxcdn": 7,
         "automattic": 7
     }

shared/data/tracker_lists/trackersWithParentCompany.json

@@ -1,41 +1,5 @@
 {
     "TopTrackerDomains": {
-        "amazon-adsystem.com": {
-            "c": "Amazon",
-            "t": "Advertising"
-        },
-        "amazon.ca": {
-            "c": "Amazon",
-            "t": "Advertising"
-        },
-        "amazon.co.jp": {
-            "c": "Amazon",
-            "t": "Advertising"
-        },
-        "amazon.co.uk": {
-            "c": "Amazon",
-            "t": "Advertising"
-        },
-        "amazon.de": {
-            "c": "Amazon",
-            "t": "Advertising"
-        },
-        "amazon.es": {
-            "c": "Amazon",
-            "t": "Advertising"
-        },
-        "amazon.fr": {
-            "c": "Amazon",
-            "t": "Advertising"
-        },
-        "amazon.it": {
-            "c": "Amazon",
-            "t": "Advertising"
-        },
-        "assoc-amazon.com": {
-            "c": "Amazon",
-            "t": "Advertising"
-        },
         "adlantic.nl": {
             "c": "AppNexus",
             "t": "Advertising"
@@ -188,10 +152,6 @@
             "c": "Google",
             "t": "Advertising"
         },
-        "alexametrics.com": {
-            "c": "Amazon",
-            "t": "Analytics"
-        },
         "polldaddy.com": {
             "c": "Automattic",
             "t": "Analytics"
@@ -1151,39 +1111,39 @@
             "u": "https://aloodo.com/"
         },
         "amazon-adsystem.com": {
-            "c": "Amazon",
+            "c": "Amazon.com",
             "u": "http://www.amazon.com/"
         },
         "amazon.ca": {
-            "c": "Amazon",
+            "c": "Amazon.com",
             "u": "http://www.amazon.com/"
         },
         "amazon.co.jp": {
-            "c": "Amazon",
+            "c": "Amazon.com",
             "u": "http://www.amazon.com/"
         },
         "amazon.co.uk": {
-            "c": "Amazon",
+            "c": "Amazon.com",
             "u": "http://www.amazon.com/"
         },
         "amazon.de": {
-            "c": "Amazon",
+            "c": "Amazon.com",
             "u": "http://www.amazon.com/"
         },
         "amazon.es": {
-            "c": "Amazon",
+            "c": "Amazon.com",
             "u": "http://www.amazon.com/"
         },
         "amazon.fr": {
-            "c": "Amazon",
+            "c": "Amazon.com",
             "u": "http://www.amazon.com/"
         },
         "amazon.it": {
-            "c": "Amazon",
+            "c": "Amazon.com",
             "u": "http://www.amazon.com/"
         },
         "assoc-amazon.com": {
-            "c": "Amazon",
+            "c": "Amazon.com",
             "u": "http://www.amazon.com/"
         },
         "adnetwork.vn": {
@@ -6261,7 +6221,7 @@
             "u": "http://amazingcounters.com/"
         },
         "alexametrics.com": {
-            "c": "Amazon",
+            "c": "Amazon.com",
             "u": "http://www.amazon.com/"
         },
         "attracta.com": {

shared/js/background/background.es6.js

@@ -133,8 +133,8 @@ chrome.webRequest.onBeforeRequest.addListener(
 
             var tracker = trackers.isTracker(requestData.url, thisTab, requestData);
 
-            // count and block trackers. Skip things that matched in the trackersWhitelist
-            if (tracker && !(tracker.type === 'trackersWhitelist')) {
+            // count and block trackers. Skip things that matched in the trackersWhitelist unless they're first party
+            if (tracker && !(tracker.type === 'trackersWhitelist' && tracker.reason !== 'first party')) {
                 // only count trackers on pages with 200 response. Trackers on these sites are still
                 // blocked below but not counted toward company stats
                 if (thisTab.statusCode === 200) {

shared/js/background/classes/tab.es6.js

@@ -75,17 +75,15 @@ class Tab {
         chrome.browserAction.setIcon({path: 'img/icon_48.png', tabId: this.id});
     };
 
-    /* Store all trackers for a given tab even if we
-     * don't block them.
-     */
+    // Store all trackers for a given tab even if we don't block them.
     addToTrackers (t) {
         let tracker = this.trackers[t.parentCompany];
         if (tracker) {
             tracker.increment();
-            tracker.addURL(t.url);
+            tracker.update(t);
         }
         else {
-            let newTracker = new Tracker(t.parentCompany, t.url, t.type);
+            let newTracker = new Tracker(t);
             this.trackers[t.parentCompany] = newTracker;
 
             // first time we have seen this network tracker on the page
@@ -99,10 +97,10 @@ class Tab {
         let tracker = this.trackersBlocked[t.parentCompany];
         if (tracker) {
             tracker.increment();
-            tracker.addURL(t.url);
+            tracker.update(t);
         }
         else {
-            let newTracker = new Tracker(t.parentCompany, t.url, t.type);
+            let newTracker = new Tracker(t);
             this.trackersBlocked[t.parentCompany] = newTracker;
             return newTracker;
         }

shared/js/background/classes/tracker.es6.js

@@ -1,11 +1,12 @@
 const Companies = require('../companies.es6')
 
 class Tracker {
-    constructor (name, url, type) {
-        this.parentCompany = Companies.get(name)
-        this.urls = [url]
+    constructor (t) {
+        this.parentCompany = Companies.get(t.parentCompany)
+        this.urls = {}
+        this.urls[t.url] = {isBlocked: t.block, reason: t.reason}
         this.count = 1 // request count
-        this.type = type || ''
+        this.type = t.type || ''
     }
 
     increment () {
@@ -16,9 +17,9 @@ class Tracker {
      * to track you through many different entities.
      * We store a list of all unique urls here.
      */
-    addURL (url) {
-        if (this.urls.indexOf(url) === -1) {
-            this.urls.push(url)
+    update (t) {
+        if (!this.urls[t.url]) {
+            this.urls[t.url] = {isBlocked: t.block, reason: t.reason}
         }
     }
 }

shared/js/background/debug.es6.js

@@ -4,8 +4,10 @@
  */
 const settings = require('./settings.es6')
 const abp = require('abp-filter-parser')
+const tabManager = require('./tab-manager.es6')
 
 window.dbg = {
   settings,
-  abp
+  abp,
+  tabManager
 }