update privacy grade (#363)

.gitmodules

@@ -4,3 +4,6 @@
 [submodule "submodules/bloom_cpp"]
 	path = submodules/bloom_cpp
 	url = https://github.com/duckduckgo/bloom_cpp
+[submodule "submodules/privacy-grade"]
+	path = submodules/privacy-grade
+	url = https://github.com/duckduckgo/privacy-grade

Core/AppUrls.swift

@@ -34,6 +34,7 @@ public struct AppUrls {
         static let easylistPrivacyBlockList = "\(base)/contentblocking.js?l=easyprivacy"
         static let trackersWhitelist = "\(base)/contentblocking/trackers-whitelist.txt"
         static let surrogates = "\(base)/contentblocking.js?l=surrogates"
+        static let entitylist = "\(base)/contentblocking.js?l=entitylist2"
         static let atb = "\(base)/atb.js\(devMode)"
         static let exti = "\(base)/exti/\(devMode)"
         static let feedback = "\(base)/feedback.js?type=app-feedback"
@@ -93,6 +94,10 @@ public struct AppUrls {
         return URL(string: Url.surrogates)!
     }
 
+    public var entitylist: URL {
+        return URL(string: Url.entitylist)!
+    }
+
     public var feedback: URL {
         return URL(string: Url.feedback)!
     }

Core/ContentBlockerLoader.swift

@@ -27,6 +27,7 @@ public class ContentBlockerLoader {
     private var disconnectStore = DisconnectMeStore()
     private var httpsUpgradeStore: HTTPSUpgradeStore = HTTPSUpgradePersistence()
     private var surrogateStore = SurrogateStore()
+    private var entityMappingStore: EntityMappingStore = DownloadedEntityMappingStore()
 
     public var hasData: Bool {
         return disconnectStore.hasData && easylistStore.hasData
@@ -52,39 +53,61 @@ public class ContentBlockerLoader {
         }
         easylistStore.removeLegacyLists()
     }
-
+    
     private func startRequests(with semaphore: DispatchSemaphore) -> Int {
-
         let contentBlockerRequest = ContentBlockerRequest()
-
+        requestEntityList(contentBlockerRequest, semaphore)
+        requestDisconnectMe(contentBlockerRequest, semaphore)
+        requestTrackerWhitelist(contentBlockerRequest, semaphore)
+        requestHttpsUpgrade(contentBlockerRequest, semaphore)
+        requestHttpsWhitelist(contentBlockerRequest, semaphore)
+        requestSurrogates(contentBlockerRequest, semaphore)
+        return contentBlockerRequest.requestCount
+    }
+
+    fileprivate func requestEntityList(_ contentBlockerRequest: ContentBlockerRequest, _ semaphore: DispatchSemaphore) {
+        contentBlockerRequest.request(.entitylist) { data, isCached in
+            if let data = data, !isCached {
+                self.newDataItems += 1
+                self.entityMappingStore.persist(data: data)
+            }
+            semaphore.signal()
+        }
+    }
+
+    fileprivate func requestDisconnectMe(_ contentBlockerRequest: ContentBlockerRequest, _ semaphore: DispatchSemaphore) {
         contentBlockerRequest.request(.disconnectMe) { data, isCached in
             if let data = data, !isCached {
                 self.newDataItems += 1
                 try? self.disconnectStore.persist(data: data)
             }
             semaphore.signal()
         }
-
+    }
+
+    fileprivate func requestTrackerWhitelist(_ contentBlockerRequest: ContentBlockerRequest, _ semaphore: DispatchSemaphore) {
         contentBlockerRequest.request(.trackersWhitelist) { data, isCached in
             if let data = data, !isCached {
                 self.newDataItems += 1
                 self.easylistStore.persistEasylistWhitelist(data: data)
             }
             semaphore.signal()
         }
-
+    }
+
+    fileprivate func requestHttpsUpgrade(_ contentBlockerRequest: ContentBlockerRequest, _ semaphore: DispatchSemaphore) {
         contentBlockerRequest.request(.httpsBloomFilterSpec) { data, _ in
             guard let data = data, let specification = try? HTTPSUpgradeParser.convertBloomFilterSpecification(fromJSONData: data) else {
                 semaphore.signal()
                 return
             }
-                
+
             if let storedSpecification = self.httpsUpgradeStore.bloomFilterSpecification(), storedSpecification == specification {
                 Logger.log(text: "Bloom filter already downloaded")
                 semaphore.signal()
                 return
             }
-                
+
             contentBlockerRequest.request(.httpsBloomFilter) { data, _ in
                 guard let data = data else {
                     semaphore.signal()
@@ -96,24 +119,25 @@ public class ContentBlockerLoader {
                 semaphore.signal()
             }
         }
-
+    }
+
+    fileprivate func requestHttpsWhitelist(_ contentBlockerRequest: ContentBlockerRequest, _ semaphore: DispatchSemaphore) {
         contentBlockerRequest.request(.httpsWhitelist) { data, isCached in
             if let data = data, !isCached, let whitelist = try? HTTPSUpgradeParser.convertWhitelist(fromJSONData: data) {
                 self.newDataItems += 1
                 self.httpsUpgradeStore.persistWhitelist(domains: whitelist)
             }
             semaphore.signal()
         }
-
+    }
+
+    fileprivate func requestSurrogates(_ contentBlockerRequest: ContentBlockerRequest, _ semaphore: DispatchSemaphore) {
         contentBlockerRequest.request(.surrogates) { data, isCached in
             if let data = data, !isCached {
                 self.newDataItems += 1
                 self.surrogateStore.parseAndPersist(data: data)
             }
             semaphore.signal()
         }
-
-        return contentBlockerRequest.requestCount
     }
-
 }

Core/ContentBlockerRequest.swift

@@ -30,6 +30,7 @@ class ContentBlockerRequest {
         case httpsBloomFilter
         case httpsWhitelist
         case surrogates
+        case entitylist = "entitylist2"
     }
 
     var requestCount = 0
@@ -82,6 +83,7 @@ class ContentBlockerRequest {
         case .httpsWhitelist: return appUrls.httpsWhitelist
         case .trackersWhitelist: return appUrls.trackersWhitelist
         case .surrogates: return appUrls.surrogates
+        case .entitylist: return appUrls.entitylist
         }
     }
 }

Core/ContentBlockerStringCache.swift

@@ -23,7 +23,7 @@ public class ContentBlockerStringCache {
 
     struct Constants {
         // bump the cache version if you know the cache should be invalidated on the next release
-        static let cacheVersion = 3
+        static let cacheVersion = 4
         static let cacheVersionKey = "com.duckduckgo.contentblockerstringcache.version"
     }
 

Core/DetectedTracker.swift

@@ -41,6 +41,10 @@ public struct DetectedTracker {
     public var isIpTracker: Bool {
         return URL.isValidIpHost(domain ?? "")
     }
+
+    public var networkNameForDisplay: String {
+        return networkName ?? domain ?? url
+    }
 
 }
 

Core/DisconnectMeStore.swift

@@ -26,25 +26,18 @@ public class DisconnectMeStore {
         static let disconnectJsonAllowed = "disconnect-json-allowed"
     }
 
-    private lazy var stringCache = ContentBlockerStringCache()
-
-    public init() {
-        stringCache = ContentBlockerStringCache()
+    public static var persistenceLocation: URL {
+        let path = FileManager.default.containerURL(forSecurityApplicationGroupIdentifier: ContentBlockerStoreConstants.groupName)
+        return path!.appendingPathComponent("disconnectme.json")
     }
 
+    private lazy var stringCache = ContentBlockerStringCache()
+
     var hasData: Bool {
-        return (try? persistenceLocation.checkResourceIsReachable()) ?? false
+        return !trackers.isEmpty
     }
 
-    public var trackers: [String: DisconnectMeTracker] {
-        do {
-            let data = try Data(contentsOf: persistenceLocation)
-            return try parse(data: data)
-        } catch {
-            Logger.log(items: "error parsing json for disconnect", error)
-            return [String: DisconnectMeTracker]()
-        }
-    }
+    public var trackers: [String: DisconnectMeTracker] = [:]
 
     var bannedTrackersJson: String {
         if let cached = stringCache.get(named: CacheKeys.disconnectJsonBanned) {
@@ -62,27 +55,39 @@ public class DisconnectMeStore {
             return cached
         }
         if let json = try? convertToInjectableJson(trackers.filter(byCategory: DisconnectMeTracker.Category.allowed)) {
+            stringCache.put(name: CacheKeys.disconnectJsonAllowed, value: json)
             return json
         }
         return "{}"
     }
 
     func persist(data: Data) throws {
-        Logger.log(items: "DisconnectMeStore", persistenceLocation)
-        try data.write(to: persistenceLocation, options: .atomic)
+        Logger.log(items: "DisconnectMeStore", DisconnectMeStore.persistenceLocation)
+        try data.write(to: DisconnectMeStore.persistenceLocation, options: .atomic)
+        loadTrackers()
         invalidateCache()
     }
 
+    public init() {
+        stringCache = ContentBlockerStringCache()
+        loadTrackers()
+    }
+
+    private func loadTrackers() {
+        do {
+            let data = try Data(contentsOf: DisconnectMeStore.persistenceLocation)
+            self.trackers = try DisconnectMeTrackersParser().convert(fromJsonData: data)
+        } catch {
+            Logger.log(items: "error parsing json for disconnect", error)
+            self.trackers = [:]
+        }
+    }
+
     private func invalidateCache() {
         stringCache.remove(named: CacheKeys.disconnectJsonAllowed)
         stringCache.remove(named: CacheKeys.disconnectJsonBanned)
     }
 
-    private func parse(data: Data) throws -> [String: DisconnectMeTracker] {
-        let parser = DisconnectMeTrackersParser()
-        return try parser.convert(fromJsonData: data)
-    }
-
     private func convertToInjectableJson(_ trackers: [String: DisconnectMeTracker]) throws -> String {
         let simplifiedTrackers = trackers.mapValues({ $0.networkName })
         let json = try JSONSerialization.data(withJSONObject: simplifiedTrackers, options: .prettyPrinted)
@@ -91,9 +96,13 @@ public class DisconnectMeStore {
         }
         return ""
     }
-
-    public var persistenceLocation: URL {
-        let path = FileManager.default.containerURL(forSecurityApplicationGroupIdentifier: ContentBlockerStoreConstants.groupName)
-        return path!.appendingPathComponent("disconnectme.json")
+
+    public func networkNameAndCategory(forDomain domain: String) -> ( networkName: String?, category: String? ) {
+        let lowercasedDomain = domain.lowercased()
+        if let tracker = trackers.first(where: { lowercasedDomain == $0.key || lowercasedDomain.hasSuffix(".\($0.key)") })?.value {
+            return ( tracker.networkName, tracker.category?.rawValue )
+        }
+        return ( nil, nil )
     }
+
 }

Core/EntityMapping.swift

@@ -0,0 +1,82 @@
+//
+//  EntityMapping.swift
+//  DuckDuckGo
+//
+//  Copyright © 2017 DuckDuckGo. All rights reserved.
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+//
+
+import Foundation
+
+public class EntityMapping {
+
+    private struct Entity: Decodable {
+
+        let properties: [String]?
+        let resources: [String]?
+
+    }
+
+    private let entities: [String: String]
+
+    public init(store: EntityMappingStore = DownloadedEntityMappingStore()) {
+
+        if let data = store.load(), let entities = try? EntityMapping.process(data) {
+            self.entities = entities
+        } else {
+            self.entities = [:]
+        }
+
+    }
+
+    func findEntity(forHost host: String) -> String? {
+        var parts = host.split(separator: ".")
+
+        while !parts.isEmpty {
+            if let entity = entities[parts.joined(separator: ".")] { return entity }
+            parts = Array(parts.dropFirst())
+        }
+
+        return nil
+    }
+
+    private static func process(_ data: Data) throws -> [String: String] {
+        if let decoded = decode(data) {
+            var entities = [String: String]()
+
+            decoded.forEach {
+                let entityName = $0.key
+                $0.value.properties?.forEach {
+                    entities[$0] = entityName
+                }
+                $0.value.resources?.forEach {
+                    entities[$0] = entityName
+                }
+            }
+
+            return entities
+        }
+        return [:]
+    }
+
+    private static func decode(_ data: Data) -> [String: Entity]? {
+        do {
+            return try JSONDecoder().decode([String: Entity].self, from: data)
+        } catch {
+            Logger.log(items: error)
+        }
+        return nil
+    }
+
+}