From 175a25757e8b5ab2866a41fcf07c1fc74a3389ff Mon Sep 17 00:00:00 2001 From: Craig Russell Date: Tue, 30 Jan 2018 17:21:50 +0000 Subject: [PATCH 01/12] WIP implementation of google analytics surrogates --- .../AnalyticsSurrogatesLoaderTest.kt | 98 +++++ .../AnalyticsSurrogatesTest.kt | 77 ++++ .../resources/binary/surrogates/surrogates_1 | 107 ++++++ .../resources/binary/surrogates/surrogates_6 | 348 ++++++++++++++++++ .../surrogates_no_empty_line_at_end_of_file | 348 ++++++++++++++++++ .../surrogates_with_different_mime_types | 166 +++++++++ .../surrogates_with_empty_line_at_end_of_file | 348 ++++++++++++++++++ .../AnalyticsSurrogates.kt | 51 +++ .../AnalyticsSurrogatesLoader.kt | 92 +++++ .../api/AnalyticsSurrogatesListDownloader.kt | 64 ++++ .../api/AnalyticsSurrogatesListService.kt | 28 ++ .../store/AnalyticsSurrogatesDataStore.kt | 46 +++ .../app/browser/BrowserWebViewClient.kt | 52 +-- .../app/browser/WebViewRequestInterceptor.kt | 100 +++++ .../com/duckduckgo/app/di/NetworkModule.kt | 5 + .../app/global/DuckDuckGoApplication.kt | 9 +- .../app/job/AppConfigurationDownloader.kt | 20 +- 17 files changed, 1905 insertions(+), 54 deletions(-) create mode 100644 app/src/androidTest/java/com/duckduckgo/app/analyticsSurrogates/AnalyticsSurrogatesLoaderTest.kt create mode 100644 app/src/androidTest/java/com/duckduckgo/app/analyticsSurrogates/AnalyticsSurrogatesTest.kt create mode 100644 app/src/androidTest/resources/binary/surrogates/surrogates_1 create mode 100644 app/src/androidTest/resources/binary/surrogates/surrogates_6 create mode 100644 app/src/androidTest/resources/binary/surrogates/surrogates_no_empty_line_at_end_of_file create mode 100644 app/src/androidTest/resources/binary/surrogates/surrogates_with_different_mime_types create mode 100644 app/src/androidTest/resources/binary/surrogates/surrogates_with_empty_line_at_end_of_file create mode 100644 app/src/main/java/com/duckduckgo/app/analyticsSurrogates/AnalyticsSurrogates.kt create mode 100644 app/src/main/java/com/duckduckgo/app/analyticsSurrogates/AnalyticsSurrogatesLoader.kt create mode 100644 app/src/main/java/com/duckduckgo/app/analyticsSurrogates/api/AnalyticsSurrogatesListDownloader.kt create mode 100644 app/src/main/java/com/duckduckgo/app/analyticsSurrogates/api/AnalyticsSurrogatesListService.kt create mode 100644 app/src/main/java/com/duckduckgo/app/analyticsSurrogates/store/AnalyticsSurrogatesDataStore.kt create mode 100644 app/src/main/java/com/duckduckgo/app/browser/WebViewRequestInterceptor.kt diff --git a/app/src/androidTest/java/com/duckduckgo/app/analyticsSurrogates/AnalyticsSurrogatesLoaderTest.kt b/app/src/androidTest/java/com/duckduckgo/app/analyticsSurrogates/AnalyticsSurrogatesLoaderTest.kt new file mode 100644 index 000000000000..6811197c171d --- /dev/null +++ b/app/src/androidTest/java/com/duckduckgo/app/analyticsSurrogates/AnalyticsSurrogatesLoaderTest.kt @@ -0,0 +1,98 @@ +/* + * Copyright (c) 2018 DuckDuckGo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.duckduckgo.app.analyticsSurrogates + +import android.support.test.InstrumentationRegistry +import com.duckduckgo.app.analyticsSurrogates.AnalyticsSurrogates.SurrogateResponse +import com.duckduckgo.app.analyticsSurrogates.store.AnalyticsSurrogatesDataStore +import org.junit.Assert +import org.junit.Assert.assertEquals +import org.junit.Before +import org.junit.Test + +class AnalyticsSurrogatesLoaderTest { + + private lateinit var testee: AnalyticsSurrogatesLoader + private lateinit var dataStore: AnalyticsSurrogatesDataStore + private lateinit var analyticsSurrogates: AnalyticsSurrogates + + @Before + fun setup() { + analyticsSurrogates = AnalyticsSurrogates() + dataStore = AnalyticsSurrogatesDataStore(InstrumentationRegistry.getTargetContext()) + testee = AnalyticsSurrogatesLoader(analyticsSurrogates, dataStore) + } + + @Test + fun whenLoading6SurrogatesThen6SurrogatesFound() { + val surrogates = initialiseFile("surrogates_6") + Assert.assertEquals(6, surrogates.size) + } + + @Test + fun whenLoading1SurrogateThen1SurrogateFound() { + val surrogates = initialiseFile("surrogates_1") + Assert.assertEquals(1, surrogates.size) + } + + @Test + fun whenLoadingWithNoEmptyLineAtEndOfFileThenLastSurrogateStillFound() { + val surrogates = initialiseFile("surrogates_no_empty_line_at_end_of_file") + assertEquals("googletagmanager.com/gtm.js", surrogates[5].name) + } + + @Test + fun whenLoadingWithEmptyLineAtEndOfFileThenLastSurrogateStillFound() { + val surrogates = initialiseFile("surrogates_with_empty_line_at_end_of_file") + assertEquals("googletagmanager.com/gtm.js", surrogates[5].name) + } + + @Test + fun whenLoadingMultipleSurrogatesThenOrderIsPreserved() { + val surrogates = initialiseFile("surrogates_6") + assertEquals("google-analytics.com/ga.js", surrogates[0].name) + assertEquals("google-analytics.com/analytics.js", surrogates[1].name) + assertEquals("google-analytics.com/inpage_linkid.js", surrogates[2].name) + assertEquals("google-analytics.com/cx/api.js", surrogates[3].name) + assertEquals("googletagservices.com/gpt.js", surrogates[4].name) + assertEquals("googletagmanager.com/gtm.js", surrogates[5].name) + } + + @Test + fun whenLoadingSurrogateThenMimeTypeIsPreserved() { + val surrogates = initialiseFile("surrogates_with_different_mime_types") + assertEquals("text/plain", surrogates[0].mimeType) + assertEquals("application/javascript", surrogates[1].mimeType) + assertEquals("application/json", surrogates[2].mimeType) + } + + @Test + fun whenLoadingSurrogateThenFunctionLengthIsPreserved() { + val surrogates = initialiseFile("surrogates_6") + val actualNumberOfLines = surrogates[0].jsFunction.reader().readLines().size + assertEquals(99, actualNumberOfLines) + } + + private fun initialiseFile(filename: String) : List { + return testee.convertBytes(readFile(filename)) + } + + private fun readFile(filename: String): ByteArray { + return javaClass.classLoader.getResource("binary/surrogates/$filename").readBytes() + } + +} \ No newline at end of file diff --git a/app/src/androidTest/java/com/duckduckgo/app/analyticsSurrogates/AnalyticsSurrogatesTest.kt b/app/src/androidTest/java/com/duckduckgo/app/analyticsSurrogates/AnalyticsSurrogatesTest.kt new file mode 100644 index 000000000000..caccae7e22da --- /dev/null +++ b/app/src/androidTest/java/com/duckduckgo/app/analyticsSurrogates/AnalyticsSurrogatesTest.kt @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2018 DuckDuckGo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.duckduckgo.app.analyticsSurrogates + +import android.net.Uri +import com.duckduckgo.app.analyticsSurrogates.AnalyticsSurrogates.SurrogateResponse +import org.junit.Assert.* +import org.junit.Before +import org.junit.Test + +class AnalyticsSurrogatesTest { + + private lateinit var testee: AnalyticsSurrogates + + @Before + fun setup() { + testee = AnalyticsSurrogates() + } + + @Test + fun whenInitialisedThenHasNoSurrogatesLoaded() { + assertEquals(0, testee.getAll().size) + } + + @Test + fun whenOneSurrogateLoadedThenOneReturnedFromFullList() { + val surrogate = SurrogateResponse() + testee.loadSurrogates(listOf(surrogate)) + assertEquals(1, testee.getAll().size) + } + + @Test + fun whenMultipleSurrogatesLoadedThenAllReturnedFromFullList() { + val surrogate = SurrogateResponse() + testee.loadSurrogates(listOf(surrogate, surrogate, surrogate)) + assertEquals(3, testee.getAll().size) + } + + @Test + fun whenSearchingForExactMatchingExistingSurrogateThenCanFindByName() { + val surrogate = SurrogateResponse(name = "foo") + testee.loadSurrogates(listOf(surrogate)) + val retrieved = testee.get(Uri.parse("foo")) + assertTrue(retrieved.responseAvailable) + } + + + @Test + fun whenSearchingForSubstringMatchingExistingSurrogateThenCanFindByName() { + val surrogate = SurrogateResponse(name = "foo.com") + testee.loadSurrogates(listOf(surrogate)) + val retrieved = testee.get(Uri.parse("foo.com/a/b/c")) + assertTrue(retrieved.responseAvailable) + } + + @Test + fun whenSearchingByNonExistentNameThenResponseUnavailableSurrogateResultReturned() { + val surrogate = SurrogateResponse(name = "foo") + testee.loadSurrogates(listOf(surrogate)) + val retrieved = testee.get(Uri.parse("bar")) + assertFalse(retrieved.responseAvailable) + } +} \ No newline at end of file diff --git a/app/src/androidTest/resources/binary/surrogates/surrogates_1 b/app/src/androidTest/resources/binary/surrogates/surrogates_1 new file mode 100644 index 000000000000..3c8cb2f442aa --- /dev/null +++ b/app/src/androidTest/resources/binary/surrogates/surrogates_1 @@ -0,0 +1,107 @@ +# To neutralize GA scripts. The goal is to provide the minimal API +# expected by clients of these scripts so that the end users are able +# to wholly block GA while minimizing risks of page breakage. +# Test cases (need way more): +# - https://github.com/chrisaljoudi/uBlock/issues/119 +# Reference API: +# - https://developers.google.com/analytics/devguides/collection/gajs/ +google-analytics.com/ga.js application/javascript +(function() { + var noopfn = function() { + ; + }; + // + var Gaq = function() { + ; + }; + Gaq.prototype.Na = noopfn; + Gaq.prototype.O = noopfn; + Gaq.prototype.Sa = noopfn; + Gaq.prototype.Ta = noopfn; + Gaq.prototype.Va = noopfn; + Gaq.prototype._createAsyncTracker = noopfn; + Gaq.prototype._getAsyncTracker = noopfn; + Gaq.prototype._getPlugin = noopfn; + Gaq.prototype.push = function(a) { + if ( typeof a === 'function' ) { + a(); return; + } + if ( Array.isArray(a) === false ) { + return; + } + // https://twitter.com/catovitch/status/776442930345218048 + // https://developers.google.com/analytics/devguides/collection/gajs/methods/gaJSApiDomainDirectory#_gat.GA_Tracker_._link + if ( a[0] === '_link' && typeof a[1] === 'string' ) { + window.location.assign(a[1]); + } + // https://github.com/gorhill/uBlock/issues/2162 + if ( a[0] === '_set' && a[1] === 'hitCallback' && typeof a[2] === 'function' ) { + a[2](); + } + }; + // + var tracker = (function() { + var out = {}; + var api = [ + '_addIgnoredOrganic _addIgnoredRef _addItem _addOrganic', + '_addTrans _clearIgnoredOrganic _clearIgnoredRef _clearOrganic', + '_cookiePathCopy _deleteCustomVar _getName _setAccount', + '_getAccount _getClientInfo _getDetectFlash _getDetectTitle', + '_getLinkerUrl _getLocalGifPath _getServiceMode _getVersion', + '_getVisitorCustomVar _initData _link _linkByPost', + '_setAllowAnchor _setAllowHash _setAllowLinker _setCampContentKey', + '_setCampMediumKey _setCampNameKey _setCampNOKey _setCampSourceKey', + '_setCampTermKey _setCampaignCookieTimeout _setCampaignTrack _setClientInfo', + '_setCookiePath _setCookiePersistence _setCookieTimeout _setCustomVar', + '_setDetectFlash _setDetectTitle _setDomainName _setLocalGifPath', + '_setLocalRemoteServerMode _setLocalServerMode _setReferrerOverride _setRemoteServerMode', + '_setSampleRate _setSessionTimeout _setSiteSpeedSampleRate _setSessionCookieTimeout', + '_setVar _setVisitorCookieTimeout _trackEvent _trackPageLoadTime', + '_trackPageview _trackSocial _trackTiming _trackTrans', + '_visitCode' + ].join(' ').split(/\s+/); + var i = api.length; + while ( i-- ) { + out[api[i]] = noopfn; + } + out._getLinkerUrl = function(a) { + return a; + }; + return out; + })(); + // + var Gat = function() { + ; + }; + Gat.prototype._anonymizeIP = noopfn; + Gat.prototype._createTracker = noopfn; + Gat.prototype._forceSSL = noopfn; + Gat.prototype._getPlugin = noopfn; + Gat.prototype._getTracker = function() { + return tracker; + }; + Gat.prototype._getTrackerByName = function() { + return tracker; + }; + Gat.prototype._getTrackers = noopfn; + Gat.prototype.aa = noopfn; + Gat.prototype.ab = noopfn; + Gat.prototype.hb = noopfn; + Gat.prototype.la = noopfn; + Gat.prototype.oa = noopfn; + Gat.prototype.pa = noopfn; + Gat.prototype.u = noopfn; + var gat = new Gat(); + window._gat = gat; + // + var gaq = new Gaq(); + (function() { + var aa = window._gaq || []; + if ( Array.isArray(aa) ) { + while ( aa[0] ) { + gaq.push(aa.shift()); + } + } + })(); + window._gaq = gaq.qf = gaq; +})(); diff --git a/app/src/androidTest/resources/binary/surrogates/surrogates_6 b/app/src/androidTest/resources/binary/surrogates/surrogates_6 new file mode 100644 index 000000000000..f2b3ecba7f14 --- /dev/null +++ b/app/src/androidTest/resources/binary/surrogates/surrogates_6 @@ -0,0 +1,348 @@ +# To neutralize GA scripts. The goal is to provide the minimal API +# expected by clients of these scripts so that the end users are able +# to wholly block GA while minimizing risks of page breakage. +# Test cases (need way more): +# - https://github.com/chrisaljoudi/uBlock/issues/119 +# Reference API: +# - https://developers.google.com/analytics/devguides/collection/gajs/ +google-analytics.com/ga.js application/javascript +(function() { + var noopfn = function() { + ; + }; + // + var Gaq = function() { + ; + }; + Gaq.prototype.Na = noopfn; + Gaq.prototype.O = noopfn; + Gaq.prototype.Sa = noopfn; + Gaq.prototype.Ta = noopfn; + Gaq.prototype.Va = noopfn; + Gaq.prototype._createAsyncTracker = noopfn; + Gaq.prototype._getAsyncTracker = noopfn; + Gaq.prototype._getPlugin = noopfn; + Gaq.prototype.push = function(a) { + if ( typeof a === 'function' ) { + a(); return; + } + if ( Array.isArray(a) === false ) { + return; + } + // https://twitter.com/catovitch/status/776442930345218048 + // https://developers.google.com/analytics/devguides/collection/gajs/methods/gaJSApiDomainDirectory#_gat.GA_Tracker_._link + if ( a[0] === '_link' && typeof a[1] === 'string' ) { + window.location.assign(a[1]); + } + // https://github.com/gorhill/uBlock/issues/2162 + if ( a[0] === '_set' && a[1] === 'hitCallback' && typeof a[2] === 'function' ) { + a[2](); + } + }; + // + var tracker = (function() { + var out = {}; + var api = [ + '_addIgnoredOrganic _addIgnoredRef _addItem _addOrganic', + '_addTrans _clearIgnoredOrganic _clearIgnoredRef _clearOrganic', + '_cookiePathCopy _deleteCustomVar _getName _setAccount', + '_getAccount _getClientInfo _getDetectFlash _getDetectTitle', + '_getLinkerUrl _getLocalGifPath _getServiceMode _getVersion', + '_getVisitorCustomVar _initData _link _linkByPost', + '_setAllowAnchor _setAllowHash _setAllowLinker _setCampContentKey', + '_setCampMediumKey _setCampNameKey _setCampNOKey _setCampSourceKey', + '_setCampTermKey _setCampaignCookieTimeout _setCampaignTrack _setClientInfo', + '_setCookiePath _setCookiePersistence _setCookieTimeout _setCustomVar', + '_setDetectFlash _setDetectTitle _setDomainName _setLocalGifPath', + '_setLocalRemoteServerMode _setLocalServerMode _setReferrerOverride _setRemoteServerMode', + '_setSampleRate _setSessionTimeout _setSiteSpeedSampleRate _setSessionCookieTimeout', + '_setVar _setVisitorCookieTimeout _trackEvent _trackPageLoadTime', + '_trackPageview _trackSocial _trackTiming _trackTrans', + '_visitCode' + ].join(' ').split(/\s+/); + var i = api.length; + while ( i-- ) { + out[api[i]] = noopfn; + } + out._getLinkerUrl = function(a) { + return a; + }; + return out; + })(); + // + var Gat = function() { + ; + }; + Gat.prototype._anonymizeIP = noopfn; + Gat.prototype._createTracker = noopfn; + Gat.prototype._forceSSL = noopfn; + Gat.prototype._getPlugin = noopfn; + Gat.prototype._getTracker = function() { + return tracker; + }; + Gat.prototype._getTrackerByName = function() { + return tracker; + }; + Gat.prototype._getTrackers = noopfn; + Gat.prototype.aa = noopfn; + Gat.prototype.ab = noopfn; + Gat.prototype.hb = noopfn; + Gat.prototype.la = noopfn; + Gat.prototype.oa = noopfn; + Gat.prototype.pa = noopfn; + Gat.prototype.u = noopfn; + var gat = new Gat(); + window._gat = gat; + // + var gaq = new Gaq(); + (function() { + var aa = window._gaq || []; + if ( Array.isArray(aa) ) { + while ( aa[0] ) { + gaq.push(aa.shift()); + } + } + })(); + window._gaq = gaq.qf = gaq; +})(); + +google-analytics.com/analytics.js application/javascript +(function() { + // https://developers.google.com/analytics/devguides/collection/analyticsjs/ + var noopfn = function() { + ; + }; + var noopnullfn = function() { + return null; + }; + // + var Tracker = function() { + ; + }; + var p = Tracker.prototype; + p.get = noopfn; + p.set = noopfn; + p.send = noopfn; + // + var w = window, + gaName = w.GoogleAnalyticsObject || 'ga'; + var ga = function() { + var len = arguments.length; + if ( len === 0 ) { + return; + } + var f = arguments[len-1]; + if ( typeof f !== 'object' || f === null || typeof f.hitCallback !== 'function' ) { + return; + } + try { + f.hitCallback(); + } catch (ex) { + } + }; + ga.create = function() { + return new Tracker(); + }; + ga.getByName = noopnullfn; + ga.getAll = function() { + return []; + }; + ga.remove = noopfn; + w[gaName] = ga; + // https://github.com/gorhill/uBlock/issues/3075 + var dl = w.dataLayer; + if ( dl instanceof Object && dl.hide instanceof Object && typeof dl.hide.end === 'function' ) { + dl.hide.end(); + } +})(); + +google-analytics.com/inpage_linkid.js application/javascript +(function() { + window._gaq = window._gaq || { + push: function() { + ; + } + }; +})(); + +# https://github.com/gorhill/uBlock/issues/2480 +# https://developers.google.com/analytics/devguides/collection/gajs/experiments#cxjs +google-analytics.com/cx/api.js application/javascript +(function() { + var noopfn = function() { + }; + window.cxApi = { + chooseVariation: function() { + return 0; + }, + getChosenVariation: noopfn, + setAllowHash: noopfn, + setChosenVariation: noopfn, + setCookiePath: noopfn, + setDomainName: noopfn + }; +})(); + +# Ubiquitous googletagservices.com: not blocked by EasyPrivacy. +# Tags are tiny bits of website code that let you measure traffic and +# visitor behavior +googletagservices.com/gpt.js application/javascript +(function() { + var p; + // https://developers.google.com/doubleclick-gpt/reference + var noopfn = function() { + ; + }.bind(); + var noopthisfn = function() { + return this; + }; + var noopnullfn = function() { + return null; + }; + var nooparrayfn = function() { + return []; + }; + var noopstrfn = function() { + return ''; + }; + // + var companionAdsService = { + addEventListener: noopthisfn, + enableSyncLoading: noopfn, + setRefreshUnfilledSlots: noopfn + }; + var contentService = { + addEventListener: noopthisfn, + setContent: noopfn + }; + var PassbackSlot = function() { + ; + }; + p = PassbackSlot.prototype; + p.display = noopfn; + p.get = noopnullfn; + p.set = noopthisfn; + p.setClickUrl = noopthisfn; + p.setTagForChildDirectedTreatment = noopthisfn; + p.setTargeting = noopthisfn; + p.updateTargetingFromMap = noopthisfn; + var pubAdsService = { + addEventListener: noopthisfn, + clear: noopfn, + clearCategoryExclusions: noopthisfn, + clearTagForChildDirectedTreatment: noopthisfn, + clearTargeting: noopthisfn, + collapseEmptyDivs: noopfn, + defineOutOfPagePassback: function() { return new PassbackSlot(); }, + definePassback: function() { return new PassbackSlot(); }, + disableInitialLoad: noopfn, + display: noopfn, + enableAsyncRendering: noopfn, + enableSingleRequest: noopfn, + enableSyncRendering: noopfn, + enableVideoAds: noopfn, + get: noopnullfn, + getAttributeKeys: nooparrayfn, + getTargeting: noopfn, + getTargetingKeys: nooparrayfn, + getSlots: nooparrayfn, + refresh: noopfn, + set: noopthisfn, + setCategoryExclusion: noopthisfn, + setCentering: noopfn, + setCookieOptions: noopthisfn, + setForceSafeFrame: noopthisfn, + setLocation: noopthisfn, + setPublisherProvidedId: noopthisfn, + setSafeFrameConfig: noopthisfn, + setTagForChildDirectedTreatment: noopthisfn, + setTargeting: noopthisfn, + setVideoContent: noopthisfn, + updateCorrelator: noopfn + }; + var SizeMappingBuilder = function() { + ; + }; + p = SizeMappingBuilder.prototype; + p.addSize = noopthisfn; + p.build = noopnullfn; + var Slot = function() { + ; + }; + p = Slot.prototype; + p.addService = noopthisfn; + p.clearCategoryExclusions = noopthisfn; + p.clearTargeting = noopthisfn; + p.defineSizeMapping = noopthisfn; + p.get = noopnullfn; + p.getAdUnitPath = nooparrayfn; + p.getAttributeKeys = nooparrayfn; + p.getCategoryExclusions = nooparrayfn; + p.getDomId = noopstrfn; + p.getSlotElementId = noopstrfn; + p.getSlotId = noopthisfn; + p.getTargeting = nooparrayfn; + p.getTargetingKeys = nooparrayfn; + p.set = noopthisfn; + p.setCategoryExclusion = noopthisfn; + p.setClickUrl = noopthisfn; + p.setCollapseEmptyDiv = noopthisfn; + p.setTargeting = noopthisfn; + // + var gpt = window.googletag || {}; + var cmd = gpt.cmd || []; + gpt.apiReady = true; + gpt.cmd = []; + gpt.cmd.push = function(a) { + try { + a(); + } catch (ex) { + } + return 1; + }; + gpt.companionAds = function() { return companionAdsService; }; + gpt.content = function() { return contentService; }; + gpt.defineOutOfPageSlot = function() { return new Slot(); }; + gpt.defineSlot = function() { return new Slot(); }; + gpt.destroySlots = noopfn; + gpt.disablePublisherConsole = noopfn; + gpt.display = noopfn; + gpt.enableServices = noopfn; + gpt.getVersion = noopstrfn; + gpt.pubads = function() { return pubAdsService; }; + gpt.pubadsReady = true; + gpt.setAdIframeTitle = noopfn; + gpt.sizeMapping = function() { return new SizeMappingBuilder(); }; + window.googletag = gpt; + while ( cmd.length !== 0 ) { + gpt.cmd.push(cmd.shift()); + } +})(); + +# Obviously more work needs to be done, but at least for now it takes care of: +# See related filter in assets/ublock/privacy.txt +# Also: +# - https://github.com/gorhill/uBlock/issues/2569 +# - https://github.com/uBlockOrigin/uAssets/issues/420 +googletagmanager.com/gtm.js application/javascript +(function() { + var noopfn = function() { + }; + var w = window; + w.ga = w.ga || noopfn; + var dl = w.dataLayer; + if ( dl instanceof Object === false ) { return; } + if ( dl.hide instanceof Object && typeof dl.hide.end === 'function' ) { + dl.hide.end(); + } + if ( typeof dl.push === 'function' ) { + dl.push = function(o) { + if ( + o instanceof Object && + typeof o.eventCallback === 'function' + ) { + setTimeout(o.eventCallback, 1); + } + }; + } +})(); \ No newline at end of file diff --git a/app/src/androidTest/resources/binary/surrogates/surrogates_no_empty_line_at_end_of_file b/app/src/androidTest/resources/binary/surrogates/surrogates_no_empty_line_at_end_of_file new file mode 100644 index 000000000000..f2b3ecba7f14 --- /dev/null +++ b/app/src/androidTest/resources/binary/surrogates/surrogates_no_empty_line_at_end_of_file @@ -0,0 +1,348 @@ +# To neutralize GA scripts. The goal is to provide the minimal API +# expected by clients of these scripts so that the end users are able +# to wholly block GA while minimizing risks of page breakage. +# Test cases (need way more): +# - https://github.com/chrisaljoudi/uBlock/issues/119 +# Reference API: +# - https://developers.google.com/analytics/devguides/collection/gajs/ +google-analytics.com/ga.js application/javascript +(function() { + var noopfn = function() { + ; + }; + // + var Gaq = function() { + ; + }; + Gaq.prototype.Na = noopfn; + Gaq.prototype.O = noopfn; + Gaq.prototype.Sa = noopfn; + Gaq.prototype.Ta = noopfn; + Gaq.prototype.Va = noopfn; + Gaq.prototype._createAsyncTracker = noopfn; + Gaq.prototype._getAsyncTracker = noopfn; + Gaq.prototype._getPlugin = noopfn; + Gaq.prototype.push = function(a) { + if ( typeof a === 'function' ) { + a(); return; + } + if ( Array.isArray(a) === false ) { + return; + } + // https://twitter.com/catovitch/status/776442930345218048 + // https://developers.google.com/analytics/devguides/collection/gajs/methods/gaJSApiDomainDirectory#_gat.GA_Tracker_._link + if ( a[0] === '_link' && typeof a[1] === 'string' ) { + window.location.assign(a[1]); + } + // https://github.com/gorhill/uBlock/issues/2162 + if ( a[0] === '_set' && a[1] === 'hitCallback' && typeof a[2] === 'function' ) { + a[2](); + } + }; + // + var tracker = (function() { + var out = {}; + var api = [ + '_addIgnoredOrganic _addIgnoredRef _addItem _addOrganic', + '_addTrans _clearIgnoredOrganic _clearIgnoredRef _clearOrganic', + '_cookiePathCopy _deleteCustomVar _getName _setAccount', + '_getAccount _getClientInfo _getDetectFlash _getDetectTitle', + '_getLinkerUrl _getLocalGifPath _getServiceMode _getVersion', + '_getVisitorCustomVar _initData _link _linkByPost', + '_setAllowAnchor _setAllowHash _setAllowLinker _setCampContentKey', + '_setCampMediumKey _setCampNameKey _setCampNOKey _setCampSourceKey', + '_setCampTermKey _setCampaignCookieTimeout _setCampaignTrack _setClientInfo', + '_setCookiePath _setCookiePersistence _setCookieTimeout _setCustomVar', + '_setDetectFlash _setDetectTitle _setDomainName _setLocalGifPath', + '_setLocalRemoteServerMode _setLocalServerMode _setReferrerOverride _setRemoteServerMode', + '_setSampleRate _setSessionTimeout _setSiteSpeedSampleRate _setSessionCookieTimeout', + '_setVar _setVisitorCookieTimeout _trackEvent _trackPageLoadTime', + '_trackPageview _trackSocial _trackTiming _trackTrans', + '_visitCode' + ].join(' ').split(/\s+/); + var i = api.length; + while ( i-- ) { + out[api[i]] = noopfn; + } + out._getLinkerUrl = function(a) { + return a; + }; + return out; + })(); + // + var Gat = function() { + ; + }; + Gat.prototype._anonymizeIP = noopfn; + Gat.prototype._createTracker = noopfn; + Gat.prototype._forceSSL = noopfn; + Gat.prototype._getPlugin = noopfn; + Gat.prototype._getTracker = function() { + return tracker; + }; + Gat.prototype._getTrackerByName = function() { + return tracker; + }; + Gat.prototype._getTrackers = noopfn; + Gat.prototype.aa = noopfn; + Gat.prototype.ab = noopfn; + Gat.prototype.hb = noopfn; + Gat.prototype.la = noopfn; + Gat.prototype.oa = noopfn; + Gat.prototype.pa = noopfn; + Gat.prototype.u = noopfn; + var gat = new Gat(); + window._gat = gat; + // + var gaq = new Gaq(); + (function() { + var aa = window._gaq || []; + if ( Array.isArray(aa) ) { + while ( aa[0] ) { + gaq.push(aa.shift()); + } + } + })(); + window._gaq = gaq.qf = gaq; +})(); + +google-analytics.com/analytics.js application/javascript +(function() { + // https://developers.google.com/analytics/devguides/collection/analyticsjs/ + var noopfn = function() { + ; + }; + var noopnullfn = function() { + return null; + }; + // + var Tracker = function() { + ; + }; + var p = Tracker.prototype; + p.get = noopfn; + p.set = noopfn; + p.send = noopfn; + // + var w = window, + gaName = w.GoogleAnalyticsObject || 'ga'; + var ga = function() { + var len = arguments.length; + if ( len === 0 ) { + return; + } + var f = arguments[len-1]; + if ( typeof f !== 'object' || f === null || typeof f.hitCallback !== 'function' ) { + return; + } + try { + f.hitCallback(); + } catch (ex) { + } + }; + ga.create = function() { + return new Tracker(); + }; + ga.getByName = noopnullfn; + ga.getAll = function() { + return []; + }; + ga.remove = noopfn; + w[gaName] = ga; + // https://github.com/gorhill/uBlock/issues/3075 + var dl = w.dataLayer; + if ( dl instanceof Object && dl.hide instanceof Object && typeof dl.hide.end === 'function' ) { + dl.hide.end(); + } +})(); + +google-analytics.com/inpage_linkid.js application/javascript +(function() { + window._gaq = window._gaq || { + push: function() { + ; + } + }; +})(); + +# https://github.com/gorhill/uBlock/issues/2480 +# https://developers.google.com/analytics/devguides/collection/gajs/experiments#cxjs +google-analytics.com/cx/api.js application/javascript +(function() { + var noopfn = function() { + }; + window.cxApi = { + chooseVariation: function() { + return 0; + }, + getChosenVariation: noopfn, + setAllowHash: noopfn, + setChosenVariation: noopfn, + setCookiePath: noopfn, + setDomainName: noopfn + }; +})(); + +# Ubiquitous googletagservices.com: not blocked by EasyPrivacy. +# Tags are tiny bits of website code that let you measure traffic and +# visitor behavior +googletagservices.com/gpt.js application/javascript +(function() { + var p; + // https://developers.google.com/doubleclick-gpt/reference + var noopfn = function() { + ; + }.bind(); + var noopthisfn = function() { + return this; + }; + var noopnullfn = function() { + return null; + }; + var nooparrayfn = function() { + return []; + }; + var noopstrfn = function() { + return ''; + }; + // + var companionAdsService = { + addEventListener: noopthisfn, + enableSyncLoading: noopfn, + setRefreshUnfilledSlots: noopfn + }; + var contentService = { + addEventListener: noopthisfn, + setContent: noopfn + }; + var PassbackSlot = function() { + ; + }; + p = PassbackSlot.prototype; + p.display = noopfn; + p.get = noopnullfn; + p.set = noopthisfn; + p.setClickUrl = noopthisfn; + p.setTagForChildDirectedTreatment = noopthisfn; + p.setTargeting = noopthisfn; + p.updateTargetingFromMap = noopthisfn; + var pubAdsService = { + addEventListener: noopthisfn, + clear: noopfn, + clearCategoryExclusions: noopthisfn, + clearTagForChildDirectedTreatment: noopthisfn, + clearTargeting: noopthisfn, + collapseEmptyDivs: noopfn, + defineOutOfPagePassback: function() { return new PassbackSlot(); }, + definePassback: function() { return new PassbackSlot(); }, + disableInitialLoad: noopfn, + display: noopfn, + enableAsyncRendering: noopfn, + enableSingleRequest: noopfn, + enableSyncRendering: noopfn, + enableVideoAds: noopfn, + get: noopnullfn, + getAttributeKeys: nooparrayfn, + getTargeting: noopfn, + getTargetingKeys: nooparrayfn, + getSlots: nooparrayfn, + refresh: noopfn, + set: noopthisfn, + setCategoryExclusion: noopthisfn, + setCentering: noopfn, + setCookieOptions: noopthisfn, + setForceSafeFrame: noopthisfn, + setLocation: noopthisfn, + setPublisherProvidedId: noopthisfn, + setSafeFrameConfig: noopthisfn, + setTagForChildDirectedTreatment: noopthisfn, + setTargeting: noopthisfn, + setVideoContent: noopthisfn, + updateCorrelator: noopfn + }; + var SizeMappingBuilder = function() { + ; + }; + p = SizeMappingBuilder.prototype; + p.addSize = noopthisfn; + p.build = noopnullfn; + var Slot = function() { + ; + }; + p = Slot.prototype; + p.addService = noopthisfn; + p.clearCategoryExclusions = noopthisfn; + p.clearTargeting = noopthisfn; + p.defineSizeMapping = noopthisfn; + p.get = noopnullfn; + p.getAdUnitPath = nooparrayfn; + p.getAttributeKeys = nooparrayfn; + p.getCategoryExclusions = nooparrayfn; + p.getDomId = noopstrfn; + p.getSlotElementId = noopstrfn; + p.getSlotId = noopthisfn; + p.getTargeting = nooparrayfn; + p.getTargetingKeys = nooparrayfn; + p.set = noopthisfn; + p.setCategoryExclusion = noopthisfn; + p.setClickUrl = noopthisfn; + p.setCollapseEmptyDiv = noopthisfn; + p.setTargeting = noopthisfn; + // + var gpt = window.googletag || {}; + var cmd = gpt.cmd || []; + gpt.apiReady = true; + gpt.cmd = []; + gpt.cmd.push = function(a) { + try { + a(); + } catch (ex) { + } + return 1; + }; + gpt.companionAds = function() { return companionAdsService; }; + gpt.content = function() { return contentService; }; + gpt.defineOutOfPageSlot = function() { return new Slot(); }; + gpt.defineSlot = function() { return new Slot(); }; + gpt.destroySlots = noopfn; + gpt.disablePublisherConsole = noopfn; + gpt.display = noopfn; + gpt.enableServices = noopfn; + gpt.getVersion = noopstrfn; + gpt.pubads = function() { return pubAdsService; }; + gpt.pubadsReady = true; + gpt.setAdIframeTitle = noopfn; + gpt.sizeMapping = function() { return new SizeMappingBuilder(); }; + window.googletag = gpt; + while ( cmd.length !== 0 ) { + gpt.cmd.push(cmd.shift()); + } +})(); + +# Obviously more work needs to be done, but at least for now it takes care of: +# See related filter in assets/ublock/privacy.txt +# Also: +# - https://github.com/gorhill/uBlock/issues/2569 +# - https://github.com/uBlockOrigin/uAssets/issues/420 +googletagmanager.com/gtm.js application/javascript +(function() { + var noopfn = function() { + }; + var w = window; + w.ga = w.ga || noopfn; + var dl = w.dataLayer; + if ( dl instanceof Object === false ) { return; } + if ( dl.hide instanceof Object && typeof dl.hide.end === 'function' ) { + dl.hide.end(); + } + if ( typeof dl.push === 'function' ) { + dl.push = function(o) { + if ( + o instanceof Object && + typeof o.eventCallback === 'function' + ) { + setTimeout(o.eventCallback, 1); + } + }; + } +})(); \ No newline at end of file diff --git a/app/src/androidTest/resources/binary/surrogates/surrogates_with_different_mime_types b/app/src/androidTest/resources/binary/surrogates/surrogates_with_different_mime_types new file mode 100644 index 000000000000..8239bb8014c8 --- /dev/null +++ b/app/src/androidTest/resources/binary/surrogates/surrogates_with_different_mime_types @@ -0,0 +1,166 @@ +# To neutralize GA scripts. The goal is to provide the minimal API +# expected by clients of these scripts so that the end users are able +# to wholly block GA while minimizing risks of page breakage. +# Test cases (need way more): +# - https://github.com/chrisaljoudi/uBlock/issues/119 +# Reference API: +# - https://developers.google.com/analytics/devguides/collection/gajs/ +google-analytics.com/ga.js text/plain +(function() { + var noopfn = function() { + ; + }; + // + var Gaq = function() { + ; + }; + Gaq.prototype.Na = noopfn; + Gaq.prototype.O = noopfn; + Gaq.prototype.Sa = noopfn; + Gaq.prototype.Ta = noopfn; + Gaq.prototype.Va = noopfn; + Gaq.prototype._createAsyncTracker = noopfn; + Gaq.prototype._getAsyncTracker = noopfn; + Gaq.prototype._getPlugin = noopfn; + Gaq.prototype.push = function(a) { + if ( typeof a === 'function' ) { + a(); return; + } + if ( Array.isArray(a) === false ) { + return; + } + // https://twitter.com/catovitch/status/776442930345218048 + // https://developers.google.com/analytics/devguides/collection/gajs/methods/gaJSApiDomainDirectory#_gat.GA_Tracker_._link + if ( a[0] === '_link' && typeof a[1] === 'string' ) { + window.location.assign(a[1]); + } + // https://github.com/gorhill/uBlock/issues/2162 + if ( a[0] === '_set' && a[1] === 'hitCallback' && typeof a[2] === 'function' ) { + a[2](); + } + }; + // + var tracker = (function() { + var out = {}; + var api = [ + '_addIgnoredOrganic _addIgnoredRef _addItem _addOrganic', + '_addTrans _clearIgnoredOrganic _clearIgnoredRef _clearOrganic', + '_cookiePathCopy _deleteCustomVar _getName _setAccount', + '_getAccount _getClientInfo _getDetectFlash _getDetectTitle', + '_getLinkerUrl _getLocalGifPath _getServiceMode _getVersion', + '_getVisitorCustomVar _initData _link _linkByPost', + '_setAllowAnchor _setAllowHash _setAllowLinker _setCampContentKey', + '_setCampMediumKey _setCampNameKey _setCampNOKey _setCampSourceKey', + '_setCampTermKey _setCampaignCookieTimeout _setCampaignTrack _setClientInfo', + '_setCookiePath _setCookiePersistence _setCookieTimeout _setCustomVar', + '_setDetectFlash _setDetectTitle _setDomainName _setLocalGifPath', + '_setLocalRemoteServerMode _setLocalServerMode _setReferrerOverride _setRemoteServerMode', + '_setSampleRate _setSessionTimeout _setSiteSpeedSampleRate _setSessionCookieTimeout', + '_setVar _setVisitorCookieTimeout _trackEvent _trackPageLoadTime', + '_trackPageview _trackSocial _trackTiming _trackTrans', + '_visitCode' + ].join(' ').split(/\s+/); + var i = api.length; + while ( i-- ) { + out[api[i]] = noopfn; + } + out._getLinkerUrl = function(a) { + return a; + }; + return out; + })(); + // + var Gat = function() { + ; + }; + Gat.prototype._anonymizeIP = noopfn; + Gat.prototype._createTracker = noopfn; + Gat.prototype._forceSSL = noopfn; + Gat.prototype._getPlugin = noopfn; + Gat.prototype._getTracker = function() { + return tracker; + }; + Gat.prototype._getTrackerByName = function() { + return tracker; + }; + Gat.prototype._getTrackers = noopfn; + Gat.prototype.aa = noopfn; + Gat.prototype.ab = noopfn; + Gat.prototype.hb = noopfn; + Gat.prototype.la = noopfn; + Gat.prototype.oa = noopfn; + Gat.prototype.pa = noopfn; + Gat.prototype.u = noopfn; + var gat = new Gat(); + window._gat = gat; + // + var gaq = new Gaq(); + (function() { + var aa = window._gaq || []; + if ( Array.isArray(aa) ) { + while ( aa[0] ) { + gaq.push(aa.shift()); + } + } + })(); + window._gaq = gaq.qf = gaq; +})(); + +google-analytics.com/analytics.js application/javascript +(function() { + // https://developers.google.com/analytics/devguides/collection/analyticsjs/ + var noopfn = function() { + ; + }; + var noopnullfn = function() { + return null; + }; + // + var Tracker = function() { + ; + }; + var p = Tracker.prototype; + p.get = noopfn; + p.set = noopfn; + p.send = noopfn; + // + var w = window, + gaName = w.GoogleAnalyticsObject || 'ga'; + var ga = function() { + var len = arguments.length; + if ( len === 0 ) { + return; + } + var f = arguments[len-1]; + if ( typeof f !== 'object' || f === null || typeof f.hitCallback !== 'function' ) { + return; + } + try { + f.hitCallback(); + } catch (ex) { + } + }; + ga.create = function() { + return new Tracker(); + }; + ga.getByName = noopnullfn; + ga.getAll = function() { + return []; + }; + ga.remove = noopfn; + w[gaName] = ga; + // https://github.com/gorhill/uBlock/issues/3075 + var dl = w.dataLayer; + if ( dl instanceof Object && dl.hide instanceof Object && typeof dl.hide.end === 'function' ) { + dl.hide.end(); + } +})(); + +google-analytics.com/inpage_linkid.js application/json +(function() { + window._gaq = window._gaq || { + push: function() { + ; + } + }; +})(); \ No newline at end of file diff --git a/app/src/androidTest/resources/binary/surrogates/surrogates_with_empty_line_at_end_of_file b/app/src/androidTest/resources/binary/surrogates/surrogates_with_empty_line_at_end_of_file new file mode 100644 index 000000000000..522ae4f44aed --- /dev/null +++ b/app/src/androidTest/resources/binary/surrogates/surrogates_with_empty_line_at_end_of_file @@ -0,0 +1,348 @@ +# To neutralize GA scripts. The goal is to provide the minimal API +# expected by clients of these scripts so that the end users are able +# to wholly block GA while minimizing risks of page breakage. +# Test cases (need way more): +# - https://github.com/chrisaljoudi/uBlock/issues/119 +# Reference API: +# - https://developers.google.com/analytics/devguides/collection/gajs/ +google-analytics.com/ga.js application/javascript +(function() { + var noopfn = function() { + ; + }; + // + var Gaq = function() { + ; + }; + Gaq.prototype.Na = noopfn; + Gaq.prototype.O = noopfn; + Gaq.prototype.Sa = noopfn; + Gaq.prototype.Ta = noopfn; + Gaq.prototype.Va = noopfn; + Gaq.prototype._createAsyncTracker = noopfn; + Gaq.prototype._getAsyncTracker = noopfn; + Gaq.prototype._getPlugin = noopfn; + Gaq.prototype.push = function(a) { + if ( typeof a === 'function' ) { + a(); return; + } + if ( Array.isArray(a) === false ) { + return; + } + // https://twitter.com/catovitch/status/776442930345218048 + // https://developers.google.com/analytics/devguides/collection/gajs/methods/gaJSApiDomainDirectory#_gat.GA_Tracker_._link + if ( a[0] === '_link' && typeof a[1] === 'string' ) { + window.location.assign(a[1]); + } + // https://github.com/gorhill/uBlock/issues/2162 + if ( a[0] === '_set' && a[1] === 'hitCallback' && typeof a[2] === 'function' ) { + a[2](); + } + }; + // + var tracker = (function() { + var out = {}; + var api = [ + '_addIgnoredOrganic _addIgnoredRef _addItem _addOrganic', + '_addTrans _clearIgnoredOrganic _clearIgnoredRef _clearOrganic', + '_cookiePathCopy _deleteCustomVar _getName _setAccount', + '_getAccount _getClientInfo _getDetectFlash _getDetectTitle', + '_getLinkerUrl _getLocalGifPath _getServiceMode _getVersion', + '_getVisitorCustomVar _initData _link _linkByPost', + '_setAllowAnchor _setAllowHash _setAllowLinker _setCampContentKey', + '_setCampMediumKey _setCampNameKey _setCampNOKey _setCampSourceKey', + '_setCampTermKey _setCampaignCookieTimeout _setCampaignTrack _setClientInfo', + '_setCookiePath _setCookiePersistence _setCookieTimeout _setCustomVar', + '_setDetectFlash _setDetectTitle _setDomainName _setLocalGifPath', + '_setLocalRemoteServerMode _setLocalServerMode _setReferrerOverride _setRemoteServerMode', + '_setSampleRate _setSessionTimeout _setSiteSpeedSampleRate _setSessionCookieTimeout', + '_setVar _setVisitorCookieTimeout _trackEvent _trackPageLoadTime', + '_trackPageview _trackSocial _trackTiming _trackTrans', + '_visitCode' + ].join(' ').split(/\s+/); + var i = api.length; + while ( i-- ) { + out[api[i]] = noopfn; + } + out._getLinkerUrl = function(a) { + return a; + }; + return out; + })(); + // + var Gat = function() { + ; + }; + Gat.prototype._anonymizeIP = noopfn; + Gat.prototype._createTracker = noopfn; + Gat.prototype._forceSSL = noopfn; + Gat.prototype._getPlugin = noopfn; + Gat.prototype._getTracker = function() { + return tracker; + }; + Gat.prototype._getTrackerByName = function() { + return tracker; + }; + Gat.prototype._getTrackers = noopfn; + Gat.prototype.aa = noopfn; + Gat.prototype.ab = noopfn; + Gat.prototype.hb = noopfn; + Gat.prototype.la = noopfn; + Gat.prototype.oa = noopfn; + Gat.prototype.pa = noopfn; + Gat.prototype.u = noopfn; + var gat = new Gat(); + window._gat = gat; + // + var gaq = new Gaq(); + (function() { + var aa = window._gaq || []; + if ( Array.isArray(aa) ) { + while ( aa[0] ) { + gaq.push(aa.shift()); + } + } + })(); + window._gaq = gaq.qf = gaq; +})(); + +google-analytics.com/analytics.js application/javascript +(function() { + // https://developers.google.com/analytics/devguides/collection/analyticsjs/ + var noopfn = function() { + ; + }; + var noopnullfn = function() { + return null; + }; + // + var Tracker = function() { + ; + }; + var p = Tracker.prototype; + p.get = noopfn; + p.set = noopfn; + p.send = noopfn; + // + var w = window, + gaName = w.GoogleAnalyticsObject || 'ga'; + var ga = function() { + var len = arguments.length; + if ( len === 0 ) { + return; + } + var f = arguments[len-1]; + if ( typeof f !== 'object' || f === null || typeof f.hitCallback !== 'function' ) { + return; + } + try { + f.hitCallback(); + } catch (ex) { + } + }; + ga.create = function() { + return new Tracker(); + }; + ga.getByName = noopnullfn; + ga.getAll = function() { + return []; + }; + ga.remove = noopfn; + w[gaName] = ga; + // https://github.com/gorhill/uBlock/issues/3075 + var dl = w.dataLayer; + if ( dl instanceof Object && dl.hide instanceof Object && typeof dl.hide.end === 'function' ) { + dl.hide.end(); + } +})(); + +google-analytics.com/inpage_linkid.js application/javascript +(function() { + window._gaq = window._gaq || { + push: function() { + ; + } + }; +})(); + +# https://github.com/gorhill/uBlock/issues/2480 +# https://developers.google.com/analytics/devguides/collection/gajs/experiments#cxjs +google-analytics.com/cx/api.js application/javascript +(function() { + var noopfn = function() { + }; + window.cxApi = { + chooseVariation: function() { + return 0; + }, + getChosenVariation: noopfn, + setAllowHash: noopfn, + setChosenVariation: noopfn, + setCookiePath: noopfn, + setDomainName: noopfn + }; +})(); + +# Ubiquitous googletagservices.com: not blocked by EasyPrivacy. +# Tags are tiny bits of website code that let you measure traffic and +# visitor behavior +googletagservices.com/gpt.js application/javascript +(function() { + var p; + // https://developers.google.com/doubleclick-gpt/reference + var noopfn = function() { + ; + }.bind(); + var noopthisfn = function() { + return this; + }; + var noopnullfn = function() { + return null; + }; + var nooparrayfn = function() { + return []; + }; + var noopstrfn = function() { + return ''; + }; + // + var companionAdsService = { + addEventListener: noopthisfn, + enableSyncLoading: noopfn, + setRefreshUnfilledSlots: noopfn + }; + var contentService = { + addEventListener: noopthisfn, + setContent: noopfn + }; + var PassbackSlot = function() { + ; + }; + p = PassbackSlot.prototype; + p.display = noopfn; + p.get = noopnullfn; + p.set = noopthisfn; + p.setClickUrl = noopthisfn; + p.setTagForChildDirectedTreatment = noopthisfn; + p.setTargeting = noopthisfn; + p.updateTargetingFromMap = noopthisfn; + var pubAdsService = { + addEventListener: noopthisfn, + clear: noopfn, + clearCategoryExclusions: noopthisfn, + clearTagForChildDirectedTreatment: noopthisfn, + clearTargeting: noopthisfn, + collapseEmptyDivs: noopfn, + defineOutOfPagePassback: function() { return new PassbackSlot(); }, + definePassback: function() { return new PassbackSlot(); }, + disableInitialLoad: noopfn, + display: noopfn, + enableAsyncRendering: noopfn, + enableSingleRequest: noopfn, + enableSyncRendering: noopfn, + enableVideoAds: noopfn, + get: noopnullfn, + getAttributeKeys: nooparrayfn, + getTargeting: noopfn, + getTargetingKeys: nooparrayfn, + getSlots: nooparrayfn, + refresh: noopfn, + set: noopthisfn, + setCategoryExclusion: noopthisfn, + setCentering: noopfn, + setCookieOptions: noopthisfn, + setForceSafeFrame: noopthisfn, + setLocation: noopthisfn, + setPublisherProvidedId: noopthisfn, + setSafeFrameConfig: noopthisfn, + setTagForChildDirectedTreatment: noopthisfn, + setTargeting: noopthisfn, + setVideoContent: noopthisfn, + updateCorrelator: noopfn + }; + var SizeMappingBuilder = function() { + ; + }; + p = SizeMappingBuilder.prototype; + p.addSize = noopthisfn; + p.build = noopnullfn; + var Slot = function() { + ; + }; + p = Slot.prototype; + p.addService = noopthisfn; + p.clearCategoryExclusions = noopthisfn; + p.clearTargeting = noopthisfn; + p.defineSizeMapping = noopthisfn; + p.get = noopnullfn; + p.getAdUnitPath = nooparrayfn; + p.getAttributeKeys = nooparrayfn; + p.getCategoryExclusions = nooparrayfn; + p.getDomId = noopstrfn; + p.getSlotElementId = noopstrfn; + p.getSlotId = noopthisfn; + p.getTargeting = nooparrayfn; + p.getTargetingKeys = nooparrayfn; + p.set = noopthisfn; + p.setCategoryExclusion = noopthisfn; + p.setClickUrl = noopthisfn; + p.setCollapseEmptyDiv = noopthisfn; + p.setTargeting = noopthisfn; + // + var gpt = window.googletag || {}; + var cmd = gpt.cmd || []; + gpt.apiReady = true; + gpt.cmd = []; + gpt.cmd.push = function(a) { + try { + a(); + } catch (ex) { + } + return 1; + }; + gpt.companionAds = function() { return companionAdsService; }; + gpt.content = function() { return contentService; }; + gpt.defineOutOfPageSlot = function() { return new Slot(); }; + gpt.defineSlot = function() { return new Slot(); }; + gpt.destroySlots = noopfn; + gpt.disablePublisherConsole = noopfn; + gpt.display = noopfn; + gpt.enableServices = noopfn; + gpt.getVersion = noopstrfn; + gpt.pubads = function() { return pubAdsService; }; + gpt.pubadsReady = true; + gpt.setAdIframeTitle = noopfn; + gpt.sizeMapping = function() { return new SizeMappingBuilder(); }; + window.googletag = gpt; + while ( cmd.length !== 0 ) { + gpt.cmd.push(cmd.shift()); + } +})(); + +# Obviously more work needs to be done, but at least for now it takes care of: +# See related filter in assets/ublock/privacy.txt +# Also: +# - https://github.com/gorhill/uBlock/issues/2569 +# - https://github.com/uBlockOrigin/uAssets/issues/420 +googletagmanager.com/gtm.js application/javascript +(function() { + var noopfn = function() { + }; + var w = window; + w.ga = w.ga || noopfn; + var dl = w.dataLayer; + if ( dl instanceof Object === false ) { return; } + if ( dl.hide instanceof Object && typeof dl.hide.end === 'function' ) { + dl.hide.end(); + } + if ( typeof dl.push === 'function' ) { + dl.push = function(o) { + if ( + o instanceof Object && + typeof o.eventCallback === 'function' + ) { + setTimeout(o.eventCallback, 1); + } + }; + } +})(); diff --git a/app/src/main/java/com/duckduckgo/app/analyticsSurrogates/AnalyticsSurrogates.kt b/app/src/main/java/com/duckduckgo/app/analyticsSurrogates/AnalyticsSurrogates.kt new file mode 100644 index 000000000000..23d8dce33ba9 --- /dev/null +++ b/app/src/main/java/com/duckduckgo/app/analyticsSurrogates/AnalyticsSurrogates.kt @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2018 DuckDuckGo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.duckduckgo.app.analyticsSurrogates + +import android.net.Uri +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class AnalyticsSurrogates @Inject constructor() { + + private val surrogates = mutableListOf() + + fun loadSurrogates(urls: List) { + surrogates.clear() + surrogates.addAll(urls) + } + + fun get(uri: Uri): SurrogateResponse { + val uriString = uri.toString() + + return surrogates.find { uriString.contains(it.name) } + ?: return SurrogateResponse(responseAvailable = false) + } + + fun getAll() : List { + return surrogates + } + + data class SurrogateResponse( + val responseAvailable: Boolean = true, + val name: String = "", + val jsFunction: String = "", + val mimeType: String = "" + ) + +} \ No newline at end of file diff --git a/app/src/main/java/com/duckduckgo/app/analyticsSurrogates/AnalyticsSurrogatesLoader.kt b/app/src/main/java/com/duckduckgo/app/analyticsSurrogates/AnalyticsSurrogatesLoader.kt new file mode 100644 index 000000000000..1db9b339a39a --- /dev/null +++ b/app/src/main/java/com/duckduckgo/app/analyticsSurrogates/AnalyticsSurrogatesLoader.kt @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2018 DuckDuckGo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.duckduckgo.app.analyticsSurrogates + +import android.support.annotation.WorkerThread +import com.duckduckgo.app.analyticsSurrogates.AnalyticsSurrogates.SurrogateResponse +import com.duckduckgo.app.analyticsSurrogates.store.AnalyticsSurrogatesDataStore +import timber.log.Timber +import java.io.ByteArrayInputStream +import javax.inject.Inject + +@WorkerThread +class AnalyticsSurrogatesLoader @Inject constructor( + private val analyticsSurrogates: AnalyticsSurrogates, + private val surrogatesDataStore: AnalyticsSurrogatesDataStore +) { + + fun loadData() { + if (surrogatesDataStore.hasData()) { + val bytes = surrogatesDataStore.loadData() + analyticsSurrogates.loadSurrogates(convertBytes(bytes)) + } + } + + fun convertBytes(bytes: ByteArray): List { + val surrogates = mutableListOf() + + val reader = ByteArrayInputStream(bytes).bufferedReader() + val existingLines = reader.readLines().toMutableList() + + if(existingLines.isNotEmpty() && existingLines.last().isNotBlank()) { + existingLines.add("") + } + + var nextLineIsNewRule = true + + var ruleName = "" + var mimeType = "" + val functionBuilder = StringBuilder() + + existingLines.forEach { + + if (it.startsWith("#")) { + return@forEach + } + + if (nextLineIsNewRule) { + + with(it.split(" ")) { + ruleName = this[0] + mimeType = this[1] + } + Timber.d("Found new surrogate rule: %s - %s", ruleName, mimeType) + nextLineIsNewRule = false + return@forEach + } + + if (it.isBlank()) { + surrogates.add( + SurrogateResponse( + name = ruleName, + mimeType = mimeType, + jsFunction = functionBuilder.toString() + ) + + ) + nextLineIsNewRule = true + return@forEach + } + + functionBuilder.append(it) + functionBuilder.append("\n") + } + + Timber.i("Processed %d surrogates", surrogates.size) + return surrogates + } +} \ No newline at end of file diff --git a/app/src/main/java/com/duckduckgo/app/analyticsSurrogates/api/AnalyticsSurrogatesListDownloader.kt b/app/src/main/java/com/duckduckgo/app/analyticsSurrogates/api/AnalyticsSurrogatesListDownloader.kt new file mode 100644 index 000000000000..42e20bb2e588 --- /dev/null +++ b/app/src/main/java/com/duckduckgo/app/analyticsSurrogates/api/AnalyticsSurrogatesListDownloader.kt @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2018 DuckDuckGo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.duckduckgo.app.analyticsSurrogates.api + +import com.duckduckgo.app.analyticsSurrogates.AnalyticsSurrogatesLoader +import com.duckduckgo.app.analyticsSurrogates.store.AnalyticsSurrogatesDataStore +import com.duckduckgo.app.global.api.isCached +import io.reactivex.Completable +import timber.log.Timber +import java.io.IOException +import javax.inject.Inject + + +class AnalyticsSurrogatesListDownloader @Inject constructor( + private val service: AnalyticsSurrogatesListService, + private val surrogatesDataStore: AnalyticsSurrogatesDataStore, + private val analyticsSurrogatesLoader: AnalyticsSurrogatesLoader +) { + + fun downloadList(): Completable { + + return Completable.fromAction { + + Timber.d("Downloading Google Analytics Surrogates data") + + val call = service.https() + val response = call.execute() + + Timber.d("Response received, success=${response.isSuccessful}") + + if (response.isCached && surrogatesDataStore.hasData()) { + Timber.d("Surrogates data already cached and stored") + return@fromAction + } + + if (response.isSuccessful) { + val bodyBytes = response.body()!!.bytes() + Timber.d("Updating surrogates data store with new data") + persistData(bodyBytes) + analyticsSurrogatesLoader.loadData() + } else { + throw IOException("Status: ${response.code()} - ${response.errorBody()?.string()}") + } + } + } + + private fun persistData(bodyBytes: ByteArray) { + surrogatesDataStore.saveData(bodyBytes) + } +} \ No newline at end of file diff --git a/app/src/main/java/com/duckduckgo/app/analyticsSurrogates/api/AnalyticsSurrogatesListService.kt b/app/src/main/java/com/duckduckgo/app/analyticsSurrogates/api/AnalyticsSurrogatesListService.kt new file mode 100644 index 000000000000..88c0af32fe3e --- /dev/null +++ b/app/src/main/java/com/duckduckgo/app/analyticsSurrogates/api/AnalyticsSurrogatesListService.kt @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2018 DuckDuckGo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.duckduckgo.app.analyticsSurrogates.api + +import okhttp3.ResponseBody +import retrofit2.Call +import retrofit2.http.GET + + +interface AnalyticsSurrogatesListService { + + @GET("/contentblocking.js?l=surrogates") + fun https(): Call +} \ No newline at end of file diff --git a/app/src/main/java/com/duckduckgo/app/analyticsSurrogates/store/AnalyticsSurrogatesDataStore.kt b/app/src/main/java/com/duckduckgo/app/analyticsSurrogates/store/AnalyticsSurrogatesDataStore.kt new file mode 100644 index 000000000000..c97247506f39 --- /dev/null +++ b/app/src/main/java/com/duckduckgo/app/analyticsSurrogates/store/AnalyticsSurrogatesDataStore.kt @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2018 DuckDuckGo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.duckduckgo.app.analyticsSurrogates.store + +import android.content.Context +import javax.inject.Inject + +class AnalyticsSurrogatesDataStore @Inject constructor(private val context: Context) { + + fun hasData(): Boolean = context.fileExists(FILENAME) + + fun loadData(): ByteArray = + context.openFileInput(FILENAME).use { it.readBytes() } + + fun saveData(byteArray: ByteArray) { + context.openFileOutput(FILENAME, Context.MODE_PRIVATE).write(byteArray) + } + + fun clearData() { + context.deleteFile(FILENAME) + } + + private fun Context.fileExists(filename: String): Boolean { + val file = getFileStreamPath(filename) + return file != null && file.exists() + } + + companion object { + private const val FILENAME = "surrogates.js" + } + +} diff --git a/app/src/main/java/com/duckduckgo/app/browser/BrowserWebViewClient.kt b/app/src/main/java/com/duckduckgo/app/browser/BrowserWebViewClient.kt index 7c842fbf999a..8d29b4baf94e 100644 --- a/app/src/main/java/com/duckduckgo/app/browser/BrowserWebViewClient.kt +++ b/app/src/main/java/com/duckduckgo/app/browser/BrowserWebViewClient.kt @@ -18,19 +18,15 @@ package com.duckduckgo.app.browser import android.graphics.Bitmap import android.net.Uri -import android.support.annotation.AnyThread import android.support.annotation.WorkerThread import android.webkit.WebResourceRequest import android.webkit.WebResourceResponse import android.webkit.WebView import android.webkit.WebViewClient -import com.duckduckgo.app.global.isHttp +import com.duckduckgo.app.analyticsSurrogates.AnalyticsSurrogates import com.duckduckgo.app.httpsupgrade.HttpsUpgrader -import com.duckduckgo.app.privacymonitor.model.TrustedSites import com.duckduckgo.app.trackerdetection.TrackerDetector -import com.duckduckgo.app.trackerdetection.model.ResourceType import timber.log.Timber -import java.util.concurrent.CountDownLatch import javax.inject.Inject @@ -38,7 +34,9 @@ class BrowserWebViewClient @Inject constructor( private val requestRewriter: DuckDuckGoRequestRewriter, private var trackerDetector: TrackerDetector, private var httpsUpgrader: HttpsUpgrader, - private val specialUrlDetector: SpecialUrlDetector + private val specialUrlDetector: SpecialUrlDetector, + private val analyticsSurrogates: AnalyticsSurrogates, + private val webViewRequestInterceptor: WebViewRequestInterceptor ) : WebViewClient() { var webViewClientListener: WebViewClientListener? = null @@ -94,45 +92,9 @@ class BrowserWebViewClient @Inject constructor( } @WorkerThread - override fun shouldInterceptRequest(view: WebView, request: WebResourceRequest): WebResourceResponse? { - Timber.v("Intercepting resource ${request.url} on page ${currentUrl}}") - - if (shouldUpgrade(request)) { - val newUri = httpsUpgrader.upgrade(request.url) - view.post { view.loadUrl(newUri.toString()) } - return WebResourceResponse(null, null, null) - } - - val documentUrl = currentUrl ?: return null - - if (TrustedSites.isTrusted(documentUrl)) { - return null - } - - if (request.url != null && request.url.isHttp) { - webViewClientListener?.pageHasHttpResources(documentUrl) - } - - if (shouldBlock(request, documentUrl)) { - return WebResourceResponse(null, null, null) - } - - return null - } - - private fun shouldUpgrade(request: WebResourceRequest) = - request.isForMainFrame && request.url != null && httpsUpgrader.shouldUpgrade(request.url) - - private fun shouldBlock(request: WebResourceRequest, documentUrl: String?): Boolean { - val url = request.url.toString() - - if (request.isForMainFrame || documentUrl == null) { - return false - } - - val trackingEvent = trackerDetector.evaluate(url, documentUrl, ResourceType.from(request)) ?: return false - webViewClientListener?.trackerDetected(trackingEvent) - return trackingEvent.blocked + override fun shouldInterceptRequest(webView: WebView, request: WebResourceRequest): WebResourceResponse? { + Timber.v("Intercepting resource ${request.url} on page $currentUrl") + return webViewRequestInterceptor.shouldIntercept(request, webView, currentUrl, webViewClientListener) } /** diff --git a/app/src/main/java/com/duckduckgo/app/browser/WebViewRequestInterceptor.kt b/app/src/main/java/com/duckduckgo/app/browser/WebViewRequestInterceptor.kt new file mode 100644 index 000000000000..4a00474cb004 --- /dev/null +++ b/app/src/main/java/com/duckduckgo/app/browser/WebViewRequestInterceptor.kt @@ -0,0 +1,100 @@ +/* + * Copyright (c) 2018 DuckDuckGo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.duckduckgo.app.browser + +import android.webkit.WebResourceRequest +import android.webkit.WebResourceResponse +import android.webkit.WebView +import com.duckduckgo.app.analyticsSurrogates.AnalyticsSurrogates +import com.duckduckgo.app.global.isHttp +import com.duckduckgo.app.httpsupgrade.HttpsUpgrader +import com.duckduckgo.app.privacymonitor.model.TrustedSites +import com.duckduckgo.app.trackerdetection.TrackerDetector +import com.duckduckgo.app.trackerdetection.model.ResourceType +import timber.log.Timber +import javax.inject.Inject + + +class WebViewRequestInterceptor @Inject constructor( + private val analyticsSurrogates: AnalyticsSurrogates, + private val trackerDetector: TrackerDetector, + private val httpsUpgrader: HttpsUpgrader +) { + + fun shouldIntercept( + request: WebResourceRequest, + webView: WebView, + currentUrl: String?, + webViewClientListener: WebViewClientListener? + ): WebResourceResponse? { + val url = request.url + + if (shouldUpgrade(request)) { + val newUri = httpsUpgrader.upgrade(url) + webView.post { webView.loadUrl(newUri.toString()) } + return WebResourceResponse(null, null, null) + } + + val documentUrl = currentUrl ?: return null + + if (TrustedSites.isTrusted(documentUrl)) { + return null + } + + if (url != null && url.isHttp) { + webViewClientListener?.pageHasHttpResources(documentUrl) + } + + val surrogate = analyticsSurrogates.get(url) + if (surrogate.responseAvailable) { + Timber.i("Surrogate found for %s", url) + return WebResourceResponse( + surrogate.mimeType, + "UTF-8", + surrogate.jsFunction.byteInputStream() + ) + } + + if (shouldBlock(request, documentUrl, webViewClientListener)) { + return WebResourceResponse(null, null, null) + } + + return null + } + + private fun shouldUpgrade(request: WebResourceRequest) = + request.isForMainFrame && request.url != null && httpsUpgrader.shouldUpgrade(request.url) + + private fun shouldBlock( + request: WebResourceRequest, + documentUrl: String?, + webViewClientListener: WebViewClientListener? + ): Boolean { + val url = request.url.toString() + + if (request.isForMainFrame || documentUrl == null) { + return false + } + + val trackingEvent = + trackerDetector.evaluate(url, documentUrl, ResourceType.from(request)) + ?: return false + webViewClientListener?.trackerDetected(trackingEvent) + return trackingEvent.blocked + } + +} \ No newline at end of file diff --git a/app/src/main/java/com/duckduckgo/app/di/NetworkModule.kt b/app/src/main/java/com/duckduckgo/app/di/NetworkModule.kt index 25605c9749b6..0d2e7d4898d7 100644 --- a/app/src/main/java/com/duckduckgo/app/di/NetworkModule.kt +++ b/app/src/main/java/com/duckduckgo/app/di/NetworkModule.kt @@ -17,6 +17,7 @@ package com.duckduckgo.app.di import android.content.Context +import com.duckduckgo.app.analyticsSurrogates.api.AnalyticsSurrogatesListService import com.duckduckgo.app.autocomplete.api.AutoCompleteService import com.duckduckgo.app.browser.R import com.duckduckgo.app.httpsupgrade.api.HttpsUpgradeListService @@ -68,6 +69,10 @@ class NetworkModule { fun autoCompleteService(retrofit: Retrofit): AutoCompleteService = retrofit.create(AutoCompleteService::class.java) + @Provides + fun surrogatesService(retrofit: Retrofit): AnalyticsSurrogatesListService = + retrofit.create(AnalyticsSurrogatesListService::class.java) + companion object { private const val CACHE_SIZE: Long = 10 * 1024 * 1024 // 10MB } diff --git a/app/src/main/java/com/duckduckgo/app/global/DuckDuckGoApplication.kt b/app/src/main/java/com/duckduckgo/app/global/DuckDuckGoApplication.kt index bf68d7d1b953..e3bb7f87a458 100644 --- a/app/src/main/java/com/duckduckgo/app/global/DuckDuckGoApplication.kt +++ b/app/src/main/java/com/duckduckgo/app/global/DuckDuckGoApplication.kt @@ -19,6 +19,7 @@ package com.duckduckgo.app.global import android.app.Activity import android.app.Application import android.app.Service +import com.duckduckgo.app.analyticsSurrogates.AnalyticsSurrogatesLoader import com.duckduckgo.app.browser.BuildConfig import com.duckduckgo.app.di.DaggerAppComponent import com.duckduckgo.app.job.AppConfigurationSyncer @@ -48,6 +49,9 @@ class DuckDuckGoApplication : HasActivityInjector, HasServiceInjector, Applicati @Inject lateinit var trackerDataLoader: TrackerDataLoader + @Inject + lateinit var analyticsSurrogatesLoader: AnalyticsSurrogatesLoader + @Inject lateinit var appConfigurationSyncer: AppConfigurationSyncer @@ -86,7 +90,10 @@ class DuckDuckGoApplication : HasActivityInjector, HasServiceInjector, Applicati } private fun loadTrackerData() { - Schedulers.io().scheduleDirect { trackerDataLoader.loadData() } + doAsync { + trackerDataLoader.loadData() + analyticsSurrogatesLoader.loadData() + } } private fun configureLogging() { diff --git a/app/src/main/java/com/duckduckgo/app/job/AppConfigurationDownloader.kt b/app/src/main/java/com/duckduckgo/app/job/AppConfigurationDownloader.kt index 64ad2ad061de..cd95e80935e0 100644 --- a/app/src/main/java/com/duckduckgo/app/job/AppConfigurationDownloader.kt +++ b/app/src/main/java/com/duckduckgo/app/job/AppConfigurationDownloader.kt @@ -16,10 +16,11 @@ package com.duckduckgo.app.job +import com.duckduckgo.app.analyticsSurrogates.api.AnalyticsSurrogatesListDownloader import com.duckduckgo.app.global.db.AppDatabase import com.duckduckgo.app.httpsupgrade.api.HttpsUpgradeListDownloader import com.duckduckgo.app.settings.db.AppConfigurationEntity -import com.duckduckgo.app.trackerdetection.Client +import com.duckduckgo.app.trackerdetection.Client.ClientName.* import com.duckduckgo.app.trackerdetection.api.TrackerDataDownloader import io.reactivex.Completable import io.reactivex.schedulers.Schedulers @@ -27,15 +28,17 @@ import timber.log.Timber import javax.inject.Inject class AppConfigurationDownloader @Inject constructor( - private val trackerDataDownloader: TrackerDataDownloader, - private val httpsUpgradeListDownloader: HttpsUpgradeListDownloader, - private val appDatabase: AppDatabase) { + private val trackerDataDownloader: TrackerDataDownloader, + private val httpsUpgradeListDownloader: HttpsUpgradeListDownloader, + private val analyticsSurrogatesDownloader: AnalyticsSurrogatesListDownloader, + private val appDatabase: AppDatabase) { fun downloadTask(): Completable { - val easyListDownload = trackerDataDownloader.downloadList(Client.ClientName.EASYLIST) - val easyPrivacyDownload = trackerDataDownloader.downloadList(Client.ClientName.EASYPRIVACY) - val trackersWhitelist = trackerDataDownloader.downloadList(Client.ClientName.TRACKERSWHITELIST) - val disconnectDownload = trackerDataDownloader.downloadList(Client.ClientName.DISCONNECT) + val easyListDownload = trackerDataDownloader.downloadList(EASYLIST) + val easyPrivacyDownload = trackerDataDownloader.downloadList(EASYPRIVACY) + val trackersWhitelist = trackerDataDownloader.downloadList(TRACKERSWHITELIST) + val disconnectDownload = trackerDataDownloader.downloadList(DISCONNECT) + val surrogatesDownload = analyticsSurrogatesDownloader.downloadList() val httpsUpgradeDownload = httpsUpgradeListDownloader.downloadList() return Completable.mergeDelayError(listOf( @@ -43,6 +46,7 @@ class AppConfigurationDownloader @Inject constructor( easyPrivacyDownload.subscribeOn(Schedulers.io()), trackersWhitelist.subscribeOn(Schedulers.io()), disconnectDownload.subscribeOn(Schedulers.io()), + surrogatesDownload.subscribeOn(Schedulers.io()), httpsUpgradeDownload.subscribeOn(Schedulers.io()) )).doOnComplete { Timber.i("Download task completed successfully") From 81f030b554e97848bedf8e5af85fef2034b7deea Mon Sep 17 00:00:00 2001 From: Craig Russell Date: Tue, 30 Jan 2018 17:22:21 +0000 Subject: [PATCH 02/12] Quieter logging --- .../app/httpsupgrade/api/HttpsUpgradeListDownloader.kt | 4 ++-- .../app/trackerdetection/TrackerDataLoader.kt | 10 +++++----- .../app/trackerdetection/api/TrackerDataDownloader.kt | 10 +++++----- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/app/src/main/java/com/duckduckgo/app/httpsupgrade/api/HttpsUpgradeListDownloader.kt b/app/src/main/java/com/duckduckgo/app/httpsupgrade/api/HttpsUpgradeListDownloader.kt index 445aa93862c4..ad7747ad4d3e 100644 --- a/app/src/main/java/com/duckduckgo/app/httpsupgrade/api/HttpsUpgradeListDownloader.kt +++ b/app/src/main/java/com/duckduckgo/app/httpsupgrade/api/HttpsUpgradeListDownloader.kt @@ -31,7 +31,7 @@ class HttpsUpgradeListDownloader @Inject constructor( fun downloadList(): Completable { - Timber.i("Downloading HTTPS Upgrade data") + Timber.d("Downloading HTTPS Upgrade data") return Completable.fromAction { @@ -39,7 +39,7 @@ class HttpsUpgradeListDownloader @Inject constructor( val response = call.execute() if (response.isCached && httpsUpgradeDao.count() != 0) { - Timber.i("HTTPS data already cached and stored") + Timber.d("HTTPS data already cached and stored") return@fromAction } diff --git a/app/src/main/java/com/duckduckgo/app/trackerdetection/TrackerDataLoader.kt b/app/src/main/java/com/duckduckgo/app/trackerdetection/TrackerDataLoader.kt index 14110137fd8c..aff961ff71ac 100644 --- a/app/src/main/java/com/duckduckgo/app/trackerdetection/TrackerDataLoader.kt +++ b/app/src/main/java/com/duckduckgo/app/trackerdetection/TrackerDataLoader.kt @@ -32,7 +32,7 @@ class TrackerDataLoader @Inject constructor( fun loadData() { - Timber.i("Loading Tracker data") + Timber.d("Loading Tracker data") // these are stored to disk, then fed to the C++ adblock module loadAdblockData(Client.ClientName.EASYLIST) @@ -44,21 +44,21 @@ class TrackerDataLoader @Inject constructor( } fun loadAdblockData(name: Client.ClientName) { - Timber.i("Looking for adblock tracker ${name.name} to load") + Timber.d("Looking for adblock tracker ${name.name} to load") if (trackerDataStore.hasData(name)) { - Timber.i("Found adblock tracker ${name.name}") + Timber.d("Found adblock tracker ${name.name}") val client = AdBlockClient(name) client.loadProcessedData(trackerDataStore.loadData(name)) trackerDetector.addClient(client) } else { - Timber.i("No adblock tracker ${name.name} found") + Timber.d("No adblock tracker ${name.name} found") } } fun loadDisconnectData() { val trackers = trackerDataDao.getAll() - Timber.i("Loaded ${trackers.size} disconnect trackers from DB") + Timber.d("Loaded ${trackers.size} disconnect trackers from DB") val client = DisconnectClient(Client.ClientName.DISCONNECT, trackers) trackerDetector.addClient(client) diff --git a/app/src/main/java/com/duckduckgo/app/trackerdetection/api/TrackerDataDownloader.kt b/app/src/main/java/com/duckduckgo/app/trackerdetection/api/TrackerDataDownloader.kt index e808e64decb8..e02311fab458 100644 --- a/app/src/main/java/com/duckduckgo/app/trackerdetection/api/TrackerDataDownloader.kt +++ b/app/src/main/java/com/duckduckgo/app/trackerdetection/api/TrackerDataDownloader.kt @@ -52,13 +52,13 @@ class TrackerDataDownloader @Inject constructor( return Completable.fromAction { - Timber.i("Downloading disconnect data") + Timber.d("Downloading disconnect data") val call = trackerListService.disconnect() val response = call.execute() if (response.isCached && trackerDataDao.count() != 0) { - Timber.i("Disconnect data already cached and stored") + Timber.d("Disconnect data already cached and stored") return@fromAction } @@ -82,18 +82,18 @@ class TrackerDataDownloader @Inject constructor( private fun easyDownload(clientName: Client.ClientName, callFactory: (clientName: Client.ClientName) -> Call): Completable { return Completable.fromAction { - Timber.i("Downloading ${clientName.name} data") + Timber.d("Downloading ${clientName.name} data") val call = callFactory(clientName) val response = call.execute() if (response.isCached && trackerDataStore.hasData(clientName)) { - Timber.i("${clientName.name} data already cached and stored") + Timber.d("${clientName.name} data already cached and stored") return@fromAction } if (response.isSuccessful) { val bodyBytes = response.body()!!.bytes() - Timber.i("Updating ${clientName.name} data store with new data") + Timber.d("Updating ${clientName.name} data store with new data") persistTrackerData(clientName, bodyBytes) trackerDataLoader.loadAdblockData(clientName) } else { From 6e2dd096dde622b58248ba46cc11f5191203e645 Mon Sep 17 00:00:00 2001 From: Craig Russell Date: Tue, 30 Jan 2018 17:25:39 +0000 Subject: [PATCH 03/12] Surrogate logic occurs only after identifying a url as one to block --- .../app/browser/WebViewRequestInterceptor.kt | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/app/src/main/java/com/duckduckgo/app/browser/WebViewRequestInterceptor.kt b/app/src/main/java/com/duckduckgo/app/browser/WebViewRequestInterceptor.kt index 4a00474cb004..3e496612422c 100644 --- a/app/src/main/java/com/duckduckgo/app/browser/WebViewRequestInterceptor.kt +++ b/app/src/main/java/com/duckduckgo/app/browser/WebViewRequestInterceptor.kt @@ -59,17 +59,18 @@ class WebViewRequestInterceptor @Inject constructor( webViewClientListener?.pageHasHttpResources(documentUrl) } - val surrogate = analyticsSurrogates.get(url) - if (surrogate.responseAvailable) { - Timber.i("Surrogate found for %s", url) - return WebResourceResponse( - surrogate.mimeType, - "UTF-8", - surrogate.jsFunction.byteInputStream() - ) - } - if (shouldBlock(request, documentUrl, webViewClientListener)) { + + val surrogate = analyticsSurrogates.get(url) + if (surrogate.responseAvailable) { + Timber.i("Surrogate found for %s", url) + return WebResourceResponse( + surrogate.mimeType, + "UTF-8", + surrogate.jsFunction.byteInputStream() + ) + } + return WebResourceResponse(null, null, null) } From 69eee7767fe655bc69273d979aec963cd3f91030 Mon Sep 17 00:00:00 2001 From: Craig Russell Date: Wed, 31 Jan 2018 12:05:47 +0000 Subject: [PATCH 04/12] Biggish refactor to allow better testing of web view request intercepor --- .../AnalyticsSurrogatesLoaderTest.kt | 3 +- .../AnalyticsSurrogatesTest.kt | 3 +- .../browser/WebViewRequestInterceptorTest.kt | 120 ++++++++++++++++++ .../app/httpsupgrade/HttpsUpgraderTest.kt | 2 +- .../db/HttpsUpgradePerformanceTest.kt | 3 +- .../TrackerDetectorListTest.kt | 4 +- .../trackerdetection/TrackerDetectorTest.kt | 2 +- .../AnalyticsSurrogates.kt | 24 ++-- .../AnalyticsSurrogatesLoader.kt | 1 - .../di/AnalyticsSurrogatesModule.kt} | 10 +- .../app/browser/BrowserWebViewClient.kt | 2 +- .../app/browser/DuckDuckGoRequestRewriter.kt | 19 ++- .../app/browser/di/BrowserModule.kt | 36 ++++++ .../app/browser/omnibar/QueryUrlConverter.kt | 4 +- .../com/duckduckgo/app/di/AppComponent.kt | 9 +- .../app/httpsupgrade/HttpsUpgrader.kt | 19 ++- .../httpsupgrade/di/HttpsUpgraderModule.kt | 32 +++++ .../app/trackerdetection/TrackerDetector.kt | 16 ++- .../di/TrackerDetectionModule.kt | 36 ++++++ 19 files changed, 297 insertions(+), 48 deletions(-) create mode 100644 app/src/androidTest/java/com/duckduckgo/app/browser/WebViewRequestInterceptorTest.kt rename app/src/main/java/com/duckduckgo/app/{di/BrowserModule.kt => analyticsSurrogates/di/AnalyticsSurrogatesModule.kt} (68%) create mode 100644 app/src/main/java/com/duckduckgo/app/browser/di/BrowserModule.kt create mode 100644 app/src/main/java/com/duckduckgo/app/httpsupgrade/di/HttpsUpgraderModule.kt create mode 100644 app/src/main/java/com/duckduckgo/app/trackerdetection/di/TrackerDetectionModule.kt diff --git a/app/src/androidTest/java/com/duckduckgo/app/analyticsSurrogates/AnalyticsSurrogatesLoaderTest.kt b/app/src/androidTest/java/com/duckduckgo/app/analyticsSurrogates/AnalyticsSurrogatesLoaderTest.kt index 6811197c171d..f2a7874b3124 100644 --- a/app/src/androidTest/java/com/duckduckgo/app/analyticsSurrogates/AnalyticsSurrogatesLoaderTest.kt +++ b/app/src/androidTest/java/com/duckduckgo/app/analyticsSurrogates/AnalyticsSurrogatesLoaderTest.kt @@ -17,7 +17,6 @@ package com.duckduckgo.app.analyticsSurrogates import android.support.test.InstrumentationRegistry -import com.duckduckgo.app.analyticsSurrogates.AnalyticsSurrogates.SurrogateResponse import com.duckduckgo.app.analyticsSurrogates.store.AnalyticsSurrogatesDataStore import org.junit.Assert import org.junit.Assert.assertEquals @@ -32,7 +31,7 @@ class AnalyticsSurrogatesLoaderTest { @Before fun setup() { - analyticsSurrogates = AnalyticsSurrogates() + analyticsSurrogates = AnalyticsSurrogatesImpl() dataStore = AnalyticsSurrogatesDataStore(InstrumentationRegistry.getTargetContext()) testee = AnalyticsSurrogatesLoader(analyticsSurrogates, dataStore) } diff --git a/app/src/androidTest/java/com/duckduckgo/app/analyticsSurrogates/AnalyticsSurrogatesTest.kt b/app/src/androidTest/java/com/duckduckgo/app/analyticsSurrogates/AnalyticsSurrogatesTest.kt index caccae7e22da..d3781f73c372 100644 --- a/app/src/androidTest/java/com/duckduckgo/app/analyticsSurrogates/AnalyticsSurrogatesTest.kt +++ b/app/src/androidTest/java/com/duckduckgo/app/analyticsSurrogates/AnalyticsSurrogatesTest.kt @@ -17,7 +17,6 @@ package com.duckduckgo.app.analyticsSurrogates import android.net.Uri -import com.duckduckgo.app.analyticsSurrogates.AnalyticsSurrogates.SurrogateResponse import org.junit.Assert.* import org.junit.Before import org.junit.Test @@ -28,7 +27,7 @@ class AnalyticsSurrogatesTest { @Before fun setup() { - testee = AnalyticsSurrogates() + testee = AnalyticsSurrogatesImpl() } @Test diff --git a/app/src/androidTest/java/com/duckduckgo/app/browser/WebViewRequestInterceptorTest.kt b/app/src/androidTest/java/com/duckduckgo/app/browser/WebViewRequestInterceptorTest.kt new file mode 100644 index 000000000000..bf6f89fb9110 --- /dev/null +++ b/app/src/androidTest/java/com/duckduckgo/app/browser/WebViewRequestInterceptorTest.kt @@ -0,0 +1,120 @@ +/* + * Copyright (c) 2018 DuckDuckGo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.duckduckgo.app.browser + +import android.net.Uri +import android.support.test.InstrumentationRegistry +import android.support.test.annotation.UiThreadTest +import android.webkit.WebResourceRequest +import android.webkit.WebView +import com.duckduckgo.app.analyticsSurrogates.AnalyticsSurrogates +import com.duckduckgo.app.analyticsSurrogates.SurrogateResponse +import com.duckduckgo.app.httpsupgrade.HttpsUpgrader +import com.duckduckgo.app.trackerdetection.Client +import com.duckduckgo.app.trackerdetection.TrackerDetector +import com.duckduckgo.app.trackerdetection.model.ResourceType +import com.duckduckgo.app.trackerdetection.model.TrackingEvent +import org.junit.Before +import org.mockito.MockitoAnnotations + +class WebViewRequestInterceptorTest { + + private lateinit var testee: WebViewRequestInterceptor + + private lateinit var stubRequestRewriter: RequestRewriter + private lateinit var stubTrackDetector: TrackerDetector + private lateinit var stubHttpsUpgrader: HttpsUpgrader + private lateinit var stubAnalyticsSurrogates: AnalyticsSurrogates + + private lateinit var webView: WebView + + @UiThreadTest + @Before + fun setup() { + MockitoAnnotations.initMocks(this) + + stubTrackDetector = object: TrackerDetector { + private val clients: MutableList = mutableListOf() + + override fun addClient(client: Client) { + clients.add(client) + } + + override fun evaluate(url: String, documentUrl: String, resourceType: ResourceType): TrackingEvent? { + return null + } + } + + stubHttpsUpgrader = object: HttpsUpgrader { + override fun shouldUpgrade(uri: Uri): Boolean { + return false + } + } + + stubAnalyticsSurrogates = object: AnalyticsSurrogates { + override fun loadSurrogates(urls: List) { + // do nothing + } + + override fun get(uri: Uri): SurrogateResponse { + return if(uri.toString().contains("good")) { + SurrogateResponse(name = "foo.com", mimeType = "application/javascript", jsFunction = "") + } else { + SurrogateResponse(responseAvailable = false) + } + } + + override fun getAll(): List { + return emptyList() + } + } + + stubRequestRewriter = object: RequestRewriter{ + override fun addCustomQueryParams(builder: Uri.Builder) {} + override fun shouldRewriteRequest(uri: Uri): Boolean = true + override fun rewriteRequestWithCustomQueryParams(request: Uri): Uri = request + } + + testee = WebViewRequestInterceptor( + trackerDetector = stubTrackDetector, + httpsUpgrader = stubHttpsUpgrader, + analyticsSurrogates = stubAnalyticsSurrogates + ) + + val context = InstrumentationRegistry.getTargetContext() + + webView = WebView(context) + } + + private fun getRequest(url: String, forMainFrame: Boolean): WebResourceRequest { + return object : WebResourceRequest { + + override fun isRedirect(): Boolean = false + + override fun getMethod(): String = "GET" + + override fun getRequestHeaders(): MutableMap = mutableMapOf() + + override fun hasGesture(): Boolean = false + + override fun isForMainFrame(): Boolean = forMainFrame + + override fun getUrl(): Uri = Uri.parse(url) + + } + } +} \ No newline at end of file diff --git a/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/HttpsUpgraderTest.kt b/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/HttpsUpgraderTest.kt index c958cf985570..e199d15ce8e9 100644 --- a/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/HttpsUpgraderTest.kt +++ b/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/HttpsUpgraderTest.kt @@ -31,7 +31,7 @@ class HttpsUpgraderTest { @Before fun before() { mockDao = mock() - testee = HttpsUpgrader(mockDao) + testee = HttpsUpgraderImpl(mockDao) } @Test diff --git a/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/db/HttpsUpgradePerformanceTest.kt b/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/db/HttpsUpgradePerformanceTest.kt index 2a726fe5536e..58c1475b3fc0 100644 --- a/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/db/HttpsUpgradePerformanceTest.kt +++ b/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/db/HttpsUpgradePerformanceTest.kt @@ -37,6 +37,7 @@ import android.net.Uri import android.support.test.InstrumentationRegistry import com.duckduckgo.app.global.db.AppDatabase import com.duckduckgo.app.httpsupgrade.HttpsUpgrader +import com.duckduckgo.app.httpsupgrade.HttpsUpgraderImpl import org.junit.Assert.assertEquals import org.junit.Assert.assertFalse import org.junit.Before @@ -55,7 +56,7 @@ class HttpsUpgraderPerformanceTest { fun setup() { db = Room.inMemoryDatabaseBuilder(InstrumentationRegistry.getContext(), AppDatabase::class.java).build() dao = db.httpsUpgradeDomainDao() - httpsUpgrader = HttpsUpgrader(dao) + httpsUpgrader = HttpsUpgraderImpl(dao) } @Test diff --git a/app/src/androidTest/java/com/duckduckgo/app/trackerdetection/TrackerDetectorListTest.kt b/app/src/androidTest/java/com/duckduckgo/app/trackerdetection/TrackerDetectorListTest.kt index 13ae4a6496db..551b9de2f427 100644 --- a/app/src/androidTest/java/com/duckduckgo/app/trackerdetection/TrackerDetectorListTest.kt +++ b/app/src/androidTest/java/com/duckduckgo/app/trackerdetection/TrackerDetectorListTest.kt @@ -56,11 +56,11 @@ class TrackerDetectorListTest { settingStore = mock() whenever(settingStore.privacyOn).thenReturn(true) - blockingOnlyTestee = TrackerDetector(TrackerNetworks(), settingStore) + blockingOnlyTestee = TrackerDetectorImpl(TrackerNetworks(), settingStore) blockingOnlyTestee.addClient(easyprivacyAdblock) blockingOnlyTestee.addClient(easylistAdblock) - testeeWithWhitelist = TrackerDetector(TrackerNetworks(), settingStore) + testeeWithWhitelist = TrackerDetectorImpl(TrackerNetworks(), settingStore) testeeWithWhitelist.addClient(trackersWhitelistAdblocks) testeeWithWhitelist.addClient(easyprivacyAdblock) testeeWithWhitelist.addClient(easylistAdblock) diff --git a/app/src/androidTest/java/com/duckduckgo/app/trackerdetection/TrackerDetectorTest.kt b/app/src/androidTest/java/com/duckduckgo/app/trackerdetection/TrackerDetectorTest.kt index 39463055e20e..40a81ce2d517 100644 --- a/app/src/androidTest/java/com/duckduckgo/app/trackerdetection/TrackerDetectorTest.kt +++ b/app/src/androidTest/java/com/duckduckgo/app/trackerdetection/TrackerDetectorTest.kt @@ -32,7 +32,7 @@ class TrackerDetectorTest { private val networkTrackers = TrackerNetworks() private val settingStore: PrivacySettingsStore = mock() - private val trackerDetector = TrackerDetector(networkTrackers, settingStore) + private val trackerDetector = TrackerDetectorImpl(networkTrackers, settingStore) companion object { private val resourceType = ResourceType.UNKNOWN diff --git a/app/src/main/java/com/duckduckgo/app/analyticsSurrogates/AnalyticsSurrogates.kt b/app/src/main/java/com/duckduckgo/app/analyticsSurrogates/AnalyticsSurrogates.kt index 23d8dce33ba9..fa21092e63e5 100644 --- a/app/src/main/java/com/duckduckgo/app/analyticsSurrogates/AnalyticsSurrogates.kt +++ b/app/src/main/java/com/duckduckgo/app/analyticsSurrogates/AnalyticsSurrogates.kt @@ -17,35 +17,37 @@ package com.duckduckgo.app.analyticsSurrogates import android.net.Uri -import javax.inject.Inject -import javax.inject.Singleton -@Singleton -class AnalyticsSurrogates @Inject constructor() { +interface AnalyticsSurrogates { + fun loadSurrogates(urls: List) + fun get(uri: Uri): SurrogateResponse + fun getAll(): List +} + +class AnalyticsSurrogatesImpl : AnalyticsSurrogates { private val surrogates = mutableListOf() - fun loadSurrogates(urls: List) { + override fun loadSurrogates(urls: List) { surrogates.clear() surrogates.addAll(urls) } - fun get(uri: Uri): SurrogateResponse { + override fun get(uri: Uri): SurrogateResponse { val uriString = uri.toString() return surrogates.find { uriString.contains(it.name) } ?: return SurrogateResponse(responseAvailable = false) } - fun getAll() : List { + override fun getAll(): List { return surrogates } +} - data class SurrogateResponse( +data class SurrogateResponse( val responseAvailable: Boolean = true, val name: String = "", val jsFunction: String = "", val mimeType: String = "" - ) - -} \ No newline at end of file +) diff --git a/app/src/main/java/com/duckduckgo/app/analyticsSurrogates/AnalyticsSurrogatesLoader.kt b/app/src/main/java/com/duckduckgo/app/analyticsSurrogates/AnalyticsSurrogatesLoader.kt index 1db9b339a39a..080e5c643ba8 100644 --- a/app/src/main/java/com/duckduckgo/app/analyticsSurrogates/AnalyticsSurrogatesLoader.kt +++ b/app/src/main/java/com/duckduckgo/app/analyticsSurrogates/AnalyticsSurrogatesLoader.kt @@ -17,7 +17,6 @@ package com.duckduckgo.app.analyticsSurrogates import android.support.annotation.WorkerThread -import com.duckduckgo.app.analyticsSurrogates.AnalyticsSurrogates.SurrogateResponse import com.duckduckgo.app.analyticsSurrogates.store.AnalyticsSurrogatesDataStore import timber.log.Timber import java.io.ByteArrayInputStream diff --git a/app/src/main/java/com/duckduckgo/app/di/BrowserModule.kt b/app/src/main/java/com/duckduckgo/app/analyticsSurrogates/di/AnalyticsSurrogatesModule.kt similarity index 68% rename from app/src/main/java/com/duckduckgo/app/di/BrowserModule.kt rename to app/src/main/java/com/duckduckgo/app/analyticsSurrogates/di/AnalyticsSurrogatesModule.kt index 8a1bf10391a9..4696848c6d46 100644 --- a/app/src/main/java/com/duckduckgo/app/di/BrowserModule.kt +++ b/app/src/main/java/com/duckduckgo/app/analyticsSurrogates/di/AnalyticsSurrogatesModule.kt @@ -14,15 +14,17 @@ * limitations under the License. */ -package com.duckduckgo.app.di +package com.duckduckgo.app.analyticsSurrogates.di -import android.webkit.CookieManager +import com.duckduckgo.app.analyticsSurrogates.AnalyticsSurrogates +import com.duckduckgo.app.analyticsSurrogates.AnalyticsSurrogatesImpl import dagger.Module import dagger.Provides + @Module -class BrowserModule { +class AnalyticsSurrogatesModule { @Provides - fun cookieManager(): CookieManager = CookieManager.getInstance() + fun analyticsSurrogates() : AnalyticsSurrogates = AnalyticsSurrogatesImpl() } \ No newline at end of file diff --git a/app/src/main/java/com/duckduckgo/app/browser/BrowserWebViewClient.kt b/app/src/main/java/com/duckduckgo/app/browser/BrowserWebViewClient.kt index 8d29b4baf94e..a77a68907886 100644 --- a/app/src/main/java/com/duckduckgo/app/browser/BrowserWebViewClient.kt +++ b/app/src/main/java/com/duckduckgo/app/browser/BrowserWebViewClient.kt @@ -31,7 +31,7 @@ import javax.inject.Inject class BrowserWebViewClient @Inject constructor( - private val requestRewriter: DuckDuckGoRequestRewriter, + private val requestRewriter: RequestRewriter, private var trackerDetector: TrackerDetector, private var httpsUpgrader: HttpsUpgrader, private val specialUrlDetector: SpecialUrlDetector, diff --git a/app/src/main/java/com/duckduckgo/app/browser/DuckDuckGoRequestRewriter.kt b/app/src/main/java/com/duckduckgo/app/browser/DuckDuckGoRequestRewriter.kt index bc8e4db264c9..8b31e140e07f 100644 --- a/app/src/main/java/com/duckduckgo/app/browser/DuckDuckGoRequestRewriter.kt +++ b/app/src/main/java/com/duckduckgo/app/browser/DuckDuckGoRequestRewriter.kt @@ -18,9 +18,14 @@ package com.duckduckgo.app.browser import android.net.Uri import timber.log.Timber -import javax.inject.Inject -class DuckDuckGoRequestRewriter @Inject constructor(private val duckDuckGoUrlDetector: DuckDuckGoUrlDetector) { +interface RequestRewriter { + fun shouldRewriteRequest(uri: Uri): Boolean + fun rewriteRequestWithCustomQueryParams(request: Uri): Uri + fun addCustomQueryParams(builder: Uri.Builder) +} + +class DuckDuckGoRequestRewriter(private val duckDuckGoUrlDetector: DuckDuckGoUrlDetector) : RequestRewriter { companion object { private const val sourceParam = "t" @@ -28,7 +33,7 @@ class DuckDuckGoRequestRewriter @Inject constructor(private val duckDuckGoUrlDet private const val querySource = "ddg_android" } - fun rewriteRequestWithCustomQueryParams(request: Uri): Uri { + override fun rewriteRequestWithCustomQueryParams(request: Uri): Uri { val builder = Uri.Builder() .authority(request.authority) .scheme(request.scheme) @@ -46,10 +51,12 @@ class DuckDuckGoRequestRewriter @Inject constructor(private val duckDuckGoUrlDet return newUri } - fun shouldRewriteRequest(uri: Uri): Boolean = - duckDuckGoUrlDetector.isDuckDuckGoUrl(uri) && !uri.queryParameterNames.containsAll(arrayListOf(sourceParam, appVersionParam)) + override fun shouldRewriteRequest(uri: Uri): Boolean { + return duckDuckGoUrlDetector.isDuckDuckGoUrl(uri) && + !uri.queryParameterNames.containsAll(arrayListOf(sourceParam, appVersionParam)) + } - fun addCustomQueryParams(builder: Uri.Builder) { + override fun addCustomQueryParams(builder: Uri.Builder) { builder.appendQueryParameter(appVersionParam, formatAppVersion()) builder.appendQueryParameter(sourceParam, querySource) } diff --git a/app/src/main/java/com/duckduckgo/app/browser/di/BrowserModule.kt b/app/src/main/java/com/duckduckgo/app/browser/di/BrowserModule.kt new file mode 100644 index 000000000000..349d2308249f --- /dev/null +++ b/app/src/main/java/com/duckduckgo/app/browser/di/BrowserModule.kt @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2018 DuckDuckGo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.duckduckgo.app.browser.di + +import android.webkit.CookieManager +import com.duckduckgo.app.browser.DuckDuckGoRequestRewriter +import com.duckduckgo.app.browser.DuckDuckGoUrlDetector +import com.duckduckgo.app.browser.RequestRewriter +import dagger.Module +import dagger.Provides + +@Module +class BrowserModule { + + @Provides + fun cookieManager(): CookieManager = CookieManager.getInstance() + + @Provides + fun duckDuckGoRequestRewriter(urlDetector: DuckDuckGoUrlDetector): RequestRewriter { + return DuckDuckGoRequestRewriter(urlDetector) + } +} \ No newline at end of file diff --git a/app/src/main/java/com/duckduckgo/app/browser/omnibar/QueryUrlConverter.kt b/app/src/main/java/com/duckduckgo/app/browser/omnibar/QueryUrlConverter.kt index 04be914a2fac..bf8c5e8278cb 100644 --- a/app/src/main/java/com/duckduckgo/app/browser/omnibar/QueryUrlConverter.kt +++ b/app/src/main/java/com/duckduckgo/app/browser/omnibar/QueryUrlConverter.kt @@ -18,13 +18,13 @@ package com.duckduckgo.app.browser.omnibar import android.net.Uri import android.support.v4.util.PatternsCompat -import com.duckduckgo.app.browser.DuckDuckGoRequestRewriter +import com.duckduckgo.app.browser.RequestRewriter import com.duckduckgo.app.global.UrlScheme.Companion.http import com.duckduckgo.app.global.UrlScheme.Companion.https import com.duckduckgo.app.global.withScheme import javax.inject.Inject -class QueryUrlConverter @Inject constructor(private val requestRewriter: DuckDuckGoRequestRewriter) : OmnibarEntryConverter { +class QueryUrlConverter @Inject constructor(private val requestRewriter: RequestRewriter) : OmnibarEntryConverter { companion object { private const val baseUrl = "duckduckgo.com" diff --git a/app/src/main/java/com/duckduckgo/app/di/AppComponent.kt b/app/src/main/java/com/duckduckgo/app/di/AppComponent.kt index b451e1e53c04..b9c63d047860 100644 --- a/app/src/main/java/com/duckduckgo/app/di/AppComponent.kt +++ b/app/src/main/java/com/duckduckgo/app/di/AppComponent.kt @@ -18,8 +18,12 @@ package com.duckduckgo.app.di import android.app.Application +import com.duckduckgo.app.analyticsSurrogates.di.AnalyticsSurrogatesModule import com.duckduckgo.app.browser.autoComplete.BrowserAutoCompleteModule +import com.duckduckgo.app.browser.di.BrowserModule import com.duckduckgo.app.global.DuckDuckGoApplication +import com.duckduckgo.app.httpsupgrade.di.HttpsUpgraderModule +import com.duckduckgo.app.trackerdetection.di.TrackerDetectionModule import dagger.BindsInstance import dagger.Component import dagger.android.AndroidInjector @@ -39,7 +43,10 @@ import javax.inject.Singleton (JsonModule::class), (StringModule::class), (BrowserModule::class), - (BrowserAutoCompleteModule::class) + (BrowserAutoCompleteModule::class), + (HttpsUpgraderModule::class), + (AnalyticsSurrogatesModule::class), + (TrackerDetectionModule::class) ]) interface AppComponent : AndroidInjector { diff --git a/app/src/main/java/com/duckduckgo/app/httpsupgrade/HttpsUpgrader.kt b/app/src/main/java/com/duckduckgo/app/httpsupgrade/HttpsUpgrader.kt index b4ee24636b6e..6afa567039ed 100644 --- a/app/src/main/java/com/duckduckgo/app/httpsupgrade/HttpsUpgrader.kt +++ b/app/src/main/java/com/duckduckgo/app/httpsupgrade/HttpsUpgrader.kt @@ -22,12 +22,21 @@ import com.duckduckgo.app.global.UrlScheme import com.duckduckgo.app.global.isHttps import com.duckduckgo.app.httpsupgrade.db.HttpsUpgradeDomainDao import timber.log.Timber -import javax.inject.Inject -class HttpsUpgrader @Inject constructor(private val dao: HttpsUpgradeDomainDao) { +interface HttpsUpgrader { @WorkerThread - fun shouldUpgrade(uri: Uri) : Boolean { + fun shouldUpgrade(uri: Uri) : Boolean + + fun upgrade(uri: Uri): Uri { + return uri.buildUpon().scheme(UrlScheme.https).build() + } +} + +class HttpsUpgraderImpl constructor(private val dao: HttpsUpgradeDomainDao) :HttpsUpgrader { + + @WorkerThread + override fun shouldUpgrade(uri: Uri) : Boolean { if (uri.isHttps) { return false } @@ -36,10 +45,6 @@ class HttpsUpgrader @Inject constructor(private val dao: HttpsUpgradeDomainDao) return dao.hasDomain(host) || matchesWildcard(host) } - fun upgrade(uri: Uri): Uri { - return uri.buildUpon().scheme(UrlScheme.https).build() - } - private fun matchesWildcard(host: String): Boolean { val domains = mutableListOf() for (part in host.split(".").reversed()) { diff --git a/app/src/main/java/com/duckduckgo/app/httpsupgrade/di/HttpsUpgraderModule.kt b/app/src/main/java/com/duckduckgo/app/httpsupgrade/di/HttpsUpgraderModule.kt new file mode 100644 index 000000000000..8afee05bdff1 --- /dev/null +++ b/app/src/main/java/com/duckduckgo/app/httpsupgrade/di/HttpsUpgraderModule.kt @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2018 DuckDuckGo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.duckduckgo.app.httpsupgrade.di + +import com.duckduckgo.app.httpsupgrade.HttpsUpgrader +import com.duckduckgo.app.httpsupgrade.HttpsUpgraderImpl +import com.duckduckgo.app.httpsupgrade.db.HttpsUpgradeDomainDao +import dagger.Module +import dagger.Provides + +@Module +class HttpsUpgraderModule { + + @Provides + fun httpsUpgrader(dao: HttpsUpgradeDomainDao): HttpsUpgrader { + return HttpsUpgraderImpl(dao) + } +} \ No newline at end of file diff --git a/app/src/main/java/com/duckduckgo/app/trackerdetection/TrackerDetector.kt b/app/src/main/java/com/duckduckgo/app/trackerdetection/TrackerDetector.kt index 080eb179a914..850bce14974d 100644 --- a/app/src/main/java/com/duckduckgo/app/trackerdetection/TrackerDetector.kt +++ b/app/src/main/java/com/duckduckgo/app/trackerdetection/TrackerDetector.kt @@ -23,23 +23,27 @@ import com.duckduckgo.app.trackerdetection.model.TrackerNetworks import com.duckduckgo.app.trackerdetection.model.TrackingEvent import timber.log.Timber import java.util.concurrent.CopyOnWriteArrayList -import javax.inject.Inject -import javax.inject.Singleton -@Singleton -class TrackerDetector @Inject constructor(private val networkTrackers: TrackerNetworks, private val settings: PrivacySettingsStore) { +interface TrackerDetector { + fun addClient(client: Client) + fun evaluate(url: String, documentUrl: String, resourceType: ResourceType): TrackingEvent? +} + +class TrackerDetectorImpl ( + private val networkTrackers: TrackerNetworks, + private val settings: PrivacySettingsStore) :TrackerDetector { private val clients = CopyOnWriteArrayList() /** * Adds a new client. If the client's name matches an existing client, old client is replaced */ - fun addClient(client: Client) { + override fun addClient(client: Client) { clients.removeAll { client.name == it.name } clients.add(client) } - fun evaluate(url: String, documentUrl: String, resourceType: ResourceType): TrackingEvent? { + override fun evaluate(url: String, documentUrl: String, resourceType: ResourceType): TrackingEvent? { val whitelisted = clients.any { it.name.type == Client.ClientType.WHITELIST && it.matches(url, documentUrl, resourceType) } if (whitelisted) { diff --git a/app/src/main/java/com/duckduckgo/app/trackerdetection/di/TrackerDetectionModule.kt b/app/src/main/java/com/duckduckgo/app/trackerdetection/di/TrackerDetectionModule.kt new file mode 100644 index 000000000000..bca1bd77ecab --- /dev/null +++ b/app/src/main/java/com/duckduckgo/app/trackerdetection/di/TrackerDetectionModule.kt @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2018 DuckDuckGo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.duckduckgo.app.trackerdetection.di + +import com.duckduckgo.app.privacymonitor.store.PrivacySettingsStore +import com.duckduckgo.app.trackerdetection.TrackerDetector +import com.duckduckgo.app.trackerdetection.TrackerDetectorImpl +import com.duckduckgo.app.trackerdetection.model.TrackerNetworks +import dagger.Module +import dagger.Provides +import javax.inject.Singleton + + +@Module +class TrackerDetectionModule { + + @Provides + @Singleton + fun trackerDetector(networkTrackers: TrackerNetworks, settings: PrivacySettingsStore): TrackerDetector { + return TrackerDetectorImpl(networkTrackers, settings) + } +} \ No newline at end of file From 79527dba71f68501d86a618b0c57eada06b9392d Mon Sep 17 00:00:00 2001 From: Craig Russell Date: Wed, 31 Jan 2018 14:26:30 +0000 Subject: [PATCH 05/12] Add more tests for the "should intercept request" logic --- .../browser/WebViewRequestInterceptorTest.kt | 317 ++++++++++++++---- .../app/browser/WebViewRequestInterceptor.kt | 11 + 2 files changed, 267 insertions(+), 61 deletions(-) diff --git a/app/src/androidTest/java/com/duckduckgo/app/browser/WebViewRequestInterceptorTest.kt b/app/src/androidTest/java/com/duckduckgo/app/browser/WebViewRequestInterceptorTest.kt index bf6f89fb9110..df5fbe84aa70 100644 --- a/app/src/androidTest/java/com/duckduckgo/app/browser/WebViewRequestInterceptorTest.kt +++ b/app/src/androidTest/java/com/duckduckgo/app/browser/WebViewRequestInterceptorTest.kt @@ -20,25 +20,33 @@ import android.net.Uri import android.support.test.InstrumentationRegistry import android.support.test.annotation.UiThreadTest import android.webkit.WebResourceRequest +import android.webkit.WebResourceResponse import android.webkit.WebView import com.duckduckgo.app.analyticsSurrogates.AnalyticsSurrogates import com.duckduckgo.app.analyticsSurrogates.SurrogateResponse import com.duckduckgo.app.httpsupgrade.HttpsUpgrader -import com.duckduckgo.app.trackerdetection.Client import com.duckduckgo.app.trackerdetection.TrackerDetector -import com.duckduckgo.app.trackerdetection.model.ResourceType import com.duckduckgo.app.trackerdetection.model.TrackingEvent +import com.nhaarman.mockito_kotlin.* +import org.junit.Assert.* import org.junit.Before +import org.junit.Test +import org.mockito.ArgumentMatchers.anyString +import org.mockito.Mock import org.mockito.MockitoAnnotations class WebViewRequestInterceptorTest { private lateinit var testee: WebViewRequestInterceptor - private lateinit var stubRequestRewriter: RequestRewriter - private lateinit var stubTrackDetector: TrackerDetector - private lateinit var stubHttpsUpgrader: HttpsUpgrader - private lateinit var stubAnalyticsSurrogates: AnalyticsSurrogates + @Mock + private lateinit var mockTrackerDetector: TrackerDetector + @Mock + private lateinit var mockHttpsUpgrader: HttpsUpgrader + @Mock + private lateinit var mockAnalyticsSurrogates: AnalyticsSurrogates + @Mock + private lateinit var mockRequest: WebResourceRequest private lateinit var webView: WebView @@ -47,52 +55,10 @@ class WebViewRequestInterceptorTest { fun setup() { MockitoAnnotations.initMocks(this) - stubTrackDetector = object: TrackerDetector { - private val clients: MutableList = mutableListOf() - - override fun addClient(client: Client) { - clients.add(client) - } - - override fun evaluate(url: String, documentUrl: String, resourceType: ResourceType): TrackingEvent? { - return null - } - } - - stubHttpsUpgrader = object: HttpsUpgrader { - override fun shouldUpgrade(uri: Uri): Boolean { - return false - } - } - - stubAnalyticsSurrogates = object: AnalyticsSurrogates { - override fun loadSurrogates(urls: List) { - // do nothing - } - - override fun get(uri: Uri): SurrogateResponse { - return if(uri.toString().contains("good")) { - SurrogateResponse(name = "foo.com", mimeType = "application/javascript", jsFunction = "") - } else { - SurrogateResponse(responseAvailable = false) - } - } - - override fun getAll(): List { - return emptyList() - } - } - - stubRequestRewriter = object: RequestRewriter{ - override fun addCustomQueryParams(builder: Uri.Builder) {} - override fun shouldRewriteRequest(uri: Uri): Boolean = true - override fun rewriteRequestWithCustomQueryParams(request: Uri): Uri = request - } - testee = WebViewRequestInterceptor( - trackerDetector = stubTrackDetector, - httpsUpgrader = stubHttpsUpgrader, - analyticsSurrogates = stubAnalyticsSurrogates + trackerDetector = mockTrackerDetector, + httpsUpgrader = mockHttpsUpgrader, + analyticsSurrogates = mockAnalyticsSurrogates ) val context = InstrumentationRegistry.getTargetContext() @@ -100,21 +66,250 @@ class WebViewRequestInterceptorTest { webView = WebView(context) } - private fun getRequest(url: String, forMainFrame: Boolean): WebResourceRequest { - return object : WebResourceRequest { + @Test + fun whenUrlShouldBeUpgradedThenUpgraderInvoked() { + configureShouldUpgrade() + testee.shouldIntercept( + request = mockRequest, + currentUrl = null, + webView = webView, + webViewClientListener = null) + + verify(mockHttpsUpgrader).upgrade(any()) + } + + @Test + fun whenUrlShouldBeUpgradedThenCancelledResponseReturned() { + configureShouldUpgrade() + val response = testee.shouldIntercept( + request = mockRequest, + currentUrl = null, + webView = webView, + webViewClientListener = null) + + assertCancelledResponse(response) + } + + @Test + fun whenUrlShouldBeUpgradedButNotOnMainFrameThenNotUpgraded() { + configureShouldUpgrade() + whenever(mockRequest.isForMainFrame).thenReturn(false) + testee.shouldIntercept( + request = mockRequest, + currentUrl = null, + webView = webView, + webViewClientListener = null) + + verify(mockHttpsUpgrader, never()).upgrade(any()) + } + + @Test + fun whenUrlShouldBeUpgradedButUrlIsNullThenNotUpgraded() { + configureShouldUpgrade() + whenever(mockRequest.url).thenReturn(null) + testee.shouldIntercept( + request = mockRequest, + currentUrl = null, + webView = webView, + webViewClientListener = null) + + verify(mockHttpsUpgrader, never()).upgrade(any()) + } + + @Test + fun whenUrlShouldNotBeUpgradedThenUpgraderNotInvoked() { + whenever(mockHttpsUpgrader.shouldUpgrade(any())).thenReturn(false) + testee.shouldIntercept( + request = mockRequest, + currentUrl = null, + webView = webView, + webViewClientListener = null) + + verify(mockHttpsUpgrader, never()).upgrade(any()) + } + + @Test + fun whenCurrentUrlIsNullThenShouldContinueToLoad() { + configureShouldNotUpgrade() + val response = testee.shouldIntercept( + request = mockRequest, + currentUrl = null, + webView = webView, + webViewClientListener = null) + assertRequestCanContinueToLoad(response) + } + + @Test + fun whenIsTrustedSite_DuckDuckGo_ThenShouldContinueToLoad() { + configureShouldNotUpgrade() + val response = testee.shouldIntercept( + request = mockRequest, + currentUrl = "duckduckgo.com/a/b/c?q=123", + webView = webView, + webViewClientListener = null) + + assertRequestCanContinueToLoad(response) + } + + @Test + fun whenIsTrustedSite_DontTrack_ThenShouldContinueToLoad() { + configureShouldNotUpgrade() + val response = testee.shouldIntercept( + request = mockRequest, + currentUrl = "donttrack.us/a/b/c?q=123", + webView = webView, + webViewClientListener = null) + + assertRequestCanContinueToLoad(response) + } + + @Test + fun whenIsTrustedSite_SpreadPrivacy_ThenShouldContinueToLoad() { + configureShouldNotUpgrade() + val response = testee.shouldIntercept( + request = mockRequest, + currentUrl = "spreadprivacy.com/a/b/c?q=123", + webView = webView, + webViewClientListener = null) + + assertRequestCanContinueToLoad(response) + } + + @Test + fun whenIsTrustedSite_DuckDuckHack_ThenShouldContinueToLoad() { + configureShouldNotUpgrade() + val response = testee.shouldIntercept( + request = mockRequest, + currentUrl = "duckduckhack.com/a/b/c?q=123", + webView = webView, + webViewClientListener = null) + + assertRequestCanContinueToLoad(response) + } + + @Test + fun whenIsTrustedSite_PrivateBrowsingMyths_ThenShouldContinueToLoad() { + configureShouldNotUpgrade() + val response = testee.shouldIntercept( + request = mockRequest, + currentUrl = "privatebrowsingmyths.com/a/b/c?q=123", + webView = webView, + webViewClientListener = null) + + assertRequestCanContinueToLoad(response) + } + + @Test + fun whenIsTrustedSite_DuckDotCo_ThenShouldContinueToLoad() { + configureShouldNotUpgrade() + val response = testee.shouldIntercept( + request = mockRequest, + currentUrl = "duck.co/a/b/c?q=123", + webView = webView, + webViewClientListener = null) + + assertRequestCanContinueToLoad(response) + } + + @Test + fun whenIsHttpRequestThenHttpRequestListenerCalled() { + configureShouldNotUpgrade() + whenever(mockRequest.url).thenReturn(Uri.parse("http://example.com")) + val mockListener = mock() + + val response = testee.shouldIntercept( + request = mockRequest, + currentUrl = "foo.com", + webView = webView, + webViewClientListener = mockListener) + + verify(mockListener).pageHasHttpResources(anyString()) + assertRequestCanContinueToLoad(response) + } + + @Test + fun whenIsHttpsRequestThenHttpRequestListenerNotCalled() { + configureShouldNotUpgrade() + whenever(mockRequest.url).thenReturn(Uri.parse("https://example.com")) + val mockListener = mock() + + val response = testee.shouldIntercept( + request = mockRequest, + currentUrl = "foo.com", + webView = webView, + webViewClientListener = mockListener) + + verify(mockListener, never()).pageHasHttpResources(anyString()) + assertRequestCanContinueToLoad(response) + } + + + @Test + fun whenRequestShouldBlockAndNoSurrogateThenCancellingResponseReturned() { + whenever(mockAnalyticsSurrogates.get(any())).thenReturn(SurrogateResponse(responseAvailable = false)) + + configureShouldNotUpgrade() + configureShouldBlock() + val response = testee.shouldIntercept( + request = mockRequest, + currentUrl = "foo.com", + webView = webView, + webViewClientListener = null) + + assertCancelledResponse(response) + } + + @Test + fun whenRequestShouldBlockButThereIsASurrogateThen() { + val availableSurrogate = SurrogateResponse( + responseAvailable = true, + mimeType = "application/javascript", + jsFunction = "javascript replacement function goes here") + whenever(mockAnalyticsSurrogates.get(any())).thenReturn(availableSurrogate) - override fun isRedirect(): Boolean = false + configureShouldNotUpgrade() + configureShouldBlock() + val response = testee.shouldIntercept( + request = mockRequest, + currentUrl = "foo.com", + webView = webView, + webViewClientListener = null) - override fun getMethod(): String = "GET" + assertEquals(availableSurrogate.jsFunction.byteInputStream().read(), response!!.data.read()) + } - override fun getRequestHeaders(): MutableMap = mutableMapOf() + private fun assertRequestCanContinueToLoad(response: WebResourceResponse?) { + assertNull(response) + } - override fun hasGesture(): Boolean = false + private fun configureShouldBlock() { + val blockTrackingEvent = TrackingEvent(blocked = true, + documentUrl = "", + trackerUrl = "", + trackerNetwork = null) + whenever(mockRequest.isForMainFrame).thenReturn(false) + whenever(mockTrackerDetector.evaluate(any(), any(), any())).thenReturn(blockTrackingEvent) + } - override fun isForMainFrame(): Boolean = forMainFrame + private fun configureShouldUpgrade() { + whenever(mockHttpsUpgrader.shouldUpgrade(any())).thenReturn(true) + whenever(mockRequest.url).thenReturn(validUri()) + whenever(mockRequest.isForMainFrame).thenReturn(true) + } - override fun getUrl(): Uri = Uri.parse(url) + private fun configureShouldNotUpgrade() { + whenever(mockHttpsUpgrader.shouldUpgrade(any())).thenReturn(false) + whenever(mockRequest.url).thenReturn(validUri()) + whenever(mockRequest.isForMainFrame).thenReturn(true) + } + + private fun validUri() = Uri.parse("example.com") - } + private fun assertCancelledResponse(response: WebResourceResponse?) { + assertNotNull(response) + assertNull(response!!.data) + assertNull(response.mimeType) + assertNull(response.encoding) } -} \ No newline at end of file + +} diff --git a/app/src/main/java/com/duckduckgo/app/browser/WebViewRequestInterceptor.kt b/app/src/main/java/com/duckduckgo/app/browser/WebViewRequestInterceptor.kt index 3e496612422c..a6e81846f0ce 100644 --- a/app/src/main/java/com/duckduckgo/app/browser/WebViewRequestInterceptor.kt +++ b/app/src/main/java/com/duckduckgo/app/browser/WebViewRequestInterceptor.kt @@ -16,6 +16,7 @@ package com.duckduckgo.app.browser +import android.support.annotation.WorkerThread import android.webkit.WebResourceRequest import android.webkit.WebResourceResponse import android.webkit.WebView @@ -35,6 +36,16 @@ class WebViewRequestInterceptor @Inject constructor( private val httpsUpgrader: HttpsUpgrader ) { + /** + * Notify the application of a resource request and allow the application to return the data. + * + * If the return value is null, the WebView will continue to load the resource as usual. + * Otherwise, the return response and data will be used. + * + * NOTE: This method is called on a thread other than the UI thread so clients should exercise + * caution when accessing private data or the view system. + */ + @WorkerThread fun shouldIntercept( request: WebResourceRequest, webView: WebView, From 5b457494b5c9dea7a198cae0374f0fd0ed2e25b2 Mon Sep 17 00:00:00 2001 From: Craig Russell Date: Wed, 31 Jan 2018 15:29:35 +0000 Subject: [PATCH 06/12] Lowering noise of log statement --- .../app/analyticsSurrogates/AnalyticsSurrogatesLoader.kt | 2 +- .../com/duckduckgo/app/browser/WebViewRequestInterceptor.kt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/src/main/java/com/duckduckgo/app/analyticsSurrogates/AnalyticsSurrogatesLoader.kt b/app/src/main/java/com/duckduckgo/app/analyticsSurrogates/AnalyticsSurrogatesLoader.kt index 080e5c643ba8..fccdb42e97e3 100644 --- a/app/src/main/java/com/duckduckgo/app/analyticsSurrogates/AnalyticsSurrogatesLoader.kt +++ b/app/src/main/java/com/duckduckgo/app/analyticsSurrogates/AnalyticsSurrogatesLoader.kt @@ -85,7 +85,7 @@ class AnalyticsSurrogatesLoader @Inject constructor( functionBuilder.append("\n") } - Timber.i("Processed %d surrogates", surrogates.size) + Timber.d("Processed %d surrogates", surrogates.size) return surrogates } } \ No newline at end of file diff --git a/app/src/main/java/com/duckduckgo/app/browser/WebViewRequestInterceptor.kt b/app/src/main/java/com/duckduckgo/app/browser/WebViewRequestInterceptor.kt index a6e81846f0ce..842ae3e86727 100644 --- a/app/src/main/java/com/duckduckgo/app/browser/WebViewRequestInterceptor.kt +++ b/app/src/main/java/com/duckduckgo/app/browser/WebViewRequestInterceptor.kt @@ -74,7 +74,7 @@ class WebViewRequestInterceptor @Inject constructor( val surrogate = analyticsSurrogates.get(url) if (surrogate.responseAvailable) { - Timber.i("Surrogate found for %s", url) + Timber.v("Surrogate found for %s", url) return WebResourceResponse( surrogate.mimeType, "UTF-8", From 2f73d3b6c8ab3f64c390a5db74967b739a6cdbe4 Mon Sep 17 00:00:00 2001 From: Craig Russell Date: Wed, 31 Jan 2018 15:30:16 +0000 Subject: [PATCH 07/12] Mark AnalyticsSurrogates as a singleton --- .../app/analyticsSurrogates/di/AnalyticsSurrogatesModule.kt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/src/main/java/com/duckduckgo/app/analyticsSurrogates/di/AnalyticsSurrogatesModule.kt b/app/src/main/java/com/duckduckgo/app/analyticsSurrogates/di/AnalyticsSurrogatesModule.kt index 4696848c6d46..d7c5e5298252 100644 --- a/app/src/main/java/com/duckduckgo/app/analyticsSurrogates/di/AnalyticsSurrogatesModule.kt +++ b/app/src/main/java/com/duckduckgo/app/analyticsSurrogates/di/AnalyticsSurrogatesModule.kt @@ -20,11 +20,13 @@ import com.duckduckgo.app.analyticsSurrogates.AnalyticsSurrogates import com.duckduckgo.app.analyticsSurrogates.AnalyticsSurrogatesImpl import dagger.Module import dagger.Provides +import javax.inject.Singleton @Module class AnalyticsSurrogatesModule { @Provides + @Singleton fun analyticsSurrogates() : AnalyticsSurrogates = AnalyticsSurrogatesImpl() } \ No newline at end of file From bbdf3d3af6136c68698d8a67be76bdc0b62d6697 Mon Sep 17 00:00:00 2001 From: Craig Russell Date: Thu, 1 Feb 2018 11:22:24 +0000 Subject: [PATCH 08/12] Clear the string builder between surrogates --- .../app/analyticsSurrogates/AnalyticsSurrogatesLoader.kt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/src/main/java/com/duckduckgo/app/analyticsSurrogates/AnalyticsSurrogatesLoader.kt b/app/src/main/java/com/duckduckgo/app/analyticsSurrogates/AnalyticsSurrogatesLoader.kt index fccdb42e97e3..3b5bdb51ea81 100644 --- a/app/src/main/java/com/duckduckgo/app/analyticsSurrogates/AnalyticsSurrogatesLoader.kt +++ b/app/src/main/java/com/duckduckgo/app/analyticsSurrogates/AnalyticsSurrogatesLoader.kt @@ -75,8 +75,10 @@ class AnalyticsSurrogatesLoader @Inject constructor( mimeType = mimeType, jsFunction = functionBuilder.toString() ) - ) + + functionBuilder.setLength(0) + nextLineIsNewRule = true return@forEach } From 6b1aaeabbc44c8a7671b3d971171301645491404 Mon Sep 17 00:00:00 2001 From: Craig Russell Date: Thu, 1 Feb 2018 11:23:00 +0000 Subject: [PATCH 09/12] Update surrogate test files to be just test functions; not the real thing --- .../AnalyticsSurrogatesLoaderTest.kt | 9 +- .../resources/binary/surrogates/surrogates_1 | 98 +----- .../resources/binary/surrogates/surrogates_6 | 330 ++---------------- .../surrogates_no_empty_line_at_end_of_file | 330 ++---------------- .../surrogates_with_different_mime_types | 192 +++------- .../surrogates_with_empty_line_at_end_of_file | 330 ++---------------- 6 files changed, 119 insertions(+), 1170 deletions(-) diff --git a/app/src/androidTest/java/com/duckduckgo/app/analyticsSurrogates/AnalyticsSurrogatesLoaderTest.kt b/app/src/androidTest/java/com/duckduckgo/app/analyticsSurrogates/AnalyticsSurrogatesLoaderTest.kt index f2a7874b3124..575ba813e711 100644 --- a/app/src/androidTest/java/com/duckduckgo/app/analyticsSurrogates/AnalyticsSurrogatesLoaderTest.kt +++ b/app/src/androidTest/java/com/duckduckgo/app/analyticsSurrogates/AnalyticsSurrogatesLoaderTest.kt @@ -83,7 +83,14 @@ class AnalyticsSurrogatesLoaderTest { fun whenLoadingSurrogateThenFunctionLengthIsPreserved() { val surrogates = initialiseFile("surrogates_6") val actualNumberOfLines = surrogates[0].jsFunction.reader().readLines().size - assertEquals(99, actualNumberOfLines) + assertEquals(3, actualNumberOfLines) + } + + @Test + fun whenLoadingSurrogateThenFunctionLengthIsPreservedJavascriptCommentsArePreserved() { + val surrogates = initialiseFile("surrogates_6") + val actualNumberOfLines = surrogates[1].jsFunction.reader().readLines().size + assertEquals(5, actualNumberOfLines) } private fun initialiseFile(filename: String) : List { diff --git a/app/src/androidTest/resources/binary/surrogates/surrogates_1 b/app/src/androidTest/resources/binary/surrogates/surrogates_1 index 3c8cb2f442aa..a62db5c27b98 100644 --- a/app/src/androidTest/resources/binary/surrogates/surrogates_1 +++ b/app/src/androidTest/resources/binary/surrogates/surrogates_1 @@ -7,101 +7,5 @@ # - https://developers.google.com/analytics/devguides/collection/gajs/ google-analytics.com/ga.js application/javascript (function() { - var noopfn = function() { - ; - }; - // - var Gaq = function() { - ; - }; - Gaq.prototype.Na = noopfn; - Gaq.prototype.O = noopfn; - Gaq.prototype.Sa = noopfn; - Gaq.prototype.Ta = noopfn; - Gaq.prototype.Va = noopfn; - Gaq.prototype._createAsyncTracker = noopfn; - Gaq.prototype._getAsyncTracker = noopfn; - Gaq.prototype._getPlugin = noopfn; - Gaq.prototype.push = function(a) { - if ( typeof a === 'function' ) { - a(); return; - } - if ( Array.isArray(a) === false ) { - return; - } - // https://twitter.com/catovitch/status/776442930345218048 - // https://developers.google.com/analytics/devguides/collection/gajs/methods/gaJSApiDomainDirectory#_gat.GA_Tracker_._link - if ( a[0] === '_link' && typeof a[1] === 'string' ) { - window.location.assign(a[1]); - } - // https://github.com/gorhill/uBlock/issues/2162 - if ( a[0] === '_set' && a[1] === 'hitCallback' && typeof a[2] === 'function' ) { - a[2](); - } - }; - // - var tracker = (function() { - var out = {}; - var api = [ - '_addIgnoredOrganic _addIgnoredRef _addItem _addOrganic', - '_addTrans _clearIgnoredOrganic _clearIgnoredRef _clearOrganic', - '_cookiePathCopy _deleteCustomVar _getName _setAccount', - '_getAccount _getClientInfo _getDetectFlash _getDetectTitle', - '_getLinkerUrl _getLocalGifPath _getServiceMode _getVersion', - '_getVisitorCustomVar _initData _link _linkByPost', - '_setAllowAnchor _setAllowHash _setAllowLinker _setCampContentKey', - '_setCampMediumKey _setCampNameKey _setCampNOKey _setCampSourceKey', - '_setCampTermKey _setCampaignCookieTimeout _setCampaignTrack _setClientInfo', - '_setCookiePath _setCookiePersistence _setCookieTimeout _setCustomVar', - '_setDetectFlash _setDetectTitle _setDomainName _setLocalGifPath', - '_setLocalRemoteServerMode _setLocalServerMode _setReferrerOverride _setRemoteServerMode', - '_setSampleRate _setSessionTimeout _setSiteSpeedSampleRate _setSessionCookieTimeout', - '_setVar _setVisitorCookieTimeout _trackEvent _trackPageLoadTime', - '_trackPageview _trackSocial _trackTiming _trackTrans', - '_visitCode' - ].join(' ').split(/\s+/); - var i = api.length; - while ( i-- ) { - out[api[i]] = noopfn; - } - out._getLinkerUrl = function(a) { - return a; - }; - return out; - })(); - // - var Gat = function() { - ; - }; - Gat.prototype._anonymizeIP = noopfn; - Gat.prototype._createTracker = noopfn; - Gat.prototype._forceSSL = noopfn; - Gat.prototype._getPlugin = noopfn; - Gat.prototype._getTracker = function() { - return tracker; - }; - Gat.prototype._getTrackerByName = function() { - return tracker; - }; - Gat.prototype._getTrackers = noopfn; - Gat.prototype.aa = noopfn; - Gat.prototype.ab = noopfn; - Gat.prototype.hb = noopfn; - Gat.prototype.la = noopfn; - Gat.prototype.oa = noopfn; - Gat.prototype.pa = noopfn; - Gat.prototype.u = noopfn; - var gat = new Gat(); - window._gat = gat; - // - var gaq = new Gaq(); - (function() { - var aa = window._gaq || []; - if ( Array.isArray(aa) ) { - while ( aa[0] ) { - gaq.push(aa.shift()); - } - } - })(); - window._gaq = gaq.qf = gaq; + alert("surrogates_1"); })(); diff --git a/app/src/androidTest/resources/binary/surrogates/surrogates_6 b/app/src/androidTest/resources/binary/surrogates/surrogates_6 index f2b3ecba7f14..b49ab0ef0fb9 100644 --- a/app/src/androidTest/resources/binary/surrogates/surrogates_6 +++ b/app/src/androidTest/resources/binary/surrogates/surrogates_6 @@ -7,180 +7,31 @@ # - https://developers.google.com/analytics/devguides/collection/gajs/ google-analytics.com/ga.js application/javascript (function() { - var noopfn = function() { - ; - }; - // - var Gaq = function() { - ; - }; - Gaq.prototype.Na = noopfn; - Gaq.prototype.O = noopfn; - Gaq.prototype.Sa = noopfn; - Gaq.prototype.Ta = noopfn; - Gaq.prototype.Va = noopfn; - Gaq.prototype._createAsyncTracker = noopfn; - Gaq.prototype._getAsyncTracker = noopfn; - Gaq.prototype._getPlugin = noopfn; - Gaq.prototype.push = function(a) { - if ( typeof a === 'function' ) { - a(); return; - } - if ( Array.isArray(a) === false ) { - return; - } - // https://twitter.com/catovitch/status/776442930345218048 - // https://developers.google.com/analytics/devguides/collection/gajs/methods/gaJSApiDomainDirectory#_gat.GA_Tracker_._link - if ( a[0] === '_link' && typeof a[1] === 'string' ) { - window.location.assign(a[1]); - } - // https://github.com/gorhill/uBlock/issues/2162 - if ( a[0] === '_set' && a[1] === 'hitCallback' && typeof a[2] === 'function' ) { - a[2](); - } - }; - // - var tracker = (function() { - var out = {}; - var api = [ - '_addIgnoredOrganic _addIgnoredRef _addItem _addOrganic', - '_addTrans _clearIgnoredOrganic _clearIgnoredRef _clearOrganic', - '_cookiePathCopy _deleteCustomVar _getName _setAccount', - '_getAccount _getClientInfo _getDetectFlash _getDetectTitle', - '_getLinkerUrl _getLocalGifPath _getServiceMode _getVersion', - '_getVisitorCustomVar _initData _link _linkByPost', - '_setAllowAnchor _setAllowHash _setAllowLinker _setCampContentKey', - '_setCampMediumKey _setCampNameKey _setCampNOKey _setCampSourceKey', - '_setCampTermKey _setCampaignCookieTimeout _setCampaignTrack _setClientInfo', - '_setCookiePath _setCookiePersistence _setCookieTimeout _setCustomVar', - '_setDetectFlash _setDetectTitle _setDomainName _setLocalGifPath', - '_setLocalRemoteServerMode _setLocalServerMode _setReferrerOverride _setRemoteServerMode', - '_setSampleRate _setSessionTimeout _setSiteSpeedSampleRate _setSessionCookieTimeout', - '_setVar _setVisitorCookieTimeout _trackEvent _trackPageLoadTime', - '_trackPageview _trackSocial _trackTiming _trackTrans', - '_visitCode' - ].join(' ').split(/\s+/); - var i = api.length; - while ( i-- ) { - out[api[i]] = noopfn; - } - out._getLinkerUrl = function(a) { - return a; - }; - return out; - })(); - // - var Gat = function() { - ; - }; - Gat.prototype._anonymizeIP = noopfn; - Gat.prototype._createTracker = noopfn; - Gat.prototype._forceSSL = noopfn; - Gat.prototype._getPlugin = noopfn; - Gat.prototype._getTracker = function() { - return tracker; - }; - Gat.prototype._getTrackerByName = function() { - return tracker; - }; - Gat.prototype._getTrackers = noopfn; - Gat.prototype.aa = noopfn; - Gat.prototype.ab = noopfn; - Gat.prototype.hb = noopfn; - Gat.prototype.la = noopfn; - Gat.prototype.oa = noopfn; - Gat.prototype.pa = noopfn; - Gat.prototype.u = noopfn; - var gat = new Gat(); - window._gat = gat; - // - var gaq = new Gaq(); - (function() { - var aa = window._gaq || []; - if ( Array.isArray(aa) ) { - while ( aa[0] ) { - gaq.push(aa.shift()); - } - } - })(); - window._gaq = gaq.qf = gaq; + alert("surrogates_1"); })(); google-analytics.com/analytics.js application/javascript (function() { - // https://developers.google.com/analytics/devguides/collection/analyticsjs/ - var noopfn = function() { - ; - }; - var noopnullfn = function() { - return null; - }; - // - var Tracker = function() { - ; - }; - var p = Tracker.prototype; - p.get = noopfn; - p.set = noopfn; - p.send = noopfn; - // - var w = window, - gaName = w.GoogleAnalyticsObject || 'ga'; - var ga = function() { - var len = arguments.length; - if ( len === 0 ) { - return; - } - var f = arguments[len-1]; - if ( typeof f !== 'object' || f === null || typeof f.hitCallback !== 'function' ) { - return; - } - try { - f.hitCallback(); - } catch (ex) { - } - }; - ga.create = function() { - return new Tracker(); - }; - ga.getByName = noopnullfn; - ga.getAll = function() { - return []; - }; - ga.remove = noopfn; - w[gaName] = ga; - // https://github.com/gorhill/uBlock/issues/3075 - var dl = w.dataLayer; - if ( dl instanceof Object && dl.hide instanceof Object && typeof dl.hide.end === 'function' ) { - dl.hide.end(); - } + // random comment + alert("surrogates_2"); + alert("surrogates_2"); })(); google-analytics.com/inpage_linkid.js application/javascript (function() { - window._gaq = window._gaq || { - push: function() { - ; - } - }; + alert("surrogates_3"); + alert("surrogates_3"); + alert("surrogates_3"); })(); # https://github.com/gorhill/uBlock/issues/2480 # https://developers.google.com/analytics/devguides/collection/gajs/experiments#cxjs google-analytics.com/cx/api.js application/javascript (function() { - var noopfn = function() { - }; - window.cxApi = { - chooseVariation: function() { - return 0; - }, - getChosenVariation: noopfn, - setAllowHash: noopfn, - setChosenVariation: noopfn, - setCookiePath: noopfn, - setDomainName: noopfn - }; + alert("surrogates_4"); + alert("surrogates_4"); + alert("surrogates_4"); + alert("surrogates_4"); })(); # Ubiquitous googletagservices.com: not blocked by EasyPrivacy. @@ -188,135 +39,11 @@ google-analytics.com/cx/api.js application/javascript # visitor behavior googletagservices.com/gpt.js application/javascript (function() { - var p; - // https://developers.google.com/doubleclick-gpt/reference - var noopfn = function() { - ; - }.bind(); - var noopthisfn = function() { - return this; - }; - var noopnullfn = function() { - return null; - }; - var nooparrayfn = function() { - return []; - }; - var noopstrfn = function() { - return ''; - }; - // - var companionAdsService = { - addEventListener: noopthisfn, - enableSyncLoading: noopfn, - setRefreshUnfilledSlots: noopfn - }; - var contentService = { - addEventListener: noopthisfn, - setContent: noopfn - }; - var PassbackSlot = function() { - ; - }; - p = PassbackSlot.prototype; - p.display = noopfn; - p.get = noopnullfn; - p.set = noopthisfn; - p.setClickUrl = noopthisfn; - p.setTagForChildDirectedTreatment = noopthisfn; - p.setTargeting = noopthisfn; - p.updateTargetingFromMap = noopthisfn; - var pubAdsService = { - addEventListener: noopthisfn, - clear: noopfn, - clearCategoryExclusions: noopthisfn, - clearTagForChildDirectedTreatment: noopthisfn, - clearTargeting: noopthisfn, - collapseEmptyDivs: noopfn, - defineOutOfPagePassback: function() { return new PassbackSlot(); }, - definePassback: function() { return new PassbackSlot(); }, - disableInitialLoad: noopfn, - display: noopfn, - enableAsyncRendering: noopfn, - enableSingleRequest: noopfn, - enableSyncRendering: noopfn, - enableVideoAds: noopfn, - get: noopnullfn, - getAttributeKeys: nooparrayfn, - getTargeting: noopfn, - getTargetingKeys: nooparrayfn, - getSlots: nooparrayfn, - refresh: noopfn, - set: noopthisfn, - setCategoryExclusion: noopthisfn, - setCentering: noopfn, - setCookieOptions: noopthisfn, - setForceSafeFrame: noopthisfn, - setLocation: noopthisfn, - setPublisherProvidedId: noopthisfn, - setSafeFrameConfig: noopthisfn, - setTagForChildDirectedTreatment: noopthisfn, - setTargeting: noopthisfn, - setVideoContent: noopthisfn, - updateCorrelator: noopfn - }; - var SizeMappingBuilder = function() { - ; - }; - p = SizeMappingBuilder.prototype; - p.addSize = noopthisfn; - p.build = noopnullfn; - var Slot = function() { - ; - }; - p = Slot.prototype; - p.addService = noopthisfn; - p.clearCategoryExclusions = noopthisfn; - p.clearTargeting = noopthisfn; - p.defineSizeMapping = noopthisfn; - p.get = noopnullfn; - p.getAdUnitPath = nooparrayfn; - p.getAttributeKeys = nooparrayfn; - p.getCategoryExclusions = nooparrayfn; - p.getDomId = noopstrfn; - p.getSlotElementId = noopstrfn; - p.getSlotId = noopthisfn; - p.getTargeting = nooparrayfn; - p.getTargetingKeys = nooparrayfn; - p.set = noopthisfn; - p.setCategoryExclusion = noopthisfn; - p.setClickUrl = noopthisfn; - p.setCollapseEmptyDiv = noopthisfn; - p.setTargeting = noopthisfn; - // - var gpt = window.googletag || {}; - var cmd = gpt.cmd || []; - gpt.apiReady = true; - gpt.cmd = []; - gpt.cmd.push = function(a) { - try { - a(); - } catch (ex) { - } - return 1; - }; - gpt.companionAds = function() { return companionAdsService; }; - gpt.content = function() { return contentService; }; - gpt.defineOutOfPageSlot = function() { return new Slot(); }; - gpt.defineSlot = function() { return new Slot(); }; - gpt.destroySlots = noopfn; - gpt.disablePublisherConsole = noopfn; - gpt.display = noopfn; - gpt.enableServices = noopfn; - gpt.getVersion = noopstrfn; - gpt.pubads = function() { return pubAdsService; }; - gpt.pubadsReady = true; - gpt.setAdIframeTitle = noopfn; - gpt.sizeMapping = function() { return new SizeMappingBuilder(); }; - window.googletag = gpt; - while ( cmd.length !== 0 ) { - gpt.cmd.push(cmd.shift()); - } + alert("surrogates_5"); + alert("surrogates_5"); + alert("surrogates_5"); + alert("surrogates_5"); + alert("surrogates_5"); })(); # Obviously more work needs to be done, but at least for now it takes care of: @@ -326,23 +53,10 @@ googletagservices.com/gpt.js application/javascript # - https://github.com/uBlockOrigin/uAssets/issues/420 googletagmanager.com/gtm.js application/javascript (function() { - var noopfn = function() { - }; - var w = window; - w.ga = w.ga || noopfn; - var dl = w.dataLayer; - if ( dl instanceof Object === false ) { return; } - if ( dl.hide instanceof Object && typeof dl.hide.end === 'function' ) { - dl.hide.end(); - } - if ( typeof dl.push === 'function' ) { - dl.push = function(o) { - if ( - o instanceof Object && - typeof o.eventCallback === 'function' - ) { - setTimeout(o.eventCallback, 1); - } - }; - } + alert("surrogates_6"); + alert("surrogates_6"); + alert("surrogates_6"); + alert("surrogates_6"); + alert("surrogates_6"); + alert("surrogates_6"); })(); \ No newline at end of file diff --git a/app/src/androidTest/resources/binary/surrogates/surrogates_no_empty_line_at_end_of_file b/app/src/androidTest/resources/binary/surrogates/surrogates_no_empty_line_at_end_of_file index f2b3ecba7f14..b49ab0ef0fb9 100644 --- a/app/src/androidTest/resources/binary/surrogates/surrogates_no_empty_line_at_end_of_file +++ b/app/src/androidTest/resources/binary/surrogates/surrogates_no_empty_line_at_end_of_file @@ -7,180 +7,31 @@ # - https://developers.google.com/analytics/devguides/collection/gajs/ google-analytics.com/ga.js application/javascript (function() { - var noopfn = function() { - ; - }; - // - var Gaq = function() { - ; - }; - Gaq.prototype.Na = noopfn; - Gaq.prototype.O = noopfn; - Gaq.prototype.Sa = noopfn; - Gaq.prototype.Ta = noopfn; - Gaq.prototype.Va = noopfn; - Gaq.prototype._createAsyncTracker = noopfn; - Gaq.prototype._getAsyncTracker = noopfn; - Gaq.prototype._getPlugin = noopfn; - Gaq.prototype.push = function(a) { - if ( typeof a === 'function' ) { - a(); return; - } - if ( Array.isArray(a) === false ) { - return; - } - // https://twitter.com/catovitch/status/776442930345218048 - // https://developers.google.com/analytics/devguides/collection/gajs/methods/gaJSApiDomainDirectory#_gat.GA_Tracker_._link - if ( a[0] === '_link' && typeof a[1] === 'string' ) { - window.location.assign(a[1]); - } - // https://github.com/gorhill/uBlock/issues/2162 - if ( a[0] === '_set' && a[1] === 'hitCallback' && typeof a[2] === 'function' ) { - a[2](); - } - }; - // - var tracker = (function() { - var out = {}; - var api = [ - '_addIgnoredOrganic _addIgnoredRef _addItem _addOrganic', - '_addTrans _clearIgnoredOrganic _clearIgnoredRef _clearOrganic', - '_cookiePathCopy _deleteCustomVar _getName _setAccount', - '_getAccount _getClientInfo _getDetectFlash _getDetectTitle', - '_getLinkerUrl _getLocalGifPath _getServiceMode _getVersion', - '_getVisitorCustomVar _initData _link _linkByPost', - '_setAllowAnchor _setAllowHash _setAllowLinker _setCampContentKey', - '_setCampMediumKey _setCampNameKey _setCampNOKey _setCampSourceKey', - '_setCampTermKey _setCampaignCookieTimeout _setCampaignTrack _setClientInfo', - '_setCookiePath _setCookiePersistence _setCookieTimeout _setCustomVar', - '_setDetectFlash _setDetectTitle _setDomainName _setLocalGifPath', - '_setLocalRemoteServerMode _setLocalServerMode _setReferrerOverride _setRemoteServerMode', - '_setSampleRate _setSessionTimeout _setSiteSpeedSampleRate _setSessionCookieTimeout', - '_setVar _setVisitorCookieTimeout _trackEvent _trackPageLoadTime', - '_trackPageview _trackSocial _trackTiming _trackTrans', - '_visitCode' - ].join(' ').split(/\s+/); - var i = api.length; - while ( i-- ) { - out[api[i]] = noopfn; - } - out._getLinkerUrl = function(a) { - return a; - }; - return out; - })(); - // - var Gat = function() { - ; - }; - Gat.prototype._anonymizeIP = noopfn; - Gat.prototype._createTracker = noopfn; - Gat.prototype._forceSSL = noopfn; - Gat.prototype._getPlugin = noopfn; - Gat.prototype._getTracker = function() { - return tracker; - }; - Gat.prototype._getTrackerByName = function() { - return tracker; - }; - Gat.prototype._getTrackers = noopfn; - Gat.prototype.aa = noopfn; - Gat.prototype.ab = noopfn; - Gat.prototype.hb = noopfn; - Gat.prototype.la = noopfn; - Gat.prototype.oa = noopfn; - Gat.prototype.pa = noopfn; - Gat.prototype.u = noopfn; - var gat = new Gat(); - window._gat = gat; - // - var gaq = new Gaq(); - (function() { - var aa = window._gaq || []; - if ( Array.isArray(aa) ) { - while ( aa[0] ) { - gaq.push(aa.shift()); - } - } - })(); - window._gaq = gaq.qf = gaq; + alert("surrogates_1"); })(); google-analytics.com/analytics.js application/javascript (function() { - // https://developers.google.com/analytics/devguides/collection/analyticsjs/ - var noopfn = function() { - ; - }; - var noopnullfn = function() { - return null; - }; - // - var Tracker = function() { - ; - }; - var p = Tracker.prototype; - p.get = noopfn; - p.set = noopfn; - p.send = noopfn; - // - var w = window, - gaName = w.GoogleAnalyticsObject || 'ga'; - var ga = function() { - var len = arguments.length; - if ( len === 0 ) { - return; - } - var f = arguments[len-1]; - if ( typeof f !== 'object' || f === null || typeof f.hitCallback !== 'function' ) { - return; - } - try { - f.hitCallback(); - } catch (ex) { - } - }; - ga.create = function() { - return new Tracker(); - }; - ga.getByName = noopnullfn; - ga.getAll = function() { - return []; - }; - ga.remove = noopfn; - w[gaName] = ga; - // https://github.com/gorhill/uBlock/issues/3075 - var dl = w.dataLayer; - if ( dl instanceof Object && dl.hide instanceof Object && typeof dl.hide.end === 'function' ) { - dl.hide.end(); - } + // random comment + alert("surrogates_2"); + alert("surrogates_2"); })(); google-analytics.com/inpage_linkid.js application/javascript (function() { - window._gaq = window._gaq || { - push: function() { - ; - } - }; + alert("surrogates_3"); + alert("surrogates_3"); + alert("surrogates_3"); })(); # https://github.com/gorhill/uBlock/issues/2480 # https://developers.google.com/analytics/devguides/collection/gajs/experiments#cxjs google-analytics.com/cx/api.js application/javascript (function() { - var noopfn = function() { - }; - window.cxApi = { - chooseVariation: function() { - return 0; - }, - getChosenVariation: noopfn, - setAllowHash: noopfn, - setChosenVariation: noopfn, - setCookiePath: noopfn, - setDomainName: noopfn - }; + alert("surrogates_4"); + alert("surrogates_4"); + alert("surrogates_4"); + alert("surrogates_4"); })(); # Ubiquitous googletagservices.com: not blocked by EasyPrivacy. @@ -188,135 +39,11 @@ google-analytics.com/cx/api.js application/javascript # visitor behavior googletagservices.com/gpt.js application/javascript (function() { - var p; - // https://developers.google.com/doubleclick-gpt/reference - var noopfn = function() { - ; - }.bind(); - var noopthisfn = function() { - return this; - }; - var noopnullfn = function() { - return null; - }; - var nooparrayfn = function() { - return []; - }; - var noopstrfn = function() { - return ''; - }; - // - var companionAdsService = { - addEventListener: noopthisfn, - enableSyncLoading: noopfn, - setRefreshUnfilledSlots: noopfn - }; - var contentService = { - addEventListener: noopthisfn, - setContent: noopfn - }; - var PassbackSlot = function() { - ; - }; - p = PassbackSlot.prototype; - p.display = noopfn; - p.get = noopnullfn; - p.set = noopthisfn; - p.setClickUrl = noopthisfn; - p.setTagForChildDirectedTreatment = noopthisfn; - p.setTargeting = noopthisfn; - p.updateTargetingFromMap = noopthisfn; - var pubAdsService = { - addEventListener: noopthisfn, - clear: noopfn, - clearCategoryExclusions: noopthisfn, - clearTagForChildDirectedTreatment: noopthisfn, - clearTargeting: noopthisfn, - collapseEmptyDivs: noopfn, - defineOutOfPagePassback: function() { return new PassbackSlot(); }, - definePassback: function() { return new PassbackSlot(); }, - disableInitialLoad: noopfn, - display: noopfn, - enableAsyncRendering: noopfn, - enableSingleRequest: noopfn, - enableSyncRendering: noopfn, - enableVideoAds: noopfn, - get: noopnullfn, - getAttributeKeys: nooparrayfn, - getTargeting: noopfn, - getTargetingKeys: nooparrayfn, - getSlots: nooparrayfn, - refresh: noopfn, - set: noopthisfn, - setCategoryExclusion: noopthisfn, - setCentering: noopfn, - setCookieOptions: noopthisfn, - setForceSafeFrame: noopthisfn, - setLocation: noopthisfn, - setPublisherProvidedId: noopthisfn, - setSafeFrameConfig: noopthisfn, - setTagForChildDirectedTreatment: noopthisfn, - setTargeting: noopthisfn, - setVideoContent: noopthisfn, - updateCorrelator: noopfn - }; - var SizeMappingBuilder = function() { - ; - }; - p = SizeMappingBuilder.prototype; - p.addSize = noopthisfn; - p.build = noopnullfn; - var Slot = function() { - ; - }; - p = Slot.prototype; - p.addService = noopthisfn; - p.clearCategoryExclusions = noopthisfn; - p.clearTargeting = noopthisfn; - p.defineSizeMapping = noopthisfn; - p.get = noopnullfn; - p.getAdUnitPath = nooparrayfn; - p.getAttributeKeys = nooparrayfn; - p.getCategoryExclusions = nooparrayfn; - p.getDomId = noopstrfn; - p.getSlotElementId = noopstrfn; - p.getSlotId = noopthisfn; - p.getTargeting = nooparrayfn; - p.getTargetingKeys = nooparrayfn; - p.set = noopthisfn; - p.setCategoryExclusion = noopthisfn; - p.setClickUrl = noopthisfn; - p.setCollapseEmptyDiv = noopthisfn; - p.setTargeting = noopthisfn; - // - var gpt = window.googletag || {}; - var cmd = gpt.cmd || []; - gpt.apiReady = true; - gpt.cmd = []; - gpt.cmd.push = function(a) { - try { - a(); - } catch (ex) { - } - return 1; - }; - gpt.companionAds = function() { return companionAdsService; }; - gpt.content = function() { return contentService; }; - gpt.defineOutOfPageSlot = function() { return new Slot(); }; - gpt.defineSlot = function() { return new Slot(); }; - gpt.destroySlots = noopfn; - gpt.disablePublisherConsole = noopfn; - gpt.display = noopfn; - gpt.enableServices = noopfn; - gpt.getVersion = noopstrfn; - gpt.pubads = function() { return pubAdsService; }; - gpt.pubadsReady = true; - gpt.setAdIframeTitle = noopfn; - gpt.sizeMapping = function() { return new SizeMappingBuilder(); }; - window.googletag = gpt; - while ( cmd.length !== 0 ) { - gpt.cmd.push(cmd.shift()); - } + alert("surrogates_5"); + alert("surrogates_5"); + alert("surrogates_5"); + alert("surrogates_5"); + alert("surrogates_5"); })(); # Obviously more work needs to be done, but at least for now it takes care of: @@ -326,23 +53,10 @@ googletagservices.com/gpt.js application/javascript # - https://github.com/uBlockOrigin/uAssets/issues/420 googletagmanager.com/gtm.js application/javascript (function() { - var noopfn = function() { - }; - var w = window; - w.ga = w.ga || noopfn; - var dl = w.dataLayer; - if ( dl instanceof Object === false ) { return; } - if ( dl.hide instanceof Object && typeof dl.hide.end === 'function' ) { - dl.hide.end(); - } - if ( typeof dl.push === 'function' ) { - dl.push = function(o) { - if ( - o instanceof Object && - typeof o.eventCallback === 'function' - ) { - setTimeout(o.eventCallback, 1); - } - }; - } + alert("surrogates_6"); + alert("surrogates_6"); + alert("surrogates_6"); + alert("surrogates_6"); + alert("surrogates_6"); + alert("surrogates_6"); })(); \ No newline at end of file diff --git a/app/src/androidTest/resources/binary/surrogates/surrogates_with_different_mime_types b/app/src/androidTest/resources/binary/surrogates/surrogates_with_different_mime_types index 8239bb8014c8..6c1834d1ddbb 100644 --- a/app/src/androidTest/resources/binary/surrogates/surrogates_with_different_mime_types +++ b/app/src/androidTest/resources/binary/surrogates/surrogates_with_different_mime_types @@ -7,160 +7,56 @@ # - https://developers.google.com/analytics/devguides/collection/gajs/ google-analytics.com/ga.js text/plain (function() { - var noopfn = function() { - ; - }; - // - var Gaq = function() { - ; - }; - Gaq.prototype.Na = noopfn; - Gaq.prototype.O = noopfn; - Gaq.prototype.Sa = noopfn; - Gaq.prototype.Ta = noopfn; - Gaq.prototype.Va = noopfn; - Gaq.prototype._createAsyncTracker = noopfn; - Gaq.prototype._getAsyncTracker = noopfn; - Gaq.prototype._getPlugin = noopfn; - Gaq.prototype.push = function(a) { - if ( typeof a === 'function' ) { - a(); return; - } - if ( Array.isArray(a) === false ) { - return; - } - // https://twitter.com/catovitch/status/776442930345218048 - // https://developers.google.com/analytics/devguides/collection/gajs/methods/gaJSApiDomainDirectory#_gat.GA_Tracker_._link - if ( a[0] === '_link' && typeof a[1] === 'string' ) { - window.location.assign(a[1]); - } - // https://github.com/gorhill/uBlock/issues/2162 - if ( a[0] === '_set' && a[1] === 'hitCallback' && typeof a[2] === 'function' ) { - a[2](); - } - }; - // - var tracker = (function() { - var out = {}; - var api = [ - '_addIgnoredOrganic _addIgnoredRef _addItem _addOrganic', - '_addTrans _clearIgnoredOrganic _clearIgnoredRef _clearOrganic', - '_cookiePathCopy _deleteCustomVar _getName _setAccount', - '_getAccount _getClientInfo _getDetectFlash _getDetectTitle', - '_getLinkerUrl _getLocalGifPath _getServiceMode _getVersion', - '_getVisitorCustomVar _initData _link _linkByPost', - '_setAllowAnchor _setAllowHash _setAllowLinker _setCampContentKey', - '_setCampMediumKey _setCampNameKey _setCampNOKey _setCampSourceKey', - '_setCampTermKey _setCampaignCookieTimeout _setCampaignTrack _setClientInfo', - '_setCookiePath _setCookiePersistence _setCookieTimeout _setCustomVar', - '_setDetectFlash _setDetectTitle _setDomainName _setLocalGifPath', - '_setLocalRemoteServerMode _setLocalServerMode _setReferrerOverride _setRemoteServerMode', - '_setSampleRate _setSessionTimeout _setSiteSpeedSampleRate _setSessionCookieTimeout', - '_setVar _setVisitorCookieTimeout _trackEvent _trackPageLoadTime', - '_trackPageview _trackSocial _trackTiming _trackTrans', - '_visitCode' - ].join(' ').split(/\s+/); - var i = api.length; - while ( i-- ) { - out[api[i]] = noopfn; - } - out._getLinkerUrl = function(a) { - return a; - }; - return out; - })(); - // - var Gat = function() { - ; - }; - Gat.prototype._anonymizeIP = noopfn; - Gat.prototype._createTracker = noopfn; - Gat.prototype._forceSSL = noopfn; - Gat.prototype._getPlugin = noopfn; - Gat.prototype._getTracker = function() { - return tracker; - }; - Gat.prototype._getTrackerByName = function() { - return tracker; - }; - Gat.prototype._getTrackers = noopfn; - Gat.prototype.aa = noopfn; - Gat.prototype.ab = noopfn; - Gat.prototype.hb = noopfn; - Gat.prototype.la = noopfn; - Gat.prototype.oa = noopfn; - Gat.prototype.pa = noopfn; - Gat.prototype.u = noopfn; - var gat = new Gat(); - window._gat = gat; - // - var gaq = new Gaq(); - (function() { - var aa = window._gaq || []; - if ( Array.isArray(aa) ) { - while ( aa[0] ) { - gaq.push(aa.shift()); - } - } - })(); - window._gaq = gaq.qf = gaq; + alert("surrogates_1"); })(); google-analytics.com/analytics.js application/javascript (function() { - // https://developers.google.com/analytics/devguides/collection/analyticsjs/ - var noopfn = function() { - ; - }; - var noopnullfn = function() { - return null; - }; - // - var Tracker = function() { - ; - }; - var p = Tracker.prototype; - p.get = noopfn; - p.set = noopfn; - p.send = noopfn; - // - var w = window, - gaName = w.GoogleAnalyticsObject || 'ga'; - var ga = function() { - var len = arguments.length; - if ( len === 0 ) { - return; - } - var f = arguments[len-1]; - if ( typeof f !== 'object' || f === null || typeof f.hitCallback !== 'function' ) { - return; - } - try { - f.hitCallback(); - } catch (ex) { - } - }; - ga.create = function() { - return new Tracker(); - }; - ga.getByName = noopnullfn; - ga.getAll = function() { - return []; - }; - ga.remove = noopfn; - w[gaName] = ga; - // https://github.com/gorhill/uBlock/issues/3075 - var dl = w.dataLayer; - if ( dl instanceof Object && dl.hide instanceof Object && typeof dl.hide.end === 'function' ) { - dl.hide.end(); - } + // random comment + alert("surrogates_2"); + alert("surrogates_2"); })(); google-analytics.com/inpage_linkid.js application/json (function() { - window._gaq = window._gaq || { - push: function() { - ; - } - }; + alert("surrogates_3"); + alert("surrogates_3"); + alert("surrogates_3"); +})(); + +# https://github.com/gorhill/uBlock/issues/2480 +# https://developers.google.com/analytics/devguides/collection/gajs/experiments#cxjs +google-analytics.com/cx/api.js application/javascript +(function() { + alert("surrogates_4"); + alert("surrogates_4"); + alert("surrogates_4"); + alert("surrogates_4"); +})(); + +# Ubiquitous googletagservices.com: not blocked by EasyPrivacy. +# Tags are tiny bits of website code that let you measure traffic and +# visitor behavior +googletagservices.com/gpt.js application/javascript +(function() { + alert("surrogates_5"); + alert("surrogates_5"); + alert("surrogates_5"); + alert("surrogates_5"); + alert("surrogates_5"); +})(); + +# Obviously more work needs to be done, but at least for now it takes care of: +# See related filter in assets/ublock/privacy.txt +# Also: +# - https://github.com/gorhill/uBlock/issues/2569 +# - https://github.com/uBlockOrigin/uAssets/issues/420 +googletagmanager.com/gtm.js application/javascript +(function() { + alert("surrogates_6"); + alert("surrogates_6"); + alert("surrogates_6"); + alert("surrogates_6"); + alert("surrogates_6"); + alert("surrogates_6"); })(); \ No newline at end of file diff --git a/app/src/androidTest/resources/binary/surrogates/surrogates_with_empty_line_at_end_of_file b/app/src/androidTest/resources/binary/surrogates/surrogates_with_empty_line_at_end_of_file index 522ae4f44aed..2787e6fb4752 100644 --- a/app/src/androidTest/resources/binary/surrogates/surrogates_with_empty_line_at_end_of_file +++ b/app/src/androidTest/resources/binary/surrogates/surrogates_with_empty_line_at_end_of_file @@ -7,180 +7,31 @@ # - https://developers.google.com/analytics/devguides/collection/gajs/ google-analytics.com/ga.js application/javascript (function() { - var noopfn = function() { - ; - }; - // - var Gaq = function() { - ; - }; - Gaq.prototype.Na = noopfn; - Gaq.prototype.O = noopfn; - Gaq.prototype.Sa = noopfn; - Gaq.prototype.Ta = noopfn; - Gaq.prototype.Va = noopfn; - Gaq.prototype._createAsyncTracker = noopfn; - Gaq.prototype._getAsyncTracker = noopfn; - Gaq.prototype._getPlugin = noopfn; - Gaq.prototype.push = function(a) { - if ( typeof a === 'function' ) { - a(); return; - } - if ( Array.isArray(a) === false ) { - return; - } - // https://twitter.com/catovitch/status/776442930345218048 - // https://developers.google.com/analytics/devguides/collection/gajs/methods/gaJSApiDomainDirectory#_gat.GA_Tracker_._link - if ( a[0] === '_link' && typeof a[1] === 'string' ) { - window.location.assign(a[1]); - } - // https://github.com/gorhill/uBlock/issues/2162 - if ( a[0] === '_set' && a[1] === 'hitCallback' && typeof a[2] === 'function' ) { - a[2](); - } - }; - // - var tracker = (function() { - var out = {}; - var api = [ - '_addIgnoredOrganic _addIgnoredRef _addItem _addOrganic', - '_addTrans _clearIgnoredOrganic _clearIgnoredRef _clearOrganic', - '_cookiePathCopy _deleteCustomVar _getName _setAccount', - '_getAccount _getClientInfo _getDetectFlash _getDetectTitle', - '_getLinkerUrl _getLocalGifPath _getServiceMode _getVersion', - '_getVisitorCustomVar _initData _link _linkByPost', - '_setAllowAnchor _setAllowHash _setAllowLinker _setCampContentKey', - '_setCampMediumKey _setCampNameKey _setCampNOKey _setCampSourceKey', - '_setCampTermKey _setCampaignCookieTimeout _setCampaignTrack _setClientInfo', - '_setCookiePath _setCookiePersistence _setCookieTimeout _setCustomVar', - '_setDetectFlash _setDetectTitle _setDomainName _setLocalGifPath', - '_setLocalRemoteServerMode _setLocalServerMode _setReferrerOverride _setRemoteServerMode', - '_setSampleRate _setSessionTimeout _setSiteSpeedSampleRate _setSessionCookieTimeout', - '_setVar _setVisitorCookieTimeout _trackEvent _trackPageLoadTime', - '_trackPageview _trackSocial _trackTiming _trackTrans', - '_visitCode' - ].join(' ').split(/\s+/); - var i = api.length; - while ( i-- ) { - out[api[i]] = noopfn; - } - out._getLinkerUrl = function(a) { - return a; - }; - return out; - })(); - // - var Gat = function() { - ; - }; - Gat.prototype._anonymizeIP = noopfn; - Gat.prototype._createTracker = noopfn; - Gat.prototype._forceSSL = noopfn; - Gat.prototype._getPlugin = noopfn; - Gat.prototype._getTracker = function() { - return tracker; - }; - Gat.prototype._getTrackerByName = function() { - return tracker; - }; - Gat.prototype._getTrackers = noopfn; - Gat.prototype.aa = noopfn; - Gat.prototype.ab = noopfn; - Gat.prototype.hb = noopfn; - Gat.prototype.la = noopfn; - Gat.prototype.oa = noopfn; - Gat.prototype.pa = noopfn; - Gat.prototype.u = noopfn; - var gat = new Gat(); - window._gat = gat; - // - var gaq = new Gaq(); - (function() { - var aa = window._gaq || []; - if ( Array.isArray(aa) ) { - while ( aa[0] ) { - gaq.push(aa.shift()); - } - } - })(); - window._gaq = gaq.qf = gaq; + alert("surrogates_1"); })(); google-analytics.com/analytics.js application/javascript (function() { - // https://developers.google.com/analytics/devguides/collection/analyticsjs/ - var noopfn = function() { - ; - }; - var noopnullfn = function() { - return null; - }; - // - var Tracker = function() { - ; - }; - var p = Tracker.prototype; - p.get = noopfn; - p.set = noopfn; - p.send = noopfn; - // - var w = window, - gaName = w.GoogleAnalyticsObject || 'ga'; - var ga = function() { - var len = arguments.length; - if ( len === 0 ) { - return; - } - var f = arguments[len-1]; - if ( typeof f !== 'object' || f === null || typeof f.hitCallback !== 'function' ) { - return; - } - try { - f.hitCallback(); - } catch (ex) { - } - }; - ga.create = function() { - return new Tracker(); - }; - ga.getByName = noopnullfn; - ga.getAll = function() { - return []; - }; - ga.remove = noopfn; - w[gaName] = ga; - // https://github.com/gorhill/uBlock/issues/3075 - var dl = w.dataLayer; - if ( dl instanceof Object && dl.hide instanceof Object && typeof dl.hide.end === 'function' ) { - dl.hide.end(); - } + // random comment + alert("surrogates_2"); + alert("surrogates_2"); })(); google-analytics.com/inpage_linkid.js application/javascript (function() { - window._gaq = window._gaq || { - push: function() { - ; - } - }; + alert("surrogates_3"); + alert("surrogates_3"); + alert("surrogates_3"); })(); # https://github.com/gorhill/uBlock/issues/2480 # https://developers.google.com/analytics/devguides/collection/gajs/experiments#cxjs google-analytics.com/cx/api.js application/javascript (function() { - var noopfn = function() { - }; - window.cxApi = { - chooseVariation: function() { - return 0; - }, - getChosenVariation: noopfn, - setAllowHash: noopfn, - setChosenVariation: noopfn, - setCookiePath: noopfn, - setDomainName: noopfn - }; + alert("surrogates_4"); + alert("surrogates_4"); + alert("surrogates_4"); + alert("surrogates_4"); })(); # Ubiquitous googletagservices.com: not blocked by EasyPrivacy. @@ -188,135 +39,11 @@ google-analytics.com/cx/api.js application/javascript # visitor behavior googletagservices.com/gpt.js application/javascript (function() { - var p; - // https://developers.google.com/doubleclick-gpt/reference - var noopfn = function() { - ; - }.bind(); - var noopthisfn = function() { - return this; - }; - var noopnullfn = function() { - return null; - }; - var nooparrayfn = function() { - return []; - }; - var noopstrfn = function() { - return ''; - }; - // - var companionAdsService = { - addEventListener: noopthisfn, - enableSyncLoading: noopfn, - setRefreshUnfilledSlots: noopfn - }; - var contentService = { - addEventListener: noopthisfn, - setContent: noopfn - }; - var PassbackSlot = function() { - ; - }; - p = PassbackSlot.prototype; - p.display = noopfn; - p.get = noopnullfn; - p.set = noopthisfn; - p.setClickUrl = noopthisfn; - p.setTagForChildDirectedTreatment = noopthisfn; - p.setTargeting = noopthisfn; - p.updateTargetingFromMap = noopthisfn; - var pubAdsService = { - addEventListener: noopthisfn, - clear: noopfn, - clearCategoryExclusions: noopthisfn, - clearTagForChildDirectedTreatment: noopthisfn, - clearTargeting: noopthisfn, - collapseEmptyDivs: noopfn, - defineOutOfPagePassback: function() { return new PassbackSlot(); }, - definePassback: function() { return new PassbackSlot(); }, - disableInitialLoad: noopfn, - display: noopfn, - enableAsyncRendering: noopfn, - enableSingleRequest: noopfn, - enableSyncRendering: noopfn, - enableVideoAds: noopfn, - get: noopnullfn, - getAttributeKeys: nooparrayfn, - getTargeting: noopfn, - getTargetingKeys: nooparrayfn, - getSlots: nooparrayfn, - refresh: noopfn, - set: noopthisfn, - setCategoryExclusion: noopthisfn, - setCentering: noopfn, - setCookieOptions: noopthisfn, - setForceSafeFrame: noopthisfn, - setLocation: noopthisfn, - setPublisherProvidedId: noopthisfn, - setSafeFrameConfig: noopthisfn, - setTagForChildDirectedTreatment: noopthisfn, - setTargeting: noopthisfn, - setVideoContent: noopthisfn, - updateCorrelator: noopfn - }; - var SizeMappingBuilder = function() { - ; - }; - p = SizeMappingBuilder.prototype; - p.addSize = noopthisfn; - p.build = noopnullfn; - var Slot = function() { - ; - }; - p = Slot.prototype; - p.addService = noopthisfn; - p.clearCategoryExclusions = noopthisfn; - p.clearTargeting = noopthisfn; - p.defineSizeMapping = noopthisfn; - p.get = noopnullfn; - p.getAdUnitPath = nooparrayfn; - p.getAttributeKeys = nooparrayfn; - p.getCategoryExclusions = nooparrayfn; - p.getDomId = noopstrfn; - p.getSlotElementId = noopstrfn; - p.getSlotId = noopthisfn; - p.getTargeting = nooparrayfn; - p.getTargetingKeys = nooparrayfn; - p.set = noopthisfn; - p.setCategoryExclusion = noopthisfn; - p.setClickUrl = noopthisfn; - p.setCollapseEmptyDiv = noopthisfn; - p.setTargeting = noopthisfn; - // - var gpt = window.googletag || {}; - var cmd = gpt.cmd || []; - gpt.apiReady = true; - gpt.cmd = []; - gpt.cmd.push = function(a) { - try { - a(); - } catch (ex) { - } - return 1; - }; - gpt.companionAds = function() { return companionAdsService; }; - gpt.content = function() { return contentService; }; - gpt.defineOutOfPageSlot = function() { return new Slot(); }; - gpt.defineSlot = function() { return new Slot(); }; - gpt.destroySlots = noopfn; - gpt.disablePublisherConsole = noopfn; - gpt.display = noopfn; - gpt.enableServices = noopfn; - gpt.getVersion = noopstrfn; - gpt.pubads = function() { return pubAdsService; }; - gpt.pubadsReady = true; - gpt.setAdIframeTitle = noopfn; - gpt.sizeMapping = function() { return new SizeMappingBuilder(); }; - window.googletag = gpt; - while ( cmd.length !== 0 ) { - gpt.cmd.push(cmd.shift()); - } + alert("surrogates_5"); + alert("surrogates_5"); + alert("surrogates_5"); + alert("surrogates_5"); + alert("surrogates_5"); })(); # Obviously more work needs to be done, but at least for now it takes care of: @@ -326,23 +53,10 @@ googletagservices.com/gpt.js application/javascript # - https://github.com/uBlockOrigin/uAssets/issues/420 googletagmanager.com/gtm.js application/javascript (function() { - var noopfn = function() { - }; - var w = window; - w.ga = w.ga || noopfn; - var dl = w.dataLayer; - if ( dl instanceof Object === false ) { return; } - if ( dl.hide instanceof Object && typeof dl.hide.end === 'function' ) { - dl.hide.end(); - } - if ( typeof dl.push === 'function' ) { - dl.push = function(o) { - if ( - o instanceof Object && - typeof o.eventCallback === 'function' - ) { - setTimeout(o.eventCallback, 1); - } - }; - } + alert("surrogates_6"); + alert("surrogates_6"); + alert("surrogates_6"); + alert("surrogates_6"); + alert("surrogates_6"); + alert("surrogates_6"); })(); From ff87c4c5e23d37e96c7dd36bb33322eb5bd53a6f Mon Sep 17 00:00:00 2001 From: Craig Russell Date: Thu, 1 Feb 2018 11:36:05 +0000 Subject: [PATCH 10/12] Remove word Analytics; more generically replaced with ResourceSurrogate --- .../browser/WebViewRequestInterceptorTest.kt | 12 ++++++------ .../ResourceSurrogateLoaderTest.kt} | 18 +++++++++--------- .../ResourceSurrogatesTest.kt} | 8 ++++---- .../app/browser/BrowserWebViewClient.kt | 6 ------ .../app/browser/WebViewRequestInterceptor.kt | 6 +++--- .../java/com/duckduckgo/app/di/AppComponent.kt | 4 ++-- .../com/duckduckgo/app/di/NetworkModule.kt | 6 +++--- .../app/global/DuckDuckGoApplication.kt | 6 +++--- .../app/job/AppConfigurationDownloader.kt | 12 ++++++------ .../ResourceSurrogateLoader.kt} | 12 ++++++------ .../ResourceSurrogates.kt} | 6 +++--- .../api/ResourceSurrogateListDownloader.kt} | 16 ++++++++-------- .../api/ResourceSurrogateListService.kt} | 4 ++-- .../di/ResourceSurrogateModule.kt} | 10 +++++----- .../store/ResourceSurrogateDataStore.kt} | 4 ++-- 15 files changed, 62 insertions(+), 68 deletions(-) rename app/src/androidTest/java/com/duckduckgo/app/{analyticsSurrogates/AnalyticsSurrogatesLoaderTest.kt => surrogates/ResourceSurrogateLoaderTest.kt} (85%) rename app/src/androidTest/java/com/duckduckgo/app/{analyticsSurrogates/AnalyticsSurrogatesTest.kt => surrogates/ResourceSurrogatesTest.kt} (92%) rename app/src/main/java/com/duckduckgo/app/{analyticsSurrogates/AnalyticsSurrogatesLoader.kt => surrogates/ResourceSurrogateLoader.kt} (86%) rename app/src/main/java/com/duckduckgo/app/{analyticsSurrogates/AnalyticsSurrogates.kt => surrogates/ResourceSurrogates.kt} (91%) rename app/src/main/java/com/duckduckgo/app/{analyticsSurrogates/api/AnalyticsSurrogatesListDownloader.kt => surrogates/api/ResourceSurrogateListDownloader.kt} (77%) rename app/src/main/java/com/duckduckgo/app/{analyticsSurrogates/api/AnalyticsSurrogatesListService.kt => surrogates/api/ResourceSurrogateListService.kt} (89%) rename app/src/main/java/com/duckduckgo/app/{analyticsSurrogates/di/AnalyticsSurrogatesModule.kt => surrogates/di/ResourceSurrogateModule.kt} (70%) rename app/src/main/java/com/duckduckgo/app/{analyticsSurrogates/store/AnalyticsSurrogatesDataStore.kt => surrogates/store/ResourceSurrogateDataStore.kt} (89%) diff --git a/app/src/androidTest/java/com/duckduckgo/app/browser/WebViewRequestInterceptorTest.kt b/app/src/androidTest/java/com/duckduckgo/app/browser/WebViewRequestInterceptorTest.kt index df5fbe84aa70..efe839263bba 100644 --- a/app/src/androidTest/java/com/duckduckgo/app/browser/WebViewRequestInterceptorTest.kt +++ b/app/src/androidTest/java/com/duckduckgo/app/browser/WebViewRequestInterceptorTest.kt @@ -22,8 +22,8 @@ import android.support.test.annotation.UiThreadTest import android.webkit.WebResourceRequest import android.webkit.WebResourceResponse import android.webkit.WebView -import com.duckduckgo.app.analyticsSurrogates.AnalyticsSurrogates -import com.duckduckgo.app.analyticsSurrogates.SurrogateResponse +import com.duckduckgo.app.surrogates.ResourceSurrogates +import com.duckduckgo.app.surrogates.SurrogateResponse import com.duckduckgo.app.httpsupgrade.HttpsUpgrader import com.duckduckgo.app.trackerdetection.TrackerDetector import com.duckduckgo.app.trackerdetection.model.TrackingEvent @@ -44,7 +44,7 @@ class WebViewRequestInterceptorTest { @Mock private lateinit var mockHttpsUpgrader: HttpsUpgrader @Mock - private lateinit var mockAnalyticsSurrogates: AnalyticsSurrogates + private lateinit var mockResourceSurrogates: ResourceSurrogates @Mock private lateinit var mockRequest: WebResourceRequest @@ -58,7 +58,7 @@ class WebViewRequestInterceptorTest { testee = WebViewRequestInterceptor( trackerDetector = mockTrackerDetector, httpsUpgrader = mockHttpsUpgrader, - analyticsSurrogates = mockAnalyticsSurrogates + resourceSurrogates = mockResourceSurrogates ) val context = InstrumentationRegistry.getTargetContext() @@ -246,7 +246,7 @@ class WebViewRequestInterceptorTest { @Test fun whenRequestShouldBlockAndNoSurrogateThenCancellingResponseReturned() { - whenever(mockAnalyticsSurrogates.get(any())).thenReturn(SurrogateResponse(responseAvailable = false)) + whenever(mockResourceSurrogates.get(any())).thenReturn(SurrogateResponse(responseAvailable = false)) configureShouldNotUpgrade() configureShouldBlock() @@ -265,7 +265,7 @@ class WebViewRequestInterceptorTest { responseAvailable = true, mimeType = "application/javascript", jsFunction = "javascript replacement function goes here") - whenever(mockAnalyticsSurrogates.get(any())).thenReturn(availableSurrogate) + whenever(mockResourceSurrogates.get(any())).thenReturn(availableSurrogate) configureShouldNotUpgrade() configureShouldBlock() diff --git a/app/src/androidTest/java/com/duckduckgo/app/analyticsSurrogates/AnalyticsSurrogatesLoaderTest.kt b/app/src/androidTest/java/com/duckduckgo/app/surrogates/ResourceSurrogateLoaderTest.kt similarity index 85% rename from app/src/androidTest/java/com/duckduckgo/app/analyticsSurrogates/AnalyticsSurrogatesLoaderTest.kt rename to app/src/androidTest/java/com/duckduckgo/app/surrogates/ResourceSurrogateLoaderTest.kt index 575ba813e711..475c09ffcf95 100644 --- a/app/src/androidTest/java/com/duckduckgo/app/analyticsSurrogates/AnalyticsSurrogatesLoaderTest.kt +++ b/app/src/androidTest/java/com/duckduckgo/app/surrogates/ResourceSurrogateLoaderTest.kt @@ -14,26 +14,26 @@ * limitations under the License. */ -package com.duckduckgo.app.analyticsSurrogates +package com.duckduckgo.app.surrogates import android.support.test.InstrumentationRegistry -import com.duckduckgo.app.analyticsSurrogates.store.AnalyticsSurrogatesDataStore +import com.duckduckgo.app.surrogates.store.ResourceSurrogateDataStore import org.junit.Assert import org.junit.Assert.assertEquals import org.junit.Before import org.junit.Test -class AnalyticsSurrogatesLoaderTest { +class ResourceSurrogateLoaderTest { - private lateinit var testee: AnalyticsSurrogatesLoader - private lateinit var dataStore: AnalyticsSurrogatesDataStore - private lateinit var analyticsSurrogates: AnalyticsSurrogates + private lateinit var testee: ResourceSurrogateLoader + private lateinit var dataStore: ResourceSurrogateDataStore + private lateinit var resourceSurrogates: ResourceSurrogates @Before fun setup() { - analyticsSurrogates = AnalyticsSurrogatesImpl() - dataStore = AnalyticsSurrogatesDataStore(InstrumentationRegistry.getTargetContext()) - testee = AnalyticsSurrogatesLoader(analyticsSurrogates, dataStore) + resourceSurrogates = ResourceSurrogatesImpl() + dataStore = ResourceSurrogateDataStore(InstrumentationRegistry.getTargetContext()) + testee = ResourceSurrogateLoader(resourceSurrogates, dataStore) } @Test diff --git a/app/src/androidTest/java/com/duckduckgo/app/analyticsSurrogates/AnalyticsSurrogatesTest.kt b/app/src/androidTest/java/com/duckduckgo/app/surrogates/ResourceSurrogatesTest.kt similarity index 92% rename from app/src/androidTest/java/com/duckduckgo/app/analyticsSurrogates/AnalyticsSurrogatesTest.kt rename to app/src/androidTest/java/com/duckduckgo/app/surrogates/ResourceSurrogatesTest.kt index d3781f73c372..cc415121ca96 100644 --- a/app/src/androidTest/java/com/duckduckgo/app/analyticsSurrogates/AnalyticsSurrogatesTest.kt +++ b/app/src/androidTest/java/com/duckduckgo/app/surrogates/ResourceSurrogatesTest.kt @@ -14,20 +14,20 @@ * limitations under the License. */ -package com.duckduckgo.app.analyticsSurrogates +package com.duckduckgo.app.surrogates import android.net.Uri import org.junit.Assert.* import org.junit.Before import org.junit.Test -class AnalyticsSurrogatesTest { +class ResourceSurrogatesTest { - private lateinit var testee: AnalyticsSurrogates + private lateinit var testee: ResourceSurrogates @Before fun setup() { - testee = AnalyticsSurrogatesImpl() + testee = ResourceSurrogatesImpl() } @Test diff --git a/app/src/main/java/com/duckduckgo/app/browser/BrowserWebViewClient.kt b/app/src/main/java/com/duckduckgo/app/browser/BrowserWebViewClient.kt index a77a68907886..78c7a5aacd6c 100644 --- a/app/src/main/java/com/duckduckgo/app/browser/BrowserWebViewClient.kt +++ b/app/src/main/java/com/duckduckgo/app/browser/BrowserWebViewClient.kt @@ -23,19 +23,13 @@ import android.webkit.WebResourceRequest import android.webkit.WebResourceResponse import android.webkit.WebView import android.webkit.WebViewClient -import com.duckduckgo.app.analyticsSurrogates.AnalyticsSurrogates -import com.duckduckgo.app.httpsupgrade.HttpsUpgrader -import com.duckduckgo.app.trackerdetection.TrackerDetector import timber.log.Timber import javax.inject.Inject class BrowserWebViewClient @Inject constructor( private val requestRewriter: RequestRewriter, - private var trackerDetector: TrackerDetector, - private var httpsUpgrader: HttpsUpgrader, private val specialUrlDetector: SpecialUrlDetector, - private val analyticsSurrogates: AnalyticsSurrogates, private val webViewRequestInterceptor: WebViewRequestInterceptor ) : WebViewClient() { diff --git a/app/src/main/java/com/duckduckgo/app/browser/WebViewRequestInterceptor.kt b/app/src/main/java/com/duckduckgo/app/browser/WebViewRequestInterceptor.kt index 842ae3e86727..b1fa52b33ab1 100644 --- a/app/src/main/java/com/duckduckgo/app/browser/WebViewRequestInterceptor.kt +++ b/app/src/main/java/com/duckduckgo/app/browser/WebViewRequestInterceptor.kt @@ -20,7 +20,7 @@ import android.support.annotation.WorkerThread import android.webkit.WebResourceRequest import android.webkit.WebResourceResponse import android.webkit.WebView -import com.duckduckgo.app.analyticsSurrogates.AnalyticsSurrogates +import com.duckduckgo.app.surrogates.ResourceSurrogates import com.duckduckgo.app.global.isHttp import com.duckduckgo.app.httpsupgrade.HttpsUpgrader import com.duckduckgo.app.privacymonitor.model.TrustedSites @@ -31,7 +31,7 @@ import javax.inject.Inject class WebViewRequestInterceptor @Inject constructor( - private val analyticsSurrogates: AnalyticsSurrogates, + private val resourceSurrogates: ResourceSurrogates, private val trackerDetector: TrackerDetector, private val httpsUpgrader: HttpsUpgrader ) { @@ -72,7 +72,7 @@ class WebViewRequestInterceptor @Inject constructor( if (shouldBlock(request, documentUrl, webViewClientListener)) { - val surrogate = analyticsSurrogates.get(url) + val surrogate = resourceSurrogates.get(url) if (surrogate.responseAvailable) { Timber.v("Surrogate found for %s", url) return WebResourceResponse( diff --git a/app/src/main/java/com/duckduckgo/app/di/AppComponent.kt b/app/src/main/java/com/duckduckgo/app/di/AppComponent.kt index b9c63d047860..8fce1e22f2d0 100644 --- a/app/src/main/java/com/duckduckgo/app/di/AppComponent.kt +++ b/app/src/main/java/com/duckduckgo/app/di/AppComponent.kt @@ -18,7 +18,7 @@ package com.duckduckgo.app.di import android.app.Application -import com.duckduckgo.app.analyticsSurrogates.di.AnalyticsSurrogatesModule +import com.duckduckgo.app.surrogates.di.ResourceSurrogateModule import com.duckduckgo.app.browser.autoComplete.BrowserAutoCompleteModule import com.duckduckgo.app.browser.di.BrowserModule import com.duckduckgo.app.global.DuckDuckGoApplication @@ -45,7 +45,7 @@ import javax.inject.Singleton (BrowserModule::class), (BrowserAutoCompleteModule::class), (HttpsUpgraderModule::class), - (AnalyticsSurrogatesModule::class), + (ResourceSurrogateModule::class), (TrackerDetectionModule::class) ]) interface AppComponent : AndroidInjector { diff --git a/app/src/main/java/com/duckduckgo/app/di/NetworkModule.kt b/app/src/main/java/com/duckduckgo/app/di/NetworkModule.kt index 0d2e7d4898d7..4c286f37d3bd 100644 --- a/app/src/main/java/com/duckduckgo/app/di/NetworkModule.kt +++ b/app/src/main/java/com/duckduckgo/app/di/NetworkModule.kt @@ -17,7 +17,7 @@ package com.duckduckgo.app.di import android.content.Context -import com.duckduckgo.app.analyticsSurrogates.api.AnalyticsSurrogatesListService +import com.duckduckgo.app.surrogates.api.ResourceSurrogateListService import com.duckduckgo.app.autocomplete.api.AutoCompleteService import com.duckduckgo.app.browser.R import com.duckduckgo.app.httpsupgrade.api.HttpsUpgradeListService @@ -70,8 +70,8 @@ class NetworkModule { retrofit.create(AutoCompleteService::class.java) @Provides - fun surrogatesService(retrofit: Retrofit): AnalyticsSurrogatesListService = - retrofit.create(AnalyticsSurrogatesListService::class.java) + fun surrogatesService(retrofit: Retrofit): ResourceSurrogateListService = + retrofit.create(ResourceSurrogateListService::class.java) companion object { private const val CACHE_SIZE: Long = 10 * 1024 * 1024 // 10MB diff --git a/app/src/main/java/com/duckduckgo/app/global/DuckDuckGoApplication.kt b/app/src/main/java/com/duckduckgo/app/global/DuckDuckGoApplication.kt index e3bb7f87a458..ee0c9cd1f7ac 100644 --- a/app/src/main/java/com/duckduckgo/app/global/DuckDuckGoApplication.kt +++ b/app/src/main/java/com/duckduckgo/app/global/DuckDuckGoApplication.kt @@ -19,7 +19,7 @@ package com.duckduckgo.app.global import android.app.Activity import android.app.Application import android.app.Service -import com.duckduckgo.app.analyticsSurrogates.AnalyticsSurrogatesLoader +import com.duckduckgo.app.surrogates.ResourceSurrogateLoader import com.duckduckgo.app.browser.BuildConfig import com.duckduckgo.app.di.DaggerAppComponent import com.duckduckgo.app.job.AppConfigurationSyncer @@ -50,7 +50,7 @@ class DuckDuckGoApplication : HasActivityInjector, HasServiceInjector, Applicati lateinit var trackerDataLoader: TrackerDataLoader @Inject - lateinit var analyticsSurrogatesLoader: AnalyticsSurrogatesLoader + lateinit var resourceSurrogateLoader: ResourceSurrogateLoader @Inject lateinit var appConfigurationSyncer: AppConfigurationSyncer @@ -92,7 +92,7 @@ class DuckDuckGoApplication : HasActivityInjector, HasServiceInjector, Applicati private fun loadTrackerData() { doAsync { trackerDataLoader.loadData() - analyticsSurrogatesLoader.loadData() + resourceSurrogateLoader.loadData() } } diff --git a/app/src/main/java/com/duckduckgo/app/job/AppConfigurationDownloader.kt b/app/src/main/java/com/duckduckgo/app/job/AppConfigurationDownloader.kt index cd95e80935e0..5cce7364cf3d 100644 --- a/app/src/main/java/com/duckduckgo/app/job/AppConfigurationDownloader.kt +++ b/app/src/main/java/com/duckduckgo/app/job/AppConfigurationDownloader.kt @@ -16,7 +16,7 @@ package com.duckduckgo.app.job -import com.duckduckgo.app.analyticsSurrogates.api.AnalyticsSurrogatesListDownloader +import com.duckduckgo.app.surrogates.api.ResourceSurrogateListDownloader import com.duckduckgo.app.global.db.AppDatabase import com.duckduckgo.app.httpsupgrade.api.HttpsUpgradeListDownloader import com.duckduckgo.app.settings.db.AppConfigurationEntity @@ -28,17 +28,17 @@ import timber.log.Timber import javax.inject.Inject class AppConfigurationDownloader @Inject constructor( - private val trackerDataDownloader: TrackerDataDownloader, - private val httpsUpgradeListDownloader: HttpsUpgradeListDownloader, - private val analyticsSurrogatesDownloader: AnalyticsSurrogatesListDownloader, - private val appDatabase: AppDatabase) { + private val trackerDataDownloader: TrackerDataDownloader, + private val httpsUpgradeListDownloader: HttpsUpgradeListDownloader, + private val resourceSurrogateDownloader: ResourceSurrogateListDownloader, + private val appDatabase: AppDatabase) { fun downloadTask(): Completable { val easyListDownload = trackerDataDownloader.downloadList(EASYLIST) val easyPrivacyDownload = trackerDataDownloader.downloadList(EASYPRIVACY) val trackersWhitelist = trackerDataDownloader.downloadList(TRACKERSWHITELIST) val disconnectDownload = trackerDataDownloader.downloadList(DISCONNECT) - val surrogatesDownload = analyticsSurrogatesDownloader.downloadList() + val surrogatesDownload = resourceSurrogateDownloader.downloadList() val httpsUpgradeDownload = httpsUpgradeListDownloader.downloadList() return Completable.mergeDelayError(listOf( diff --git a/app/src/main/java/com/duckduckgo/app/analyticsSurrogates/AnalyticsSurrogatesLoader.kt b/app/src/main/java/com/duckduckgo/app/surrogates/ResourceSurrogateLoader.kt similarity index 86% rename from app/src/main/java/com/duckduckgo/app/analyticsSurrogates/AnalyticsSurrogatesLoader.kt rename to app/src/main/java/com/duckduckgo/app/surrogates/ResourceSurrogateLoader.kt index 3b5bdb51ea81..06e5ae419751 100644 --- a/app/src/main/java/com/duckduckgo/app/analyticsSurrogates/AnalyticsSurrogatesLoader.kt +++ b/app/src/main/java/com/duckduckgo/app/surrogates/ResourceSurrogateLoader.kt @@ -14,24 +14,24 @@ * limitations under the License. */ -package com.duckduckgo.app.analyticsSurrogates +package com.duckduckgo.app.surrogates import android.support.annotation.WorkerThread -import com.duckduckgo.app.analyticsSurrogates.store.AnalyticsSurrogatesDataStore +import com.duckduckgo.app.surrogates.store.ResourceSurrogateDataStore import timber.log.Timber import java.io.ByteArrayInputStream import javax.inject.Inject @WorkerThread -class AnalyticsSurrogatesLoader @Inject constructor( - private val analyticsSurrogates: AnalyticsSurrogates, - private val surrogatesDataStore: AnalyticsSurrogatesDataStore +class ResourceSurrogateLoader @Inject constructor( + private val resourceSurrogates: ResourceSurrogates, + private val surrogatesDataStore: ResourceSurrogateDataStore ) { fun loadData() { if (surrogatesDataStore.hasData()) { val bytes = surrogatesDataStore.loadData() - analyticsSurrogates.loadSurrogates(convertBytes(bytes)) + resourceSurrogates.loadSurrogates(convertBytes(bytes)) } } diff --git a/app/src/main/java/com/duckduckgo/app/analyticsSurrogates/AnalyticsSurrogates.kt b/app/src/main/java/com/duckduckgo/app/surrogates/ResourceSurrogates.kt similarity index 91% rename from app/src/main/java/com/duckduckgo/app/analyticsSurrogates/AnalyticsSurrogates.kt rename to app/src/main/java/com/duckduckgo/app/surrogates/ResourceSurrogates.kt index fa21092e63e5..8856b7e8b7dc 100644 --- a/app/src/main/java/com/duckduckgo/app/analyticsSurrogates/AnalyticsSurrogates.kt +++ b/app/src/main/java/com/duckduckgo/app/surrogates/ResourceSurrogates.kt @@ -14,17 +14,17 @@ * limitations under the License. */ -package com.duckduckgo.app.analyticsSurrogates +package com.duckduckgo.app.surrogates import android.net.Uri -interface AnalyticsSurrogates { +interface ResourceSurrogates { fun loadSurrogates(urls: List) fun get(uri: Uri): SurrogateResponse fun getAll(): List } -class AnalyticsSurrogatesImpl : AnalyticsSurrogates { +class ResourceSurrogatesImpl : ResourceSurrogates { private val surrogates = mutableListOf() diff --git a/app/src/main/java/com/duckduckgo/app/analyticsSurrogates/api/AnalyticsSurrogatesListDownloader.kt b/app/src/main/java/com/duckduckgo/app/surrogates/api/ResourceSurrogateListDownloader.kt similarity index 77% rename from app/src/main/java/com/duckduckgo/app/analyticsSurrogates/api/AnalyticsSurrogatesListDownloader.kt rename to app/src/main/java/com/duckduckgo/app/surrogates/api/ResourceSurrogateListDownloader.kt index 42e20bb2e588..c7c67c3a57fd 100644 --- a/app/src/main/java/com/duckduckgo/app/analyticsSurrogates/api/AnalyticsSurrogatesListDownloader.kt +++ b/app/src/main/java/com/duckduckgo/app/surrogates/api/ResourceSurrogateListDownloader.kt @@ -14,10 +14,10 @@ * limitations under the License. */ -package com.duckduckgo.app.analyticsSurrogates.api +package com.duckduckgo.app.surrogates.api -import com.duckduckgo.app.analyticsSurrogates.AnalyticsSurrogatesLoader -import com.duckduckgo.app.analyticsSurrogates.store.AnalyticsSurrogatesDataStore +import com.duckduckgo.app.surrogates.ResourceSurrogateLoader +import com.duckduckgo.app.surrogates.store.ResourceSurrogateDataStore import com.duckduckgo.app.global.api.isCached import io.reactivex.Completable import timber.log.Timber @@ -25,10 +25,10 @@ import java.io.IOException import javax.inject.Inject -class AnalyticsSurrogatesListDownloader @Inject constructor( - private val service: AnalyticsSurrogatesListService, - private val surrogatesDataStore: AnalyticsSurrogatesDataStore, - private val analyticsSurrogatesLoader: AnalyticsSurrogatesLoader +class ResourceSurrogateListDownloader @Inject constructor( + private val service: ResourceSurrogateListService, + private val surrogatesDataStore: ResourceSurrogateDataStore, + private val resourceSurrogateLoader: ResourceSurrogateLoader ) { fun downloadList(): Completable { @@ -51,7 +51,7 @@ class AnalyticsSurrogatesListDownloader @Inject constructor( val bodyBytes = response.body()!!.bytes() Timber.d("Updating surrogates data store with new data") persistData(bodyBytes) - analyticsSurrogatesLoader.loadData() + resourceSurrogateLoader.loadData() } else { throw IOException("Status: ${response.code()} - ${response.errorBody()?.string()}") } diff --git a/app/src/main/java/com/duckduckgo/app/analyticsSurrogates/api/AnalyticsSurrogatesListService.kt b/app/src/main/java/com/duckduckgo/app/surrogates/api/ResourceSurrogateListService.kt similarity index 89% rename from app/src/main/java/com/duckduckgo/app/analyticsSurrogates/api/AnalyticsSurrogatesListService.kt rename to app/src/main/java/com/duckduckgo/app/surrogates/api/ResourceSurrogateListService.kt index 88c0af32fe3e..1066ebec33e8 100644 --- a/app/src/main/java/com/duckduckgo/app/analyticsSurrogates/api/AnalyticsSurrogatesListService.kt +++ b/app/src/main/java/com/duckduckgo/app/surrogates/api/ResourceSurrogateListService.kt @@ -14,14 +14,14 @@ * limitations under the License. */ -package com.duckduckgo.app.analyticsSurrogates.api +package com.duckduckgo.app.surrogates.api import okhttp3.ResponseBody import retrofit2.Call import retrofit2.http.GET -interface AnalyticsSurrogatesListService { +interface ResourceSurrogateListService { @GET("/contentblocking.js?l=surrogates") fun https(): Call diff --git a/app/src/main/java/com/duckduckgo/app/analyticsSurrogates/di/AnalyticsSurrogatesModule.kt b/app/src/main/java/com/duckduckgo/app/surrogates/di/ResourceSurrogateModule.kt similarity index 70% rename from app/src/main/java/com/duckduckgo/app/analyticsSurrogates/di/AnalyticsSurrogatesModule.kt rename to app/src/main/java/com/duckduckgo/app/surrogates/di/ResourceSurrogateModule.kt index d7c5e5298252..15d6629a8b38 100644 --- a/app/src/main/java/com/duckduckgo/app/analyticsSurrogates/di/AnalyticsSurrogatesModule.kt +++ b/app/src/main/java/com/duckduckgo/app/surrogates/di/ResourceSurrogateModule.kt @@ -14,19 +14,19 @@ * limitations under the License. */ -package com.duckduckgo.app.analyticsSurrogates.di +package com.duckduckgo.app.surrogates.di -import com.duckduckgo.app.analyticsSurrogates.AnalyticsSurrogates -import com.duckduckgo.app.analyticsSurrogates.AnalyticsSurrogatesImpl +import com.duckduckgo.app.surrogates.ResourceSurrogates +import com.duckduckgo.app.surrogates.ResourceSurrogatesImpl import dagger.Module import dagger.Provides import javax.inject.Singleton @Module -class AnalyticsSurrogatesModule { +class ResourceSurrogateModule { @Provides @Singleton - fun analyticsSurrogates() : AnalyticsSurrogates = AnalyticsSurrogatesImpl() + fun analyticsSurrogates() : ResourceSurrogates = ResourceSurrogatesImpl() } \ No newline at end of file diff --git a/app/src/main/java/com/duckduckgo/app/analyticsSurrogates/store/AnalyticsSurrogatesDataStore.kt b/app/src/main/java/com/duckduckgo/app/surrogates/store/ResourceSurrogateDataStore.kt similarity index 89% rename from app/src/main/java/com/duckduckgo/app/analyticsSurrogates/store/AnalyticsSurrogatesDataStore.kt rename to app/src/main/java/com/duckduckgo/app/surrogates/store/ResourceSurrogateDataStore.kt index c97247506f39..e7d92c6b4728 100644 --- a/app/src/main/java/com/duckduckgo/app/analyticsSurrogates/store/AnalyticsSurrogatesDataStore.kt +++ b/app/src/main/java/com/duckduckgo/app/surrogates/store/ResourceSurrogateDataStore.kt @@ -14,12 +14,12 @@ * limitations under the License. */ -package com.duckduckgo.app.analyticsSurrogates.store +package com.duckduckgo.app.surrogates.store import android.content.Context import javax.inject.Inject -class AnalyticsSurrogatesDataStore @Inject constructor(private val context: Context) { +class ResourceSurrogateDataStore @Inject constructor(private val context: Context) { fun hasData(): Boolean = context.fileExists(FILENAME) From bea9dba0ada8367e9dd2b05f9e4aefe199b054ec Mon Sep 17 00:00:00 2001 From: Craig Russell Date: Thu, 1 Feb 2018 11:41:01 +0000 Subject: [PATCH 11/12] Replace all real comments with fake comments for surrogate test data --- .../resources/binary/surrogates/surrogates_1 | 9 ++----- .../resources/binary/surrogates/surrogates_6 | 23 +++++------------ .../surrogates_no_empty_line_at_end_of_file | 25 ++++++------------- .../surrogates_with_different_mime_types | 24 ++++++------------ .../surrogates_with_empty_line_at_end_of_file | 23 +++++------------ 5 files changed, 29 insertions(+), 75 deletions(-) diff --git a/app/src/androidTest/resources/binary/surrogates/surrogates_1 b/app/src/androidTest/resources/binary/surrogates/surrogates_1 index a62db5c27b98..7bd5659717de 100644 --- a/app/src/androidTest/resources/binary/surrogates/surrogates_1 +++ b/app/src/androidTest/resources/binary/surrogates/surrogates_1 @@ -1,10 +1,5 @@ -# To neutralize GA scripts. The goal is to provide the minimal API -# expected by clients of these scripts so that the end users are able -# to wholly block GA while minimizing risks of page breakage. -# Test cases (need way more): -# - https://github.com/chrisaljoudi/uBlock/issues/119 -# Reference API: -# - https://developers.google.com/analytics/devguides/collection/gajs/ +# Some comments here +# This comment is also here google-analytics.com/ga.js application/javascript (function() { alert("surrogates_1"); diff --git a/app/src/androidTest/resources/binary/surrogates/surrogates_6 b/app/src/androidTest/resources/binary/surrogates/surrogates_6 index b49ab0ef0fb9..a9bd0b3ea49c 100644 --- a/app/src/androidTest/resources/binary/surrogates/surrogates_6 +++ b/app/src/androidTest/resources/binary/surrogates/surrogates_6 @@ -1,10 +1,5 @@ -# To neutralize GA scripts. The goal is to provide the minimal API -# expected by clients of these scripts so that the end users are able -# to wholly block GA while minimizing risks of page breakage. -# Test cases (need way more): -# - https://github.com/chrisaljoudi/uBlock/issues/119 -# Reference API: -# - https://developers.google.com/analytics/devguides/collection/gajs/ +# Some comments here +# This comment is also here google-analytics.com/ga.js application/javascript (function() { alert("surrogates_1"); @@ -24,8 +19,7 @@ google-analytics.com/inpage_linkid.js application/javascript alert("surrogates_3"); })(); -# https://github.com/gorhill/uBlock/issues/2480 -# https://developers.google.com/analytics/devguides/collection/gajs/experiments#cxjs +# A comment google-analytics.com/cx/api.js application/javascript (function() { alert("surrogates_4"); @@ -34,9 +28,8 @@ google-analytics.com/cx/api.js application/javascript alert("surrogates_4"); })(); -# Ubiquitous googletagservices.com: not blocked by EasyPrivacy. -# Tags are tiny bits of website code that let you measure traffic and -# visitor behavior +# A comment +# A comment googletagservices.com/gpt.js application/javascript (function() { alert("surrogates_5"); @@ -46,11 +39,7 @@ googletagservices.com/gpt.js application/javascript alert("surrogates_5"); })(); -# Obviously more work needs to be done, but at least for now it takes care of: -# See related filter in assets/ublock/privacy.txt -# Also: -# - https://github.com/gorhill/uBlock/issues/2569 -# - https://github.com/uBlockOrigin/uAssets/issues/420 +# A comment googletagmanager.com/gtm.js application/javascript (function() { alert("surrogates_6"); diff --git a/app/src/androidTest/resources/binary/surrogates/surrogates_no_empty_line_at_end_of_file b/app/src/androidTest/resources/binary/surrogates/surrogates_no_empty_line_at_end_of_file index b49ab0ef0fb9..bdd2d9a9f80b 100644 --- a/app/src/androidTest/resources/binary/surrogates/surrogates_no_empty_line_at_end_of_file +++ b/app/src/androidTest/resources/binary/surrogates/surrogates_no_empty_line_at_end_of_file @@ -1,10 +1,5 @@ -# To neutralize GA scripts. The goal is to provide the minimal API -# expected by clients of these scripts so that the end users are able -# to wholly block GA while minimizing risks of page breakage. -# Test cases (need way more): -# - https://github.com/chrisaljoudi/uBlock/issues/119 -# Reference API: -# - https://developers.google.com/analytics/devguides/collection/gajs/ +# Some comments here +# This comment is also here google-analytics.com/ga.js application/javascript (function() { alert("surrogates_1"); @@ -24,8 +19,9 @@ google-analytics.com/inpage_linkid.js application/javascript alert("surrogates_3"); })(); -# https://github.com/gorhill/uBlock/issues/2480 -# https://developers.google.com/analytics/devguides/collection/gajs/experiments#cxjs +# A comment +# A comment +# A comment google-analytics.com/cx/api.js application/javascript (function() { alert("surrogates_4"); @@ -34,9 +30,8 @@ google-analytics.com/cx/api.js application/javascript alert("surrogates_4"); })(); -# Ubiquitous googletagservices.com: not blocked by EasyPrivacy. -# Tags are tiny bits of website code that let you measure traffic and -# visitor behavior +# A comment +# A comment googletagservices.com/gpt.js application/javascript (function() { alert("surrogates_5"); @@ -46,11 +41,7 @@ googletagservices.com/gpt.js application/javascript alert("surrogates_5"); })(); -# Obviously more work needs to be done, but at least for now it takes care of: -# See related filter in assets/ublock/privacy.txt -# Also: -# - https://github.com/gorhill/uBlock/issues/2569 -# - https://github.com/uBlockOrigin/uAssets/issues/420 +# A comment googletagmanager.com/gtm.js application/javascript (function() { alert("surrogates_6"); diff --git a/app/src/androidTest/resources/binary/surrogates/surrogates_with_different_mime_types b/app/src/androidTest/resources/binary/surrogates/surrogates_with_different_mime_types index 6c1834d1ddbb..b09dacecb855 100644 --- a/app/src/androidTest/resources/binary/surrogates/surrogates_with_different_mime_types +++ b/app/src/androidTest/resources/binary/surrogates/surrogates_with_different_mime_types @@ -1,10 +1,5 @@ -# To neutralize GA scripts. The goal is to provide the minimal API -# expected by clients of these scripts so that the end users are able -# to wholly block GA while minimizing risks of page breakage. -# Test cases (need way more): -# - https://github.com/chrisaljoudi/uBlock/issues/119 -# Reference API: -# - https://developers.google.com/analytics/devguides/collection/gajs/ +# Some comments here +# This comment is also here google-analytics.com/ga.js text/plain (function() { alert("surrogates_1"); @@ -24,8 +19,8 @@ google-analytics.com/inpage_linkid.js application/json alert("surrogates_3"); })(); -# https://github.com/gorhill/uBlock/issues/2480 -# https://developers.google.com/analytics/devguides/collection/gajs/experiments#cxjs +# A comment +# A comment google-analytics.com/cx/api.js application/javascript (function() { alert("surrogates_4"); @@ -34,9 +29,7 @@ google-analytics.com/cx/api.js application/javascript alert("surrogates_4"); })(); -# Ubiquitous googletagservices.com: not blocked by EasyPrivacy. -# Tags are tiny bits of website code that let you measure traffic and -# visitor behavior +# A comment googletagservices.com/gpt.js application/javascript (function() { alert("surrogates_5"); @@ -46,11 +39,8 @@ googletagservices.com/gpt.js application/javascript alert("surrogates_5"); })(); -# Obviously more work needs to be done, but at least for now it takes care of: -# See related filter in assets/ublock/privacy.txt -# Also: -# - https://github.com/gorhill/uBlock/issues/2569 -# - https://github.com/uBlockOrigin/uAssets/issues/420 +# A comment +# A comment googletagmanager.com/gtm.js application/javascript (function() { alert("surrogates_6"); diff --git a/app/src/androidTest/resources/binary/surrogates/surrogates_with_empty_line_at_end_of_file b/app/src/androidTest/resources/binary/surrogates/surrogates_with_empty_line_at_end_of_file index 2787e6fb4752..b8454585c7c1 100644 --- a/app/src/androidTest/resources/binary/surrogates/surrogates_with_empty_line_at_end_of_file +++ b/app/src/androidTest/resources/binary/surrogates/surrogates_with_empty_line_at_end_of_file @@ -1,10 +1,5 @@ -# To neutralize GA scripts. The goal is to provide the minimal API -# expected by clients of these scripts so that the end users are able -# to wholly block GA while minimizing risks of page breakage. -# Test cases (need way more): -# - https://github.com/chrisaljoudi/uBlock/issues/119 -# Reference API: -# - https://developers.google.com/analytics/devguides/collection/gajs/ +# Some comments here +# This comment is also here google-analytics.com/ga.js application/javascript (function() { alert("surrogates_1"); @@ -24,8 +19,7 @@ google-analytics.com/inpage_linkid.js application/javascript alert("surrogates_3"); })(); -# https://github.com/gorhill/uBlock/issues/2480 -# https://developers.google.com/analytics/devguides/collection/gajs/experiments#cxjs +# A comment google-analytics.com/cx/api.js application/javascript (function() { alert("surrogates_4"); @@ -34,9 +28,8 @@ google-analytics.com/cx/api.js application/javascript alert("surrogates_4"); })(); -# Ubiquitous googletagservices.com: not blocked by EasyPrivacy. -# Tags are tiny bits of website code that let you measure traffic and -# visitor behavior +# A comment +# A comment googletagservices.com/gpt.js application/javascript (function() { alert("surrogates_5"); @@ -46,11 +39,7 @@ googletagservices.com/gpt.js application/javascript alert("surrogates_5"); })(); -# Obviously more work needs to be done, but at least for now it takes care of: -# See related filter in assets/ublock/privacy.txt -# Also: -# - https://github.com/gorhill/uBlock/issues/2569 -# - https://github.com/uBlockOrigin/uAssets/issues/420 +# A comment googletagmanager.com/gtm.js application/javascript (function() { alert("surrogates_6"); From 5336ba241860f6ca99f8ca7b62959ce752611a89 Mon Sep 17 00:00:00 2001 From: Craig Russell Date: Thu, 1 Feb 2018 12:22:08 +0000 Subject: [PATCH 12/12] Add extra tests and catch all errors for extra parsing safety --- .../surrogates/ResourceSurrogateLoaderTest.kt | 12 +++++ ...urrogates_invalid_format_missing_mimetypes | 14 +++++ ...t_unexpected_extra_space_in_function_close | 51 +++++++++++++++++++ .../app/surrogates/ResourceSurrogateLoader.kt | 21 +++++--- 4 files changed, 92 insertions(+), 6 deletions(-) create mode 100644 app/src/androidTest/resources/binary/surrogates/surrogates_invalid_format_missing_mimetypes create mode 100644 app/src/androidTest/resources/binary/surrogates/surrogates_valid_but_unexpected_extra_space_in_function_close diff --git a/app/src/androidTest/java/com/duckduckgo/app/surrogates/ResourceSurrogateLoaderTest.kt b/app/src/androidTest/java/com/duckduckgo/app/surrogates/ResourceSurrogateLoaderTest.kt index 475c09ffcf95..06555435a080 100644 --- a/app/src/androidTest/java/com/duckduckgo/app/surrogates/ResourceSurrogateLoaderTest.kt +++ b/app/src/androidTest/java/com/duckduckgo/app/surrogates/ResourceSurrogateLoaderTest.kt @@ -93,6 +93,18 @@ class ResourceSurrogateLoaderTest { assertEquals(5, actualNumberOfLines) } + @Test + fun whenSurrogateFileIsMissingMimeTypeEmptyListReturned() { + val surrogates = initialiseFile("surrogates_invalid_format_missing_mimetypes") + assertEquals(0, surrogates.size) + } + + @Test + fun whenSurrogateFileIsHasSpaceInFinalFunctionBlock() { + val surrogates = initialiseFile("surrogates_valid_but_unexpected_extra_space_in_function_close") + assertEquals(6, surrogates.size) + } + private fun initialiseFile(filename: String) : List { return testee.convertBytes(readFile(filename)) } diff --git a/app/src/androidTest/resources/binary/surrogates/surrogates_invalid_format_missing_mimetypes b/app/src/androidTest/resources/binary/surrogates/surrogates_invalid_format_missing_mimetypes new file mode 100644 index 000000000000..0d0cd328d25a --- /dev/null +++ b/app/src/androidTest/resources/binary/surrogates/surrogates_invalid_format_missing_mimetypes @@ -0,0 +1,14 @@ +# Some comments here +# This comment is also here +google-analytics.com/ga.js +(function() { + alert("surrogates_1"); +})(); + +google-analytics.com/analytics.js +(function() { + // random comment + alert("surrogates_2"); + alert("surrogates_2"); +})(); + diff --git a/app/src/androidTest/resources/binary/surrogates/surrogates_valid_but_unexpected_extra_space_in_function_close b/app/src/androidTest/resources/binary/surrogates/surrogates_valid_but_unexpected_extra_space_in_function_close new file mode 100644 index 000000000000..a440603f0841 --- /dev/null +++ b/app/src/androidTest/resources/binary/surrogates/surrogates_valid_but_unexpected_extra_space_in_function_close @@ -0,0 +1,51 @@ +# Some comments here +# This comment is also here +google-analytics.com/ga.js application/javascript +(function() { + alert("surrogates_1"); +}) (); + +google-analytics.com/analytics.js application/javascript +(function() { + // random comment + alert("surrogates_2"); + alert("surrogates_2"); +}) (); + +google-analytics.com/inpage_linkid.js application/javascript +(function() { + alert("surrogates_3"); + alert("surrogates_3"); + alert("surrogates_3"); +}) (); + +# A comment +google-analytics.com/cx/api.js application/javascript +(function() { + alert("surrogates_4"); + alert("surrogates_4"); + alert("surrogates_4"); + alert("surrogates_4"); +}) (); + +# A comment +# A comment +googletagservices.com/gpt.js application/javascript +(function() { + alert("surrogates_5"); + alert("surrogates_5"); + alert("surrogates_5"); + alert("surrogates_5"); + alert("surrogates_5"); +}) (); + +# A comment +googletagmanager.com/gtm.js application/javascript +(function() { + alert("surrogates_6"); + alert("surrogates_6"); + alert("surrogates_6"); + alert("surrogates_6"); + alert("surrogates_6"); + alert("surrogates_6"); +}) (); \ No newline at end of file diff --git a/app/src/main/java/com/duckduckgo/app/surrogates/ResourceSurrogateLoader.kt b/app/src/main/java/com/duckduckgo/app/surrogates/ResourceSurrogateLoader.kt index 06e5ae419751..f2cd69aff792 100644 --- a/app/src/main/java/com/duckduckgo/app/surrogates/ResourceSurrogateLoader.kt +++ b/app/src/main/java/com/duckduckgo/app/surrogates/ResourceSurrogateLoader.kt @@ -36,12 +36,21 @@ class ResourceSurrogateLoader @Inject constructor( } fun convertBytes(bytes: ByteArray): List { + return try { + parse(bytes) + } catch (e: Throwable) { + Timber.w(e, "Failed to parse surrogates file; file may be corrupt or badly formatted") + emptyList() + } + } + + private fun parse(bytes: ByteArray): List { val surrogates = mutableListOf() val reader = ByteArrayInputStream(bytes).bufferedReader() val existingLines = reader.readLines().toMutableList() - if(existingLines.isNotEmpty() && existingLines.last().isNotBlank()) { + if (existingLines.isNotEmpty() && existingLines.last().isNotBlank()) { existingLines.add("") } @@ -70,11 +79,11 @@ class ResourceSurrogateLoader @Inject constructor( if (it.isBlank()) { surrogates.add( - SurrogateResponse( - name = ruleName, - mimeType = mimeType, - jsFunction = functionBuilder.toString() - ) + SurrogateResponse( + name = ruleName, + mimeType = mimeType, + jsFunction = functionBuilder.toString() + ) ) functionBuilder.setLength(0)