diff --git a/app/src/androidTest/java/com/duckduckgo/app/browser/WebViewRequestInterceptorTest.kt b/app/src/androidTest/java/com/duckduckgo/app/browser/WebViewRequestInterceptorTest.kt new file mode 100644 index 000000000000..efe839263bba --- /dev/null +++ b/app/src/androidTest/java/com/duckduckgo/app/browser/WebViewRequestInterceptorTest.kt @@ -0,0 +1,315 @@ +/* + * Copyright (c) 2018 DuckDuckGo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.duckduckgo.app.browser + +import android.net.Uri +import android.support.test.InstrumentationRegistry +import android.support.test.annotation.UiThreadTest +import android.webkit.WebResourceRequest +import android.webkit.WebResourceResponse +import android.webkit.WebView +import com.duckduckgo.app.surrogates.ResourceSurrogates +import com.duckduckgo.app.surrogates.SurrogateResponse +import com.duckduckgo.app.httpsupgrade.HttpsUpgrader +import com.duckduckgo.app.trackerdetection.TrackerDetector +import com.duckduckgo.app.trackerdetection.model.TrackingEvent +import com.nhaarman.mockito_kotlin.* +import org.junit.Assert.* +import org.junit.Before +import org.junit.Test +import org.mockito.ArgumentMatchers.anyString +import org.mockito.Mock +import org.mockito.MockitoAnnotations + +class WebViewRequestInterceptorTest { + + private lateinit var testee: WebViewRequestInterceptor + + @Mock + private lateinit var mockTrackerDetector: TrackerDetector + @Mock + private lateinit var mockHttpsUpgrader: HttpsUpgrader + @Mock + private lateinit var mockResourceSurrogates: ResourceSurrogates + @Mock + private lateinit var mockRequest: WebResourceRequest + + private lateinit var webView: WebView + + @UiThreadTest + @Before + fun setup() { + MockitoAnnotations.initMocks(this) + + testee = WebViewRequestInterceptor( + trackerDetector = mockTrackerDetector, + httpsUpgrader = mockHttpsUpgrader, + resourceSurrogates = mockResourceSurrogates + ) + + val context = InstrumentationRegistry.getTargetContext() + + webView = WebView(context) + } + + @Test + fun whenUrlShouldBeUpgradedThenUpgraderInvoked() { + configureShouldUpgrade() + testee.shouldIntercept( + request = mockRequest, + currentUrl = null, + webView = webView, + webViewClientListener = null) + + verify(mockHttpsUpgrader).upgrade(any()) + } + + @Test + fun whenUrlShouldBeUpgradedThenCancelledResponseReturned() { + configureShouldUpgrade() + val response = testee.shouldIntercept( + request = mockRequest, + currentUrl = null, + webView = webView, + webViewClientListener = null) + + assertCancelledResponse(response) + } + + @Test + fun whenUrlShouldBeUpgradedButNotOnMainFrameThenNotUpgraded() { + configureShouldUpgrade() + whenever(mockRequest.isForMainFrame).thenReturn(false) + testee.shouldIntercept( + request = mockRequest, + currentUrl = null, + webView = webView, + webViewClientListener = null) + + verify(mockHttpsUpgrader, never()).upgrade(any()) + } + + @Test + fun whenUrlShouldBeUpgradedButUrlIsNullThenNotUpgraded() { + configureShouldUpgrade() + whenever(mockRequest.url).thenReturn(null) + testee.shouldIntercept( + request = mockRequest, + currentUrl = null, + webView = webView, + webViewClientListener = null) + + verify(mockHttpsUpgrader, never()).upgrade(any()) + } + + @Test + fun whenUrlShouldNotBeUpgradedThenUpgraderNotInvoked() { + whenever(mockHttpsUpgrader.shouldUpgrade(any())).thenReturn(false) + testee.shouldIntercept( + request = mockRequest, + currentUrl = null, + webView = webView, + webViewClientListener = null) + + verify(mockHttpsUpgrader, never()).upgrade(any()) + } + + @Test + fun whenCurrentUrlIsNullThenShouldContinueToLoad() { + configureShouldNotUpgrade() + val response = testee.shouldIntercept( + request = mockRequest, + currentUrl = null, + webView = webView, + webViewClientListener = null) + assertRequestCanContinueToLoad(response) + } + + @Test + fun whenIsTrustedSite_DuckDuckGo_ThenShouldContinueToLoad() { + configureShouldNotUpgrade() + val response = testee.shouldIntercept( + request = mockRequest, + currentUrl = "duckduckgo.com/a/b/c?q=123", + webView = webView, + webViewClientListener = null) + + assertRequestCanContinueToLoad(response) + } + + @Test + fun whenIsTrustedSite_DontTrack_ThenShouldContinueToLoad() { + configureShouldNotUpgrade() + val response = testee.shouldIntercept( + request = mockRequest, + currentUrl = "donttrack.us/a/b/c?q=123", + webView = webView, + webViewClientListener = null) + + assertRequestCanContinueToLoad(response) + } + + @Test + fun whenIsTrustedSite_SpreadPrivacy_ThenShouldContinueToLoad() { + configureShouldNotUpgrade() + val response = testee.shouldIntercept( + request = mockRequest, + currentUrl = "spreadprivacy.com/a/b/c?q=123", + webView = webView, + webViewClientListener = null) + + assertRequestCanContinueToLoad(response) + } + + @Test + fun whenIsTrustedSite_DuckDuckHack_ThenShouldContinueToLoad() { + configureShouldNotUpgrade() + val response = testee.shouldIntercept( + request = mockRequest, + currentUrl = "duckduckhack.com/a/b/c?q=123", + webView = webView, + webViewClientListener = null) + + assertRequestCanContinueToLoad(response) + } + + @Test + fun whenIsTrustedSite_PrivateBrowsingMyths_ThenShouldContinueToLoad() { + configureShouldNotUpgrade() + val response = testee.shouldIntercept( + request = mockRequest, + currentUrl = "privatebrowsingmyths.com/a/b/c?q=123", + webView = webView, + webViewClientListener = null) + + assertRequestCanContinueToLoad(response) + } + + @Test + fun whenIsTrustedSite_DuckDotCo_ThenShouldContinueToLoad() { + configureShouldNotUpgrade() + val response = testee.shouldIntercept( + request = mockRequest, + currentUrl = "duck.co/a/b/c?q=123", + webView = webView, + webViewClientListener = null) + + assertRequestCanContinueToLoad(response) + } + + @Test + fun whenIsHttpRequestThenHttpRequestListenerCalled() { + configureShouldNotUpgrade() + whenever(mockRequest.url).thenReturn(Uri.parse("http://example.com")) + val mockListener = mock() + + val response = testee.shouldIntercept( + request = mockRequest, + currentUrl = "foo.com", + webView = webView, + webViewClientListener = mockListener) + + verify(mockListener).pageHasHttpResources(anyString()) + assertRequestCanContinueToLoad(response) + } + + @Test + fun whenIsHttpsRequestThenHttpRequestListenerNotCalled() { + configureShouldNotUpgrade() + whenever(mockRequest.url).thenReturn(Uri.parse("https://example.com")) + val mockListener = mock() + + val response = testee.shouldIntercept( + request = mockRequest, + currentUrl = "foo.com", + webView = webView, + webViewClientListener = mockListener) + + verify(mockListener, never()).pageHasHttpResources(anyString()) + assertRequestCanContinueToLoad(response) + } + + + @Test + fun whenRequestShouldBlockAndNoSurrogateThenCancellingResponseReturned() { + whenever(mockResourceSurrogates.get(any())).thenReturn(SurrogateResponse(responseAvailable = false)) + + configureShouldNotUpgrade() + configureShouldBlock() + val response = testee.shouldIntercept( + request = mockRequest, + currentUrl = "foo.com", + webView = webView, + webViewClientListener = null) + + assertCancelledResponse(response) + } + + @Test + fun whenRequestShouldBlockButThereIsASurrogateThen() { + val availableSurrogate = SurrogateResponse( + responseAvailable = true, + mimeType = "application/javascript", + jsFunction = "javascript replacement function goes here") + whenever(mockResourceSurrogates.get(any())).thenReturn(availableSurrogate) + + configureShouldNotUpgrade() + configureShouldBlock() + val response = testee.shouldIntercept( + request = mockRequest, + currentUrl = "foo.com", + webView = webView, + webViewClientListener = null) + + assertEquals(availableSurrogate.jsFunction.byteInputStream().read(), response!!.data.read()) + } + + private fun assertRequestCanContinueToLoad(response: WebResourceResponse?) { + assertNull(response) + } + + private fun configureShouldBlock() { + val blockTrackingEvent = TrackingEvent(blocked = true, + documentUrl = "", + trackerUrl = "", + trackerNetwork = null) + whenever(mockRequest.isForMainFrame).thenReturn(false) + whenever(mockTrackerDetector.evaluate(any(), any(), any())).thenReturn(blockTrackingEvent) + } + + private fun configureShouldUpgrade() { + whenever(mockHttpsUpgrader.shouldUpgrade(any())).thenReturn(true) + whenever(mockRequest.url).thenReturn(validUri()) + whenever(mockRequest.isForMainFrame).thenReturn(true) + } + + private fun configureShouldNotUpgrade() { + whenever(mockHttpsUpgrader.shouldUpgrade(any())).thenReturn(false) + whenever(mockRequest.url).thenReturn(validUri()) + whenever(mockRequest.isForMainFrame).thenReturn(true) + } + + private fun validUri() = Uri.parse("example.com") + + private fun assertCancelledResponse(response: WebResourceResponse?) { + assertNotNull(response) + assertNull(response!!.data) + assertNull(response.mimeType) + assertNull(response.encoding) + } + +} diff --git a/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/HttpsUpgraderTest.kt b/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/HttpsUpgraderTest.kt index c958cf985570..e199d15ce8e9 100644 --- a/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/HttpsUpgraderTest.kt +++ b/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/HttpsUpgraderTest.kt @@ -31,7 +31,7 @@ class HttpsUpgraderTest { @Before fun before() { mockDao = mock() - testee = HttpsUpgrader(mockDao) + testee = HttpsUpgraderImpl(mockDao) } @Test diff --git a/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/db/HttpsUpgradePerformanceTest.kt b/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/db/HttpsUpgradePerformanceTest.kt index 2a726fe5536e..58c1475b3fc0 100644 --- a/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/db/HttpsUpgradePerformanceTest.kt +++ b/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/db/HttpsUpgradePerformanceTest.kt @@ -37,6 +37,7 @@ import android.net.Uri import android.support.test.InstrumentationRegistry import com.duckduckgo.app.global.db.AppDatabase import com.duckduckgo.app.httpsupgrade.HttpsUpgrader +import com.duckduckgo.app.httpsupgrade.HttpsUpgraderImpl import org.junit.Assert.assertEquals import org.junit.Assert.assertFalse import org.junit.Before @@ -55,7 +56,7 @@ class HttpsUpgraderPerformanceTest { fun setup() { db = Room.inMemoryDatabaseBuilder(InstrumentationRegistry.getContext(), AppDatabase::class.java).build() dao = db.httpsUpgradeDomainDao() - httpsUpgrader = HttpsUpgrader(dao) + httpsUpgrader = HttpsUpgraderImpl(dao) } @Test diff --git a/app/src/androidTest/java/com/duckduckgo/app/surrogates/ResourceSurrogateLoaderTest.kt b/app/src/androidTest/java/com/duckduckgo/app/surrogates/ResourceSurrogateLoaderTest.kt new file mode 100644 index 000000000000..06555435a080 --- /dev/null +++ b/app/src/androidTest/java/com/duckduckgo/app/surrogates/ResourceSurrogateLoaderTest.kt @@ -0,0 +1,116 @@ +/* + * Copyright (c) 2018 DuckDuckGo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.duckduckgo.app.surrogates + +import android.support.test.InstrumentationRegistry +import com.duckduckgo.app.surrogates.store.ResourceSurrogateDataStore +import org.junit.Assert +import org.junit.Assert.assertEquals +import org.junit.Before +import org.junit.Test + +class ResourceSurrogateLoaderTest { + + private lateinit var testee: ResourceSurrogateLoader + private lateinit var dataStore: ResourceSurrogateDataStore + private lateinit var resourceSurrogates: ResourceSurrogates + + @Before + fun setup() { + resourceSurrogates = ResourceSurrogatesImpl() + dataStore = ResourceSurrogateDataStore(InstrumentationRegistry.getTargetContext()) + testee = ResourceSurrogateLoader(resourceSurrogates, dataStore) + } + + @Test + fun whenLoading6SurrogatesThen6SurrogatesFound() { + val surrogates = initialiseFile("surrogates_6") + Assert.assertEquals(6, surrogates.size) + } + + @Test + fun whenLoading1SurrogateThen1SurrogateFound() { + val surrogates = initialiseFile("surrogates_1") + Assert.assertEquals(1, surrogates.size) + } + + @Test + fun whenLoadingWithNoEmptyLineAtEndOfFileThenLastSurrogateStillFound() { + val surrogates = initialiseFile("surrogates_no_empty_line_at_end_of_file") + assertEquals("googletagmanager.com/gtm.js", surrogates[5].name) + } + + @Test + fun whenLoadingWithEmptyLineAtEndOfFileThenLastSurrogateStillFound() { + val surrogates = initialiseFile("surrogates_with_empty_line_at_end_of_file") + assertEquals("googletagmanager.com/gtm.js", surrogates[5].name) + } + + @Test + fun whenLoadingMultipleSurrogatesThenOrderIsPreserved() { + val surrogates = initialiseFile("surrogates_6") + assertEquals("google-analytics.com/ga.js", surrogates[0].name) + assertEquals("google-analytics.com/analytics.js", surrogates[1].name) + assertEquals("google-analytics.com/inpage_linkid.js", surrogates[2].name) + assertEquals("google-analytics.com/cx/api.js", surrogates[3].name) + assertEquals("googletagservices.com/gpt.js", surrogates[4].name) + assertEquals("googletagmanager.com/gtm.js", surrogates[5].name) + } + + @Test + fun whenLoadingSurrogateThenMimeTypeIsPreserved() { + val surrogates = initialiseFile("surrogates_with_different_mime_types") + assertEquals("text/plain", surrogates[0].mimeType) + assertEquals("application/javascript", surrogates[1].mimeType) + assertEquals("application/json", surrogates[2].mimeType) + } + + @Test + fun whenLoadingSurrogateThenFunctionLengthIsPreserved() { + val surrogates = initialiseFile("surrogates_6") + val actualNumberOfLines = surrogates[0].jsFunction.reader().readLines().size + assertEquals(3, actualNumberOfLines) + } + + @Test + fun whenLoadingSurrogateThenFunctionLengthIsPreservedJavascriptCommentsArePreserved() { + val surrogates = initialiseFile("surrogates_6") + val actualNumberOfLines = surrogates[1].jsFunction.reader().readLines().size + assertEquals(5, actualNumberOfLines) + } + + @Test + fun whenSurrogateFileIsMissingMimeTypeEmptyListReturned() { + val surrogates = initialiseFile("surrogates_invalid_format_missing_mimetypes") + assertEquals(0, surrogates.size) + } + + @Test + fun whenSurrogateFileIsHasSpaceInFinalFunctionBlock() { + val surrogates = initialiseFile("surrogates_valid_but_unexpected_extra_space_in_function_close") + assertEquals(6, surrogates.size) + } + + private fun initialiseFile(filename: String) : List { + return testee.convertBytes(readFile(filename)) + } + + private fun readFile(filename: String): ByteArray { + return javaClass.classLoader.getResource("binary/surrogates/$filename").readBytes() + } + +} \ No newline at end of file diff --git a/app/src/androidTest/java/com/duckduckgo/app/surrogates/ResourceSurrogatesTest.kt b/app/src/androidTest/java/com/duckduckgo/app/surrogates/ResourceSurrogatesTest.kt new file mode 100644 index 000000000000..cc415121ca96 --- /dev/null +++ b/app/src/androidTest/java/com/duckduckgo/app/surrogates/ResourceSurrogatesTest.kt @@ -0,0 +1,76 @@ +/* + * Copyright (c) 2018 DuckDuckGo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.duckduckgo.app.surrogates + +import android.net.Uri +import org.junit.Assert.* +import org.junit.Before +import org.junit.Test + +class ResourceSurrogatesTest { + + private lateinit var testee: ResourceSurrogates + + @Before + fun setup() { + testee = ResourceSurrogatesImpl() + } + + @Test + fun whenInitialisedThenHasNoSurrogatesLoaded() { + assertEquals(0, testee.getAll().size) + } + + @Test + fun whenOneSurrogateLoadedThenOneReturnedFromFullList() { + val surrogate = SurrogateResponse() + testee.loadSurrogates(listOf(surrogate)) + assertEquals(1, testee.getAll().size) + } + + @Test + fun whenMultipleSurrogatesLoadedThenAllReturnedFromFullList() { + val surrogate = SurrogateResponse() + testee.loadSurrogates(listOf(surrogate, surrogate, surrogate)) + assertEquals(3, testee.getAll().size) + } + + @Test + fun whenSearchingForExactMatchingExistingSurrogateThenCanFindByName() { + val surrogate = SurrogateResponse(name = "foo") + testee.loadSurrogates(listOf(surrogate)) + val retrieved = testee.get(Uri.parse("foo")) + assertTrue(retrieved.responseAvailable) + } + + + @Test + fun whenSearchingForSubstringMatchingExistingSurrogateThenCanFindByName() { + val surrogate = SurrogateResponse(name = "foo.com") + testee.loadSurrogates(listOf(surrogate)) + val retrieved = testee.get(Uri.parse("foo.com/a/b/c")) + assertTrue(retrieved.responseAvailable) + } + + @Test + fun whenSearchingByNonExistentNameThenResponseUnavailableSurrogateResultReturned() { + val surrogate = SurrogateResponse(name = "foo") + testee.loadSurrogates(listOf(surrogate)) + val retrieved = testee.get(Uri.parse("bar")) + assertFalse(retrieved.responseAvailable) + } +} \ No newline at end of file diff --git a/app/src/androidTest/java/com/duckduckgo/app/trackerdetection/TrackerDetectorListTest.kt b/app/src/androidTest/java/com/duckduckgo/app/trackerdetection/TrackerDetectorListTest.kt index 13ae4a6496db..551b9de2f427 100644 --- a/app/src/androidTest/java/com/duckduckgo/app/trackerdetection/TrackerDetectorListTest.kt +++ b/app/src/androidTest/java/com/duckduckgo/app/trackerdetection/TrackerDetectorListTest.kt @@ -56,11 +56,11 @@ class TrackerDetectorListTest { settingStore = mock() whenever(settingStore.privacyOn).thenReturn(true) - blockingOnlyTestee = TrackerDetector(TrackerNetworks(), settingStore) + blockingOnlyTestee = TrackerDetectorImpl(TrackerNetworks(), settingStore) blockingOnlyTestee.addClient(easyprivacyAdblock) blockingOnlyTestee.addClient(easylistAdblock) - testeeWithWhitelist = TrackerDetector(TrackerNetworks(), settingStore) + testeeWithWhitelist = TrackerDetectorImpl(TrackerNetworks(), settingStore) testeeWithWhitelist.addClient(trackersWhitelistAdblocks) testeeWithWhitelist.addClient(easyprivacyAdblock) testeeWithWhitelist.addClient(easylistAdblock) diff --git a/app/src/androidTest/java/com/duckduckgo/app/trackerdetection/TrackerDetectorTest.kt b/app/src/androidTest/java/com/duckduckgo/app/trackerdetection/TrackerDetectorTest.kt index 39463055e20e..40a81ce2d517 100644 --- a/app/src/androidTest/java/com/duckduckgo/app/trackerdetection/TrackerDetectorTest.kt +++ b/app/src/androidTest/java/com/duckduckgo/app/trackerdetection/TrackerDetectorTest.kt @@ -32,7 +32,7 @@ class TrackerDetectorTest { private val networkTrackers = TrackerNetworks() private val settingStore: PrivacySettingsStore = mock() - private val trackerDetector = TrackerDetector(networkTrackers, settingStore) + private val trackerDetector = TrackerDetectorImpl(networkTrackers, settingStore) companion object { private val resourceType = ResourceType.UNKNOWN diff --git a/app/src/androidTest/resources/binary/surrogates/surrogates_1 b/app/src/androidTest/resources/binary/surrogates/surrogates_1 new file mode 100644 index 000000000000..7bd5659717de --- /dev/null +++ b/app/src/androidTest/resources/binary/surrogates/surrogates_1 @@ -0,0 +1,6 @@ +# Some comments here +# This comment is also here +google-analytics.com/ga.js application/javascript +(function() { + alert("surrogates_1"); +})(); diff --git a/app/src/androidTest/resources/binary/surrogates/surrogates_6 b/app/src/androidTest/resources/binary/surrogates/surrogates_6 new file mode 100644 index 000000000000..a9bd0b3ea49c --- /dev/null +++ b/app/src/androidTest/resources/binary/surrogates/surrogates_6 @@ -0,0 +1,51 @@ +# Some comments here +# This comment is also here +google-analytics.com/ga.js application/javascript +(function() { + alert("surrogates_1"); +})(); + +google-analytics.com/analytics.js application/javascript +(function() { + // random comment + alert("surrogates_2"); + alert("surrogates_2"); +})(); + +google-analytics.com/inpage_linkid.js application/javascript +(function() { + alert("surrogates_3"); + alert("surrogates_3"); + alert("surrogates_3"); +})(); + +# A comment +google-analytics.com/cx/api.js application/javascript +(function() { + alert("surrogates_4"); + alert("surrogates_4"); + alert("surrogates_4"); + alert("surrogates_4"); +})(); + +# A comment +# A comment +googletagservices.com/gpt.js application/javascript +(function() { + alert("surrogates_5"); + alert("surrogates_5"); + alert("surrogates_5"); + alert("surrogates_5"); + alert("surrogates_5"); +})(); + +# A comment +googletagmanager.com/gtm.js application/javascript +(function() { + alert("surrogates_6"); + alert("surrogates_6"); + alert("surrogates_6"); + alert("surrogates_6"); + alert("surrogates_6"); + alert("surrogates_6"); +})(); \ No newline at end of file diff --git a/app/src/androidTest/resources/binary/surrogates/surrogates_invalid_format_missing_mimetypes b/app/src/androidTest/resources/binary/surrogates/surrogates_invalid_format_missing_mimetypes new file mode 100644 index 000000000000..0d0cd328d25a --- /dev/null +++ b/app/src/androidTest/resources/binary/surrogates/surrogates_invalid_format_missing_mimetypes @@ -0,0 +1,14 @@ +# Some comments here +# This comment is also here +google-analytics.com/ga.js +(function() { + alert("surrogates_1"); +})(); + +google-analytics.com/analytics.js +(function() { + // random comment + alert("surrogates_2"); + alert("surrogates_2"); +})(); + diff --git a/app/src/androidTest/resources/binary/surrogates/surrogates_no_empty_line_at_end_of_file b/app/src/androidTest/resources/binary/surrogates/surrogates_no_empty_line_at_end_of_file new file mode 100644 index 000000000000..bdd2d9a9f80b --- /dev/null +++ b/app/src/androidTest/resources/binary/surrogates/surrogates_no_empty_line_at_end_of_file @@ -0,0 +1,53 @@ +# Some comments here +# This comment is also here +google-analytics.com/ga.js application/javascript +(function() { + alert("surrogates_1"); +})(); + +google-analytics.com/analytics.js application/javascript +(function() { + // random comment + alert("surrogates_2"); + alert("surrogates_2"); +})(); + +google-analytics.com/inpage_linkid.js application/javascript +(function() { + alert("surrogates_3"); + alert("surrogates_3"); + alert("surrogates_3"); +})(); + +# A comment +# A comment +# A comment +google-analytics.com/cx/api.js application/javascript +(function() { + alert("surrogates_4"); + alert("surrogates_4"); + alert("surrogates_4"); + alert("surrogates_4"); +})(); + +# A comment +# A comment +googletagservices.com/gpt.js application/javascript +(function() { + alert("surrogates_5"); + alert("surrogates_5"); + alert("surrogates_5"); + alert("surrogates_5"); + alert("surrogates_5"); +})(); + +# A comment +googletagmanager.com/gtm.js application/javascript +(function() { + alert("surrogates_6"); + alert("surrogates_6"); + alert("surrogates_6"); + alert("surrogates_6"); + alert("surrogates_6"); + alert("surrogates_6"); +})(); \ No newline at end of file diff --git a/app/src/androidTest/resources/binary/surrogates/surrogates_valid_but_unexpected_extra_space_in_function_close b/app/src/androidTest/resources/binary/surrogates/surrogates_valid_but_unexpected_extra_space_in_function_close new file mode 100644 index 000000000000..a440603f0841 --- /dev/null +++ b/app/src/androidTest/resources/binary/surrogates/surrogates_valid_but_unexpected_extra_space_in_function_close @@ -0,0 +1,51 @@ +# Some comments here +# This comment is also here +google-analytics.com/ga.js application/javascript +(function() { + alert("surrogates_1"); +}) (); + +google-analytics.com/analytics.js application/javascript +(function() { + // random comment + alert("surrogates_2"); + alert("surrogates_2"); +}) (); + +google-analytics.com/inpage_linkid.js application/javascript +(function() { + alert("surrogates_3"); + alert("surrogates_3"); + alert("surrogates_3"); +}) (); + +# A comment +google-analytics.com/cx/api.js application/javascript +(function() { + alert("surrogates_4"); + alert("surrogates_4"); + alert("surrogates_4"); + alert("surrogates_4"); +}) (); + +# A comment +# A comment +googletagservices.com/gpt.js application/javascript +(function() { + alert("surrogates_5"); + alert("surrogates_5"); + alert("surrogates_5"); + alert("surrogates_5"); + alert("surrogates_5"); +}) (); + +# A comment +googletagmanager.com/gtm.js application/javascript +(function() { + alert("surrogates_6"); + alert("surrogates_6"); + alert("surrogates_6"); + alert("surrogates_6"); + alert("surrogates_6"); + alert("surrogates_6"); +}) (); \ No newline at end of file diff --git a/app/src/androidTest/resources/binary/surrogates/surrogates_with_different_mime_types b/app/src/androidTest/resources/binary/surrogates/surrogates_with_different_mime_types new file mode 100644 index 000000000000..b09dacecb855 --- /dev/null +++ b/app/src/androidTest/resources/binary/surrogates/surrogates_with_different_mime_types @@ -0,0 +1,52 @@ +# Some comments here +# This comment is also here +google-analytics.com/ga.js text/plain +(function() { + alert("surrogates_1"); +})(); + +google-analytics.com/analytics.js application/javascript +(function() { + // random comment + alert("surrogates_2"); + alert("surrogates_2"); +})(); + +google-analytics.com/inpage_linkid.js application/json +(function() { + alert("surrogates_3"); + alert("surrogates_3"); + alert("surrogates_3"); +})(); + +# A comment +# A comment +google-analytics.com/cx/api.js application/javascript +(function() { + alert("surrogates_4"); + alert("surrogates_4"); + alert("surrogates_4"); + alert("surrogates_4"); +})(); + +# A comment +googletagservices.com/gpt.js application/javascript +(function() { + alert("surrogates_5"); + alert("surrogates_5"); + alert("surrogates_5"); + alert("surrogates_5"); + alert("surrogates_5"); +})(); + +# A comment +# A comment +googletagmanager.com/gtm.js application/javascript +(function() { + alert("surrogates_6"); + alert("surrogates_6"); + alert("surrogates_6"); + alert("surrogates_6"); + alert("surrogates_6"); + alert("surrogates_6"); +})(); \ No newline at end of file diff --git a/app/src/androidTest/resources/binary/surrogates/surrogates_with_empty_line_at_end_of_file b/app/src/androidTest/resources/binary/surrogates/surrogates_with_empty_line_at_end_of_file new file mode 100644 index 000000000000..b8454585c7c1 --- /dev/null +++ b/app/src/androidTest/resources/binary/surrogates/surrogates_with_empty_line_at_end_of_file @@ -0,0 +1,51 @@ +# Some comments here +# This comment is also here +google-analytics.com/ga.js application/javascript +(function() { + alert("surrogates_1"); +})(); + +google-analytics.com/analytics.js application/javascript +(function() { + // random comment + alert("surrogates_2"); + alert("surrogates_2"); +})(); + +google-analytics.com/inpage_linkid.js application/javascript +(function() { + alert("surrogates_3"); + alert("surrogates_3"); + alert("surrogates_3"); +})(); + +# A comment +google-analytics.com/cx/api.js application/javascript +(function() { + alert("surrogates_4"); + alert("surrogates_4"); + alert("surrogates_4"); + alert("surrogates_4"); +})(); + +# A comment +# A comment +googletagservices.com/gpt.js application/javascript +(function() { + alert("surrogates_5"); + alert("surrogates_5"); + alert("surrogates_5"); + alert("surrogates_5"); + alert("surrogates_5"); +})(); + +# A comment +googletagmanager.com/gtm.js application/javascript +(function() { + alert("surrogates_6"); + alert("surrogates_6"); + alert("surrogates_6"); + alert("surrogates_6"); + alert("surrogates_6"); + alert("surrogates_6"); +})(); diff --git a/app/src/main/java/com/duckduckgo/app/browser/BrowserWebViewClient.kt b/app/src/main/java/com/duckduckgo/app/browser/BrowserWebViewClient.kt index 7c842fbf999a..78c7a5aacd6c 100644 --- a/app/src/main/java/com/duckduckgo/app/browser/BrowserWebViewClient.kt +++ b/app/src/main/java/com/duckduckgo/app/browser/BrowserWebViewClient.kt @@ -18,27 +18,19 @@ package com.duckduckgo.app.browser import android.graphics.Bitmap import android.net.Uri -import android.support.annotation.AnyThread import android.support.annotation.WorkerThread import android.webkit.WebResourceRequest import android.webkit.WebResourceResponse import android.webkit.WebView import android.webkit.WebViewClient -import com.duckduckgo.app.global.isHttp -import com.duckduckgo.app.httpsupgrade.HttpsUpgrader -import com.duckduckgo.app.privacymonitor.model.TrustedSites -import com.duckduckgo.app.trackerdetection.TrackerDetector -import com.duckduckgo.app.trackerdetection.model.ResourceType import timber.log.Timber -import java.util.concurrent.CountDownLatch import javax.inject.Inject class BrowserWebViewClient @Inject constructor( - private val requestRewriter: DuckDuckGoRequestRewriter, - private var trackerDetector: TrackerDetector, - private var httpsUpgrader: HttpsUpgrader, - private val specialUrlDetector: SpecialUrlDetector + private val requestRewriter: RequestRewriter, + private val specialUrlDetector: SpecialUrlDetector, + private val webViewRequestInterceptor: WebViewRequestInterceptor ) : WebViewClient() { var webViewClientListener: WebViewClientListener? = null @@ -94,45 +86,9 @@ class BrowserWebViewClient @Inject constructor( } @WorkerThread - override fun shouldInterceptRequest(view: WebView, request: WebResourceRequest): WebResourceResponse? { - Timber.v("Intercepting resource ${request.url} on page ${currentUrl}}") - - if (shouldUpgrade(request)) { - val newUri = httpsUpgrader.upgrade(request.url) - view.post { view.loadUrl(newUri.toString()) } - return WebResourceResponse(null, null, null) - } - - val documentUrl = currentUrl ?: return null - - if (TrustedSites.isTrusted(documentUrl)) { - return null - } - - if (request.url != null && request.url.isHttp) { - webViewClientListener?.pageHasHttpResources(documentUrl) - } - - if (shouldBlock(request, documentUrl)) { - return WebResourceResponse(null, null, null) - } - - return null - } - - private fun shouldUpgrade(request: WebResourceRequest) = - request.isForMainFrame && request.url != null && httpsUpgrader.shouldUpgrade(request.url) - - private fun shouldBlock(request: WebResourceRequest, documentUrl: String?): Boolean { - val url = request.url.toString() - - if (request.isForMainFrame || documentUrl == null) { - return false - } - - val trackingEvent = trackerDetector.evaluate(url, documentUrl, ResourceType.from(request)) ?: return false - webViewClientListener?.trackerDetected(trackingEvent) - return trackingEvent.blocked + override fun shouldInterceptRequest(webView: WebView, request: WebResourceRequest): WebResourceResponse? { + Timber.v("Intercepting resource ${request.url} on page $currentUrl") + return webViewRequestInterceptor.shouldIntercept(request, webView, currentUrl, webViewClientListener) } /** diff --git a/app/src/main/java/com/duckduckgo/app/browser/DuckDuckGoRequestRewriter.kt b/app/src/main/java/com/duckduckgo/app/browser/DuckDuckGoRequestRewriter.kt index bc8e4db264c9..8b31e140e07f 100644 --- a/app/src/main/java/com/duckduckgo/app/browser/DuckDuckGoRequestRewriter.kt +++ b/app/src/main/java/com/duckduckgo/app/browser/DuckDuckGoRequestRewriter.kt @@ -18,9 +18,14 @@ package com.duckduckgo.app.browser import android.net.Uri import timber.log.Timber -import javax.inject.Inject -class DuckDuckGoRequestRewriter @Inject constructor(private val duckDuckGoUrlDetector: DuckDuckGoUrlDetector) { +interface RequestRewriter { + fun shouldRewriteRequest(uri: Uri): Boolean + fun rewriteRequestWithCustomQueryParams(request: Uri): Uri + fun addCustomQueryParams(builder: Uri.Builder) +} + +class DuckDuckGoRequestRewriter(private val duckDuckGoUrlDetector: DuckDuckGoUrlDetector) : RequestRewriter { companion object { private const val sourceParam = "t" @@ -28,7 +33,7 @@ class DuckDuckGoRequestRewriter @Inject constructor(private val duckDuckGoUrlDet private const val querySource = "ddg_android" } - fun rewriteRequestWithCustomQueryParams(request: Uri): Uri { + override fun rewriteRequestWithCustomQueryParams(request: Uri): Uri { val builder = Uri.Builder() .authority(request.authority) .scheme(request.scheme) @@ -46,10 +51,12 @@ class DuckDuckGoRequestRewriter @Inject constructor(private val duckDuckGoUrlDet return newUri } - fun shouldRewriteRequest(uri: Uri): Boolean = - duckDuckGoUrlDetector.isDuckDuckGoUrl(uri) && !uri.queryParameterNames.containsAll(arrayListOf(sourceParam, appVersionParam)) + override fun shouldRewriteRequest(uri: Uri): Boolean { + return duckDuckGoUrlDetector.isDuckDuckGoUrl(uri) && + !uri.queryParameterNames.containsAll(arrayListOf(sourceParam, appVersionParam)) + } - fun addCustomQueryParams(builder: Uri.Builder) { + override fun addCustomQueryParams(builder: Uri.Builder) { builder.appendQueryParameter(appVersionParam, formatAppVersion()) builder.appendQueryParameter(sourceParam, querySource) } diff --git a/app/src/main/java/com/duckduckgo/app/browser/WebViewRequestInterceptor.kt b/app/src/main/java/com/duckduckgo/app/browser/WebViewRequestInterceptor.kt new file mode 100644 index 000000000000..b1fa52b33ab1 --- /dev/null +++ b/app/src/main/java/com/duckduckgo/app/browser/WebViewRequestInterceptor.kt @@ -0,0 +1,112 @@ +/* + * Copyright (c) 2018 DuckDuckGo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.duckduckgo.app.browser + +import android.support.annotation.WorkerThread +import android.webkit.WebResourceRequest +import android.webkit.WebResourceResponse +import android.webkit.WebView +import com.duckduckgo.app.surrogates.ResourceSurrogates +import com.duckduckgo.app.global.isHttp +import com.duckduckgo.app.httpsupgrade.HttpsUpgrader +import com.duckduckgo.app.privacymonitor.model.TrustedSites +import com.duckduckgo.app.trackerdetection.TrackerDetector +import com.duckduckgo.app.trackerdetection.model.ResourceType +import timber.log.Timber +import javax.inject.Inject + + +class WebViewRequestInterceptor @Inject constructor( + private val resourceSurrogates: ResourceSurrogates, + private val trackerDetector: TrackerDetector, + private val httpsUpgrader: HttpsUpgrader +) { + + /** + * Notify the application of a resource request and allow the application to return the data. + * + * If the return value is null, the WebView will continue to load the resource as usual. + * Otherwise, the return response and data will be used. + * + * NOTE: This method is called on a thread other than the UI thread so clients should exercise + * caution when accessing private data or the view system. + */ + @WorkerThread + fun shouldIntercept( + request: WebResourceRequest, + webView: WebView, + currentUrl: String?, + webViewClientListener: WebViewClientListener? + ): WebResourceResponse? { + val url = request.url + + if (shouldUpgrade(request)) { + val newUri = httpsUpgrader.upgrade(url) + webView.post { webView.loadUrl(newUri.toString()) } + return WebResourceResponse(null, null, null) + } + + val documentUrl = currentUrl ?: return null + + if (TrustedSites.isTrusted(documentUrl)) { + return null + } + + if (url != null && url.isHttp) { + webViewClientListener?.pageHasHttpResources(documentUrl) + } + + if (shouldBlock(request, documentUrl, webViewClientListener)) { + + val surrogate = resourceSurrogates.get(url) + if (surrogate.responseAvailable) { + Timber.v("Surrogate found for %s", url) + return WebResourceResponse( + surrogate.mimeType, + "UTF-8", + surrogate.jsFunction.byteInputStream() + ) + } + + return WebResourceResponse(null, null, null) + } + + return null + } + + private fun shouldUpgrade(request: WebResourceRequest) = + request.isForMainFrame && request.url != null && httpsUpgrader.shouldUpgrade(request.url) + + private fun shouldBlock( + request: WebResourceRequest, + documentUrl: String?, + webViewClientListener: WebViewClientListener? + ): Boolean { + val url = request.url.toString() + + if (request.isForMainFrame || documentUrl == null) { + return false + } + + val trackingEvent = + trackerDetector.evaluate(url, documentUrl, ResourceType.from(request)) + ?: return false + webViewClientListener?.trackerDetected(trackingEvent) + return trackingEvent.blocked + } + +} \ No newline at end of file diff --git a/app/src/main/java/com/duckduckgo/app/browser/di/BrowserModule.kt b/app/src/main/java/com/duckduckgo/app/browser/di/BrowserModule.kt new file mode 100644 index 000000000000..349d2308249f --- /dev/null +++ b/app/src/main/java/com/duckduckgo/app/browser/di/BrowserModule.kt @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2018 DuckDuckGo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.duckduckgo.app.browser.di + +import android.webkit.CookieManager +import com.duckduckgo.app.browser.DuckDuckGoRequestRewriter +import com.duckduckgo.app.browser.DuckDuckGoUrlDetector +import com.duckduckgo.app.browser.RequestRewriter +import dagger.Module +import dagger.Provides + +@Module +class BrowserModule { + + @Provides + fun cookieManager(): CookieManager = CookieManager.getInstance() + + @Provides + fun duckDuckGoRequestRewriter(urlDetector: DuckDuckGoUrlDetector): RequestRewriter { + return DuckDuckGoRequestRewriter(urlDetector) + } +} \ No newline at end of file diff --git a/app/src/main/java/com/duckduckgo/app/browser/omnibar/QueryUrlConverter.kt b/app/src/main/java/com/duckduckgo/app/browser/omnibar/QueryUrlConverter.kt index 04be914a2fac..bf8c5e8278cb 100644 --- a/app/src/main/java/com/duckduckgo/app/browser/omnibar/QueryUrlConverter.kt +++ b/app/src/main/java/com/duckduckgo/app/browser/omnibar/QueryUrlConverter.kt @@ -18,13 +18,13 @@ package com.duckduckgo.app.browser.omnibar import android.net.Uri import android.support.v4.util.PatternsCompat -import com.duckduckgo.app.browser.DuckDuckGoRequestRewriter +import com.duckduckgo.app.browser.RequestRewriter import com.duckduckgo.app.global.UrlScheme.Companion.http import com.duckduckgo.app.global.UrlScheme.Companion.https import com.duckduckgo.app.global.withScheme import javax.inject.Inject -class QueryUrlConverter @Inject constructor(private val requestRewriter: DuckDuckGoRequestRewriter) : OmnibarEntryConverter { +class QueryUrlConverter @Inject constructor(private val requestRewriter: RequestRewriter) : OmnibarEntryConverter { companion object { private const val baseUrl = "duckduckgo.com" diff --git a/app/src/main/java/com/duckduckgo/app/di/AppComponent.kt b/app/src/main/java/com/duckduckgo/app/di/AppComponent.kt index b451e1e53c04..8fce1e22f2d0 100644 --- a/app/src/main/java/com/duckduckgo/app/di/AppComponent.kt +++ b/app/src/main/java/com/duckduckgo/app/di/AppComponent.kt @@ -18,8 +18,12 @@ package com.duckduckgo.app.di import android.app.Application +import com.duckduckgo.app.surrogates.di.ResourceSurrogateModule import com.duckduckgo.app.browser.autoComplete.BrowserAutoCompleteModule +import com.duckduckgo.app.browser.di.BrowserModule import com.duckduckgo.app.global.DuckDuckGoApplication +import com.duckduckgo.app.httpsupgrade.di.HttpsUpgraderModule +import com.duckduckgo.app.trackerdetection.di.TrackerDetectionModule import dagger.BindsInstance import dagger.Component import dagger.android.AndroidInjector @@ -39,7 +43,10 @@ import javax.inject.Singleton (JsonModule::class), (StringModule::class), (BrowserModule::class), - (BrowserAutoCompleteModule::class) + (BrowserAutoCompleteModule::class), + (HttpsUpgraderModule::class), + (ResourceSurrogateModule::class), + (TrackerDetectionModule::class) ]) interface AppComponent : AndroidInjector { diff --git a/app/src/main/java/com/duckduckgo/app/di/NetworkModule.kt b/app/src/main/java/com/duckduckgo/app/di/NetworkModule.kt index 25605c9749b6..4c286f37d3bd 100644 --- a/app/src/main/java/com/duckduckgo/app/di/NetworkModule.kt +++ b/app/src/main/java/com/duckduckgo/app/di/NetworkModule.kt @@ -17,6 +17,7 @@ package com.duckduckgo.app.di import android.content.Context +import com.duckduckgo.app.surrogates.api.ResourceSurrogateListService import com.duckduckgo.app.autocomplete.api.AutoCompleteService import com.duckduckgo.app.browser.R import com.duckduckgo.app.httpsupgrade.api.HttpsUpgradeListService @@ -68,6 +69,10 @@ class NetworkModule { fun autoCompleteService(retrofit: Retrofit): AutoCompleteService = retrofit.create(AutoCompleteService::class.java) + @Provides + fun surrogatesService(retrofit: Retrofit): ResourceSurrogateListService = + retrofit.create(ResourceSurrogateListService::class.java) + companion object { private const val CACHE_SIZE: Long = 10 * 1024 * 1024 // 10MB } diff --git a/app/src/main/java/com/duckduckgo/app/global/DuckDuckGoApplication.kt b/app/src/main/java/com/duckduckgo/app/global/DuckDuckGoApplication.kt index bf68d7d1b953..ee0c9cd1f7ac 100644 --- a/app/src/main/java/com/duckduckgo/app/global/DuckDuckGoApplication.kt +++ b/app/src/main/java/com/duckduckgo/app/global/DuckDuckGoApplication.kt @@ -19,6 +19,7 @@ package com.duckduckgo.app.global import android.app.Activity import android.app.Application import android.app.Service +import com.duckduckgo.app.surrogates.ResourceSurrogateLoader import com.duckduckgo.app.browser.BuildConfig import com.duckduckgo.app.di.DaggerAppComponent import com.duckduckgo.app.job.AppConfigurationSyncer @@ -48,6 +49,9 @@ class DuckDuckGoApplication : HasActivityInjector, HasServiceInjector, Applicati @Inject lateinit var trackerDataLoader: TrackerDataLoader + @Inject + lateinit var resourceSurrogateLoader: ResourceSurrogateLoader + @Inject lateinit var appConfigurationSyncer: AppConfigurationSyncer @@ -86,7 +90,10 @@ class DuckDuckGoApplication : HasActivityInjector, HasServiceInjector, Applicati } private fun loadTrackerData() { - Schedulers.io().scheduleDirect { trackerDataLoader.loadData() } + doAsync { + trackerDataLoader.loadData() + resourceSurrogateLoader.loadData() + } } private fun configureLogging() { diff --git a/app/src/main/java/com/duckduckgo/app/httpsupgrade/HttpsUpgrader.kt b/app/src/main/java/com/duckduckgo/app/httpsupgrade/HttpsUpgrader.kt index b4ee24636b6e..6afa567039ed 100644 --- a/app/src/main/java/com/duckduckgo/app/httpsupgrade/HttpsUpgrader.kt +++ b/app/src/main/java/com/duckduckgo/app/httpsupgrade/HttpsUpgrader.kt @@ -22,12 +22,21 @@ import com.duckduckgo.app.global.UrlScheme import com.duckduckgo.app.global.isHttps import com.duckduckgo.app.httpsupgrade.db.HttpsUpgradeDomainDao import timber.log.Timber -import javax.inject.Inject -class HttpsUpgrader @Inject constructor(private val dao: HttpsUpgradeDomainDao) { +interface HttpsUpgrader { @WorkerThread - fun shouldUpgrade(uri: Uri) : Boolean { + fun shouldUpgrade(uri: Uri) : Boolean + + fun upgrade(uri: Uri): Uri { + return uri.buildUpon().scheme(UrlScheme.https).build() + } +} + +class HttpsUpgraderImpl constructor(private val dao: HttpsUpgradeDomainDao) :HttpsUpgrader { + + @WorkerThread + override fun shouldUpgrade(uri: Uri) : Boolean { if (uri.isHttps) { return false } @@ -36,10 +45,6 @@ class HttpsUpgrader @Inject constructor(private val dao: HttpsUpgradeDomainDao) return dao.hasDomain(host) || matchesWildcard(host) } - fun upgrade(uri: Uri): Uri { - return uri.buildUpon().scheme(UrlScheme.https).build() - } - private fun matchesWildcard(host: String): Boolean { val domains = mutableListOf() for (part in host.split(".").reversed()) { diff --git a/app/src/main/java/com/duckduckgo/app/httpsupgrade/api/HttpsUpgradeListDownloader.kt b/app/src/main/java/com/duckduckgo/app/httpsupgrade/api/HttpsUpgradeListDownloader.kt index 445aa93862c4..ad7747ad4d3e 100644 --- a/app/src/main/java/com/duckduckgo/app/httpsupgrade/api/HttpsUpgradeListDownloader.kt +++ b/app/src/main/java/com/duckduckgo/app/httpsupgrade/api/HttpsUpgradeListDownloader.kt @@ -31,7 +31,7 @@ class HttpsUpgradeListDownloader @Inject constructor( fun downloadList(): Completable { - Timber.i("Downloading HTTPS Upgrade data") + Timber.d("Downloading HTTPS Upgrade data") return Completable.fromAction { @@ -39,7 +39,7 @@ class HttpsUpgradeListDownloader @Inject constructor( val response = call.execute() if (response.isCached && httpsUpgradeDao.count() != 0) { - Timber.i("HTTPS data already cached and stored") + Timber.d("HTTPS data already cached and stored") return@fromAction } diff --git a/app/src/main/java/com/duckduckgo/app/httpsupgrade/di/HttpsUpgraderModule.kt b/app/src/main/java/com/duckduckgo/app/httpsupgrade/di/HttpsUpgraderModule.kt new file mode 100644 index 000000000000..8afee05bdff1 --- /dev/null +++ b/app/src/main/java/com/duckduckgo/app/httpsupgrade/di/HttpsUpgraderModule.kt @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2018 DuckDuckGo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.duckduckgo.app.httpsupgrade.di + +import com.duckduckgo.app.httpsupgrade.HttpsUpgrader +import com.duckduckgo.app.httpsupgrade.HttpsUpgraderImpl +import com.duckduckgo.app.httpsupgrade.db.HttpsUpgradeDomainDao +import dagger.Module +import dagger.Provides + +@Module +class HttpsUpgraderModule { + + @Provides + fun httpsUpgrader(dao: HttpsUpgradeDomainDao): HttpsUpgrader { + return HttpsUpgraderImpl(dao) + } +} \ No newline at end of file diff --git a/app/src/main/java/com/duckduckgo/app/job/AppConfigurationDownloader.kt b/app/src/main/java/com/duckduckgo/app/job/AppConfigurationDownloader.kt index 64ad2ad061de..5cce7364cf3d 100644 --- a/app/src/main/java/com/duckduckgo/app/job/AppConfigurationDownloader.kt +++ b/app/src/main/java/com/duckduckgo/app/job/AppConfigurationDownloader.kt @@ -16,10 +16,11 @@ package com.duckduckgo.app.job +import com.duckduckgo.app.surrogates.api.ResourceSurrogateListDownloader import com.duckduckgo.app.global.db.AppDatabase import com.duckduckgo.app.httpsupgrade.api.HttpsUpgradeListDownloader import com.duckduckgo.app.settings.db.AppConfigurationEntity -import com.duckduckgo.app.trackerdetection.Client +import com.duckduckgo.app.trackerdetection.Client.ClientName.* import com.duckduckgo.app.trackerdetection.api.TrackerDataDownloader import io.reactivex.Completable import io.reactivex.schedulers.Schedulers @@ -29,13 +30,15 @@ import javax.inject.Inject class AppConfigurationDownloader @Inject constructor( private val trackerDataDownloader: TrackerDataDownloader, private val httpsUpgradeListDownloader: HttpsUpgradeListDownloader, + private val resourceSurrogateDownloader: ResourceSurrogateListDownloader, private val appDatabase: AppDatabase) { fun downloadTask(): Completable { - val easyListDownload = trackerDataDownloader.downloadList(Client.ClientName.EASYLIST) - val easyPrivacyDownload = trackerDataDownloader.downloadList(Client.ClientName.EASYPRIVACY) - val trackersWhitelist = trackerDataDownloader.downloadList(Client.ClientName.TRACKERSWHITELIST) - val disconnectDownload = trackerDataDownloader.downloadList(Client.ClientName.DISCONNECT) + val easyListDownload = trackerDataDownloader.downloadList(EASYLIST) + val easyPrivacyDownload = trackerDataDownloader.downloadList(EASYPRIVACY) + val trackersWhitelist = trackerDataDownloader.downloadList(TRACKERSWHITELIST) + val disconnectDownload = trackerDataDownloader.downloadList(DISCONNECT) + val surrogatesDownload = resourceSurrogateDownloader.downloadList() val httpsUpgradeDownload = httpsUpgradeListDownloader.downloadList() return Completable.mergeDelayError(listOf( @@ -43,6 +46,7 @@ class AppConfigurationDownloader @Inject constructor( easyPrivacyDownload.subscribeOn(Schedulers.io()), trackersWhitelist.subscribeOn(Schedulers.io()), disconnectDownload.subscribeOn(Schedulers.io()), + surrogatesDownload.subscribeOn(Schedulers.io()), httpsUpgradeDownload.subscribeOn(Schedulers.io()) )).doOnComplete { Timber.i("Download task completed successfully") diff --git a/app/src/main/java/com/duckduckgo/app/surrogates/ResourceSurrogateLoader.kt b/app/src/main/java/com/duckduckgo/app/surrogates/ResourceSurrogateLoader.kt new file mode 100644 index 000000000000..f2cd69aff792 --- /dev/null +++ b/app/src/main/java/com/duckduckgo/app/surrogates/ResourceSurrogateLoader.kt @@ -0,0 +1,102 @@ +/* + * Copyright (c) 2018 DuckDuckGo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.duckduckgo.app.surrogates + +import android.support.annotation.WorkerThread +import com.duckduckgo.app.surrogates.store.ResourceSurrogateDataStore +import timber.log.Timber +import java.io.ByteArrayInputStream +import javax.inject.Inject + +@WorkerThread +class ResourceSurrogateLoader @Inject constructor( + private val resourceSurrogates: ResourceSurrogates, + private val surrogatesDataStore: ResourceSurrogateDataStore +) { + + fun loadData() { + if (surrogatesDataStore.hasData()) { + val bytes = surrogatesDataStore.loadData() + resourceSurrogates.loadSurrogates(convertBytes(bytes)) + } + } + + fun convertBytes(bytes: ByteArray): List { + return try { + parse(bytes) + } catch (e: Throwable) { + Timber.w(e, "Failed to parse surrogates file; file may be corrupt or badly formatted") + emptyList() + } + } + + private fun parse(bytes: ByteArray): List { + val surrogates = mutableListOf() + + val reader = ByteArrayInputStream(bytes).bufferedReader() + val existingLines = reader.readLines().toMutableList() + + if (existingLines.isNotEmpty() && existingLines.last().isNotBlank()) { + existingLines.add("") + } + + var nextLineIsNewRule = true + + var ruleName = "" + var mimeType = "" + val functionBuilder = StringBuilder() + + existingLines.forEach { + + if (it.startsWith("#")) { + return@forEach + } + + if (nextLineIsNewRule) { + + with(it.split(" ")) { + ruleName = this[0] + mimeType = this[1] + } + Timber.d("Found new surrogate rule: %s - %s", ruleName, mimeType) + nextLineIsNewRule = false + return@forEach + } + + if (it.isBlank()) { + surrogates.add( + SurrogateResponse( + name = ruleName, + mimeType = mimeType, + jsFunction = functionBuilder.toString() + ) + ) + + functionBuilder.setLength(0) + + nextLineIsNewRule = true + return@forEach + } + + functionBuilder.append(it) + functionBuilder.append("\n") + } + + Timber.d("Processed %d surrogates", surrogates.size) + return surrogates + } +} \ No newline at end of file diff --git a/app/src/main/java/com/duckduckgo/app/surrogates/ResourceSurrogates.kt b/app/src/main/java/com/duckduckgo/app/surrogates/ResourceSurrogates.kt new file mode 100644 index 000000000000..8856b7e8b7dc --- /dev/null +++ b/app/src/main/java/com/duckduckgo/app/surrogates/ResourceSurrogates.kt @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2018 DuckDuckGo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.duckduckgo.app.surrogates + +import android.net.Uri + +interface ResourceSurrogates { + fun loadSurrogates(urls: List) + fun get(uri: Uri): SurrogateResponse + fun getAll(): List +} + +class ResourceSurrogatesImpl : ResourceSurrogates { + + private val surrogates = mutableListOf() + + override fun loadSurrogates(urls: List) { + surrogates.clear() + surrogates.addAll(urls) + } + + override fun get(uri: Uri): SurrogateResponse { + val uriString = uri.toString() + + return surrogates.find { uriString.contains(it.name) } + ?: return SurrogateResponse(responseAvailable = false) + } + + override fun getAll(): List { + return surrogates + } +} + +data class SurrogateResponse( + val responseAvailable: Boolean = true, + val name: String = "", + val jsFunction: String = "", + val mimeType: String = "" +) diff --git a/app/src/main/java/com/duckduckgo/app/surrogates/api/ResourceSurrogateListDownloader.kt b/app/src/main/java/com/duckduckgo/app/surrogates/api/ResourceSurrogateListDownloader.kt new file mode 100644 index 000000000000..c7c67c3a57fd --- /dev/null +++ b/app/src/main/java/com/duckduckgo/app/surrogates/api/ResourceSurrogateListDownloader.kt @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2018 DuckDuckGo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.duckduckgo.app.surrogates.api + +import com.duckduckgo.app.surrogates.ResourceSurrogateLoader +import com.duckduckgo.app.surrogates.store.ResourceSurrogateDataStore +import com.duckduckgo.app.global.api.isCached +import io.reactivex.Completable +import timber.log.Timber +import java.io.IOException +import javax.inject.Inject + + +class ResourceSurrogateListDownloader @Inject constructor( + private val service: ResourceSurrogateListService, + private val surrogatesDataStore: ResourceSurrogateDataStore, + private val resourceSurrogateLoader: ResourceSurrogateLoader +) { + + fun downloadList(): Completable { + + return Completable.fromAction { + + Timber.d("Downloading Google Analytics Surrogates data") + + val call = service.https() + val response = call.execute() + + Timber.d("Response received, success=${response.isSuccessful}") + + if (response.isCached && surrogatesDataStore.hasData()) { + Timber.d("Surrogates data already cached and stored") + return@fromAction + } + + if (response.isSuccessful) { + val bodyBytes = response.body()!!.bytes() + Timber.d("Updating surrogates data store with new data") + persistData(bodyBytes) + resourceSurrogateLoader.loadData() + } else { + throw IOException("Status: ${response.code()} - ${response.errorBody()?.string()}") + } + } + } + + private fun persistData(bodyBytes: ByteArray) { + surrogatesDataStore.saveData(bodyBytes) + } +} \ No newline at end of file diff --git a/app/src/main/java/com/duckduckgo/app/di/BrowserModule.kt b/app/src/main/java/com/duckduckgo/app/surrogates/api/ResourceSurrogateListService.kt similarity index 71% rename from app/src/main/java/com/duckduckgo/app/di/BrowserModule.kt rename to app/src/main/java/com/duckduckgo/app/surrogates/api/ResourceSurrogateListService.kt index 8a1bf10391a9..1066ebec33e8 100644 --- a/app/src/main/java/com/duckduckgo/app/di/BrowserModule.kt +++ b/app/src/main/java/com/duckduckgo/app/surrogates/api/ResourceSurrogateListService.kt @@ -14,15 +14,15 @@ * limitations under the License. */ -package com.duckduckgo.app.di +package com.duckduckgo.app.surrogates.api -import android.webkit.CookieManager -import dagger.Module -import dagger.Provides +import okhttp3.ResponseBody +import retrofit2.Call +import retrofit2.http.GET -@Module -class BrowserModule { - @Provides - fun cookieManager(): CookieManager = CookieManager.getInstance() +interface ResourceSurrogateListService { + + @GET("/contentblocking.js?l=surrogates") + fun https(): Call } \ No newline at end of file diff --git a/app/src/main/java/com/duckduckgo/app/surrogates/di/ResourceSurrogateModule.kt b/app/src/main/java/com/duckduckgo/app/surrogates/di/ResourceSurrogateModule.kt new file mode 100644 index 000000000000..15d6629a8b38 --- /dev/null +++ b/app/src/main/java/com/duckduckgo/app/surrogates/di/ResourceSurrogateModule.kt @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2018 DuckDuckGo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.duckduckgo.app.surrogates.di + +import com.duckduckgo.app.surrogates.ResourceSurrogates +import com.duckduckgo.app.surrogates.ResourceSurrogatesImpl +import dagger.Module +import dagger.Provides +import javax.inject.Singleton + + +@Module +class ResourceSurrogateModule { + + @Provides + @Singleton + fun analyticsSurrogates() : ResourceSurrogates = ResourceSurrogatesImpl() +} \ No newline at end of file diff --git a/app/src/main/java/com/duckduckgo/app/surrogates/store/ResourceSurrogateDataStore.kt b/app/src/main/java/com/duckduckgo/app/surrogates/store/ResourceSurrogateDataStore.kt new file mode 100644 index 000000000000..e7d92c6b4728 --- /dev/null +++ b/app/src/main/java/com/duckduckgo/app/surrogates/store/ResourceSurrogateDataStore.kt @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2018 DuckDuckGo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.duckduckgo.app.surrogates.store + +import android.content.Context +import javax.inject.Inject + +class ResourceSurrogateDataStore @Inject constructor(private val context: Context) { + + fun hasData(): Boolean = context.fileExists(FILENAME) + + fun loadData(): ByteArray = + context.openFileInput(FILENAME).use { it.readBytes() } + + fun saveData(byteArray: ByteArray) { + context.openFileOutput(FILENAME, Context.MODE_PRIVATE).write(byteArray) + } + + fun clearData() { + context.deleteFile(FILENAME) + } + + private fun Context.fileExists(filename: String): Boolean { + val file = getFileStreamPath(filename) + return file != null && file.exists() + } + + companion object { + private const val FILENAME = "surrogates.js" + } + +} diff --git a/app/src/main/java/com/duckduckgo/app/trackerdetection/TrackerDataLoader.kt b/app/src/main/java/com/duckduckgo/app/trackerdetection/TrackerDataLoader.kt index 14110137fd8c..aff961ff71ac 100644 --- a/app/src/main/java/com/duckduckgo/app/trackerdetection/TrackerDataLoader.kt +++ b/app/src/main/java/com/duckduckgo/app/trackerdetection/TrackerDataLoader.kt @@ -32,7 +32,7 @@ class TrackerDataLoader @Inject constructor( fun loadData() { - Timber.i("Loading Tracker data") + Timber.d("Loading Tracker data") // these are stored to disk, then fed to the C++ adblock module loadAdblockData(Client.ClientName.EASYLIST) @@ -44,21 +44,21 @@ class TrackerDataLoader @Inject constructor( } fun loadAdblockData(name: Client.ClientName) { - Timber.i("Looking for adblock tracker ${name.name} to load") + Timber.d("Looking for adblock tracker ${name.name} to load") if (trackerDataStore.hasData(name)) { - Timber.i("Found adblock tracker ${name.name}") + Timber.d("Found adblock tracker ${name.name}") val client = AdBlockClient(name) client.loadProcessedData(trackerDataStore.loadData(name)) trackerDetector.addClient(client) } else { - Timber.i("No adblock tracker ${name.name} found") + Timber.d("No adblock tracker ${name.name} found") } } fun loadDisconnectData() { val trackers = trackerDataDao.getAll() - Timber.i("Loaded ${trackers.size} disconnect trackers from DB") + Timber.d("Loaded ${trackers.size} disconnect trackers from DB") val client = DisconnectClient(Client.ClientName.DISCONNECT, trackers) trackerDetector.addClient(client) diff --git a/app/src/main/java/com/duckduckgo/app/trackerdetection/TrackerDetector.kt b/app/src/main/java/com/duckduckgo/app/trackerdetection/TrackerDetector.kt index 080eb179a914..850bce14974d 100644 --- a/app/src/main/java/com/duckduckgo/app/trackerdetection/TrackerDetector.kt +++ b/app/src/main/java/com/duckduckgo/app/trackerdetection/TrackerDetector.kt @@ -23,23 +23,27 @@ import com.duckduckgo.app.trackerdetection.model.TrackerNetworks import com.duckduckgo.app.trackerdetection.model.TrackingEvent import timber.log.Timber import java.util.concurrent.CopyOnWriteArrayList -import javax.inject.Inject -import javax.inject.Singleton -@Singleton -class TrackerDetector @Inject constructor(private val networkTrackers: TrackerNetworks, private val settings: PrivacySettingsStore) { +interface TrackerDetector { + fun addClient(client: Client) + fun evaluate(url: String, documentUrl: String, resourceType: ResourceType): TrackingEvent? +} + +class TrackerDetectorImpl ( + private val networkTrackers: TrackerNetworks, + private val settings: PrivacySettingsStore) :TrackerDetector { private val clients = CopyOnWriteArrayList() /** * Adds a new client. If the client's name matches an existing client, old client is replaced */ - fun addClient(client: Client) { + override fun addClient(client: Client) { clients.removeAll { client.name == it.name } clients.add(client) } - fun evaluate(url: String, documentUrl: String, resourceType: ResourceType): TrackingEvent? { + override fun evaluate(url: String, documentUrl: String, resourceType: ResourceType): TrackingEvent? { val whitelisted = clients.any { it.name.type == Client.ClientType.WHITELIST && it.matches(url, documentUrl, resourceType) } if (whitelisted) { diff --git a/app/src/main/java/com/duckduckgo/app/trackerdetection/api/TrackerDataDownloader.kt b/app/src/main/java/com/duckduckgo/app/trackerdetection/api/TrackerDataDownloader.kt index e808e64decb8..e02311fab458 100644 --- a/app/src/main/java/com/duckduckgo/app/trackerdetection/api/TrackerDataDownloader.kt +++ b/app/src/main/java/com/duckduckgo/app/trackerdetection/api/TrackerDataDownloader.kt @@ -52,13 +52,13 @@ class TrackerDataDownloader @Inject constructor( return Completable.fromAction { - Timber.i("Downloading disconnect data") + Timber.d("Downloading disconnect data") val call = trackerListService.disconnect() val response = call.execute() if (response.isCached && trackerDataDao.count() != 0) { - Timber.i("Disconnect data already cached and stored") + Timber.d("Disconnect data already cached and stored") return@fromAction } @@ -82,18 +82,18 @@ class TrackerDataDownloader @Inject constructor( private fun easyDownload(clientName: Client.ClientName, callFactory: (clientName: Client.ClientName) -> Call): Completable { return Completable.fromAction { - Timber.i("Downloading ${clientName.name} data") + Timber.d("Downloading ${clientName.name} data") val call = callFactory(clientName) val response = call.execute() if (response.isCached && trackerDataStore.hasData(clientName)) { - Timber.i("${clientName.name} data already cached and stored") + Timber.d("${clientName.name} data already cached and stored") return@fromAction } if (response.isSuccessful) { val bodyBytes = response.body()!!.bytes() - Timber.i("Updating ${clientName.name} data store with new data") + Timber.d("Updating ${clientName.name} data store with new data") persistTrackerData(clientName, bodyBytes) trackerDataLoader.loadAdblockData(clientName) } else { diff --git a/app/src/main/java/com/duckduckgo/app/trackerdetection/di/TrackerDetectionModule.kt b/app/src/main/java/com/duckduckgo/app/trackerdetection/di/TrackerDetectionModule.kt new file mode 100644 index 000000000000..bca1bd77ecab --- /dev/null +++ b/app/src/main/java/com/duckduckgo/app/trackerdetection/di/TrackerDetectionModule.kt @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2018 DuckDuckGo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.duckduckgo.app.trackerdetection.di + +import com.duckduckgo.app.privacymonitor.store.PrivacySettingsStore +import com.duckduckgo.app.trackerdetection.TrackerDetector +import com.duckduckgo.app.trackerdetection.TrackerDetectorImpl +import com.duckduckgo.app.trackerdetection.model.TrackerNetworks +import dagger.Module +import dagger.Provides +import javax.inject.Singleton + + +@Module +class TrackerDetectionModule { + + @Provides + @Singleton + fun trackerDetector(networkTrackers: TrackerNetworks, settings: PrivacySettingsStore): TrackerDetector { + return TrackerDetectorImpl(networkTrackers, settings) + } +} \ No newline at end of file