diff --git a/.gitmodules b/.gitmodules index c375a5977496..472b11f5ed40 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ -[submodule "app/libs/OnionKit"] - path = app/libs/OnionKit - url = https://github.com/guardianproject/NetCipher.git +[submodule "app/src/main/cpp/third-party/bloom_cpp"] + path = app/src/main/cpp/third-party/bloom_cpp + url = https://github.com/duckduckgo/bloom_cpp.git diff --git a/app/CMakeLists.txt b/app/CMakeLists.txt index adef54822180..85685fa663ae 100644 --- a/app/CMakeLists.txt +++ b/app/CMakeLists.txt @@ -18,12 +18,23 @@ add_library( # Sets the name of the library. # Provides a relative path to your source file(s). src/main/cpp/adblockclient-lib.cpp - src/main/cpp/ad-block/ad_block_client.cc - src/main/cpp/ad-block/cosmetic_filter.cc - src/main/cpp/ad-block/filter.cc - src/main/cpp/bloom-filter-cpp/BloomFilter.cpp - src/main/cpp/bloom-filter-cpp/hashFn.cpp - src/main/cpp/hashset-cpp/HashSet.cpp + src/main/cpp/third-party/ad-block/ad_block_client.cc + src/main/cpp/third-party/ad-block/cosmetic_filter.cc + src/main/cpp/third-party/ad-block/filter.cc + src/main/cpp/third-party/bloom-filter-cpp/BloomFilter.cpp + src/main/cpp/third-party/bloom-filter-cpp/hashFn.cpp + src/main/cpp/third-party/hashset-cpp/HashSet.cpp + ) + +add_library( # Sets the name of the library. + https-bloom-lib + + # Sets the library as a shared library. + SHARED + + # Provides a relative path to your source file(s). + src/main/cpp/https-bloom-lib.cpp + src/main/cpp/third-party/bloom_cpp/src/BloomFilter.cpp ) # Searches for a specified prebuilt library and stores the path as a diff --git a/app/build.gradle b/app/build.gradle index c10ccbf33f0b..20ae7acddc5c 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -78,7 +78,7 @@ android { ext { supportLibrary = "27.1.1" - architectureComponents = "1.1.0" + architectureComponents = "1.1.1" architectureComponentsExtensions = "1.1.1" androidKtx = "0.3" dagger = "2.14.1" diff --git a/app/schemas/com.duckduckgo.app.global.db.AppDatabase/4.json b/app/schemas/com.duckduckgo.app.global.db.AppDatabase/4.json new file mode 100644 index 000000000000..68af9f9a8403 --- /dev/null +++ b/app/schemas/com.duckduckgo.app.global.db.AppDatabase/4.json @@ -0,0 +1,302 @@ +{ + "formatVersion": 1, + "database": { + "version": 4, + "identityHash": "a80686f1ea7760530a37dec6214a19f4", + "entities": [ + { + "tableName": "disconnect_tracker", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`url` TEXT NOT NULL, `category` TEXT NOT NULL, `networkName` TEXT NOT NULL, `networkUrl` TEXT NOT NULL, PRIMARY KEY(`url`))", + "fields": [ + { + "fieldPath": "url", + "columnName": "url", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "category", + "columnName": "category", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "networkName", + "columnName": "networkName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "networkUrl", + "columnName": "networkUrl", + "affinity": "TEXT", + "notNull": true + } + ], + "primaryKey": { + "columnNames": [ + "url" + ], + "autoGenerate": false + }, + "indices": [], + "foreignKeys": [] + }, + { + "tableName": "https_bloom_filter_spec", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER NOT NULL, `errorRate` REAL NOT NULL, `totalEntries` INTEGER NOT NULL, `sha256` TEXT NOT NULL, PRIMARY KEY(`id`))", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "errorRate", + "columnName": "errorRate", + "affinity": "REAL", + "notNull": true + }, + { + "fieldPath": "totalEntries", + "columnName": "totalEntries", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "sha256", + "columnName": "sha256", + "affinity": "TEXT", + "notNull": true + } + ], + "primaryKey": { + "columnNames": [ + "id" + ], + "autoGenerate": false + }, + "indices": [], + "foreignKeys": [] + }, + { + "tableName": "https_whitelisted_domain", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`domain` TEXT NOT NULL, PRIMARY KEY(`domain`))", + "fields": [ + { + "fieldPath": "domain", + "columnName": "domain", + "affinity": "TEXT", + "notNull": true + } + ], + "primaryKey": { + "columnNames": [ + "domain" + ], + "autoGenerate": false + }, + "indices": [], + "foreignKeys": [] + }, + { + "tableName": "network_leaderboard", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`networkName` TEXT NOT NULL, `domainVisited` TEXT NOT NULL, PRIMARY KEY(`networkName`, `domainVisited`))", + "fields": [ + { + "fieldPath": "networkName", + "columnName": "networkName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "domainVisited", + "columnName": "domainVisited", + "affinity": "TEXT", + "notNull": true + } + ], + "primaryKey": { + "columnNames": [ + "networkName", + "domainVisited" + ], + "autoGenerate": false + }, + "indices": [], + "foreignKeys": [] + }, + { + "tableName": "site_visited", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`domain` TEXT NOT NULL, PRIMARY KEY(`domain`))", + "fields": [ + { + "fieldPath": "domain", + "columnName": "domain", + "affinity": "TEXT", + "notNull": true + } + ], + "primaryKey": { + "columnNames": [ + "domain" + ], + "autoGenerate": false + }, + "indices": [], + "foreignKeys": [] + }, + { + "tableName": "app_configuration", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`key` TEXT NOT NULL, `appConfigurationDownloaded` INTEGER NOT NULL, PRIMARY KEY(`key`))", + "fields": [ + { + "fieldPath": "key", + "columnName": "key", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "appConfigurationDownloaded", + "columnName": "appConfigurationDownloaded", + "affinity": "INTEGER", + "notNull": true + } + ], + "primaryKey": { + "columnNames": [ + "key" + ], + "autoGenerate": false + }, + "indices": [], + "foreignKeys": [] + }, + { + "tableName": "tabs", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`tabId` TEXT NOT NULL, `url` TEXT, `title` TEXT, PRIMARY KEY(`tabId`))", + "fields": [ + { + "fieldPath": "tabId", + "columnName": "tabId", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "url", + "columnName": "url", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "title", + "columnName": "title", + "affinity": "TEXT", + "notNull": false + } + ], + "primaryKey": { + "columnNames": [ + "tabId" + ], + "autoGenerate": false + }, + "indices": [ + { + "name": "index_tabs_tabId", + "unique": false, + "columnNames": [ + "tabId" + ], + "createSql": "CREATE INDEX `index_tabs_tabId` ON `${TABLE_NAME}` (`tabId`)" + } + ], + "foreignKeys": [] + }, + { + "tableName": "tab_selection", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER NOT NULL, `tabId` TEXT, PRIMARY KEY(`id`), FOREIGN KEY(`tabId`) REFERENCES `tabs`(`tabId`) ON UPDATE NO ACTION ON DELETE SET NULL )", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "tabId", + "columnName": "tabId", + "affinity": "TEXT", + "notNull": false + } + ], + "primaryKey": { + "columnNames": [ + "id" + ], + "autoGenerate": false + }, + "indices": [ + { + "name": "index_tab_selection_tabId", + "unique": false, + "columnNames": [ + "tabId" + ], + "createSql": "CREATE INDEX `index_tab_selection_tabId` ON `${TABLE_NAME}` (`tabId`)" + } + ], + "foreignKeys": [ + { + "table": "tabs", + "onDelete": "SET NULL", + "onUpdate": "NO ACTION", + "columns": [ + "tabId" + ], + "referencedColumns": [ + "tabId" + ] + } + ] + }, + { + "tableName": "bookmarks", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `title` TEXT, `url` TEXT NOT NULL)", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "title", + "columnName": "title", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "url", + "columnName": "url", + "affinity": "TEXT", + "notNull": true + } + ], + "primaryKey": { + "columnNames": [ + "id" + ], + "autoGenerate": true + }, + "indices": [], + "foreignKeys": [] + } + ], + "setupQueries": [ + "CREATE TABLE IF NOT EXISTS room_master_table (id INTEGER PRIMARY KEY,identity_hash TEXT)", + "INSERT OR REPLACE INTO room_master_table (id,identity_hash) VALUES(42, \"a80686f1ea7760530a37dec6214a19f4\")" + ] + } +} \ No newline at end of file diff --git a/app/src/androidTest/java/com/duckduckgo/app/FileUtilities.kt b/app/src/androidTest/java/com/duckduckgo/app/FileUtilities.kt index 482c99e34ddf..e088270bf83e 100644 --- a/app/src/androidTest/java/com/duckduckgo/app/FileUtilities.kt +++ b/app/src/androidTest/java/com/duckduckgo/app/FileUtilities.kt @@ -21,4 +21,8 @@ object FileUtilities { fun loadText(resourceName: String): String = javaClass.classLoader.getResource(resourceName).openStream().bufferedReader().use { it.readText() } + + fun loadLines(resourceName: String): List = + javaClass.classLoader.getResource(resourceName).openStream().bufferedReader().use { it.readLines() } + } \ No newline at end of file diff --git a/app/src/androidTest/java/com/duckduckgo/app/feedback/ui/FeedbackViewModelTest.kt b/app/src/androidTest/java/com/duckduckgo/app/feedback/ui/FeedbackViewModelTest.kt index 84c9eecdd972..d480f81b4259 100644 --- a/app/src/androidTest/java/com/duckduckgo/app/feedback/ui/FeedbackViewModelTest.kt +++ b/app/src/androidTest/java/com/duckduckgo/app/feedback/ui/FeedbackViewModelTest.kt @@ -55,7 +55,7 @@ class FeedbackViewModelTest { @Test fun whenBrokenUrlSwitchedOnWithUrlThenMessageFocused() { - testee.onBrokenSiteUrlChanged(Constants.url) + testee.onBrokenSiteUrlChanged(url) testee.onBrokenSiteChanged(true) verify(mockCommandObserver).onChanged(Command.FocusMessage) } @@ -63,8 +63,8 @@ class FeedbackViewModelTest { @Test fun whenBrokenUrlOnWithUrlAndMessageThenCanSubmit() { testee.onBrokenSiteChanged(true) - testee.onBrokenSiteUrlChanged(Constants.url) - testee.onFeedbackMessageChanged(Constants.message) + testee.onBrokenSiteUrlChanged(url) + testee.onFeedbackMessageChanged(message) assertTrue(viewState.submitAllowed) } @@ -72,14 +72,14 @@ class FeedbackViewModelTest { fun whenBrokenUrlOnWithNullUrlThenCannotSubmit() { testee.onBrokenSiteChanged(true) testee.onBrokenSiteUrlChanged(null) - testee.onFeedbackMessageChanged(Constants.message) + testee.onFeedbackMessageChanged(message) assertFalse(viewState.submitAllowed) } @Test fun whenBrokenUrlOnWithNullMessageThenCannotSubmit() { testee.onBrokenSiteChanged(true) - testee.onBrokenSiteUrlChanged(Constants.url) + testee.onBrokenSiteUrlChanged(url) testee.onFeedbackMessageChanged(null) assertFalse(viewState.submitAllowed) } @@ -88,14 +88,14 @@ class FeedbackViewModelTest { fun whenBrokenUrlOnWithBlankUrlThenCannotSubmit() { testee.onBrokenSiteChanged(true) testee.onBrokenSiteUrlChanged(" ") - testee.onFeedbackMessageChanged(Constants.message) + testee.onFeedbackMessageChanged(message) assertFalse(viewState.submitAllowed) } @Test fun whenBrokenUrlOnWithBlankMessageThenCannotSubmit() { testee.onBrokenSiteChanged(true) - testee.onBrokenSiteUrlChanged(Constants.url) + testee.onBrokenSiteUrlChanged(url) testee.onFeedbackMessageChanged(" ") assertFalse(viewState.submitAllowed) } @@ -103,7 +103,7 @@ class FeedbackViewModelTest { @Test fun whenBrokenUrlOffWithMessageThenCanSubmit() { testee.onBrokenSiteChanged(false) - testee.onFeedbackMessageChanged(Constants.message) + testee.onFeedbackMessageChanged(message) assertTrue(viewState.submitAllowed) } @@ -124,11 +124,11 @@ class FeedbackViewModelTest { @Test fun whenCanSubmitBrokenSiteAndSubmitPressedThenFeedbackSubmitted() { testee.onBrokenSiteChanged(true) - testee.onBrokenSiteUrlChanged(Constants.url) - testee.onFeedbackMessageChanged(Constants.message) + testee.onBrokenSiteUrlChanged(url) + testee.onFeedbackMessageChanged(message) testee.onSubmitPressed() - verify(mockFeedbackSender).submitBrokenSiteFeedback(Constants.message, Constants.url) + verify(mockFeedbackSender).submitBrokenSiteFeedback(message, url) verify(mockCommandObserver).onChanged(Command.ConfirmAndFinish) } @@ -143,9 +143,9 @@ class FeedbackViewModelTest { @Test fun whenCanSubmitMessageAndSubmitPressedThenFeedbackSubmitted() { testee.onBrokenSiteChanged(false) - testee.onFeedbackMessageChanged(Constants.message) + testee.onFeedbackMessageChanged(message) testee.onSubmitPressed() - verify(mockFeedbackSender).submitGeneralFeedback(Constants.message) + verify(mockFeedbackSender).submitGeneralFeedback(message) verify(mockCommandObserver).onChanged(Command.ConfirmAndFinish) } diff --git a/app/src/androidTest/java/com/duckduckgo/app/global/ChecksumTest.kt b/app/src/androidTest/java/com/duckduckgo/app/global/ChecksumTest.kt new file mode 100644 index 000000000000..c6b5c3e93374 --- /dev/null +++ b/app/src/androidTest/java/com/duckduckgo/app/global/ChecksumTest.kt @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2018 DuckDuckGo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.duckduckgo.app.global + +import org.junit.Assert.* +import org.junit.Test + +class ChecksumTest { + + @Test + fun whenSha256ChecksumCalledThenChecksumIsCorrect() { + val result = helloWorldText.toByteArray().sha256 + assertEquals(helloWorldSha256, result) + } + + @Test + fun whenSha256CheckumCorrectThenVerifyIsTrue() { + assertTrue(helloWorldText.toByteArray().verifySha256(helloWorldSha256)) + } + + @Test + fun whenSha256CheckumIncorrectThenVerifyIsFalse() { + assertFalse(helloWorldText.toByteArray().verifySha256(otherSha256)) + } + + companion object { + const val helloWorldText = "Hello World!" + const val helloWorldSha256 = "7f83b1657ff1fc53b92dc18148a1d65dfc2d4b1fa3d677284addd200126d9069" + const val otherSha256 = "f97e9da0e3b879f0a9df979ae260a5f7e1371edb127c1862d4f861981166cdc1" + } +} \ No newline at end of file diff --git a/app/src/androidTest/java/com/duckduckgo/app/global/db/AppDatabaseTest.kt b/app/src/androidTest/java/com/duckduckgo/app/global/db/AppDatabaseTest.kt index 0fadc438b3d2..3a7d3dbedf01 100644 --- a/app/src/androidTest/java/com/duckduckgo/app/global/db/AppDatabaseTest.kt +++ b/app/src/androidTest/java/com/duckduckgo/app/global/db/AppDatabaseTest.kt @@ -26,6 +26,7 @@ import org.junit.Assert.assertTrue import org.junit.Rule import org.junit.Test + class AppDatabaseTest { @get:Rule @@ -48,16 +49,21 @@ class AppDatabaseTest { testHelper.createDatabase(TEST_DB_NAME, 2).use { it.execSQL("INSERT INTO `network_leaderboard` VALUES ('Network2', 'example.com')") } - assertTrue(database().networkLeaderboardDao().trackerNetworkTally().blockingObserve()!!.isEmpty()) } + @Test + fun whenMigratingFromVersion3To4ThenValidationSucceeds() { + testHelper.createDatabase(TEST_DB_NAME, 3).close() + testHelper.runMigrationsAndValidate(TEST_DB_NAME, 4, true, AppDatabase.MIGRATION_3_TO_4) + } + private fun database(): AppDatabase { val database = Room - .databaseBuilder(InstrumentationRegistry.getTargetContext(), AppDatabase::class.java, TEST_DB_NAME) - .addMigrations(AppDatabase.MIGRATION_1_TO_2, AppDatabase.MIGRATION_2_TO_3) - .allowMainThreadQueries() - .build() + .databaseBuilder(InstrumentationRegistry.getTargetContext(), AppDatabase::class.java, TEST_DB_NAME) + .addMigrations(AppDatabase.MIGRATION_1_TO_2, AppDatabase.MIGRATION_2_TO_3, AppDatabase.MIGRATION_3_TO_4) + .allowMainThreadQueries() + .build() testHelper.closeWhenFinished(database) diff --git a/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/BloomFilterTest.kt b/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/BloomFilterTest.kt new file mode 100644 index 000000000000..509f85cc4b2f --- /dev/null +++ b/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/BloomFilterTest.kt @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2018 DuckDuckGo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.duckduckgo.app.httpsupgrade + +import org.junit.Assert.* +import org.junit.Test +import java.util.* +import kotlin.collections.ArrayList + + +class BloomFilterTest { + + private lateinit var testee: BloomFilter + + @Test + fun whenBloomFilterEmptyThenContainsIsFalse() { + testee = BloomFilter(FILTER_ELEMENT_COUNT, TARGET_ERROR_RATE) + assertFalse(testee.contains("abc")) + } + + @Test + fun whenBloomFilterContainsElementThenContainsIsTrue() { + testee = BloomFilter(FILTER_ELEMENT_COUNT, TARGET_ERROR_RATE) + testee.add("abc") + assertTrue(testee.contains("abc")) + } + + @Test + fun whenBloomFilterContainsItemsThenLookupResultsAreWithinRange() { + + val bloomData = createRandomStrings(FILTER_ELEMENT_COUNT) + val testData = bloomData + createRandomStrings(ADDITIONAL_TEST_DATA_ELEMENT_COUNT) + + testee = BloomFilter(bloomData.size, TARGET_ERROR_RATE) + bloomData.forEach { testee.add(it) } + + var (falsePositives, truePositives, falseNegatives, trueNegatives) = arrayOf(0, 0, 0, 0) + for (element in testData) { + val result = testee.contains(element) + when { + bloomData.contains(element) && !result -> falseNegatives++ + !bloomData.contains(element) && result -> falsePositives++ + !bloomData.contains(element) && !result -> trueNegatives++ + bloomData.contains(element) && result -> truePositives++ + } + } + + val errorRate = falsePositives / testData.size + assertEquals(0, falseNegatives) + assertEquals(bloomData.size, truePositives) + assertTrue(trueNegatives <= testData.size - bloomData.size) + assertTrue(errorRate <= ACCEPTABLE_ERROR_RATE) + } + + private fun createRandomStrings(items: Int): ArrayList { + var list = ArrayList() + repeat(items) { list.add(UUID.randomUUID().toString()) } + return list + } + + companion object { + const val FILTER_ELEMENT_COUNT = 1000 + const val ADDITIONAL_TEST_DATA_ELEMENT_COUNT = 9000 + const val TARGET_ERROR_RATE = 0.001 + const val ACCEPTABLE_ERROR_RATE = TARGET_ERROR_RATE * 1.1 + } +} \ No newline at end of file diff --git a/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/HttpsUpgraderTest.kt b/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/HttpsUpgraderTest.kt index e199d15ce8e9..7768bbfe65ea 100644 --- a/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/HttpsUpgraderTest.kt +++ b/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/HttpsUpgraderTest.kt @@ -16,81 +16,68 @@ package com.duckduckgo.app.httpsupgrade +import android.arch.core.executor.testing.InstantTaskExecutorRule import android.net.Uri -import com.duckduckgo.app.httpsupgrade.db.HttpsUpgradeDomainDao -import com.nhaarman.mockito_kotlin.* -import org.junit.Assert.* +import com.duckduckgo.app.InstantSchedulersRule +import com.duckduckgo.app.httpsupgrade.api.HttpsBloomFilterFactory +import com.duckduckgo.app.httpsupgrade.db.HttpsWhitelistDao +import com.nhaarman.mockito_kotlin.mock +import com.nhaarman.mockito_kotlin.whenever +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue import org.junit.Before +import org.junit.Rule import org.junit.Test class HttpsUpgraderTest { - lateinit var testee: HttpsUpgrader - lateinit var mockDao: HttpsUpgradeDomainDao + @get:Rule + @Suppress("unused") + var instantTaskExecutorRule = InstantTaskExecutorRule() - @Before - fun before() { - mockDao = mock() - testee = HttpsUpgraderImpl(mockDao) - } + @get:Rule + @Suppress("unused") + val schedulers = InstantSchedulersRule() - @Test - fun whenHostContainsTwoPartsThenUpgradeStillChecksWildcard() { - whenever(mockDao.hasDomain("macpro.localhost")).thenReturn(false) - assertFalse(testee.shouldUpgrade(Uri.parse("http://macpro.localhost"))) - verify(mockDao).hasDomain("*.localhost") - } + lateinit var testee: HttpsUpgrader - @Test - fun whenHostContainsSinglePartThenUpgradeStillChecksWildcard() { - whenever(mockDao.hasDomain("localhost")).thenReturn(false) - assertFalse(testee.shouldUpgrade(Uri.parse("http://localhost"))) - verify(mockDao, times(1)).hasDomain(any()) - } + private var mockHttpsBloomFilterFactory: HttpsBloomFilterFactory = mock() + private var mockWhitelistDao: HttpsWhitelistDao = mock() + private var bloomFilter = BloomFilter(100, 0.01) - @Test - fun whenMixedCaseDomainIsASubdominOfAWildCardInTheDatabaseThenShouldUpgrade() { - whenever(mockDao.hasDomain("www.example.com")).thenReturn(false) - whenever(mockDao.hasDomain("*.example.com")).thenReturn(true) - assertTrue(testee.shouldUpgrade(Uri.parse("http://www.EXAMPLE.com"))) - verify(mockDao).hasDomain("*.example.com") - } - - @Test - fun whenMixedCaseUriIsHttpAndInUpgradeListThenShouldUpgrade() { - whenever(mockDao.hasDomain("www.example.com")).thenReturn(true) - assertTrue(testee.shouldUpgrade(Uri.parse("http://www.EXAMPLE.com"))) + @Before + fun before() { + whenever(mockHttpsBloomFilterFactory.create()).thenReturn(bloomFilter) + testee = HttpsUpgraderImpl(mockWhitelistDao, mockHttpsBloomFilterFactory) } @Test - fun whenGivenUriItIsUpgradedToHttps() { - val input = Uri.parse("http://www.example.com/some/path/to/a/file.txt") - val expected = Uri.parse("https://www.example.com/some/path/to/a/file.txt") - assertEquals(expected, testee.upgrade(input)) + fun whenUriIsHttpsThenShouldNotUpgrade() { + assertFalse(testee.shouldUpgrade(Uri.parse("https://www.example.com"))) } @Test - fun whenDomainIsASubdominOfAWildCardInTheDatabaseThenShouldUpgrade() { - whenever(mockDao.hasDomain("www.example.com")).thenReturn(false) - whenever(mockDao.hasDomain("*.example.com")).thenReturn(true) - assertTrue(testee.shouldUpgrade(Uri.parse("http://www.example.com"))) - verify(mockDao).hasDomain("*.example.com") + fun whenHttpUriIsNotInBloomFilterThenShouldNotUpgrade() { + assertFalse(testee.shouldUpgrade(Uri.parse("http://www.example.com"))) } @Test - fun whenUriIsHttpAndInUpgradeListThenShouldUpgrade() { - whenever(mockDao.hasDomain("www.example.com")).thenReturn(true) + fun whenHttpUriIsInBloomFilterThenShouldUpgrade() { + bloomFilter.add("www.example.com") assertTrue(testee.shouldUpgrade(Uri.parse("http://www.example.com"))) } @Test - fun whenUriIsHttpAndIsNotInUpgradeListThenShouldNotUpgrade() { + fun whenHttpUriHasOnlyPartDomainInBloomFilterThenShouldNotUpgrade() { + bloomFilter.add("example.com") assertFalse(testee.shouldUpgrade(Uri.parse("http://www.example.com"))) } @Test - fun whenUriIsHttpsThenShouldNotUpgrade() { - assertFalse(testee.shouldUpgrade(Uri.parse("https://www.example.com"))) + fun whenHttpUriIsInBloomFilterAndInWhitelistThenShouldNotUpgrade() { + bloomFilter.add("www.example.com") + whenever(mockWhitelistDao.contains("www.example.com")).thenReturn(true) + assertFalse(testee.shouldUpgrade(Uri.parse("http://www.example.com"))) } } diff --git a/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/api/HttpsBloomFilterSpecJsonTest.kt b/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/api/HttpsBloomFilterSpecJsonTest.kt new file mode 100644 index 000000000000..2a48489f4297 --- /dev/null +++ b/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/api/HttpsBloomFilterSpecJsonTest.kt @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2018 DuckDuckGo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.duckduckgo.app.httpsupgrade.api + +import com.duckduckgo.app.httpsupgrade.model.HttpsBloomFilterSpec +import com.squareup.moshi.Moshi +import org.junit.Assert.assertEquals +import org.junit.Test + +class HttpsBloomFilterSpecJsonTest { + + @Test + fun whenGivenValidJsonThenParsesCorrectly() { + val moshi = Moshi.Builder().build() + val jsonAdapter = moshi.adapter(HttpsBloomFilterSpec::class.java) + val result = jsonAdapter.fromJson(json()) + assertEquals(2858372, result.totalEntries) + assertEquals(0.0001, result.errorRate, 0.00001) + assertEquals("932ae1481fc33d94320a3b072638c0df8005482506933897e35feb1294693c84", result.sha256) + } + + private fun json(): String = """ + { + "totalEntries":2858372, + "errorRate" : 0.0001, + "sha256" : "932ae1481fc33d94320a3b072638c0df8005482506933897e35feb1294693c84" + } + """ + +} diff --git a/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/api/HttpsUpgradeJsonTest.kt b/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/api/HttpsWhitelistJsonTest.kt similarity index 54% rename from app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/api/HttpsUpgradeJsonTest.kt rename to app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/api/HttpsWhitelistJsonTest.kt index 8744b8c69128..966e46c026b1 100644 --- a/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/api/HttpsUpgradeJsonTest.kt +++ b/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/api/HttpsWhitelistJsonTest.kt @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017 DuckDuckGo + * Copyright (c) 2018 DuckDuckGo * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,33 +16,37 @@ package com.duckduckgo.app.httpsupgrade.api -import com.duckduckgo.app.httpsupgrade.db.HttpsUpgradeDomain +import com.duckduckgo.app.httpsupgrade.model.HttpsWhitelistedDomain import com.squareup.moshi.JsonAdapter import com.squareup.moshi.Moshi import com.squareup.moshi.Types import org.junit.Assert.assertEquals import org.junit.Test -class HttpsUpgradeJsonTest { +class HttpsWhitelistJsonTest { @Test fun whenGivenValidJsonThenParsesCorrectly() { - val moshi = Moshi.Builder().add(HttpsUpgradeDomainFromStringAdapter()).build() - val type = Types.newParameterizedType(List::class.java, HttpsUpgradeDomain::class.java) - val adapter: JsonAdapter> = moshi.adapter(type) - val list = adapter.fromJson(json()) - assertEquals(5, list.count()) - } + val moshi = Moshi.Builder().add(HttpsWhitelistJsonAdapter()).build() + val type = Types.newParameterizedType(List::class.java, HttpsWhitelistedDomain::class.java) + val jsonAdapter: JsonAdapter> = moshi.adapter(type) - private fun json() : String = """ - [ - "1337x.to", - "1688.com", - "2ch.net", - "adobe.com", - "alibaba.com" - ] - """ + val list = jsonAdapter.fromJson(json()) + assertEquals(7, list.count()) + } -} \ No newline at end of file + private fun json(): String = """ + { + "data": [ + "mlb.mlb.com", + "proa.accuweather.com", + "jbhard.org", + "lody.net", + "ci.brookfield.wi.us", + "cocktaildreams.de", + "anwalt-im-netz.de" + ] + } + """ +} diff --git a/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/db/HttpsBloomFilterSpecDaoTest.kt b/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/db/HttpsBloomFilterSpecDaoTest.kt new file mode 100644 index 000000000000..1a8c9adf87e6 --- /dev/null +++ b/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/db/HttpsBloomFilterSpecDaoTest.kt @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2018 DuckDuckGo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.duckduckgo.app.httpsupgrade.db + +import android.arch.persistence.room.Room +import android.support.test.InstrumentationRegistry +import com.duckduckgo.app.global.db.AppDatabase +import com.duckduckgo.app.httpsupgrade.model.HttpsBloomFilterSpec +import org.junit.After +import org.junit.Assert.* +import org.junit.Before +import org.junit.Test + +class HttpsBloomFilterSpecDaoTest { + + private lateinit var db: AppDatabase + private lateinit var dao: HttpsBloomFilterSpecDao + + @Before + fun before() { + db = Room.inMemoryDatabaseBuilder(InstrumentationRegistry.getContext(), AppDatabase::class.java).build() + dao = db.httpsBloomFilterSpecDao() + } + + @After + fun after() { + db.close() + } + + @Test + fun whenModelIsEmptyThenGetIsNull() { + assertNull(dao.get()) + } + + @Test + fun whenModelIsInsertedThenGetIsNotNull() { + dao.insert(HttpsBloomFilterSpec(errorRate = 0.1, totalEntries = 55, sha256 = "abc")) + assertNotNull(dao.get()) + } + + @Test + fun whenNewModelIsInsertedThenGetIsNotNullAndDetailsUpdates() { + dao.insert(HttpsBloomFilterSpec(errorRate = 0.1, totalEntries = 55, sha256 = "abc")) + dao.insert(HttpsBloomFilterSpec(errorRate = 0.2, totalEntries = 60, sha256 = "xyz")) + + val specification = dao.get() + assertNotNull(specification) + assertEquals(0.2, specification!!.errorRate, 0.01) + assertEquals(60, specification!!.totalEntries) + assertEquals("xyz", specification!!.sha256) + } + +} \ No newline at end of file diff --git a/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/db/HttpsUpgradeDomainDAOTest.kt b/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/db/HttpsUpgradeDomainDAOTest.kt deleted file mode 100644 index 067ac7c774d3..000000000000 --- a/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/db/HttpsUpgradeDomainDAOTest.kt +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Copyright (c) 2017 DuckDuckGo - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.duckduckgo.app.httpsupgrade.db - -import android.arch.persistence.room.Room -import android.support.test.InstrumentationRegistry -import com.duckduckgo.app.global.db.AppDatabase -import org.junit.After -import org.junit.Assert.* -import org.junit.Before -import org.junit.Test - -class HttpsUpgradeDomainDaoTest { - - companion object { - var exactMatchDomain = "bbc.co.uk" - var otherDomain = "other.com" - var wildcardDomain = "*.wordpress.com" - var otherWildcardDomain = "*.google.com" - var exampleWildcardDomain = "example.wordpress.com" - var parentOfWildcardDomain = "wordpress.com" - } - - private lateinit var db: AppDatabase - private lateinit var dao: HttpsUpgradeDomainDao - - @Before - fun before() { - db = Room.inMemoryDatabaseBuilder(InstrumentationRegistry.getContext(), AppDatabase::class.java).build() - dao = db.httpsUpgradeDomainDao() - } - - @After - fun after() { - db.close() - } - - @Test - fun whenExactMatchDomainAddedAndThenAllDeletedThenDoesNotContainExactMatchDomain() { - dao.insertAll(listOf(HttpsUpgradeDomain(exactMatchDomain))) - dao.deleteAll() - assertFalse(dao.hasDomain(exactMatchDomain)) - } - - @Test - fun whenWildcardDomainInsertedModelThenDoesNotContainParentOfWildcardDomain() { - dao.insertAll(listOf(HttpsUpgradeDomain(wildcardDomain))) - assertFalse(dao.hasDomain(parentOfWildcardDomain)) - } - - @Test - fun whenOtherWildcardDomainInsertedThenModelDoesNotContainExampleWildcardDomain() { - dao.insertAll(listOf(HttpsUpgradeDomain(otherWildcardDomain))) - assertFalse(dao.hasDomain(exampleWildcardDomain)) - } - - @Test - fun whenWildcardDomainInsertedThenModelDoesNotContainExactMatchDomain() { - dao.insertAll(listOf(HttpsUpgradeDomain(wildcardDomain))) - assertFalse(dao.hasDomain(exactMatchDomain)) - } - - @Test - fun whenWildcardDomainInsertedThenModelContainsExampleWildcardDomain() { - dao.insertAll(listOf(HttpsUpgradeDomain(wildcardDomain))) - assertTrue(dao.hasDomain("*.$parentOfWildcardDomain")) - } - - @Test - fun whenExactMatchDomainInsertedThenModelDoesNotContainOtherDomain() { - dao.insertAll(listOf(HttpsUpgradeDomain(exactMatchDomain))) - assertFalse(dao.hasDomain(otherDomain)) - } - - @Test - fun whenExactMatchDomainIsInsertedThenModelContainsExactMatchDomain() { - dao.insertAll(listOf(HttpsUpgradeDomain(exactMatchDomain))) - assertTrue(dao.hasDomain(exactMatchDomain)) - } - - @Test - fun whenModelIsEmptyThenModelDoesNotContainExactMatchDomain() { - assertFalse(dao.hasDomain(exactMatchDomain)) - } - - @Test - fun whenModelIsEmptyThenCountIsZero() { - assertEquals(0, dao.count()) - } - - @Test - fun whenModelContainsTwoItemsThenCountIsTwo() { - dao.insertAll(listOf(HttpsUpgradeDomain(exactMatchDomain), HttpsUpgradeDomain(otherDomain))) - assertEquals(2, dao.count()) - } - -} \ No newline at end of file diff --git a/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/db/HttpsUpgradePerformanceTest.kt b/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/db/HttpsUpgradePerformanceTest.kt deleted file mode 100644 index f665e381afec..000000000000 --- a/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/db/HttpsUpgradePerformanceTest.kt +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Copyright (c) 2018 DuckDuckGo - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.duckduckgo.app.httpsupgrade.db - -/* - * Copyright (c) 2018 DuckDuckGo - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import android.arch.persistence.room.Room -import android.net.Uri -import android.support.test.InstrumentationRegistry -import com.duckduckgo.app.global.db.AppDatabase -import com.duckduckgo.app.httpsupgrade.HttpsUpgrader -import com.duckduckgo.app.httpsupgrade.HttpsUpgraderImpl -import org.junit.Assert.assertEquals -import org.junit.Assert.assertFalse -import org.junit.Before -import org.junit.Ignore -import org.junit.Test -import timber.log.Timber - -@Ignore("Can run slow due to amount of data") -class HttpsUpgraderPerformanceTest { - - lateinit var dao: HttpsUpgradeDomainDao - lateinit var httpsUpgrader: HttpsUpgrader - lateinit var db: AppDatabase - - @Before - fun setup() { - db = Room.inMemoryDatabaseBuilder(InstrumentationRegistry.getContext(), AppDatabase::class.java).build() - dao = db.httpsUpgradeDomainDao() - httpsUpgrader = HttpsUpgraderImpl(dao) - } - - @Test - fun test2800() { - val testDomains = ingest(2800) - assertEquals(2800, testDomains.size) - checkPerformance(testDomains) - } - - @Test - fun test200000() { - val testDomains = ingest(200000) - assertEquals(200000, testDomains.size) - checkPerformance(testDomains) - } - - @Test - fun test500000() { - val testDomains = ingest(500000) - assertEquals(500000, testDomains.size) - checkPerformance(testDomains) - } - - private fun checkPerformance(testDomains: Array) { - var start = System.currentTimeMillis() - var index = 0 - for (domain in testDomains) { - assertFalse(httpsUpgrader.shouldUpgrade(Uri.parse("http://$domain"))) - - if ((index % 100) == 0) { - val diff = System.currentTimeMillis() - start - Timber.i("$diff $domain") - start = System.currentTimeMillis() - } - index++ - } - - val diff = System.currentTimeMillis() - start - Timber.i("$index $diff") - } - - private fun ingest(size: Int): Array { - val start = System.currentTimeMillis() - for (i in 0 .. size) { - dao.insertAll(listOf(HttpsUpgradeDomain("domain$i.com"))) - } - - val testDomains = Array(size, { - "testdomain$it.com" - }) - - val diff = System.currentTimeMillis() - start - Timber.i("ingest finished $diff") - return testDomains - } - -} diff --git a/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/db/HttpsWhitelistDaoTest.kt b/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/db/HttpsWhitelistDaoTest.kt new file mode 100644 index 000000000000..23894ea1aeee --- /dev/null +++ b/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/db/HttpsWhitelistDaoTest.kt @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2018 DuckDuckGo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.duckduckgo.app.httpsupgrade.db + +import android.arch.persistence.room.Room +import android.support.test.InstrumentationRegistry.getContext +import com.duckduckgo.app.global.db.AppDatabase +import com.duckduckgo.app.httpsupgrade.model.HttpsWhitelistedDomain +import org.junit.After +import org.junit.Assert.* +import org.junit.Before +import org.junit.Test + +class HttpsWhitelistDaoTest { + + private lateinit var db: AppDatabase + private lateinit var dao: HttpsWhitelistDao + + @Before + fun before() { + db = Room.inMemoryDatabaseBuilder(getContext(), AppDatabase::class.java).build() + dao = db.httpsWhitelistedDao() + } + + @After + fun after() { + db.close() + } + + @Test + fun whenModelIsEmptyThenCountIsZero() { + assertEquals(0, dao.count()) + } + + @Test + fun whenModelIsEmptyThenContainsDomainIsFalse() { + assertFalse(dao.contains(domain)) + } + + @Test + fun whenDomainInsertedThenContainsDomainIsTrue() { + dao.insertAll(listOf(HttpsWhitelistedDomain(domain))) + assertTrue(dao.contains(domain)) + } + + @Test + fun whenDomainInsertedThenCountIsOne() { + dao.insertAll(listOf(HttpsWhitelistedDomain(domain))) + assertEquals(1, dao.count()) + } + + @Test + fun whenSecondUniqueDomainInsertedThenCountIsTwo() { + dao.insertAll(listOf(HttpsWhitelistedDomain(domain))) + dao.insertAll(listOf(HttpsWhitelistedDomain(anotherDomain))) + assertEquals(2, dao.count()) + } + + @Test + fun whenSecondDuplicateDomainInsertedThenCountIsOne() { + dao.insertAll(listOf(HttpsWhitelistedDomain(domain))) + dao.insertAll(listOf(HttpsWhitelistedDomain(domain))) + assertEquals(1, dao.count()) + } + + @Test + fun whenAllUpdatedThenPreviousValuesAreReplaced() { + dao.insertAll(listOf(HttpsWhitelistedDomain(domain))) + dao.updateAll(listOf(HttpsWhitelistedDomain(anotherDomain))) + assertEquals(1, dao.count()) + assertTrue(dao.contains(anotherDomain)) + } + + @Test + fun whenhAllDeletedThenContainsDomainIsFalse() { + dao.insertAll(listOf(HttpsWhitelistedDomain(domain))) + dao.deleteAll() + assertFalse(dao.contains(domain)) + } + + companion object { + var domain = "domain.com" + var anotherDomain = "another.com" + } +} diff --git a/app/src/androidTest/java/com/duckduckgo/app/trackerdetection/db/TrackerDataDaoTest.kt b/app/src/androidTest/java/com/duckduckgo/app/trackerdetection/db/TrackerDataDaoTest.kt new file mode 100644 index 000000000000..0073f096ba65 --- /dev/null +++ b/app/src/androidTest/java/com/duckduckgo/app/trackerdetection/db/TrackerDataDaoTest.kt @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2018 DuckDuckGo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.duckduckgo.app.trackerdetection.db + +import android.arch.persistence.room.Room +import android.support.test.InstrumentationRegistry +import com.duckduckgo.app.global.db.AppDatabase +import com.duckduckgo.app.trackerdetection.model.DisconnectTracker +import org.junit.After +import org.junit.Assert.* +import org.junit.Before +import org.junit.Test + +class TrackerDataDaoTest { + + private lateinit var db: AppDatabase + private lateinit var dao: TrackerDataDao + + @Before + fun before() { + db = Room.inMemoryDatabaseBuilder(InstrumentationRegistry.getContext(), AppDatabase::class.java).build() + dao = db.trackerDataDao() + } + + @After + fun after() { + db.close() + } + + @Test + fun whenModelIsEmptyThenCountIsZero() { + assertEquals(0, dao.count()) + } + + @Test + fun whenTrackerInsertedThenCountIsOne() { + dao.insertAll(listOf(createTracker(trackerUrl))) + assertEquals(1, dao.count()) + } + + @Test + fun whenTrackerInsertedThenContainsTracker() { + val tracker = createTracker(trackerUrl) + dao.insertAll(listOf(tracker)) + assertTrue(dao.getAll().contains(tracker)) + } + + @Test + fun whenSecondUniqueTrackerInsertedThenCountIsTwo() { + dao.insertAll(listOf(createTracker(trackerUrl))) + dao.insertAll(listOf(createTracker(anotherTrackerUrl))) + assertEquals(2, dao.count()) + } + + @Test + fun whenSecondDuplicateTrackerInsertedThenCountIsOne() { + dao.insertAll(listOf(createTracker(trackerUrl))) + dao.insertAll(listOf(createTracker(trackerUrl))) + assertEquals(1, dao.count()) + } + + @Test + fun whenAllUpdatedThenPreviousValuesAreReplaced() { + val initialTracker = createTracker(trackerUrl) + val replacementTracker = createTracker(anotherTrackerUrl) + + dao.insertAll(listOf(initialTracker)) + dao.updateAll(listOf(replacementTracker)) + assertEquals(1, dao.count()) + assertTrue(dao.getAll().contains(replacementTracker)) + } + + @Test + fun whenhAllDeletedThenCountIsZero() { + val tracker = createTracker(trackerUrl) + dao.insertAll(listOf(tracker)) + dao.deleteAll() + assertEquals(0, dao.count()) + } + + fun createTracker(url: String): DisconnectTracker { + return DisconnectTracker(url, "","", "") + } + + companion object { + var trackerUrl = "tracker.com" + var anotherTrackerUrl = "anotherTracker.com" + } +} diff --git a/app/src/main/cpp/adblockclient-lib.cpp b/app/src/main/cpp/adblockclient-lib.cpp index f8bbc066334d..39c55ae41bb5 100644 --- a/app/src/main/cpp/adblockclient-lib.cpp +++ b/app/src/main/cpp/adblockclient-lib.cpp @@ -1,11 +1,11 @@ #include -#include "ad-block/ad_block_client.h" +#include "third-party/ad-block/ad_block_client.h" extern "C" JNIEXPORT jlong JNICALL Java_com_duckduckgo_app_trackerdetection_AdBlockClient_createClient(JNIEnv *env, - jobject /* this */) { + jobject) { AdBlockClient *client = new AdBlockClient(); return (long) client; } @@ -14,7 +14,7 @@ extern "C" JNIEXPORT void JNICALL Java_com_duckduckgo_app_trackerdetection_AdBlockClient_releaseClient(JNIEnv *env, - jobject /* this */, + jobject, jlong clientPointer, jlong rawDataPointer, jlong processedDataPointer) { @@ -33,9 +33,9 @@ extern "C" JNIEXPORT jlong JNICALL Java_com_duckduckgo_app_trackerdetection_AdBlockClient_loadBasicData(JNIEnv *env, - jobject /* this */, - jlong clientPointer, - jbyteArray data) { + jobject, + jlong clientPointer, + jbyteArray data) { int dataLength = env->GetArrayLength(data); char *dataChars = new char[dataLength]; diff --git a/app/src/main/cpp/https-bloom-lib.cpp b/app/src/main/cpp/https-bloom-lib.cpp new file mode 100644 index 000000000000..672e9848694f --- /dev/null +++ b/app/src/main/cpp/https-bloom-lib.cpp @@ -0,0 +1,67 @@ +#include +#include "third-party/bloom_cpp/src/BloomFilter.hpp" + +extern "C" +JNIEXPORT long +JNICALL +Java_com_duckduckgo_app_httpsupgrade_BloomFilter_createBloomFilter(JNIEnv *env, + jobject, + jint maxItems, + jdouble targetProbability) { + BloomFilter* filter = new BloomFilter(maxItems, targetProbability); + return (long) filter; +} + + +extern "C" +JNIEXPORT long +JNICALL +Java_com_duckduckgo_app_httpsupgrade_BloomFilter_createBloomFilterFromFile(JNIEnv *env, + jobject, + jstring path, + jint maxItems) { + jboolean isElementCopy; + const char *pathChars = env->GetStringUTFChars(path, &isElementCopy); + + BloomFilter* filter = new BloomFilter(pathChars, maxItems); + return (long) filter; +} + + +extern "C" +JNIEXPORT void +JNICALL +Java_com_duckduckgo_app_httpsupgrade_BloomFilter_releaseBloomFilter(JNIEnv *env, + jobject, + jlong pointer) { + BloomFilter *filter = (BloomFilter *) pointer; + delete filter; +} + +extern "C" +JNIEXPORT void +JNICALL +Java_com_duckduckgo_app_httpsupgrade_BloomFilter_add(JNIEnv *env, + jobject, + jlong pointer, + jstring element) { + jboolean isElementCopy; + const char *elementChars = env->GetStringUTFChars(element, &isElementCopy); + + BloomFilter *filter = (BloomFilter *) pointer; + filter->add(elementChars); +} + +extern "C" +JNIEXPORT jboolean +JNICALL +Java_com_duckduckgo_app_httpsupgrade_BloomFilter_contains(JNIEnv *env, + jobject, + jlong pointer, + jstring element) { + jboolean isElementCopy; + const char *elementChars = env->GetStringUTFChars(element, &isElementCopy); + + BloomFilter *filter = (BloomFilter *) pointer; + return filter->contains(elementChars); +} \ No newline at end of file diff --git a/app/src/main/cpp/ad-block/ad_block_client.cc b/app/src/main/cpp/third-party/ad-block/ad_block_client.cc similarity index 100% rename from app/src/main/cpp/ad-block/ad_block_client.cc rename to app/src/main/cpp/third-party/ad-block/ad_block_client.cc diff --git a/app/src/main/cpp/ad-block/ad_block_client.h b/app/src/main/cpp/third-party/ad-block/ad_block_client.h similarity index 100% rename from app/src/main/cpp/ad-block/ad_block_client.h rename to app/src/main/cpp/third-party/ad-block/ad_block_client.h diff --git a/app/src/main/cpp/ad-block/bad_fingerprint.h b/app/src/main/cpp/third-party/ad-block/bad_fingerprint.h similarity index 100% rename from app/src/main/cpp/ad-block/bad_fingerprint.h rename to app/src/main/cpp/third-party/ad-block/bad_fingerprint.h diff --git a/app/src/main/cpp/ad-block/bad_fingerprints.h b/app/src/main/cpp/third-party/ad-block/bad_fingerprints.h similarity index 100% rename from app/src/main/cpp/ad-block/bad_fingerprints.h rename to app/src/main/cpp/third-party/ad-block/bad_fingerprints.h diff --git a/app/src/main/cpp/ad-block/base.h b/app/src/main/cpp/third-party/ad-block/base.h similarity index 100% rename from app/src/main/cpp/ad-block/base.h rename to app/src/main/cpp/third-party/ad-block/base.h diff --git a/app/src/main/cpp/ad-block/cosmetic_filter.cc b/app/src/main/cpp/third-party/ad-block/cosmetic_filter.cc similarity index 100% rename from app/src/main/cpp/ad-block/cosmetic_filter.cc rename to app/src/main/cpp/third-party/ad-block/cosmetic_filter.cc diff --git a/app/src/main/cpp/ad-block/cosmetic_filter.h b/app/src/main/cpp/third-party/ad-block/cosmetic_filter.h similarity index 100% rename from app/src/main/cpp/ad-block/cosmetic_filter.h rename to app/src/main/cpp/third-party/ad-block/cosmetic_filter.h diff --git a/app/src/main/cpp/ad-block/filter.cc b/app/src/main/cpp/third-party/ad-block/filter.cc similarity index 100% rename from app/src/main/cpp/ad-block/filter.cc rename to app/src/main/cpp/third-party/ad-block/filter.cc diff --git a/app/src/main/cpp/ad-block/filter.h b/app/src/main/cpp/third-party/ad-block/filter.h similarity index 100% rename from app/src/main/cpp/ad-block/filter.h rename to app/src/main/cpp/third-party/ad-block/filter.h diff --git a/app/src/main/cpp/bloom-filter-cpp/BloomFilter.cpp b/app/src/main/cpp/third-party/bloom-filter-cpp/BloomFilter.cpp similarity index 100% rename from app/src/main/cpp/bloom-filter-cpp/BloomFilter.cpp rename to app/src/main/cpp/third-party/bloom-filter-cpp/BloomFilter.cpp diff --git a/app/src/main/cpp/bloom-filter-cpp/BloomFilter.h b/app/src/main/cpp/third-party/bloom-filter-cpp/BloomFilter.h similarity index 100% rename from app/src/main/cpp/bloom-filter-cpp/BloomFilter.h rename to app/src/main/cpp/third-party/bloom-filter-cpp/BloomFilter.h diff --git a/app/src/main/cpp/bloom-filter-cpp/base.h b/app/src/main/cpp/third-party/bloom-filter-cpp/base.h similarity index 100% rename from app/src/main/cpp/bloom-filter-cpp/base.h rename to app/src/main/cpp/third-party/bloom-filter-cpp/base.h diff --git a/app/src/main/cpp/bloom-filter-cpp/hashFn.cpp b/app/src/main/cpp/third-party/bloom-filter-cpp/hashFn.cpp similarity index 100% rename from app/src/main/cpp/bloom-filter-cpp/hashFn.cpp rename to app/src/main/cpp/third-party/bloom-filter-cpp/hashFn.cpp diff --git a/app/src/main/cpp/bloom-filter-cpp/hashFn.h b/app/src/main/cpp/third-party/bloom-filter-cpp/hashFn.h similarity index 100% rename from app/src/main/cpp/bloom-filter-cpp/hashFn.h rename to app/src/main/cpp/third-party/bloom-filter-cpp/hashFn.h diff --git a/app/src/main/cpp/third-party/bloom_cpp b/app/src/main/cpp/third-party/bloom_cpp new file mode 160000 index 000000000000..e3d110ad0b0c --- /dev/null +++ b/app/src/main/cpp/third-party/bloom_cpp @@ -0,0 +1 @@ +Subproject commit e3d110ad0b0cd43140a23ee42f98b7bdeaa7ee36 diff --git a/app/src/main/cpp/hashset-cpp/HashItem.h b/app/src/main/cpp/third-party/hashset-cpp/HashItem.h similarity index 100% rename from app/src/main/cpp/hashset-cpp/HashItem.h rename to app/src/main/cpp/third-party/hashset-cpp/HashItem.h diff --git a/app/src/main/cpp/hashset-cpp/HashSet.cpp b/app/src/main/cpp/third-party/hashset-cpp/HashSet.cpp similarity index 100% rename from app/src/main/cpp/hashset-cpp/HashSet.cpp rename to app/src/main/cpp/third-party/hashset-cpp/HashSet.cpp diff --git a/app/src/main/cpp/hashset-cpp/HashSet.h b/app/src/main/cpp/third-party/hashset-cpp/HashSet.h similarity index 100% rename from app/src/main/cpp/hashset-cpp/HashSet.h rename to app/src/main/cpp/third-party/hashset-cpp/HashSet.h diff --git a/app/src/main/cpp/hashset-cpp/base.h b/app/src/main/cpp/third-party/hashset-cpp/base.h similarity index 100% rename from app/src/main/cpp/hashset-cpp/base.h rename to app/src/main/cpp/third-party/hashset-cpp/base.h diff --git a/app/src/main/cpp/hashset-cpp/hashFn.h b/app/src/main/cpp/third-party/hashset-cpp/hashFn.h similarity index 100% rename from app/src/main/cpp/hashset-cpp/hashFn.h rename to app/src/main/cpp/third-party/hashset-cpp/hashFn.h diff --git a/app/src/main/java/com/duckduckgo/app/di/AppConfigurationDownloadModule.kt b/app/src/main/java/com/duckduckgo/app/di/AppConfigurationDownloadModule.kt index 7a1678bc4231..37bbc1d4210a 100644 --- a/app/src/main/java/com/duckduckgo/app/di/AppConfigurationDownloadModule.kt +++ b/app/src/main/java/com/duckduckgo/app/di/AppConfigurationDownloadModule.kt @@ -17,7 +17,7 @@ package com.duckduckgo.app.di import com.duckduckgo.app.global.db.AppDatabase -import com.duckduckgo.app.httpsupgrade.api.HttpsUpgradeListDownloader +import com.duckduckgo.app.httpsupgrade.api.HttpsUpgradeDataDownloader import com.duckduckgo.app.job.AppConfigurationDownloader import com.duckduckgo.app.job.ConfigurationDownloader import com.duckduckgo.app.surrogates.api.ResourceSurrogateListDownloader @@ -30,9 +30,9 @@ open class AppConfigurationDownloaderModule { @Provides open fun appConfigurationDownloader(trackerDataDownloader: TrackerDataDownloader, - httpsUpgradeListDownloader: HttpsUpgradeListDownloader, + httpsUpgradeDataDownloader: HttpsUpgradeDataDownloader, resourceSurrogateDownloader: ResourceSurrogateListDownloader, appDatabase: AppDatabase) : ConfigurationDownloader { - return AppConfigurationDownloader(trackerDataDownloader, httpsUpgradeListDownloader, resourceSurrogateDownloader, appDatabase) + return AppConfigurationDownloader(trackerDataDownloader, httpsUpgradeDataDownloader, resourceSurrogateDownloader, appDatabase) } } \ No newline at end of file diff --git a/app/src/main/java/com/duckduckgo/app/di/ApplicationModule.kt b/app/src/main/java/com/duckduckgo/app/di/ApplicationModule.kt index cce2063c7a90..ccce7dd1ef41 100644 --- a/app/src/main/java/com/duckduckgo/app/di/ApplicationModule.kt +++ b/app/src/main/java/com/duckduckgo/app/di/ApplicationModule.kt @@ -18,7 +18,6 @@ package com.duckduckgo.app.di import android.app.Application import android.content.Context -import com.duckduckgo.app.global.device.DeviceInfo import dagger.Binds import dagger.Module import javax.inject.Singleton diff --git a/app/src/main/java/com/duckduckgo/app/di/DaoModule.kt b/app/src/main/java/com/duckduckgo/app/di/DaoModule.kt index 01297a5f9923..75d82eb8fc03 100644 --- a/app/src/main/java/com/duckduckgo/app/di/DaoModule.kt +++ b/app/src/main/java/com/duckduckgo/app/di/DaoModule.kt @@ -24,10 +24,13 @@ import dagger.Provides class DaoModule { @Provides - fun provideHttpsUpgradeDomainDao(database: AppDatabase) = database.httpsUpgradeDomainDao() + fun providesHttpsWhitelistDao(database: AppDatabase) = database.httpsWhitelistedDao() @Provides - fun provideDisconnectTrackDao(database: AppDatabase) = database.trackerDataDao() + fun provideHttpsBloomFilterSpecDao(database: AppDatabase) = database.httpsBloomFilterSpecDao() + + @Provides + fun providesDisconnectTrackDao(database: AppDatabase) = database.trackerDataDao() @Provides fun providesNetworkLeaderboardDao(database: AppDatabase) = database.networkLeaderboardDao() diff --git a/app/src/main/java/com/duckduckgo/app/di/DatabaseModule.kt b/app/src/main/java/com/duckduckgo/app/di/DatabaseModule.kt index 448d7d84e500..82bba57eee78 100644 --- a/app/src/main/java/com/duckduckgo/app/di/DatabaseModule.kt +++ b/app/src/main/java/com/duckduckgo/app/di/DatabaseModule.kt @@ -21,6 +21,7 @@ import android.content.Context import com.duckduckgo.app.global.db.AppDatabase import com.duckduckgo.app.global.db.AppDatabase.Companion.MIGRATION_1_TO_2 import com.duckduckgo.app.global.db.AppDatabase.Companion.MIGRATION_2_TO_3 +import com.duckduckgo.app.global.db.AppDatabase.Companion.MIGRATION_3_TO_4 import dagger.Module import dagger.Provides import javax.inject.Singleton @@ -32,7 +33,7 @@ class DatabaseModule { @Singleton fun provideDatabase(context: Context): AppDatabase { return Room.databaseBuilder(context, AppDatabase::class.java, "app.db") - .addMigrations(MIGRATION_1_TO_2, MIGRATION_2_TO_3) + .addMigrations(MIGRATION_1_TO_2, MIGRATION_2_TO_3, MIGRATION_3_TO_4) .build() } diff --git a/app/src/main/java/com/duckduckgo/app/di/JsonModule.kt b/app/src/main/java/com/duckduckgo/app/di/JsonModule.kt index 77a2c7c07f6a..0f00b4919554 100644 --- a/app/src/main/java/com/duckduckgo/app/di/JsonModule.kt +++ b/app/src/main/java/com/duckduckgo/app/di/JsonModule.kt @@ -16,13 +16,10 @@ package com.duckduckgo.app.di -import com.duckduckgo.app.httpsupgrade.api.HttpsUpgradeDomainFromStringAdapter -import com.duckduckgo.app.httpsupgrade.db.HttpsUpgradeDomain +import com.duckduckgo.app.httpsupgrade.api.HttpsWhitelistJsonAdapter import com.duckduckgo.app.privacy.api.TermsOfServiceListAdapter import com.duckduckgo.app.trackerdetection.api.DisconnectListJsonAdapter -import com.squareup.moshi.JsonAdapter import com.squareup.moshi.Moshi -import com.squareup.moshi.Types import dagger.Module import dagger.Provides import javax.inject.Singleton @@ -33,15 +30,8 @@ class JsonModule { @Provides @Singleton fun moshi(): Moshi = Moshi.Builder() - .add(DisconnectListJsonAdapter()) + .add(HttpsWhitelistJsonAdapter()) + .add(DisconnectListJsonAdapter()) .add(TermsOfServiceListAdapter()) - .add(HttpsUpgradeDomainFromStringAdapter()) .build() - - @Provides - fun httpsUpgradeDomainAdapter(moshi: Moshi): JsonAdapter> { - val type = Types.newParameterizedType(List::class.java, HttpsUpgradeDomain::class.java) - return moshi.adapter(type) - } - } \ No newline at end of file diff --git a/app/src/main/java/com/duckduckgo/app/di/NetworkModule.kt b/app/src/main/java/com/duckduckgo/app/di/NetworkModule.kt index 2d4cfcc046c7..fff8690d2ecd 100644 --- a/app/src/main/java/com/duckduckgo/app/di/NetworkModule.kt +++ b/app/src/main/java/com/duckduckgo/app/di/NetworkModule.kt @@ -25,7 +25,7 @@ import com.duckduckgo.app.feedback.api.FeedbackSubmitter import com.duckduckgo.app.global.AppUrl.Url import com.duckduckgo.app.global.api.ApiRequestInterceptor import com.duckduckgo.app.global.job.JobBuilder -import com.duckduckgo.app.httpsupgrade.api.HttpsUpgradeListService +import com.duckduckgo.app.httpsupgrade.api.HttpsUpgradeService import com.duckduckgo.app.job.AppConfigurationSyncer import com.duckduckgo.app.job.ConfigurationDownloader import com.duckduckgo.app.statistics.VariantManager @@ -100,8 +100,8 @@ class NetworkModule { retrofit.create(TrackerListService::class.java) @Provides - fun httpsUpgradeListService(@Named("api") retrofit: Retrofit): HttpsUpgradeListService = - retrofit.create(HttpsUpgradeListService::class.java) + fun httpsUpgradeService(@Named("api") retrofit: Retrofit): HttpsUpgradeService = + retrofit.create(HttpsUpgradeService::class.java) @Provides fun autoCompleteService(@Named("api") retrofit: Retrofit): AutoCompleteService = diff --git a/app/src/main/java/com/duckduckgo/app/di/StatisticsModule.kt b/app/src/main/java/com/duckduckgo/app/di/StatisticsModule.kt index b72dbbbe5f12..46791db9eaec 100644 --- a/app/src/main/java/com/duckduckgo/app/di/StatisticsModule.kt +++ b/app/src/main/java/com/duckduckgo/app/di/StatisticsModule.kt @@ -17,8 +17,6 @@ package com.duckduckgo.app.di import android.content.Context -import com.duckduckgo.app.global.AppUrl -import com.duckduckgo.app.global.api.ApiRequestInterceptor import com.duckduckgo.app.global.device.ContextDeviceInfo import com.duckduckgo.app.global.device.DeviceInfo import com.duckduckgo.app.statistics.VariantManager @@ -33,7 +31,6 @@ import dagger.Module import dagger.Provides import okhttp3.OkHttpClient import retrofit2.Retrofit -import retrofit2.adapter.rxjava2.RxJava2CallAdapterFactory import javax.inject.Named diff --git a/app/src/main/java/com/duckduckgo/app/global/Checksum.kt b/app/src/main/java/com/duckduckgo/app/global/Checksum.kt new file mode 100644 index 000000000000..403d271b6296 --- /dev/null +++ b/app/src/main/java/com/duckduckgo/app/global/Checksum.kt @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2018 DuckDuckGo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.duckduckgo.app.global + +import java.math.BigInteger +import java.security.MessageDigest + + +val ByteArray.sha256: String + get() { + val md = MessageDigest.getInstance("SHA-256") + md.reset() + val digest = md.digest(this) + return String.format("%0" + digest.size * 2 + "x", BigInteger(1, digest)) + } + +fun ByteArray.verifySha256(sha256: String): Boolean { + return this.sha256 == sha256 +} \ No newline at end of file diff --git a/app/src/main/java/com/duckduckgo/app/global/db/AppDatabase.kt b/app/src/main/java/com/duckduckgo/app/global/db/AppDatabase.kt index 858c7a6e7780..9f9d3ba156d4 100644 --- a/app/src/main/java/com/duckduckgo/app/global/db/AppDatabase.kt +++ b/app/src/main/java/com/duckduckgo/app/global/db/AppDatabase.kt @@ -22,8 +22,10 @@ import android.arch.persistence.room.RoomDatabase import android.arch.persistence.room.migration.Migration import com.duckduckgo.app.bookmarks.db.BookmarkEntity import com.duckduckgo.app.bookmarks.db.BookmarksDao -import com.duckduckgo.app.httpsupgrade.db.HttpsUpgradeDomain -import com.duckduckgo.app.httpsupgrade.db.HttpsUpgradeDomainDao +import com.duckduckgo.app.httpsupgrade.db.HttpsBloomFilterSpecDao +import com.duckduckgo.app.httpsupgrade.db.HttpsWhitelistDao +import com.duckduckgo.app.httpsupgrade.model.HttpsBloomFilterSpec +import com.duckduckgo.app.httpsupgrade.model.HttpsWhitelistedDomain import com.duckduckgo.app.privacy.db.NetworkLeaderboardDao import com.duckduckgo.app.privacy.db.NetworkLeaderboardEntry import com.duckduckgo.app.privacy.db.SiteVisitedEntity @@ -33,9 +35,10 @@ import com.duckduckgo.app.tabs.model.TabSelectionEntity import com.duckduckgo.app.trackerdetection.db.TrackerDataDao import com.duckduckgo.app.trackerdetection.model.DisconnectTracker -@Database(exportSchema = true, version = 3, entities = [ - HttpsUpgradeDomain::class, +@Database(exportSchema = true, version = 4, entities = [ DisconnectTracker::class, + HttpsBloomFilterSpec::class, + HttpsWhitelistedDomain::class, NetworkLeaderboardEntry::class, SiteVisitedEntity::class, AppConfigurationEntity::class, @@ -46,8 +49,9 @@ import com.duckduckgo.app.trackerdetection.model.DisconnectTracker abstract class AppDatabase : RoomDatabase() { - abstract fun httpsUpgradeDomainDao(): HttpsUpgradeDomainDao abstract fun trackerDataDao(): TrackerDataDao + abstract fun httpsWhitelistedDao(): HttpsWhitelistDao + abstract fun httpsBloomFilterSpecDao(): HttpsBloomFilterSpecDao abstract fun networkLeaderboardDao(): NetworkLeaderboardDao abstract fun appConfigurationDao(): AppConfigurationDao abstract fun tabsDao(): TabsDao @@ -69,5 +73,13 @@ abstract class AppDatabase : RoomDatabase() { database.execSQL("DELETE FROM `network_leaderboard`") } } + + val MIGRATION_3_TO_4: Migration = object : Migration(3, 4) { + override fun migrate(database: SupportSQLiteDatabase) { + database.execSQL("DROP TABLE https_upgrade_domain") + database.execSQL("CREATE TABLE `https_bloom_filter_spec` (`id` INTEGER NOT NULL, `errorRate` REAL NOT NULL, `totalEntries` INTEGER NOT NULL, `sha256` TEXT NOT NULL, PRIMARY KEY(`id`))") + database.execSQL("CREATE TABLE `https_whitelisted_domain` (`domain` TEXT NOT NULL, PRIMARY KEY(`domain`))") + } + } } } \ No newline at end of file diff --git a/app/src/main/java/com/duckduckgo/app/global/store/BinaryDataStore.kt b/app/src/main/java/com/duckduckgo/app/global/store/BinaryDataStore.kt new file mode 100644 index 000000000000..2f60cca071df --- /dev/null +++ b/app/src/main/java/com/duckduckgo/app/global/store/BinaryDataStore.kt @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2017 DuckDuckGo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.duckduckgo.app.global.store + +import android.content.Context +import com.duckduckgo.app.global.verifySha256 +import javax.inject.Inject + +class BinaryDataStore @Inject constructor(private val context: Context) { + + fun hasData(name: String): Boolean = context.fileExists(name) + + fun loadData(name: String): ByteArray = + context.openFileInput(name).use { it.readBytes() } + + fun saveData(name: String, byteArray: ByteArray) { + context.openFileOutput(name, Context.MODE_PRIVATE).write(byteArray) + } + + fun clearData(name: String) { + context.deleteFile(name) + } + + fun dataFilePath(name: String): String? { + return context.filePath(name) + } + + fun verifyCheckSum(name: String, sha256: String): Boolean { + if (context.fileExists(name)) { + return verifyCheckSum(loadData(name), sha256) + } + return false + } + + fun verifyCheckSum(bytes: ByteArray, sha256: String): Boolean { + return bytes.verifySha256(sha256) + } + + private fun Context.filePath(filename: String): String? { + val file = getFileStreamPath(filename) + return if (file != null && file.exists()) file.path else null + } + + private fun Context.fileExists(filename: String): Boolean { + return filePath(filename) != null + } +} diff --git a/app/src/main/java/com/duckduckgo/app/httpsupgrade/BloomFilter.kt b/app/src/main/java/com/duckduckgo/app/httpsupgrade/BloomFilter.kt new file mode 100644 index 000000000000..b9a66bcab54b --- /dev/null +++ b/app/src/main/java/com/duckduckgo/app/httpsupgrade/BloomFilter.kt @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2018 DuckDuckGo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.duckduckgo.app.httpsupgrade + +class BloomFilter { + + private val nativePointer: Long + + init { + System.loadLibrary("https-bloom-lib") + } + + constructor(maxItems: Int, targetProbability: Double) { + nativePointer = createBloomFilter(maxItems, targetProbability) + } + + constructor(path: String, maxItems: Int) { + nativePointer = createBloomFilterFromFile(path, maxItems) + } + + private external fun createBloomFilter(maxItems: Int, targetProbability: Double): Long + + private external fun createBloomFilterFromFile(path: String, maxItems: Int): Long + + + fun add(element: String) { + add(nativePointer, element) + } + + private external fun add(nativePointer: Long, element: String) + + fun contains(element: String): Boolean { + return contains(nativePointer, element) + } + + private external fun contains(nativePointer: Long, element: String): Boolean + + @Suppress("unused", "protectedInFinal") + protected fun finalize() { + releaseBloomFilter(nativePointer) + } + + private external fun releaseBloomFilter(nativePointer: Long) +} \ No newline at end of file diff --git a/app/src/main/java/com/duckduckgo/app/httpsupgrade/HttpsUpgrader.kt b/app/src/main/java/com/duckduckgo/app/httpsupgrade/HttpsUpgrader.kt index 6afa567039ed..9b6845102bee 100644 --- a/app/src/main/java/com/duckduckgo/app/httpsupgrade/HttpsUpgrader.kt +++ b/app/src/main/java/com/duckduckgo/app/httpsupgrade/HttpsUpgrader.kt @@ -20,52 +20,68 @@ import android.net.Uri import android.support.annotation.WorkerThread import com.duckduckgo.app.global.UrlScheme import com.duckduckgo.app.global.isHttps -import com.duckduckgo.app.httpsupgrade.db.HttpsUpgradeDomainDao +import com.duckduckgo.app.httpsupgrade.api.HttpsBloomFilterFactory +import com.duckduckgo.app.httpsupgrade.db.HttpsWhitelistDao +import io.reactivex.schedulers.Schedulers import timber.log.Timber interface HttpsUpgrader { @WorkerThread - fun shouldUpgrade(uri: Uri) : Boolean + fun shouldUpgrade(uri: Uri): Boolean fun upgrade(uri: Uri): Uri { return uri.buildUpon().scheme(UrlScheme.https).build() } + + fun reloadData() } -class HttpsUpgraderImpl constructor(private val dao: HttpsUpgradeDomainDao) :HttpsUpgrader { +class HttpsUpgraderImpl( + private val whitelistedDao: HttpsWhitelistDao, + private val bloomFactory: HttpsBloomFilterFactory +) : HttpsUpgrader { + + private var httpsBloomFilter: BloomFilter? = null + + init { + reloadData() + } @WorkerThread - override fun shouldUpgrade(uri: Uri) : Boolean { + override fun shouldUpgrade(uri: Uri): Boolean { + if (uri.isHttps) { return false } - val host = (uri.host ?: return false).toLowerCase() - return dao.hasDomain(host) || matchesWildcard(host) - } - - private fun matchesWildcard(host: String): Boolean { - val domains = mutableListOf() - for (part in host.split(".").reversed()) { - if (domains.isEmpty()) { - domains.add(".$part") - } else { - val last = domains.last() - domains.add(".$part$last") - } + val host = uri.host + if (whitelistedDao.contains(host)) { + Timber.d("${host} is in whitelist and so not upgradable") + return false } - domains.asReversed().removeAt(0) - Timber.d("domains $domains") + httpsBloomFilter?.let { - for (domain in domains) { - if (dao.hasDomain("*$domain")) { - return true - } + val initialTime = System.nanoTime() + val shouldUpgrade = it.contains(host) + val totalTime = System.nanoTime() - initialTime + Timber.d("${host} ${if (shouldUpgrade) "is" else "is not"} upgradable, lookup in ${totalTime/NANO_TO_MILLIS_DIVISOR}ms") + + return shouldUpgrade } return false } -} \ No newline at end of file + override fun reloadData() { + Schedulers.io().scheduleDirect { + httpsBloomFilter = bloomFactory.create() + } + } + + companion object { + const val NANO_TO_MILLIS_DIVISOR = 1_000_000.0 + } + +} diff --git a/app/src/main/java/com/duckduckgo/app/httpsupgrade/api/HttpsBloomFilterFactory.kt b/app/src/main/java/com/duckduckgo/app/httpsupgrade/api/HttpsBloomFilterFactory.kt new file mode 100644 index 000000000000..36c43ca6e60d --- /dev/null +++ b/app/src/main/java/com/duckduckgo/app/httpsupgrade/api/HttpsBloomFilterFactory.kt @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2018 DuckDuckGo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.duckduckgo.app.httpsupgrade.api + +import android.support.annotation.WorkerThread +import com.duckduckgo.app.global.store.BinaryDataStore +import com.duckduckgo.app.httpsupgrade.BloomFilter +import com.duckduckgo.app.httpsupgrade.db.HttpsBloomFilterSpecDao +import com.duckduckgo.app.httpsupgrade.model.HttpsBloomFilterSpec.Companion.HTTPS_BINARY_FILE +import timber.log.Timber +import javax.inject.Inject + +interface HttpsBloomFilterFactory { + fun create(): BloomFilter? +} + + +class HttpsBloomFilterFactoryImpl @Inject constructor(private val dao: HttpsBloomFilterSpecDao, private val binaryDataStore: BinaryDataStore) : + HttpsBloomFilterFactory { + + @WorkerThread + override fun create(): BloomFilter? { + + val specification = dao.get() + val dataPath = binaryDataStore.dataFilePath(HTTPS_BINARY_FILE) + + if (dataPath == null || specification == null) { + Timber.d("Https update data not found") + return null + } + + val initialTimestamp = System.currentTimeMillis() + Timber.d("Found https data at $dataPath, building filter") + var bloomFilter = BloomFilter(dataPath, specification.totalEntries) + Timber.v("Loading took ${System.currentTimeMillis() - initialTimestamp}ms") + + return bloomFilter + } +} \ No newline at end of file diff --git a/app/src/main/java/com/duckduckgo/app/httpsupgrade/api/HttpsUpgradeDataDownloader.kt b/app/src/main/java/com/duckduckgo/app/httpsupgrade/api/HttpsUpgradeDataDownloader.kt new file mode 100644 index 000000000000..f881b88c3250 --- /dev/null +++ b/app/src/main/java/com/duckduckgo/app/httpsupgrade/api/HttpsUpgradeDataDownloader.kt @@ -0,0 +1,106 @@ +/* + * Copyright (c) 2018 DuckDuckGo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.duckduckgo.app.httpsupgrade.api + +import com.duckduckgo.app.global.api.isCached +import com.duckduckgo.app.global.store.BinaryDataStore +import com.duckduckgo.app.httpsupgrade.HttpsUpgrader +import com.duckduckgo.app.httpsupgrade.db.HttpsBloomFilterSpecDao +import com.duckduckgo.app.httpsupgrade.db.HttpsWhitelistDao +import com.duckduckgo.app.httpsupgrade.model.HttpsBloomFilterSpec +import com.duckduckgo.app.httpsupgrade.model.HttpsBloomFilterSpec.Companion.HTTPS_BINARY_FILE +import io.reactivex.Completable +import io.reactivex.Completable.fromAction +import timber.log.Timber +import java.io.IOException +import javax.inject.Inject + +class HttpsUpgradeDataDownloader @Inject constructor( + private val service: HttpsUpgradeService, + private val httpsUpgrader: HttpsUpgrader, + private val httpsBloomSpecDao: HttpsBloomFilterSpecDao, + private val whitelistDao: HttpsWhitelistDao, + private val binaryDataStore: BinaryDataStore +) { + + fun download(): Completable { + + val filter = service.httpsBloomFilterSpec() + .flatMapCompletable { + downloadBloomFilter(it) + } + val whitelist = downloadWhitelist() + + return Completable.mergeDelayError(listOf(filter, whitelist)) + .doOnComplete { + Timber.i("Https download task completed successfully") + } + } + + private fun downloadBloomFilter(specification: HttpsBloomFilterSpec): Completable { + return fromAction { + Timber.d("Downloading https bloom filter binary") + val call = service.httpsBloomFilter() + val response = call.execute() + val fileName = HTTPS_BINARY_FILE + + if (response.isCached && binaryDataStore.verifyCheckSum(fileName, specification.sha256)) { + Timber.d("Https bloom data already cached and stored for this spec") + return@fromAction + } + + if (!response.isSuccessful) { + throw IOException("Status: ${response.code()} - ${response.errorBody()?.string()}") + } + + val bytes = response.body()!!.bytes() + if (!binaryDataStore.verifyCheckSum(bytes, specification.sha256)) { + throw IOException("Https binary has incorrect checksum, throwisng away file") + } + + Timber.d("Updating https bloom data store with new data") + httpsBloomSpecDao.insert(specification) + binaryDataStore.saveData(fileName, bytes) + httpsUpgrader.reloadData() + } + } + + private fun downloadWhitelist(): Completable { + + Timber.d("Downloading HTTPS whitelist") + return fromAction { + + val call = service.whitelist() + val response = call.execute() + + if (response.isCached && whitelistDao.count() > 0) { + Timber.d("Https whitelist already cached and stored") + return@fromAction + } + + if (response.isSuccessful) { + val whitelist = response.body()!! + Timber.d("Updating https whitelist with new data") + whitelistDao.updateAll(whitelist) + } else { + throw IOException("Status: ${response.code()} - ${response.errorBody()?.string()}") + } + } + + } + +} \ No newline at end of file diff --git a/app/src/main/java/com/duckduckgo/app/httpsupgrade/api/HttpsUpgradeListDownloader.kt b/app/src/main/java/com/duckduckgo/app/httpsupgrade/api/HttpsUpgradeListDownloader.kt deleted file mode 100644 index a37fd4154028..000000000000 --- a/app/src/main/java/com/duckduckgo/app/httpsupgrade/api/HttpsUpgradeListDownloader.kt +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Copyright (c) 2017 DuckDuckGo - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.duckduckgo.app.httpsupgrade.api - -import com.duckduckgo.app.global.api.isCached -import com.duckduckgo.app.global.db.AppDatabase -import com.duckduckgo.app.httpsupgrade.db.HttpsUpgradeDbWriteStatusStore -import com.duckduckgo.app.httpsupgrade.db.HttpsUpgradeDomainDao -import io.reactivex.Completable -import timber.log.Timber -import java.io.IOException -import javax.inject.Inject - -/** - * For performance reasons, we can't insert all the HTTPS rows in a single transaction as it blocks other DB writes from happening for a while - * To avoid the performance problem of a single transaction, we can chunk the data and perform many smaller transactions. - * However, if we don't use a single transaction, there is the risk the process could die or exception be thrown while iterating through the HTTPS inserts, leaving the DB in a bad state. - * The network cache could therefore deliver the cached value and we could skip writing to the DB thinking we'd already successfully written it all. - * - * To counter this, we need an external mechanism to track whether the write completed successfully or not. - * Using this, we can detect if the write failed when we next download the data. We'll only skip the DB inserts if cache returns the list and the - * the eTag received in the response headers matches the eTag we stored on last successful write, indicating that it finished writing successfully. - * @see HttpsUpgradeDbWriteStatusStore for how this status flag is stored. - */ -class HttpsUpgradeListDownloader @Inject constructor( - private val service: HttpsUpgradeListService, - private val database: AppDatabase, - private val httpsUpgradeDao: HttpsUpgradeDomainDao, - private val dbWriteStatusStore: HttpsUpgradeDbWriteStatusStore -) { - - fun downloadList(chunkSize: Int = INSERTION_CHUNK_SIZE): Completable { - - Timber.d("Downloading HTTPS Upgrade data") - - return Completable.fromAction { - - val call = service.https() - val response = call.execute() - val eTag = response.headers().get(HEADER_ETAG) - - if (response.isCached && dbWriteStatusStore.isMatchingETag(eTag)) { - Timber.d("HTTPS data already cached and stored") - return@fromAction - } - - if (response.isSuccessful) { - Timber.d("Updating HTTPS upgrade list from server") - - val domains = response.body() ?: throw IllegalStateException("Failed to obtain HTTPS upgrade list") - - val startTime = System.currentTimeMillis() - - httpsUpgradeDao.deleteAll() - Timber.v("Took ${System.currentTimeMillis() - startTime}ms to delete existing records") - - val chunks = domains.chunked(chunkSize) - Timber.i("Received ${domains.size} HTTPS domains; chunking by $chunkSize into ${chunks.size} separate DB transactions") - - chunks.forEach { - database.runInTransaction { - httpsUpgradeDao.insertAll(it) - } - } - - dbWriteStatusStore.saveETag(eTag) - Timber.i("Successfully wrote HTTPS data; took ${System.currentTimeMillis() - startTime}ms") - - } else { - throw IOException("Status: ${response.code()} - ${response.errorBody()?.string()}") - } - } - } - - companion object { - private const val INSERTION_CHUNK_SIZE = 1_000 - private const val HEADER_ETAG = "etag" - } -} \ No newline at end of file diff --git a/app/src/main/java/com/duckduckgo/app/httpsupgrade/api/HttpsUpgradeService.kt b/app/src/main/java/com/duckduckgo/app/httpsupgrade/api/HttpsUpgradeService.kt new file mode 100644 index 000000000000..9a129f592ea5 --- /dev/null +++ b/app/src/main/java/com/duckduckgo/app/httpsupgrade/api/HttpsUpgradeService.kt @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2018 DuckDuckGo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.duckduckgo.app.httpsupgrade.api + +import com.duckduckgo.app.httpsupgrade.model.HttpsBloomFilterSpec +import com.duckduckgo.app.httpsupgrade.model.HttpsWhitelistedDomain +import io.reactivex.Observable +import okhttp3.ResponseBody +import retrofit2.Call +import retrofit2.http.GET + +interface HttpsUpgradeService { + + @GET("https://staticcdn.duckduckgo.com/https/https-mobile-whitelist.json") + fun whitelist(): Call> + + @GET("https://staticcdn.duckduckgo.com/https/https-mobile-bloom-spec.json") + fun httpsBloomFilterSpec(): Observable + + @GET("https://staticcdn.duckduckgo.com/https/https-mobile-bloom.bin") + fun httpsBloomFilter(): Call +} diff --git a/app/src/main/java/com/duckduckgo/app/httpsupgrade/api/HttpsUpgradeDomainFromStringAdapter.kt b/app/src/main/java/com/duckduckgo/app/httpsupgrade/api/HttpsWhitelistJsonAdapter.kt similarity index 67% rename from app/src/main/java/com/duckduckgo/app/httpsupgrade/api/HttpsUpgradeDomainFromStringAdapter.kt rename to app/src/main/java/com/duckduckgo/app/httpsupgrade/api/HttpsWhitelistJsonAdapter.kt index 50833051e4cf..4a8d3f364e79 100644 --- a/app/src/main/java/com/duckduckgo/app/httpsupgrade/api/HttpsUpgradeDomainFromStringAdapter.kt +++ b/app/src/main/java/com/duckduckgo/app/httpsupgrade/api/HttpsWhitelistJsonAdapter.kt @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017 DuckDuckGo + * Copyright (c) 2018 DuckDuckGo * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,11 +16,14 @@ package com.duckduckgo.app.httpsupgrade.api -import com.duckduckgo.app.httpsupgrade.db.HttpsUpgradeDomain +import com.duckduckgo.app.httpsupgrade.model.HttpsWhitelistedDomain import com.squareup.moshi.FromJson -class HttpsUpgradeDomainFromStringAdapter { +class HttpsWhitelistJsonAdapter { - @FromJson fun adapt(domain: String) = HttpsUpgradeDomain(domain) + @FromJson + fun adapt(data: Map>): List { + return data.getValue("data").map { HttpsWhitelistedDomain(it) } + } } \ No newline at end of file diff --git a/app/src/main/java/com/duckduckgo/app/httpsupgrade/db/HttpsUpgradeDomainDao.kt b/app/src/main/java/com/duckduckgo/app/httpsupgrade/db/HttpsBloomFilterSpecDao.kt similarity index 68% rename from app/src/main/java/com/duckduckgo/app/httpsupgrade/db/HttpsUpgradeDomainDao.kt rename to app/src/main/java/com/duckduckgo/app/httpsupgrade/db/HttpsBloomFilterSpecDao.kt index ecdb3cab4fa4..7deb207c0c92 100644 --- a/app/src/main/java/com/duckduckgo/app/httpsupgrade/db/HttpsUpgradeDomainDao.kt +++ b/app/src/main/java/com/duckduckgo/app/httpsupgrade/db/HttpsBloomFilterSpecDao.kt @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017 DuckDuckGo + * Copyright (c) 2018 DuckDuckGo * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,20 +20,16 @@ import android.arch.persistence.room.Dao import android.arch.persistence.room.Insert import android.arch.persistence.room.OnConflictStrategy import android.arch.persistence.room.Query +import com.duckduckgo.app.httpsupgrade.model.HttpsBloomFilterSpec +import javax.inject.Singleton @Dao -interface HttpsUpgradeDomainDao { +@Singleton +interface HttpsBloomFilterSpecDao { @Insert(onConflict = OnConflictStrategy.REPLACE) - fun insertAll(domains: List) - - @Query("select count(1) > 0 from https_upgrade_domain where domain = :domain") - fun hasDomain(domain: String) : Boolean - - @Query("delete from https_upgrade_domain") - fun deleteAll() - - @Query("select count(1) from https_upgrade_domain") - fun count(): Int + fun insert(specification: HttpsBloomFilterSpec) + @Query("select * from https_bloom_filter_spec limit 1") + fun get(): HttpsBloomFilterSpec? } \ No newline at end of file diff --git a/app/src/main/java/com/duckduckgo/app/httpsupgrade/db/HttpsUpgradeDbWriteStatusStore.kt b/app/src/main/java/com/duckduckgo/app/httpsupgrade/db/HttpsUpgradeDbWriteStatusStore.kt deleted file mode 100644 index 84dde1cb9fce..000000000000 --- a/app/src/main/java/com/duckduckgo/app/httpsupgrade/db/HttpsUpgradeDbWriteStatusStore.kt +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright (c) 2018 DuckDuckGo - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.duckduckgo.app.httpsupgrade.db - -import android.content.Context -import android.content.SharedPreferences -import android.support.annotation.VisibleForTesting -import androidx.core.content.edit -import javax.inject.Inject - -interface HttpsUpgradeDbWriteStatusStore { - - fun saveETag(eTag: String?) - fun isMatchingETag(eTag: String?): Boolean -} - -class HttpsUpgradeDbWriteStatusSharedPreferences @Inject constructor(private val context: Context) : HttpsUpgradeDbWriteStatusStore { - - override fun saveETag(eTag: String?) { - preferences.edit { putString(KEY_ETAG_OF_LAST_FULL_WRITE, eTag) } - } - - override fun isMatchingETag(eTag: String?): Boolean { - val existingETag = preferences.getString(KEY_ETAG_OF_LAST_FULL_WRITE, null) - return eTag == existingETag - } - - private val preferences: SharedPreferences - get() = context.getSharedPreferences(FILENAME, Context.MODE_PRIVATE) - - companion object { - - @VisibleForTesting - const val FILENAME = "com.duckduckgo.app.httpsupgrade.db.HttpsUpgradeDbWriteStatus" - const val KEY_ETAG_OF_LAST_FULL_WRITE = "KEY_ETAG_OF_LAST_FULL_WRITE" - } -} \ No newline at end of file diff --git a/app/src/main/java/com/duckduckgo/app/httpsupgrade/db/HttpsWhitelistDao.kt b/app/src/main/java/com/duckduckgo/app/httpsupgrade/db/HttpsWhitelistDao.kt new file mode 100644 index 000000000000..edb42e5b4029 --- /dev/null +++ b/app/src/main/java/com/duckduckgo/app/httpsupgrade/db/HttpsWhitelistDao.kt @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2018 DuckDuckGo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.duckduckgo.app.httpsupgrade.db + +import android.arch.persistence.room.* +import com.duckduckgo.app.httpsupgrade.model.HttpsWhitelistedDomain +import javax.inject.Singleton + +@Dao +@Singleton +abstract class HttpsWhitelistDao { + + @Insert(onConflict = OnConflictStrategy.REPLACE) + abstract fun insertAll(domains: List) + + @Transaction + open fun updateAll(domains: List) { + deleteAll() + insertAll(domains) + } + + @Query("select count(1) > 0 from https_whitelisted_domain where domain = :domain") + abstract fun contains(domain: String): Boolean + + @Query("delete from https_whitelisted_domain") + abstract fun deleteAll() + + @Query("select count(1) from https_whitelisted_domain") + abstract fun count(): Int +} \ No newline at end of file diff --git a/app/src/main/java/com/duckduckgo/app/httpsupgrade/di/HttpsUpgraderModule.kt b/app/src/main/java/com/duckduckgo/app/httpsupgrade/di/HttpsUpgraderModule.kt index 66fe18172a3c..583578ddb3ee 100644 --- a/app/src/main/java/com/duckduckgo/app/httpsupgrade/di/HttpsUpgraderModule.kt +++ b/app/src/main/java/com/duckduckgo/app/httpsupgrade/di/HttpsUpgraderModule.kt @@ -16,12 +16,13 @@ package com.duckduckgo.app.httpsupgrade.di -import android.content.Context +import com.duckduckgo.app.global.store.BinaryDataStore import com.duckduckgo.app.httpsupgrade.HttpsUpgrader import com.duckduckgo.app.httpsupgrade.HttpsUpgraderImpl -import com.duckduckgo.app.httpsupgrade.db.HttpsUpgradeDbWriteStatusSharedPreferences -import com.duckduckgo.app.httpsupgrade.db.HttpsUpgradeDbWriteStatusStore -import com.duckduckgo.app.httpsupgrade.db.HttpsUpgradeDomainDao +import com.duckduckgo.app.httpsupgrade.api.HttpsBloomFilterFactory +import com.duckduckgo.app.httpsupgrade.api.HttpsBloomFilterFactoryImpl +import com.duckduckgo.app.httpsupgrade.db.HttpsBloomFilterSpecDao +import com.duckduckgo.app.httpsupgrade.db.HttpsWhitelistDao import dagger.Module import dagger.Provides import javax.inject.Singleton @@ -29,14 +30,14 @@ import javax.inject.Singleton @Module class HttpsUpgraderModule { + @Singleton @Provides - fun httpsUpgrader(dao: HttpsUpgradeDomainDao): HttpsUpgrader { - return HttpsUpgraderImpl(dao) + fun httpsUpgrader(whitelistDao: HttpsWhitelistDao, bloomFilterFactory: HttpsBloomFilterFactory): HttpsUpgrader { + return HttpsUpgraderImpl(whitelistDao, bloomFilterFactory) } @Provides - @Singleton - fun httpsUpgradeDbWriteStatusStore(context: Context): HttpsUpgradeDbWriteStatusStore { - return HttpsUpgradeDbWriteStatusSharedPreferences(context) + fun bloomFilterFactory(specificationDao: HttpsBloomFilterSpecDao, binaryDataStore: BinaryDataStore): HttpsBloomFilterFactory { + return HttpsBloomFilterFactoryImpl(specificationDao, binaryDataStore) } } \ No newline at end of file diff --git a/app/src/main/java/com/duckduckgo/app/httpsupgrade/api/HttpsUpgradeListService.kt b/app/src/main/java/com/duckduckgo/app/httpsupgrade/model/HttpsBloomFilterSpec.kt similarity index 55% rename from app/src/main/java/com/duckduckgo/app/httpsupgrade/api/HttpsUpgradeListService.kt rename to app/src/main/java/com/duckduckgo/app/httpsupgrade/model/HttpsBloomFilterSpec.kt index db9eded6b897..6147c2809f86 100644 --- a/app/src/main/java/com/duckduckgo/app/httpsupgrade/api/HttpsUpgradeListService.kt +++ b/app/src/main/java/com/duckduckgo/app/httpsupgrade/model/HttpsBloomFilterSpec.kt @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017 DuckDuckGo + * Copyright (c) 2018 DuckDuckGo * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,15 +14,20 @@ * limitations under the License. */ -package com.duckduckgo.app.httpsupgrade.api +package com.duckduckgo.app.httpsupgrade.model -import com.duckduckgo.app.httpsupgrade.db.HttpsUpgradeDomain -import retrofit2.Call -import retrofit2.http.GET +import android.arch.persistence.room.Entity +import android.arch.persistence.room.PrimaryKey -interface HttpsUpgradeListService { - @GET("/contentblocking.js?l=https2") - fun https(): Call> - -} +@Entity(tableName = "https_bloom_filter_spec") +class HttpsBloomFilterSpec( + @PrimaryKey val id: Int = 1, + val errorRate: Double, + val totalEntries: Int, + val sha256: String +) { + companion object { + const val HTTPS_BINARY_FILE = "HTTPS_BINARY_FILE" + } +} \ No newline at end of file diff --git a/app/src/main/java/com/duckduckgo/app/httpsupgrade/db/HttpsUpgradeDomain.kt b/app/src/main/java/com/duckduckgo/app/httpsupgrade/model/HttpsWhitelistedDomain.kt similarity index 76% rename from app/src/main/java/com/duckduckgo/app/httpsupgrade/db/HttpsUpgradeDomain.kt rename to app/src/main/java/com/duckduckgo/app/httpsupgrade/model/HttpsWhitelistedDomain.kt index aa47ec735903..8e5de745069a 100644 --- a/app/src/main/java/com/duckduckgo/app/httpsupgrade/db/HttpsUpgradeDomain.kt +++ b/app/src/main/java/com/duckduckgo/app/httpsupgrade/model/HttpsWhitelistedDomain.kt @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017 DuckDuckGo + * Copyright (c) 2018 DuckDuckGo * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,10 +14,12 @@ * limitations under the License. */ -package com.duckduckgo.app.httpsupgrade.db +package com.duckduckgo.app.httpsupgrade.model import android.arch.persistence.room.Entity import android.arch.persistence.room.PrimaryKey -@Entity(tableName = "https_upgrade_domain") -data class HttpsUpgradeDomain(@PrimaryKey var domain: String) \ No newline at end of file +@Entity(tableName = "https_whitelisted_domain") +data class HttpsWhitelistedDomain( + @PrimaryKey var domain: String +) \ No newline at end of file diff --git a/app/src/main/java/com/duckduckgo/app/job/AppConfigurationDownloader.kt b/app/src/main/java/com/duckduckgo/app/job/AppConfigurationDownloader.kt index 877953efd42c..80f5cf2acf42 100644 --- a/app/src/main/java/com/duckduckgo/app/job/AppConfigurationDownloader.kt +++ b/app/src/main/java/com/duckduckgo/app/job/AppConfigurationDownloader.kt @@ -18,7 +18,7 @@ package com.duckduckgo.app.job import com.duckduckgo.app.global.db.AppConfigurationEntity import com.duckduckgo.app.global.db.AppDatabase -import com.duckduckgo.app.httpsupgrade.api.HttpsUpgradeListDownloader +import com.duckduckgo.app.httpsupgrade.api.HttpsUpgradeDataDownloader import com.duckduckgo.app.surrogates.api.ResourceSurrogateListDownloader import com.duckduckgo.app.trackerdetection.Client.ClientName.* import com.duckduckgo.app.trackerdetection.api.TrackerDataDownloader @@ -31,7 +31,7 @@ interface ConfigurationDownloader { class AppConfigurationDownloader( private val trackerDataDownloader: TrackerDataDownloader, - private val httpsUpgradeListDownloader: HttpsUpgradeListDownloader, + private val httpsUpgradeDataDownloader: HttpsUpgradeDataDownloader, private val resourceSurrogateDownloader: ResourceSurrogateListDownloader, private val appDatabase: AppDatabase) : ConfigurationDownloader { @@ -41,7 +41,7 @@ class AppConfigurationDownloader( val trackersWhitelist = trackerDataDownloader.downloadList(TRACKERSWHITELIST) val disconnectDownload = trackerDataDownloader.downloadList(DISCONNECT) val surrogatesDownload = resourceSurrogateDownloader.downloadList() - val httpsUpgradeDownload = httpsUpgradeListDownloader.downloadList() + val httpsUpgradeDownload = httpsUpgradeDataDownloader.download() return Completable.mergeDelayError(listOf( easyListDownload, diff --git a/app/src/main/java/com/duckduckgo/app/surrogates/api/ResourceSurrogateListDownloader.kt b/app/src/main/java/com/duckduckgo/app/surrogates/api/ResourceSurrogateListDownloader.kt index c7c67c3a57fd..ba0b96e9fe93 100644 --- a/app/src/main/java/com/duckduckgo/app/surrogates/api/ResourceSurrogateListDownloader.kt +++ b/app/src/main/java/com/duckduckgo/app/surrogates/api/ResourceSurrogateListDownloader.kt @@ -37,7 +37,7 @@ class ResourceSurrogateListDownloader @Inject constructor( Timber.d("Downloading Google Analytics Surrogates data") - val call = service.https() + val call = service.surrogates() val response = call.execute() Timber.d("Response received, success=${response.isSuccessful}") diff --git a/app/src/main/java/com/duckduckgo/app/surrogates/api/ResourceSurrogateListService.kt b/app/src/main/java/com/duckduckgo/app/surrogates/api/ResourceSurrogateListService.kt index 1066ebec33e8..2ac0928f674d 100644 --- a/app/src/main/java/com/duckduckgo/app/surrogates/api/ResourceSurrogateListService.kt +++ b/app/src/main/java/com/duckduckgo/app/surrogates/api/ResourceSurrogateListService.kt @@ -24,5 +24,5 @@ import retrofit2.http.GET interface ResourceSurrogateListService { @GET("/contentblocking.js?l=surrogates") - fun https(): Call + fun surrogates(): Call } \ No newline at end of file diff --git a/app/src/main/java/com/duckduckgo/app/trackerdetection/TrackerDataLoader.kt b/app/src/main/java/com/duckduckgo/app/trackerdetection/TrackerDataLoader.kt index 8e8407ab0304..d18c632effdc 100644 --- a/app/src/main/java/com/duckduckgo/app/trackerdetection/TrackerDataLoader.kt +++ b/app/src/main/java/com/duckduckgo/app/trackerdetection/TrackerDataLoader.kt @@ -19,16 +19,16 @@ package com.duckduckgo.app.trackerdetection import android.support.annotation.WorkerThread import com.duckduckgo.app.trackerdetection.db.TrackerDataDao import com.duckduckgo.app.trackerdetection.model.TrackerNetworks -import com.duckduckgo.app.trackerdetection.store.TrackerDataStore +import com.duckduckgo.app.global.store.BinaryDataStore import timber.log.Timber import javax.inject.Inject @WorkerThread class TrackerDataLoader @Inject constructor( - private val trackerDetector: TrackerDetector, - private val trackerDataStore: TrackerDataStore, - private val trackerDataDao: TrackerDataDao, - private val networkTrackers: TrackerNetworks) { + private val trackerDetector: TrackerDetector, + private val binaryDataStore: BinaryDataStore, + private val trackerDataDao: TrackerDataDao, + private val networkTrackers: TrackerNetworks) { fun loadData() { @@ -44,10 +44,10 @@ class TrackerDataLoader @Inject constructor( fun loadAdblockData(name: Client.ClientName) { Timber.d("Looking for adblock tracker ${name.name} to load") - if (trackerDataStore.hasData(name)) { + if (binaryDataStore.hasData(name.name)) { Timber.d("Found adblock tracker ${name.name}") val client = AdBlockClient(name) - client.loadProcessedData(trackerDataStore.loadData(name)) + client.loadProcessedData(binaryDataStore.loadData(name.name)) trackerDetector.addClient(client) } else { Timber.d("No adblock tracker ${name.name} found") diff --git a/app/src/main/java/com/duckduckgo/app/trackerdetection/api/TrackerDataDownloader.kt b/app/src/main/java/com/duckduckgo/app/trackerdetection/api/TrackerDataDownloader.kt index 7be5a84c6361..826c6b04a30f 100644 --- a/app/src/main/java/com/duckduckgo/app/trackerdetection/api/TrackerDataDownloader.kt +++ b/app/src/main/java/com/duckduckgo/app/trackerdetection/api/TrackerDataDownloader.kt @@ -23,7 +23,7 @@ import com.duckduckgo.app.trackerdetection.Client import com.duckduckgo.app.trackerdetection.Client.ClientName.* import com.duckduckgo.app.trackerdetection.TrackerDataLoader import com.duckduckgo.app.trackerdetection.db.TrackerDataDao -import com.duckduckgo.app.trackerdetection.store.TrackerDataStore +import com.duckduckgo.app.global.store.BinaryDataStore import io.reactivex.Completable import okhttp3.ResponseBody import retrofit2.Call @@ -33,11 +33,11 @@ import javax.inject.Inject class TrackerDataDownloader @Inject constructor( - private val trackerListService: TrackerListService, - private val trackerDataStore: TrackerDataStore, - private val trackerDataLoader: TrackerDataLoader, - private val trackerDataDao: TrackerDataDao, - private val appDatabase: AppDatabase) { + private val trackerListService: TrackerListService, + private val binaryDataStore: BinaryDataStore, + private val trackerDataLoader: TrackerDataLoader, + private val trackerDataDao: TrackerDataDao, + private val appDatabase: AppDatabase) { fun downloadList(clientName: Client.ClientName): Completable { @@ -67,12 +67,10 @@ class TrackerDataDownloader @Inject constructor( val body = response.body()!! appDatabase.runInTransaction { - trackerDataDao.deleteAll() - trackerDataDao.insertAll(body.trackers) + trackerDataDao.updateAll(body.trackers) trackerDataLoader.loadDisconnectData() } - } else { throw IOException("Status: ${response.code()} - ${response.errorBody()?.string()}") } @@ -86,7 +84,7 @@ class TrackerDataDownloader @Inject constructor( val call = callFactory(clientName) val response = call.execute() - if (response.isCached && trackerDataStore.hasData(clientName)) { + if (response.isCached && binaryDataStore.hasData(clientName.name)) { Timber.d("${clientName.name} data already cached and stored") return@fromAction } @@ -105,13 +103,13 @@ class TrackerDataDownloader @Inject constructor( private fun persistTrackerData(clientName: Client.ClientName, bodyBytes: ByteArray) { val client = AdBlockClient(clientName) client.loadBasicData(bodyBytes) - trackerDataStore.saveData(clientName, client.getProcessedData()) + binaryDataStore.saveData(clientName.name, client.getProcessedData()) } private fun removeLegacyList(clientName: Client.ClientName): Completable { return Completable.fromAction { - if (trackerDataStore.hasData(clientName)) { - trackerDataStore.clearData(clientName) + if (binaryDataStore.hasData(clientName.name)) { + binaryDataStore.clearData(clientName.name) } return@fromAction } diff --git a/app/src/main/java/com/duckduckgo/app/trackerdetection/db/TrackerDataDao.kt b/app/src/main/java/com/duckduckgo/app/trackerdetection/db/TrackerDataDao.kt index 189dfbdc05cb..0cfd2ae4dcd7 100644 --- a/app/src/main/java/com/duckduckgo/app/trackerdetection/db/TrackerDataDao.kt +++ b/app/src/main/java/com/duckduckgo/app/trackerdetection/db/TrackerDataDao.kt @@ -16,25 +16,28 @@ package com.duckduckgo.app.trackerdetection.db -import android.arch.persistence.room.Dao -import android.arch.persistence.room.Insert -import android.arch.persistence.room.OnConflictStrategy -import android.arch.persistence.room.Query +import android.arch.persistence.room.* import com.duckduckgo.app.trackerdetection.model.DisconnectTracker @Dao -interface TrackerDataDao { +abstract class TrackerDataDao { @Insert(onConflict = OnConflictStrategy.REPLACE) - fun insertAll(trackers: List) + abstract fun insertAll(trackers: List) - @Query("Select * from disconnect_tracker") - fun getAll() : List + @Transaction + open fun updateAll(trackers: List) { + deleteAll() + insertAll(trackers) + } - @Query("Select count(*) from disconnect_tracker") - fun count(): Int + @Query("select * from disconnect_tracker") + abstract fun getAll() : List - @Query("DELETE FROM disconnect_tracker") - fun deleteAll() + @Query("select count(*) from disconnect_tracker") + abstract fun count(): Int + + @Query("delete from disconnect_tracker") + abstract fun deleteAll() } \ No newline at end of file diff --git a/app/src/main/java/com/duckduckgo/app/trackerdetection/store/TrackerDataStore.kt b/app/src/main/java/com/duckduckgo/app/trackerdetection/store/TrackerDataStore.kt deleted file mode 100644 index 5fd4bc2369c4..000000000000 --- a/app/src/main/java/com/duckduckgo/app/trackerdetection/store/TrackerDataStore.kt +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright (c) 2017 DuckDuckGo - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.duckduckgo.app.trackerdetection.store - -import android.content.Context -import com.duckduckgo.app.trackerdetection.Client.ClientName -import javax.inject.Inject - -class TrackerDataStore @Inject constructor(private val context: Context) { - - fun hasData(client: ClientName): Boolean = context.fileExists(client.name) - - fun loadData(client: ClientName): ByteArray = - context.openFileInput(client.name).use { it.readBytes() } - - fun saveData(client: ClientName, byteArray: ByteArray) { - context.openFileOutput(client.name, Context.MODE_PRIVATE).write(byteArray) - } - - fun clearData(client: ClientName) { - context.deleteFile(client.name) - } - - private fun Context.fileExists(filename: String): Boolean { - val file = getFileStreamPath(filename) - return file != null && file.exists() - } -} diff --git a/bitrise.yml b/bitrise.yml index 1d401c4be63f..2f2205ec48be 100644 --- a/bitrise.yml +++ b/bitrise.yml @@ -10,7 +10,9 @@ trigger_map: workflows: primary: steps: - - avd-manager@0.9.2: {} + - avd-manager@0.9.2: + inputs: + - version: '27' - activate-ssh-key@3.1.1: run_if: '{{getenv "SSH_RSA_PRIVATE_KEY" | ne ""}}' - git-clone@4.0.11: {} diff --git a/build.gradle b/build.gradle index 98386d4a4592..ad70cbff4d42 100644 --- a/build.gradle +++ b/build.gradle @@ -7,7 +7,7 @@ buildscript { jcenter() } dependencies { - classpath 'com.android.tools.build:gradle:3.2.0-beta03' + classpath 'com.android.tools.build:gradle:3.2.0-beta04' classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version" // NOTE: Do not place your application dependencies here; they belong