From ced264e910c95ba4f61af049507dfe1d7796e80f Mon Sep 17 00:00:00 2001 From: Mia Alexiou Date: Sat, 21 Jul 2018 00:54:02 +0100 Subject: [PATCH 01/25] Remove old https solution and add large scale https with bloom filter * Remove old component and replace with new * Download https data, validate and persist * Add migration including deleting old records * Unit tests --- app/CMakeLists.txt | 12 + .../4.json | 302 ++++++++++++++++++ .../java/com/duckduckgo/app/FileUtilities.kt | 4 + .../com/duckduckgo/app/global/ChecksumTest.kt | 47 +++ .../app/global/db/AppDatabaseTest.kt | 16 +- .../app/httpsupgrade/BloomFilterTest.kt | 79 +++++ .../app/httpsupgrade/HttpsUpgraderTest.kt | 72 ++--- .../api/HttpsBloomFilterSpecJsonTest.kt | 44 +++ ...eJsonTest.kt => HttpsWhitelistJsonTest.kt} | 42 +-- .../db/HttpsBloomFilterSpecDaoTest.kt | 67 ++++ .../db/HttpsUpgradeDomainDAOTest.kt | 111 ------- .../db/HttpsUpgradePerformanceTest.kt | 116 ------- .../httpsupgrade/db/HttpsWhitelistDaoTest.kt | 91 ++++++ app/src/main/cpp/adblockclient-lib.cpp | 10 +- app/src/main/cpp/https-bloom-lib.cpp | 67 ++++ app/src/main/cpp/https/BloomFilter.cpp | 222 +++++++++++++ app/src/main/cpp/https/BloomFilter.hpp | 23 ++ .../app/di/AppConfigurationDownloadModule.kt | 6 +- .../java/com/duckduckgo/app/di/DaoModule.kt | 5 +- .../com/duckduckgo/app/di/DatabaseModule.kt | 3 +- .../java/com/duckduckgo/app/di/JsonModule.kt | 16 +- .../com/duckduckgo/app/di/NetworkModule.kt | 6 +- .../com/duckduckgo/app/global/Checksum.kt | 33 ++ .../duckduckgo/app/global/db/AppDatabase.kt | 22 +- .../app/global/store/BinaryDataStore.kt | 61 ++++ .../app/httpsupgrade/BloomFilter.kt | 58 ++++ .../app/httpsupgrade/HttpsUpgrader.kt | 54 ++-- .../api/HttpsBloomFilterFactory.kt | 58 ++++ .../api/HttpsUpgradeDataDownloader.kt | 108 +++++++ .../api/HttpsUpgradeListDownloader.kt | 93 ------ .../httpsupgrade/api/HttpsUpgradeService.kt | 39 +++ ...dapter.kt => HttpsWhitelistJsonAdapter.kt} | 11 +- ...omainDao.kt => HttpsBloomFilterSpecDao.kt} | 20 +- .../db/HttpsUpgradeDbWriteStatusStore.kt | 51 --- .../app/httpsupgrade/db/HttpsWhitelistDao.kt | 41 +++ .../httpsupgrade/di/HttpsUpgraderModule.kt | 19 +- .../HttpsBloomFilterSpec.kt} | 25 +- .../HttpsWhitelistedDomain.kt} | 10 +- .../app/job/AppConfigurationDownloader.kt | 6 +- .../api/ResourceSurrogateListDownloader.kt | 2 +- .../api/ResourceSurrogateListService.kt | 2 +- .../app/trackerdetection/TrackerDataLoader.kt | 14 +- .../api/TrackerDataDownloader.kt | 20 +- .../store/TrackerDataStore.kt | 42 --- 44 files changed, 1548 insertions(+), 602 deletions(-) create mode 100644 app/schemas/com.duckduckgo.app.global.db.AppDatabase/4.json create mode 100644 app/src/androidTest/java/com/duckduckgo/app/global/ChecksumTest.kt create mode 100644 app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/BloomFilterTest.kt create mode 100644 app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/api/HttpsBloomFilterSpecJsonTest.kt rename app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/api/{HttpsUpgradeJsonTest.kt => HttpsWhitelistJsonTest.kt} (54%) create mode 100644 app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/db/HttpsBloomFilterSpecDaoTest.kt delete mode 100644 app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/db/HttpsUpgradeDomainDAOTest.kt delete mode 100644 app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/db/HttpsUpgradePerformanceTest.kt create mode 100644 app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/db/HttpsWhitelistDaoTest.kt create mode 100644 app/src/main/cpp/https-bloom-lib.cpp create mode 100644 app/src/main/cpp/https/BloomFilter.cpp create mode 100644 app/src/main/cpp/https/BloomFilter.hpp create mode 100644 app/src/main/java/com/duckduckgo/app/global/Checksum.kt create mode 100644 app/src/main/java/com/duckduckgo/app/global/store/BinaryDataStore.kt create mode 100644 app/src/main/java/com/duckduckgo/app/httpsupgrade/BloomFilter.kt create mode 100644 app/src/main/java/com/duckduckgo/app/httpsupgrade/api/HttpsBloomFilterFactory.kt create mode 100644 app/src/main/java/com/duckduckgo/app/httpsupgrade/api/HttpsUpgradeDataDownloader.kt delete mode 100644 app/src/main/java/com/duckduckgo/app/httpsupgrade/api/HttpsUpgradeListDownloader.kt create mode 100644 app/src/main/java/com/duckduckgo/app/httpsupgrade/api/HttpsUpgradeService.kt rename app/src/main/java/com/duckduckgo/app/httpsupgrade/api/{HttpsUpgradeDomainFromStringAdapter.kt => HttpsWhitelistJsonAdapter.kt} (67%) rename app/src/main/java/com/duckduckgo/app/httpsupgrade/db/{HttpsUpgradeDomainDao.kt => HttpsBloomFilterSpecDao.kt} (68%) delete mode 100644 app/src/main/java/com/duckduckgo/app/httpsupgrade/db/HttpsUpgradeDbWriteStatusStore.kt create mode 100644 app/src/main/java/com/duckduckgo/app/httpsupgrade/db/HttpsWhitelistDao.kt rename app/src/main/java/com/duckduckgo/app/httpsupgrade/{api/HttpsUpgradeListService.kt => model/HttpsBloomFilterSpec.kt} (55%) rename app/src/main/java/com/duckduckgo/app/httpsupgrade/{db/HttpsUpgradeDomain.kt => model/HttpsWhitelistedDomain.kt} (76%) delete mode 100644 app/src/main/java/com/duckduckgo/app/trackerdetection/store/TrackerDataStore.kt diff --git a/app/CMakeLists.txt b/app/CMakeLists.txt index adef54822180..11eae725cf38 100644 --- a/app/CMakeLists.txt +++ b/app/CMakeLists.txt @@ -26,6 +26,18 @@ add_library( # Sets the name of the library. src/main/cpp/hashset-cpp/HashSet.cpp ) + +add_library( # Sets the name of the library. + https-bloom-lib + + # Sets the library as a shared library. + SHARED + + # Provides a relative path to your source file(s). + src/main/cpp/https-bloom-lib.cpp + src/main/cpp/https/BloomFilter.cpp + ) + # Searches for a specified prebuilt library and stores the path as a # variable. Because CMake includes system libraries in the search path by # default, you only need to specify the name of the public NDK library diff --git a/app/schemas/com.duckduckgo.app.global.db.AppDatabase/4.json b/app/schemas/com.duckduckgo.app.global.db.AppDatabase/4.json new file mode 100644 index 000000000000..68af9f9a8403 --- /dev/null +++ b/app/schemas/com.duckduckgo.app.global.db.AppDatabase/4.json @@ -0,0 +1,302 @@ +{ + "formatVersion": 1, + "database": { + "version": 4, + "identityHash": "a80686f1ea7760530a37dec6214a19f4", + "entities": [ + { + "tableName": "disconnect_tracker", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`url` TEXT NOT NULL, `category` TEXT NOT NULL, `networkName` TEXT NOT NULL, `networkUrl` TEXT NOT NULL, PRIMARY KEY(`url`))", + "fields": [ + { + "fieldPath": "url", + "columnName": "url", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "category", + "columnName": "category", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "networkName", + "columnName": "networkName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "networkUrl", + "columnName": "networkUrl", + "affinity": "TEXT", + "notNull": true + } + ], + "primaryKey": { + "columnNames": [ + "url" + ], + "autoGenerate": false + }, + "indices": [], + "foreignKeys": [] + }, + { + "tableName": "https_bloom_filter_spec", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER NOT NULL, `errorRate` REAL NOT NULL, `totalEntries` INTEGER NOT NULL, `sha256` TEXT NOT NULL, PRIMARY KEY(`id`))", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "errorRate", + "columnName": "errorRate", + "affinity": "REAL", + "notNull": true + }, + { + "fieldPath": "totalEntries", + "columnName": "totalEntries", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "sha256", + "columnName": "sha256", + "affinity": "TEXT", + "notNull": true + } + ], + "primaryKey": { + "columnNames": [ + "id" + ], + "autoGenerate": false + }, + "indices": [], + "foreignKeys": [] + }, + { + "tableName": "https_whitelisted_domain", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`domain` TEXT NOT NULL, PRIMARY KEY(`domain`))", + "fields": [ + { + "fieldPath": "domain", + "columnName": "domain", + "affinity": "TEXT", + "notNull": true + } + ], + "primaryKey": { + "columnNames": [ + "domain" + ], + "autoGenerate": false + }, + "indices": [], + "foreignKeys": [] + }, + { + "tableName": "network_leaderboard", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`networkName` TEXT NOT NULL, `domainVisited` TEXT NOT NULL, PRIMARY KEY(`networkName`, `domainVisited`))", + "fields": [ + { + "fieldPath": "networkName", + "columnName": "networkName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "domainVisited", + "columnName": "domainVisited", + "affinity": "TEXT", + "notNull": true + } + ], + "primaryKey": { + "columnNames": [ + "networkName", + "domainVisited" + ], + "autoGenerate": false + }, + "indices": [], + "foreignKeys": [] + }, + { + "tableName": "site_visited", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`domain` TEXT NOT NULL, PRIMARY KEY(`domain`))", + "fields": [ + { + "fieldPath": "domain", + "columnName": "domain", + "affinity": "TEXT", + "notNull": true + } + ], + "primaryKey": { + "columnNames": [ + "domain" + ], + "autoGenerate": false + }, + "indices": [], + "foreignKeys": [] + }, + { + "tableName": "app_configuration", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`key` TEXT NOT NULL, `appConfigurationDownloaded` INTEGER NOT NULL, PRIMARY KEY(`key`))", + "fields": [ + { + "fieldPath": "key", + "columnName": "key", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "appConfigurationDownloaded", + "columnName": "appConfigurationDownloaded", + "affinity": "INTEGER", + "notNull": true + } + ], + "primaryKey": { + "columnNames": [ + "key" + ], + "autoGenerate": false + }, + "indices": [], + "foreignKeys": [] + }, + { + "tableName": "tabs", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`tabId` TEXT NOT NULL, `url` TEXT, `title` TEXT, PRIMARY KEY(`tabId`))", + "fields": [ + { + "fieldPath": "tabId", + "columnName": "tabId", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "url", + "columnName": "url", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "title", + "columnName": "title", + "affinity": "TEXT", + "notNull": false + } + ], + "primaryKey": { + "columnNames": [ + "tabId" + ], + "autoGenerate": false + }, + "indices": [ + { + "name": "index_tabs_tabId", + "unique": false, + "columnNames": [ + "tabId" + ], + "createSql": "CREATE INDEX `index_tabs_tabId` ON `${TABLE_NAME}` (`tabId`)" + } + ], + "foreignKeys": [] + }, + { + "tableName": "tab_selection", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER NOT NULL, `tabId` TEXT, PRIMARY KEY(`id`), FOREIGN KEY(`tabId`) REFERENCES `tabs`(`tabId`) ON UPDATE NO ACTION ON DELETE SET NULL )", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "tabId", + "columnName": "tabId", + "affinity": "TEXT", + "notNull": false + } + ], + "primaryKey": { + "columnNames": [ + "id" + ], + "autoGenerate": false + }, + "indices": [ + { + "name": "index_tab_selection_tabId", + "unique": false, + "columnNames": [ + "tabId" + ], + "createSql": "CREATE INDEX `index_tab_selection_tabId` ON `${TABLE_NAME}` (`tabId`)" + } + ], + "foreignKeys": [ + { + "table": "tabs", + "onDelete": "SET NULL", + "onUpdate": "NO ACTION", + "columns": [ + "tabId" + ], + "referencedColumns": [ + "tabId" + ] + } + ] + }, + { + "tableName": "bookmarks", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `title` TEXT, `url` TEXT NOT NULL)", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "title", + "columnName": "title", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "url", + "columnName": "url", + "affinity": "TEXT", + "notNull": true + } + ], + "primaryKey": { + "columnNames": [ + "id" + ], + "autoGenerate": true + }, + "indices": [], + "foreignKeys": [] + } + ], + "setupQueries": [ + "CREATE TABLE IF NOT EXISTS room_master_table (id INTEGER PRIMARY KEY,identity_hash TEXT)", + "INSERT OR REPLACE INTO room_master_table (id,identity_hash) VALUES(42, \"a80686f1ea7760530a37dec6214a19f4\")" + ] + } +} \ No newline at end of file diff --git a/app/src/androidTest/java/com/duckduckgo/app/FileUtilities.kt b/app/src/androidTest/java/com/duckduckgo/app/FileUtilities.kt index 482c99e34ddf..e088270bf83e 100644 --- a/app/src/androidTest/java/com/duckduckgo/app/FileUtilities.kt +++ b/app/src/androidTest/java/com/duckduckgo/app/FileUtilities.kt @@ -21,4 +21,8 @@ object FileUtilities { fun loadText(resourceName: String): String = javaClass.classLoader.getResource(resourceName).openStream().bufferedReader().use { it.readText() } + + fun loadLines(resourceName: String): List = + javaClass.classLoader.getResource(resourceName).openStream().bufferedReader().use { it.readLines() } + } \ No newline at end of file diff --git a/app/src/androidTest/java/com/duckduckgo/app/global/ChecksumTest.kt b/app/src/androidTest/java/com/duckduckgo/app/global/ChecksumTest.kt new file mode 100644 index 000000000000..a00d047936aa --- /dev/null +++ b/app/src/androidTest/java/com/duckduckgo/app/global/ChecksumTest.kt @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2018 DuckDuckGo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.duckduckgo.app.global + +import junit.framework.Assert.assertEquals +import junit.framework.Assert.assertTrue +import org.junit.Assert.assertFalse +import org.junit.Test + +class ChecksumTest { + + @Test + fun whenSha256ChecksumCalledThenChecksumIsCorrect() { + val result = helloWorldText.toByteArray().sha256 + assertEquals(helloWorldSha256, result) + } + + @Test + fun whenSha256CheckumCorrectThenVerifyIsTrue() { + assertTrue(helloWorldText.toByteArray().verifySha256(helloWorldSha256)) + } + + @Test + fun whenSha256CheckumIncorrectThenVerifyIsFalse() { + assertFalse(helloWorldText.toByteArray().verifySha256(otherSha256)) + } + + companion object { + const val helloWorldText = "Hello World!" + const val helloWorldSha256 = "7f83b1657ff1fc53b92dc18148a1d65dfc2d4b1fa3d677284addd200126d9069" + const val otherSha256 = "f97e9da0e3b879f0a9df979ae260a5f7e1371edb127c1862d4f861981166cdc1" + } +} \ No newline at end of file diff --git a/app/src/androidTest/java/com/duckduckgo/app/global/db/AppDatabaseTest.kt b/app/src/androidTest/java/com/duckduckgo/app/global/db/AppDatabaseTest.kt index 0fadc438b3d2..3a7d3dbedf01 100644 --- a/app/src/androidTest/java/com/duckduckgo/app/global/db/AppDatabaseTest.kt +++ b/app/src/androidTest/java/com/duckduckgo/app/global/db/AppDatabaseTest.kt @@ -26,6 +26,7 @@ import org.junit.Assert.assertTrue import org.junit.Rule import org.junit.Test + class AppDatabaseTest { @get:Rule @@ -48,16 +49,21 @@ class AppDatabaseTest { testHelper.createDatabase(TEST_DB_NAME, 2).use { it.execSQL("INSERT INTO `network_leaderboard` VALUES ('Network2', 'example.com')") } - assertTrue(database().networkLeaderboardDao().trackerNetworkTally().blockingObserve()!!.isEmpty()) } + @Test + fun whenMigratingFromVersion3To4ThenValidationSucceeds() { + testHelper.createDatabase(TEST_DB_NAME, 3).close() + testHelper.runMigrationsAndValidate(TEST_DB_NAME, 4, true, AppDatabase.MIGRATION_3_TO_4) + } + private fun database(): AppDatabase { val database = Room - .databaseBuilder(InstrumentationRegistry.getTargetContext(), AppDatabase::class.java, TEST_DB_NAME) - .addMigrations(AppDatabase.MIGRATION_1_TO_2, AppDatabase.MIGRATION_2_TO_3) - .allowMainThreadQueries() - .build() + .databaseBuilder(InstrumentationRegistry.getTargetContext(), AppDatabase::class.java, TEST_DB_NAME) + .addMigrations(AppDatabase.MIGRATION_1_TO_2, AppDatabase.MIGRATION_2_TO_3, AppDatabase.MIGRATION_3_TO_4) + .allowMainThreadQueries() + .build() testHelper.closeWhenFinished(database) diff --git a/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/BloomFilterTest.kt b/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/BloomFilterTest.kt new file mode 100644 index 000000000000..9bfbec8c83a1 --- /dev/null +++ b/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/BloomFilterTest.kt @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2018 DuckDuckGo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.duckduckgo.app.httpsupgrade + +import org.junit.Assert.* +import org.junit.Test +import java.util.* +import kotlin.collections.ArrayList + + +class BloomFilterTest { + + private lateinit var testee: BloomFilter + + @Test + fun whenBloomFilterEmptyContainsIsFalse() { + testee = BloomFilter(1000, TARGET_FALSE_POSITIVE_RATE) + assertFalse(testee.contains("abc")) + } + + @Test + fun whenBloomFilterContainsElementContainsIsTrue() { + testee = BloomFilter(1000, TARGET_FALSE_POSITIVE_RATE) + testee.add("abc") + assertTrue(testee.contains("abc")) + } + + @Test + fun whenBloomFilterContainsItemsThenLookupResultsAreWithinRange() { + + val bloomData = createRandomStrings(1000) + val testData = bloomData + createRandomStrings(9000) + + testee = BloomFilter(bloomData.size, TARGET_FALSE_POSITIVE_RATE) + bloomData.forEach { testee.add(it) } + + var (falsePositives, truePositives, falseNegatives, trueNegatives) = arrayOf(0, 0, 0, 0) + for (element in testData) { + val result = testee.contains(element) + when { + bloomData.contains(element) && !result -> falseNegatives++ + !bloomData.contains(element) && result -> falsePositives++ + !bloomData.contains(element) && !result -> trueNegatives++ + bloomData.contains(element) && result == true -> truePositives++ + } + } + + val falsePositiveRate = falsePositives / testData.size + assertEquals(0, falseNegatives) + assertEquals(bloomData.size, truePositives) + assertTrue(trueNegatives <= testData.size - bloomData.size) + assertTrue(falsePositiveRate < ACCEPTABLE_FALSE_POSITIVE_RATE) + } + + private fun createRandomStrings(items: Int): ArrayList { + var list = ArrayList() + repeat(items) { list.add(UUID.randomUUID().toString()) } + return list + } + + companion object { + const val TARGET_FALSE_POSITIVE_RATE = 0.001 + const val ACCEPTABLE_FALSE_POSITIVE_RATE = TARGET_FALSE_POSITIVE_RATE * 1.1 + } +} \ No newline at end of file diff --git a/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/HttpsUpgraderTest.kt b/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/HttpsUpgraderTest.kt index e199d15ce8e9..92896d7ca63b 100644 --- a/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/HttpsUpgraderTest.kt +++ b/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/HttpsUpgraderTest.kt @@ -17,80 +17,56 @@ package com.duckduckgo.app.httpsupgrade import android.net.Uri -import com.duckduckgo.app.httpsupgrade.db.HttpsUpgradeDomainDao -import com.nhaarman.mockito_kotlin.* -import org.junit.Assert.* +import com.duckduckgo.app.httpsupgrade.api.HttpsBloomFilterFactory +import com.duckduckgo.app.httpsupgrade.db.HttpsWhitelistDao +import com.nhaarman.mockito_kotlin.mock +import com.nhaarman.mockito_kotlin.whenever +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue import org.junit.Before import org.junit.Test class HttpsUpgraderTest { lateinit var testee: HttpsUpgrader - lateinit var mockDao: HttpsUpgradeDomainDao + + private var mockHttpsBloomFilterFactory: HttpsBloomFilterFactory = mock() + private var mockWhitelistDao: HttpsWhitelistDao = mock() + private var bloomFilter = BloomFilter(100, 0.01) @Before fun before() { - mockDao = mock() - testee = HttpsUpgraderImpl(mockDao) - } - - @Test - fun whenHostContainsTwoPartsThenUpgradeStillChecksWildcard() { - whenever(mockDao.hasDomain("macpro.localhost")).thenReturn(false) - assertFalse(testee.shouldUpgrade(Uri.parse("http://macpro.localhost"))) - verify(mockDao).hasDomain("*.localhost") - } - - @Test - fun whenHostContainsSinglePartThenUpgradeStillChecksWildcard() { - whenever(mockDao.hasDomain("localhost")).thenReturn(false) - assertFalse(testee.shouldUpgrade(Uri.parse("http://localhost"))) - verify(mockDao, times(1)).hasDomain(any()) + whenever(mockHttpsBloomFilterFactory.create()).thenReturn(bloomFilter) + testee = HttpsUpgraderImpl(mockWhitelistDao, mockHttpsBloomFilterFactory) } @Test - fun whenMixedCaseDomainIsASubdominOfAWildCardInTheDatabaseThenShouldUpgrade() { - whenever(mockDao.hasDomain("www.example.com")).thenReturn(false) - whenever(mockDao.hasDomain("*.example.com")).thenReturn(true) - assertTrue(testee.shouldUpgrade(Uri.parse("http://www.EXAMPLE.com"))) - verify(mockDao).hasDomain("*.example.com") - } - - @Test - fun whenMixedCaseUriIsHttpAndInUpgradeListThenShouldUpgrade() { - whenever(mockDao.hasDomain("www.example.com")).thenReturn(true) - assertTrue(testee.shouldUpgrade(Uri.parse("http://www.EXAMPLE.com"))) - } - - @Test - fun whenGivenUriItIsUpgradedToHttps() { - val input = Uri.parse("http://www.example.com/some/path/to/a/file.txt") - val expected = Uri.parse("https://www.example.com/some/path/to/a/file.txt") - assertEquals(expected, testee.upgrade(input)) + fun whenUriIsHttpsThenShouldNotUpgrade() { + assertFalse(testee.shouldUpgrade(Uri.parse("https://www.example.com"))) } @Test - fun whenDomainIsASubdominOfAWildCardInTheDatabaseThenShouldUpgrade() { - whenever(mockDao.hasDomain("www.example.com")).thenReturn(false) - whenever(mockDao.hasDomain("*.example.com")).thenReturn(true) - assertTrue(testee.shouldUpgrade(Uri.parse("http://www.example.com"))) - verify(mockDao).hasDomain("*.example.com") + fun whenHttpUriIsNotInBloomFilterThenShouldNotUpgrade() { + assertFalse(testee.shouldUpgrade(Uri.parse("http://www.example.com"))) } @Test - fun whenUriIsHttpAndInUpgradeListThenShouldUpgrade() { - whenever(mockDao.hasDomain("www.example.com")).thenReturn(true) + fun whenHttpUriIsInBloomFilterThenShouldUpgrade() { + bloomFilter.add("www.example.com") assertTrue(testee.shouldUpgrade(Uri.parse("http://www.example.com"))) } @Test - fun whenUriIsHttpAndIsNotInUpgradeListThenShouldNotUpgrade() { + fun whenHttpUriHasOnlyPartDomainInBloomFilterThenShouldNotUpgrade() { + bloomFilter.add("example.com") assertFalse(testee.shouldUpgrade(Uri.parse("http://www.example.com"))) } @Test - fun whenUriIsHttpsThenShouldNotUpgrade() { - assertFalse(testee.shouldUpgrade(Uri.parse("https://www.example.com"))) + fun whenHttpUriIsInBloomFilterAndInWhitelistThenShouldNotUpgrade() { + bloomFilter.add("www.example.com") + whenever(mockWhitelistDao.contains("www.example.com")).thenReturn(true) + assertFalse(testee.shouldUpgrade(Uri.parse("http://www.example.com"))) } } diff --git a/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/api/HttpsBloomFilterSpecJsonTest.kt b/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/api/HttpsBloomFilterSpecJsonTest.kt new file mode 100644 index 000000000000..2a48489f4297 --- /dev/null +++ b/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/api/HttpsBloomFilterSpecJsonTest.kt @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2018 DuckDuckGo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.duckduckgo.app.httpsupgrade.api + +import com.duckduckgo.app.httpsupgrade.model.HttpsBloomFilterSpec +import com.squareup.moshi.Moshi +import org.junit.Assert.assertEquals +import org.junit.Test + +class HttpsBloomFilterSpecJsonTest { + + @Test + fun whenGivenValidJsonThenParsesCorrectly() { + val moshi = Moshi.Builder().build() + val jsonAdapter = moshi.adapter(HttpsBloomFilterSpec::class.java) + val result = jsonAdapter.fromJson(json()) + assertEquals(2858372, result.totalEntries) + assertEquals(0.0001, result.errorRate, 0.00001) + assertEquals("932ae1481fc33d94320a3b072638c0df8005482506933897e35feb1294693c84", result.sha256) + } + + private fun json(): String = """ + { + "totalEntries":2858372, + "errorRate" : 0.0001, + "sha256" : "932ae1481fc33d94320a3b072638c0df8005482506933897e35feb1294693c84" + } + """ + +} diff --git a/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/api/HttpsUpgradeJsonTest.kt b/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/api/HttpsWhitelistJsonTest.kt similarity index 54% rename from app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/api/HttpsUpgradeJsonTest.kt rename to app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/api/HttpsWhitelistJsonTest.kt index 8744b8c69128..966e46c026b1 100644 --- a/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/api/HttpsUpgradeJsonTest.kt +++ b/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/api/HttpsWhitelistJsonTest.kt @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017 DuckDuckGo + * Copyright (c) 2018 DuckDuckGo * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,33 +16,37 @@ package com.duckduckgo.app.httpsupgrade.api -import com.duckduckgo.app.httpsupgrade.db.HttpsUpgradeDomain +import com.duckduckgo.app.httpsupgrade.model.HttpsWhitelistedDomain import com.squareup.moshi.JsonAdapter import com.squareup.moshi.Moshi import com.squareup.moshi.Types import org.junit.Assert.assertEquals import org.junit.Test -class HttpsUpgradeJsonTest { +class HttpsWhitelistJsonTest { @Test fun whenGivenValidJsonThenParsesCorrectly() { - val moshi = Moshi.Builder().add(HttpsUpgradeDomainFromStringAdapter()).build() - val type = Types.newParameterizedType(List::class.java, HttpsUpgradeDomain::class.java) - val adapter: JsonAdapter> = moshi.adapter(type) - val list = adapter.fromJson(json()) - assertEquals(5, list.count()) - } + val moshi = Moshi.Builder().add(HttpsWhitelistJsonAdapter()).build() + val type = Types.newParameterizedType(List::class.java, HttpsWhitelistedDomain::class.java) + val jsonAdapter: JsonAdapter> = moshi.adapter(type) - private fun json() : String = """ - [ - "1337x.to", - "1688.com", - "2ch.net", - "adobe.com", - "alibaba.com" - ] - """ + val list = jsonAdapter.fromJson(json()) + assertEquals(7, list.count()) + } -} \ No newline at end of file + private fun json(): String = """ + { + "data": [ + "mlb.mlb.com", + "proa.accuweather.com", + "jbhard.org", + "lody.net", + "ci.brookfield.wi.us", + "cocktaildreams.de", + "anwalt-im-netz.de" + ] + } + """ +} diff --git a/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/db/HttpsBloomFilterSpecDaoTest.kt b/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/db/HttpsBloomFilterSpecDaoTest.kt new file mode 100644 index 000000000000..1a8c9adf87e6 --- /dev/null +++ b/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/db/HttpsBloomFilterSpecDaoTest.kt @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2018 DuckDuckGo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.duckduckgo.app.httpsupgrade.db + +import android.arch.persistence.room.Room +import android.support.test.InstrumentationRegistry +import com.duckduckgo.app.global.db.AppDatabase +import com.duckduckgo.app.httpsupgrade.model.HttpsBloomFilterSpec +import org.junit.After +import org.junit.Assert.* +import org.junit.Before +import org.junit.Test + +class HttpsBloomFilterSpecDaoTest { + + private lateinit var db: AppDatabase + private lateinit var dao: HttpsBloomFilterSpecDao + + @Before + fun before() { + db = Room.inMemoryDatabaseBuilder(InstrumentationRegistry.getContext(), AppDatabase::class.java).build() + dao = db.httpsBloomFilterSpecDao() + } + + @After + fun after() { + db.close() + } + + @Test + fun whenModelIsEmptyThenGetIsNull() { + assertNull(dao.get()) + } + + @Test + fun whenModelIsInsertedThenGetIsNotNull() { + dao.insert(HttpsBloomFilterSpec(errorRate = 0.1, totalEntries = 55, sha256 = "abc")) + assertNotNull(dao.get()) + } + + @Test + fun whenNewModelIsInsertedThenGetIsNotNullAndDetailsUpdates() { + dao.insert(HttpsBloomFilterSpec(errorRate = 0.1, totalEntries = 55, sha256 = "abc")) + dao.insert(HttpsBloomFilterSpec(errorRate = 0.2, totalEntries = 60, sha256 = "xyz")) + + val specification = dao.get() + assertNotNull(specification) + assertEquals(0.2, specification!!.errorRate, 0.01) + assertEquals(60, specification!!.totalEntries) + assertEquals("xyz", specification!!.sha256) + } + +} \ No newline at end of file diff --git a/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/db/HttpsUpgradeDomainDAOTest.kt b/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/db/HttpsUpgradeDomainDAOTest.kt deleted file mode 100644 index 067ac7c774d3..000000000000 --- a/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/db/HttpsUpgradeDomainDAOTest.kt +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Copyright (c) 2017 DuckDuckGo - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.duckduckgo.app.httpsupgrade.db - -import android.arch.persistence.room.Room -import android.support.test.InstrumentationRegistry -import com.duckduckgo.app.global.db.AppDatabase -import org.junit.After -import org.junit.Assert.* -import org.junit.Before -import org.junit.Test - -class HttpsUpgradeDomainDaoTest { - - companion object { - var exactMatchDomain = "bbc.co.uk" - var otherDomain = "other.com" - var wildcardDomain = "*.wordpress.com" - var otherWildcardDomain = "*.google.com" - var exampleWildcardDomain = "example.wordpress.com" - var parentOfWildcardDomain = "wordpress.com" - } - - private lateinit var db: AppDatabase - private lateinit var dao: HttpsUpgradeDomainDao - - @Before - fun before() { - db = Room.inMemoryDatabaseBuilder(InstrumentationRegistry.getContext(), AppDatabase::class.java).build() - dao = db.httpsUpgradeDomainDao() - } - - @After - fun after() { - db.close() - } - - @Test - fun whenExactMatchDomainAddedAndThenAllDeletedThenDoesNotContainExactMatchDomain() { - dao.insertAll(listOf(HttpsUpgradeDomain(exactMatchDomain))) - dao.deleteAll() - assertFalse(dao.hasDomain(exactMatchDomain)) - } - - @Test - fun whenWildcardDomainInsertedModelThenDoesNotContainParentOfWildcardDomain() { - dao.insertAll(listOf(HttpsUpgradeDomain(wildcardDomain))) - assertFalse(dao.hasDomain(parentOfWildcardDomain)) - } - - @Test - fun whenOtherWildcardDomainInsertedThenModelDoesNotContainExampleWildcardDomain() { - dao.insertAll(listOf(HttpsUpgradeDomain(otherWildcardDomain))) - assertFalse(dao.hasDomain(exampleWildcardDomain)) - } - - @Test - fun whenWildcardDomainInsertedThenModelDoesNotContainExactMatchDomain() { - dao.insertAll(listOf(HttpsUpgradeDomain(wildcardDomain))) - assertFalse(dao.hasDomain(exactMatchDomain)) - } - - @Test - fun whenWildcardDomainInsertedThenModelContainsExampleWildcardDomain() { - dao.insertAll(listOf(HttpsUpgradeDomain(wildcardDomain))) - assertTrue(dao.hasDomain("*.$parentOfWildcardDomain")) - } - - @Test - fun whenExactMatchDomainInsertedThenModelDoesNotContainOtherDomain() { - dao.insertAll(listOf(HttpsUpgradeDomain(exactMatchDomain))) - assertFalse(dao.hasDomain(otherDomain)) - } - - @Test - fun whenExactMatchDomainIsInsertedThenModelContainsExactMatchDomain() { - dao.insertAll(listOf(HttpsUpgradeDomain(exactMatchDomain))) - assertTrue(dao.hasDomain(exactMatchDomain)) - } - - @Test - fun whenModelIsEmptyThenModelDoesNotContainExactMatchDomain() { - assertFalse(dao.hasDomain(exactMatchDomain)) - } - - @Test - fun whenModelIsEmptyThenCountIsZero() { - assertEquals(0, dao.count()) - } - - @Test - fun whenModelContainsTwoItemsThenCountIsTwo() { - dao.insertAll(listOf(HttpsUpgradeDomain(exactMatchDomain), HttpsUpgradeDomain(otherDomain))) - assertEquals(2, dao.count()) - } - -} \ No newline at end of file diff --git a/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/db/HttpsUpgradePerformanceTest.kt b/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/db/HttpsUpgradePerformanceTest.kt deleted file mode 100644 index f665e381afec..000000000000 --- a/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/db/HttpsUpgradePerformanceTest.kt +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Copyright (c) 2018 DuckDuckGo - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.duckduckgo.app.httpsupgrade.db - -/* - * Copyright (c) 2018 DuckDuckGo - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import android.arch.persistence.room.Room -import android.net.Uri -import android.support.test.InstrumentationRegistry -import com.duckduckgo.app.global.db.AppDatabase -import com.duckduckgo.app.httpsupgrade.HttpsUpgrader -import com.duckduckgo.app.httpsupgrade.HttpsUpgraderImpl -import org.junit.Assert.assertEquals -import org.junit.Assert.assertFalse -import org.junit.Before -import org.junit.Ignore -import org.junit.Test -import timber.log.Timber - -@Ignore("Can run slow due to amount of data") -class HttpsUpgraderPerformanceTest { - - lateinit var dao: HttpsUpgradeDomainDao - lateinit var httpsUpgrader: HttpsUpgrader - lateinit var db: AppDatabase - - @Before - fun setup() { - db = Room.inMemoryDatabaseBuilder(InstrumentationRegistry.getContext(), AppDatabase::class.java).build() - dao = db.httpsUpgradeDomainDao() - httpsUpgrader = HttpsUpgraderImpl(dao) - } - - @Test - fun test2800() { - val testDomains = ingest(2800) - assertEquals(2800, testDomains.size) - checkPerformance(testDomains) - } - - @Test - fun test200000() { - val testDomains = ingest(200000) - assertEquals(200000, testDomains.size) - checkPerformance(testDomains) - } - - @Test - fun test500000() { - val testDomains = ingest(500000) - assertEquals(500000, testDomains.size) - checkPerformance(testDomains) - } - - private fun checkPerformance(testDomains: Array) { - var start = System.currentTimeMillis() - var index = 0 - for (domain in testDomains) { - assertFalse(httpsUpgrader.shouldUpgrade(Uri.parse("http://$domain"))) - - if ((index % 100) == 0) { - val diff = System.currentTimeMillis() - start - Timber.i("$diff $domain") - start = System.currentTimeMillis() - } - index++ - } - - val diff = System.currentTimeMillis() - start - Timber.i("$index $diff") - } - - private fun ingest(size: Int): Array { - val start = System.currentTimeMillis() - for (i in 0 .. size) { - dao.insertAll(listOf(HttpsUpgradeDomain("domain$i.com"))) - } - - val testDomains = Array(size, { - "testdomain$it.com" - }) - - val diff = System.currentTimeMillis() - start - Timber.i("ingest finished $diff") - return testDomains - } - -} diff --git a/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/db/HttpsWhitelistDaoTest.kt b/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/db/HttpsWhitelistDaoTest.kt new file mode 100644 index 000000000000..f2214efefa5b --- /dev/null +++ b/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/db/HttpsWhitelistDaoTest.kt @@ -0,0 +1,91 @@ +/* + * Copyright (c) 2018 DuckDuckGo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.duckduckgo.app.httpsupgrade.db + +import android.arch.persistence.room.Room +import android.support.test.InstrumentationRegistry.getContext +import com.duckduckgo.app.global.db.AppDatabase +import com.duckduckgo.app.httpsupgrade.model.HttpsWhitelistedDomain +import org.junit.After +import org.junit.Assert.* +import org.junit.Before +import org.junit.Test + +class HttpsWhitelistDaoTest { + + private lateinit var db: AppDatabase + private lateinit var dao: HttpsWhitelistDao + + @Before + fun before() { + db = Room.inMemoryDatabaseBuilder(getContext(), AppDatabase::class.java).build() + dao = db.httpsWhitelistedDao() + } + + @After + fun after() { + db.close() + } + + @Test + fun whenModelIsEmptyThenCountIsZero() { + assertEquals(0, dao.count()) + } + + @Test + fun whenModelIsEmptyThenContainsDomainIsFalse() { + assertFalse(dao.contains(domain)) + } + + @Test + fun whenDomainInsertedThenContainsDomainIsTrue() { + dao.insertAll(listOf(HttpsWhitelistedDomain(domain))) + assertTrue(dao.contains(domain)) + } + + @Test + fun whenDomainInsertedThenCountIsOne() { + dao.insertAll(listOf(HttpsWhitelistedDomain(domain))) + assertEquals(1, dao.count()) + } + + @Test + fun whenSecondUniqueDomainInsertedThenCountIsTwo() { + dao.insertAll(listOf(HttpsWhitelistedDomain(domain))) + dao.insertAll(listOf(HttpsWhitelistedDomain(anotherDomain))) + assertEquals(2, dao.count()) + } + + @Test + fun whenSecondDuplicateDomainInsertedThenCountIsOne() { + dao.insertAll(listOf(HttpsWhitelistedDomain(domain))) + dao.insertAll(listOf(HttpsWhitelistedDomain(domain))) + assertEquals(1, dao.count()) + } + + @Test + fun whenhAllDeletedThenContainsDomainIsFalse() { + dao.insertAll(listOf(HttpsWhitelistedDomain(domain))) + dao.deleteAll() + assertFalse(dao.contains(domain)) + } + + companion object { + var domain = "domain.com" + var anotherDomain = "another.com" + } +} diff --git a/app/src/main/cpp/adblockclient-lib.cpp b/app/src/main/cpp/adblockclient-lib.cpp index f8bbc066334d..73315509f09f 100644 --- a/app/src/main/cpp/adblockclient-lib.cpp +++ b/app/src/main/cpp/adblockclient-lib.cpp @@ -5,7 +5,7 @@ extern "C" JNIEXPORT jlong JNICALL Java_com_duckduckgo_app_trackerdetection_AdBlockClient_createClient(JNIEnv *env, - jobject /* this */) { + jobject) { AdBlockClient *client = new AdBlockClient(); return (long) client; } @@ -14,7 +14,7 @@ extern "C" JNIEXPORT void JNICALL Java_com_duckduckgo_app_trackerdetection_AdBlockClient_releaseClient(JNIEnv *env, - jobject /* this */, + jobject, jlong clientPointer, jlong rawDataPointer, jlong processedDataPointer) { @@ -33,9 +33,9 @@ extern "C" JNIEXPORT jlong JNICALL Java_com_duckduckgo_app_trackerdetection_AdBlockClient_loadBasicData(JNIEnv *env, - jobject /* this */, - jlong clientPointer, - jbyteArray data) { + jobject, + jlong clientPointer, + jbyteArray data) { int dataLength = env->GetArrayLength(data); char *dataChars = new char[dataLength]; diff --git a/app/src/main/cpp/https-bloom-lib.cpp b/app/src/main/cpp/https-bloom-lib.cpp new file mode 100644 index 000000000000..340221e8bffc --- /dev/null +++ b/app/src/main/cpp/https-bloom-lib.cpp @@ -0,0 +1,67 @@ +#include +#include "https/BloomFilter.hpp" + +extern "C" +JNIEXPORT long +JNICALL +Java_com_duckduckgo_app_httpsupgrade_BloomFilter_createBloomFilter(JNIEnv *env, + jobject, + jint maxItems, + jdouble targetProbability) { + BloomFilter* filter = new BloomFilter(maxItems, targetProbability); + return (long) filter; +} + + +extern "C" +JNIEXPORT long +JNICALL +Java_com_duckduckgo_app_httpsupgrade_BloomFilter_createBloomFilterFromFile(JNIEnv *env, + jobject, + jstring path, + jint maxItems) { + jboolean isElementCopy; + const char *pathChars = env->GetStringUTFChars(path, &isElementCopy); + + BloomFilter* filter = new BloomFilter(pathChars, maxItems); + return (long) filter; +} + + +extern "C" +JNIEXPORT void +JNICALL +Java_com_duckduckgo_app_httpsupgrade_BloomFilter_releaseBloomFilter(JNIEnv *env, + jobject, + jlong pointer) { + BloomFilter *filter = (BloomFilter *) pointer; + delete filter; +} + +extern "C" +JNIEXPORT void +JNICALL +Java_com_duckduckgo_app_httpsupgrade_BloomFilter_add(JNIEnv *env, + jobject, + jlong pointer, + jstring element) { + jboolean isElementCopy; + const char *elementChars = env->GetStringUTFChars(element, &isElementCopy); + + BloomFilter *filter = (BloomFilter *) pointer; + filter->add(elementChars); +} + +extern "C" +JNIEXPORT jboolean +JNICALL +Java_com_duckduckgo_app_httpsupgrade_BloomFilter_contains(JNIEnv *env, + jobject, + jlong pointer, + jstring element) { + jboolean isElementCopy; + const char *elementChars = env->GetStringUTFChars(element, &isElementCopy); + + BloomFilter *filter = (BloomFilter *) pointer; + return filter->contains(elementChars); +} \ No newline at end of file diff --git a/app/src/main/cpp/https/BloomFilter.cpp b/app/src/main/cpp/https/BloomFilter.cpp new file mode 100644 index 000000000000..df9493aa6900 --- /dev/null +++ b/app/src/main/cpp/https/BloomFilter.cpp @@ -0,0 +1,222 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include "BloomFilter.hpp" + + +// Forward declarations of free functions + +unsigned int calculateHashRounds(unsigned int size, unsigned int maxItems); + +unsigned int djb2Hash(std::string text); + +unsigned int sdbmHash(std::string text); + +unsigned int doubleHash(unsigned int hash1, unsigned int hash2, unsigned int round); + +static void writeVectorToStream(std::vector& bloomVector, BinaryOutputStream& out); + +template T pack(const std::vector& filter, const size_t block, const size_t bits); + +static std::vector readVectorFromFile(std::string path); + +static std::vector readVectorFromStream(BinaryInputStream& in); + +void unpackIntoVector(std::vector& bloomVector, + const size_t offset, + const size_t bitsInThisBlock, + BinaryInputStream& in); + + +// Implementation + +BloomFilter::BloomFilter(unsigned int maxItems, double targetProbability) +{ + unsigned int size = ceil((maxItems * log(targetProbability)) / log(1.0 / (pow(2.0, log(2.0))))); + bloomVector = std::vector(size); + hashRounds = calculateHashRounds(size, maxItems); +} + +BloomFilter::BloomFilter(std::string importFilePath, unsigned int maxItems) +{ + bloomVector = readVectorFromFile(importFilePath); + hashRounds = calculateHashRounds((unsigned int) bloomVector.size(), maxItems); +} + +BloomFilter::BloomFilter(BinaryInputStream& in, unsigned int maxItems) +{ + bloomVector = readVectorFromStream(in); + hashRounds = calculateHashRounds((unsigned int) bloomVector.size(), maxItems); +} + +unsigned int calculateHashRounds(unsigned int size, unsigned int maxItems) +{ + return round(log(2.0) * size / maxItems); +} + +void BloomFilter::add(std::string element) +{ + unsigned int hash1 = djb2Hash(element); + unsigned int hash2 = sdbmHash(element); + + for (int i = 0; i < hashRounds; i++) + { + unsigned int hash = doubleHash(hash1, hash2, i); + unsigned int index = hash % bloomVector.size(); + bloomVector[index] = true; + } +} + +bool BloomFilter::contains(std::string element) +{ + unsigned int hash1 = djb2Hash(element); + unsigned int hash2 = sdbmHash(element); + + for (int i = 0; i < hashRounds; i++) + { + unsigned int hash = doubleHash(hash1, hash2, i); + unsigned int index = hash % bloomVector.size(); + if (bloomVector[index] == false) + { + return false; + } + } + + return true; +} + +unsigned int djb2Hash(std::string text) +{ + unsigned int hash = 5381; + for (auto iterator = text.begin(); iterator != text.end(); iterator++) + { + hash = ((hash << 5) + hash) + *iterator; + } + return hash; +} + +unsigned int sdbmHash(std::string text) +{ + unsigned int hash = 0; + for (auto iterator = text.begin(); iterator != text.end(); iterator++) + { + hash = *iterator + ((hash << 6) + (hash << 16) - hash); + } + return hash; +} + +unsigned int doubleHash(unsigned int hash1, unsigned int hash2, unsigned int round) +{ + switch (round) + { + case 0: + return hash1; + case 1: + return hash2; + default: + return (hash1 + (round * hash2) + (round^2)); + } +} + +void BloomFilter::writeToFile(std::string path) +{ + std::basic_ofstream out(path.c_str(), std::ofstream::binary); + writeVectorToStream(bloomVector, out); +} + + +void BloomFilter::writeToStream(BinaryOutputStream& out) +{ + writeVectorToStream(bloomVector, out); +} + +void writeVectorToStream(std::vector& bloomVector, BinaryOutputStream& out) +{ + const size_t elements = bloomVector.size(); + out.put(((elements & 0x000000ff) >> 0)); + out.put(((elements & 0x0000ff00) >> 8)); + out.put(((elements & 0x00ff0000) >> 16)); + out.put(((elements & 0xff000000) >> 24)); + + const size_t bitsPerBlock = sizeof(BlockType) * 8; + for (size_t i = 0; i < elements / bitsPerBlock; i++) + { + const BlockType buffer = pack(bloomVector, i, bitsPerBlock); + out.put( buffer ); + } + + const size_t bitsInLastBlock = elements % bitsPerBlock; + if (bitsInLastBlock > 0) + { + const size_t lastBlock = elements / bitsPerBlock; + const BlockType buffer = pack(bloomVector, lastBlock, bitsInLastBlock); + out.put( buffer ); + } +} + +template T pack(const std::vector& filter, const size_t block, const size_t bits) +{ + const size_t sizeOfTInBits = sizeof(T) * 8; + assert( bits <= sizeOfTInBits ); + T buffer = 0; + for (int j = 0; j < bits; ++j) + { + const size_t offset = (block * sizeOfTInBits) + j; + const T bit = filter[offset] << j; + buffer |= bit; + } + return buffer; +} + +std::vector readVectorFromFile(std::string path) +{ + std::basic_ifstream inFile(path, std::ifstream::binary); + return readVectorFromStream(inFile); +} + +std::vector readVectorFromStream(BinaryInputStream& in) +{ + const size_t component1 = in.get() << 0; + const size_t component2 = in.get() << 8; + const size_t component3 = in.get() << 16; + const size_t component4 = in.get() << 24; + const size_t elementCount = component1 + component2 + component3 + component4; + + std::vector bloomVector(elementCount); + + const size_t bitsPerBlock = sizeof(BlockType) * 8; + const size_t fullBlocks = elementCount / bitsPerBlock; + for (int i = 0; i < fullBlocks; ++i) + { + const size_t offset = i * bitsPerBlock; + unpackIntoVector(bloomVector, offset, bitsPerBlock, in); + } + + const size_t bitsInLastBlock = elementCount % bitsPerBlock; + if (bitsInLastBlock > 0) + { + const size_t offset = bitsPerBlock * fullBlocks; + unpackIntoVector(bloomVector, offset, bitsInLastBlock, in); + } + + return bloomVector; +} + +void unpackIntoVector(std::vector& bloomVector, + const size_t offset, + const size_t bitsInThisBlock, + BinaryInputStream& in) +{ + const BlockType block = in.get(); + + for (int j = 0; j < bitsInThisBlock; j++) + { + const BlockType mask = 1 << j; + bloomVector[offset + j] = (block & mask) != 0; + } +} diff --git a/app/src/main/cpp/https/BloomFilter.hpp b/app/src/main/cpp/https/BloomFilter.hpp new file mode 100644 index 000000000000..8786b6103753 --- /dev/null +++ b/app/src/main/cpp/https/BloomFilter.hpp @@ -0,0 +1,23 @@ +#include +#include +#include +#include + +typedef char BlockType; +typedef std::basic_istream BinaryInputStream; +typedef std::basic_ostream BinaryOutputStream; + +class BloomFilter { + +private: + unsigned int hashRounds; + std::vector bloomVector; +public: + BloomFilter(unsigned int maxItems, double targetProbability); + BloomFilter(std::string importFilePath, unsigned int maxItems); + BloomFilter(BinaryInputStream& in, unsigned int maxItems); + void add(std::string element); + bool contains(std::string element); + void writeToFile(std::string exportFilePath); + void writeToStream(BinaryOutputStream& out); +}; diff --git a/app/src/main/java/com/duckduckgo/app/di/AppConfigurationDownloadModule.kt b/app/src/main/java/com/duckduckgo/app/di/AppConfigurationDownloadModule.kt index 7a1678bc4231..37bbc1d4210a 100644 --- a/app/src/main/java/com/duckduckgo/app/di/AppConfigurationDownloadModule.kt +++ b/app/src/main/java/com/duckduckgo/app/di/AppConfigurationDownloadModule.kt @@ -17,7 +17,7 @@ package com.duckduckgo.app.di import com.duckduckgo.app.global.db.AppDatabase -import com.duckduckgo.app.httpsupgrade.api.HttpsUpgradeListDownloader +import com.duckduckgo.app.httpsupgrade.api.HttpsUpgradeDataDownloader import com.duckduckgo.app.job.AppConfigurationDownloader import com.duckduckgo.app.job.ConfigurationDownloader import com.duckduckgo.app.surrogates.api.ResourceSurrogateListDownloader @@ -30,9 +30,9 @@ open class AppConfigurationDownloaderModule { @Provides open fun appConfigurationDownloader(trackerDataDownloader: TrackerDataDownloader, - httpsUpgradeListDownloader: HttpsUpgradeListDownloader, + httpsUpgradeDataDownloader: HttpsUpgradeDataDownloader, resourceSurrogateDownloader: ResourceSurrogateListDownloader, appDatabase: AppDatabase) : ConfigurationDownloader { - return AppConfigurationDownloader(trackerDataDownloader, httpsUpgradeListDownloader, resourceSurrogateDownloader, appDatabase) + return AppConfigurationDownloader(trackerDataDownloader, httpsUpgradeDataDownloader, resourceSurrogateDownloader, appDatabase) } } \ No newline at end of file diff --git a/app/src/main/java/com/duckduckgo/app/di/DaoModule.kt b/app/src/main/java/com/duckduckgo/app/di/DaoModule.kt index 01297a5f9923..27575480ab85 100644 --- a/app/src/main/java/com/duckduckgo/app/di/DaoModule.kt +++ b/app/src/main/java/com/duckduckgo/app/di/DaoModule.kt @@ -24,7 +24,10 @@ import dagger.Provides class DaoModule { @Provides - fun provideHttpsUpgradeDomainDao(database: AppDatabase) = database.httpsUpgradeDomainDao() + fun provideHttpsWhitelistDao(database: AppDatabase) = database.httpsWhitelistedDao() + + @Provides + fun provideHttpsBloomFilterSpecDao(database: AppDatabase) = database.httpsBloomFilterSpecDao() @Provides fun provideDisconnectTrackDao(database: AppDatabase) = database.trackerDataDao() diff --git a/app/src/main/java/com/duckduckgo/app/di/DatabaseModule.kt b/app/src/main/java/com/duckduckgo/app/di/DatabaseModule.kt index 448d7d84e500..82bba57eee78 100644 --- a/app/src/main/java/com/duckduckgo/app/di/DatabaseModule.kt +++ b/app/src/main/java/com/duckduckgo/app/di/DatabaseModule.kt @@ -21,6 +21,7 @@ import android.content.Context import com.duckduckgo.app.global.db.AppDatabase import com.duckduckgo.app.global.db.AppDatabase.Companion.MIGRATION_1_TO_2 import com.duckduckgo.app.global.db.AppDatabase.Companion.MIGRATION_2_TO_3 +import com.duckduckgo.app.global.db.AppDatabase.Companion.MIGRATION_3_TO_4 import dagger.Module import dagger.Provides import javax.inject.Singleton @@ -32,7 +33,7 @@ class DatabaseModule { @Singleton fun provideDatabase(context: Context): AppDatabase { return Room.databaseBuilder(context, AppDatabase::class.java, "app.db") - .addMigrations(MIGRATION_1_TO_2, MIGRATION_2_TO_3) + .addMigrations(MIGRATION_1_TO_2, MIGRATION_2_TO_3, MIGRATION_3_TO_4) .build() } diff --git a/app/src/main/java/com/duckduckgo/app/di/JsonModule.kt b/app/src/main/java/com/duckduckgo/app/di/JsonModule.kt index 77a2c7c07f6a..0f00b4919554 100644 --- a/app/src/main/java/com/duckduckgo/app/di/JsonModule.kt +++ b/app/src/main/java/com/duckduckgo/app/di/JsonModule.kt @@ -16,13 +16,10 @@ package com.duckduckgo.app.di -import com.duckduckgo.app.httpsupgrade.api.HttpsUpgradeDomainFromStringAdapter -import com.duckduckgo.app.httpsupgrade.db.HttpsUpgradeDomain +import com.duckduckgo.app.httpsupgrade.api.HttpsWhitelistJsonAdapter import com.duckduckgo.app.privacy.api.TermsOfServiceListAdapter import com.duckduckgo.app.trackerdetection.api.DisconnectListJsonAdapter -import com.squareup.moshi.JsonAdapter import com.squareup.moshi.Moshi -import com.squareup.moshi.Types import dagger.Module import dagger.Provides import javax.inject.Singleton @@ -33,15 +30,8 @@ class JsonModule { @Provides @Singleton fun moshi(): Moshi = Moshi.Builder() - .add(DisconnectListJsonAdapter()) + .add(HttpsWhitelistJsonAdapter()) + .add(DisconnectListJsonAdapter()) .add(TermsOfServiceListAdapter()) - .add(HttpsUpgradeDomainFromStringAdapter()) .build() - - @Provides - fun httpsUpgradeDomainAdapter(moshi: Moshi): JsonAdapter> { - val type = Types.newParameterizedType(List::class.java, HttpsUpgradeDomain::class.java) - return moshi.adapter(type) - } - } \ No newline at end of file diff --git a/app/src/main/java/com/duckduckgo/app/di/NetworkModule.kt b/app/src/main/java/com/duckduckgo/app/di/NetworkModule.kt index 2d4cfcc046c7..a4d9711b33b3 100644 --- a/app/src/main/java/com/duckduckgo/app/di/NetworkModule.kt +++ b/app/src/main/java/com/duckduckgo/app/di/NetworkModule.kt @@ -25,7 +25,7 @@ import com.duckduckgo.app.feedback.api.FeedbackSubmitter import com.duckduckgo.app.global.AppUrl.Url import com.duckduckgo.app.global.api.ApiRequestInterceptor import com.duckduckgo.app.global.job.JobBuilder -import com.duckduckgo.app.httpsupgrade.api.HttpsUpgradeListService +import com.duckduckgo.app.httpsupgrade.api.HttpsUpgradeService import com.duckduckgo.app.job.AppConfigurationSyncer import com.duckduckgo.app.job.ConfigurationDownloader import com.duckduckgo.app.statistics.VariantManager @@ -100,8 +100,8 @@ class NetworkModule { retrofit.create(TrackerListService::class.java) @Provides - fun httpsUpgradeListService(@Named("api") retrofit: Retrofit): HttpsUpgradeListService = - retrofit.create(HttpsUpgradeListService::class.java) + fun httpsUpgradeService(retrofit: Retrofit): HttpsUpgradeService = + retrofit.create(HttpsUpgradeService::class.java) @Provides fun autoCompleteService(@Named("api") retrofit: Retrofit): AutoCompleteService = diff --git a/app/src/main/java/com/duckduckgo/app/global/Checksum.kt b/app/src/main/java/com/duckduckgo/app/global/Checksum.kt new file mode 100644 index 000000000000..403d271b6296 --- /dev/null +++ b/app/src/main/java/com/duckduckgo/app/global/Checksum.kt @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2018 DuckDuckGo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.duckduckgo.app.global + +import java.math.BigInteger +import java.security.MessageDigest + + +val ByteArray.sha256: String + get() { + val md = MessageDigest.getInstance("SHA-256") + md.reset() + val digest = md.digest(this) + return String.format("%0" + digest.size * 2 + "x", BigInteger(1, digest)) + } + +fun ByteArray.verifySha256(sha256: String): Boolean { + return this.sha256 == sha256 +} \ No newline at end of file diff --git a/app/src/main/java/com/duckduckgo/app/global/db/AppDatabase.kt b/app/src/main/java/com/duckduckgo/app/global/db/AppDatabase.kt index 858c7a6e7780..9f9d3ba156d4 100644 --- a/app/src/main/java/com/duckduckgo/app/global/db/AppDatabase.kt +++ b/app/src/main/java/com/duckduckgo/app/global/db/AppDatabase.kt @@ -22,8 +22,10 @@ import android.arch.persistence.room.RoomDatabase import android.arch.persistence.room.migration.Migration import com.duckduckgo.app.bookmarks.db.BookmarkEntity import com.duckduckgo.app.bookmarks.db.BookmarksDao -import com.duckduckgo.app.httpsupgrade.db.HttpsUpgradeDomain -import com.duckduckgo.app.httpsupgrade.db.HttpsUpgradeDomainDao +import com.duckduckgo.app.httpsupgrade.db.HttpsBloomFilterSpecDao +import com.duckduckgo.app.httpsupgrade.db.HttpsWhitelistDao +import com.duckduckgo.app.httpsupgrade.model.HttpsBloomFilterSpec +import com.duckduckgo.app.httpsupgrade.model.HttpsWhitelistedDomain import com.duckduckgo.app.privacy.db.NetworkLeaderboardDao import com.duckduckgo.app.privacy.db.NetworkLeaderboardEntry import com.duckduckgo.app.privacy.db.SiteVisitedEntity @@ -33,9 +35,10 @@ import com.duckduckgo.app.tabs.model.TabSelectionEntity import com.duckduckgo.app.trackerdetection.db.TrackerDataDao import com.duckduckgo.app.trackerdetection.model.DisconnectTracker -@Database(exportSchema = true, version = 3, entities = [ - HttpsUpgradeDomain::class, +@Database(exportSchema = true, version = 4, entities = [ DisconnectTracker::class, + HttpsBloomFilterSpec::class, + HttpsWhitelistedDomain::class, NetworkLeaderboardEntry::class, SiteVisitedEntity::class, AppConfigurationEntity::class, @@ -46,8 +49,9 @@ import com.duckduckgo.app.trackerdetection.model.DisconnectTracker abstract class AppDatabase : RoomDatabase() { - abstract fun httpsUpgradeDomainDao(): HttpsUpgradeDomainDao abstract fun trackerDataDao(): TrackerDataDao + abstract fun httpsWhitelistedDao(): HttpsWhitelistDao + abstract fun httpsBloomFilterSpecDao(): HttpsBloomFilterSpecDao abstract fun networkLeaderboardDao(): NetworkLeaderboardDao abstract fun appConfigurationDao(): AppConfigurationDao abstract fun tabsDao(): TabsDao @@ -69,5 +73,13 @@ abstract class AppDatabase : RoomDatabase() { database.execSQL("DELETE FROM `network_leaderboard`") } } + + val MIGRATION_3_TO_4: Migration = object : Migration(3, 4) { + override fun migrate(database: SupportSQLiteDatabase) { + database.execSQL("DROP TABLE https_upgrade_domain") + database.execSQL("CREATE TABLE `https_bloom_filter_spec` (`id` INTEGER NOT NULL, `errorRate` REAL NOT NULL, `totalEntries` INTEGER NOT NULL, `sha256` TEXT NOT NULL, PRIMARY KEY(`id`))") + database.execSQL("CREATE TABLE `https_whitelisted_domain` (`domain` TEXT NOT NULL, PRIMARY KEY(`domain`))") + } + } } } \ No newline at end of file diff --git a/app/src/main/java/com/duckduckgo/app/global/store/BinaryDataStore.kt b/app/src/main/java/com/duckduckgo/app/global/store/BinaryDataStore.kt new file mode 100644 index 000000000000..2f60cca071df --- /dev/null +++ b/app/src/main/java/com/duckduckgo/app/global/store/BinaryDataStore.kt @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2017 DuckDuckGo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.duckduckgo.app.global.store + +import android.content.Context +import com.duckduckgo.app.global.verifySha256 +import javax.inject.Inject + +class BinaryDataStore @Inject constructor(private val context: Context) { + + fun hasData(name: String): Boolean = context.fileExists(name) + + fun loadData(name: String): ByteArray = + context.openFileInput(name).use { it.readBytes() } + + fun saveData(name: String, byteArray: ByteArray) { + context.openFileOutput(name, Context.MODE_PRIVATE).write(byteArray) + } + + fun clearData(name: String) { + context.deleteFile(name) + } + + fun dataFilePath(name: String): String? { + return context.filePath(name) + } + + fun verifyCheckSum(name: String, sha256: String): Boolean { + if (context.fileExists(name)) { + return verifyCheckSum(loadData(name), sha256) + } + return false + } + + fun verifyCheckSum(bytes: ByteArray, sha256: String): Boolean { + return bytes.verifySha256(sha256) + } + + private fun Context.filePath(filename: String): String? { + val file = getFileStreamPath(filename) + return if (file != null && file.exists()) file.path else null + } + + private fun Context.fileExists(filename: String): Boolean { + return filePath(filename) != null + } +} diff --git a/app/src/main/java/com/duckduckgo/app/httpsupgrade/BloomFilter.kt b/app/src/main/java/com/duckduckgo/app/httpsupgrade/BloomFilter.kt new file mode 100644 index 000000000000..b9a66bcab54b --- /dev/null +++ b/app/src/main/java/com/duckduckgo/app/httpsupgrade/BloomFilter.kt @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2018 DuckDuckGo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.duckduckgo.app.httpsupgrade + +class BloomFilter { + + private val nativePointer: Long + + init { + System.loadLibrary("https-bloom-lib") + } + + constructor(maxItems: Int, targetProbability: Double) { + nativePointer = createBloomFilter(maxItems, targetProbability) + } + + constructor(path: String, maxItems: Int) { + nativePointer = createBloomFilterFromFile(path, maxItems) + } + + private external fun createBloomFilter(maxItems: Int, targetProbability: Double): Long + + private external fun createBloomFilterFromFile(path: String, maxItems: Int): Long + + + fun add(element: String) { + add(nativePointer, element) + } + + private external fun add(nativePointer: Long, element: String) + + fun contains(element: String): Boolean { + return contains(nativePointer, element) + } + + private external fun contains(nativePointer: Long, element: String): Boolean + + @Suppress("unused", "protectedInFinal") + protected fun finalize() { + releaseBloomFilter(nativePointer) + } + + private external fun releaseBloomFilter(nativePointer: Long) +} \ No newline at end of file diff --git a/app/src/main/java/com/duckduckgo/app/httpsupgrade/HttpsUpgrader.kt b/app/src/main/java/com/duckduckgo/app/httpsupgrade/HttpsUpgrader.kt index 6afa567039ed..a9b2daf99979 100644 --- a/app/src/main/java/com/duckduckgo/app/httpsupgrade/HttpsUpgrader.kt +++ b/app/src/main/java/com/duckduckgo/app/httpsupgrade/HttpsUpgrader.kt @@ -20,52 +20,56 @@ import android.net.Uri import android.support.annotation.WorkerThread import com.duckduckgo.app.global.UrlScheme import com.duckduckgo.app.global.isHttps -import com.duckduckgo.app.httpsupgrade.db.HttpsUpgradeDomainDao +import com.duckduckgo.app.httpsupgrade.api.HttpsBloomFilterFactory +import com.duckduckgo.app.httpsupgrade.db.HttpsWhitelistDao import timber.log.Timber interface HttpsUpgrader { @WorkerThread - fun shouldUpgrade(uri: Uri) : Boolean + fun shouldUpgrade(uri: Uri): Boolean fun upgrade(uri: Uri): Uri { return uri.buildUpon().scheme(UrlScheme.https).build() } + + fun reloadData() } -class HttpsUpgraderImpl constructor(private val dao: HttpsUpgradeDomainDao) :HttpsUpgrader { +class HttpsUpgraderImpl( + private val whitelistedDao: HttpsWhitelistDao, + private val bloomFactory: HttpsBloomFilterFactory +) : HttpsUpgrader { + + private var httpsBloomFilter: BloomFilter? = null + + init { + reloadData() + } @WorkerThread - override fun shouldUpgrade(uri: Uri) : Boolean { + override fun shouldUpgrade(uri: Uri): Boolean { + if (uri.isHttps) { return false } - val host = (uri.host ?: return false).toLowerCase() - return dao.hasDomain(host) || matchesWildcard(host) - } - - private fun matchesWildcard(host: String): Boolean { - val domains = mutableListOf() - for (part in host.split(".").reversed()) { - if (domains.isEmpty()) { - domains.add(".$part") - } else { - val last = domains.last() - domains.add(".$part$last") - } + if (whitelistedDao.contains(uri.host)) { + Timber.d("${uri.host} is in whitelist and so not upgradable") + return false } - domains.asReversed().removeAt(0) - Timber.d("domains $domains") - - for (domain in domains) { - if (dao.hasDomain("*$domain")) { - return true - } + httpsBloomFilter?.let { + val shouldUpgrade = it.contains(uri.host) + Timber.d("${uri.host} ${if (shouldUpgrade) "is" else "is not"} upgradable") + return shouldUpgrade } return false } -} \ No newline at end of file + override fun reloadData() { + httpsBloomFilter = bloomFactory.create() + } + +} diff --git a/app/src/main/java/com/duckduckgo/app/httpsupgrade/api/HttpsBloomFilterFactory.kt b/app/src/main/java/com/duckduckgo/app/httpsupgrade/api/HttpsBloomFilterFactory.kt new file mode 100644 index 000000000000..964cb11e1851 --- /dev/null +++ b/app/src/main/java/com/duckduckgo/app/httpsupgrade/api/HttpsBloomFilterFactory.kt @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2018 DuckDuckGo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.duckduckgo.app.httpsupgrade.api + +import com.duckduckgo.app.global.store.BinaryDataStore +import com.duckduckgo.app.httpsupgrade.BloomFilter +import com.duckduckgo.app.httpsupgrade.db.HttpsBloomFilterSpecDao +import com.duckduckgo.app.httpsupgrade.model.HttpsBloomFilterSpec.Companion.HTTPS_BINARY_FILE +import io.reactivex.Observable +import io.reactivex.schedulers.Schedulers +import timber.log.Timber +import javax.inject.Inject + +interface HttpsBloomFilterFactory { + fun create(): BloomFilter? +} + +class OptionalWrapper(val value: T?) + +class HttpsBloomFilterFactoryImpl @Inject constructor(private val dao: HttpsBloomFilterSpecDao, private val binaryDataStore: BinaryDataStore) : + HttpsBloomFilterFactory { + + override fun create(): BloomFilter? { + + val specificationWrapper = Observable.just(dao) + .observeOn(Schedulers.io()) + .map { + OptionalWrapper(dao.get()) + } + .blockingSingle() + + val specification = specificationWrapper.value + val dataPath = binaryDataStore.dataFilePath(HTTPS_BINARY_FILE) + + if (dataPath == null || specification == null) { + Timber.d("Https update data not found") + return null + } + + Timber.d("Found https data at $dataPath, building filter") + return BloomFilter(dataPath, specification.totalEntries) + + } +} \ No newline at end of file diff --git a/app/src/main/java/com/duckduckgo/app/httpsupgrade/api/HttpsUpgradeDataDownloader.kt b/app/src/main/java/com/duckduckgo/app/httpsupgrade/api/HttpsUpgradeDataDownloader.kt new file mode 100644 index 000000000000..8c94b3ed8272 --- /dev/null +++ b/app/src/main/java/com/duckduckgo/app/httpsupgrade/api/HttpsUpgradeDataDownloader.kt @@ -0,0 +1,108 @@ +/* + * Copyright (c) 2018 DuckDuckGo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.duckduckgo.app.httpsupgrade.api + +import com.duckduckgo.app.global.api.isCached +import com.duckduckgo.app.global.store.BinaryDataStore +import com.duckduckgo.app.httpsupgrade.HttpsUpgrader +import com.duckduckgo.app.httpsupgrade.db.HttpsBloomFilterSpecDao +import com.duckduckgo.app.httpsupgrade.db.HttpsWhitelistDao +import com.duckduckgo.app.httpsupgrade.model.HttpsBloomFilterSpec +import com.duckduckgo.app.httpsupgrade.model.HttpsBloomFilterSpec.Companion.HTTPS_BINARY_FILE +import io.reactivex.Completable +import io.reactivex.Completable.fromAction +import timber.log.Timber +import java.io.IOException +import javax.inject.Inject + +class HttpsUpgradeDataDownloader @Inject constructor( + private val service: HttpsUpgradeService, + private val httpsUpgrader: HttpsUpgrader, + private val httpsBloomSpecDao: HttpsBloomFilterSpecDao, + private val whitelistDao: HttpsWhitelistDao, + private val binaryDataStore: BinaryDataStore +) { + + fun download(): Completable { + + val filter = service.httpsBloomFilterSpec() + .flatMapCompletable { + downloadBloomFilter(it) + } + val whitelist = downloadWhitelist() + + return Completable.mergeDelayError(listOf(filter, whitelist)) + .doOnComplete { + Timber.i("Https download task completed successfully") + } + } + + private fun downloadBloomFilter(specification: HttpsBloomFilterSpec): Completable { + return fromAction { + Timber.d("Downloading https bloom filter binary") + val call = service.httpsBloomFilter() + val response = call.execute() + val fileName = HTTPS_BINARY_FILE + + if (response.isCached && binaryDataStore.verifyCheckSum(fileName, specification.sha256)) { + Timber.d("Https bloom data already cached and stored for this spec") + return@fromAction + } + + if (!response.isSuccessful) { + throw IOException("Status: ${response.code()} - ${response.errorBody()?.string()}") + } + + val bytes = response.body()!!.bytes() + if (!binaryDataStore.verifyCheckSum(bytes, specification.sha256)) { + throw IOException("Https binary has incorrect checksum, throwisng away file") + } + + Timber.d("Updating https bloom data store with new data") + httpsBloomSpecDao.insert(specification) + binaryDataStore.saveData(fileName, bytes) + whitelistDao.deleteAll() + httpsUpgrader.reloadData() + } + } + + private fun downloadWhitelist(): Completable { + + Timber.d("Downloading HTTPS whitelist") + return fromAction { + + val call = service.whitelist() + val response = call.execute() + + if (response.isCached && whitelistDao.count() > 0) { + Timber.d("Https whitelist already cached and stored") + return@fromAction + } + + if (response.isSuccessful) { + val whitelist = response.body()!! + Timber.d("Updating https whitelist with new data") + whitelistDao.deleteAll() + whitelistDao.insertAll(whitelist) + } else { + throw IOException("Status: ${response.code()} - ${response.errorBody()?.string()}") + } + } + + } + +} \ No newline at end of file diff --git a/app/src/main/java/com/duckduckgo/app/httpsupgrade/api/HttpsUpgradeListDownloader.kt b/app/src/main/java/com/duckduckgo/app/httpsupgrade/api/HttpsUpgradeListDownloader.kt deleted file mode 100644 index a37fd4154028..000000000000 --- a/app/src/main/java/com/duckduckgo/app/httpsupgrade/api/HttpsUpgradeListDownloader.kt +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Copyright (c) 2017 DuckDuckGo - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.duckduckgo.app.httpsupgrade.api - -import com.duckduckgo.app.global.api.isCached -import com.duckduckgo.app.global.db.AppDatabase -import com.duckduckgo.app.httpsupgrade.db.HttpsUpgradeDbWriteStatusStore -import com.duckduckgo.app.httpsupgrade.db.HttpsUpgradeDomainDao -import io.reactivex.Completable -import timber.log.Timber -import java.io.IOException -import javax.inject.Inject - -/** - * For performance reasons, we can't insert all the HTTPS rows in a single transaction as it blocks other DB writes from happening for a while - * To avoid the performance problem of a single transaction, we can chunk the data and perform many smaller transactions. - * However, if we don't use a single transaction, there is the risk the process could die or exception be thrown while iterating through the HTTPS inserts, leaving the DB in a bad state. - * The network cache could therefore deliver the cached value and we could skip writing to the DB thinking we'd already successfully written it all. - * - * To counter this, we need an external mechanism to track whether the write completed successfully or not. - * Using this, we can detect if the write failed when we next download the data. We'll only skip the DB inserts if cache returns the list and the - * the eTag received in the response headers matches the eTag we stored on last successful write, indicating that it finished writing successfully. - * @see HttpsUpgradeDbWriteStatusStore for how this status flag is stored. - */ -class HttpsUpgradeListDownloader @Inject constructor( - private val service: HttpsUpgradeListService, - private val database: AppDatabase, - private val httpsUpgradeDao: HttpsUpgradeDomainDao, - private val dbWriteStatusStore: HttpsUpgradeDbWriteStatusStore -) { - - fun downloadList(chunkSize: Int = INSERTION_CHUNK_SIZE): Completable { - - Timber.d("Downloading HTTPS Upgrade data") - - return Completable.fromAction { - - val call = service.https() - val response = call.execute() - val eTag = response.headers().get(HEADER_ETAG) - - if (response.isCached && dbWriteStatusStore.isMatchingETag(eTag)) { - Timber.d("HTTPS data already cached and stored") - return@fromAction - } - - if (response.isSuccessful) { - Timber.d("Updating HTTPS upgrade list from server") - - val domains = response.body() ?: throw IllegalStateException("Failed to obtain HTTPS upgrade list") - - val startTime = System.currentTimeMillis() - - httpsUpgradeDao.deleteAll() - Timber.v("Took ${System.currentTimeMillis() - startTime}ms to delete existing records") - - val chunks = domains.chunked(chunkSize) - Timber.i("Received ${domains.size} HTTPS domains; chunking by $chunkSize into ${chunks.size} separate DB transactions") - - chunks.forEach { - database.runInTransaction { - httpsUpgradeDao.insertAll(it) - } - } - - dbWriteStatusStore.saveETag(eTag) - Timber.i("Successfully wrote HTTPS data; took ${System.currentTimeMillis() - startTime}ms") - - } else { - throw IOException("Status: ${response.code()} - ${response.errorBody()?.string()}") - } - } - } - - companion object { - private const val INSERTION_CHUNK_SIZE = 1_000 - private const val HEADER_ETAG = "etag" - } -} \ No newline at end of file diff --git a/app/src/main/java/com/duckduckgo/app/httpsupgrade/api/HttpsUpgradeService.kt b/app/src/main/java/com/duckduckgo/app/httpsupgrade/api/HttpsUpgradeService.kt new file mode 100644 index 000000000000..0bcdb7f4cd6b --- /dev/null +++ b/app/src/main/java/com/duckduckgo/app/httpsupgrade/api/HttpsUpgradeService.kt @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2018 DuckDuckGo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.duckduckgo.app.httpsupgrade.api + +import com.duckduckgo.app.httpsupgrade.model.HttpsBloomFilterSpec +import com.duckduckgo.app.httpsupgrade.model.HttpsWhitelistedDomain +import io.reactivex.Observable +import okhttp3.ResponseBody +import retrofit2.Call +import retrofit2.http.GET + +interface HttpsUpgradeService { + + // TODO final url + @GET("https://staticcdn.duckduckgo.com/https/https-whitelist-test.json") + fun whitelist(): Call> + + // TODO final url + @GET("http://192.168.86.132:8000/response.json") + fun httpsBloomFilterSpec(): Observable + + // TODO final url + @GET("http://192.168.86.132:8000/https_bloom_filter.bin") + fun httpsBloomFilter(): Call +} diff --git a/app/src/main/java/com/duckduckgo/app/httpsupgrade/api/HttpsUpgradeDomainFromStringAdapter.kt b/app/src/main/java/com/duckduckgo/app/httpsupgrade/api/HttpsWhitelistJsonAdapter.kt similarity index 67% rename from app/src/main/java/com/duckduckgo/app/httpsupgrade/api/HttpsUpgradeDomainFromStringAdapter.kt rename to app/src/main/java/com/duckduckgo/app/httpsupgrade/api/HttpsWhitelistJsonAdapter.kt index 50833051e4cf..4a8d3f364e79 100644 --- a/app/src/main/java/com/duckduckgo/app/httpsupgrade/api/HttpsUpgradeDomainFromStringAdapter.kt +++ b/app/src/main/java/com/duckduckgo/app/httpsupgrade/api/HttpsWhitelistJsonAdapter.kt @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017 DuckDuckGo + * Copyright (c) 2018 DuckDuckGo * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,11 +16,14 @@ package com.duckduckgo.app.httpsupgrade.api -import com.duckduckgo.app.httpsupgrade.db.HttpsUpgradeDomain +import com.duckduckgo.app.httpsupgrade.model.HttpsWhitelistedDomain import com.squareup.moshi.FromJson -class HttpsUpgradeDomainFromStringAdapter { +class HttpsWhitelistJsonAdapter { - @FromJson fun adapt(domain: String) = HttpsUpgradeDomain(domain) + @FromJson + fun adapt(data: Map>): List { + return data.getValue("data").map { HttpsWhitelistedDomain(it) } + } } \ No newline at end of file diff --git a/app/src/main/java/com/duckduckgo/app/httpsupgrade/db/HttpsUpgradeDomainDao.kt b/app/src/main/java/com/duckduckgo/app/httpsupgrade/db/HttpsBloomFilterSpecDao.kt similarity index 68% rename from app/src/main/java/com/duckduckgo/app/httpsupgrade/db/HttpsUpgradeDomainDao.kt rename to app/src/main/java/com/duckduckgo/app/httpsupgrade/db/HttpsBloomFilterSpecDao.kt index ecdb3cab4fa4..7deb207c0c92 100644 --- a/app/src/main/java/com/duckduckgo/app/httpsupgrade/db/HttpsUpgradeDomainDao.kt +++ b/app/src/main/java/com/duckduckgo/app/httpsupgrade/db/HttpsBloomFilterSpecDao.kt @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017 DuckDuckGo + * Copyright (c) 2018 DuckDuckGo * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,20 +20,16 @@ import android.arch.persistence.room.Dao import android.arch.persistence.room.Insert import android.arch.persistence.room.OnConflictStrategy import android.arch.persistence.room.Query +import com.duckduckgo.app.httpsupgrade.model.HttpsBloomFilterSpec +import javax.inject.Singleton @Dao -interface HttpsUpgradeDomainDao { +@Singleton +interface HttpsBloomFilterSpecDao { @Insert(onConflict = OnConflictStrategy.REPLACE) - fun insertAll(domains: List) - - @Query("select count(1) > 0 from https_upgrade_domain where domain = :domain") - fun hasDomain(domain: String) : Boolean - - @Query("delete from https_upgrade_domain") - fun deleteAll() - - @Query("select count(1) from https_upgrade_domain") - fun count(): Int + fun insert(specification: HttpsBloomFilterSpec) + @Query("select * from https_bloom_filter_spec limit 1") + fun get(): HttpsBloomFilterSpec? } \ No newline at end of file diff --git a/app/src/main/java/com/duckduckgo/app/httpsupgrade/db/HttpsUpgradeDbWriteStatusStore.kt b/app/src/main/java/com/duckduckgo/app/httpsupgrade/db/HttpsUpgradeDbWriteStatusStore.kt deleted file mode 100644 index 84dde1cb9fce..000000000000 --- a/app/src/main/java/com/duckduckgo/app/httpsupgrade/db/HttpsUpgradeDbWriteStatusStore.kt +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright (c) 2018 DuckDuckGo - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.duckduckgo.app.httpsupgrade.db - -import android.content.Context -import android.content.SharedPreferences -import android.support.annotation.VisibleForTesting -import androidx.core.content.edit -import javax.inject.Inject - -interface HttpsUpgradeDbWriteStatusStore { - - fun saveETag(eTag: String?) - fun isMatchingETag(eTag: String?): Boolean -} - -class HttpsUpgradeDbWriteStatusSharedPreferences @Inject constructor(private val context: Context) : HttpsUpgradeDbWriteStatusStore { - - override fun saveETag(eTag: String?) { - preferences.edit { putString(KEY_ETAG_OF_LAST_FULL_WRITE, eTag) } - } - - override fun isMatchingETag(eTag: String?): Boolean { - val existingETag = preferences.getString(KEY_ETAG_OF_LAST_FULL_WRITE, null) - return eTag == existingETag - } - - private val preferences: SharedPreferences - get() = context.getSharedPreferences(FILENAME, Context.MODE_PRIVATE) - - companion object { - - @VisibleForTesting - const val FILENAME = "com.duckduckgo.app.httpsupgrade.db.HttpsUpgradeDbWriteStatus" - const val KEY_ETAG_OF_LAST_FULL_WRITE = "KEY_ETAG_OF_LAST_FULL_WRITE" - } -} \ No newline at end of file diff --git a/app/src/main/java/com/duckduckgo/app/httpsupgrade/db/HttpsWhitelistDao.kt b/app/src/main/java/com/duckduckgo/app/httpsupgrade/db/HttpsWhitelistDao.kt new file mode 100644 index 000000000000..261902d9b8bf --- /dev/null +++ b/app/src/main/java/com/duckduckgo/app/httpsupgrade/db/HttpsWhitelistDao.kt @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2018 DuckDuckGo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.duckduckgo.app.httpsupgrade.db + +import android.arch.persistence.room.Dao +import android.arch.persistence.room.Insert +import android.arch.persistence.room.OnConflictStrategy +import android.arch.persistence.room.Query +import com.duckduckgo.app.httpsupgrade.model.HttpsWhitelistedDomain +import javax.inject.Singleton + +@Dao +@Singleton +interface HttpsWhitelistDao { + + @Insert(onConflict = OnConflictStrategy.REPLACE) + fun insertAll(domains: List) + + @Query("select count(1) > 0 from https_whitelisted_domain where domain = :domain") + fun contains(domain: String): Boolean + + @Query("delete from https_whitelisted_domain") + fun deleteAll() + + @Query("select count(1) from https_whitelisted_domain") + fun count(): Int +} \ No newline at end of file diff --git a/app/src/main/java/com/duckduckgo/app/httpsupgrade/di/HttpsUpgraderModule.kt b/app/src/main/java/com/duckduckgo/app/httpsupgrade/di/HttpsUpgraderModule.kt index 66fe18172a3c..583578ddb3ee 100644 --- a/app/src/main/java/com/duckduckgo/app/httpsupgrade/di/HttpsUpgraderModule.kt +++ b/app/src/main/java/com/duckduckgo/app/httpsupgrade/di/HttpsUpgraderModule.kt @@ -16,12 +16,13 @@ package com.duckduckgo.app.httpsupgrade.di -import android.content.Context +import com.duckduckgo.app.global.store.BinaryDataStore import com.duckduckgo.app.httpsupgrade.HttpsUpgrader import com.duckduckgo.app.httpsupgrade.HttpsUpgraderImpl -import com.duckduckgo.app.httpsupgrade.db.HttpsUpgradeDbWriteStatusSharedPreferences -import com.duckduckgo.app.httpsupgrade.db.HttpsUpgradeDbWriteStatusStore -import com.duckduckgo.app.httpsupgrade.db.HttpsUpgradeDomainDao +import com.duckduckgo.app.httpsupgrade.api.HttpsBloomFilterFactory +import com.duckduckgo.app.httpsupgrade.api.HttpsBloomFilterFactoryImpl +import com.duckduckgo.app.httpsupgrade.db.HttpsBloomFilterSpecDao +import com.duckduckgo.app.httpsupgrade.db.HttpsWhitelistDao import dagger.Module import dagger.Provides import javax.inject.Singleton @@ -29,14 +30,14 @@ import javax.inject.Singleton @Module class HttpsUpgraderModule { + @Singleton @Provides - fun httpsUpgrader(dao: HttpsUpgradeDomainDao): HttpsUpgrader { - return HttpsUpgraderImpl(dao) + fun httpsUpgrader(whitelistDao: HttpsWhitelistDao, bloomFilterFactory: HttpsBloomFilterFactory): HttpsUpgrader { + return HttpsUpgraderImpl(whitelistDao, bloomFilterFactory) } @Provides - @Singleton - fun httpsUpgradeDbWriteStatusStore(context: Context): HttpsUpgradeDbWriteStatusStore { - return HttpsUpgradeDbWriteStatusSharedPreferences(context) + fun bloomFilterFactory(specificationDao: HttpsBloomFilterSpecDao, binaryDataStore: BinaryDataStore): HttpsBloomFilterFactory { + return HttpsBloomFilterFactoryImpl(specificationDao, binaryDataStore) } } \ No newline at end of file diff --git a/app/src/main/java/com/duckduckgo/app/httpsupgrade/api/HttpsUpgradeListService.kt b/app/src/main/java/com/duckduckgo/app/httpsupgrade/model/HttpsBloomFilterSpec.kt similarity index 55% rename from app/src/main/java/com/duckduckgo/app/httpsupgrade/api/HttpsUpgradeListService.kt rename to app/src/main/java/com/duckduckgo/app/httpsupgrade/model/HttpsBloomFilterSpec.kt index db9eded6b897..6147c2809f86 100644 --- a/app/src/main/java/com/duckduckgo/app/httpsupgrade/api/HttpsUpgradeListService.kt +++ b/app/src/main/java/com/duckduckgo/app/httpsupgrade/model/HttpsBloomFilterSpec.kt @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017 DuckDuckGo + * Copyright (c) 2018 DuckDuckGo * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,15 +14,20 @@ * limitations under the License. */ -package com.duckduckgo.app.httpsupgrade.api +package com.duckduckgo.app.httpsupgrade.model -import com.duckduckgo.app.httpsupgrade.db.HttpsUpgradeDomain -import retrofit2.Call -import retrofit2.http.GET +import android.arch.persistence.room.Entity +import android.arch.persistence.room.PrimaryKey -interface HttpsUpgradeListService { - @GET("/contentblocking.js?l=https2") - fun https(): Call> - -} +@Entity(tableName = "https_bloom_filter_spec") +class HttpsBloomFilterSpec( + @PrimaryKey val id: Int = 1, + val errorRate: Double, + val totalEntries: Int, + val sha256: String +) { + companion object { + const val HTTPS_BINARY_FILE = "HTTPS_BINARY_FILE" + } +} \ No newline at end of file diff --git a/app/src/main/java/com/duckduckgo/app/httpsupgrade/db/HttpsUpgradeDomain.kt b/app/src/main/java/com/duckduckgo/app/httpsupgrade/model/HttpsWhitelistedDomain.kt similarity index 76% rename from app/src/main/java/com/duckduckgo/app/httpsupgrade/db/HttpsUpgradeDomain.kt rename to app/src/main/java/com/duckduckgo/app/httpsupgrade/model/HttpsWhitelistedDomain.kt index aa47ec735903..8e5de745069a 100644 --- a/app/src/main/java/com/duckduckgo/app/httpsupgrade/db/HttpsUpgradeDomain.kt +++ b/app/src/main/java/com/duckduckgo/app/httpsupgrade/model/HttpsWhitelistedDomain.kt @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017 DuckDuckGo + * Copyright (c) 2018 DuckDuckGo * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,10 +14,12 @@ * limitations under the License. */ -package com.duckduckgo.app.httpsupgrade.db +package com.duckduckgo.app.httpsupgrade.model import android.arch.persistence.room.Entity import android.arch.persistence.room.PrimaryKey -@Entity(tableName = "https_upgrade_domain") -data class HttpsUpgradeDomain(@PrimaryKey var domain: String) \ No newline at end of file +@Entity(tableName = "https_whitelisted_domain") +data class HttpsWhitelistedDomain( + @PrimaryKey var domain: String +) \ No newline at end of file diff --git a/app/src/main/java/com/duckduckgo/app/job/AppConfigurationDownloader.kt b/app/src/main/java/com/duckduckgo/app/job/AppConfigurationDownloader.kt index 877953efd42c..80f5cf2acf42 100644 --- a/app/src/main/java/com/duckduckgo/app/job/AppConfigurationDownloader.kt +++ b/app/src/main/java/com/duckduckgo/app/job/AppConfigurationDownloader.kt @@ -18,7 +18,7 @@ package com.duckduckgo.app.job import com.duckduckgo.app.global.db.AppConfigurationEntity import com.duckduckgo.app.global.db.AppDatabase -import com.duckduckgo.app.httpsupgrade.api.HttpsUpgradeListDownloader +import com.duckduckgo.app.httpsupgrade.api.HttpsUpgradeDataDownloader import com.duckduckgo.app.surrogates.api.ResourceSurrogateListDownloader import com.duckduckgo.app.trackerdetection.Client.ClientName.* import com.duckduckgo.app.trackerdetection.api.TrackerDataDownloader @@ -31,7 +31,7 @@ interface ConfigurationDownloader { class AppConfigurationDownloader( private val trackerDataDownloader: TrackerDataDownloader, - private val httpsUpgradeListDownloader: HttpsUpgradeListDownloader, + private val httpsUpgradeDataDownloader: HttpsUpgradeDataDownloader, private val resourceSurrogateDownloader: ResourceSurrogateListDownloader, private val appDatabase: AppDatabase) : ConfigurationDownloader { @@ -41,7 +41,7 @@ class AppConfigurationDownloader( val trackersWhitelist = trackerDataDownloader.downloadList(TRACKERSWHITELIST) val disconnectDownload = trackerDataDownloader.downloadList(DISCONNECT) val surrogatesDownload = resourceSurrogateDownloader.downloadList() - val httpsUpgradeDownload = httpsUpgradeListDownloader.downloadList() + val httpsUpgradeDownload = httpsUpgradeDataDownloader.download() return Completable.mergeDelayError(listOf( easyListDownload, diff --git a/app/src/main/java/com/duckduckgo/app/surrogates/api/ResourceSurrogateListDownloader.kt b/app/src/main/java/com/duckduckgo/app/surrogates/api/ResourceSurrogateListDownloader.kt index c7c67c3a57fd..ba0b96e9fe93 100644 --- a/app/src/main/java/com/duckduckgo/app/surrogates/api/ResourceSurrogateListDownloader.kt +++ b/app/src/main/java/com/duckduckgo/app/surrogates/api/ResourceSurrogateListDownloader.kt @@ -37,7 +37,7 @@ class ResourceSurrogateListDownloader @Inject constructor( Timber.d("Downloading Google Analytics Surrogates data") - val call = service.https() + val call = service.surrogates() val response = call.execute() Timber.d("Response received, success=${response.isSuccessful}") diff --git a/app/src/main/java/com/duckduckgo/app/surrogates/api/ResourceSurrogateListService.kt b/app/src/main/java/com/duckduckgo/app/surrogates/api/ResourceSurrogateListService.kt index 1066ebec33e8..2ac0928f674d 100644 --- a/app/src/main/java/com/duckduckgo/app/surrogates/api/ResourceSurrogateListService.kt +++ b/app/src/main/java/com/duckduckgo/app/surrogates/api/ResourceSurrogateListService.kt @@ -24,5 +24,5 @@ import retrofit2.http.GET interface ResourceSurrogateListService { @GET("/contentblocking.js?l=surrogates") - fun https(): Call + fun surrogates(): Call } \ No newline at end of file diff --git a/app/src/main/java/com/duckduckgo/app/trackerdetection/TrackerDataLoader.kt b/app/src/main/java/com/duckduckgo/app/trackerdetection/TrackerDataLoader.kt index 8e8407ab0304..d18c632effdc 100644 --- a/app/src/main/java/com/duckduckgo/app/trackerdetection/TrackerDataLoader.kt +++ b/app/src/main/java/com/duckduckgo/app/trackerdetection/TrackerDataLoader.kt @@ -19,16 +19,16 @@ package com.duckduckgo.app.trackerdetection import android.support.annotation.WorkerThread import com.duckduckgo.app.trackerdetection.db.TrackerDataDao import com.duckduckgo.app.trackerdetection.model.TrackerNetworks -import com.duckduckgo.app.trackerdetection.store.TrackerDataStore +import com.duckduckgo.app.global.store.BinaryDataStore import timber.log.Timber import javax.inject.Inject @WorkerThread class TrackerDataLoader @Inject constructor( - private val trackerDetector: TrackerDetector, - private val trackerDataStore: TrackerDataStore, - private val trackerDataDao: TrackerDataDao, - private val networkTrackers: TrackerNetworks) { + private val trackerDetector: TrackerDetector, + private val binaryDataStore: BinaryDataStore, + private val trackerDataDao: TrackerDataDao, + private val networkTrackers: TrackerNetworks) { fun loadData() { @@ -44,10 +44,10 @@ class TrackerDataLoader @Inject constructor( fun loadAdblockData(name: Client.ClientName) { Timber.d("Looking for adblock tracker ${name.name} to load") - if (trackerDataStore.hasData(name)) { + if (binaryDataStore.hasData(name.name)) { Timber.d("Found adblock tracker ${name.name}") val client = AdBlockClient(name) - client.loadProcessedData(trackerDataStore.loadData(name)) + client.loadProcessedData(binaryDataStore.loadData(name.name)) trackerDetector.addClient(client) } else { Timber.d("No adblock tracker ${name.name} found") diff --git a/app/src/main/java/com/duckduckgo/app/trackerdetection/api/TrackerDataDownloader.kt b/app/src/main/java/com/duckduckgo/app/trackerdetection/api/TrackerDataDownloader.kt index 7be5a84c6361..23475558567c 100644 --- a/app/src/main/java/com/duckduckgo/app/trackerdetection/api/TrackerDataDownloader.kt +++ b/app/src/main/java/com/duckduckgo/app/trackerdetection/api/TrackerDataDownloader.kt @@ -23,7 +23,7 @@ import com.duckduckgo.app.trackerdetection.Client import com.duckduckgo.app.trackerdetection.Client.ClientName.* import com.duckduckgo.app.trackerdetection.TrackerDataLoader import com.duckduckgo.app.trackerdetection.db.TrackerDataDao -import com.duckduckgo.app.trackerdetection.store.TrackerDataStore +import com.duckduckgo.app.global.store.BinaryDataStore import io.reactivex.Completable import okhttp3.ResponseBody import retrofit2.Call @@ -33,11 +33,11 @@ import javax.inject.Inject class TrackerDataDownloader @Inject constructor( - private val trackerListService: TrackerListService, - private val trackerDataStore: TrackerDataStore, - private val trackerDataLoader: TrackerDataLoader, - private val trackerDataDao: TrackerDataDao, - private val appDatabase: AppDatabase) { + private val trackerListService: TrackerListService, + private val binaryDataStore: BinaryDataStore, + private val trackerDataLoader: TrackerDataLoader, + private val trackerDataDao: TrackerDataDao, + private val appDatabase: AppDatabase) { fun downloadList(clientName: Client.ClientName): Completable { @@ -86,7 +86,7 @@ class TrackerDataDownloader @Inject constructor( val call = callFactory(clientName) val response = call.execute() - if (response.isCached && trackerDataStore.hasData(clientName)) { + if (response.isCached && binaryDataStore.hasData(clientName.name)) { Timber.d("${clientName.name} data already cached and stored") return@fromAction } @@ -105,13 +105,13 @@ class TrackerDataDownloader @Inject constructor( private fun persistTrackerData(clientName: Client.ClientName, bodyBytes: ByteArray) { val client = AdBlockClient(clientName) client.loadBasicData(bodyBytes) - trackerDataStore.saveData(clientName, client.getProcessedData()) + binaryDataStore.saveData(clientName.name, client.getProcessedData()) } private fun removeLegacyList(clientName: Client.ClientName): Completable { return Completable.fromAction { - if (trackerDataStore.hasData(clientName)) { - trackerDataStore.clearData(clientName) + if (binaryDataStore.hasData(clientName.name)) { + binaryDataStore.clearData(clientName.name) } return@fromAction } diff --git a/app/src/main/java/com/duckduckgo/app/trackerdetection/store/TrackerDataStore.kt b/app/src/main/java/com/duckduckgo/app/trackerdetection/store/TrackerDataStore.kt deleted file mode 100644 index 5fd4bc2369c4..000000000000 --- a/app/src/main/java/com/duckduckgo/app/trackerdetection/store/TrackerDataStore.kt +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright (c) 2017 DuckDuckGo - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.duckduckgo.app.trackerdetection.store - -import android.content.Context -import com.duckduckgo.app.trackerdetection.Client.ClientName -import javax.inject.Inject - -class TrackerDataStore @Inject constructor(private val context: Context) { - - fun hasData(client: ClientName): Boolean = context.fileExists(client.name) - - fun loadData(client: ClientName): ByteArray = - context.openFileInput(client.name).use { it.readBytes() } - - fun saveData(client: ClientName, byteArray: ByteArray) { - context.openFileOutput(client.name, Context.MODE_PRIVATE).write(byteArray) - } - - fun clearData(client: ClientName) { - context.deleteFile(client.name) - } - - private fun Context.fileExists(filename: String): Boolean { - val file = getFileStreamPath(filename) - return file != null && file.exists() - } -} From 8caf81b1b71783e551333d74f874cfaedf0e9cc1 Mon Sep 17 00:00:00 2001 From: Mia Alexiou Date: Thu, 26 Jul 2018 04:51:04 +0100 Subject: [PATCH 02/25] Tidy up, remove unneeded companion references --- .../app/feedback/ui/FeedbackViewModelTest.kt | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/app/src/androidTest/java/com/duckduckgo/app/feedback/ui/FeedbackViewModelTest.kt b/app/src/androidTest/java/com/duckduckgo/app/feedback/ui/FeedbackViewModelTest.kt index 84c9eecdd972..d480f81b4259 100644 --- a/app/src/androidTest/java/com/duckduckgo/app/feedback/ui/FeedbackViewModelTest.kt +++ b/app/src/androidTest/java/com/duckduckgo/app/feedback/ui/FeedbackViewModelTest.kt @@ -55,7 +55,7 @@ class FeedbackViewModelTest { @Test fun whenBrokenUrlSwitchedOnWithUrlThenMessageFocused() { - testee.onBrokenSiteUrlChanged(Constants.url) + testee.onBrokenSiteUrlChanged(url) testee.onBrokenSiteChanged(true) verify(mockCommandObserver).onChanged(Command.FocusMessage) } @@ -63,8 +63,8 @@ class FeedbackViewModelTest { @Test fun whenBrokenUrlOnWithUrlAndMessageThenCanSubmit() { testee.onBrokenSiteChanged(true) - testee.onBrokenSiteUrlChanged(Constants.url) - testee.onFeedbackMessageChanged(Constants.message) + testee.onBrokenSiteUrlChanged(url) + testee.onFeedbackMessageChanged(message) assertTrue(viewState.submitAllowed) } @@ -72,14 +72,14 @@ class FeedbackViewModelTest { fun whenBrokenUrlOnWithNullUrlThenCannotSubmit() { testee.onBrokenSiteChanged(true) testee.onBrokenSiteUrlChanged(null) - testee.onFeedbackMessageChanged(Constants.message) + testee.onFeedbackMessageChanged(message) assertFalse(viewState.submitAllowed) } @Test fun whenBrokenUrlOnWithNullMessageThenCannotSubmit() { testee.onBrokenSiteChanged(true) - testee.onBrokenSiteUrlChanged(Constants.url) + testee.onBrokenSiteUrlChanged(url) testee.onFeedbackMessageChanged(null) assertFalse(viewState.submitAllowed) } @@ -88,14 +88,14 @@ class FeedbackViewModelTest { fun whenBrokenUrlOnWithBlankUrlThenCannotSubmit() { testee.onBrokenSiteChanged(true) testee.onBrokenSiteUrlChanged(" ") - testee.onFeedbackMessageChanged(Constants.message) + testee.onFeedbackMessageChanged(message) assertFalse(viewState.submitAllowed) } @Test fun whenBrokenUrlOnWithBlankMessageThenCannotSubmit() { testee.onBrokenSiteChanged(true) - testee.onBrokenSiteUrlChanged(Constants.url) + testee.onBrokenSiteUrlChanged(url) testee.onFeedbackMessageChanged(" ") assertFalse(viewState.submitAllowed) } @@ -103,7 +103,7 @@ class FeedbackViewModelTest { @Test fun whenBrokenUrlOffWithMessageThenCanSubmit() { testee.onBrokenSiteChanged(false) - testee.onFeedbackMessageChanged(Constants.message) + testee.onFeedbackMessageChanged(message) assertTrue(viewState.submitAllowed) } @@ -124,11 +124,11 @@ class FeedbackViewModelTest { @Test fun whenCanSubmitBrokenSiteAndSubmitPressedThenFeedbackSubmitted() { testee.onBrokenSiteChanged(true) - testee.onBrokenSiteUrlChanged(Constants.url) - testee.onFeedbackMessageChanged(Constants.message) + testee.onBrokenSiteUrlChanged(url) + testee.onFeedbackMessageChanged(message) testee.onSubmitPressed() - verify(mockFeedbackSender).submitBrokenSiteFeedback(Constants.message, Constants.url) + verify(mockFeedbackSender).submitBrokenSiteFeedback(message, url) verify(mockCommandObserver).onChanged(Command.ConfirmAndFinish) } @@ -143,9 +143,9 @@ class FeedbackViewModelTest { @Test fun whenCanSubmitMessageAndSubmitPressedThenFeedbackSubmitted() { testee.onBrokenSiteChanged(false) - testee.onFeedbackMessageChanged(Constants.message) + testee.onFeedbackMessageChanged(message) testee.onSubmitPressed() - verify(mockFeedbackSender).submitGeneralFeedback(Constants.message) + verify(mockFeedbackSender).submitGeneralFeedback(message) verify(mockCommandObserver).onChanged(Command.ConfirmAndFinish) } From 64046e739b8965bcad3228fb9009eb2ee1d15de3 Mon Sep 17 00:00:00 2001 From: Mia Alexiou Date: Thu, 26 Jul 2018 23:48:15 +0100 Subject: [PATCH 03/25] Update tooling --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 98386d4a4592..ad70cbff4d42 100644 --- a/build.gradle +++ b/build.gradle @@ -7,7 +7,7 @@ buildscript { jcenter() } dependencies { - classpath 'com.android.tools.build:gradle:3.2.0-beta03' + classpath 'com.android.tools.build:gradle:3.2.0-beta04' classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version" // NOTE: Do not place your application dependencies here; they belong From 77852414f7373a0f11141eead540738b3c4dbe63 Mon Sep 17 00:00:00 2001 From: Mia Alexiou Date: Fri, 27 Jul 2018 00:41:06 +0100 Subject: [PATCH 04/25] Move bloom filter from project to submodule. Also move other 3rd party libs into correct folder --- .gitmodules | 7 +- app/CMakeLists.txt | 15 +- app/src/main/cpp/adblockclient-lib.cpp | 2 +- app/src/main/cpp/https-bloom-lib.cpp | 2 +- app/src/main/cpp/https/BloomFilter.cpp | 222 ------------------ app/src/main/cpp/https/BloomFilter.hpp | 23 -- .../ad-block/ad_block_client.cc | 0 .../ad-block/ad_block_client.h | 0 .../ad-block/bad_fingerprint.h | 0 .../ad-block/bad_fingerprints.h | 0 .../cpp/{ => third-party}/ad-block/base.h | 0 .../ad-block/cosmetic_filter.cc | 0 .../ad-block/cosmetic_filter.h | 0 .../cpp/{ => third-party}/ad-block/filter.cc | 0 .../cpp/{ => third-party}/ad-block/filter.h | 0 .../bloom-filter-cpp/BloomFilter.cpp | 0 .../bloom-filter-cpp/BloomFilter.h | 0 .../{ => third-party}/bloom-filter-cpp/base.h | 0 .../bloom-filter-cpp/hashFn.cpp | 0 .../bloom-filter-cpp/hashFn.h | 0 app/src/main/cpp/third-party/bloom_cpp | 1 + .../{ => third-party}/hashset-cpp/HashItem.h | 0 .../{ => third-party}/hashset-cpp/HashSet.cpp | 0 .../{ => third-party}/hashset-cpp/HashSet.h | 0 .../cpp/{ => third-party}/hashset-cpp/base.h | 0 .../{ => third-party}/hashset-cpp/hashFn.h | 0 26 files changed, 14 insertions(+), 258 deletions(-) delete mode 100644 app/src/main/cpp/https/BloomFilter.cpp delete mode 100644 app/src/main/cpp/https/BloomFilter.hpp rename app/src/main/cpp/{ => third-party}/ad-block/ad_block_client.cc (100%) rename app/src/main/cpp/{ => third-party}/ad-block/ad_block_client.h (100%) rename app/src/main/cpp/{ => third-party}/ad-block/bad_fingerprint.h (100%) rename app/src/main/cpp/{ => third-party}/ad-block/bad_fingerprints.h (100%) rename app/src/main/cpp/{ => third-party}/ad-block/base.h (100%) rename app/src/main/cpp/{ => third-party}/ad-block/cosmetic_filter.cc (100%) rename app/src/main/cpp/{ => third-party}/ad-block/cosmetic_filter.h (100%) rename app/src/main/cpp/{ => third-party}/ad-block/filter.cc (100%) rename app/src/main/cpp/{ => third-party}/ad-block/filter.h (100%) rename app/src/main/cpp/{ => third-party}/bloom-filter-cpp/BloomFilter.cpp (100%) rename app/src/main/cpp/{ => third-party}/bloom-filter-cpp/BloomFilter.h (100%) rename app/src/main/cpp/{ => third-party}/bloom-filter-cpp/base.h (100%) rename app/src/main/cpp/{ => third-party}/bloom-filter-cpp/hashFn.cpp (100%) rename app/src/main/cpp/{ => third-party}/bloom-filter-cpp/hashFn.h (100%) create mode 160000 app/src/main/cpp/third-party/bloom_cpp rename app/src/main/cpp/{ => third-party}/hashset-cpp/HashItem.h (100%) rename app/src/main/cpp/{ => third-party}/hashset-cpp/HashSet.cpp (100%) rename app/src/main/cpp/{ => third-party}/hashset-cpp/HashSet.h (100%) rename app/src/main/cpp/{ => third-party}/hashset-cpp/base.h (100%) rename app/src/main/cpp/{ => third-party}/hashset-cpp/hashFn.h (100%) diff --git a/.gitmodules b/.gitmodules index c375a5977496..3170b024bf77 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,4 @@ -[submodule "app/libs/OnionKit"] - path = app/libs/OnionKit - url = https://github.com/guardianproject/NetCipher.git +[submodule "app/src/main/cpp/third-party/bloom_cpp"] + path = app/src/main/cpp/third-party/bloom_cpp + url = git@github.com:duckduckgo/bloom_cpp.git + branch = feature/initial_filter diff --git a/app/CMakeLists.txt b/app/CMakeLists.txt index 11eae725cf38..85685fa663ae 100644 --- a/app/CMakeLists.txt +++ b/app/CMakeLists.txt @@ -18,15 +18,14 @@ add_library( # Sets the name of the library. # Provides a relative path to your source file(s). src/main/cpp/adblockclient-lib.cpp - src/main/cpp/ad-block/ad_block_client.cc - src/main/cpp/ad-block/cosmetic_filter.cc - src/main/cpp/ad-block/filter.cc - src/main/cpp/bloom-filter-cpp/BloomFilter.cpp - src/main/cpp/bloom-filter-cpp/hashFn.cpp - src/main/cpp/hashset-cpp/HashSet.cpp + src/main/cpp/third-party/ad-block/ad_block_client.cc + src/main/cpp/third-party/ad-block/cosmetic_filter.cc + src/main/cpp/third-party/ad-block/filter.cc + src/main/cpp/third-party/bloom-filter-cpp/BloomFilter.cpp + src/main/cpp/third-party/bloom-filter-cpp/hashFn.cpp + src/main/cpp/third-party/hashset-cpp/HashSet.cpp ) - add_library( # Sets the name of the library. https-bloom-lib @@ -35,7 +34,7 @@ add_library( # Sets the name of the library. # Provides a relative path to your source file(s). src/main/cpp/https-bloom-lib.cpp - src/main/cpp/https/BloomFilter.cpp + src/main/cpp/third-party/bloom_cpp/src/BloomFilter.cpp ) # Searches for a specified prebuilt library and stores the path as a diff --git a/app/src/main/cpp/adblockclient-lib.cpp b/app/src/main/cpp/adblockclient-lib.cpp index 73315509f09f..39c55ae41bb5 100644 --- a/app/src/main/cpp/adblockclient-lib.cpp +++ b/app/src/main/cpp/adblockclient-lib.cpp @@ -1,5 +1,5 @@ #include -#include "ad-block/ad_block_client.h" +#include "third-party/ad-block/ad_block_client.h" extern "C" JNIEXPORT jlong diff --git a/app/src/main/cpp/https-bloom-lib.cpp b/app/src/main/cpp/https-bloom-lib.cpp index 340221e8bffc..672e9848694f 100644 --- a/app/src/main/cpp/https-bloom-lib.cpp +++ b/app/src/main/cpp/https-bloom-lib.cpp @@ -1,5 +1,5 @@ #include -#include "https/BloomFilter.hpp" +#include "third-party/bloom_cpp/src/BloomFilter.hpp" extern "C" JNIEXPORT long diff --git a/app/src/main/cpp/https/BloomFilter.cpp b/app/src/main/cpp/https/BloomFilter.cpp deleted file mode 100644 index df9493aa6900..000000000000 --- a/app/src/main/cpp/https/BloomFilter.cpp +++ /dev/null @@ -1,222 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include "BloomFilter.hpp" - - -// Forward declarations of free functions - -unsigned int calculateHashRounds(unsigned int size, unsigned int maxItems); - -unsigned int djb2Hash(std::string text); - -unsigned int sdbmHash(std::string text); - -unsigned int doubleHash(unsigned int hash1, unsigned int hash2, unsigned int round); - -static void writeVectorToStream(std::vector& bloomVector, BinaryOutputStream& out); - -template T pack(const std::vector& filter, const size_t block, const size_t bits); - -static std::vector readVectorFromFile(std::string path); - -static std::vector readVectorFromStream(BinaryInputStream& in); - -void unpackIntoVector(std::vector& bloomVector, - const size_t offset, - const size_t bitsInThisBlock, - BinaryInputStream& in); - - -// Implementation - -BloomFilter::BloomFilter(unsigned int maxItems, double targetProbability) -{ - unsigned int size = ceil((maxItems * log(targetProbability)) / log(1.0 / (pow(2.0, log(2.0))))); - bloomVector = std::vector(size); - hashRounds = calculateHashRounds(size, maxItems); -} - -BloomFilter::BloomFilter(std::string importFilePath, unsigned int maxItems) -{ - bloomVector = readVectorFromFile(importFilePath); - hashRounds = calculateHashRounds((unsigned int) bloomVector.size(), maxItems); -} - -BloomFilter::BloomFilter(BinaryInputStream& in, unsigned int maxItems) -{ - bloomVector = readVectorFromStream(in); - hashRounds = calculateHashRounds((unsigned int) bloomVector.size(), maxItems); -} - -unsigned int calculateHashRounds(unsigned int size, unsigned int maxItems) -{ - return round(log(2.0) * size / maxItems); -} - -void BloomFilter::add(std::string element) -{ - unsigned int hash1 = djb2Hash(element); - unsigned int hash2 = sdbmHash(element); - - for (int i = 0; i < hashRounds; i++) - { - unsigned int hash = doubleHash(hash1, hash2, i); - unsigned int index = hash % bloomVector.size(); - bloomVector[index] = true; - } -} - -bool BloomFilter::contains(std::string element) -{ - unsigned int hash1 = djb2Hash(element); - unsigned int hash2 = sdbmHash(element); - - for (int i = 0; i < hashRounds; i++) - { - unsigned int hash = doubleHash(hash1, hash2, i); - unsigned int index = hash % bloomVector.size(); - if (bloomVector[index] == false) - { - return false; - } - } - - return true; -} - -unsigned int djb2Hash(std::string text) -{ - unsigned int hash = 5381; - for (auto iterator = text.begin(); iterator != text.end(); iterator++) - { - hash = ((hash << 5) + hash) + *iterator; - } - return hash; -} - -unsigned int sdbmHash(std::string text) -{ - unsigned int hash = 0; - for (auto iterator = text.begin(); iterator != text.end(); iterator++) - { - hash = *iterator + ((hash << 6) + (hash << 16) - hash); - } - return hash; -} - -unsigned int doubleHash(unsigned int hash1, unsigned int hash2, unsigned int round) -{ - switch (round) - { - case 0: - return hash1; - case 1: - return hash2; - default: - return (hash1 + (round * hash2) + (round^2)); - } -} - -void BloomFilter::writeToFile(std::string path) -{ - std::basic_ofstream out(path.c_str(), std::ofstream::binary); - writeVectorToStream(bloomVector, out); -} - - -void BloomFilter::writeToStream(BinaryOutputStream& out) -{ - writeVectorToStream(bloomVector, out); -} - -void writeVectorToStream(std::vector& bloomVector, BinaryOutputStream& out) -{ - const size_t elements = bloomVector.size(); - out.put(((elements & 0x000000ff) >> 0)); - out.put(((elements & 0x0000ff00) >> 8)); - out.put(((elements & 0x00ff0000) >> 16)); - out.put(((elements & 0xff000000) >> 24)); - - const size_t bitsPerBlock = sizeof(BlockType) * 8; - for (size_t i = 0; i < elements / bitsPerBlock; i++) - { - const BlockType buffer = pack(bloomVector, i, bitsPerBlock); - out.put( buffer ); - } - - const size_t bitsInLastBlock = elements % bitsPerBlock; - if (bitsInLastBlock > 0) - { - const size_t lastBlock = elements / bitsPerBlock; - const BlockType buffer = pack(bloomVector, lastBlock, bitsInLastBlock); - out.put( buffer ); - } -} - -template T pack(const std::vector& filter, const size_t block, const size_t bits) -{ - const size_t sizeOfTInBits = sizeof(T) * 8; - assert( bits <= sizeOfTInBits ); - T buffer = 0; - for (int j = 0; j < bits; ++j) - { - const size_t offset = (block * sizeOfTInBits) + j; - const T bit = filter[offset] << j; - buffer |= bit; - } - return buffer; -} - -std::vector readVectorFromFile(std::string path) -{ - std::basic_ifstream inFile(path, std::ifstream::binary); - return readVectorFromStream(inFile); -} - -std::vector readVectorFromStream(BinaryInputStream& in) -{ - const size_t component1 = in.get() << 0; - const size_t component2 = in.get() << 8; - const size_t component3 = in.get() << 16; - const size_t component4 = in.get() << 24; - const size_t elementCount = component1 + component2 + component3 + component4; - - std::vector bloomVector(elementCount); - - const size_t bitsPerBlock = sizeof(BlockType) * 8; - const size_t fullBlocks = elementCount / bitsPerBlock; - for (int i = 0; i < fullBlocks; ++i) - { - const size_t offset = i * bitsPerBlock; - unpackIntoVector(bloomVector, offset, bitsPerBlock, in); - } - - const size_t bitsInLastBlock = elementCount % bitsPerBlock; - if (bitsInLastBlock > 0) - { - const size_t offset = bitsPerBlock * fullBlocks; - unpackIntoVector(bloomVector, offset, bitsInLastBlock, in); - } - - return bloomVector; -} - -void unpackIntoVector(std::vector& bloomVector, - const size_t offset, - const size_t bitsInThisBlock, - BinaryInputStream& in) -{ - const BlockType block = in.get(); - - for (int j = 0; j < bitsInThisBlock; j++) - { - const BlockType mask = 1 << j; - bloomVector[offset + j] = (block & mask) != 0; - } -} diff --git a/app/src/main/cpp/https/BloomFilter.hpp b/app/src/main/cpp/https/BloomFilter.hpp deleted file mode 100644 index 8786b6103753..000000000000 --- a/app/src/main/cpp/https/BloomFilter.hpp +++ /dev/null @@ -1,23 +0,0 @@ -#include -#include -#include -#include - -typedef char BlockType; -typedef std::basic_istream BinaryInputStream; -typedef std::basic_ostream BinaryOutputStream; - -class BloomFilter { - -private: - unsigned int hashRounds; - std::vector bloomVector; -public: - BloomFilter(unsigned int maxItems, double targetProbability); - BloomFilter(std::string importFilePath, unsigned int maxItems); - BloomFilter(BinaryInputStream& in, unsigned int maxItems); - void add(std::string element); - bool contains(std::string element); - void writeToFile(std::string exportFilePath); - void writeToStream(BinaryOutputStream& out); -}; diff --git a/app/src/main/cpp/ad-block/ad_block_client.cc b/app/src/main/cpp/third-party/ad-block/ad_block_client.cc similarity index 100% rename from app/src/main/cpp/ad-block/ad_block_client.cc rename to app/src/main/cpp/third-party/ad-block/ad_block_client.cc diff --git a/app/src/main/cpp/ad-block/ad_block_client.h b/app/src/main/cpp/third-party/ad-block/ad_block_client.h similarity index 100% rename from app/src/main/cpp/ad-block/ad_block_client.h rename to app/src/main/cpp/third-party/ad-block/ad_block_client.h diff --git a/app/src/main/cpp/ad-block/bad_fingerprint.h b/app/src/main/cpp/third-party/ad-block/bad_fingerprint.h similarity index 100% rename from app/src/main/cpp/ad-block/bad_fingerprint.h rename to app/src/main/cpp/third-party/ad-block/bad_fingerprint.h diff --git a/app/src/main/cpp/ad-block/bad_fingerprints.h b/app/src/main/cpp/third-party/ad-block/bad_fingerprints.h similarity index 100% rename from app/src/main/cpp/ad-block/bad_fingerprints.h rename to app/src/main/cpp/third-party/ad-block/bad_fingerprints.h diff --git a/app/src/main/cpp/ad-block/base.h b/app/src/main/cpp/third-party/ad-block/base.h similarity index 100% rename from app/src/main/cpp/ad-block/base.h rename to app/src/main/cpp/third-party/ad-block/base.h diff --git a/app/src/main/cpp/ad-block/cosmetic_filter.cc b/app/src/main/cpp/third-party/ad-block/cosmetic_filter.cc similarity index 100% rename from app/src/main/cpp/ad-block/cosmetic_filter.cc rename to app/src/main/cpp/third-party/ad-block/cosmetic_filter.cc diff --git a/app/src/main/cpp/ad-block/cosmetic_filter.h b/app/src/main/cpp/third-party/ad-block/cosmetic_filter.h similarity index 100% rename from app/src/main/cpp/ad-block/cosmetic_filter.h rename to app/src/main/cpp/third-party/ad-block/cosmetic_filter.h diff --git a/app/src/main/cpp/ad-block/filter.cc b/app/src/main/cpp/third-party/ad-block/filter.cc similarity index 100% rename from app/src/main/cpp/ad-block/filter.cc rename to app/src/main/cpp/third-party/ad-block/filter.cc diff --git a/app/src/main/cpp/ad-block/filter.h b/app/src/main/cpp/third-party/ad-block/filter.h similarity index 100% rename from app/src/main/cpp/ad-block/filter.h rename to app/src/main/cpp/third-party/ad-block/filter.h diff --git a/app/src/main/cpp/bloom-filter-cpp/BloomFilter.cpp b/app/src/main/cpp/third-party/bloom-filter-cpp/BloomFilter.cpp similarity index 100% rename from app/src/main/cpp/bloom-filter-cpp/BloomFilter.cpp rename to app/src/main/cpp/third-party/bloom-filter-cpp/BloomFilter.cpp diff --git a/app/src/main/cpp/bloom-filter-cpp/BloomFilter.h b/app/src/main/cpp/third-party/bloom-filter-cpp/BloomFilter.h similarity index 100% rename from app/src/main/cpp/bloom-filter-cpp/BloomFilter.h rename to app/src/main/cpp/third-party/bloom-filter-cpp/BloomFilter.h diff --git a/app/src/main/cpp/bloom-filter-cpp/base.h b/app/src/main/cpp/third-party/bloom-filter-cpp/base.h similarity index 100% rename from app/src/main/cpp/bloom-filter-cpp/base.h rename to app/src/main/cpp/third-party/bloom-filter-cpp/base.h diff --git a/app/src/main/cpp/bloom-filter-cpp/hashFn.cpp b/app/src/main/cpp/third-party/bloom-filter-cpp/hashFn.cpp similarity index 100% rename from app/src/main/cpp/bloom-filter-cpp/hashFn.cpp rename to app/src/main/cpp/third-party/bloom-filter-cpp/hashFn.cpp diff --git a/app/src/main/cpp/bloom-filter-cpp/hashFn.h b/app/src/main/cpp/third-party/bloom-filter-cpp/hashFn.h similarity index 100% rename from app/src/main/cpp/bloom-filter-cpp/hashFn.h rename to app/src/main/cpp/third-party/bloom-filter-cpp/hashFn.h diff --git a/app/src/main/cpp/third-party/bloom_cpp b/app/src/main/cpp/third-party/bloom_cpp new file mode 160000 index 000000000000..849e85f3abba --- /dev/null +++ b/app/src/main/cpp/third-party/bloom_cpp @@ -0,0 +1 @@ +Subproject commit 849e85f3abba237e3f7792ad19b05448a232ea8e diff --git a/app/src/main/cpp/hashset-cpp/HashItem.h b/app/src/main/cpp/third-party/hashset-cpp/HashItem.h similarity index 100% rename from app/src/main/cpp/hashset-cpp/HashItem.h rename to app/src/main/cpp/third-party/hashset-cpp/HashItem.h diff --git a/app/src/main/cpp/hashset-cpp/HashSet.cpp b/app/src/main/cpp/third-party/hashset-cpp/HashSet.cpp similarity index 100% rename from app/src/main/cpp/hashset-cpp/HashSet.cpp rename to app/src/main/cpp/third-party/hashset-cpp/HashSet.cpp diff --git a/app/src/main/cpp/hashset-cpp/HashSet.h b/app/src/main/cpp/third-party/hashset-cpp/HashSet.h similarity index 100% rename from app/src/main/cpp/hashset-cpp/HashSet.h rename to app/src/main/cpp/third-party/hashset-cpp/HashSet.h diff --git a/app/src/main/cpp/hashset-cpp/base.h b/app/src/main/cpp/third-party/hashset-cpp/base.h similarity index 100% rename from app/src/main/cpp/hashset-cpp/base.h rename to app/src/main/cpp/third-party/hashset-cpp/base.h diff --git a/app/src/main/cpp/hashset-cpp/hashFn.h b/app/src/main/cpp/third-party/hashset-cpp/hashFn.h similarity index 100% rename from app/src/main/cpp/hashset-cpp/hashFn.h rename to app/src/main/cpp/third-party/hashset-cpp/hashFn.h From 388e1c926f04fa7ae5cf0791974af926343f0505 Mon Sep 17 00:00:00 2001 From: Mia Alexiou Date: Fri, 27 Jul 2018 10:21:40 +0100 Subject: [PATCH 05/25] Tidy up test names --- .../java/com/duckduckgo/app/httpsupgrade/BloomFilterTest.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/BloomFilterTest.kt b/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/BloomFilterTest.kt index 9bfbec8c83a1..e09583fac952 100644 --- a/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/BloomFilterTest.kt +++ b/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/BloomFilterTest.kt @@ -27,13 +27,13 @@ class BloomFilterTest { private lateinit var testee: BloomFilter @Test - fun whenBloomFilterEmptyContainsIsFalse() { + fun whenBloomFilterEmptyThenContainsIsFalse() { testee = BloomFilter(1000, TARGET_FALSE_POSITIVE_RATE) assertFalse(testee.contains("abc")) } @Test - fun whenBloomFilterContainsElementContainsIsTrue() { + fun whenBloomFilterContainsElementThenContainsIsTrue() { testee = BloomFilter(1000, TARGET_FALSE_POSITIVE_RATE) testee.add("abc") assertTrue(testee.contains("abc")) From 3498c36567ab60c84cb79dd1e14f9114fe926df4 Mon Sep 17 00:00:00 2001 From: Mia Alexiou Date: Fri, 27 Jul 2018 10:22:12 +0100 Subject: [PATCH 06/25] Add comments to indicate final urls --- .../app/httpsupgrade/api/HttpsUpgradeService.kt | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/app/src/main/java/com/duckduckgo/app/httpsupgrade/api/HttpsUpgradeService.kt b/app/src/main/java/com/duckduckgo/app/httpsupgrade/api/HttpsUpgradeService.kt index 0bcdb7f4cd6b..f4bdb48ff6f8 100644 --- a/app/src/main/java/com/duckduckgo/app/httpsupgrade/api/HttpsUpgradeService.kt +++ b/app/src/main/java/com/duckduckgo/app/httpsupgrade/api/HttpsUpgradeService.kt @@ -23,17 +23,18 @@ import okhttp3.ResponseBody import retrofit2.Call import retrofit2.http.GET +// TODO update to final urls interface HttpsUpgradeService { - // TODO final url - @GET("https://staticcdn.duckduckgo.com/https/https-whitelist-test.json") + //@GET("https://staticcdn.duckduckgo.com/https/https-mobile-whitelist.json") + @GET("http://192.168.86.132:8000/https-mobile-whitelist.json") fun whitelist(): Call> - // TODO final url - @GET("http://192.168.86.132:8000/response.json") + //@GET("https://staticcdn.duckduckgo.com/https/https-mobile-bloom-spec.json") + @GET("http://192.168.86.132:8000/https-mobile-bloom-spec.json") fun httpsBloomFilterSpec(): Observable - // TODO final url - @GET("http://192.168.86.132:8000/https_bloom_filter.bin") + //@GET("https://staticcdn.duckduckgo.com/https/https-mobile-bloom.bin") + @GET("http://192.168.86.132:8000/https-mobile-bloom.bin") fun httpsBloomFilter(): Call } From 832c700d4efe9b24a7e623ae7fd577e66efe862a Mon Sep 17 00:00:00 2001 From: Mia Alexiou Date: Fri, 27 Jul 2018 17:51:50 +0100 Subject: [PATCH 07/25] Update submodule --- app/src/main/cpp/third-party/bloom_cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/src/main/cpp/third-party/bloom_cpp b/app/src/main/cpp/third-party/bloom_cpp index 849e85f3abba..a3de7fcc6eae 160000 --- a/app/src/main/cpp/third-party/bloom_cpp +++ b/app/src/main/cpp/third-party/bloom_cpp @@ -1 +1 @@ -Subproject commit 849e85f3abba237e3f7792ad19b05448a232ea8e +Subproject commit a3de7fcc6eae1908c0ad6a3a4a05ff3172c8fd27 From 4413fb0fd1829f8011e0b409a2ec1f6c5d1f5364 Mon Sep 17 00:00:00 2001 From: Mia Alexiou Date: Mon, 30 Jul 2018 16:22:32 +0100 Subject: [PATCH 08/25] Update submodule to use https to see if BitRise copes with this better --- .gitmodules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index 3170b024bf77..077c27689563 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,4 +1,4 @@ [submodule "app/src/main/cpp/third-party/bloom_cpp"] path = app/src/main/cpp/third-party/bloom_cpp - url = git@github.com:duckduckgo/bloom_cpp.git + url = https://github.com/duckduckgo/bloom_cpp.git branch = feature/initial_filter From 11c2067d0d212b136c6486872fb75fdebee72eda Mon Sep 17 00:00:00 2001 From: Mia Alexiou Date: Mon, 30 Jul 2018 22:11:27 +0100 Subject: [PATCH 09/25] Tidy up post-merge --- app/src/main/java/com/duckduckgo/app/di/ApplicationModule.kt | 1 - app/src/main/java/com/duckduckgo/app/di/DaoModule.kt | 4 ++-- app/src/main/java/com/duckduckgo/app/di/NetworkModule.kt | 2 +- app/src/main/java/com/duckduckgo/app/di/StatisticsModule.kt | 3 --- 4 files changed, 3 insertions(+), 7 deletions(-) diff --git a/app/src/main/java/com/duckduckgo/app/di/ApplicationModule.kt b/app/src/main/java/com/duckduckgo/app/di/ApplicationModule.kt index cce2063c7a90..ccce7dd1ef41 100644 --- a/app/src/main/java/com/duckduckgo/app/di/ApplicationModule.kt +++ b/app/src/main/java/com/duckduckgo/app/di/ApplicationModule.kt @@ -18,7 +18,6 @@ package com.duckduckgo.app.di import android.app.Application import android.content.Context -import com.duckduckgo.app.global.device.DeviceInfo import dagger.Binds import dagger.Module import javax.inject.Singleton diff --git a/app/src/main/java/com/duckduckgo/app/di/DaoModule.kt b/app/src/main/java/com/duckduckgo/app/di/DaoModule.kt index 27575480ab85..75d82eb8fc03 100644 --- a/app/src/main/java/com/duckduckgo/app/di/DaoModule.kt +++ b/app/src/main/java/com/duckduckgo/app/di/DaoModule.kt @@ -24,13 +24,13 @@ import dagger.Provides class DaoModule { @Provides - fun provideHttpsWhitelistDao(database: AppDatabase) = database.httpsWhitelistedDao() + fun providesHttpsWhitelistDao(database: AppDatabase) = database.httpsWhitelistedDao() @Provides fun provideHttpsBloomFilterSpecDao(database: AppDatabase) = database.httpsBloomFilterSpecDao() @Provides - fun provideDisconnectTrackDao(database: AppDatabase) = database.trackerDataDao() + fun providesDisconnectTrackDao(database: AppDatabase) = database.trackerDataDao() @Provides fun providesNetworkLeaderboardDao(database: AppDatabase) = database.networkLeaderboardDao() diff --git a/app/src/main/java/com/duckduckgo/app/di/NetworkModule.kt b/app/src/main/java/com/duckduckgo/app/di/NetworkModule.kt index a4d9711b33b3..fff8690d2ecd 100644 --- a/app/src/main/java/com/duckduckgo/app/di/NetworkModule.kt +++ b/app/src/main/java/com/duckduckgo/app/di/NetworkModule.kt @@ -100,7 +100,7 @@ class NetworkModule { retrofit.create(TrackerListService::class.java) @Provides - fun httpsUpgradeService(retrofit: Retrofit): HttpsUpgradeService = + fun httpsUpgradeService(@Named("api") retrofit: Retrofit): HttpsUpgradeService = retrofit.create(HttpsUpgradeService::class.java) @Provides diff --git a/app/src/main/java/com/duckduckgo/app/di/StatisticsModule.kt b/app/src/main/java/com/duckduckgo/app/di/StatisticsModule.kt index b72dbbbe5f12..46791db9eaec 100644 --- a/app/src/main/java/com/duckduckgo/app/di/StatisticsModule.kt +++ b/app/src/main/java/com/duckduckgo/app/di/StatisticsModule.kt @@ -17,8 +17,6 @@ package com.duckduckgo.app.di import android.content.Context -import com.duckduckgo.app.global.AppUrl -import com.duckduckgo.app.global.api.ApiRequestInterceptor import com.duckduckgo.app.global.device.ContextDeviceInfo import com.duckduckgo.app.global.device.DeviceInfo import com.duckduckgo.app.statistics.VariantManager @@ -33,7 +31,6 @@ import dagger.Module import dagger.Provides import okhttp3.OkHttpClient import retrofit2.Retrofit -import retrofit2.adapter.rxjava2.RxJava2CallAdapterFactory import javax.inject.Named From 0e2fcd10a158598b4e0889c46b40889c0bc00670 Mon Sep 17 00:00:00 2001 From: Mia Alexiou Date: Wed, 1 Aug 2018 09:28:44 +0100 Subject: [PATCH 10/25] Update submodule and tidy up tests --- .../duckduckgo/app/httpsupgrade/BloomFilterTest.kt | 14 ++++++++------ app/src/main/cpp/third-party/bloom_cpp | 2 +- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/BloomFilterTest.kt b/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/BloomFilterTest.kt index e09583fac952..5f6b029c187e 100644 --- a/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/BloomFilterTest.kt +++ b/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/BloomFilterTest.kt @@ -28,13 +28,13 @@ class BloomFilterTest { @Test fun whenBloomFilterEmptyThenContainsIsFalse() { - testee = BloomFilter(1000, TARGET_FALSE_POSITIVE_RATE) + testee = BloomFilter(FILTER_ELEMENT_COUNT, TARGET_FALSE_POSITIVE_RATE) assertFalse(testee.contains("abc")) } @Test fun whenBloomFilterContainsElementThenContainsIsTrue() { - testee = BloomFilter(1000, TARGET_FALSE_POSITIVE_RATE) + testee = BloomFilter(FILTER_ELEMENT_COUNT, TARGET_FALSE_POSITIVE_RATE) testee.add("abc") assertTrue(testee.contains("abc")) } @@ -42,8 +42,8 @@ class BloomFilterTest { @Test fun whenBloomFilterContainsItemsThenLookupResultsAreWithinRange() { - val bloomData = createRandomStrings(1000) - val testData = bloomData + createRandomStrings(9000) + val bloomData = createRandomStrings(FILTER_ELEMENT_COUNT) + val testData = bloomData + createRandomStrings(ADDITIONAL_TEST_DATA_ELEMENT_COUNT) testee = BloomFilter(bloomData.size, TARGET_FALSE_POSITIVE_RATE) bloomData.forEach { testee.add(it) } @@ -55,7 +55,7 @@ class BloomFilterTest { bloomData.contains(element) && !result -> falseNegatives++ !bloomData.contains(element) && result -> falsePositives++ !bloomData.contains(element) && !result -> trueNegatives++ - bloomData.contains(element) && result == true -> truePositives++ + bloomData.contains(element) && result -> truePositives++ } } @@ -63,7 +63,7 @@ class BloomFilterTest { assertEquals(0, falseNegatives) assertEquals(bloomData.size, truePositives) assertTrue(trueNegatives <= testData.size - bloomData.size) - assertTrue(falsePositiveRate < ACCEPTABLE_FALSE_POSITIVE_RATE) + assertTrue(falsePositiveRate <= ACCEPTABLE_FALSE_POSITIVE_RATE) } private fun createRandomStrings(items: Int): ArrayList { @@ -73,6 +73,8 @@ class BloomFilterTest { } companion object { + const val FILTER_ELEMENT_COUNT = 1000 + const val ADDITIONAL_TEST_DATA_ELEMENT_COUNT = 9000 const val TARGET_FALSE_POSITIVE_RATE = 0.001 const val ACCEPTABLE_FALSE_POSITIVE_RATE = TARGET_FALSE_POSITIVE_RATE * 1.1 } diff --git a/app/src/main/cpp/third-party/bloom_cpp b/app/src/main/cpp/third-party/bloom_cpp index a3de7fcc6eae..bb794e96f9a3 160000 --- a/app/src/main/cpp/third-party/bloom_cpp +++ b/app/src/main/cpp/third-party/bloom_cpp @@ -1 +1 @@ -Subproject commit a3de7fcc6eae1908c0ad6a3a4a05ff3172c8fd27 +Subproject commit bb794e96f9a30ebd1049ac429c7de3b00b3ea812 From b601e82fc48ae0feb470c994ba3b37360f2fd8ea Mon Sep 17 00:00:00 2001 From: Mia Alexiou Date: Wed, 1 Aug 2018 15:25:43 +0100 Subject: [PATCH 11/25] Update dependency --- app/src/main/cpp/third-party/bloom_cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/src/main/cpp/third-party/bloom_cpp b/app/src/main/cpp/third-party/bloom_cpp index bb794e96f9a3..dcf676873654 160000 --- a/app/src/main/cpp/third-party/bloom_cpp +++ b/app/src/main/cpp/third-party/bloom_cpp @@ -1 +1 @@ -Subproject commit bb794e96f9a30ebd1049ac429c7de3b00b3ea812 +Subproject commit dcf6768736543bea90c364145b0c3855aef4399b From 9630543a6954552366be46488660069c63dc5125 Mon Sep 17 00:00:00 2001 From: Mia Alexiou Date: Wed, 1 Aug 2018 15:33:10 +0100 Subject: [PATCH 12/25] Fix junit import --- .../java/com/duckduckgo/app/global/ChecksumTest.kt | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/app/src/androidTest/java/com/duckduckgo/app/global/ChecksumTest.kt b/app/src/androidTest/java/com/duckduckgo/app/global/ChecksumTest.kt index a00d047936aa..c6b5c3e93374 100644 --- a/app/src/androidTest/java/com/duckduckgo/app/global/ChecksumTest.kt +++ b/app/src/androidTest/java/com/duckduckgo/app/global/ChecksumTest.kt @@ -16,9 +16,7 @@ package com.duckduckgo.app.global -import junit.framework.Assert.assertEquals -import junit.framework.Assert.assertTrue -import org.junit.Assert.assertFalse +import org.junit.Assert.* import org.junit.Test class ChecksumTest { From 671af1044ef3cd89842976e4f94c8a4115b3e297 Mon Sep 17 00:00:00 2001 From: Mia Alexiou Date: Fri, 10 Aug 2018 21:53:57 +0100 Subject: [PATCH 13/25] Update to final urls --- app/src/main/cpp/third-party/bloom_cpp | 2 +- .../app/httpsupgrade/api/HttpsUpgradeService.kt | 10 +++------- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/app/src/main/cpp/third-party/bloom_cpp b/app/src/main/cpp/third-party/bloom_cpp index dcf676873654..516d51ee4e88 160000 --- a/app/src/main/cpp/third-party/bloom_cpp +++ b/app/src/main/cpp/third-party/bloom_cpp @@ -1 +1 @@ -Subproject commit dcf6768736543bea90c364145b0c3855aef4399b +Subproject commit 516d51ee4e8834a65b792adf1e175c8372e6198f diff --git a/app/src/main/java/com/duckduckgo/app/httpsupgrade/api/HttpsUpgradeService.kt b/app/src/main/java/com/duckduckgo/app/httpsupgrade/api/HttpsUpgradeService.kt index f4bdb48ff6f8..9a129f592ea5 100644 --- a/app/src/main/java/com/duckduckgo/app/httpsupgrade/api/HttpsUpgradeService.kt +++ b/app/src/main/java/com/duckduckgo/app/httpsupgrade/api/HttpsUpgradeService.kt @@ -23,18 +23,14 @@ import okhttp3.ResponseBody import retrofit2.Call import retrofit2.http.GET -// TODO update to final urls interface HttpsUpgradeService { - //@GET("https://staticcdn.duckduckgo.com/https/https-mobile-whitelist.json") - @GET("http://192.168.86.132:8000/https-mobile-whitelist.json") + @GET("https://staticcdn.duckduckgo.com/https/https-mobile-whitelist.json") fun whitelist(): Call> - //@GET("https://staticcdn.duckduckgo.com/https/https-mobile-bloom-spec.json") - @GET("http://192.168.86.132:8000/https-mobile-bloom-spec.json") + @GET("https://staticcdn.duckduckgo.com/https/https-mobile-bloom-spec.json") fun httpsBloomFilterSpec(): Observable - //@GET("https://staticcdn.duckduckgo.com/https/https-mobile-bloom.bin") - @GET("http://192.168.86.132:8000/https-mobile-bloom.bin") + @GET("https://staticcdn.duckduckgo.com/https/https-mobile-bloom.bin") fun httpsBloomFilter(): Call } From bee47a1048751ea7523b81b850bb1d7754e15a08 Mon Sep 17 00:00:00 2001 From: Mia Alexiou Date: Fri, 10 Aug 2018 22:09:05 +0100 Subject: [PATCH 14/25] Remove additional whitelist clear --- .../app/httpsupgrade/api/HttpsUpgradeDataDownloader.kt | 1 - 1 file changed, 1 deletion(-) diff --git a/app/src/main/java/com/duckduckgo/app/httpsupgrade/api/HttpsUpgradeDataDownloader.kt b/app/src/main/java/com/duckduckgo/app/httpsupgrade/api/HttpsUpgradeDataDownloader.kt index 8c94b3ed8272..548f87545f63 100644 --- a/app/src/main/java/com/duckduckgo/app/httpsupgrade/api/HttpsUpgradeDataDownloader.kt +++ b/app/src/main/java/com/duckduckgo/app/httpsupgrade/api/HttpsUpgradeDataDownloader.kt @@ -75,7 +75,6 @@ class HttpsUpgradeDataDownloader @Inject constructor( Timber.d("Updating https bloom data store with new data") httpsBloomSpecDao.insert(specification) binaryDataStore.saveData(fileName, bytes) - whitelistDao.deleteAll() httpsUpgrader.reloadData() } } From 2b8e67f8991ee7eeae6800ebfc68f89227ea65ad Mon Sep 17 00:00:00 2001 From: Mia Alexiou Date: Fri, 10 Aug 2018 22:14:35 +0100 Subject: [PATCH 15/25] Remove old branch from module --- .gitmodules | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index 077c27689563..472b11f5ed40 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,4 +1,3 @@ [submodule "app/src/main/cpp/third-party/bloom_cpp"] path = app/src/main/cpp/third-party/bloom_cpp url = https://github.com/duckduckgo/bloom_cpp.git - branch = feature/initial_filter From 85a05f39d385a21b2034c0e7ff344b4529e9a76b Mon Sep 17 00:00:00 2001 From: Mia Alexiou Date: Fri, 10 Aug 2018 22:17:42 +0100 Subject: [PATCH 16/25] Improve variable name --- .../java/com/duckduckgo/app/httpsupgrade/BloomFilterTest.kt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/BloomFilterTest.kt b/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/BloomFilterTest.kt index 5f6b029c187e..d5566d16c3e6 100644 --- a/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/BloomFilterTest.kt +++ b/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/BloomFilterTest.kt @@ -59,11 +59,11 @@ class BloomFilterTest { } } - val falsePositiveRate = falsePositives / testData.size + val errorRate = falsePositives / testData.size assertEquals(0, falseNegatives) assertEquals(bloomData.size, truePositives) assertTrue(trueNegatives <= testData.size - bloomData.size) - assertTrue(falsePositiveRate <= ACCEPTABLE_FALSE_POSITIVE_RATE) + assertTrue(errorRate <= ACCEPTABLE_ERROR_RATE) } private fun createRandomStrings(items: Int): ArrayList { @@ -76,6 +76,6 @@ class BloomFilterTest { const val FILTER_ELEMENT_COUNT = 1000 const val ADDITIONAL_TEST_DATA_ELEMENT_COUNT = 9000 const val TARGET_FALSE_POSITIVE_RATE = 0.001 - const val ACCEPTABLE_FALSE_POSITIVE_RATE = TARGET_FALSE_POSITIVE_RATE * 1.1 + const val ACCEPTABLE_ERROR_RATE = TARGET_FALSE_POSITIVE_RATE * 1.1 } } \ No newline at end of file From 5ce6fc5e907b9f5a5201cd13fd545b93e0d030d2 Mon Sep 17 00:00:00 2001 From: Mia Alexiou Date: Mon, 13 Aug 2018 12:00:43 +0100 Subject: [PATCH 17/25] Update bitrise config to include avd version --- bitrise.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bitrise.yml b/bitrise.yml index 1d401c4be63f..2f2205ec48be 100644 --- a/bitrise.yml +++ b/bitrise.yml @@ -10,7 +10,9 @@ trigger_map: workflows: primary: steps: - - avd-manager@0.9.2: {} + - avd-manager@0.9.2: + inputs: + - version: '27' - activate-ssh-key@3.1.1: run_if: '{{getenv "SSH_RSA_PRIVATE_KEY" | ne ""}}' - git-clone@4.0.11: {} From 15967f6ac166a9c280255e0cb2292838a2d8ff06 Mon Sep 17 00:00:00 2001 From: Mia Alexiou Date: Mon, 13 Aug 2018 12:01:10 +0100 Subject: [PATCH 18/25] Rename constants for better clarity --- .../com/duckduckgo/app/httpsupgrade/BloomFilterTest.kt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/BloomFilterTest.kt b/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/BloomFilterTest.kt index d5566d16c3e6..509f85cc4b2f 100644 --- a/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/BloomFilterTest.kt +++ b/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/BloomFilterTest.kt @@ -28,13 +28,13 @@ class BloomFilterTest { @Test fun whenBloomFilterEmptyThenContainsIsFalse() { - testee = BloomFilter(FILTER_ELEMENT_COUNT, TARGET_FALSE_POSITIVE_RATE) + testee = BloomFilter(FILTER_ELEMENT_COUNT, TARGET_ERROR_RATE) assertFalse(testee.contains("abc")) } @Test fun whenBloomFilterContainsElementThenContainsIsTrue() { - testee = BloomFilter(FILTER_ELEMENT_COUNT, TARGET_FALSE_POSITIVE_RATE) + testee = BloomFilter(FILTER_ELEMENT_COUNT, TARGET_ERROR_RATE) testee.add("abc") assertTrue(testee.contains("abc")) } @@ -45,7 +45,7 @@ class BloomFilterTest { val bloomData = createRandomStrings(FILTER_ELEMENT_COUNT) val testData = bloomData + createRandomStrings(ADDITIONAL_TEST_DATA_ELEMENT_COUNT) - testee = BloomFilter(bloomData.size, TARGET_FALSE_POSITIVE_RATE) + testee = BloomFilter(bloomData.size, TARGET_ERROR_RATE) bloomData.forEach { testee.add(it) } var (falsePositives, truePositives, falseNegatives, trueNegatives) = arrayOf(0, 0, 0, 0) @@ -75,7 +75,7 @@ class BloomFilterTest { companion object { const val FILTER_ELEMENT_COUNT = 1000 const val ADDITIONAL_TEST_DATA_ELEMENT_COUNT = 9000 - const val TARGET_FALSE_POSITIVE_RATE = 0.001 - const val ACCEPTABLE_ERROR_RATE = TARGET_FALSE_POSITIVE_RATE * 1.1 + const val TARGET_ERROR_RATE = 0.001 + const val ACCEPTABLE_ERROR_RATE = TARGET_ERROR_RATE * 1.1 } } \ No newline at end of file From e090202a4555e16d0b1c6811813da10f689ee3ec Mon Sep 17 00:00:00 2001 From: Mia Alexiou Date: Tue, 14 Aug 2018 18:00:22 +0100 Subject: [PATCH 19/25] Simplify upgrade threading and add logging --- .../app/httpsupgrade/HttpsUpgraderTest.kt | 11 ++++++++++ .../app/httpsupgrade/HttpsUpgrader.kt | 22 ++++++++++++++----- .../api/HttpsBloomFilterFactory.kt | 20 +++++++---------- 3 files changed, 36 insertions(+), 17 deletions(-) diff --git a/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/HttpsUpgraderTest.kt b/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/HttpsUpgraderTest.kt index 92896d7ca63b..7768bbfe65ea 100644 --- a/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/HttpsUpgraderTest.kt +++ b/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/HttpsUpgraderTest.kt @@ -16,7 +16,9 @@ package com.duckduckgo.app.httpsupgrade +import android.arch.core.executor.testing.InstantTaskExecutorRule import android.net.Uri +import com.duckduckgo.app.InstantSchedulersRule import com.duckduckgo.app.httpsupgrade.api.HttpsBloomFilterFactory import com.duckduckgo.app.httpsupgrade.db.HttpsWhitelistDao import com.nhaarman.mockito_kotlin.mock @@ -24,10 +26,19 @@ import com.nhaarman.mockito_kotlin.whenever import org.junit.Assert.assertFalse import org.junit.Assert.assertTrue import org.junit.Before +import org.junit.Rule import org.junit.Test class HttpsUpgraderTest { + @get:Rule + @Suppress("unused") + var instantTaskExecutorRule = InstantTaskExecutorRule() + + @get:Rule + @Suppress("unused") + val schedulers = InstantSchedulersRule() + lateinit var testee: HttpsUpgrader private var mockHttpsBloomFilterFactory: HttpsBloomFilterFactory = mock() diff --git a/app/src/main/java/com/duckduckgo/app/httpsupgrade/HttpsUpgrader.kt b/app/src/main/java/com/duckduckgo/app/httpsupgrade/HttpsUpgrader.kt index a9b2daf99979..9b6845102bee 100644 --- a/app/src/main/java/com/duckduckgo/app/httpsupgrade/HttpsUpgrader.kt +++ b/app/src/main/java/com/duckduckgo/app/httpsupgrade/HttpsUpgrader.kt @@ -22,6 +22,7 @@ import com.duckduckgo.app.global.UrlScheme import com.duckduckgo.app.global.isHttps import com.duckduckgo.app.httpsupgrade.api.HttpsBloomFilterFactory import com.duckduckgo.app.httpsupgrade.db.HttpsWhitelistDao +import io.reactivex.schedulers.Schedulers import timber.log.Timber interface HttpsUpgrader { @@ -54,14 +55,19 @@ class HttpsUpgraderImpl( return false } - if (whitelistedDao.contains(uri.host)) { - Timber.d("${uri.host} is in whitelist and so not upgradable") + val host = uri.host + if (whitelistedDao.contains(host)) { + Timber.d("${host} is in whitelist and so not upgradable") return false } httpsBloomFilter?.let { - val shouldUpgrade = it.contains(uri.host) - Timber.d("${uri.host} ${if (shouldUpgrade) "is" else "is not"} upgradable") + + val initialTime = System.nanoTime() + val shouldUpgrade = it.contains(host) + val totalTime = System.nanoTime() - initialTime + Timber.d("${host} ${if (shouldUpgrade) "is" else "is not"} upgradable, lookup in ${totalTime/NANO_TO_MILLIS_DIVISOR}ms") + return shouldUpgrade } @@ -69,7 +75,13 @@ class HttpsUpgraderImpl( } override fun reloadData() { - httpsBloomFilter = bloomFactory.create() + Schedulers.io().scheduleDirect { + httpsBloomFilter = bloomFactory.create() + } + } + + companion object { + const val NANO_TO_MILLIS_DIVISOR = 1_000_000.0 } } diff --git a/app/src/main/java/com/duckduckgo/app/httpsupgrade/api/HttpsBloomFilterFactory.kt b/app/src/main/java/com/duckduckgo/app/httpsupgrade/api/HttpsBloomFilterFactory.kt index 964cb11e1851..530be277d598 100644 --- a/app/src/main/java/com/duckduckgo/app/httpsupgrade/api/HttpsBloomFilterFactory.kt +++ b/app/src/main/java/com/duckduckgo/app/httpsupgrade/api/HttpsBloomFilterFactory.kt @@ -16,34 +16,27 @@ package com.duckduckgo.app.httpsupgrade.api +import android.support.annotation.WorkerThread import com.duckduckgo.app.global.store.BinaryDataStore import com.duckduckgo.app.httpsupgrade.BloomFilter import com.duckduckgo.app.httpsupgrade.db.HttpsBloomFilterSpecDao import com.duckduckgo.app.httpsupgrade.model.HttpsBloomFilterSpec.Companion.HTTPS_BINARY_FILE -import io.reactivex.Observable -import io.reactivex.schedulers.Schedulers import timber.log.Timber +import java.util.* import javax.inject.Inject interface HttpsBloomFilterFactory { fun create(): BloomFilter? } -class OptionalWrapper(val value: T?) class HttpsBloomFilterFactoryImpl @Inject constructor(private val dao: HttpsBloomFilterSpecDao, private val binaryDataStore: BinaryDataStore) : HttpsBloomFilterFactory { + @WorkerThread override fun create(): BloomFilter? { - val specificationWrapper = Observable.just(dao) - .observeOn(Schedulers.io()) - .map { - OptionalWrapper(dao.get()) - } - .blockingSingle() - - val specification = specificationWrapper.value + val specification = dao.get() val dataPath = binaryDataStore.dataFilePath(HTTPS_BINARY_FILE) if (dataPath == null || specification == null) { @@ -51,8 +44,11 @@ class HttpsBloomFilterFactoryImpl @Inject constructor(private val dao: HttpsBloo return null } + val initialTimestamp = Date().time Timber.d("Found https data at $dataPath, building filter") - return BloomFilter(dataPath, specification.totalEntries) + var bloomFilter = BloomFilter(dataPath, specification.totalEntries) + Timber.v("Loading took ${Date().time - initialTimestamp}ms") + return bloomFilter } } \ No newline at end of file From 1eaf1c41e3b2a17f76cc12b02b850253b9f96d76 Mon Sep 17 00:00:00 2001 From: Mia Alexiou Date: Tue, 14 Aug 2018 18:33:53 +0100 Subject: [PATCH 20/25] Update bloom to point to release tag --- app/src/main/cpp/third-party/bloom_cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/src/main/cpp/third-party/bloom_cpp b/app/src/main/cpp/third-party/bloom_cpp index 516d51ee4e88..e3d110ad0b0c 160000 --- a/app/src/main/cpp/third-party/bloom_cpp +++ b/app/src/main/cpp/third-party/bloom_cpp @@ -1 +1 @@ -Subproject commit 516d51ee4e8834a65b792adf1e175c8372e6198f +Subproject commit e3d110ad0b0cd43140a23ee42f98b7bdeaa7ee36 From 844d00e7208592114d1d462f01100a69473968ca Mon Sep 17 00:00:00 2001 From: Mia Alexiou Date: Wed, 15 Aug 2018 11:58:17 +0100 Subject: [PATCH 21/25] Update timestamp call --- .../app/httpsupgrade/api/HttpsBloomFilterFactory.kt | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/app/src/main/java/com/duckduckgo/app/httpsupgrade/api/HttpsBloomFilterFactory.kt b/app/src/main/java/com/duckduckgo/app/httpsupgrade/api/HttpsBloomFilterFactory.kt index 530be277d598..36c43ca6e60d 100644 --- a/app/src/main/java/com/duckduckgo/app/httpsupgrade/api/HttpsBloomFilterFactory.kt +++ b/app/src/main/java/com/duckduckgo/app/httpsupgrade/api/HttpsBloomFilterFactory.kt @@ -22,7 +22,6 @@ import com.duckduckgo.app.httpsupgrade.BloomFilter import com.duckduckgo.app.httpsupgrade.db.HttpsBloomFilterSpecDao import com.duckduckgo.app.httpsupgrade.model.HttpsBloomFilterSpec.Companion.HTTPS_BINARY_FILE import timber.log.Timber -import java.util.* import javax.inject.Inject interface HttpsBloomFilterFactory { @@ -44,10 +43,10 @@ class HttpsBloomFilterFactoryImpl @Inject constructor(private val dao: HttpsBloo return null } - val initialTimestamp = Date().time + val initialTimestamp = System.currentTimeMillis() Timber.d("Found https data at $dataPath, building filter") var bloomFilter = BloomFilter(dataPath, specification.totalEntries) - Timber.v("Loading took ${Date().time - initialTimestamp}ms") + Timber.v("Loading took ${System.currentTimeMillis() - initialTimestamp}ms") return bloomFilter } From 0086bda3937fdbee1e527a12c1e646565c1cfe1c Mon Sep 17 00:00:00 2001 From: Mia Alexiou Date: Wed, 15 Aug 2018 12:15:26 +0100 Subject: [PATCH 22/25] Increase architecture components to 1.1.1 This fixes room migration bug. 1.1.1 See May 16 2018 release notes (1.1.1 is identical to 1.1.1-rc1) https://developer.android.com/jetpack/docs/release-notes --- app/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/build.gradle b/app/build.gradle index c10ccbf33f0b..20ae7acddc5c 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -78,7 +78,7 @@ android { ext { supportLibrary = "27.1.1" - architectureComponents = "1.1.0" + architectureComponents = "1.1.1" architectureComponentsExtensions = "1.1.1" androidKtx = "0.3" dagger = "2.14.1" From e60eec5269dbf47289396fdb003f6e74402b6e90 Mon Sep 17 00:00:00 2001 From: Mia Alexiou Date: Wed, 15 Aug 2018 12:59:18 +0100 Subject: [PATCH 23/25] Add transaction for full updates (where we delete then insert new records) of https whitelist and tracker entries --- .../httpsupgrade/db/HttpsWhitelistDaoTest.kt | 8 ++ .../trackerdetection/db/TrackerDataDaoTest.kt | 103 ++++++++++++++++++ .../api/HttpsUpgradeDataDownloader.kt | 3 +- .../app/httpsupgrade/db/HttpsWhitelistDao.kt | 21 ++-- .../api/TrackerDataDownloader.kt | 4 +- .../app/trackerdetection/db/TrackerDataDao.kt | 21 ++-- 6 files changed, 137 insertions(+), 23 deletions(-) create mode 100644 app/src/androidTest/java/com/duckduckgo/app/trackerdetection/db/TrackerDataDaoTest.kt diff --git a/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/db/HttpsWhitelistDaoTest.kt b/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/db/HttpsWhitelistDaoTest.kt index f2214efefa5b..23894ea1aeee 100644 --- a/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/db/HttpsWhitelistDaoTest.kt +++ b/app/src/androidTest/java/com/duckduckgo/app/httpsupgrade/db/HttpsWhitelistDaoTest.kt @@ -77,6 +77,14 @@ class HttpsWhitelistDaoTest { assertEquals(1, dao.count()) } + @Test + fun whenAllUpdatedThenPreviousValuesAreReplaced() { + dao.insertAll(listOf(HttpsWhitelistedDomain(domain))) + dao.updateAll(listOf(HttpsWhitelistedDomain(anotherDomain))) + assertEquals(1, dao.count()) + assertTrue(dao.contains(anotherDomain)) + } + @Test fun whenhAllDeletedThenContainsDomainIsFalse() { dao.insertAll(listOf(HttpsWhitelistedDomain(domain))) diff --git a/app/src/androidTest/java/com/duckduckgo/app/trackerdetection/db/TrackerDataDaoTest.kt b/app/src/androidTest/java/com/duckduckgo/app/trackerdetection/db/TrackerDataDaoTest.kt new file mode 100644 index 000000000000..0073f096ba65 --- /dev/null +++ b/app/src/androidTest/java/com/duckduckgo/app/trackerdetection/db/TrackerDataDaoTest.kt @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2018 DuckDuckGo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.duckduckgo.app.trackerdetection.db + +import android.arch.persistence.room.Room +import android.support.test.InstrumentationRegistry +import com.duckduckgo.app.global.db.AppDatabase +import com.duckduckgo.app.trackerdetection.model.DisconnectTracker +import org.junit.After +import org.junit.Assert.* +import org.junit.Before +import org.junit.Test + +class TrackerDataDaoTest { + + private lateinit var db: AppDatabase + private lateinit var dao: TrackerDataDao + + @Before + fun before() { + db = Room.inMemoryDatabaseBuilder(InstrumentationRegistry.getContext(), AppDatabase::class.java).build() + dao = db.trackerDataDao() + } + + @After + fun after() { + db.close() + } + + @Test + fun whenModelIsEmptyThenCountIsZero() { + assertEquals(0, dao.count()) + } + + @Test + fun whenTrackerInsertedThenCountIsOne() { + dao.insertAll(listOf(createTracker(trackerUrl))) + assertEquals(1, dao.count()) + } + + @Test + fun whenTrackerInsertedThenContainsTracker() { + val tracker = createTracker(trackerUrl) + dao.insertAll(listOf(tracker)) + assertTrue(dao.getAll().contains(tracker)) + } + + @Test + fun whenSecondUniqueTrackerInsertedThenCountIsTwo() { + dao.insertAll(listOf(createTracker(trackerUrl))) + dao.insertAll(listOf(createTracker(anotherTrackerUrl))) + assertEquals(2, dao.count()) + } + + @Test + fun whenSecondDuplicateTrackerInsertedThenCountIsOne() { + dao.insertAll(listOf(createTracker(trackerUrl))) + dao.insertAll(listOf(createTracker(trackerUrl))) + assertEquals(1, dao.count()) + } + + @Test + fun whenAllUpdatedThenPreviousValuesAreReplaced() { + val initialTracker = createTracker(trackerUrl) + val replacementTracker = createTracker(anotherTrackerUrl) + + dao.insertAll(listOf(initialTracker)) + dao.updateAll(listOf(replacementTracker)) + assertEquals(1, dao.count()) + assertTrue(dao.getAll().contains(replacementTracker)) + } + + @Test + fun whenhAllDeletedThenCountIsZero() { + val tracker = createTracker(trackerUrl) + dao.insertAll(listOf(tracker)) + dao.deleteAll() + assertEquals(0, dao.count()) + } + + fun createTracker(url: String): DisconnectTracker { + return DisconnectTracker(url, "","", "") + } + + companion object { + var trackerUrl = "tracker.com" + var anotherTrackerUrl = "anotherTracker.com" + } +} diff --git a/app/src/main/java/com/duckduckgo/app/httpsupgrade/api/HttpsUpgradeDataDownloader.kt b/app/src/main/java/com/duckduckgo/app/httpsupgrade/api/HttpsUpgradeDataDownloader.kt index 548f87545f63..f881b88c3250 100644 --- a/app/src/main/java/com/duckduckgo/app/httpsupgrade/api/HttpsUpgradeDataDownloader.kt +++ b/app/src/main/java/com/duckduckgo/app/httpsupgrade/api/HttpsUpgradeDataDownloader.kt @@ -95,8 +95,7 @@ class HttpsUpgradeDataDownloader @Inject constructor( if (response.isSuccessful) { val whitelist = response.body()!! Timber.d("Updating https whitelist with new data") - whitelistDao.deleteAll() - whitelistDao.insertAll(whitelist) + whitelistDao.updateAll(whitelist) } else { throw IOException("Status: ${response.code()} - ${response.errorBody()?.string()}") } diff --git a/app/src/main/java/com/duckduckgo/app/httpsupgrade/db/HttpsWhitelistDao.kt b/app/src/main/java/com/duckduckgo/app/httpsupgrade/db/HttpsWhitelistDao.kt index 261902d9b8bf..edb42e5b4029 100644 --- a/app/src/main/java/com/duckduckgo/app/httpsupgrade/db/HttpsWhitelistDao.kt +++ b/app/src/main/java/com/duckduckgo/app/httpsupgrade/db/HttpsWhitelistDao.kt @@ -16,26 +16,29 @@ package com.duckduckgo.app.httpsupgrade.db -import android.arch.persistence.room.Dao -import android.arch.persistence.room.Insert -import android.arch.persistence.room.OnConflictStrategy -import android.arch.persistence.room.Query +import android.arch.persistence.room.* import com.duckduckgo.app.httpsupgrade.model.HttpsWhitelistedDomain import javax.inject.Singleton @Dao @Singleton -interface HttpsWhitelistDao { +abstract class HttpsWhitelistDao { @Insert(onConflict = OnConflictStrategy.REPLACE) - fun insertAll(domains: List) + abstract fun insertAll(domains: List) + + @Transaction + open fun updateAll(domains: List) { + deleteAll() + insertAll(domains) + } @Query("select count(1) > 0 from https_whitelisted_domain where domain = :domain") - fun contains(domain: String): Boolean + abstract fun contains(domain: String): Boolean @Query("delete from https_whitelisted_domain") - fun deleteAll() + abstract fun deleteAll() @Query("select count(1) from https_whitelisted_domain") - fun count(): Int + abstract fun count(): Int } \ No newline at end of file diff --git a/app/src/main/java/com/duckduckgo/app/trackerdetection/api/TrackerDataDownloader.kt b/app/src/main/java/com/duckduckgo/app/trackerdetection/api/TrackerDataDownloader.kt index 23475558567c..826c6b04a30f 100644 --- a/app/src/main/java/com/duckduckgo/app/trackerdetection/api/TrackerDataDownloader.kt +++ b/app/src/main/java/com/duckduckgo/app/trackerdetection/api/TrackerDataDownloader.kt @@ -67,12 +67,10 @@ class TrackerDataDownloader @Inject constructor( val body = response.body()!! appDatabase.runInTransaction { - trackerDataDao.deleteAll() - trackerDataDao.insertAll(body.trackers) + trackerDataDao.updateAll(body.trackers) trackerDataLoader.loadDisconnectData() } - } else { throw IOException("Status: ${response.code()} - ${response.errorBody()?.string()}") } diff --git a/app/src/main/java/com/duckduckgo/app/trackerdetection/db/TrackerDataDao.kt b/app/src/main/java/com/duckduckgo/app/trackerdetection/db/TrackerDataDao.kt index 189dfbdc05cb..7ce9cc83d4c1 100644 --- a/app/src/main/java/com/duckduckgo/app/trackerdetection/db/TrackerDataDao.kt +++ b/app/src/main/java/com/duckduckgo/app/trackerdetection/db/TrackerDataDao.kt @@ -16,25 +16,28 @@ package com.duckduckgo.app.trackerdetection.db -import android.arch.persistence.room.Dao -import android.arch.persistence.room.Insert -import android.arch.persistence.room.OnConflictStrategy -import android.arch.persistence.room.Query +import android.arch.persistence.room.* import com.duckduckgo.app.trackerdetection.model.DisconnectTracker @Dao -interface TrackerDataDao { +abstract class TrackerDataDao { @Insert(onConflict = OnConflictStrategy.REPLACE) - fun insertAll(trackers: List) + abstract fun insertAll(trackers: List) + + @Transaction + open fun updateAll(trackers: List) { + deleteAll() + insertAll(trackers) + } @Query("Select * from disconnect_tracker") - fun getAll() : List + abstract fun getAll() : List @Query("Select count(*) from disconnect_tracker") - fun count(): Int + abstract fun count(): Int @Query("DELETE FROM disconnect_tracker") - fun deleteAll() + abstract fun deleteAll() } \ No newline at end of file From 0651883ab410abac23dbfa2389430236f5b44912 Mon Sep 17 00:00:00 2001 From: Mia Alexiou Date: Wed, 15 Aug 2018 13:02:21 +0100 Subject: [PATCH 24/25] Make query case consistent with rest of project --- .../duckduckgo/app/trackerdetection/db/TrackerDataDao.kt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/src/main/java/com/duckduckgo/app/trackerdetection/db/TrackerDataDao.kt b/app/src/main/java/com/duckduckgo/app/trackerdetection/db/TrackerDataDao.kt index 7ce9cc83d4c1..e52b5d723482 100644 --- a/app/src/main/java/com/duckduckgo/app/trackerdetection/db/TrackerDataDao.kt +++ b/app/src/main/java/com/duckduckgo/app/trackerdetection/db/TrackerDataDao.kt @@ -32,12 +32,12 @@ abstract class TrackerDataDao { insertAll(trackers) } - @Query("Select * from disconnect_tracker") + @Query("select * from disconnect_tracker") abstract fun getAll() : List - @Query("Select count(*) from disconnect_tracker") + @Query("select count(*) from disconnect_tracker") abstract fun count(): Int - @Query("DELETE FROM disconnect_tracker") + @Query("delete FROM disconnect_tracker") abstract fun deleteAll() } \ No newline at end of file From 6d52256c9abb1cdae6e5d75d0dbdff3359df21bd Mon Sep 17 00:00:00 2001 From: Mia Alexiou Date: Wed, 15 Aug 2018 13:42:47 +0100 Subject: [PATCH 25/25] Another update to case --- .../com/duckduckgo/app/trackerdetection/db/TrackerDataDao.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/src/main/java/com/duckduckgo/app/trackerdetection/db/TrackerDataDao.kt b/app/src/main/java/com/duckduckgo/app/trackerdetection/db/TrackerDataDao.kt index e52b5d723482..0cfd2ae4dcd7 100644 --- a/app/src/main/java/com/duckduckgo/app/trackerdetection/db/TrackerDataDao.kt +++ b/app/src/main/java/com/duckduckgo/app/trackerdetection/db/TrackerDataDao.kt @@ -38,6 +38,6 @@ abstract class TrackerDataDao { @Query("select count(*) from disconnect_tracker") abstract fun count(): Int - @Query("delete FROM disconnect_tracker") + @Query("delete from disconnect_tracker") abstract fun deleteAll() } \ No newline at end of file