From 326a3b728cb99ee410b9bf1b81d78e7c5da35fbd Mon Sep 17 00:00:00 2001 From: David Gonzalez Date: Wed, 3 Mar 2021 17:27:58 +0100 Subject: [PATCH 1/6] added a httpsdatapersister that does not persist embedded data for froid --- .../HttpsEmbeddedDataIntegrationTest.kt | 4 +- .../duckduckgo/app/di/HttpsPersisterModule.kt | 41 ++++++++ .../store/FDroidHttpsDataPersister.kt | 74 +++++++++++++++ app/src/main/AndroidManifest.xml | 1 + .../com/duckduckgo/app/di/AppComponent.kt | 3 +- .../httpsupgrade/store/HttpsDataPersister.kt | 65 ++----------- .../duckduckgo/app/di/HttpsPersisterModule.kt | 45 +++++++++ .../store/PlayHttpsDataPersister.kt | 88 ++++++++++++++++++ .../res/raw/https_mobile_v2_bloom.bin | Bin .../res/raw/https_mobile_v2_bloom_spec.json | 0 .../raw/https_mobile_v2_false_positives.json | 0 11 files changed, 260 insertions(+), 61 deletions(-) create mode 100644 app/src/fdroid/java/com/duckduckgo/app/di/HttpsPersisterModule.kt create mode 100644 app/src/fdroid/java/com/duckduckgo/httpsupgrade/store/FDroidHttpsDataPersister.kt create mode 100644 app/src/play/java/com/duckduckgo/app/di/HttpsPersisterModule.kt create mode 100644 app/src/play/java/com/duckduckgo/httpsupgrade/store/PlayHttpsDataPersister.kt rename app/src/{main => play}/res/raw/https_mobile_v2_bloom.bin (100%) rename app/src/{main => play}/res/raw/https_mobile_v2_bloom_spec.json (100%) rename app/src/{main => play}/res/raw/https_mobile_v2_false_positives.json (100%) diff --git a/app/src/androidTest/java/com/duckduckgo/app/integration/HttpsEmbeddedDataIntegrationTest.kt b/app/src/androidTest/java/com/duckduckgo/app/integration/HttpsEmbeddedDataIntegrationTest.kt index 33638a9f0de1..205d95140f1e 100644 --- a/app/src/androidTest/java/com/duckduckgo/app/integration/HttpsEmbeddedDataIntegrationTest.kt +++ b/app/src/androidTest/java/com/duckduckgo/app/integration/HttpsEmbeddedDataIntegrationTest.kt @@ -25,7 +25,7 @@ import com.duckduckgo.app.httpsupgrade.HttpsBloomFilterFactoryImpl import com.duckduckgo.app.httpsupgrade.HttpsUpgrader import com.duckduckgo.app.httpsupgrade.HttpsUpgraderImpl import com.duckduckgo.app.httpsupgrade.api.HttpsFalsePositivesJsonAdapter -import com.duckduckgo.app.httpsupgrade.store.HttpsDataPersister +import com.duckduckgo.httpsupgrade.store.PlayHttpsDataPersister import com.duckduckgo.app.privacy.db.UserWhitelistDao import com.duckduckgo.app.statistics.pixels.Pixel import com.nhaarman.mockitokotlin2.mock @@ -55,7 +55,7 @@ class HttpsEmbeddedDataIntegrationTest { var httpsFalsePositivesDao = db.httpsFalsePositivesDao() var binaryDataStore: BinaryDataStore = BinaryDataStore(context) - val persister = HttpsDataPersister( + val persister = PlayHttpsDataPersister( binaryDataStore, httpsBloomSpecDao, httpsFalsePositivesDao, diff --git a/app/src/fdroid/java/com/duckduckgo/app/di/HttpsPersisterModule.kt b/app/src/fdroid/java/com/duckduckgo/app/di/HttpsPersisterModule.kt new file mode 100644 index 000000000000..dc0bb9b6ed86 --- /dev/null +++ b/app/src/fdroid/java/com/duckduckgo/app/di/HttpsPersisterModule.kt @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2021 DuckDuckGo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.duckduckgo.app.di + +import com.duckduckgo.app.global.db.AppDatabase +import com.duckduckgo.app.global.store.BinaryDataStore +import com.duckduckgo.app.httpsupgrade.store.HttpsBloomFilterSpecDao +import com.duckduckgo.app.httpsupgrade.store.HttpsDataPersister +import com.duckduckgo.app.httpsupgrade.store.HttpsFalsePositivesDao +import com.duckduckgo.httpsupgrade.store.FDroidHttpsDataPersister +import dagger.Module +import dagger.Provides + +@Module +class HttpsPersisterModule { + + @Provides + fun providesHttpsDataPersister( + binaryDataStore: BinaryDataStore, + httpsBloomSpecDao: HttpsBloomFilterSpecDao, + httpsFalsePositivesDao: HttpsFalsePositivesDao, + appDatabase: AppDatabase + ): HttpsDataPersister { + return FDroidHttpsDataPersister(binaryDataStore, httpsBloomSpecDao, httpsFalsePositivesDao, appDatabase) + } + +} diff --git a/app/src/fdroid/java/com/duckduckgo/httpsupgrade/store/FDroidHttpsDataPersister.kt b/app/src/fdroid/java/com/duckduckgo/httpsupgrade/store/FDroidHttpsDataPersister.kt new file mode 100644 index 000000000000..cd4ea77a80c4 --- /dev/null +++ b/app/src/fdroid/java/com/duckduckgo/httpsupgrade/store/FDroidHttpsDataPersister.kt @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2021 DuckDuckGo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.duckduckgo.httpsupgrade.store + +import com.duckduckgo.app.global.db.AppDatabase +import com.duckduckgo.app.global.store.BinaryDataStore +import com.duckduckgo.app.httpsupgrade.model.HttpsBloomFilterSpec +import com.duckduckgo.app.httpsupgrade.model.HttpsFalsePositiveDomain +import com.duckduckgo.app.httpsupgrade.store.HttpsBloomFilterSpecDao +import com.duckduckgo.app.httpsupgrade.store.HttpsDataPersister +import com.duckduckgo.app.httpsupgrade.store.HttpsFalsePositivesDao +import timber.log.Timber +import java.io.IOException +import javax.inject.Inject + +class FDroidHttpsDataPersister @Inject constructor( + private val binaryDataStore: BinaryDataStore, + private val httpsBloomSpecDao: HttpsBloomFilterSpecDao, + private val httpsFalsePositivesDao: HttpsFalsePositivesDao, + private val appDatabase: AppDatabase +) : HttpsDataPersister { + + override fun persistBloomFilter(specification: HttpsBloomFilterSpec, bytes: ByteArray) { + if (!binaryDataStore.verifyCheckSum(bytes, specification.sha256)) { + throw IOException("Https binary has incorrect sha, throwing away file") + } + + Timber.d("Updating https bloom data store with new data") + appDatabase.runInTransaction { + httpsBloomSpecDao.insert(specification) + binaryDataStore.saveData(HttpsBloomFilterSpec.HTTPS_BINARY_FILE, bytes) + } + } + + private fun persistBloomFilter(specification: HttpsBloomFilterSpec, bytes: ByteArray, falsePositives: List) { + appDatabase.runInTransaction { + persistBloomFilter(specification, bytes) + persistFalsePositives(falsePositives) + } + } + + override fun persistFalsePositives(falsePositives: List) { + httpsFalsePositivesDao.updateAll(falsePositives) + } + + override fun persistEmbeddedData() { + Timber.d("F-Droid does not get https data from embedded files") + return + } + + override fun isPersisted(specification: HttpsBloomFilterSpec): Boolean { + return specification == httpsBloomSpecDao.get() && binaryDataStore.verifyCheckSum(HttpsBloomFilterSpec.HTTPS_BINARY_FILE, specification.sha256) + } + + override fun isPersisted(): Boolean { + val specification = httpsBloomSpecDao.get() ?: return false + return binaryDataStore.verifyCheckSum(HttpsBloomFilterSpec.HTTPS_BINARY_FILE, specification.sha256) + } + +} diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml index 193cef9f5e8f..0a8de6669125 100644 --- a/app/src/main/AndroidManifest.xml +++ b/app/src/main/AndroidManifest.xml @@ -186,6 +186,7 @@ android:configChanges="keyboardHidden|orientation|screenSize" android:label="@string/appDescription" android:launchMode="singleTask" + android:supportsPictureInPicture="true" android:windowSoftInputMode="adjustResize|stateHidden"> diff --git a/app/src/main/java/com/duckduckgo/app/di/AppComponent.kt b/app/src/main/java/com/duckduckgo/app/di/AppComponent.kt index 4ea47a2ca7f9..c81814f862aa 100644 --- a/app/src/main/java/com/duckduckgo/app/di/AppComponent.kt +++ b/app/src/main/java/com/duckduckgo/app/di/AppComponent.kt @@ -78,7 +78,8 @@ import javax.inject.Singleton StoreReferralModule::class, CoroutinesModule::class, CertificateTrustedStoreModule::class, - WelcomePageModule::class + WelcomePageModule::class, + HttpsPersisterModule::class ] ) interface AppComponent : AndroidInjector { diff --git a/app/src/main/java/com/duckduckgo/app/httpsupgrade/store/HttpsDataPersister.kt b/app/src/main/java/com/duckduckgo/app/httpsupgrade/store/HttpsDataPersister.kt index ff9a9adf7dec..42ff43f6ba28 100644 --- a/app/src/main/java/com/duckduckgo/app/httpsupgrade/store/HttpsDataPersister.kt +++ b/app/src/main/java/com/duckduckgo/app/httpsupgrade/store/HttpsDataPersister.kt @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020 DuckDuckGo + * Copyright (c) 2021 DuckDuckGo * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,70 +16,19 @@ package com.duckduckgo.app.httpsupgrade.store -import android.content.Context -import com.duckduckgo.app.browser.R -import com.duckduckgo.app.global.db.AppDatabase -import com.duckduckgo.app.global.store.BinaryDataStore import com.duckduckgo.app.httpsupgrade.model.HttpsBloomFilterSpec import com.duckduckgo.app.httpsupgrade.model.HttpsFalsePositiveDomain -import com.squareup.moshi.JsonAdapter -import com.squareup.moshi.Moshi -import com.squareup.moshi.Types -import timber.log.Timber -import java.io.IOException -import javax.inject.Inject -class HttpsDataPersister @Inject constructor( - private val binaryDataStore: BinaryDataStore, - private val httpsBloomSpecDao: HttpsBloomFilterSpecDao, - private val httpsFalsePositivesDao: HttpsFalsePositivesDao, - private val appDatabase: AppDatabase, - private val context: Context, - private val moshi: Moshi -) { +interface HttpsDataPersister { - fun persistBloomFilter(specification: HttpsBloomFilterSpec, bytes: ByteArray) { - if (!binaryDataStore.verifyCheckSum(bytes, specification.sha256)) { - throw IOException("Https binary has incorrect sha, throwing away file") - } + fun persistBloomFilter(specification: HttpsBloomFilterSpec, bytes: ByteArray) - Timber.d("Updating https bloom data store with new data") - appDatabase.runInTransaction { - httpsBloomSpecDao.insert(specification) - binaryDataStore.saveData(HttpsBloomFilterSpec.HTTPS_BINARY_FILE, bytes) - } - } + fun persistFalsePositives(falsePositives: List) - private fun persistBloomFilter(specification: HttpsBloomFilterSpec, bytes: ByteArray, falsePositives: List) { - appDatabase.runInTransaction { - persistBloomFilter(specification, bytes) - persistFalsePositives(falsePositives) - } - } + fun persistEmbeddedData() - fun persistFalsePositives(falsePositives: List) { - httpsFalsePositivesDao.updateAll(falsePositives) - } + fun isPersisted(specification: HttpsBloomFilterSpec): Boolean - fun persistEmbeddedData() { - Timber.d("Updating https data from embedded files") - val specJson = context.resources.openRawResource(R.raw.https_mobile_v2_bloom_spec).bufferedReader().use { it.readText() } - val specAdapter = moshi.adapter(HttpsBloomFilterSpec::class.java) + fun isPersisted(): Boolean - val falsePositivesJson = context.resources.openRawResource(R.raw.https_mobile_v2_false_positives).bufferedReader().use { it.readText() } - val falsePositivesType = Types.newParameterizedType(List::class.java, HttpsFalsePositiveDomain::class.java) - val falsePositivesAdapter: JsonAdapter> = moshi.adapter(falsePositivesType) - - val bytes = context.resources.openRawResource(R.raw.https_mobile_v2_bloom).use { it.readBytes() } - persistBloomFilter(specAdapter.fromJson(specJson)!!, bytes, falsePositivesAdapter.fromJson(falsePositivesJson)!!) - } - - fun isPersisted(specification: HttpsBloomFilterSpec): Boolean { - return specification == httpsBloomSpecDao.get() && binaryDataStore.verifyCheckSum(HttpsBloomFilterSpec.HTTPS_BINARY_FILE, specification.sha256) - } - - fun isPersisted(): Boolean { - val specification = httpsBloomSpecDao.get() ?: return false - return binaryDataStore.verifyCheckSum(HttpsBloomFilterSpec.HTTPS_BINARY_FILE, specification.sha256) - } } diff --git a/app/src/play/java/com/duckduckgo/app/di/HttpsPersisterModule.kt b/app/src/play/java/com/duckduckgo/app/di/HttpsPersisterModule.kt new file mode 100644 index 000000000000..537a9bb63fac --- /dev/null +++ b/app/src/play/java/com/duckduckgo/app/di/HttpsPersisterModule.kt @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2021 DuckDuckGo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.duckduckgo.app.di + +import android.content.Context +import com.duckduckgo.app.global.db.AppDatabase +import com.duckduckgo.app.global.store.BinaryDataStore +import com.duckduckgo.app.httpsupgrade.store.HttpsBloomFilterSpecDao +import com.duckduckgo.app.httpsupgrade.store.HttpsDataPersister +import com.duckduckgo.app.httpsupgrade.store.HttpsFalsePositivesDao +import com.duckduckgo.httpsupgrade.store.PlayHttpsDataPersister +import com.squareup.moshi.Moshi +import dagger.Module +import dagger.Provides + +@Module +class HttpsPersisterModule { + + @Provides + fun providesHttpsDataPersister( + binaryDataStore: BinaryDataStore, + httpsBloomSpecDao: HttpsBloomFilterSpecDao, + httpsFalsePositivesDao: HttpsFalsePositivesDao, + appDatabase: AppDatabase, + context: Context, + moshi: Moshi + ): HttpsDataPersister { + return PlayHttpsDataPersister(binaryDataStore, httpsBloomSpecDao, httpsFalsePositivesDao, appDatabase, context, moshi) + } + +} diff --git a/app/src/play/java/com/duckduckgo/httpsupgrade/store/PlayHttpsDataPersister.kt b/app/src/play/java/com/duckduckgo/httpsupgrade/store/PlayHttpsDataPersister.kt new file mode 100644 index 000000000000..bb30a7ddd0ed --- /dev/null +++ b/app/src/play/java/com/duckduckgo/httpsupgrade/store/PlayHttpsDataPersister.kt @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2021 DuckDuckGo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.duckduckgo.httpsupgrade.store + +import android.content.Context +import com.duckduckgo.app.browser.R +import com.duckduckgo.app.global.db.AppDatabase +import com.duckduckgo.app.global.store.BinaryDataStore +import com.duckduckgo.app.httpsupgrade.model.HttpsBloomFilterSpec +import com.duckduckgo.app.httpsupgrade.model.HttpsFalsePositiveDomain +import com.duckduckgo.app.httpsupgrade.store.HttpsBloomFilterSpecDao +import com.duckduckgo.app.httpsupgrade.store.HttpsDataPersister +import com.duckduckgo.app.httpsupgrade.store.HttpsFalsePositivesDao +import com.squareup.moshi.JsonAdapter +import com.squareup.moshi.Moshi +import com.squareup.moshi.Types +import timber.log.Timber +import java.io.IOException +import javax.inject.Inject + +class PlayHttpsDataPersister @Inject constructor( + private val binaryDataStore: BinaryDataStore, + private val httpsBloomSpecDao: HttpsBloomFilterSpecDao, + private val httpsFalsePositivesDao: HttpsFalsePositivesDao, + private val appDatabase: AppDatabase, + private val context: Context, + private val moshi: Moshi +) : HttpsDataPersister { + + override fun persistBloomFilter(specification: HttpsBloomFilterSpec, bytes: ByteArray) { + if (!binaryDataStore.verifyCheckSum(bytes, specification.sha256)) { + throw IOException("Https binary has incorrect sha, throwing away file") + } + + Timber.d("Updating https bloom data store with new data") + appDatabase.runInTransaction { + httpsBloomSpecDao.insert(specification) + binaryDataStore.saveData(HttpsBloomFilterSpec.HTTPS_BINARY_FILE, bytes) + } + } + + private fun persistBloomFilter(specification: HttpsBloomFilterSpec, bytes: ByteArray, falsePositives: List) { + appDatabase.runInTransaction { + persistBloomFilter(specification, bytes) + persistFalsePositives(falsePositives) + } + } + + override fun persistFalsePositives(falsePositives: List) { + httpsFalsePositivesDao.updateAll(falsePositives) + } + + override fun persistEmbeddedData() { + Timber.d("Updating https data from embedded files") + val specJson = context.resources.openRawResource(R.raw.https_mobile_v2_bloom_spec).bufferedReader().use { it.readText() } + val specAdapter = moshi.adapter(HttpsBloomFilterSpec::class.java) + + val falsePositivesJson = context.resources.openRawResource(R.raw.https_mobile_v2_false_positives).bufferedReader().use { it.readText() } + val falsePositivesType = Types.newParameterizedType(List::class.java, HttpsFalsePositiveDomain::class.java) + val falsePositivesAdapter: JsonAdapter> = moshi.adapter(falsePositivesType) + + val bytes = context.resources.openRawResource(R.raw.https_mobile_v2_bloom).use { it.readBytes() } + persistBloomFilter(specAdapter.fromJson(specJson)!!, bytes, falsePositivesAdapter.fromJson(falsePositivesJson)!!) + } + + override fun isPersisted(specification: HttpsBloomFilterSpec): Boolean { + return specification == httpsBloomSpecDao.get() && binaryDataStore.verifyCheckSum(HttpsBloomFilterSpec.HTTPS_BINARY_FILE, specification.sha256) + } + + fun isPersisted(): Boolean { + val specification = httpsBloomSpecDao.get() ?: return false + return binaryDataStore.verifyCheckSum(HttpsBloomFilterSpec.HTTPS_BINARY_FILE, specification.sha256) + } +} diff --git a/app/src/main/res/raw/https_mobile_v2_bloom.bin b/app/src/play/res/raw/https_mobile_v2_bloom.bin similarity index 100% rename from app/src/main/res/raw/https_mobile_v2_bloom.bin rename to app/src/play/res/raw/https_mobile_v2_bloom.bin diff --git a/app/src/main/res/raw/https_mobile_v2_bloom_spec.json b/app/src/play/res/raw/https_mobile_v2_bloom_spec.json similarity index 100% rename from app/src/main/res/raw/https_mobile_v2_bloom_spec.json rename to app/src/play/res/raw/https_mobile_v2_bloom_spec.json diff --git a/app/src/main/res/raw/https_mobile_v2_false_positives.json b/app/src/play/res/raw/https_mobile_v2_false_positives.json similarity index 100% rename from app/src/main/res/raw/https_mobile_v2_false_positives.json rename to app/src/play/res/raw/https_mobile_v2_false_positives.json From 4baf44c39eab370f964d62eef939c0b1003b598d Mon Sep 17 00:00:00 2001 From: David Gonzalez Date: Wed, 3 Mar 2021 17:35:58 +0100 Subject: [PATCH 2/6] more code cleanup --- app/src/main/AndroidManifest.xml | 1 - .../com/duckduckgo/httpsupgrade/store/PlayHttpsDataPersister.kt | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml index 0a8de6669125..193cef9f5e8f 100644 --- a/app/src/main/AndroidManifest.xml +++ b/app/src/main/AndroidManifest.xml @@ -186,7 +186,6 @@ android:configChanges="keyboardHidden|orientation|screenSize" android:label="@string/appDescription" android:launchMode="singleTask" - android:supportsPictureInPicture="true" android:windowSoftInputMode="adjustResize|stateHidden"> diff --git a/app/src/play/java/com/duckduckgo/httpsupgrade/store/PlayHttpsDataPersister.kt b/app/src/play/java/com/duckduckgo/httpsupgrade/store/PlayHttpsDataPersister.kt index bb30a7ddd0ed..0ee4e2a12a2c 100644 --- a/app/src/play/java/com/duckduckgo/httpsupgrade/store/PlayHttpsDataPersister.kt +++ b/app/src/play/java/com/duckduckgo/httpsupgrade/store/PlayHttpsDataPersister.kt @@ -81,7 +81,7 @@ class PlayHttpsDataPersister @Inject constructor( return specification == httpsBloomSpecDao.get() && binaryDataStore.verifyCheckSum(HttpsBloomFilterSpec.HTTPS_BINARY_FILE, specification.sha256) } - fun isPersisted(): Boolean { + override fun isPersisted(): Boolean { val specification = httpsBloomSpecDao.get() ?: return false return binaryDataStore.verifyCheckSum(HttpsBloomFilterSpec.HTTPS_BINARY_FILE, specification.sha256) } From e462616fdda76c26de64a49ab74e5f846876cfd5 Mon Sep 17 00:00:00 2001 From: David Gonzalez Date: Thu, 4 Mar 2021 17:24:54 +0100 Subject: [PATCH 3/6] we don't need to duplicate so much code --- .../HttpsEmbeddedDataIntegrationTest.kt | 1 - .../duckduckgo/app/di/HttpsPersisterModule.kt | 17 +--- .../store/FDroidHttpsDataPersister.kt | 74 ---------------- .../store/FDroidHttpsEmbeddedDataPersister.kt | 32 +++++++ .../httpsupgrade/HttpsBloomFilterFactory.kt | 10 ++- .../httpsupgrade/di/HttpsUpgraderModule.kt | 6 +- .../httpsupgrade/store/HttpsDataPersister.kt | 52 ++++++++--- .../store/HttpsEmbeddedDataPersister.kt | 25 ++++++ .../duckduckgo/app/di/HttpsPersisterModule.kt | 14 ++- .../store/PlayHttpsDataPersister.kt | 88 ------------------- .../store/PlayHttpsEmbeddedDataPersister.kt | 42 +++++++++ 11 files changed, 158 insertions(+), 203 deletions(-) delete mode 100644 app/src/fdroid/java/com/duckduckgo/httpsupgrade/store/FDroidHttpsDataPersister.kt create mode 100644 app/src/fdroid/java/com/duckduckgo/httpsupgrade/store/FDroidHttpsEmbeddedDataPersister.kt create mode 100644 app/src/main/java/com/duckduckgo/app/httpsupgrade/store/HttpsEmbeddedDataPersister.kt delete mode 100644 app/src/play/java/com/duckduckgo/httpsupgrade/store/PlayHttpsDataPersister.kt create mode 100644 app/src/play/java/com/duckduckgo/httpsupgrade/store/PlayHttpsEmbeddedDataPersister.kt diff --git a/app/src/androidTest/java/com/duckduckgo/app/integration/HttpsEmbeddedDataIntegrationTest.kt b/app/src/androidTest/java/com/duckduckgo/app/integration/HttpsEmbeddedDataIntegrationTest.kt index 205d95140f1e..0bd0ef001cbc 100644 --- a/app/src/androidTest/java/com/duckduckgo/app/integration/HttpsEmbeddedDataIntegrationTest.kt +++ b/app/src/androidTest/java/com/duckduckgo/app/integration/HttpsEmbeddedDataIntegrationTest.kt @@ -25,7 +25,6 @@ import com.duckduckgo.app.httpsupgrade.HttpsBloomFilterFactoryImpl import com.duckduckgo.app.httpsupgrade.HttpsUpgrader import com.duckduckgo.app.httpsupgrade.HttpsUpgraderImpl import com.duckduckgo.app.httpsupgrade.api.HttpsFalsePositivesJsonAdapter -import com.duckduckgo.httpsupgrade.store.PlayHttpsDataPersister import com.duckduckgo.app.privacy.db.UserWhitelistDao import com.duckduckgo.app.statistics.pixels.Pixel import com.nhaarman.mockitokotlin2.mock diff --git a/app/src/fdroid/java/com/duckduckgo/app/di/HttpsPersisterModule.kt b/app/src/fdroid/java/com/duckduckgo/app/di/HttpsPersisterModule.kt index dc0bb9b6ed86..ae859e869c3a 100644 --- a/app/src/fdroid/java/com/duckduckgo/app/di/HttpsPersisterModule.kt +++ b/app/src/fdroid/java/com/duckduckgo/app/di/HttpsPersisterModule.kt @@ -16,12 +16,8 @@ package com.duckduckgo.app.di -import com.duckduckgo.app.global.db.AppDatabase -import com.duckduckgo.app.global.store.BinaryDataStore -import com.duckduckgo.app.httpsupgrade.store.HttpsBloomFilterSpecDao -import com.duckduckgo.app.httpsupgrade.store.HttpsDataPersister -import com.duckduckgo.app.httpsupgrade.store.HttpsFalsePositivesDao -import com.duckduckgo.httpsupgrade.store.FDroidHttpsDataPersister +import com.duckduckgo.app.httpsupgrade.store.HttpsEmbeddedDataPersister +import com.duckduckgo.httpsupgrade.store.FDroidHttpsEmbeddedDataPersister import dagger.Module import dagger.Provides @@ -29,13 +25,8 @@ import dagger.Provides class HttpsPersisterModule { @Provides - fun providesHttpsDataPersister( - binaryDataStore: BinaryDataStore, - httpsBloomSpecDao: HttpsBloomFilterSpecDao, - httpsFalsePositivesDao: HttpsFalsePositivesDao, - appDatabase: AppDatabase - ): HttpsDataPersister { - return FDroidHttpsDataPersister(binaryDataStore, httpsBloomSpecDao, httpsFalsePositivesDao, appDatabase) + fun providesHttpsDataManager(): HttpsEmbeddedDataPersister { + return FDroidHttpsEmbeddedDataPersister() } } diff --git a/app/src/fdroid/java/com/duckduckgo/httpsupgrade/store/FDroidHttpsDataPersister.kt b/app/src/fdroid/java/com/duckduckgo/httpsupgrade/store/FDroidHttpsDataPersister.kt deleted file mode 100644 index cd4ea77a80c4..000000000000 --- a/app/src/fdroid/java/com/duckduckgo/httpsupgrade/store/FDroidHttpsDataPersister.kt +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Copyright (c) 2021 DuckDuckGo - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.duckduckgo.httpsupgrade.store - -import com.duckduckgo.app.global.db.AppDatabase -import com.duckduckgo.app.global.store.BinaryDataStore -import com.duckduckgo.app.httpsupgrade.model.HttpsBloomFilterSpec -import com.duckduckgo.app.httpsupgrade.model.HttpsFalsePositiveDomain -import com.duckduckgo.app.httpsupgrade.store.HttpsBloomFilterSpecDao -import com.duckduckgo.app.httpsupgrade.store.HttpsDataPersister -import com.duckduckgo.app.httpsupgrade.store.HttpsFalsePositivesDao -import timber.log.Timber -import java.io.IOException -import javax.inject.Inject - -class FDroidHttpsDataPersister @Inject constructor( - private val binaryDataStore: BinaryDataStore, - private val httpsBloomSpecDao: HttpsBloomFilterSpecDao, - private val httpsFalsePositivesDao: HttpsFalsePositivesDao, - private val appDatabase: AppDatabase -) : HttpsDataPersister { - - override fun persistBloomFilter(specification: HttpsBloomFilterSpec, bytes: ByteArray) { - if (!binaryDataStore.verifyCheckSum(bytes, specification.sha256)) { - throw IOException("Https binary has incorrect sha, throwing away file") - } - - Timber.d("Updating https bloom data store with new data") - appDatabase.runInTransaction { - httpsBloomSpecDao.insert(specification) - binaryDataStore.saveData(HttpsBloomFilterSpec.HTTPS_BINARY_FILE, bytes) - } - } - - private fun persistBloomFilter(specification: HttpsBloomFilterSpec, bytes: ByteArray, falsePositives: List) { - appDatabase.runInTransaction { - persistBloomFilter(specification, bytes) - persistFalsePositives(falsePositives) - } - } - - override fun persistFalsePositives(falsePositives: List) { - httpsFalsePositivesDao.updateAll(falsePositives) - } - - override fun persistEmbeddedData() { - Timber.d("F-Droid does not get https data from embedded files") - return - } - - override fun isPersisted(specification: HttpsBloomFilterSpec): Boolean { - return specification == httpsBloomSpecDao.get() && binaryDataStore.verifyCheckSum(HttpsBloomFilterSpec.HTTPS_BINARY_FILE, specification.sha256) - } - - override fun isPersisted(): Boolean { - val specification = httpsBloomSpecDao.get() ?: return false - return binaryDataStore.verifyCheckSum(HttpsBloomFilterSpec.HTTPS_BINARY_FILE, specification.sha256) - } - -} diff --git a/app/src/fdroid/java/com/duckduckgo/httpsupgrade/store/FDroidHttpsEmbeddedDataPersister.kt b/app/src/fdroid/java/com/duckduckgo/httpsupgrade/store/FDroidHttpsEmbeddedDataPersister.kt new file mode 100644 index 000000000000..5a3301358cce --- /dev/null +++ b/app/src/fdroid/java/com/duckduckgo/httpsupgrade/store/FDroidHttpsEmbeddedDataPersister.kt @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2021 DuckDuckGo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.duckduckgo.httpsupgrade.store + +import com.duckduckgo.app.httpsupgrade.store.HttpsEmbeddedDataPersister +import timber.log.Timber + +class FDroidHttpsEmbeddedDataPersister : HttpsEmbeddedDataPersister { + + override fun shouldPersistEmbeddedData(): Boolean { + Timber.d("Https update data not found, but F-Droid does not use the binary to load embedded data") + return false + } + + override fun persistEmbeddedData() { + Timber.d("Https update data not found, but F-Droid does not use the binary to load embedded data") + } +} diff --git a/app/src/main/java/com/duckduckgo/app/httpsupgrade/HttpsBloomFilterFactory.kt b/app/src/main/java/com/duckduckgo/app/httpsupgrade/HttpsBloomFilterFactory.kt index ae0a16f7c44f..3eccf7820ef1 100644 --- a/app/src/main/java/com/duckduckgo/app/httpsupgrade/HttpsBloomFilterFactory.kt +++ b/app/src/main/java/com/duckduckgo/app/httpsupgrade/HttpsBloomFilterFactory.kt @@ -20,6 +20,7 @@ import androidx.annotation.WorkerThread import com.duckduckgo.app.global.store.BinaryDataStore import com.duckduckgo.app.httpsupgrade.model.HttpsBloomFilterSpec.Companion.HTTPS_BINARY_FILE import com.duckduckgo.app.httpsupgrade.store.HttpsBloomFilterSpecDao +import com.duckduckgo.app.httpsupgrade.store.HttpsEmbeddedDataPersister import com.duckduckgo.app.httpsupgrade.store.HttpsDataPersister import timber.log.Timber import javax.inject.Inject @@ -31,20 +32,21 @@ interface HttpsBloomFilterFactory { class HttpsBloomFilterFactoryImpl @Inject constructor( private val dao: HttpsBloomFilterSpecDao, private val binaryDataStore: BinaryDataStore, - private val persister: HttpsDataPersister + private val httpsEmbeddedDataPersister: HttpsEmbeddedDataPersister, + private val httpsDataPersister: HttpsDataPersister ) : HttpsBloomFilterFactory { @WorkerThread override fun create(): BloomFilter? { - if (!persister.isPersisted()) { + if (httpsEmbeddedDataPersister.shouldPersistEmbeddedData()) { Timber.d("Https update data not found, loading embedded data") - persister.persistEmbeddedData() + httpsEmbeddedDataPersister.persistEmbeddedData() } val specification = dao.get() val dataPath = binaryDataStore.dataFilePath(HTTPS_BINARY_FILE) - if (dataPath == null || specification == null || !persister.isPersisted(specification)) { + if (dataPath == null || specification == null || !httpsDataPersister.isPersisted(specification)) { Timber.d("Embedded https update data failed to load") return null } diff --git a/app/src/main/java/com/duckduckgo/app/httpsupgrade/di/HttpsUpgraderModule.kt b/app/src/main/java/com/duckduckgo/app/httpsupgrade/di/HttpsUpgraderModule.kt index 4fba40010ecb..46df4880904f 100644 --- a/app/src/main/java/com/duckduckgo/app/httpsupgrade/di/HttpsUpgraderModule.kt +++ b/app/src/main/java/com/duckduckgo/app/httpsupgrade/di/HttpsUpgraderModule.kt @@ -23,6 +23,7 @@ import com.duckduckgo.app.httpsupgrade.HttpsBloomFilterFactory import com.duckduckgo.app.httpsupgrade.HttpsBloomFilterFactoryImpl import com.duckduckgo.app.httpsupgrade.store.HttpsBloomFilterSpecDao import com.duckduckgo.app.httpsupgrade.store.HttpsDataPersister +import com.duckduckgo.app.httpsupgrade.store.HttpsEmbeddedDataPersister import com.duckduckgo.app.httpsupgrade.store.HttpsFalsePositivesDao import com.duckduckgo.app.privacy.db.UserWhitelistDao import com.duckduckgo.app.statistics.pixels.Pixel @@ -48,8 +49,9 @@ class HttpsUpgraderModule { fun bloomFilterFactory( specificationDao: HttpsBloomFilterSpecDao, binaryDataStore: BinaryDataStore, - persister: HttpsDataPersister + embeddedDataPersister: HttpsEmbeddedDataPersister, + dataPersister: HttpsDataPersister ): HttpsBloomFilterFactory { - return HttpsBloomFilterFactoryImpl(specificationDao, binaryDataStore, persister) + return HttpsBloomFilterFactoryImpl(specificationDao, binaryDataStore, embeddedDataPersister, dataPersister) } } diff --git a/app/src/main/java/com/duckduckgo/app/httpsupgrade/store/HttpsDataPersister.kt b/app/src/main/java/com/duckduckgo/app/httpsupgrade/store/HttpsDataPersister.kt index 42ff43f6ba28..e86c0c9fa7b8 100644 --- a/app/src/main/java/com/duckduckgo/app/httpsupgrade/store/HttpsDataPersister.kt +++ b/app/src/main/java/com/duckduckgo/app/httpsupgrade/store/HttpsDataPersister.kt @@ -16,19 +16,45 @@ package com.duckduckgo.app.httpsupgrade.store +import com.duckduckgo.app.global.db.AppDatabase +import com.duckduckgo.app.global.store.BinaryDataStore import com.duckduckgo.app.httpsupgrade.model.HttpsBloomFilterSpec import com.duckduckgo.app.httpsupgrade.model.HttpsFalsePositiveDomain - -interface HttpsDataPersister { - - fun persistBloomFilter(specification: HttpsBloomFilterSpec, bytes: ByteArray) - - fun persistFalsePositives(falsePositives: List) - - fun persistEmbeddedData() - - fun isPersisted(specification: HttpsBloomFilterSpec): Boolean - - fun isPersisted(): Boolean - +import timber.log.Timber +import java.io.IOException +import javax.inject.Inject + +class HttpsDataPersister @Inject constructor( + private val binaryDataStore: BinaryDataStore, + private val httpsBloomSpecDao: HttpsBloomFilterSpecDao, + private val httpsFalsePositivesDao: HttpsFalsePositivesDao, + private val appDatabase: AppDatabase +) { + + fun persistBloomFilter(specification: HttpsBloomFilterSpec, bytes: ByteArray, falsePositives: List) { + appDatabase.runInTransaction { + persistBloomFilter(specification, bytes) + persistFalsePositives(falsePositives) + } + } + + fun persistBloomFilter(specification: HttpsBloomFilterSpec, bytes: ByteArray) { + if (!binaryDataStore.verifyCheckSum(bytes, specification.sha256)) { + throw IOException("Https binary has incorrect sha, throwing away file") + } + + Timber.d("Updating https bloom data store with new data") + appDatabase.runInTransaction { + httpsBloomSpecDao.insert(specification) + binaryDataStore.saveData(HttpsBloomFilterSpec.HTTPS_BINARY_FILE, bytes) + } + } + + fun persistFalsePositives(falsePositives: List) { + httpsFalsePositivesDao.updateAll(falsePositives) + } + + fun isPersisted(specification: HttpsBloomFilterSpec): Boolean { + return specification == httpsBloomSpecDao.get() && binaryDataStore.verifyCheckSum(HttpsBloomFilterSpec.HTTPS_BINARY_FILE, specification.sha256) + } } diff --git a/app/src/main/java/com/duckduckgo/app/httpsupgrade/store/HttpsEmbeddedDataPersister.kt b/app/src/main/java/com/duckduckgo/app/httpsupgrade/store/HttpsEmbeddedDataPersister.kt new file mode 100644 index 000000000000..a4c6a0bfa3d2 --- /dev/null +++ b/app/src/main/java/com/duckduckgo/app/httpsupgrade/store/HttpsEmbeddedDataPersister.kt @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2021 DuckDuckGo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.duckduckgo.app.httpsupgrade.store + +interface HttpsEmbeddedDataPersister { + + fun shouldPersistEmbeddedData(): Boolean + + fun persistEmbeddedData() + +} diff --git a/app/src/play/java/com/duckduckgo/app/di/HttpsPersisterModule.kt b/app/src/play/java/com/duckduckgo/app/di/HttpsPersisterModule.kt index 537a9bb63fac..1c3e73eb0b5c 100644 --- a/app/src/play/java/com/duckduckgo/app/di/HttpsPersisterModule.kt +++ b/app/src/play/java/com/duckduckgo/app/di/HttpsPersisterModule.kt @@ -17,12 +17,11 @@ package com.duckduckgo.app.di import android.content.Context -import com.duckduckgo.app.global.db.AppDatabase import com.duckduckgo.app.global.store.BinaryDataStore import com.duckduckgo.app.httpsupgrade.store.HttpsBloomFilterSpecDao import com.duckduckgo.app.httpsupgrade.store.HttpsDataPersister -import com.duckduckgo.app.httpsupgrade.store.HttpsFalsePositivesDao -import com.duckduckgo.httpsupgrade.store.PlayHttpsDataPersister +import com.duckduckgo.app.httpsupgrade.store.HttpsEmbeddedDataPersister +import com.duckduckgo.httpsupgrade.store.PlayHttpsEmbeddedDataPersister import com.squareup.moshi.Moshi import dagger.Module import dagger.Provides @@ -31,15 +30,14 @@ import dagger.Provides class HttpsPersisterModule { @Provides - fun providesHttpsDataPersister( + fun providesPlayHttpsEmbeddedDataPersister( + httpsDataPersister: HttpsDataPersister, binaryDataStore: BinaryDataStore, httpsBloomSpecDao: HttpsBloomFilterSpecDao, - httpsFalsePositivesDao: HttpsFalsePositivesDao, - appDatabase: AppDatabase, context: Context, moshi: Moshi - ): HttpsDataPersister { - return PlayHttpsDataPersister(binaryDataStore, httpsBloomSpecDao, httpsFalsePositivesDao, appDatabase, context, moshi) + ): HttpsEmbeddedDataPersister { + return PlayHttpsEmbeddedDataPersister(httpsDataPersister, binaryDataStore, httpsBloomSpecDao, context, moshi) } } diff --git a/app/src/play/java/com/duckduckgo/httpsupgrade/store/PlayHttpsDataPersister.kt b/app/src/play/java/com/duckduckgo/httpsupgrade/store/PlayHttpsDataPersister.kt deleted file mode 100644 index 0ee4e2a12a2c..000000000000 --- a/app/src/play/java/com/duckduckgo/httpsupgrade/store/PlayHttpsDataPersister.kt +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Copyright (c) 2021 DuckDuckGo - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.duckduckgo.httpsupgrade.store - -import android.content.Context -import com.duckduckgo.app.browser.R -import com.duckduckgo.app.global.db.AppDatabase -import com.duckduckgo.app.global.store.BinaryDataStore -import com.duckduckgo.app.httpsupgrade.model.HttpsBloomFilterSpec -import com.duckduckgo.app.httpsupgrade.model.HttpsFalsePositiveDomain -import com.duckduckgo.app.httpsupgrade.store.HttpsBloomFilterSpecDao -import com.duckduckgo.app.httpsupgrade.store.HttpsDataPersister -import com.duckduckgo.app.httpsupgrade.store.HttpsFalsePositivesDao -import com.squareup.moshi.JsonAdapter -import com.squareup.moshi.Moshi -import com.squareup.moshi.Types -import timber.log.Timber -import java.io.IOException -import javax.inject.Inject - -class PlayHttpsDataPersister @Inject constructor( - private val binaryDataStore: BinaryDataStore, - private val httpsBloomSpecDao: HttpsBloomFilterSpecDao, - private val httpsFalsePositivesDao: HttpsFalsePositivesDao, - private val appDatabase: AppDatabase, - private val context: Context, - private val moshi: Moshi -) : HttpsDataPersister { - - override fun persistBloomFilter(specification: HttpsBloomFilterSpec, bytes: ByteArray) { - if (!binaryDataStore.verifyCheckSum(bytes, specification.sha256)) { - throw IOException("Https binary has incorrect sha, throwing away file") - } - - Timber.d("Updating https bloom data store with new data") - appDatabase.runInTransaction { - httpsBloomSpecDao.insert(specification) - binaryDataStore.saveData(HttpsBloomFilterSpec.HTTPS_BINARY_FILE, bytes) - } - } - - private fun persistBloomFilter(specification: HttpsBloomFilterSpec, bytes: ByteArray, falsePositives: List) { - appDatabase.runInTransaction { - persistBloomFilter(specification, bytes) - persistFalsePositives(falsePositives) - } - } - - override fun persistFalsePositives(falsePositives: List) { - httpsFalsePositivesDao.updateAll(falsePositives) - } - - override fun persistEmbeddedData() { - Timber.d("Updating https data from embedded files") - val specJson = context.resources.openRawResource(R.raw.https_mobile_v2_bloom_spec).bufferedReader().use { it.readText() } - val specAdapter = moshi.adapter(HttpsBloomFilterSpec::class.java) - - val falsePositivesJson = context.resources.openRawResource(R.raw.https_mobile_v2_false_positives).bufferedReader().use { it.readText() } - val falsePositivesType = Types.newParameterizedType(List::class.java, HttpsFalsePositiveDomain::class.java) - val falsePositivesAdapter: JsonAdapter> = moshi.adapter(falsePositivesType) - - val bytes = context.resources.openRawResource(R.raw.https_mobile_v2_bloom).use { it.readBytes() } - persistBloomFilter(specAdapter.fromJson(specJson)!!, bytes, falsePositivesAdapter.fromJson(falsePositivesJson)!!) - } - - override fun isPersisted(specification: HttpsBloomFilterSpec): Boolean { - return specification == httpsBloomSpecDao.get() && binaryDataStore.verifyCheckSum(HttpsBloomFilterSpec.HTTPS_BINARY_FILE, specification.sha256) - } - - override fun isPersisted(): Boolean { - val specification = httpsBloomSpecDao.get() ?: return false - return binaryDataStore.verifyCheckSum(HttpsBloomFilterSpec.HTTPS_BINARY_FILE, specification.sha256) - } -} diff --git a/app/src/play/java/com/duckduckgo/httpsupgrade/store/PlayHttpsEmbeddedDataPersister.kt b/app/src/play/java/com/duckduckgo/httpsupgrade/store/PlayHttpsEmbeddedDataPersister.kt new file mode 100644 index 000000000000..2b6d70eeedfa --- /dev/null +++ b/app/src/play/java/com/duckduckgo/httpsupgrade/store/PlayHttpsEmbeddedDataPersister.kt @@ -0,0 +1,42 @@ +package com.duckduckgo.httpsupgrade.store + +import android.content.Context +import com.duckduckgo.app.browser.R +import com.duckduckgo.app.global.store.BinaryDataStore +import com.duckduckgo.app.httpsupgrade.model.HttpsBloomFilterSpec +import com.duckduckgo.app.httpsupgrade.model.HttpsFalsePositiveDomain +import com.duckduckgo.app.httpsupgrade.store.HttpsBloomFilterSpecDao +import com.duckduckgo.app.httpsupgrade.store.HttpsDataPersister +import com.duckduckgo.app.httpsupgrade.store.HttpsEmbeddedDataPersister +import com.squareup.moshi.JsonAdapter +import com.squareup.moshi.Moshi +import com.squareup.moshi.Types +import timber.log.Timber + +class PlayHttpsEmbeddedDataPersister( + private val httpsDataPersister: HttpsDataPersister, + private val binaryDataStore: BinaryDataStore, + private val httpsBloomSpecDao: HttpsBloomFilterSpecDao, + private val context: Context, + private val moshi: Moshi +) : HttpsEmbeddedDataPersister { + + override fun shouldPersistEmbeddedData(): Boolean { + val specification = httpsBloomSpecDao.get() ?: return true + return !binaryDataStore.verifyCheckSum(HttpsBloomFilterSpec.HTTPS_BINARY_FILE, specification.sha256) + } + + override fun persistEmbeddedData() { + Timber.d("Updating https data from embedded files") + val specJson = context.resources.openRawResource(R.raw.https_mobile_v2_bloom_spec).bufferedReader().use { it.readText() } + val specAdapter = moshi.adapter(HttpsBloomFilterSpec::class.java) + + val falsePositivesJson = context.resources.openRawResource(R.raw.https_mobile_v2_false_positives).bufferedReader().use { it.readText() } + val falsePositivesType = Types.newParameterizedType(List::class.java, HttpsFalsePositiveDomain::class.java) + val falsePositivesAdapter: JsonAdapter> = moshi.adapter(falsePositivesType) + + val bytes = context.resources.openRawResource(R.raw.https_mobile_v2_bloom).use { it.readBytes() } + httpsDataPersister.persistBloomFilter(specAdapter.fromJson(specJson)!!, bytes, falsePositivesAdapter.fromJson(falsePositivesJson)!!) + } + +} From 3a24f0ec1f0a9ec4fae9ffbe14d8915f91b5a7e4 Mon Sep 17 00:00:00 2001 From: David Gonzalez Date: Thu, 4 Mar 2021 17:42:44 +0100 Subject: [PATCH 4/6] fixed bloom filter test --- .../HttpsEmbeddedDataIntegrationTest.kt | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/app/src/androidTest/java/com/duckduckgo/app/integration/HttpsEmbeddedDataIntegrationTest.kt b/app/src/androidTest/java/com/duckduckgo/app/integration/HttpsEmbeddedDataIntegrationTest.kt index 0bd0ef001cbc..784a67d61f18 100644 --- a/app/src/androidTest/java/com/duckduckgo/app/integration/HttpsEmbeddedDataIntegrationTest.kt +++ b/app/src/androidTest/java/com/duckduckgo/app/integration/HttpsEmbeddedDataIntegrationTest.kt @@ -25,8 +25,11 @@ import com.duckduckgo.app.httpsupgrade.HttpsBloomFilterFactoryImpl import com.duckduckgo.app.httpsupgrade.HttpsUpgrader import com.duckduckgo.app.httpsupgrade.HttpsUpgraderImpl import com.duckduckgo.app.httpsupgrade.api.HttpsFalsePositivesJsonAdapter +import com.duckduckgo.app.httpsupgrade.store.HttpsDataPersister +import com.duckduckgo.app.httpsupgrade.store.HttpsEmbeddedDataPersister import com.duckduckgo.app.privacy.db.UserWhitelistDao import com.duckduckgo.app.statistics.pixels.Pixel +import com.duckduckgo.httpsupgrade.store.PlayHttpsEmbeddedDataPersister import com.nhaarman.mockitokotlin2.mock import com.squareup.moshi.Moshi import org.junit.After @@ -50,20 +53,21 @@ class HttpsEmbeddedDataIntegrationTest { db = Room.inMemoryDatabaseBuilder(context, AppDatabase::class.java) .allowMainThreadQueries() .build() + var httpsBloomSpecDao = db.httpsBloomFilterSpecDao() var httpsFalsePositivesDao = db.httpsFalsePositivesDao() - var binaryDataStore: BinaryDataStore = BinaryDataStore(context) + var binaryDataStore = BinaryDataStore(context) - val persister = PlayHttpsDataPersister( + val persister = HttpsDataPersister( binaryDataStore, httpsBloomSpecDao, httpsFalsePositivesDao, - db, - context, - moshi + db ) - val factory = HttpsBloomFilterFactoryImpl(httpsBloomSpecDao, binaryDataStore, persister) + var embeddedDataPersister = PlayHttpsEmbeddedDataPersister(persister, binaryDataStore, httpsBloomSpecDao, context, moshi) + + val factory = HttpsBloomFilterFactoryImpl(httpsBloomSpecDao, binaryDataStore, embeddedDataPersister, persister) httpsUpgrader = HttpsUpgraderImpl(factory, httpsFalsePositivesDao, mockUserAllowlistDao, mockPixel) httpsUpgrader.reloadData() } From 0aee3348a4e57e396d589c88b0ab0353cd54c6cf Mon Sep 17 00:00:00 2001 From: David Gonzalez Date: Fri, 5 Mar 2021 07:39:46 +0100 Subject: [PATCH 5/6] spotless check --- .../app/integration/HttpsEmbeddedDataIntegrationTest.kt | 1 - 1 file changed, 1 deletion(-) diff --git a/app/src/androidTest/java/com/duckduckgo/app/integration/HttpsEmbeddedDataIntegrationTest.kt b/app/src/androidTest/java/com/duckduckgo/app/integration/HttpsEmbeddedDataIntegrationTest.kt index 784a67d61f18..396cfa96b7d4 100644 --- a/app/src/androidTest/java/com/duckduckgo/app/integration/HttpsEmbeddedDataIntegrationTest.kt +++ b/app/src/androidTest/java/com/duckduckgo/app/integration/HttpsEmbeddedDataIntegrationTest.kt @@ -26,7 +26,6 @@ import com.duckduckgo.app.httpsupgrade.HttpsUpgrader import com.duckduckgo.app.httpsupgrade.HttpsUpgraderImpl import com.duckduckgo.app.httpsupgrade.api.HttpsFalsePositivesJsonAdapter import com.duckduckgo.app.httpsupgrade.store.HttpsDataPersister -import com.duckduckgo.app.httpsupgrade.store.HttpsEmbeddedDataPersister import com.duckduckgo.app.privacy.db.UserWhitelistDao import com.duckduckgo.app.statistics.pixels.Pixel import com.duckduckgo.httpsupgrade.store.PlayHttpsEmbeddedDataPersister From d4be13cbd06ea5c4e33bbd39f52b7df2fbefb9eb Mon Sep 17 00:00:00 2001 From: David Gonzalez Date: Mon, 8 Mar 2021 10:58:42 +0100 Subject: [PATCH 6/6] improved logging statements --- .../httpsupgrade/store/FDroidHttpsEmbeddedDataPersister.kt | 4 ++-- .../duckduckgo/app/httpsupgrade/HttpsBloomFilterFactory.kt | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/src/fdroid/java/com/duckduckgo/httpsupgrade/store/FDroidHttpsEmbeddedDataPersister.kt b/app/src/fdroid/java/com/duckduckgo/httpsupgrade/store/FDroidHttpsEmbeddedDataPersister.kt index 5a3301358cce..4a71eb12d550 100644 --- a/app/src/fdroid/java/com/duckduckgo/httpsupgrade/store/FDroidHttpsEmbeddedDataPersister.kt +++ b/app/src/fdroid/java/com/duckduckgo/httpsupgrade/store/FDroidHttpsEmbeddedDataPersister.kt @@ -22,11 +22,11 @@ import timber.log.Timber class FDroidHttpsEmbeddedDataPersister : HttpsEmbeddedDataPersister { override fun shouldPersistEmbeddedData(): Boolean { - Timber.d("Https update data not found, but F-Droid does not use the binary to load embedded data") + Timber.d("Ignoring, FDroid does not use embedded data due to binary data restrictions") return false } override fun persistEmbeddedData() { - Timber.d("Https update data not found, but F-Droid does not use the binary to load embedded data") + Timber.d("Ignoring, FDroid does not use embedded data due to binary data restrictions") } } diff --git a/app/src/main/java/com/duckduckgo/app/httpsupgrade/HttpsBloomFilterFactory.kt b/app/src/main/java/com/duckduckgo/app/httpsupgrade/HttpsBloomFilterFactory.kt index 3eccf7820ef1..0bad229575c4 100644 --- a/app/src/main/java/com/duckduckgo/app/httpsupgrade/HttpsBloomFilterFactory.kt +++ b/app/src/main/java/com/duckduckgo/app/httpsupgrade/HttpsBloomFilterFactory.kt @@ -47,7 +47,7 @@ class HttpsBloomFilterFactoryImpl @Inject constructor( val specification = dao.get() val dataPath = binaryDataStore.dataFilePath(HTTPS_BINARY_FILE) if (dataPath == null || specification == null || !httpsDataPersister.isPersisted(specification)) { - Timber.d("Embedded https update data failed to load") + Timber.d("Https update data not available") return null }