diff --git a/app/schemas/com.duckduckgo.app.global.db.AppDatabase/32.json b/app/schemas/com.duckduckgo.app.global.db.AppDatabase/32.json new file mode 100644 index 000000000000..432d52fe44c4 --- /dev/null +++ b/app/schemas/com.duckduckgo.app.global.db.AppDatabase/32.json @@ -0,0 +1,866 @@ +{ + "formatVersion": 1, + "database": { + "version": 32, + "identityHash": "be6380785dcbf8f6793f852097f0d224", + "entities": [ + { + "tableName": "tds_tracker", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`domain` TEXT NOT NULL, `defaultAction` TEXT NOT NULL, `ownerName` TEXT NOT NULL, `categories` TEXT NOT NULL, `rules` TEXT NOT NULL, PRIMARY KEY(`domain`))", + "fields": [ + { + "fieldPath": "domain", + "columnName": "domain", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "defaultAction", + "columnName": "defaultAction", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "ownerName", + "columnName": "ownerName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "categories", + "columnName": "categories", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "rules", + "columnName": "rules", + "affinity": "TEXT", + "notNull": true + } + ], + "primaryKey": { + "columnNames": [ + "domain" + ], + "autoGenerate": false + }, + "indices": [], + "foreignKeys": [] + }, + { + "tableName": "tds_entity", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`name` TEXT NOT NULL, `displayName` TEXT NOT NULL, `prevalence` REAL NOT NULL, PRIMARY KEY(`name`))", + "fields": [ + { + "fieldPath": "name", + "columnName": "name", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "displayName", + "columnName": "displayName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "prevalence", + "columnName": "prevalence", + "affinity": "REAL", + "notNull": true + } + ], + "primaryKey": { + "columnNames": [ + "name" + ], + "autoGenerate": false + }, + "indices": [], + "foreignKeys": [] + }, + { + "tableName": "tds_domain_entity", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`domain` TEXT NOT NULL, `entityName` TEXT NOT NULL, PRIMARY KEY(`domain`))", + "fields": [ + { + "fieldPath": "domain", + "columnName": "domain", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "entityName", + "columnName": "entityName", + "affinity": "TEXT", + "notNull": true + } + ], + "primaryKey": { + "columnNames": [ + "domain" + ], + "autoGenerate": false + }, + "indices": [], + "foreignKeys": [] + }, + { + "tableName": "temporary_tracking_whitelist", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`domain` TEXT NOT NULL, PRIMARY KEY(`domain`))", + "fields": [ + { + "fieldPath": "domain", + "columnName": "domain", + "affinity": "TEXT", + "notNull": true + } + ], + "primaryKey": { + "columnNames": [ + "domain" + ], + "autoGenerate": false + }, + "indices": [], + "foreignKeys": [] + }, + { + "tableName": "user_whitelist", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`domain` TEXT NOT NULL, PRIMARY KEY(`domain`))", + "fields": [ + { + "fieldPath": "domain", + "columnName": "domain", + "affinity": "TEXT", + "notNull": true + } + ], + "primaryKey": { + "columnNames": [ + "domain" + ], + "autoGenerate": false + }, + "indices": [], + "foreignKeys": [] + }, + { + "tableName": "https_bloom_filter_spec", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER NOT NULL, `bitCount` INTEGER NOT NULL, `errorRate` REAL NOT NULL, `totalEntries` INTEGER NOT NULL, `sha256` TEXT NOT NULL, PRIMARY KEY(`id`))", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "bitCount", + "columnName": "bitCount", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "errorRate", + "columnName": "errorRate", + "affinity": "REAL", + "notNull": true + }, + { + "fieldPath": "totalEntries", + "columnName": "totalEntries", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "sha256", + "columnName": "sha256", + "affinity": "TEXT", + "notNull": true + } + ], + "primaryKey": { + "columnNames": [ + "id" + ], + "autoGenerate": false + }, + "indices": [], + "foreignKeys": [] + }, + { + "tableName": "https_false_positive_domain", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`domain` TEXT NOT NULL, PRIMARY KEY(`domain`))", + "fields": [ + { + "fieldPath": "domain", + "columnName": "domain", + "affinity": "TEXT", + "notNull": true + } + ], + "primaryKey": { + "columnNames": [ + "domain" + ], + "autoGenerate": false + }, + "indices": [], + "foreignKeys": [] + }, + { + "tableName": "network_leaderboard", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`networkName` TEXT NOT NULL, `count` INTEGER NOT NULL, PRIMARY KEY(`networkName`))", + "fields": [ + { + "fieldPath": "networkName", + "columnName": "networkName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "count", + "columnName": "count", + "affinity": "INTEGER", + "notNull": true + } + ], + "primaryKey": { + "columnNames": [ + "networkName" + ], + "autoGenerate": false + }, + "indices": [], + "foreignKeys": [] + }, + { + "tableName": "sites_visited", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`key` TEXT NOT NULL, `count` INTEGER NOT NULL, PRIMARY KEY(`key`))", + "fields": [ + { + "fieldPath": "key", + "columnName": "key", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "count", + "columnName": "count", + "affinity": "INTEGER", + "notNull": true + } + ], + "primaryKey": { + "columnNames": [ + "key" + ], + "autoGenerate": false + }, + "indices": [], + "foreignKeys": [] + }, + { + "tableName": "tabs", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`tabId` TEXT NOT NULL, `url` TEXT, `title` TEXT, `skipHome` INTEGER NOT NULL, `viewed` INTEGER NOT NULL, `position` INTEGER NOT NULL, `tabPreviewFile` TEXT, `sourceTabId` TEXT, `deletable` INTEGER NOT NULL, PRIMARY KEY(`tabId`), FOREIGN KEY(`sourceTabId`) REFERENCES `tabs`(`tabId`) ON UPDATE SET NULL ON DELETE SET NULL )", + "fields": [ + { + "fieldPath": "tabId", + "columnName": "tabId", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "url", + "columnName": "url", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "title", + "columnName": "title", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "skipHome", + "columnName": "skipHome", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "viewed", + "columnName": "viewed", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "position", + "columnName": "position", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "tabPreviewFile", + "columnName": "tabPreviewFile", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "sourceTabId", + "columnName": "sourceTabId", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "deletable", + "columnName": "deletable", + "affinity": "INTEGER", + "notNull": true + } + ], + "primaryKey": { + "columnNames": [ + "tabId" + ], + "autoGenerate": false + }, + "indices": [ + { + "name": "index_tabs_tabId", + "unique": false, + "columnNames": [ + "tabId" + ], + "createSql": "CREATE INDEX IF NOT EXISTS `index_tabs_tabId` ON `${TABLE_NAME}` (`tabId`)" + } + ], + "foreignKeys": [ + { + "table": "tabs", + "onDelete": "SET NULL", + "onUpdate": "SET NULL", + "columns": [ + "sourceTabId" + ], + "referencedColumns": [ + "tabId" + ] + } + ] + }, + { + "tableName": "tab_selection", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER NOT NULL, `tabId` TEXT, PRIMARY KEY(`id`), FOREIGN KEY(`tabId`) REFERENCES `tabs`(`tabId`) ON UPDATE NO ACTION ON DELETE SET NULL )", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "tabId", + "columnName": "tabId", + "affinity": "TEXT", + "notNull": false + } + ], + "primaryKey": { + "columnNames": [ + "id" + ], + "autoGenerate": false + }, + "indices": [ + { + "name": "index_tab_selection_tabId", + "unique": false, + "columnNames": [ + "tabId" + ], + "createSql": "CREATE INDEX IF NOT EXISTS `index_tab_selection_tabId` ON `${TABLE_NAME}` (`tabId`)" + } + ], + "foreignKeys": [ + { + "table": "tabs", + "onDelete": "SET NULL", + "onUpdate": "NO ACTION", + "columns": [ + "tabId" + ], + "referencedColumns": [ + "tabId" + ] + } + ] + }, + { + "tableName": "bookmarks", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `title` TEXT, `url` TEXT NOT NULL)", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "title", + "columnName": "title", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "url", + "columnName": "url", + "affinity": "TEXT", + "notNull": true + } + ], + "primaryKey": { + "columnNames": [ + "id" + ], + "autoGenerate": true + }, + "indices": [], + "foreignKeys": [] + }, + { + "tableName": "survey", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`surveyId` TEXT NOT NULL, `url` TEXT, `daysInstalled` INTEGER, `status` TEXT NOT NULL, PRIMARY KEY(`surveyId`))", + "fields": [ + { + "fieldPath": "surveyId", + "columnName": "surveyId", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "url", + "columnName": "url", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "daysInstalled", + "columnName": "daysInstalled", + "affinity": "INTEGER", + "notNull": false + }, + { + "fieldPath": "status", + "columnName": "status", + "affinity": "TEXT", + "notNull": true + } + ], + "primaryKey": { + "columnNames": [ + "surveyId" + ], + "autoGenerate": false + }, + "indices": [], + "foreignKeys": [] + }, + { + "tableName": "dismissed_cta", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`ctaId` TEXT NOT NULL, PRIMARY KEY(`ctaId`))", + "fields": [ + { + "fieldPath": "ctaId", + "columnName": "ctaId", + "affinity": "TEXT", + "notNull": true + } + ], + "primaryKey": { + "columnNames": [ + "ctaId" + ], + "autoGenerate": false + }, + "indices": [], + "foreignKeys": [] + }, + { + "tableName": "search_count", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`key` TEXT NOT NULL, `count` INTEGER NOT NULL, PRIMARY KEY(`key`))", + "fields": [ + { + "fieldPath": "key", + "columnName": "key", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "count", + "columnName": "count", + "affinity": "INTEGER", + "notNull": true + } + ], + "primaryKey": { + "columnNames": [ + "key" + ], + "autoGenerate": false + }, + "indices": [], + "foreignKeys": [] + }, + { + "tableName": "app_days_used", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`date` TEXT NOT NULL, PRIMARY KEY(`date`))", + "fields": [ + { + "fieldPath": "date", + "columnName": "date", + "affinity": "TEXT", + "notNull": true + } + ], + "primaryKey": { + "columnNames": [ + "date" + ], + "autoGenerate": false + }, + "indices": [], + "foreignKeys": [] + }, + { + "tableName": "app_enjoyment", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`eventType` INTEGER NOT NULL, `promptCount` INTEGER NOT NULL, `timestamp` INTEGER NOT NULL, `primaryKey` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL)", + "fields": [ + { + "fieldPath": "eventType", + "columnName": "eventType", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "promptCount", + "columnName": "promptCount", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "timestamp", + "columnName": "timestamp", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "primaryKey", + "columnName": "primaryKey", + "affinity": "INTEGER", + "notNull": true + } + ], + "primaryKey": { + "columnNames": [ + "primaryKey" + ], + "autoGenerate": true + }, + "indices": [], + "foreignKeys": [] + }, + { + "tableName": "notification", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`notificationId` TEXT NOT NULL, PRIMARY KEY(`notificationId`))", + "fields": [ + { + "fieldPath": "notificationId", + "columnName": "notificationId", + "affinity": "TEXT", + "notNull": true + } + ], + "primaryKey": { + "columnNames": [ + "notificationId" + ], + "autoGenerate": false + }, + "indices": [], + "foreignKeys": [] + }, + { + "tableName": "privacy_protection_count", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`key` TEXT NOT NULL, `blocked_tracker_count` INTEGER NOT NULL, `upgrade_count` INTEGER NOT NULL, PRIMARY KEY(`key`))", + "fields": [ + { + "fieldPath": "key", + "columnName": "key", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "blockedTrackerCount", + "columnName": "blocked_tracker_count", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "upgradeCount", + "columnName": "upgrade_count", + "affinity": "INTEGER", + "notNull": true + } + ], + "primaryKey": { + "columnNames": [ + "key" + ], + "autoGenerate": false + }, + "indices": [], + "foreignKeys": [] + }, + { + "tableName": "UncaughtExceptionEntity", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `exceptionSource` TEXT NOT NULL, `message` TEXT NOT NULL, `version` TEXT NOT NULL, `timestamp` INTEGER NOT NULL)", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "exceptionSource", + "columnName": "exceptionSource", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "message", + "columnName": "message", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "version", + "columnName": "version", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "timestamp", + "columnName": "timestamp", + "affinity": "INTEGER", + "notNull": true + } + ], + "primaryKey": { + "columnNames": [ + "id" + ], + "autoGenerate": true + }, + "indices": [], + "foreignKeys": [] + }, + { + "tableName": "tdsMetadata", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER NOT NULL, `eTag` TEXT NOT NULL, PRIMARY KEY(`id`))", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "eTag", + "columnName": "eTag", + "affinity": "TEXT", + "notNull": true + } + ], + "primaryKey": { + "columnNames": [ + "id" + ], + "autoGenerate": false + }, + "indices": [], + "foreignKeys": [] + }, + { + "tableName": "userStage", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`key` INTEGER NOT NULL, `appStage` TEXT NOT NULL, PRIMARY KEY(`key`))", + "fields": [ + { + "fieldPath": "key", + "columnName": "key", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "appStage", + "columnName": "appStage", + "affinity": "TEXT", + "notNull": true + } + ], + "primaryKey": { + "columnNames": [ + "key" + ], + "autoGenerate": false + }, + "indices": [], + "foreignKeys": [] + }, + { + "tableName": "fireproofWebsites", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`domain` TEXT NOT NULL, PRIMARY KEY(`domain`))", + "fields": [ + { + "fieldPath": "domain", + "columnName": "domain", + "affinity": "TEXT", + "notNull": true + } + ], + "primaryKey": { + "columnNames": [ + "domain" + ], + "autoGenerate": false + }, + "indices": [], + "foreignKeys": [] + }, + { + "tableName": "user_events", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` TEXT NOT NULL, `timestamp` INTEGER NOT NULL, PRIMARY KEY(`id`))", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "timestamp", + "columnName": "timestamp", + "affinity": "INTEGER", + "notNull": true + } + ], + "primaryKey": { + "columnNames": [ + "id" + ], + "autoGenerate": false + }, + "indices": [], + "foreignKeys": [] + }, + { + "tableName": "locationPermissions", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`domain` TEXT NOT NULL, `permission` INTEGER NOT NULL, PRIMARY KEY(`domain`))", + "fields": [ + { + "fieldPath": "domain", + "columnName": "domain", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "permission", + "columnName": "permission", + "affinity": "INTEGER", + "notNull": true + } + ], + "primaryKey": { + "columnNames": [ + "domain" + ], + "autoGenerate": false + }, + "indices": [], + "foreignKeys": [] + }, + { + "tableName": "pixel_store", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `pixelName` TEXT NOT NULL, `atb` TEXT NOT NULL, `additionalQueryParams` TEXT NOT NULL, `encodedQueryParams` TEXT NOT NULL)", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "pixelName", + "columnName": "pixelName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "atb", + "columnName": "atb", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "additionalQueryParams", + "columnName": "additionalQueryParams", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "encodedQueryParams", + "columnName": "encodedQueryParams", + "affinity": "TEXT", + "notNull": true + } + ], + "primaryKey": { + "columnNames": [ + "id" + ], + "autoGenerate": true + }, + "indices": [], + "foreignKeys": [] + }, + { + "tableName": "auth_cookies_allowed_domains", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`domain` TEXT NOT NULL, PRIMARY KEY(`domain`))", + "fields": [ + { + "fieldPath": "domain", + "columnName": "domain", + "affinity": "TEXT", + "notNull": true + } + ], + "primaryKey": { + "columnNames": [ + "domain" + ], + "autoGenerate": false + }, + "indices": [], + "foreignKeys": [] + } + ], + "views": [], + "setupQueries": [ + "CREATE TABLE IF NOT EXISTS room_master_table (id INTEGER PRIMARY KEY,identity_hash TEXT)", + "INSERT OR REPLACE INTO room_master_table (id,identity_hash) VALUES(42, 'be6380785dcbf8f6793f852097f0d224')" + ] + } +} \ No newline at end of file diff --git a/app/src/androidTest/java/com/duckduckgo/app/brokensite/BrokenSiteDataTest.kt b/app/src/androidTest/java/com/duckduckgo/app/brokensite/BrokenSiteDataTest.kt index 1ce4528e7de4..5250ceed3799 100644 --- a/app/src/androidTest/java/com/duckduckgo/app/brokensite/BrokenSiteDataTest.kt +++ b/app/src/androidTest/java/com/duckduckgo/app/brokensite/BrokenSiteDataTest.kt @@ -65,8 +65,8 @@ class BrokenSiteDataTest { @Test fun whenSiteHasBlockedTrackersThenBlockedTrackersExist() { val site = buildSite(SITE_URL) - val event = TrackingEvent("http://www.example.com", "http://www.tracker.com/tracker.js", emptyList(), null, false) - val anotherEvent = TrackingEvent("http://www.example.com/test", "http://www.anothertracker.com/tracker.js", emptyList(), null, false) + val event = TrackingEvent("http://www.example.com", "http://www.tracker.com/tracker.js", emptyList(), null, false, null) + val anotherEvent = TrackingEvent("http://www.example.com/test", "http://www.anothertracker.com/tracker.js", emptyList(), null, false, null) site.trackerDetected(event) site.trackerDetected(anotherEvent) assertEquals("www.tracker.com,www.anothertracker.com", BrokenSiteData.fromSite(site).blockedTrackers) @@ -75,8 +75,8 @@ class BrokenSiteDataTest { @Test fun whenSiteHasSameHostBlockedTrackersThenOnlyUniqueTrackersIncludedInData() { val site = buildSite(SITE_URL) - val event = TrackingEvent("http://www.example.com", "http://www.tracker.com/tracker.js", emptyList(), null, false) - val anotherEvent = TrackingEvent("http://www.example.com/test", "http://www.tracker.com/tracker2.js", emptyList(), null, false) + val event = TrackingEvent("http://www.example.com", "http://www.tracker.com/tracker.js", emptyList(), null, false, null) + val anotherEvent = TrackingEvent("http://www.example.com/test", "http://www.tracker.com/tracker2.js", emptyList(), null, false, null) site.trackerDetected(event) site.trackerDetected(anotherEvent) assertEquals("www.tracker.com", BrokenSiteData.fromSite(site).blockedTrackers) @@ -91,8 +91,8 @@ class BrokenSiteDataTest { @Test fun whenSiteHasSurrogatesThenSurrogatesExist() { - val surrogate = SurrogateResponse(true, "surrogate.com/test.js", "", "") - val anotherSurrogate = SurrogateResponse(true, "anothersurrogate.com/test.js", "", "") + val surrogate = SurrogateResponse("test.js", true, "surrogate.com/test.js", "", "") + val anotherSurrogate = SurrogateResponse("test.js", true, "anothersurrogate.com/test.js", "", "") val site = buildSite(SITE_URL) site.surrogateDetected(surrogate) site.surrogateDetected(anotherSurrogate) @@ -101,8 +101,8 @@ class BrokenSiteDataTest { @Test fun whenSiteHasSameHostSurrogatesThenOnlyUniqueSurrogateIncludedInData() { - val surrogate = SurrogateResponse(true, "surrogate.com/test.js", "", "") - val anotherSurrogate = SurrogateResponse(true, "surrogate.com/test2.js", "", "") + val surrogate = SurrogateResponse("test.js", true, "surrogate.com/test.js", "", "") + val anotherSurrogate = SurrogateResponse("test.js", true, "surrogate.com/test2.js", "", "") val site = buildSite(SITE_URL) site.surrogateDetected(surrogate) site.surrogateDetected(anotherSurrogate) diff --git a/app/src/androidTest/java/com/duckduckgo/app/browser/BrowserTabViewModelTest.kt b/app/src/androidTest/java/com/duckduckgo/app/browser/BrowserTabViewModelTest.kt index 2c5f345717bf..5c200ddb31aa 100644 --- a/app/src/androidTest/java/com/duckduckgo/app/browser/BrowserTabViewModelTest.kt +++ b/app/src/androidTest/java/com/duckduckgo/app/browser/BrowserTabViewModelTest.kt @@ -526,7 +526,7 @@ class BrowserTabViewModelTest { @Test fun whenTrackerDetectedThenNetworkLeaderboardUpdated() { val networkEntity = TestEntity("Network1", "Network1", 10.0) - val event = TrackingEvent("http://www.example.com", "http://www.tracker.com/tracker.js", emptyList(), networkEntity, false) + val event = TrackingEvent("http://www.example.com", "http://www.tracker.com/tracker.js", emptyList(), networkEntity, false, null) testee.trackerDetected(event) verify(mockNetworkLeaderboardDao).incrementNetworkCount("Network1") } @@ -777,7 +777,7 @@ class BrowserTabViewModelTest { loadUrl("https://example.com") val entity = TestEntity("Network1", "Network1", 10.0) for (i in 1..10) { - testee.trackerDetected(TrackingEvent("https://example.com", "", null, entity, false)) + testee.trackerDetected(TrackingEvent("https://example.com", "", null, entity, false, null)) } assertNotEquals(grade, privacyGradeState().privacyGrade) } diff --git a/app/src/androidTest/java/com/duckduckgo/app/browser/WebViewRequestInterceptorTest.kt b/app/src/androidTest/java/com/duckduckgo/app/browser/WebViewRequestInterceptorTest.kt index 57c61a376279..68f6a6414336 100644 --- a/app/src/androidTest/java/com/duckduckgo/app/browser/WebViewRequestInterceptorTest.kt +++ b/app/src/androidTest/java/com/duckduckgo/app/browser/WebViewRequestInterceptorTest.kt @@ -285,8 +285,9 @@ class WebViewRequestInterceptorTest { } @Test - fun whenRequestShouldBlockButThereIsASurrogateThen() = runBlocking { + fun whenRequestShouldBlockButThereIsASurrogateThenResponseReturnedContainsTheSurrogateData() = runBlocking { val availableSurrogate = SurrogateResponse( + scriptId = "testId", responseAvailable = true, mimeType = "application/javascript", jsFunction = "javascript replacement function goes here" @@ -308,6 +309,7 @@ class WebViewRequestInterceptorTest { @Test fun whenRequestShouldBlockButThereIsASurrogateThenCallSurrogateDetected() = runBlocking { val availableSurrogate = SurrogateResponse( + scriptId = "testId", responseAvailable = true, mimeType = "application/javascript", jsFunction = "javascript replacement function goes here" @@ -499,7 +501,8 @@ class WebViewRequestInterceptorTest { documentUrl = "", trackerUrl = "", entity = null, - categories = null + categories = null, + surrogateId = "testId" ) whenever(mockRequest.isForMainFrame).thenReturn(false) whenever(mockTrackerDetector.evaluate(any(), any())).thenReturn(blockTrackingEvent) diff --git a/app/src/androidTest/java/com/duckduckgo/app/cta/ui/CtaTest.kt b/app/src/androidTest/java/com/duckduckgo/app/cta/ui/CtaTest.kt index 80a58089e8a3..ab8c8a927881 100644 --- a/app/src/androidTest/java/com/duckduckgo/app/cta/ui/CtaTest.kt +++ b/app/src/androidTest/java/com/duckduckgo/app/cta/ui/CtaTest.kt @@ -297,8 +297,8 @@ class CtaTest { @Test fun whenTrackersBlockedReturnThemSortingByPrevalence() { val trackers = listOf( - TrackingEvent("facebook.com", "facebook.com", blocked = true, entity = TestEntity("Facebook", "Facebook", 3.0), categories = null), - TrackingEvent("other.com", "other.com", blocked = true, entity = TestEntity("Other", "Other", 9.0), categories = null) + TrackingEvent("facebook.com", "facebook.com", blocked = true, entity = TestEntity("Facebook", "Facebook", 3.0), categories = null, surrogateId = null), + TrackingEvent("other.com", "other.com", blocked = true, entity = TestEntity("Other", "Other", 9.0), categories = null, surrogateId = null) ) val site = site(events = trackers) @@ -312,8 +312,8 @@ class CtaTest { @Test fun whenTrackersBlockedReturnOnlyTrackersWithDisplayName() { val trackers = listOf( - TrackingEvent("facebook.com", "facebook.com", blocked = true, entity = TestEntity("Facebook", "Facebook", 3.0), categories = null), - TrackingEvent("other.com", "other.com", blocked = true, entity = TestEntity("Other", "", 9.0), categories = null) + TrackingEvent("facebook.com", "facebook.com", blocked = true, entity = TestEntity("Facebook", "Facebook", 3.0), categories = null, surrogateId = null), + TrackingEvent("other.com", "other.com", blocked = true, entity = TestEntity("Other", "", 9.0), categories = null, surrogateId = null) ) val site = site(events = trackers) diff --git a/app/src/androidTest/java/com/duckduckgo/app/cta/ui/CtaViewModelTest.kt b/app/src/androidTest/java/com/duckduckgo/app/cta/ui/CtaViewModelTest.kt index 5f5b145920cd..9807c0de498c 100644 --- a/app/src/androidTest/java/com/duckduckgo/app/cta/ui/CtaViewModelTest.kt +++ b/app/src/androidTest/java/com/duckduckgo/app/cta/ui/CtaViewModelTest.kt @@ -415,7 +415,7 @@ class CtaViewModelTest { @Test fun whenRefreshCtaWhileBrowsingThenReturnTrackersBlockedCta() = coroutineRule.runBlocking { givenDaxOnboardingActive() - val trackingEvent = TrackingEvent("test.com", "test.com", null, TestEntity("test", "test", 9.0), true) + val trackingEvent = TrackingEvent("test.com", "test.com", null, TestEntity("test", "test", 9.0), true, null) val site = site(url = "http://www.cnn.com", trackerCount = 1, events = listOf(trackingEvent)) val value = testee.refreshCta(coroutineRule.testDispatcher, isBrowserShowing = true, site = site) @@ -425,7 +425,7 @@ class CtaViewModelTest { @Test fun whenRefreshCtaWhileBrowsingAndTrackersAreNotMajorThenReturnTrackersBlockedCta() = coroutineRule.runBlocking { givenDaxOnboardingActive() - val trackingEvent = TrackingEvent("test.com", "test.com", null, TestEntity("test", "test", 0.123), true) + val trackingEvent = TrackingEvent("test.com", "test.com", null, TestEntity("test", "test", 0.123), true, null) val site = site(url = "http://www.cnn.com", trackerCount = 1, events = listOf(trackingEvent)) val value = testee.refreshCta(coroutineRule.testDispatcher, isBrowserShowing = true, site = site) diff --git a/app/src/androidTest/java/com/duckduckgo/app/global/db/AppDatabaseTest.kt b/app/src/androidTest/java/com/duckduckgo/app/global/db/AppDatabaseTest.kt index 8447a8da6841..b4757728c5ad 100644 --- a/app/src/androidTest/java/com/duckduckgo/app/global/db/AppDatabaseTest.kt +++ b/app/src/androidTest/java/com/duckduckgo/app/global/db/AppDatabaseTest.kt @@ -350,6 +350,11 @@ class AppDatabaseTest { createDatabaseAndMigrate(30, 31, migrationsProvider.MIGRATION_30_TO_31) } + @Test + fun whenMigratingFromVersion31To32ThenValidationSucceeds() { + createDatabaseAndMigrate(31, 32, migrationsProvider.MIGRATION_31_TO_32) + } + private fun givenUserStageIs(database: SupportSQLiteDatabase, appStage: AppStage) { database.execSQL("INSERT INTO `userStage` values (1, '${appStage.name}') ") } diff --git a/app/src/androidTest/java/com/duckduckgo/app/global/model/SiteMonitorTest.kt b/app/src/androidTest/java/com/duckduckgo/app/global/model/SiteMonitorTest.kt index ebe0a6a33859..9034e3117cd9 100644 --- a/app/src/androidTest/java/com/duckduckgo/app/global/model/SiteMonitorTest.kt +++ b/app/src/androidTest/java/com/duckduckgo/app/global/model/SiteMonitorTest.kt @@ -91,30 +91,30 @@ class SiteMonitorTest { @Test fun whenTrackersAreDetectedThenTrackerCountIsIncremented() { val testee = SiteMonitor(document, null) - testee.trackerDetected(TrackingEvent(document, trackerA, null, null, true)) - testee.trackerDetected(TrackingEvent(document, trackerB, null, null, true)) + testee.trackerDetected(TrackingEvent(document, trackerA, null, null, true, null)) + testee.trackerDetected(TrackingEvent(document, trackerB, null, null, true, null)) assertEquals(2, testee.trackerCount) } @Test fun whenNonMajorNetworkTrackerIsDetectedThenMajorNetworkCoutnIsZero() { val testee = SiteMonitor(document, null) - testee.trackerDetected(TrackingEvent(document, trackerA, null, network, true)) + testee.trackerDetected(TrackingEvent(document, trackerA, null, network, true, null)) assertEquals(0, testee.majorNetworkCount) } @Test fun whenMajorNetworkTrackerIsDetectedThenMajorNetworkCountIsOne() { val testee = SiteMonitor(document, null) - testee.trackerDetected(TrackingEvent(document, majorNetworkTracker, null, majorNetwork, true)) + testee.trackerDetected(TrackingEvent(document, majorNetworkTracker, null, majorNetwork, true, null)) assertEquals(1, testee.majorNetworkCount) } @Test fun whenDuplicateMajorNetworkIsDetectedThenMajorNetworkCountIsStillOne() { val testee = SiteMonitor(document, null) - testee.trackerDetected(TrackingEvent(document, trackerA, null, majorNetwork, true)) - testee.trackerDetected(TrackingEvent(document, trackerB, null, majorNetwork, true)) + testee.trackerDetected(TrackingEvent(document, trackerA, null, majorNetwork, true, null)) + testee.trackerDetected(TrackingEvent(document, trackerB, null, majorNetwork, true, null)) assertEquals(1, testee.majorNetworkCount) } diff --git a/app/src/androidTest/java/com/duckduckgo/app/privacy/ui/TrackerNetworksAdapterTest.kt b/app/src/androidTest/java/com/duckduckgo/app/privacy/ui/TrackerNetworksAdapterTest.kt index 23ce109966b1..5b3843db5f78 100644 --- a/app/src/androidTest/java/com/duckduckgo/app/privacy/ui/TrackerNetworksAdapterTest.kt +++ b/app/src/androidTest/java/com/duckduckgo/app/privacy/ui/TrackerNetworksAdapterTest.kt @@ -50,7 +50,7 @@ class TrackerNetworksAdapterTest { } private fun data(): SortedMap> { - val trackingEvent = TrackingEvent("", "", null, null, true) + val trackingEvent = TrackingEvent("", "", null, null, true, null) val minorNetworkList = arrayListOf(trackingEvent, trackingEvent) val majorNetworkList = arrayListOf(trackingEvent, trackingEvent) return mapOf( diff --git a/app/src/androidTest/java/com/duckduckgo/app/privacy/ui/TrackerNetworksViewModelTest.kt b/app/src/androidTest/java/com/duckduckgo/app/privacy/ui/TrackerNetworksViewModelTest.kt index 1ecf4696518e..b16aa762e32d 100644 --- a/app/src/androidTest/java/com/duckduckgo/app/privacy/ui/TrackerNetworksViewModelTest.kt +++ b/app/src/androidTest/java/com/duckduckgo/app/privacy/ui/TrackerNetworksViewModelTest.kt @@ -65,7 +65,7 @@ class TrackerNetworksViewModelTest { @Test fun whenTrackersUpdatedWithNoTrackersThenViewModelListIsEmpty() { - val input = listOf(TrackingEvent(Url.DOCUMENT, Url.tracker(1), null, Entity.MINOR_ENTITY_A, true)) + val input = listOf(TrackingEvent(Url.DOCUMENT, Url.tracker(1), null, Entity.MINOR_ENTITY_A, true, null)) testee.onSiteChanged(site(trackingEvents = input)) assertTrue(testee.viewState.value!!.trackingEventsByNetwork.isNotEmpty()) @@ -78,21 +78,21 @@ class TrackerNetworksViewModelTest { val input = listOf( // Minor entity with 3 distinct trackers - TrackingEvent(Url.DOCUMENT, Url.tracker(1), null, Entity.MINOR_ENTITY_A, true), - TrackingEvent(Url.DOCUMENT, Url.tracker(1), null, Entity.MINOR_ENTITY_A, true), - TrackingEvent(Url.DOCUMENT, Url.tracker(2), null, Entity.MINOR_ENTITY_A, true), - TrackingEvent(Url.DOCUMENT, Url.tracker(3), null, Entity.MINOR_ENTITY_A, true), + TrackingEvent(Url.DOCUMENT, Url.tracker(1), null, Entity.MINOR_ENTITY_A, true, null), + TrackingEvent(Url.DOCUMENT, Url.tracker(1), null, Entity.MINOR_ENTITY_A, true, null), + TrackingEvent(Url.DOCUMENT, Url.tracker(2), null, Entity.MINOR_ENTITY_A, true, null), + TrackingEvent(Url.DOCUMENT, Url.tracker(3), null, Entity.MINOR_ENTITY_A, true, null), // Minor entity with 1 distinct tracker - TrackingEvent(Url.DOCUMENT, Url.tracker(4), null, Entity.MINOR_ENTITY_B, true), + TrackingEvent(Url.DOCUMENT, Url.tracker(4), null, Entity.MINOR_ENTITY_B, true, null), // Major entity with 2 distinct tracker - TrackingEvent(Url.DOCUMENT, Url.tracker(6), null, Entity.MAJOR_ENTITY_B, true), - TrackingEvent(Url.DOCUMENT, Url.tracker(6), null, Entity.MAJOR_ENTITY_B, true), - TrackingEvent(Url.DOCUMENT, Url.tracker(7), null, Entity.MAJOR_ENTITY_B, true), + TrackingEvent(Url.DOCUMENT, Url.tracker(6), null, Entity.MAJOR_ENTITY_B, true, null), + TrackingEvent(Url.DOCUMENT, Url.tracker(6), null, Entity.MAJOR_ENTITY_B, true, null), + TrackingEvent(Url.DOCUMENT, Url.tracker(7), null, Entity.MAJOR_ENTITY_B, true, null), // Major entity with 1 distinct tracker - TrackingEvent(Url.DOCUMENT, Url.tracker(5), null, Entity.MAJOR_ENTITY_A, true) + TrackingEvent(Url.DOCUMENT, Url.tracker(5), null, Entity.MAJOR_ENTITY_A, true, null) ) testee.onSiteChanged(site(trackingEvents = input)) diff --git a/app/src/androidTest/java/com/duckduckgo/app/surrogates/ResourceSurrogateLoaderTest.kt b/app/src/androidTest/java/com/duckduckgo/app/surrogates/ResourceSurrogateLoaderTest.kt index e7b3bbf3591a..c496ef2b71f0 100644 --- a/app/src/androidTest/java/com/duckduckgo/app/surrogates/ResourceSurrogateLoaderTest.kt +++ b/app/src/androidTest/java/com/duckduckgo/app/surrogates/ResourceSurrogateLoaderTest.kt @@ -105,6 +105,17 @@ class ResourceSurrogateLoaderTest { assertEquals(6, surrogates.size) } + @Test + fun whenLoadingSurrogatesThenCorrectScriptIdStored() { + val surrogates = initialiseFile("surrogates_6") + assertEquals("ga.js", surrogates[0].scriptId) + assertEquals("analytics.js", surrogates[1].scriptId) + assertEquals("inpage_linkid.js", surrogates[2].scriptId) + assertEquals("api.js", surrogates[3].scriptId) + assertEquals("gpt.js", surrogates[4].scriptId) + assertEquals("gtm.js", surrogates[5].scriptId) + } + private fun initialiseFile(filename: String): List { return testee.convertBytes(readFile(filename)) } diff --git a/app/src/androidTest/java/com/duckduckgo/app/surrogates/ResourceSurrogatesTest.kt b/app/src/androidTest/java/com/duckduckgo/app/surrogates/ResourceSurrogatesTest.kt index 7c385705adbb..d0dffbf83e1b 100644 --- a/app/src/androidTest/java/com/duckduckgo/app/surrogates/ResourceSurrogatesTest.kt +++ b/app/src/androidTest/java/com/duckduckgo/app/surrogates/ResourceSurrogatesTest.kt @@ -16,7 +16,6 @@ package com.duckduckgo.app.surrogates -import android.net.Uri import org.junit.Assert.* import org.junit.Before import org.junit.Test @@ -50,26 +49,18 @@ class ResourceSurrogatesTest { } @Test - fun whenSearchingForExactMatchingExistingSurrogateThenCanFindByName() { - val surrogate = SurrogateResponse(name = "foo") + fun whenSearchingForExactMatchingExistingSurrogateThenCanFindByScriptId() { + val surrogate = SurrogateResponse(scriptId = "fooId", name = "foo") testee.loadSurrogates(listOf(surrogate)) - val retrieved = testee.get(Uri.parse("foo")) + val retrieved = testee.get("fooId") assertTrue(retrieved.responseAvailable) } @Test - fun whenSearchingForSubstringMatchingExistingSurrogateThenCanFindByName() { - val surrogate = SurrogateResponse(name = "foo.com") + fun whenSearchingByNonExistentScriptIdThenResponseUnavailableSurrogateResultReturned() { + val surrogate = SurrogateResponse(scriptId = "fooId", name = "foo") testee.loadSurrogates(listOf(surrogate)) - val retrieved = testee.get(Uri.parse("foo.com/a/b/c")) - assertTrue(retrieved.responseAvailable) - } - - @Test - fun whenSearchingByNonExistentNameThenResponseUnavailableSurrogateResultReturned() { - val surrogate = SurrogateResponse(name = "foo") - testee.loadSurrogates(listOf(surrogate)) - val retrieved = testee.get(Uri.parse("bar")) + val retrieved = testee.get("bar") assertFalse(retrieved.responseAvailable) } } diff --git a/app/src/androidTest/java/com/duckduckgo/app/trackerdetection/TdsClientTest.kt b/app/src/androidTest/java/com/duckduckgo/app/trackerdetection/TdsClientTest.kt index d506666f3032..dc6f3ba2de65 100644 --- a/app/src/androidTest/java/com/duckduckgo/app/trackerdetection/TdsClientTest.kt +++ b/app/src/androidTest/java/com/duckduckgo/app/trackerdetection/TdsClientTest.kt @@ -22,6 +22,7 @@ import com.duckduckgo.app.trackerdetection.model.Action.IGNORE import com.duckduckgo.app.trackerdetection.model.Rule import com.duckduckgo.app.trackerdetection.model.RuleExceptions import com.duckduckgo.app.trackerdetection.model.TdsTracker +import org.junit.Assert.assertEquals import org.junit.Assert.assertFalse import org.junit.Assert.assertTrue import org.junit.Test @@ -78,7 +79,7 @@ class TdsClientTest { @Test fun whenUrlMatchesRuleWithNoExceptionsAndRuleActionBlockThenMatchesIsTrue() { - val rule = Rule("api\\.tracker\\.com\\/auth", BLOCK, null) + val rule = Rule("api\\.tracker\\.com\\/auth", BLOCK, null, null) val data = listOf(TdsTracker("tracker.com", BLOCK, OWNER, CATEGORY, listOf(rule))) val testee = TdsClient(TDS, data) val result = testee.matches("http://api.tracker.com/auth/script.js", DOCUMENT_URL) @@ -87,7 +88,7 @@ class TdsClientTest { @Test fun whenUrlMatchesRuleWithNoExceptionsAndRuleActionIgnoreThenMatchesIsFalse() { - val rule = Rule("api\\.tracker\\.com\\/auth", IGNORE, null) + val rule = Rule("api\\.tracker\\.com\\/auth", IGNORE, null, null) val data = listOf(TdsTracker("tracker.com", BLOCK, OWNER, CATEGORY, listOf(rule))) val testee = TdsClient(TDS, data) val result = testee.matches("http://api.tracker.com/auth/script.js", DOCUMENT_URL) @@ -96,7 +97,7 @@ class TdsClientTest { @Test fun whenUrlMatchesDomainWithDefaultBlockAndRuleWithNoExceptionsAndNoActionThenMatchesIsTrue() { - val rule = Rule("api\\.tracker\\.com\\/auth", null, null) + val rule = Rule("api\\.tracker\\.com\\/auth", null, null, null) val data = listOf(TdsTracker("tracker.com", BLOCK, OWNER, CATEGORY, listOf(rule))) val testee = TdsClient(TDS, data) val result = testee.matches("http://api.tracker.com/auth/script.js", DOCUMENT_URL) @@ -105,7 +106,7 @@ class TdsClientTest { @Test fun whenUrlMatchesDomainWithDefaultIgnoreAndRuleWithNoExceptionsAndNoActionThenMatchesIsTrue() { - val rule = Rule("api\\.tracker\\.com\\/auth", null, null) + val rule = Rule("api\\.tracker\\.com\\/auth", null, null, null) val data = listOf(TdsTracker("tracker.com", IGNORE, OWNER, CATEGORY, listOf(rule))) val testee = TdsClient(TDS, data) val result = testee.matches("http://api.tracker.com/auth/script.js", DOCUMENT_URL) @@ -116,9 +117,9 @@ class TdsClientTest { fun whenUrlMatchesRuleWithExceptionsAndExceptionDomainMatchesDocumentThenMatchesIsFalseIrrespectiveOfAction() { val exceptions = RuleExceptions(listOf("example.com"), null) - val ruleBlock = Rule("api\\.tracker\\.com\\/auth", BLOCK, exceptions) - val ruleIgnore = Rule("api\\.tracker\\.com\\/auth", IGNORE, exceptions) - val ruleNone = Rule("api\\.tracker\\.com\\/auth", null, exceptions) + val ruleBlock = Rule("api\\.tracker\\.com\\/auth", BLOCK, exceptions, null) + val ruleIgnore = Rule("api\\.tracker\\.com\\/auth", IGNORE, exceptions, null) + val ruleNone = Rule("api\\.tracker\\.com\\/auth", null, exceptions, null) val testeeBlockRuleBlock = TdsClient(TDS, listOf(TdsTracker("tracker.com", BLOCK, OWNER, CATEGORY, listOf(ruleBlock)))) val testeeIgnoreRuleBlock = TdsClient(TDS, listOf(TdsTracker("tracker.com", IGNORE, OWNER, CATEGORY, listOf(ruleBlock)))) @@ -139,9 +140,9 @@ class TdsClientTest { fun whenUrlMatchesRuleWithExceptionsAndExceptionDomainDoesNotMatchDocumentThenMatchesBehaviorIsStandard() { val exceptions = RuleExceptions(listOf("nonmatching.com"), null) - val ruleBlock = Rule("api\\.tracker\\.com\\/auth", BLOCK, exceptions) - val ruleIgnore = Rule("api\\.tracker\\.com\\/auth", IGNORE, exceptions) - val ruleNone = Rule("api\\.tracker\\.com\\/auth", null, exceptions) + val ruleBlock = Rule("api\\.tracker\\.com\\/auth", BLOCK, exceptions, null) + val ruleIgnore = Rule("api\\.tracker\\.com\\/auth", IGNORE, exceptions, null) + val ruleNone = Rule("api\\.tracker\\.com\\/auth", null, exceptions, null) val testeeBlockRuleBlock = TdsClient(TDS, listOf(TdsTracker("tracker.com", BLOCK, OWNER, CATEGORY, listOf(ruleBlock)))) val testeeIgnoreRuleBlock = TdsClient(TDS, listOf(TdsTracker("tracker.com", IGNORE, OWNER, CATEGORY, listOf(ruleBlock)))) @@ -162,9 +163,9 @@ class TdsClientTest { fun whenUrlMatchesRuleWithExceptionsWithNoDomainsAndTypesIsNotNullThenMatchesIsFalseIrrespectiveOfAction() { val exceptions = RuleExceptions(null, listOf("something")) - val ruleBlock = Rule("api\\.tracker\\.com\\/auth", BLOCK, exceptions) - val ruleIgnore = Rule("api\\.tracker\\.com\\/auth", IGNORE, exceptions) - val ruleNone = Rule("api\\.tracker\\.com\\/auth", null, exceptions) + val ruleBlock = Rule("api\\.tracker\\.com\\/auth", BLOCK, exceptions, null) + val ruleIgnore = Rule("api\\.tracker\\.com\\/auth", IGNORE, exceptions, null) + val ruleNone = Rule("api\\.tracker\\.com\\/auth", null, exceptions, null) val testeeBlockRuleBlock = TdsClient(TDS, listOf(TdsTracker("tracker.com", BLOCK, OWNER, CATEGORY, listOf(ruleBlock)))) val testeeIgnoreRuleBlock = TdsClient(TDS, listOf(TdsTracker("tracker.com", IGNORE, OWNER, CATEGORY, listOf(ruleBlock)))) @@ -181,6 +182,38 @@ class TdsClientTest { assertFalse(testeeIgnoreRuleNone.matches("http://api.tracker.com/auth/script.js", DOCUMENT_URL).matches) } + @Test + fun whenUrlMatchesRuleWithSurrogateThenMatchesIsTrueIrrespectiveOfAction() { + val exceptions = RuleExceptions(null, listOf("something")) + + val ruleBlock = Rule("api\\.tracker\\.com\\/auth", BLOCK, exceptions, "testId") + val ruleIgnore = Rule("api\\.tracker\\.com\\/auth", IGNORE, exceptions, "testId") + val ruleNone = Rule("api\\.tracker\\.com\\/auth", null, exceptions, "testId") + + val testeeBlockRuleBlock = TdsClient(TDS, listOf(TdsTracker("tracker.com", BLOCK, OWNER, CATEGORY, listOf(ruleBlock)))) + val testeeIgnoreRuleBlock = TdsClient(TDS, listOf(TdsTracker("tracker.com", IGNORE, OWNER, CATEGORY, listOf(ruleBlock)))) + val testeeBlockRuleIgnore = TdsClient(TDS, listOf(TdsTracker("tracker.com", BLOCK, OWNER, CATEGORY, listOf(ruleIgnore)))) + val testeeIgnoreRuleIgnore = TdsClient(TDS, listOf(TdsTracker("tracker.com", IGNORE, OWNER, CATEGORY, listOf(ruleIgnore)))) + val testeeBlockRuleNone = TdsClient(TDS, listOf(TdsTracker("tracker.com", BLOCK, OWNER, CATEGORY, listOf(ruleNone)))) + val testeeIgnoreRuleNone = TdsClient(TDS, listOf(TdsTracker("tracker.com", IGNORE, OWNER, CATEGORY, listOf(ruleNone)))) + + assertFalse(testeeBlockRuleBlock.matches("http://api.tracker.com/auth/script.js", DOCUMENT_URL).matches) + assertFalse(testeeIgnoreRuleBlock.matches("http://api.tracker.com/auth/script.js", DOCUMENT_URL).matches) + assertFalse(testeeBlockRuleIgnore.matches("http://api.tracker.com/auth/script.js", DOCUMENT_URL).matches) + assertFalse(testeeIgnoreRuleIgnore.matches("http://api.tracker.com/auth/script.js", DOCUMENT_URL).matches) + assertFalse(testeeBlockRuleNone.matches("http://api.tracker.com/auth/script.js", DOCUMENT_URL).matches) + assertFalse(testeeIgnoreRuleNone.matches("http://api.tracker.com/auth/script.js", DOCUMENT_URL).matches) + } + + @Test + fun whenUrlMatchesRuleWithSurrogateThenSurrogateScriptIdReturned() { + val rule = Rule("api\\.tracker\\.com\\/auth", BLOCK, null, "script.js") + + val testee = TdsClient(TDS, listOf(TdsTracker("tracker.com", BLOCK, OWNER, CATEGORY, listOf(rule)))) + + assertEquals("script.js", testee.matches("http://api.tracker.com/auth/script.js", DOCUMENT_URL).surrogate) + } + companion object { private const val OWNER = "A Network Owner" private const val DOCUMENT_URL = "http://example.com/index.htm" diff --git a/app/src/androidTest/java/com/duckduckgo/app/trackerdetection/TrackerDetectorClientTypeTest.kt b/app/src/androidTest/java/com/duckduckgo/app/trackerdetection/TrackerDetectorClientTypeTest.kt index 596d5724f5f2..79ef2c411c02 100644 --- a/app/src/androidTest/java/com/duckduckgo/app/trackerdetection/TrackerDetectorClientTypeTest.kt +++ b/app/src/androidTest/java/com/duckduckgo/app/trackerdetection/TrackerDetectorClientTypeTest.kt @@ -58,7 +58,7 @@ class TrackerDetectorClientTypeTest { @Test fun whenUrlMatchesOnlyInBlockingClientThenEvaluateReturnsTrackingEvent() { val url = Url.BLOCKED - val expected = TrackingEvent(documentUrl, url, null, null, true) + val expected = TrackingEvent(documentUrl, url, null, null, true, null) assertEquals(expected, testee.evaluate(url, documentUrl)) } diff --git a/app/src/androidTest/java/com/duckduckgo/app/trackerdetection/TrackerDetectorTest.kt b/app/src/androidTest/java/com/duckduckgo/app/trackerdetection/TrackerDetectorTest.kt index c64e1067105f..3aeacffa48bd 100644 --- a/app/src/androidTest/java/com/duckduckgo/app/trackerdetection/TrackerDetectorTest.kt +++ b/app/src/androidTest/java/com/duckduckgo/app/trackerdetection/TrackerDetectorTest.kt @@ -79,7 +79,7 @@ class TrackerDetectorTest { whenever(mockUserWhitelistDao.contains("example.com")).thenReturn(false) trackerDetector.addClient(alwaysMatchingClient(CLIENT_A)) trackerDetector.addClient(alwaysMatchingClient(CLIENT_B)) - val expected = TrackingEvent("http://example.com/index.com", "http://thirdparty.com/update.js", null, null, true) + val expected = TrackingEvent("http://example.com/index.com", "http://thirdparty.com/update.js", null, null, true, null) val actual = trackerDetector.evaluate("http://thirdparty.com/update.js", "http://example.com/index.com") assertEquals(expected, actual) } @@ -89,7 +89,7 @@ class TrackerDetectorTest { whenever(mockUserWhitelistDao.contains("example.com")).thenReturn(true) trackerDetector.addClient(alwaysMatchingClient(CLIENT_A)) trackerDetector.addClient(alwaysMatchingClient(CLIENT_B)) - val expected = TrackingEvent("http://example.com/index.com", "http://thirdparty.com/update.js", null, null, false) + val expected = TrackingEvent("http://example.com/index.com", "http://thirdparty.com/update.js", null, null, false, null) val actual = trackerDetector.evaluate("http://thirdparty.com/update.js", "http://example.com/index.com") assertEquals(expected, actual) } @@ -99,7 +99,7 @@ class TrackerDetectorTest { whenever(mockUserWhitelistDao.contains("example.com")).thenReturn(false) trackerDetector.addClient(neverMatchingClient(CLIENT_A)) trackerDetector.addClient(alwaysMatchingClient(CLIENT_B)) - val expected = TrackingEvent("http://example.com/index.com", "http://thirdparty.com/update.js", null, null, true) + val expected = TrackingEvent("http://example.com/index.com", "http://thirdparty.com/update.js", null, null, true, null) val actual = trackerDetector.evaluate("http://thirdparty.com/update.js", "http://example.com/index.com") assertEquals(expected, actual) } @@ -109,7 +109,16 @@ class TrackerDetectorTest { whenever(mockUserWhitelistDao.contains("example.com")).thenReturn(true) trackerDetector.addClient(neverMatchingClient(CLIENT_A)) trackerDetector.addClient(alwaysMatchingClient(CLIENT_B)) - val expected = TrackingEvent("http://example.com/index.com", "http://thirdparty.com/update.js", null, null, false) + val expected = TrackingEvent("http://example.com/index.com", "http://thirdparty.com/update.js", null, null, false, null) + val actual = trackerDetector.evaluate("http://thirdparty.com/update.js", "http://example.com/index.com") + assertEquals(expected, actual) + } + + @Test + fun whenSiteIsNotUserWhitelistedAndSomeClientsMatchWithSurrogateThenEvaluateReturnsBlockedTrackingEventWithSurrogate() { + whenever(mockUserWhitelistDao.contains("example.com")).thenReturn(false) + trackerDetector.addClient(alwaysMatchingClientWithSurrogate(CLIENT_A)) + val expected = TrackingEvent("http://example.com/index.com", "http://thirdparty.com/update.js", null, null, true, "testId") val actual = trackerDetector.evaluate("http://thirdparty.com/update.js", "http://example.com/index.com") assertEquals(expected, actual) } @@ -146,6 +155,13 @@ class TrackerDetectorTest { return client } + private fun alwaysMatchingClientWithSurrogate(name: ClientName): Client { + val client: Client = mock() + whenever(client.name).thenReturn(name) + whenever(client.matches(anyString(), anyString())).thenReturn(Client.Result(matches = true, surrogate = "testId")) + return client + } + companion object { // It doesn't matter what the value of these is they just need to be different private val CLIENT_A = EASYLIST diff --git a/app/src/main/java/com/duckduckgo/app/browser/WebViewRequestInterceptor.kt b/app/src/main/java/com/duckduckgo/app/browser/WebViewRequestInterceptor.kt index cd0f6a0d2225..ce6c13aac1f4 100644 --- a/app/src/main/java/com/duckduckgo/app/browser/WebViewRequestInterceptor.kt +++ b/app/src/main/java/com/duckduckgo/app/browser/WebViewRequestInterceptor.kt @@ -30,6 +30,7 @@ import com.duckduckgo.app.privacy.db.PrivacyProtectionCountDao import com.duckduckgo.app.privacy.model.TrustedSites import com.duckduckgo.app.surrogates.ResourceSurrogates import com.duckduckgo.app.trackerdetection.TrackerDetector +import com.duckduckgo.app.trackerdetection.model.TrackingEvent import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext import timber.log.Timber @@ -111,12 +112,15 @@ class WebViewRequestInterceptor( webViewClientListener?.pageHasHttpResources(documentUrl) } - if (shouldBlock(request, documentUrl, webViewClientListener)) { - val surrogate = resourceSurrogates.get(url) - if (surrogate.responseAvailable) { - Timber.d("Surrogate found for $url") - webViewClientListener?.surrogateDetected(surrogate) - return WebResourceResponse(surrogate.mimeType, "UTF-8", surrogate.jsFunction.byteInputStream()) + val trackingEvent = trackingEvent(request, documentUrl, webViewClientListener) + if (trackingEvent?.blocked == true) { + trackingEvent.surrogateId?.let { surrogateId -> + val surrogate = resourceSurrogates.get(surrogateId) + if (surrogate.responseAvailable) { + Timber.d("Surrogate found for $url") + webViewClientListener?.surrogateDetected(surrogate) + return WebResourceResponse(surrogate.mimeType, "UTF-8", surrogate.jsFunction.byteInputStream()) + } } Timber.d("Blocking request $url") @@ -175,15 +179,15 @@ class WebViewRequestInterceptor( private fun shouldUpgrade(request: WebResourceRequest) = request.isForMainFrame && request.url != null && httpsUpgrader.shouldUpgrade(request.url) - private fun shouldBlock(request: WebResourceRequest, documentUrl: String?, webViewClientListener: WebViewClientListener?): Boolean { + private fun trackingEvent(request: WebResourceRequest, documentUrl: String?, webViewClientListener: WebViewClientListener?): TrackingEvent? { val url = request.url.toString() if (request.isForMainFrame || documentUrl == null) { - return false + return null } - val trackingEvent = trackerDetector.evaluate(url, documentUrl) ?: return false + val trackingEvent = trackerDetector.evaluate(url, documentUrl) ?: return null webViewClientListener?.trackerDetected(trackingEvent) - return trackingEvent.blocked + return trackingEvent } } diff --git a/app/src/main/java/com/duckduckgo/app/global/db/AppDatabase.kt b/app/src/main/java/com/duckduckgo/app/global/db/AppDatabase.kt index 7172bc74b375..1489a338bd07 100644 --- a/app/src/main/java/com/duckduckgo/app/global/db/AppDatabase.kt +++ b/app/src/main/java/com/duckduckgo/app/global/db/AppDatabase.kt @@ -67,7 +67,7 @@ import com.duckduckgo.app.usage.search.SearchCountDao import com.duckduckgo.app.usage.search.SearchCountEntity @Database( - exportSchema = true, version = 31, + exportSchema = true, version = 32, entities = [ TdsTracker::class, TdsEntity::class, @@ -397,6 +397,14 @@ class MigrationsProvider( } } + val MIGRATION_31_TO_32: Migration = object : Migration(31, 32) { + override fun migrate(database: SupportSQLiteDatabase) { + database.execSQL("DELETE FROM tds_domain_entity") + database.execSQL("DELETE FROM tds_entity") + database.execSQL("DELETE FROM tds_tracker") + } + } + val BOOKMARKS_DB_ON_CREATE = object : RoomDatabase.Callback() { override fun onCreate(database: SupportSQLiteDatabase) { MIGRATION_29_TO_30.migrate(database) @@ -440,7 +448,8 @@ class MigrationsProvider( MIGRATION_27_TO_28, MIGRATION_28_TO_29, MIGRATION_29_TO_30, - MIGRATION_30_TO_31 + MIGRATION_30_TO_31, + MIGRATION_31_TO_32 ) @Deprecated( diff --git a/app/src/main/java/com/duckduckgo/app/surrogates/ResourceSurrogateLoader.kt b/app/src/main/java/com/duckduckgo/app/surrogates/ResourceSurrogateLoader.kt index 6f48eb58b497..20ce9715d728 100644 --- a/app/src/main/java/com/duckduckgo/app/surrogates/ResourceSurrogateLoader.kt +++ b/app/src/main/java/com/duckduckgo/app/surrogates/ResourceSurrogateLoader.kt @@ -50,6 +50,7 @@ class ResourceSurrogateLoader @Inject constructor( var nextLineIsNewRule = true + var scriptId = "" var ruleName = "" var mimeType = "" val functionBuilder = StringBuilder() @@ -66,7 +67,10 @@ class ResourceSurrogateLoader @Inject constructor( ruleName = this[0] mimeType = this[1] } - Timber.d("Found new surrogate rule: $ruleName - $mimeType") + with(ruleName.split("/")) { + scriptId = this.last() + } + Timber.d("Found new surrogate rule: $scriptId - $ruleName - $mimeType") nextLineIsNewRule = false return@forEach } @@ -74,6 +78,7 @@ class ResourceSurrogateLoader @Inject constructor( if (it.isBlank()) { surrogates.add( SurrogateResponse( + scriptId = scriptId, name = ruleName, mimeType = mimeType, jsFunction = functionBuilder.toString() diff --git a/app/src/main/java/com/duckduckgo/app/surrogates/ResourceSurrogates.kt b/app/src/main/java/com/duckduckgo/app/surrogates/ResourceSurrogates.kt index 46d718b8a0dd..bb8a97271d82 100644 --- a/app/src/main/java/com/duckduckgo/app/surrogates/ResourceSurrogates.kt +++ b/app/src/main/java/com/duckduckgo/app/surrogates/ResourceSurrogates.kt @@ -16,11 +16,9 @@ package com.duckduckgo.app.surrogates -import android.net.Uri - interface ResourceSurrogates { fun loadSurrogates(urls: List) - fun get(uri: Uri): SurrogateResponse + fun get(scriptId: String): SurrogateResponse fun getAll(): List } @@ -33,10 +31,9 @@ class ResourceSurrogatesImpl : ResourceSurrogates { surrogates.addAll(urls) } - override fun get(uri: Uri): SurrogateResponse { - val uriString = uri.toString() + override fun get(scriptId: String): SurrogateResponse { - return surrogates.find { uriString.contains(it.name) } + return surrogates.find { it.scriptId == scriptId } ?: return SurrogateResponse(responseAvailable = false) } @@ -46,6 +43,7 @@ class ResourceSurrogatesImpl : ResourceSurrogates { } data class SurrogateResponse( + val scriptId: String = "", val responseAvailable: Boolean = true, val name: String = "", val jsFunction: String = "", diff --git a/app/src/main/java/com/duckduckgo/app/trackerdetection/Client.kt b/app/src/main/java/com/duckduckgo/app/trackerdetection/Client.kt index 72c6dd552ca8..8c4cb6eefde1 100644 --- a/app/src/main/java/com/duckduckgo/app/trackerdetection/Client.kt +++ b/app/src/main/java/com/duckduckgo/app/trackerdetection/Client.kt @@ -37,7 +37,8 @@ interface Client { data class Result( val matches: Boolean, val entityName: String? = null, - val categories: List? = null + val categories: List? = null, + val surrogate: String? = null ) val name: ClientName diff --git a/app/src/main/java/com/duckduckgo/app/trackerdetection/TdsClient.kt b/app/src/main/java/com/duckduckgo/app/trackerdetection/TdsClient.kt index fb2853209bc0..769d97179ef9 100644 --- a/app/src/main/java/com/duckduckgo/app/trackerdetection/TdsClient.kt +++ b/app/src/main/java/com/duckduckgo/app/trackerdetection/TdsClient.kt @@ -27,24 +27,27 @@ class TdsClient(override val name: Client.ClientName, private val trackers: List override fun matches(url: String, documentUrl: String): Client.Result { val tracker = trackers.firstOrNull { sameOrSubdomain(url, it.domain) } ?: return Client.Result(false) val matches = matchesTrackerEntry(tracker, url, documentUrl) - return Client.Result(matches, tracker.ownerName, tracker.categories) + return Client.Result(matches = matches.shouldBlock, entityName = tracker.ownerName, categories = tracker.categories, surrogate = matches.surrogate) } - private fun matchesTrackerEntry(tracker: TdsTracker, url: String, documentUrl: String): Boolean { + private fun matchesTrackerEntry(tracker: TdsTracker, url: String, documentUrl: String): MatchedResult { tracker.rules.forEach { rule -> val regex = ".*${rule.rule}.*".toRegex() if (url.matches(regex)) { if (matchedException(rule.exceptions, documentUrl)) { - return false + return MatchedResult(shouldBlock = false) } if (rule.action == IGNORE) { - return false + return MatchedResult(shouldBlock = false) } - return true + if (rule.surrogate?.isNotEmpty() == true) { + return MatchedResult(shouldBlock = true, surrogate = rule.surrogate) + } + return MatchedResult(shouldBlock = true) } } - return tracker.defaultAction == BLOCK + return MatchedResult(shouldBlock = (tracker.defaultAction == BLOCK)) } private fun matchedException(exceptions: RuleExceptions?, documentUrl: String): Boolean { @@ -67,4 +70,6 @@ class TdsClient(override val name: Client.ClientName, private val trackers: List } return false } + + private data class MatchedResult(val shouldBlock: Boolean, val surrogate: String? = null) } diff --git a/app/src/main/java/com/duckduckgo/app/trackerdetection/TrackerDetector.kt b/app/src/main/java/com/duckduckgo/app/trackerdetection/TrackerDetector.kt index 2ecfea5cf965..b07967cebcdf 100644 --- a/app/src/main/java/com/duckduckgo/app/trackerdetection/TrackerDetector.kt +++ b/app/src/main/java/com/duckduckgo/app/trackerdetection/TrackerDetector.kt @@ -63,7 +63,7 @@ class TrackerDetectorImpl( if (result != null) { Timber.v("$documentUrl resource $url WAS identified as a tracker") val entity = if (result.entityName != null) entityLookup.entityForName(result.entityName) else null - return TrackingEvent(documentUrl, url, result.categories, entity, !userWhitelistDao.isDocumentWhitelisted(documentUrl)) + return TrackingEvent(documentUrl, url, result.categories, entity, !userWhitelistDao.isDocumentWhitelisted(documentUrl), result.surrogate) } Timber.v("$documentUrl resource $url was not identified as a tracker") diff --git a/app/src/main/java/com/duckduckgo/app/trackerdetection/model/TdsTracker.kt b/app/src/main/java/com/duckduckgo/app/trackerdetection/model/TdsTracker.kt index 7983b758bb46..3ae6ccd48b3a 100644 --- a/app/src/main/java/com/duckduckgo/app/trackerdetection/model/TdsTracker.kt +++ b/app/src/main/java/com/duckduckgo/app/trackerdetection/model/TdsTracker.kt @@ -40,7 +40,8 @@ enum class Action { class Rule( val rule: String, val action: Action?, - val exceptions: RuleExceptions? + val exceptions: RuleExceptions?, + val surrogate: String? ) class RuleExceptions( diff --git a/app/src/main/java/com/duckduckgo/app/trackerdetection/model/TrackingEvent.kt b/app/src/main/java/com/duckduckgo/app/trackerdetection/model/TrackingEvent.kt index a580d52a0fa3..36699cf85e80 100644 --- a/app/src/main/java/com/duckduckgo/app/trackerdetection/model/TrackingEvent.kt +++ b/app/src/main/java/com/duckduckgo/app/trackerdetection/model/TrackingEvent.kt @@ -21,5 +21,6 @@ data class TrackingEvent( val trackerUrl: String, val categories: List?, val entity: Entity?, - val blocked: Boolean + val blocked: Boolean, + val surrogateId: String? )