Fix surrogates algorithm

app/src/androidTest/java/com/duckduckgo/app/brokensite/BrokenSiteDataTest.kt

@@ -65,8 +65,8 @@ class BrokenSiteDataTest {
     @Test
     fun whenSiteHasBlockedTrackersThenBlockedTrackersExist() {
         val site = buildSite(SITE_URL)
-        val event = TrackingEvent("http://www.example.com", "http://www.tracker.com/tracker.js", emptyList(), null, false)
-        val anotherEvent = TrackingEvent("http://www.example.com/test", "http://www.anothertracker.com/tracker.js", emptyList(), null, false)
+        val event = TrackingEvent("http://www.example.com", "http://www.tracker.com/tracker.js", emptyList(), null, false, null)
+        val anotherEvent = TrackingEvent("http://www.example.com/test", "http://www.anothertracker.com/tracker.js", emptyList(), null, false, null)
         site.trackerDetected(event)
         site.trackerDetected(anotherEvent)
         assertEquals("www.tracker.com,www.anothertracker.com", BrokenSiteData.fromSite(site).blockedTrackers)
@@ -75,8 +75,8 @@ class BrokenSiteDataTest {
     @Test
     fun whenSiteHasSameHostBlockedTrackersThenOnlyUniqueTrackersIncludedInData() {
         val site = buildSite(SITE_URL)
-        val event = TrackingEvent("http://www.example.com", "http://www.tracker.com/tracker.js", emptyList(), null, false)
-        val anotherEvent = TrackingEvent("http://www.example.com/test", "http://www.tracker.com/tracker2.js", emptyList(), null, false)
+        val event = TrackingEvent("http://www.example.com", "http://www.tracker.com/tracker.js", emptyList(), null, false, null)
+        val anotherEvent = TrackingEvent("http://www.example.com/test", "http://www.tracker.com/tracker2.js", emptyList(), null, false, null)
         site.trackerDetected(event)
         site.trackerDetected(anotherEvent)
         assertEquals("www.tracker.com", BrokenSiteData.fromSite(site).blockedTrackers)
@@ -91,8 +91,8 @@ class BrokenSiteDataTest {
 
     @Test
     fun whenSiteHasSurrogatesThenSurrogatesExist() {
-        val surrogate = SurrogateResponse(true, "surrogate.com/test.js", "", "")
-        val anotherSurrogate = SurrogateResponse(true, "anothersurrogate.com/test.js", "", "")
+        val surrogate = SurrogateResponse("test.js", true, "surrogate.com/test.js", "", "")
+        val anotherSurrogate = SurrogateResponse("test.js", true, "anothersurrogate.com/test.js", "", "")
         val site = buildSite(SITE_URL)
         site.surrogateDetected(surrogate)
         site.surrogateDetected(anotherSurrogate)
@@ -101,8 +101,8 @@ class BrokenSiteDataTest {
 
     @Test
     fun whenSiteHasSameHostSurrogatesThenOnlyUniqueSurrogateIncludedInData() {
-        val surrogate = SurrogateResponse(true, "surrogate.com/test.js", "", "")
-        val anotherSurrogate = SurrogateResponse(true, "surrogate.com/test2.js", "", "")
+        val surrogate = SurrogateResponse("test.js", true, "surrogate.com/test.js", "", "")
+        val anotherSurrogate = SurrogateResponse("test.js", true, "surrogate.com/test2.js", "", "")
         val site = buildSite(SITE_URL)
         site.surrogateDetected(surrogate)
         site.surrogateDetected(anotherSurrogate)

app/src/androidTest/java/com/duckduckgo/app/browser/BrowserTabViewModelTest.kt

@@ -526,7 +526,7 @@ class BrowserTabViewModelTest {
     @Test
     fun whenTrackerDetectedThenNetworkLeaderboardUpdated() {
         val networkEntity = TestEntity("Network1", "Network1", 10.0)
-        val event = TrackingEvent("http://www.example.com", "http://www.tracker.com/tracker.js", emptyList(), networkEntity, false)
+        val event = TrackingEvent("http://www.example.com", "http://www.tracker.com/tracker.js", emptyList(), networkEntity, false, null)
         testee.trackerDetected(event)
         verify(mockNetworkLeaderboardDao).incrementNetworkCount("Network1")
     }
@@ -777,7 +777,7 @@ class BrowserTabViewModelTest {
         loadUrl("https://example.com")
         val entity = TestEntity("Network1", "Network1", 10.0)
         for (i in 1..10) {
-            testee.trackerDetected(TrackingEvent("https://example.com", "", null, entity, false))
+            testee.trackerDetected(TrackingEvent("https://example.com", "", null, entity, false, null))
         }
         assertNotEquals(grade, privacyGradeState().privacyGrade)
     }

app/src/androidTest/java/com/duckduckgo/app/browser/WebViewRequestInterceptorTest.kt

@@ -285,8 +285,9 @@ class WebViewRequestInterceptorTest {
     }
 
     @Test
-    fun whenRequestShouldBlockButThereIsASurrogateThen() = runBlocking<Unit> {
+    fun whenRequestShouldBlockButThereIsASurrogateThenResponseReturnedContainsTheSurrogateData() = runBlocking<Unit> {
         val availableSurrogate = SurrogateResponse(
+            scriptId = "testId",
             responseAvailable = true,
             mimeType = "application/javascript",
             jsFunction = "javascript replacement function goes here"
@@ -308,6 +309,7 @@ class WebViewRequestInterceptorTest {
     @Test
     fun whenRequestShouldBlockButThereIsASurrogateThenCallSurrogateDetected() = runBlocking<Unit> {
         val availableSurrogate = SurrogateResponse(
+            scriptId = "testId",
             responseAvailable = true,
             mimeType = "application/javascript",
             jsFunction = "javascript replacement function goes here"
@@ -499,7 +501,8 @@ class WebViewRequestInterceptorTest {
             documentUrl = "",
             trackerUrl = "",
             entity = null,
-            categories = null
+            categories = null,
+            surrogateId = "testId"
         )
         whenever(mockRequest.isForMainFrame).thenReturn(false)
         whenever(mockTrackerDetector.evaluate(any(), any())).thenReturn(blockTrackingEvent)

app/src/androidTest/java/com/duckduckgo/app/cta/ui/CtaTest.kt

@@ -297,8 +297,8 @@ class CtaTest {
     @Test
     fun whenTrackersBlockedReturnThemSortingByPrevalence() {
         val trackers = listOf(
-            TrackingEvent("facebook.com", "facebook.com", blocked = true, entity = TestEntity("Facebook", "Facebook", 3.0), categories = null),
-            TrackingEvent("other.com", "other.com", blocked = true, entity = TestEntity("Other", "Other", 9.0), categories = null)
+            TrackingEvent("facebook.com", "facebook.com", blocked = true, entity = TestEntity("Facebook", "Facebook", 3.0), categories = null, surrogateId = null),
+            TrackingEvent("other.com", "other.com", blocked = true, entity = TestEntity("Other", "Other", 9.0), categories = null, surrogateId = null)
         )
         val site = site(events = trackers)
 
@@ -312,8 +312,8 @@ class CtaTest {
     @Test
     fun whenTrackersBlockedReturnOnlyTrackersWithDisplayName() {
         val trackers = listOf(
-            TrackingEvent("facebook.com", "facebook.com", blocked = true, entity = TestEntity("Facebook", "Facebook", 3.0), categories = null),
-            TrackingEvent("other.com", "other.com", blocked = true, entity = TestEntity("Other", "", 9.0), categories = null)
+            TrackingEvent("facebook.com", "facebook.com", blocked = true, entity = TestEntity("Facebook", "Facebook", 3.0), categories = null, surrogateId = null),
+            TrackingEvent("other.com", "other.com", blocked = true, entity = TestEntity("Other", "", 9.0), categories = null, surrogateId = null)
         )
         val site = site(events = trackers)
 

app/src/androidTest/java/com/duckduckgo/app/cta/ui/CtaViewModelTest.kt

@@ -415,7 +415,7 @@ class CtaViewModelTest {
     @Test
     fun whenRefreshCtaWhileBrowsingThenReturnTrackersBlockedCta() = coroutineRule.runBlocking {
         givenDaxOnboardingActive()
-        val trackingEvent = TrackingEvent("test.com", "test.com", null, TestEntity("test", "test", 9.0), true)
+        val trackingEvent = TrackingEvent("test.com", "test.com", null, TestEntity("test", "test", 9.0), true, null)
         val site = site(url = "http://www.cnn.com", trackerCount = 1, events = listOf(trackingEvent))
         val value = testee.refreshCta(coroutineRule.testDispatcher, isBrowserShowing = true, site = site)
 
@@ -425,7 +425,7 @@ class CtaViewModelTest {
     @Test
     fun whenRefreshCtaWhileBrowsingAndTrackersAreNotMajorThenReturnTrackersBlockedCta() = coroutineRule.runBlocking {
         givenDaxOnboardingActive()
-        val trackingEvent = TrackingEvent("test.com", "test.com", null, TestEntity("test", "test", 0.123), true)
+        val trackingEvent = TrackingEvent("test.com", "test.com", null, TestEntity("test", "test", 0.123), true, null)
         val site = site(url = "http://www.cnn.com", trackerCount = 1, events = listOf(trackingEvent))
         val value = testee.refreshCta(coroutineRule.testDispatcher, isBrowserShowing = true, site = site)
 

app/src/androidTest/java/com/duckduckgo/app/global/db/AppDatabaseTest.kt

@@ -350,6 +350,11 @@ class AppDatabaseTest {
         createDatabaseAndMigrate(30, 31, migrationsProvider.MIGRATION_30_TO_31)
     }
 
+    @Test
+    fun whenMigratingFromVersion31To32ThenValidationSucceeds() {
+        createDatabaseAndMigrate(31, 32, migrationsProvider.MIGRATION_31_TO_32)
+    }
+
     private fun givenUserStageIs(database: SupportSQLiteDatabase, appStage: AppStage) {
         database.execSQL("INSERT INTO `userStage` values (1, '${appStage.name}') ")
     }

app/src/androidTest/java/com/duckduckgo/app/global/model/SiteMonitorTest.kt

@@ -91,30 +91,30 @@ class SiteMonitorTest {
     @Test
     fun whenTrackersAreDetectedThenTrackerCountIsIncremented() {
         val testee = SiteMonitor(document, null)
-        testee.trackerDetected(TrackingEvent(document, trackerA, null, null, true))
-        testee.trackerDetected(TrackingEvent(document, trackerB, null, null, true))
+        testee.trackerDetected(TrackingEvent(document, trackerA, null, null, true, null))
+        testee.trackerDetected(TrackingEvent(document, trackerB, null, null, true, null))
         assertEquals(2, testee.trackerCount)
     }
 
     @Test
     fun whenNonMajorNetworkTrackerIsDetectedThenMajorNetworkCoutnIsZero() {
         val testee = SiteMonitor(document, null)
-        testee.trackerDetected(TrackingEvent(document, trackerA, null, network, true))
+        testee.trackerDetected(TrackingEvent(document, trackerA, null, network, true, null))
         assertEquals(0, testee.majorNetworkCount)
     }
 
     @Test
     fun whenMajorNetworkTrackerIsDetectedThenMajorNetworkCountIsOne() {
         val testee = SiteMonitor(document, null)
-        testee.trackerDetected(TrackingEvent(document, majorNetworkTracker, null, majorNetwork, true))
+        testee.trackerDetected(TrackingEvent(document, majorNetworkTracker, null, majorNetwork, true, null))
         assertEquals(1, testee.majorNetworkCount)
     }
 
     @Test
     fun whenDuplicateMajorNetworkIsDetectedThenMajorNetworkCountIsStillOne() {
         val testee = SiteMonitor(document, null)
-        testee.trackerDetected(TrackingEvent(document, trackerA, null, majorNetwork, true))
-        testee.trackerDetected(TrackingEvent(document, trackerB, null, majorNetwork, true))
+        testee.trackerDetected(TrackingEvent(document, trackerA, null, majorNetwork, true, null))
+        testee.trackerDetected(TrackingEvent(document, trackerB, null, majorNetwork, true, null))
         assertEquals(1, testee.majorNetworkCount)
     }
 

app/src/androidTest/java/com/duckduckgo/app/privacy/ui/TrackerNetworksAdapterTest.kt

@@ -50,7 +50,7 @@ class TrackerNetworksAdapterTest {
     }
 
     private fun data(): SortedMap<Entity, List<TrackingEvent>> {
-        val trackingEvent = TrackingEvent("", "", null, null, true)
+        val trackingEvent = TrackingEvent("", "", null, null, true, null)
         val minorNetworkList = arrayListOf(trackingEvent, trackingEvent)
         val majorNetworkList = arrayListOf(trackingEvent, trackingEvent)
         return mapOf(

app/src/androidTest/java/com/duckduckgo/app/privacy/ui/TrackerNetworksViewModelTest.kt

@@ -65,7 +65,7 @@ class TrackerNetworksViewModelTest {
 
     @Test
     fun whenTrackersUpdatedWithNoTrackersThenViewModelListIsEmpty() {
-        val input = listOf(TrackingEvent(Url.DOCUMENT, Url.tracker(1), null, Entity.MINOR_ENTITY_A, true))
+        val input = listOf(TrackingEvent(Url.DOCUMENT, Url.tracker(1), null, Entity.MINOR_ENTITY_A, true, null))
         testee.onSiteChanged(site(trackingEvents = input))
         assertTrue(testee.viewState.value!!.trackingEventsByNetwork.isNotEmpty())
 
@@ -78,21 +78,21 @@ class TrackerNetworksViewModelTest {
 
         val input = listOf(
             // Minor entity with 3 distinct trackers
-            TrackingEvent(Url.DOCUMENT, Url.tracker(1), null, Entity.MINOR_ENTITY_A, true),
-            TrackingEvent(Url.DOCUMENT, Url.tracker(1), null, Entity.MINOR_ENTITY_A, true),
-            TrackingEvent(Url.DOCUMENT, Url.tracker(2), null, Entity.MINOR_ENTITY_A, true),
-            TrackingEvent(Url.DOCUMENT, Url.tracker(3), null, Entity.MINOR_ENTITY_A, true),
+            TrackingEvent(Url.DOCUMENT, Url.tracker(1), null, Entity.MINOR_ENTITY_A, true, null),
+            TrackingEvent(Url.DOCUMENT, Url.tracker(1), null, Entity.MINOR_ENTITY_A, true, null),
+            TrackingEvent(Url.DOCUMENT, Url.tracker(2), null, Entity.MINOR_ENTITY_A, true, null),
+            TrackingEvent(Url.DOCUMENT, Url.tracker(3), null, Entity.MINOR_ENTITY_A, true, null),
 
             // Minor entity with 1 distinct tracker
-            TrackingEvent(Url.DOCUMENT, Url.tracker(4), null, Entity.MINOR_ENTITY_B, true),
+            TrackingEvent(Url.DOCUMENT, Url.tracker(4), null, Entity.MINOR_ENTITY_B, true, null),
 
             // Major entity with 2 distinct tracker
-            TrackingEvent(Url.DOCUMENT, Url.tracker(6), null, Entity.MAJOR_ENTITY_B, true),
-            TrackingEvent(Url.DOCUMENT, Url.tracker(6), null, Entity.MAJOR_ENTITY_B, true),
-            TrackingEvent(Url.DOCUMENT, Url.tracker(7), null, Entity.MAJOR_ENTITY_B, true),
+            TrackingEvent(Url.DOCUMENT, Url.tracker(6), null, Entity.MAJOR_ENTITY_B, true, null),
+            TrackingEvent(Url.DOCUMENT, Url.tracker(6), null, Entity.MAJOR_ENTITY_B, true, null),
+            TrackingEvent(Url.DOCUMENT, Url.tracker(7), null, Entity.MAJOR_ENTITY_B, true, null),
 
             // Major entity with 1 distinct tracker
-            TrackingEvent(Url.DOCUMENT, Url.tracker(5), null, Entity.MAJOR_ENTITY_A, true)
+            TrackingEvent(Url.DOCUMENT, Url.tracker(5), null, Entity.MAJOR_ENTITY_A, true, null)
         )
 
         testee.onSiteChanged(site(trackingEvents = input))

app/src/androidTest/java/com/duckduckgo/app/surrogates/ResourceSurrogateLoaderTest.kt

@@ -105,6 +105,17 @@ class ResourceSurrogateLoaderTest {
         assertEquals(6, surrogates.size)
     }
 
+    @Test
+    fun whenLoadingSurrogatesThenCorrectScriptIdStored() {
+        val surrogates = initialiseFile("surrogates_6")
+        assertEquals("ga.js", surrogates[0].scriptId)
+        assertEquals("analytics.js", surrogates[1].scriptId)
+        assertEquals("inpage_linkid.js", surrogates[2].scriptId)
+        assertEquals("api.js", surrogates[3].scriptId)
+        assertEquals("gpt.js", surrogates[4].scriptId)
+        assertEquals("gtm.js", surrogates[5].scriptId)
+    }
+
     private fun initialiseFile(filename: String): List<SurrogateResponse> {
         return testee.convertBytes(readFile(filename))
     }

app/src/androidTest/java/com/duckduckgo/app/surrogates/ResourceSurrogatesTest.kt

@@ -16,7 +16,6 @@
 
 package com.duckduckgo.app.surrogates
 
-import android.net.Uri
 import org.junit.Assert.*
 import org.junit.Before
 import org.junit.Test
@@ -50,26 +49,18 @@ class ResourceSurrogatesTest {
     }
 
     @Test
-    fun whenSearchingForExactMatchingExistingSurrogateThenCanFindByName() {
-        val surrogate = SurrogateResponse(name = "foo")
+    fun whenSearchingForExactMatchingExistingSurrogateThenCanFindByScriptId() {
+        val surrogate = SurrogateResponse(scriptId = "fooId", name = "foo")
         testee.loadSurrogates(listOf(surrogate))
-        val retrieved = testee.get(Uri.parse("foo"))
+        val retrieved = testee.get("fooId")
         assertTrue(retrieved.responseAvailable)
     }
 
     @Test
-    fun whenSearchingForSubstringMatchingExistingSurrogateThenCanFindByName() {
-        val surrogate = SurrogateResponse(name = "foo.com")
+    fun whenSearchingByNonExistentScriptIdThenResponseUnavailableSurrogateResultReturned() {
+        val surrogate = SurrogateResponse(scriptId = "fooId", name = "foo")
         testee.loadSurrogates(listOf(surrogate))
-        val retrieved = testee.get(Uri.parse("foo.com/a/b/c"))
-        assertTrue(retrieved.responseAvailable)
-    }
-
-    @Test
-    fun whenSearchingByNonExistentNameThenResponseUnavailableSurrogateResultReturned() {
-        val surrogate = SurrogateResponse(name = "foo")
-        testee.loadSurrogates(listOf(surrogate))
-        val retrieved = testee.get(Uri.parse("bar"))
+        val retrieved = testee.get("bar")
         assertFalse(retrieved.responseAvailable)
     }
 }