Update #1313 for MV3 (#1450)

* started on mv3 ad click

* passes integration tests

* use ddg2dnr

* Update shared/js/background/classes/ad-click-attribution-policy.js

Co-authored-by: Jonathan Kingston <jkingston@duckduckgo.com>

* add unique rule id, oneadclick per tab

* lint

* check that tab exists

* use ad attribution priority

* lint

* lint

* remove debug lines

* remove debug lines

* check for adClickDNR for mv2

* Update shared/js/background/classes/ad-click-attribution-policy.js

Co-authored-by: Dave Vandyke <kzar@kzar.co.uk>

* rename module

* unused function

* Apply suggestions from code review

Co-authored-by: Jonathan Kingston <jkingston@duckduckgo.com>
Co-authored-by: Dave Vandyke <kzar@kzar.co.uk>

* Apply suggestions from code review

Co-authored-by: Jonathan Kingston <jkingston@duckduckgo.com>

* simplify session ids, move dnr stuff around

* fix removal

* everything in adclick

* get offset from storage

* set session id on startup

* Update shared/js/background/dnr-session-rule-id.js

Co-authored-by: Jonathan Kingston <jkingston@duckduckgo.com>

* Update shared/js/background/dnr-session-rule-id.js

Co-authored-by: Dave Vandyke <kzar@kzar.co.uk>

* Update shared/js/background/classes/ad-click-attribution-policy.js

Co-authored-by: Jonathan Kingston <jkingston@duckduckgo.com>

* add tests

* fix test

* changes from pr review

* add ready flag to session id

* make linter happy

Co-authored-by: Jonathan Kingston <jkingston@duckduckgo.com>
Co-authored-by: Dave Vandyke <kzar@kzar.co.uk>

integration-test/config-mv3.json

@@ -5,6 +5,7 @@
         "background/request-blocking.js",
         "background/test-fingerprint.js",
         "background/storage.js",
+        "background/click-attribution.js",
         "content-scripts/gpc.js"
     ]
 }

shared/js/background/classes/ad-click-attribution-policy.js

@@ -1,4 +1,16 @@
+import {
+    AD_ATTRIBUTION_POLICY_PRIORITY
+} from '@duckduckgo/ddg2dnr/lib/rulePriorities'
+
+import {
+    generateDNRRule
+} from '@duckduckgo/ddg2dnr/lib/utils'
+
 const { getFeatureSettings, getBaseDomain } = require('../utils.es6')
+const browserWrapper = require('../wrapper.es6')
+const { getNextSessionRuleId } = require('../dnr-session-rule-id')
+
+const manifestVersion = browserWrapper.getManifestVersion()
 
 /**
  * @typedef AdClickAttributionLinkFormat
@@ -65,15 +77,18 @@ export class AdClickAttributionPolicy {
         const linkFormat = this.getMatchingLinkFormat(resourceURL)
         if (!linkFormat) return
 
-        const adClick = new AdClick(this.navigationExpiration, this.totalExpiration)
+        const adClick = new AdClick(this.navigationExpiration, this.totalExpiration, this.allowlist)
+
+        if (manifestVersion === 3) {
+            adClick.createDNR(tab.id)
+        }
 
         if (linkFormat.adDomainParameterName) {
             const parameterDomain = resourceURL.searchParams.get(linkFormat.adDomainParameterName)
             if (parameterDomain && this.domainDetectionEnabled) {
                 const parsedParameterDomain = getBaseDomain(parameterDomain)
                 if (parsedParameterDomain) {
-                    adClick.adBaseDomain = parsedParameterDomain
-                    adClick.adClickRedirect = false
+                    adClick.setAdBaseDomain(parsedParameterDomain)
                     return adClick
                 }
             }
@@ -109,34 +124,65 @@ export class AdClick {
      * @param {number} navigationExpiration in seconds
      * @param {number} totalExpiration in seconds
      */
-    constructor (navigationExpiration, totalExpiration) {
+    constructor (navigationExpiration, totalExpiration, allowlist) {
         /** @type {string | null} */
         this.adBaseDomain = null
         this.adClickRedirect = false
         this.navigationExpiration = navigationExpiration
         this.totalExpiration = totalExpiration
         this.expires = Date.now() + (this.totalExpiration * 1000)
         this.clickExpires = Date.now() + (this.navigationExpiration * 1000)
+        this.allowlist = allowlist
+        this.adClickDNR = null
     }
 
     clone () {
-        const adClick = new AdClick(this.navigationExpiration, this.totalExpiration)
+        const adClick = new AdClick(this.navigationExpiration, this.totalExpiration, this.allowlist)
         adClick.adBaseDomain = this.adBaseDomain
         adClick.adClickRedirect = this.adClickRedirect
         adClick.expires = this.expires
         adClick.clickExpires = Date.now() + (this.navigationExpiration * 1000)
+        adClick.adClickDNR = this.adClickDNR
+        return adClick
+    }
+
+    /**
+     * Propagate an adclick to a new tab, used when a user navigates to a new tab.
+     * @param {number} tabId
+     * @returns {AdClick} adClick
+     */
+    propagate (tabId) {
+        const adClick = this.clone()
+
+        if (this.adClickDNR) {
+            this.createDNR(tabId)
+        }
+
         return adClick
     }
 
     static restore (adClick) {
-        const restoredAdClick = new AdClick(adClick.navigationExpiration, adClick.totalExpiration)
+        const restoredAdClick = new AdClick(adClick.navigationExpiration, adClick.totalExpiration, adClick.allowlist)
         restoredAdClick.adBaseDomain = adClick.adBaseDomain
         restoredAdClick.adClickRedirect = adClick.adClickRedirect
         restoredAdClick.expires = adClick.expires
         restoredAdClick.clickExpires = adClick.clickExpires
+        restoredAdClick.adClickDNR = adClick.adClickDNR
         return restoredAdClick
     }
 
+    /**
+     * @param {string} domain
+     **/
+    setAdBaseDomain (domain) {
+        this.adBaseDomain = domain
+        this.adClickRedirect = false
+
+        if (this.adClickDNR) {
+            this.updateDNRInitiator(domain)
+        }
+    }
+
     /**
      * @param {Tab} tab
      * @returns {boolean} true if a new tab should have the ad attribution policy applied
@@ -160,7 +206,12 @@ export class AdClick {
     }
 
     hasNotExpired () {
-        return this.expires > Date.now()
+        if (this.expires > Date.now()) {
+            return true
+        } else {
+            this.removeDNR()
+            return false
+        }
     }
 
     /**
@@ -173,4 +224,45 @@ export class AdClick {
         if (tab.site.baseDomain !== this.adBaseDomain) return false
         return this.hasNotExpired()
     }
+
+    getAdClickDNR (tabId) {
+        const adClickDNR = {
+            rule: generateDNRRule({
+                id: null,
+                priority: AD_ATTRIBUTION_POLICY_PRIORITY,
+                actionType: 'allow',
+                requestDomains: this.allowlist.map((entry) => entry.host)
+            })
+        }
+        adClickDNR.rule.condition.tabIds = [tabId]
+        return adClickDNR
+    }
+
+    updateDNRInitiator (domain) {
+        if (this.adClickDNR && domain) {
+            this.adClickDNR.rule.condition.initiatorDomains = [domain]
+            this.updateDNR()
+        }
+    }
+
+    createDNR (tabId) {
+        this.adClickDNR = this.getAdClickDNR(tabId)
+        this.adClickDNR.rule.id = getNextSessionRuleId()
+        chrome.declarativeNetRequest.updateSessionRules({ addRules: [this.adClickDNR.rule] })
+    }
+
+    updateDNR () {
+        if (this.adClickDNR) {
+            chrome.declarativeNetRequest.updateSessionRules({
+                removeRuleIds: [this.adClickDNR.rule.id],
+                addRules: [this.adClickDNR.rule]
+            })
+        }
+    }
+
+    removeDNR () {
+        if (this.adClickDNR) {
+            chrome.declarativeNetRequest.updateSessionRules({ removeRuleIds: [this.adClickDNR.rule.id] })
+        }
+    }
 }

shared/js/background/dnr-session-rule-id.js

@@ -0,0 +1,33 @@
+/**
+ * For managing dynamically created MV3 session rules
+ * getNextSessionRuleId will return the next unique session rule id to use when creating new session rules
+ **/
+import * as browserWrapper from './wrapper.es6'
+const SESSION_RULE_ID_START = 100000
+const SESSION_RULE_STORAGE_KEY = 'sessionRuleOffset'
+let sessionRuleOffset = 0
+let ready = false
+
+export async function setSessionRuleOffsetFromStorage () {
+    const offset = await browserWrapper.getFromSessionStorage(SESSION_RULE_STORAGE_KEY)
+    if (offset) {
+        sessionRuleOffset = offset
+    }
+    ready = true
+}
+
+/**
+ * Get the next unique session rule id to use when craeting session DNR rules
+ * @returns {number | null} nextRuleId
+ */
+export function getNextSessionRuleId () {
+    if (!ready) {
+        console.warn('Tried to get session rule id before reading offset from storage')
+        return null
+    }
+
+    const nextRuleId = SESSION_RULE_ID_START + sessionRuleOffset
+    sessionRuleOffset += 1
+    browserWrapper.setToSessionStorage(SESSION_RULE_STORAGE_KEY, sessionRuleOffset)
+    return nextRuleId
+}

shared/js/background/events.es6.js

@@ -183,75 +183,79 @@ browser.webRequest.onBeforeRequest.addListener(
     additionalOptions
 )
 
+// MV2 needs blocking for webRequest
+// MV3 still needs some info from response headers
+const extraInfoSpec = ['responseHeaders']
 if (manifestVersion === 2) {
-    const extraInfoSpec = ['blocking', 'responseHeaders']
-    if (browser.webRequest.OnHeadersReceivedOptions.EXTRA_HEADERS) {
-        extraInfoSpec.push(browser.webRequest.OnHeadersReceivedOptions.EXTRA_HEADERS)
-    }
+    extraInfoSpec.push('blocking')
+}
 
-    // We determine if browsingTopics is enabled by testing for availability of its
-    // JS API.
-    // Note: This approach will not work with MV3 since the background
-    //       ServiceWorker does not have access to a `document` Object.
-    const isTopicsEnabled = ('browsingTopics' in document) && utils.isFeatureEnabled('googleRejected')
-    browser.webRequest.onHeadersReceived.addListener(
-        request => {
-            if (request.type === 'main_frame') {
-                tabManager.updateTabUrl(request)
-
-                const tab = tabManager.get({ tabId: request.tabId })
-                // SERP ad click detection
-                if (
-                    utils.isRedirect(request.statusCode)
-                ) {
-                    tab.setAdClickIfValidRedirect(request.url)
-                } else if (tab && tab.adClick && tab.adClick.adClickRedirect && !utils.isRedirect(request.statusCode)) {
-                    tab.adClick.adClickRedirect = false
-                    tab.adClick.adBaseDomain = tab.site.baseDomain
-                }
-            }
+if (browser.webRequest.OnHeadersReceivedOptions.EXTRA_HEADERS) {
+    extraInfoSpec.push(browser.webRequest.OnHeadersReceivedOptions.EXTRA_HEADERS)
+}
 
-            if (ATB.shouldUpdateSetAtb(request)) {
-                // returns a promise
-                return ATB.updateSetAtb()
-            }
+// We determine if browsingTopics is enabled by testing for availability of its
+// JS API.
+// Note: This approach will not work with MV3 since the background
+//       ServiceWorker does not have access to a `document` Object.
+const isTopicsEnabled = manifestVersion === 2 && 'browsingTopics' in document && utils.isFeatureEnabled('googleRejected')
 
-            const responseHeaders = request.responseHeaders
+browser.webRequest.onHeadersReceived.addListener(
+    request => {
+        if (request.type === 'main_frame') {
+            tabManager.updateTabUrl(request)
 
-            if (isTopicsEnabled && responseHeaders && (request.type === 'main_frame' || request.type === 'sub_frame')) {
-                // there can be multiple permissions-policy headers, so we are good always appending one
-                // According to Google's docs a site can opt out of browsing topics the same way as opting out of FLoC
-                // https://privacysandbox.com/proposals/topics (See FAQ)
-                responseHeaders.push({ name: 'permissions-policy', value: 'interest-cohort=()' })
+            const tab = tabManager.get({ tabId: request.tabId })
+            // SERP ad click detection
+            if (
+                utils.isRedirect(request.statusCode)
+            ) {
+                tab.setAdClickIfValidRedirect(request.url)
+            } else if (tab && tab.adClick && tab.adClick.adClickRedirect && !utils.isRedirect(request.statusCode)) {
+                tab.adClick.setAdBaseDomain(tab.site.baseDomain)
             }
+        }
 
-            return { responseHeaders }
-        },
+        if (ATB.shouldUpdateSetAtb(request)) {
+            // returns a promise
+            return ATB.updateSetAtb()
+        }
+
+        const responseHeaders = request.responseHeaders
+
+        if (isTopicsEnabled && responseHeaders && (request.type === 'main_frame' || request.type === 'sub_frame')) {
+            // there can be multiple permissions-policy headers, so we are good always appending one
+            // According to Google's docs a site can opt out of browsing topics the same way as opting out of FLoC
+            // https://privacysandbox.com/proposals/topics (See FAQ)
+            responseHeaders.push({ name: 'permissions-policy', value: 'interest-cohort=()' })
+        }
+
+        return { responseHeaders }
+    },
+    { urls: ['<all_urls>'] },
+    extraInfoSpec
+)
+
+// Wait until the extension configuration has finished loading and then
+// start dropping tracking cookies.
+// Note: Event listeners must be registered in the top-level of the script
+//       to be compatible with MV3. Registering the listener asynchronously
+//       is only possible here as this is a MV2-only event listener!
+// See https://developer.chrome.com/docs/extensions/mv3/migrating_to_service_workers/#event_listeners
+startup.ready().then(() => {
+    browser.webRequest.onHeadersReceived.addListener(
+        dropTracking3pCookiesFromResponse,
         { urls: ['<all_urls>'] },
         extraInfoSpec
     )
-
-    // Wait until the extension configuration has finished loading and then
-    // start dropping tracking cookies.
-    // Note: Event listeners must be registered in the top-level of the script
-    //       to be compatible with MV3. Registering the listener asynchronously
-    //       is only possible here as this is a MV2-only event listener!
-    // See https://developer.chrome.com/docs/extensions/mv3/migrating_to_service_workers/#event_listeners
-    startup.ready().then(() => {
-        browser.webRequest.onHeadersReceived.addListener(
-            dropTracking3pCookiesFromResponse,
-            { urls: ['<all_urls>'] },
-            extraInfoSpec
-        )
-    })
-}
+})
 
 browser.webNavigation.onCreatedNavigationTarget.addListener(details => {
     const currentTab = tabManager.get({ tabId: details.sourceTabId })
     if (currentTab && currentTab.adClick) {
         const newTab = tabManager.createOrUpdateTab(details.tabId, { url: details.url })
         if (currentTab.adClick.shouldPropagateAdClickForNewTab(newTab)) {
-            newTab.adClick = currentTab.adClick
+            newTab.adClick = currentTab.adClick.propagate(newTab.id)
         }
     }
 })

shared/js/background/startup.es6.js

@@ -1,6 +1,7 @@
 import browser from 'webextension-polyfill'
 const { getCurrentTab } = require('./utils.es6')
 const browserUIWrapper = require('../ui/base/ui-wrapper.es6')
+const browserWrapper = require('./wrapper.es6')
 const Companies = require('./companies.es6')
 const experiment = require('./experiments.es6')
 const https = require('./https.es6')
@@ -9,14 +10,19 @@ const settings = require('./settings.es6')
 const tabManager = require('./tab-manager.es6')
 const tdsStorage = require('./storage/tds.es6')
 const trackers = require('./trackers.es6')
+const dnrSessionId = require('./dnr-session-rule-id')
 const { fetchAlias, showContextMenuAction } = require('./email-utils.es6')
-
+const manifestVersion = browserWrapper.getManifestVersion()
 /** @module */
 
 let resolveReadyPromise
 const readyPromise = new Promise(resolve => { resolveReadyPromise = resolve })
 
 export async function onStartup () {
+    if (manifestVersion === 3) {
+        await dnrSessionId.setSessionRuleOffsetFromStorage()
+    }
+
     registerUnloadHandler()
     await settings.ready()
     experiment.setActiveExperiment()

shared/js/background/tab-manager.es6.js

@@ -52,6 +52,7 @@ class TabManager {
     }
 
     delete (id) {
+        this.tabContainer[id]?.adClick?.removeDNR()
         delete this.tabContainer[id]
         TabState.delete(id)
     }