Add seen before day on site breakage reports (#2306)

* Add seen before day on site breakage reports

* Rename field to lastSentDay

* Expire storage and hash hostname

* Adjust clear timer

* Add support for multiple report reference tests

* Support for matches and current day

* Fix plural file name

* Switch over to branch for ref tests

* Fix time output for not UTC

* Change reference tests back

* Fix pixel lint

* Make new tests await

shared/js/background/broken-site-report.js

@@ -88,6 +88,57 @@ const requestCategoryMapping = {
     'ignore-user': 'ignoredByUserRequests'
 }
 
+async function digestMessage (message) {
+    const msgUint8 = new TextEncoder().encode(message)
+    const hashBuffer = await crypto.subtle.digest('SHA-256', msgUint8)
+    const hashArray = Array.from(new Uint8Array(hashBuffer))
+    const hashHex = hashArray
+        .map((b) => b.toString(16).padStart(2, '0'))
+        .join('')
+    return hashHex
+}
+
+async function computeLastSentDay (urlString) {
+    const url = new URL(urlString)
+    // Output time as a string in the format YYYY-MM-DD
+    const dayOutput = new Date().toISOString().split('T')[0]
+
+    // Use a sha256 hash prefix of the hostname so that we don't store the full hostname
+    const hash = await digestMessage(url.hostname)
+    const hostnameHashPrefix = hash.slice(0, 6)
+
+    const reportTimes = settings.getSetting('brokenSiteReportTimes') || {}
+    const lastSentDay = reportTimes[hostnameHashPrefix]
+
+    // Update existing time
+    reportTimes[hostnameHashPrefix] = dayOutput
+    settings.updateSetting('brokenSiteReportTimes', reportTimes)
+
+    return lastSentDay
+}
+
+/**
+ * Clears any expired broken site report times
+ * Called by an alarm every hour to remove entries older than 30 days
+ */
+export async function clearExpiredBrokenSiteReportTimes () {
+    await settings.ready()
+    const brokenSiteReports = settings.getSetting('brokenSiteReportTimes') || {}
+    // Expiry of 30 days
+    const expiryTime = new Date().getTime() - 30 * 24 * 60 * 60 * 1000
+    for (const hashPrefix in brokenSiteReports) {
+        const reportTime = new Date(brokenSiteReports[hashPrefix])
+        if (reportTime.getTime() < expiryTime) {
+            delete brokenSiteReports[hashPrefix]
+        }
+    }
+    settings.updateSetting('brokenSiteReportTimes', brokenSiteReports)
+}
+
+export async function clearAllBrokenSiteReportTimes () {
+    settings.updateSetting('brokenSiteReportTimes', {})
+}
+
 /**
  * Given an optional category and description, create a report for a given Tab instance.
  *
@@ -103,7 +154,7 @@ const requestCategoryMapping = {
  * @prop {string | undefined} arg.category - optional category
  * @prop {string | undefined} arg.description - optional description
  */
-export function breakageReportForTab ({
+export async function breakageReportForTab ({
     tab, tds, remoteConfigEtag, remoteConfigVersion,
     category, description
 }) {
@@ -137,6 +188,7 @@ export function breakageReportForTab ({
     const debugFlags = tab.debugFlags.join(',')
     const errorDescriptions = JSON.stringify(tab.errorDescriptions)
     const httpErrorCodes = tab.httpErrorCodes.join(',')
+    const lastSentDay = await computeLastSentDay(tab.url)
 
     const brokenSiteParams = new URLSearchParams({
         siteUrl,
@@ -155,6 +207,7 @@ export function breakageReportForTab ({
         brokenSiteParams.append(key, value.join(','))
     }
 
+    if (lastSentDay) brokenSiteParams.set('lastSentDay', lastSentDay)
     if (ampUrl) brokenSiteParams.set('ampUrl', ampUrl)
     if (category) brokenSiteParams.set('category', category)
     if (debugFlags) brokenSiteParams.set('debugFlags', debugFlags)

shared/js/background/events.js

@@ -17,6 +17,7 @@ import {
 import tdsStorage from './storage/tds'
 import httpsStorage from './storage/https'
 import ATB from './atb'
+import { clearExpiredBrokenSiteReportTimes } from './broken-site-report'
 const utils = require('./utils')
 const experiment = require('./experiments')
 const settings = require('./settings')
@@ -434,7 +435,9 @@ browserWrapper.createAlarm('clearExpiredHTTPSServiceCache', { periodInMinutes: 6
 // Rotate the user agent spoofed
 browserWrapper.createAlarm('rotateUserAgent', { periodInMinutes: 24 * 60 })
 // Rotate the sessionKey
-browserWrapper.createAlarm('rotateSessionKey', { periodInMinutes: 24 * 60 })
+browserWrapper.createAlarm('rotateSessionKey', { periodInMinutes: 60 })
+// Expire site breakage reports
+browserWrapper.createAlarm('clearExpiredBrokenSiteReportTimes', { periodInMinutes: 60 })
 
 browser.alarms.onAlarm.addListener(async alarmEvent => {
     // Warning: Awaiting in this function doesn't actually wait for the promise to resolve before unblocking the main thread.
@@ -461,6 +464,8 @@ browser.alarms.onAlarm.addListener(async alarmEvent => {
         await utils.resetSessionKey()
     } else if (alarmEvent.name === REFETCH_ALIAS_ALARM) {
         fetchAlias()
+    } else if (alarmEvent.name === 'clearExpiredBrokenSiteReportTimes') {
+        await clearExpiredBrokenSiteReportTimes()
     }
 })
 

shared/js/background/startup.js

@@ -3,6 +3,7 @@ import { NewTabTrackerStats } from './newtab-tracker-stats'
 import { TrackerStats } from './classes/tracker-stats.js'
 import httpsStorage from './storage/https'
 import tdsStorage from './storage/tds'
+import { clearExpiredBrokenSiteReportTimes } from './broken-site-report'
 const utils = require('./utils')
 const Companies = require('./companies')
 const experiment = require('./experiments')
@@ -69,6 +70,8 @@ export async function onStartup () {
         showContextMenuAction()
     }
 
+    await clearExpiredBrokenSiteReportTimes()
+
     if (resolveReadyPromise) {
         resolveReadyPromise()
         resolveReadyPromise = null