From dbefa0788716236e3fa3918023d07186f0eecbbf Mon Sep 17 00:00:00 2001 From: Karl Dimla Date: Mon, 10 Nov 2025 15:24:38 +0100 Subject: [PATCH 1/7] Add pixel definition for optout success rate --- .../pixels/personal_information_removal.json5 | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/PixelDefinitions/pixels/personal_information_removal.json5 b/PixelDefinitions/pixels/personal_information_removal.json5 index c5f4f5611e6c..f0901e130abd 100644 --- a/PixelDefinitions/pixels/personal_information_removal.json5 +++ b/PixelDefinitions/pixels/personal_information_removal.json5 @@ -129,5 +129,24 @@ "enum": ["on", "off", "unsupported"] } ] + }, + "dbp_databroker_custom_stats_optoutsubmit": { + "description": "Fired every 24 hours for every broker to measure opt-out request success rate.", + "owners": ["karlenDimla", "landomen"], + "triggers": ["other"], + "suffixes": ["form_factor"], + "parameters": [ + "appVersion", + { + "key": "data_broker", + "description": "The URL of the data broker that this opt-out attempt targets", + "type": "string" + }, + { + "key": "optout_submit_success_rate", + "description": "The success rate of how many opt-out jobs successfully requested within 24 hours of creation", + "type": "string" + } + ] } } From acd4f9a7a6016c3f4f84a267d4dcb7defe04e30d Mon Sep 17 00:00:00 2001 From: Karl Dimla Date: Mon, 10 Nov 2025 15:25:14 +0100 Subject: [PATCH 2/7] Add pixel and pixel send for broker optout success rate --- .../duckduckgo/pir/impl/pixels/PirPixel.kt | 5 ++++ .../pir/impl/pixels/PirPixelSender.kt | 25 +++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirPixel.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirPixel.kt index c849c89f07a4..1b70031b8118 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirPixel.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirPixel.kt @@ -135,6 +135,11 @@ enum class PirPixel( PIR_OPTOUT_SUBMIT_FAILURE( baseName = "dbp_optout_process_failure", types = setOf(Count), + ), + + PIR_BROKER_CUSTOM_STATS_OPTOUT_SUBMIT_SUCCESSRATE( + baseName = "dbp_databroker_custom_stats_optoutsubmit", + type = Count, ), ; constructor( diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirPixelSender.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirPixelSender.kt index 85ac7b2be51d..53ff572fabaa 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirPixelSender.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirPixelSender.kt @@ -18,6 +18,7 @@ package com.duckduckgo.pir.impl.pixels import com.duckduckgo.app.statistics.pixels.Pixel import com.duckduckgo.di.scopes.AppScope +import com.duckduckgo.pir.impl.pixels.PirPixel.PIR_BROKER_CUSTOM_STATS_OPTOUT_SUBMIT_SUCCESSRATE import com.duckduckgo.pir.impl.pixels.PirPixel.PIR_EMAIL_CONFIRMATION_ATTEMPT_FAILED import com.duckduckgo.pir.impl.pixels.PirPixel.PIR_EMAIL_CONFIRMATION_ATTEMPT_START import com.duckduckgo.pir.impl.pixels.PirPixel.PIR_EMAIL_CONFIRMATION_ATTEMPT_SUCCESS @@ -321,6 +322,17 @@ interface PirPixelSender { * Emits a pixel to signal that PIR encrypted database is unavailable. */ fun reportSecureStorageUnavailable() + + /** + * Emits a pixel containing the opt-out submit success rate for a broker for the last 24 hours + * + * @param brokerUrl url of the Broker for which the opt-out submit rate is for + * @param optOutSuccessRate opt out submit success rate for the past 24 hours + */ + fun reportBrokerCustomStateOptOutSubmitRate( + brokerUrl: String, + optOutSuccessRate: Double, + ) } @ContributesBinding(AppScope::class) @@ -603,6 +615,18 @@ class RealPirPixelSender @Inject constructor( fire(PIR_INTERNAL_SECURE_STORAGE_UNAVAILABLE) } + override fun reportBrokerCustomStateOptOutSubmitRate( + brokerUrl: String, + optOutSuccessRate: Double, + ) { + val params = mapOf( + PARAM_KEY_BROKER to brokerUrl, + PARAM_KEY_OPTOUT_SUBMIT_SUCCESS_RATE to optOutSuccessRate.toString(), + ) + + fire(PIR_BROKER_CUSTOM_STATS_OPTOUT_SUBMIT_SUCCESSRATE, params) + } + private fun fire( pixel: PirPixel, params: Map = emptyMap(), @@ -638,5 +662,6 @@ class RealPirPixelSender @Inject constructor( private const val PARAM_KEY_STAGE = "stage" private const val PARAM_KEY_PATTERN = "pattern" private const val PARAM_KEY_ACTION_TYPE = "action_type" + private const val PARAM_KEY_OPTOUT_SUBMIT_SUCCESS_RATE = "optout_submit_success_rate" } } From d3a244112552ba6fb3e3e63c0a009888c6ebdebf Mon Sep 17 00:00:00 2001 From: Karl Dimla Date: Mon, 10 Nov 2025 15:54:10 +0100 Subject: [PATCH 3/7] Add date created for ScanJobRecord and OptOutJobRecord --- .../15.json | 952 ++++++++++++++++++ .../pir/impl/models/scheduling/JobRecord.kt | 2 + .../duckduckgo/pir/impl/store/PirDatabase.kt | 2 +- .../pir/impl/store/PirSchedulingRepository.kt | 12 + .../impl/store/db/JobSchedulingEntities.kt | 2 + .../store/RealPirSchedulingRepositoryTest.kt | 10 + 6 files changed, 979 insertions(+), 1 deletion(-) create mode 100644 pir/pir-impl/schemas/com.duckduckgo.pir.impl.store.PirDatabase/15.json diff --git a/pir/pir-impl/schemas/com.duckduckgo.pir.impl.store.PirDatabase/15.json b/pir/pir-impl/schemas/com.duckduckgo.pir.impl.store.PirDatabase/15.json new file mode 100644 index 000000000000..1c354877cbb9 --- /dev/null +++ b/pir/pir-impl/schemas/com.duckduckgo.pir.impl.store.PirDatabase/15.json @@ -0,0 +1,952 @@ +{ + "formatVersion": 1, + "database": { + "version": 15, + "identityHash": "584648b8b3065521786fb33f44214f2e", + "entities": [ + { + "tableName": "pir_broker_json_etag", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`fileName` TEXT NOT NULL, `etag` TEXT NOT NULL, PRIMARY KEY(`fileName`))", + "fields": [ + { + "fieldPath": "fileName", + "columnName": "fileName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "etag", + "columnName": "etag", + "affinity": "TEXT", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "fileName" + ] + }, + "indices": [], + "foreignKeys": [] + }, + { + "tableName": "pir_broker_details", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`name` TEXT NOT NULL, `fileName` TEXT NOT NULL, `url` TEXT NOT NULL, `version` TEXT NOT NULL, `parent` TEXT, `addedDatetime` INTEGER NOT NULL, `removedAt` INTEGER NOT NULL, PRIMARY KEY(`name`))", + "fields": [ + { + "fieldPath": "name", + "columnName": "name", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "fileName", + "columnName": "fileName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "url", + "columnName": "url", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "version", + "columnName": "version", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "parent", + "columnName": "parent", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "addedDatetime", + "columnName": "addedDatetime", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "removedAt", + "columnName": "removedAt", + "affinity": "INTEGER", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "name" + ] + }, + "indices": [], + "foreignKeys": [] + }, + { + "tableName": "pir_broker_opt_out", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`brokerName` TEXT NOT NULL, `stepsJson` TEXT NOT NULL, `optOutUrl` TEXT, PRIMARY KEY(`brokerName`), FOREIGN KEY(`brokerName`) REFERENCES `pir_broker_details`(`name`) ON UPDATE NO ACTION ON DELETE CASCADE )", + "fields": [ + { + "fieldPath": "brokerName", + "columnName": "brokerName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "stepsJson", + "columnName": "stepsJson", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "optOutUrl", + "columnName": "optOutUrl", + "affinity": "TEXT", + "notNull": false + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "brokerName" + ] + }, + "indices": [], + "foreignKeys": [ + { + "table": "pir_broker_details", + "onDelete": "CASCADE", + "onUpdate": "NO ACTION", + "columns": [ + "brokerName" + ], + "referencedColumns": [ + "name" + ] + } + ] + }, + { + "tableName": "pir_broker_scan", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`brokerName` TEXT NOT NULL, `stepsJson` TEXT, PRIMARY KEY(`brokerName`), FOREIGN KEY(`brokerName`) REFERENCES `pir_broker_details`(`name`) ON UPDATE NO ACTION ON DELETE CASCADE )", + "fields": [ + { + "fieldPath": "brokerName", + "columnName": "brokerName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "stepsJson", + "columnName": "stepsJson", + "affinity": "TEXT", + "notNull": false + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "brokerName" + ] + }, + "indices": [], + "foreignKeys": [ + { + "table": "pir_broker_details", + "onDelete": "CASCADE", + "onUpdate": "NO ACTION", + "columns": [ + "brokerName" + ], + "referencedColumns": [ + "name" + ] + } + ] + }, + { + "tableName": "pir_broker_scheduling_config", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`brokerName` TEXT NOT NULL, `retryError` INTEGER NOT NULL, `confirmOptOutScan` INTEGER NOT NULL, `maintenanceScan` INTEGER NOT NULL, `maxAttempts` INTEGER, PRIMARY KEY(`brokerName`), FOREIGN KEY(`brokerName`) REFERENCES `pir_broker_details`(`name`) ON UPDATE NO ACTION ON DELETE CASCADE )", + "fields": [ + { + "fieldPath": "brokerName", + "columnName": "brokerName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "retryError", + "columnName": "retryError", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "confirmOptOutScan", + "columnName": "confirmOptOutScan", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "maintenanceScan", + "columnName": "maintenanceScan", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "maxAttempts", + "columnName": "maxAttempts", + "affinity": "INTEGER", + "notNull": false + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "brokerName" + ] + }, + "indices": [], + "foreignKeys": [ + { + "table": "pir_broker_details", + "onDelete": "CASCADE", + "onUpdate": "NO ACTION", + "columns": [ + "brokerName" + ], + "referencedColumns": [ + "name" + ] + } + ] + }, + { + "tableName": "pir_user_profile", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `birthYear` INTEGER NOT NULL, `phone` TEXT, `deprecated` INTEGER NOT NULL, `user_firstName` TEXT NOT NULL, `user_lastName` TEXT NOT NULL, `user_middleName` TEXT, `user_suffix` TEXT, `address_city` TEXT NOT NULL, `address_state` TEXT NOT NULL, `address_street` TEXT, `address_zip` TEXT)", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "birthYear", + "columnName": "birthYear", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "phone", + "columnName": "phone", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "deprecated", + "columnName": "deprecated", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "userName.firstName", + "columnName": "user_firstName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "userName.lastName", + "columnName": "user_lastName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "userName.middleName", + "columnName": "user_middleName", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "userName.suffix", + "columnName": "user_suffix", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "addresses.city", + "columnName": "address_city", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "addresses.state", + "columnName": "address_state", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "addresses.street", + "columnName": "address_street", + "affinity": "TEXT", + "notNull": false + }, + { + "fieldPath": "addresses.zip", + "columnName": "address_zip", + "affinity": "TEXT", + "notNull": false + } + ], + "primaryKey": { + "autoGenerate": true, + "columnNames": [ + "id" + ] + }, + "indices": [], + "foreignKeys": [] + }, + { + "tableName": "pir_events_log", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`eventTimeInMillis` INTEGER NOT NULL, `eventType` TEXT NOT NULL, PRIMARY KEY(`eventTimeInMillis`))", + "fields": [ + { + "fieldPath": "eventTimeInMillis", + "columnName": "eventTimeInMillis", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "eventType", + "columnName": "eventType", + "affinity": "TEXT", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "eventTimeInMillis" + ] + }, + "indices": [], + "foreignKeys": [] + }, + { + "tableName": "pir_broker_scan_log", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`eventTimeInMillis` INTEGER NOT NULL, `brokerName` TEXT NOT NULL, `eventType` TEXT NOT NULL, PRIMARY KEY(`eventTimeInMillis`))", + "fields": [ + { + "fieldPath": "eventTimeInMillis", + "columnName": "eventTimeInMillis", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "brokerName", + "columnName": "brokerName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "eventType", + "columnName": "eventType", + "affinity": "TEXT", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "eventTimeInMillis" + ] + }, + "indices": [], + "foreignKeys": [] + }, + { + "tableName": "pir_scan_complete_brokers", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`brokerName` TEXT NOT NULL, `profileQueryId` INTEGER NOT NULL, `startTimeInMillis` INTEGER NOT NULL, `endTimeInMillis` INTEGER NOT NULL, `isSuccess` INTEGER NOT NULL, PRIMARY KEY(`brokerName`, `profileQueryId`))", + "fields": [ + { + "fieldPath": "brokerName", + "columnName": "brokerName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "profileQueryId", + "columnName": "profileQueryId", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "startTimeInMillis", + "columnName": "startTimeInMillis", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "endTimeInMillis", + "columnName": "endTimeInMillis", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "isSuccess", + "columnName": "isSuccess", + "affinity": "INTEGER", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "brokerName", + "profileQueryId" + ] + }, + "indices": [], + "foreignKeys": [] + }, + { + "tableName": "pir_opt_out_complete_brokers", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `brokerName` TEXT NOT NULL, `extractedProfile` TEXT NOT NULL, `startTimeInMillis` INTEGER NOT NULL, `endTimeInMillis` INTEGER NOT NULL, `isSubmitSuccess` INTEGER NOT NULL)", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "brokerName", + "columnName": "brokerName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "extractedProfile", + "columnName": "extractedProfile", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "startTimeInMillis", + "columnName": "startTimeInMillis", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "endTimeInMillis", + "columnName": "endTimeInMillis", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "isSubmitSuccess", + "columnName": "isSubmitSuccess", + "affinity": "INTEGER", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": true, + "columnNames": [ + "id" + ] + }, + "indices": [], + "foreignKeys": [] + }, + { + "tableName": "pir_opt_out_action_log", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `brokerName` TEXT NOT NULL, `extractedProfile` TEXT NOT NULL, `completionTimeInMillis` INTEGER NOT NULL, `actionType` TEXT NOT NULL, `isError` INTEGER NOT NULL, `result` TEXT NOT NULL)", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "brokerName", + "columnName": "brokerName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "extractedProfile", + "columnName": "extractedProfile", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "completionTimeInMillis", + "columnName": "completionTimeInMillis", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "actionType", + "columnName": "actionType", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "isError", + "columnName": "isError", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "result", + "columnName": "result", + "affinity": "TEXT", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": true, + "columnNames": [ + "id" + ] + }, + "indices": [], + "foreignKeys": [] + }, + { + "tableName": "pir_extracted_profiles", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `profileQueryId` INTEGER NOT NULL, `brokerName` TEXT NOT NULL, `name` TEXT NOT NULL, `alternativeNames` TEXT NOT NULL, `age` TEXT NOT NULL, `addresses` TEXT NOT NULL, `phoneNumbers` TEXT NOT NULL, `relatives` TEXT NOT NULL, `profileUrl` TEXT NOT NULL, `identifier` TEXT NOT NULL, `reportId` TEXT NOT NULL, `email` TEXT NOT NULL, `fullName` TEXT NOT NULL, `dateAddedInMillis` INTEGER NOT NULL, `deprecated` INTEGER NOT NULL)", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "profileQueryId", + "columnName": "profileQueryId", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "brokerName", + "columnName": "brokerName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "name", + "columnName": "name", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "alternativeNames", + "columnName": "alternativeNames", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "age", + "columnName": "age", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "addresses", + "columnName": "addresses", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "phoneNumbers", + "columnName": "phoneNumbers", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "relatives", + "columnName": "relatives", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "profileUrl", + "columnName": "profileUrl", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "identifier", + "columnName": "identifier", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "reportId", + "columnName": "reportId", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "email", + "columnName": "email", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "fullName", + "columnName": "fullName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "dateAddedInMillis", + "columnName": "dateAddedInMillis", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "deprecated", + "columnName": "deprecated", + "affinity": "INTEGER", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": true, + "columnNames": [ + "id" + ] + }, + "indices": [ + { + "name": "index_pir_extracted_profiles_profileQueryId_brokerName_name_profileUrl_identifier", + "unique": true, + "columnNames": [ + "profileQueryId", + "brokerName", + "name", + "profileUrl", + "identifier" + ], + "orders": [], + "createSql": "CREATE UNIQUE INDEX IF NOT EXISTS `index_pir_extracted_profiles_profileQueryId_brokerName_name_profileUrl_identifier` ON `${TABLE_NAME}` (`profileQueryId`, `brokerName`, `name`, `profileUrl`, `identifier`)" + } + ], + "foreignKeys": [] + }, + { + "tableName": "pir_scan_job_record", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`brokerName` TEXT NOT NULL, `userProfileId` INTEGER NOT NULL, `status` TEXT NOT NULL, `lastScanDateInMillis` INTEGER, `deprecated` INTEGER NOT NULL, `dateCreatedInMillis` INTEGER NOT NULL, PRIMARY KEY(`brokerName`, `userProfileId`))", + "fields": [ + { + "fieldPath": "brokerName", + "columnName": "brokerName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "userProfileId", + "columnName": "userProfileId", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "status", + "columnName": "status", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "lastScanDateInMillis", + "columnName": "lastScanDateInMillis", + "affinity": "INTEGER", + "notNull": false + }, + { + "fieldPath": "deprecated", + "columnName": "deprecated", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "dateCreatedInMillis", + "columnName": "dateCreatedInMillis", + "affinity": "INTEGER", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "brokerName", + "userProfileId" + ] + }, + "indices": [], + "foreignKeys": [] + }, + { + "tableName": "pir_optout_job_record", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`extractedProfileId` INTEGER NOT NULL, `brokerName` TEXT NOT NULL, `userProfileId` INTEGER NOT NULL, `status` TEXT NOT NULL, `attemptCount` INTEGER NOT NULL, `lastOptOutAttemptDate` INTEGER, `optOutRequestedDate` INTEGER NOT NULL, `optOutRemovedDate` INTEGER NOT NULL, `deprecated` INTEGER NOT NULL, `dateCreatedInMillis` INTEGER NOT NULL, PRIMARY KEY(`extractedProfileId`))", + "fields": [ + { + "fieldPath": "extractedProfileId", + "columnName": "extractedProfileId", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "brokerName", + "columnName": "brokerName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "userProfileId", + "columnName": "userProfileId", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "status", + "columnName": "status", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "attemptCount", + "columnName": "attemptCount", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "lastOptOutAttemptDate", + "columnName": "lastOptOutAttemptDate", + "affinity": "INTEGER", + "notNull": false + }, + { + "fieldPath": "optOutRequestedDate", + "columnName": "optOutRequestedDate", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "optOutRemovedDate", + "columnName": "optOutRemovedDate", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "deprecated", + "columnName": "deprecated", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "dateCreatedInMillis", + "columnName": "dateCreatedInMillis", + "affinity": "INTEGER", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "extractedProfileId" + ] + }, + "indices": [], + "foreignKeys": [] + }, + { + "tableName": "pir_broker_mirror_sites", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`name` TEXT NOT NULL, `url` TEXT NOT NULL, `addedAt` INTEGER NOT NULL, `removedAt` INTEGER NOT NULL, `optOutUrl` TEXT NOT NULL, `parentSite` TEXT NOT NULL, PRIMARY KEY(`name`))", + "fields": [ + { + "fieldPath": "name", + "columnName": "name", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "url", + "columnName": "url", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "addedAt", + "columnName": "addedAt", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "removedAt", + "columnName": "removedAt", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "optOutUrl", + "columnName": "optOutUrl", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "parentSite", + "columnName": "parentSite", + "affinity": "TEXT", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "name" + ] + }, + "indices": [], + "foreignKeys": [] + }, + { + "tableName": "pir_email_confirmation_job_record", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`extractedProfileId` INTEGER NOT NULL, `brokerName` TEXT NOT NULL, `userProfileId` INTEGER NOT NULL, `email` TEXT NOT NULL, `attemptId` TEXT NOT NULL, `dateCreatedInMillis` INTEGER NOT NULL, `emailConfirmationLink` TEXT NOT NULL, `linkFetchAttemptCount` INTEGER NOT NULL, `lastLinkFetchDateInMillis` INTEGER NOT NULL, `jobAttemptCount` INTEGER NOT NULL, `lastJobAttemptDateInMillis` INTEGER NOT NULL, `lastJobAttemptActionId` TEXT NOT NULL, `deprecated` INTEGER NOT NULL, PRIMARY KEY(`extractedProfileId`))", + "fields": [ + { + "fieldPath": "extractedProfileId", + "columnName": "extractedProfileId", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "brokerName", + "columnName": "brokerName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "userProfileId", + "columnName": "userProfileId", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "email", + "columnName": "email", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "attemptId", + "columnName": "attemptId", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "dateCreatedInMillis", + "columnName": "dateCreatedInMillis", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "emailConfirmationLink", + "columnName": "emailConfirmationLink", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "linkFetchAttemptCount", + "columnName": "linkFetchAttemptCount", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "lastLinkFetchDateInMillis", + "columnName": "lastLinkFetchDateInMillis", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "jobAttemptCount", + "columnName": "jobAttemptCount", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "lastJobAttemptDateInMillis", + "columnName": "lastJobAttemptDateInMillis", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "lastJobAttemptActionId", + "columnName": "lastJobAttemptActionId", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "deprecated", + "columnName": "deprecated", + "affinity": "INTEGER", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "extractedProfileId" + ] + }, + "indices": [], + "foreignKeys": [] + }, + { + "tableName": "pir_email_confirmation_log", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`eventTimeInMillis` INTEGER NOT NULL, `eventType` TEXT NOT NULL, `value` TEXT NOT NULL, PRIMARY KEY(`eventTimeInMillis`))", + "fields": [ + { + "fieldPath": "eventTimeInMillis", + "columnName": "eventTimeInMillis", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "eventType", + "columnName": "eventType", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "value", + "columnName": "value", + "affinity": "TEXT", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "eventTimeInMillis" + ] + }, + "indices": [], + "foreignKeys": [] + } + ], + "views": [], + "setupQueries": [ + "CREATE TABLE IF NOT EXISTS room_master_table (id INTEGER PRIMARY KEY,identity_hash TEXT)", + "INSERT OR REPLACE INTO room_master_table (id,identity_hash) VALUES(42, '584648b8b3065521786fb33f44214f2e')" + ] + } +} \ No newline at end of file diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/models/scheduling/JobRecord.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/models/scheduling/JobRecord.kt index ae996deedd8e..95f7b6735ed8 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/models/scheduling/JobRecord.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/models/scheduling/JobRecord.kt @@ -55,6 +55,7 @@ sealed class JobRecord( val optOutRequestedDateInMillis: Long = 0L, val optOutRemovedDateInMillis: Long = 0L, val deprecated: Boolean = false, + val dateCreatedInMillis: Long = 0L, ) : JobRecord(brokerName, userProfileId) { enum class OptOutJobStatus { /** Opt-out has not been executed yet and should be executed when possible */ @@ -90,6 +91,7 @@ sealed class JobRecord( val status: ScanJobStatus = ScanJobStatus.NOT_EXECUTED, val lastScanDateInMillis: Long = 0L, val deprecated: Boolean = false, + val dateCreatedInMillis: Long = 0L, ) : JobRecord(brokerName, userProfileId) { enum class ScanJobStatus { /** Scan has not been executed yet and should be executed when possible */ diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/PirDatabase.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/PirDatabase.kt index ef026e0b66a5..071fbcc70255 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/PirDatabase.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/PirDatabase.kt @@ -53,7 +53,7 @@ import com.squareup.moshi.Types @Database( exportSchema = true, - version = 14, + version = 15, entities = [ BrokerJsonEtag::class, BrokerEntity::class, diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/PirSchedulingRepository.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/PirSchedulingRepository.kt index 311db495677e..4e32921e71d0 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/PirSchedulingRepository.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/PirSchedulingRepository.kt @@ -311,6 +311,7 @@ class RealPirSchedulingRepository @Inject constructor( status = ScanJobStatus.entries.find { it.name == this.status } ?: ScanJobStatus.ERROR, lastScanDateInMillis = this.lastScanDateInMillis ?: 0L, deprecated = this.deprecated, + dateCreatedInMillis = this.dateCreatedInMillis, ) private fun ScanJobRecord.toEntity(): ScanJobRecordEntity = @@ -320,6 +321,11 @@ class RealPirSchedulingRepository @Inject constructor( status = this.status.name, lastScanDateInMillis = this.lastScanDateInMillis, deprecated = this.deprecated, + dateCreatedInMillis = if (this.dateCreatedInMillis != 0L) { + this.dateCreatedInMillis + } else { + currentTimeProvider.currentTimeMillis() + }, ) private fun OptOutJobRecordEntity.toRecord(): OptOutJobRecord = @@ -333,6 +339,7 @@ class RealPirSchedulingRepository @Inject constructor( optOutRequestedDateInMillis = this.optOutRequestedDate, optOutRemovedDateInMillis = this.optOutRemovedDate, deprecated = this.deprecated, + dateCreatedInMillis = this.dateCreatedInMillis, ) private fun OptOutJobRecord.toEntity(): OptOutJobRecordEntity = @@ -346,6 +353,11 @@ class RealPirSchedulingRepository @Inject constructor( optOutRequestedDate = this.optOutRequestedDateInMillis, optOutRemovedDate = this.optOutRemovedDateInMillis, deprecated = this.deprecated, + dateCreatedInMillis = if (this.dateCreatedInMillis != 0L) { + this.dateCreatedInMillis + } else { + currentTimeProvider.currentTimeMillis() + }, ) private fun EmailConfirmationJobRecord.toEntity(): EmailConfirmationJobRecordEntity = diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/db/JobSchedulingEntities.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/db/JobSchedulingEntities.kt index adacba5232a0..62e3bc1092be 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/db/JobSchedulingEntities.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/db/JobSchedulingEntities.kt @@ -29,6 +29,7 @@ data class ScanJobRecordEntity( val status: String, val lastScanDateInMillis: Long? = null, val deprecated: Boolean = false, + val dateCreatedInMillis: Long, ) @Entity(tableName = "pir_optout_job_record") @@ -42,6 +43,7 @@ data class OptOutJobRecordEntity( val optOutRequestedDate: Long = 0L, val optOutRemovedDate: Long = 0L, val deprecated: Boolean = false, + val dateCreatedInMillis: Long, ) @Entity(tableName = "pir_email_confirmation_job_record") diff --git a/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/store/RealPirSchedulingRepositoryTest.kt b/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/store/RealPirSchedulingRepositoryTest.kt index ad93c1383d0e..301541d36c7b 100644 --- a/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/store/RealPirSchedulingRepositoryTest.kt +++ b/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/store/RealPirSchedulingRepositoryTest.kt @@ -79,6 +79,7 @@ class RealPirSchedulingRepositoryTest { userProfileId = 123L, status = ScanJobStatus.NOT_EXECUTED.name, lastScanDateInMillis = 1000L, + dateCreatedInMillis = 100L, ) private val deprecatedScanJobEntity = @@ -88,6 +89,7 @@ class RealPirSchedulingRepositoryTest { status = ScanJobStatus.MATCHES_FOUND.name, deprecated = true, lastScanDateInMillis = 2000L, + dateCreatedInMillis = 100L, ) private val validOptOutJobEntity = @@ -100,6 +102,7 @@ class RealPirSchedulingRepositoryTest { lastOptOutAttemptDate = 1000L, optOutRequestedDate = 2000L, optOutRemovedDate = 0L, + dateCreatedInMillis = 100L, ) private val deprecatedOptOutJobEntity = @@ -113,6 +116,7 @@ class RealPirSchedulingRepositoryTest { lastOptOutAttemptDate = 3000L, optOutRequestedDate = 4000L, optOutRemovedDate = 0L, + dateCreatedInMillis = 100L, ) private val scanJobRecord = @@ -243,6 +247,7 @@ class RealPirSchedulingRepositoryTest { userProfileId = 123L, status = "NOT_EXECUTED", lastScanDateInMillis = 1000L, + dateCreatedInMillis = 9000L, ), ) } @@ -268,12 +273,14 @@ class RealPirSchedulingRepositoryTest { userProfileId = 123L, status = "NOT_EXECUTED", lastScanDateInMillis = 1000L, + dateCreatedInMillis = 9000L, ), ScanJobRecordEntity( brokerName = "another-broker", userProfileId = 456L, status = "MATCHES_FOUND", lastScanDateInMillis = 5000L, + dateCreatedInMillis = 9000L, ), ), ) @@ -397,6 +404,7 @@ class RealPirSchedulingRepositoryTest { lastOptOutAttemptDate = 1000L, optOutRequestedDate = 2000L, optOutRemovedDate = 0L, + dateCreatedInMillis = 9000L, ), ) } @@ -430,6 +438,7 @@ class RealPirSchedulingRepositoryTest { lastOptOutAttemptDate = 1000L, optOutRequestedDate = 2000L, optOutRemovedDate = 0L, + dateCreatedInMillis = 9000L, ), OptOutJobRecordEntity( extractedProfileId = 999L, @@ -440,6 +449,7 @@ class RealPirSchedulingRepositoryTest { lastOptOutAttemptDate = 3000L, optOutRequestedDate = 4000L, optOutRemovedDate = 5000L, + dateCreatedInMillis = 9000L, ), ), ) From ea707df33fe64b2f3c79e702d77e3050fda6d7d4 Mon Sep 17 00:00:00 2001 From: Karl Dimla Date: Tue, 11 Nov 2025 14:59:17 +0100 Subject: [PATCH 4/7] Integrate and emit custom stats pixel --- pir/pir-impl/build.gradle | 1 + .../impl/pixels/OptOutSubmitRateCalculator.kt | 75 ++ .../pir/impl/pixels/PirCustomStatsWorker.kt | 96 ++ .../pir/impl/scan/PirScanScheduler.kt | 17 + .../duckduckgo/pir/impl/store/PirDataStore.kt | 10 + .../pir/impl/store/PirRepository.kt | 12 + .../pir/impl/store/PirSchedulingRepository.kt | 15 + .../pir/impl/store/db/JobSchedulingDao.kt | 3 + .../impl/pixels/PirCustomStatsWorkerTest.kt | 440 ++++++++++ .../RealOptOutSubmitRateCalculatorTest.kt | 828 ++++++++++++++++++ 10 files changed, 1497 insertions(+) create mode 100644 pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/OptOutSubmitRateCalculator.kt create mode 100644 pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirCustomStatsWorker.kt create mode 100644 pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/pixels/PirCustomStatsWorkerTest.kt create mode 100644 pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/pixels/RealOptOutSubmitRateCalculatorTest.kt diff --git a/pir/pir-impl/build.gradle b/pir/pir-impl/build.gradle index 58c4926233f8..3a9dbc204b75 100644 --- a/pir/pir-impl/build.gradle +++ b/pir/pir-impl/build.gradle @@ -70,6 +70,7 @@ dependencies { testImplementation AndroidX.test.ext.junit testImplementation Testing.robolectric testImplementation "androidx.lifecycle:lifecycle-runtime-testing:_" + testImplementation AndroidX.work.testing testImplementation(KotlinX.coroutines.test) { // https://github.com/Kotlin/kotlinx.coroutines/issues/2023 // conflicts with mockito due to direct inclusion of byte buddy diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/OptOutSubmitRateCalculator.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/OptOutSubmitRateCalculator.kt new file mode 100644 index 000000000000..23840aacd614 --- /dev/null +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/OptOutSubmitRateCalculator.kt @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2025 DuckDuckGo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.duckduckgo.pir.impl.pixels + +import com.duckduckgo.common.utils.DispatcherProvider +import com.duckduckgo.di.scopes.AppScope +import com.duckduckgo.pir.impl.models.scheduling.JobRecord.OptOutJobRecord.OptOutJobStatus.REMOVED +import com.duckduckgo.pir.impl.models.scheduling.JobRecord.OptOutJobRecord.OptOutJobStatus.REQUESTED +import com.duckduckgo.pir.impl.store.PirSchedulingRepository +import com.squareup.anvil.annotations.ContributesBinding +import kotlinx.coroutines.withContext +import java.util.concurrent.TimeUnit +import javax.inject.Inject +import kotlin.math.round + +interface OptOutSubmitRateCalculator { + /** + * Calculates the opt-out 24h submit rate for a given broker within the specified date range. + * + * @param brokerName name of the broker to calculate the opt-out submit rate for. + * @param startDateMs The opt-out records to include should be created on or after this date. Default is 0L (epoch). + * @param endDateMs tThe opt-out records to include should be created on or before this date. Default is 0L (epoch). + */ + suspend fun calculateOptOutSubmitRate( + brokerName: String, + startDateMs: Long = 0L, + endDateMs: Long, + ): Double? +} + +@ContributesBinding(AppScope::class) +class RealOptOutSubmitRateCalculator @Inject constructor( + private val dispatcherProvider: DispatcherProvider, + private val schedulingRepository: PirSchedulingRepository, +) : OptOutSubmitRateCalculator { + override suspend fun calculateOptOutSubmitRate( + brokerName: String, + startDateMs: Long, + endDateMs: Long, + ): Double? = withContext(dispatcherProvider.io()) { + // Get all opt out job records created within the given range for the specified broker + val recordsCreatedWithinRange = schedulingRepository.getAllValidOptOutJobRecordsForBroker(brokerName).filter { + it.brokerName == brokerName && it.dateCreatedInMillis in startDateMs..endDateMs + } + + // We don't need to calculate the rate if there are no records + if (recordsCreatedWithinRange.isEmpty()) return@withContext null + + // Filter the records to only include those that were requested within 24 hours of creation + val requestedRecordsWithinRange = recordsCreatedWithinRange.filter { + (it.status == REQUESTED || it.status == REMOVED) && it.optOutRequestedDateInMillis > it.dateCreatedInMillis && + it.optOutRequestedDateInMillis <= it.dateCreatedInMillis + TimeUnit.HOURS.toMillis( + 24, + ) + } + + val optOutSuccessRate = requestedRecordsWithinRange.size.toDouble() / recordsCreatedWithinRange.size.toDouble() + val roundedOptOutSuccessRate = round(optOutSuccessRate * 100) / 100 + return@withContext roundedOptOutSuccessRate + } +} diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirCustomStatsWorker.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirCustomStatsWorker.kt new file mode 100644 index 000000000000..ddda63414b51 --- /dev/null +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirCustomStatsWorker.kt @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2025 DuckDuckGo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.duckduckgo.pir.impl.pixels + +import android.content.Context +import androidx.work.CoroutineWorker +import androidx.work.WorkerParameters +import com.duckduckgo.anvil.annotations.ContributesWorker +import com.duckduckgo.common.utils.CurrentTimeProvider +import com.duckduckgo.di.scopes.AppScope +import com.duckduckgo.pir.impl.store.PirRepository +import java.util.concurrent.TimeUnit +import javax.inject.Inject +import kotlin.math.abs + +@ContributesWorker(AppScope::class) +class PirCustomStatsWorker( + context: Context, + workerParameters: WorkerParameters, +) : CoroutineWorker(context, workerParameters) { + @Inject + lateinit var optOutSubmitRateCalculator: OptOutSubmitRateCalculator + + @Inject + lateinit var pirRepository: PirRepository + + @Inject + lateinit var currentTimeProvider: CurrentTimeProvider + + @Inject + lateinit var pirPixelSender: PirPixelSender + + override suspend fun doWork(): Result { + val startDate = pirRepository.getCustomStatsPixelsLastSentMs() + val now = currentTimeProvider.currentTimeMillis() + + if (shouldFirePixel(startDate, now)) { + val endDate = now - TimeUnit.HOURS.toMillis(24) + val activeBrokers = pirRepository.getAllActiveBrokerObjects() + val hasUserProfiles = pirRepository.getAllUserProfileQueries().isNotEmpty() + + if (activeBrokers.isNotEmpty() && hasUserProfiles) { + activeBrokers.forEach { + val successRate = optOutSubmitRateCalculator.calculateOptOutSubmitRate( + it.name, + startDate, + endDate, + ) + + if (successRate != null) { + pirPixelSender.reportBrokerCustomStateOptOutSubmitRate( + brokerUrl = it.url, + optOutSuccessRate = successRate, + ) + } + } + + pirRepository.setCustomStatsPixelsLastSentMs(endDate) + } + } + + return Result.success() + } + + private fun shouldFirePixel( + startDate: Long, + now: Long, + ): Boolean { + return if (startDate == 0L) { + // IF first run, we emit the custom stats pixel + true + } else { + // Else we check if at least 24 hours have passed since last emission + val nowDiffFromStart = abs(now - startDate) + nowDiffFromStart > TimeUnit.HOURS.toMillis(24) + } + } + + companion object { + const val TAG_PIR_RECURRING_CUSTOM_STATS = "TAG_PIR_RECURRING_CUSTOM_STATS" + } +} diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/scan/PirScanScheduler.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/scan/PirScanScheduler.kt index ee14153d07a7..28f3a414ce38 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/scan/PirScanScheduler.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/scan/PirScanScheduler.kt @@ -24,6 +24,7 @@ import androidx.work.Data import androidx.work.ExistingPeriodicWorkPolicy import androidx.work.NetworkType import androidx.work.PeriodicWorkRequest +import androidx.work.PeriodicWorkRequestBuilder import androidx.work.WorkManager import androidx.work.multiprocess.RemoteListenableWorker import com.duckduckgo.app.di.AppCoroutineScope @@ -34,6 +35,8 @@ import com.duckduckgo.pir.impl.common.PirJobConstants.EMAIL_CONFIRMATION_INTERVA import com.duckduckgo.pir.impl.common.PirJobConstants.SCHEDULED_SCAN_INTERVAL_HOURS import com.duckduckgo.pir.impl.email.PirEmailConfirmationRemoteWorker import com.duckduckgo.pir.impl.email.PirEmailConfirmationRemoteWorker.Companion.TAG_EMAIL_CONFIRMATION +import com.duckduckgo.pir.impl.pixels.PirCustomStatsWorker +import com.duckduckgo.pir.impl.pixels.PirCustomStatsWorker.Companion.TAG_PIR_RECURRING_CUSTOM_STATS import com.duckduckgo.pir.impl.pixels.PirPixelSender import com.duckduckgo.pir.impl.scan.PirScheduledScanRemoteWorker.Companion.TAG_SCHEDULED_SCAN import com.duckduckgo.pir.impl.store.PirEventsRepository @@ -66,6 +69,7 @@ class RealPirScanScheduler @Inject constructor( schedulePirScans() scheduleEmailConfirmation() + scheduleRecurringPixelStats() } private fun schedulePirScans() { @@ -129,9 +133,22 @@ class RealPirScanScheduler @Inject constructor( ) } + private fun scheduleRecurringPixelStats() { + val periodicWorkRequest = PeriodicWorkRequestBuilder(5, TimeUnit.HOURS) + .addTag(TAG_PIR_RECURRING_CUSTOM_STATS) + .build() + + workManager.enqueueUniquePeriodicWork( + TAG_PIR_RECURRING_CUSTOM_STATS, + ExistingPeriodicWorkPolicy.UPDATE, + periodicWorkRequest, + ) + } + override fun cancelScheduledScans(context: Context) { workManager.cancelUniqueWork(TAG_SCHEDULED_SCAN) workManager.cancelUniqueWork(TAG_EMAIL_CONFIRMATION) + workManager.cancelUniqueWork(TAG_PIR_RECURRING_CUSTOM_STATS) context.stopService(Intent(context, PirRemoteWorkerService::class.java)) } diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/PirDataStore.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/PirDataStore.kt index 38e81fb7b60f..c2db7aec25f4 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/PirDataStore.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/PirDataStore.kt @@ -22,6 +22,7 @@ import com.duckduckgo.data.store.api.SharedPreferencesProvider interface PirDataStore { var mainConfigEtag: String? + var customStatsPixelsLastSentMs: Long } internal class RealPirDataStore( @@ -42,8 +43,17 @@ internal class RealPirDataStore( } } + override var customStatsPixelsLastSentMs: Long + get() = preferences.getLong(KEY_CUSTOM_STATS_PIXEL_LAST_SENT_MS, 0L) + set(value) { + preferences.edit { + putLong(KEY_CUSTOM_STATS_PIXEL_LAST_SENT_MS, value) + } + } + companion object { private const val FILENAME = "com.duckduckgo.pir.v1" private const val KEY_MAIN_ETAG = "KEY_MAIN_ETAG" + private const val KEY_CUSTOM_STATS_PIXEL_LAST_SENT_MS = "KEY_CUSTOM_STATS_PIXEL_LAST_SENT_MS" } } diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/PirRepository.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/PirRepository.kt index 248919fe9abc..9843d2724f6c 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/PirRepository.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/PirRepository.kt @@ -174,6 +174,10 @@ interface PirRepository { suspend fun deleteEmailData(emailData: List) + suspend fun getCustomStatsPixelsLastSentMs(): Long + + suspend fun setCustomStatsPixelsLastSentMs(timeMs: Long) + data class GeneratedEmailData( val emailAddress: String, val pattern: String, @@ -663,6 +667,14 @@ class RealPirRepository( return@withContext } + override suspend fun getCustomStatsPixelsLastSentMs(): Long = withContext(dispatcherProvider.io()) { + pirDataStore.customStatsPixelsLastSentMs + } + + override suspend fun setCustomStatsPixelsLastSentMs(timeMs: Long) = withContext(dispatcherProvider.io()) { + pirDataStore.customStatsPixelsLastSentMs = timeMs + } + private fun List.toRequest(): PirEmailConfirmationDataRequest = PirEmailConfirmationDataRequest( items = diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/PirSchedulingRepository.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/PirSchedulingRepository.kt index 4e32921e71d0..d5461c69f7ed 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/PirSchedulingRepository.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/PirSchedulingRepository.kt @@ -60,6 +60,11 @@ interface PirSchedulingRepository { */ suspend fun getAllValidOptOutJobRecords(): List + /** + * Returns all ScanJobRecord whose state is not INVALID for a specific broker + */ + suspend fun getAllValidOptOutJobRecordsForBroker(brokerName: String): List + /** * Returns a matching [OptOutJobRecord] whose state is not INVALID * @@ -181,6 +186,16 @@ class RealPirSchedulingRepository @Inject constructor( .orEmpty() } + override suspend fun getAllValidOptOutJobRecordsForBroker(brokerName: String): List = + withContext(dispatcherProvider.io()) { + return@withContext jobSchedulingDao() + ?.getAllOptOutJobRecordsForBroker(brokerName) + ?.map { record -> record.toRecord() } + // do not pick-up deprecated jobs as they belong to removed profiles + ?.filter { !it.deprecated } + .orEmpty() + } + override suspend fun saveScanJobRecord(scanJobRecord: ScanJobRecord) { withContext(dispatcherProvider.io()) { jobSchedulingDao()?.saveScanJobRecord(scanJobRecord.toEntity()) diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/db/JobSchedulingDao.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/db/JobSchedulingDao.kt index 30d040df1fde..e45f1bbad106 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/db/JobSchedulingDao.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/db/JobSchedulingDao.kt @@ -40,6 +40,9 @@ interface JobSchedulingDao { @Query("SELECT * FROM pir_optout_job_record ORDER BY attemptCount") fun getAllOptOutJobRecords(): List + @Query("SELECT * FROM pir_optout_job_record WHERE brokerName = :brokerName ORDER BY attemptCount") + fun getAllOptOutJobRecordsForBroker(brokerName: String): List + @Query("SELECT * FROM pir_optout_job_record ORDER BY attemptCount") fun getAllOptOutJobRecordsFlow(): Flow> diff --git a/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/pixels/PirCustomStatsWorkerTest.kt b/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/pixels/PirCustomStatsWorkerTest.kt new file mode 100644 index 000000000000..1b502ff2b595 --- /dev/null +++ b/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/pixels/PirCustomStatsWorkerTest.kt @@ -0,0 +1,440 @@ +/* + * Copyright (c) 2025 DuckDuckGo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.duckduckgo.pir.impl.pixels + +import android.content.Context +import androidx.work.testing.TestListenableWorkerBuilder +import com.duckduckgo.common.test.CoroutineTestRule +import com.duckduckgo.common.utils.CurrentTimeProvider +import com.duckduckgo.pir.impl.models.Broker +import com.duckduckgo.pir.impl.models.ProfileQuery +import com.duckduckgo.pir.impl.store.PirRepository +import kotlinx.coroutines.test.runTest +import org.junit.Before +import org.junit.Rule +import org.junit.Test +import org.mockito.kotlin.any +import org.mockito.kotlin.eq +import org.mockito.kotlin.mock +import org.mockito.kotlin.never +import org.mockito.kotlin.verify +import org.mockito.kotlin.whenever +import java.util.concurrent.TimeUnit + +class PirCustomStatsWorkerTest { + + @get:Rule + var coroutineRule = CoroutineTestRule() + + private val mockPirRepository: PirRepository = mock() + private val mockCurrentTimeProvider: CurrentTimeProvider = mock() + private val mockOptOutSubmitRateCalculator: OptOutSubmitRateCalculator = mock() + private val mockPirPixelSender: PirPixelSender = mock() + private val context: Context = mock() + + private lateinit var worker: PirCustomStatsWorker + + // Test data + // January 15, 2024 10:00:00 UTC + private val baseTime = 1705309200000L + private val oneHour = TimeUnit.HOURS.toMillis(1) + private val twentyFourHours = TimeUnit.HOURS.toMillis(24) + + private val testBroker1 = Broker( + name = "test-broker-1", + fileName = "test-broker-1.json", + url = "https://test-broker-1.com", + version = "1.0", + parent = null, + addedDatetime = baseTime, + removedAt = 0L, + ) + + private val testBroker2 = Broker( + name = "test-broker-2", + fileName = "test-broker-2.json", + url = "https://test-broker-2.com", + version = "1.0", + parent = null, + addedDatetime = baseTime, + removedAt = 0L, + ) + + private val testProfileQuery = ProfileQuery( + id = 1L, + firstName = "John", + lastName = "Doe", + city = "New York", + state = "NY", + addresses = emptyList(), + birthYear = 1990, + fullName = "John Doe", + age = 33, + deprecated = false, + ) + + @Before + fun setUp() { + worker = TestListenableWorkerBuilder + .from(context, PirCustomStatsWorker::class.java) + .build() + worker.pirRepository = mockPirRepository + worker.currentTimeProvider = mockCurrentTimeProvider + worker.optOutSubmitRateCalculator = mockOptOutSubmitRateCalculator + worker.pirPixelSender = mockPirPixelSender + } + + @Test + fun whenFirstRunThenShouldFirePixel() = runTest { + val now = baseTime + whenever(mockPirRepository.getCustomStatsPixelsLastSentMs()).thenReturn(0L) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker1)) + whenever(mockPirRepository.getAllUserProfileQueries()).thenReturn(listOf(testProfileQuery)) + whenever( + mockOptOutSubmitRateCalculator.calculateOptOutSubmitRate( + any(), + any(), + any(), + ), + ).thenReturn(0.5) + + worker.doWork() + + verify(mockPirPixelSender).reportBrokerCustomStateOptOutSubmitRate( + brokerUrl = testBroker1.url, + optOutSuccessRate = 0.5, + ) + verify(mockPirRepository).setCustomStatsPixelsLastSentMs(now - twentyFourHours) + } + + @Test + fun whenLessThan24HoursPassedThenShouldNotFirePixel() = runTest { + val startDate = baseTime + val now = baseTime + oneHour // Only 1 hour passed + + whenever(mockPirRepository.getCustomStatsPixelsLastSentMs()).thenReturn(startDate) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + + worker.doWork() + verify(mockPirRepository, never()).getAllActiveBrokerObjects() + verify(mockPirPixelSender, never()).reportBrokerCustomStateOptOutSubmitRate(any(), any()) + verify(mockPirRepository, never()).setCustomStatsPixelsLastSentMs(any()) + } + + @Test + fun whenMoreThan24HoursPassedThenShouldFirePixel() = runTest { + val startDate = baseTime + val now = baseTime + twentyFourHours + oneHour // 25 hours passed + + whenever(mockPirRepository.getCustomStatsPixelsLastSentMs()).thenReturn(startDate) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker1)) + whenever(mockPirRepository.getAllUserProfileQueries()).thenReturn(listOf(testProfileQuery)) + whenever( + mockOptOutSubmitRateCalculator.calculateOptOutSubmitRate( + any(), + any(), + any(), + ), + ).thenReturn(0.75) + + worker.doWork() + verify(mockPirPixelSender).reportBrokerCustomStateOptOutSubmitRate( + brokerUrl = testBroker1.url, + optOutSuccessRate = 0.75, + ) + verify(mockPirRepository).setCustomStatsPixelsLastSentMs(now - twentyFourHours) + } + + @Test + fun whenExactly24HoursPassedThenShouldNotFirePixel() = runTest { + val startDate = baseTime + val now = baseTime + twentyFourHours // Exactly 24 hours + + whenever(mockPirRepository.getCustomStatsPixelsLastSentMs()).thenReturn(startDate) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + + worker.doWork() + verify(mockPirRepository, never()).getAllActiveBrokerObjects() + verify(mockPirPixelSender, never()).reportBrokerCustomStateOptOutSubmitRate(any(), any()) + } + + @Test + fun whenNoActiveBrokersThenShouldNotFirePixel() = runTest { + val now = baseTime + whenever(mockPirRepository.getCustomStatsPixelsLastSentMs()).thenReturn(0L) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(emptyList()) + whenever(mockPirRepository.getAllUserProfileQueries()).thenReturn(listOf(testProfileQuery)) + + worker.doWork() + verify(mockOptOutSubmitRateCalculator, never()).calculateOptOutSubmitRate( + any(), + any(), + any(), + ) + verify(mockPirPixelSender, never()).reportBrokerCustomStateOptOutSubmitRate(any(), any()) + verify(mockPirRepository, never()).setCustomStatsPixelsLastSentMs(any()) + } + + @Test + fun whenNoUserProfilesThenShouldNotFirePixel() = runTest { + val now = baseTime + whenever(mockPirRepository.getCustomStatsPixelsLastSentMs()).thenReturn(0L) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker1)) + whenever(mockPirRepository.getAllUserProfileQueries()).thenReturn(emptyList()) + + worker.doWork() + verify(mockOptOutSubmitRateCalculator, never()).calculateOptOutSubmitRate( + any(), + any(), + any(), + ) + verify(mockPirPixelSender, never()).reportBrokerCustomStateOptOutSubmitRate(any(), any()) + verify(mockPirRepository, never()).setCustomStatsPixelsLastSentMs(any()) + } + + @Test + fun whenMultipleBrokersThenShouldFirePixelForEach() = runTest { + val now = baseTime + whenever(mockPirRepository.getCustomStatsPixelsLastSentMs()).thenReturn(0L) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn( + listOf( + testBroker1, + testBroker2, + ), + ) + whenever(mockPirRepository.getAllUserProfileQueries()).thenReturn(listOf(testProfileQuery)) + whenever( + mockOptOutSubmitRateCalculator.calculateOptOutSubmitRate( + testBroker1.name, + 0L, + now - twentyFourHours, + ), + ) + .thenReturn(0.5) + whenever( + mockOptOutSubmitRateCalculator.calculateOptOutSubmitRate( + testBroker2.name, + 0L, + now - twentyFourHours, + ), + ) + .thenReturn(0.8) + + worker.doWork() + verify(mockPirPixelSender).reportBrokerCustomStateOptOutSubmitRate( + brokerUrl = testBroker1.url, + optOutSuccessRate = 0.5, + ) + verify(mockPirPixelSender).reportBrokerCustomStateOptOutSubmitRate( + brokerUrl = testBroker2.url, + optOutSuccessRate = 0.8, + ) + verify(mockPirRepository).setCustomStatsPixelsLastSentMs(now - twentyFourHours) + } + + @Test + fun whenSuccessRateIsNullThenShouldNotFirePixelForThatBroker() = runTest { + val now = baseTime + whenever(mockPirRepository.getCustomStatsPixelsLastSentMs()).thenReturn(0L) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn( + listOf( + testBroker1, + testBroker2, + ), + ) + whenever(mockPirRepository.getAllUserProfileQueries()).thenReturn(listOf(testProfileQuery)) + whenever( + mockOptOutSubmitRateCalculator.calculateOptOutSubmitRate( + testBroker1.name, + 0L, + now - twentyFourHours, + ), + ) + .thenReturn(0.5) + whenever( + mockOptOutSubmitRateCalculator.calculateOptOutSubmitRate( + testBroker2.name, + 0L, + now - twentyFourHours, + ), + ) + .thenReturn(null) + + worker.doWork() + verify(mockPirPixelSender).reportBrokerCustomStateOptOutSubmitRate( + brokerUrl = testBroker1.url, + optOutSuccessRate = 0.5, + ) + verify(mockPirPixelSender, never()).reportBrokerCustomStateOptOutSubmitRate( + brokerUrl = eq(testBroker2.url), + optOutSuccessRate = any(), + ) + verify(mockPirRepository).setCustomStatsPixelsLastSentMs(now - twentyFourHours) + } + + @Test + fun whenAllSuccessRatesAreNullThenShouldNotFireAnyPixels() = runTest { + val now = baseTime + whenever(mockPirRepository.getCustomStatsPixelsLastSentMs()).thenReturn(0L) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker1)) + whenever(mockPirRepository.getAllUserProfileQueries()).thenReturn(listOf(testProfileQuery)) + whenever( + mockOptOutSubmitRateCalculator.calculateOptOutSubmitRate( + any(), + any(), + any(), + ), + ).thenReturn(null) + + worker.doWork() + verify(mockPirPixelSender, never()).reportBrokerCustomStateOptOutSubmitRate(any(), any()) + verify(mockPirRepository).setCustomStatsPixelsLastSentMs(now - twentyFourHours) + } + + @Test + fun whenShouldFirePixelThenUsesCorrectDateRange() = runTest { + val startDate = baseTime + val now = baseTime + twentyFourHours + oneHour + val expectedEndDate = now - twentyFourHours + + whenever(mockPirRepository.getCustomStatsPixelsLastSentMs()).thenReturn(startDate) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker1)) + whenever(mockPirRepository.getAllUserProfileQueries()).thenReturn(listOf(testProfileQuery)) + whenever( + mockOptOutSubmitRateCalculator.calculateOptOutSubmitRate( + any(), + any(), + any(), + ), + ).thenReturn(0.5) + + worker.doWork() + + verify(mockOptOutSubmitRateCalculator).calculateOptOutSubmitRate( + brokerName = testBroker1.name, + startDateMs = startDate, + endDateMs = expectedEndDate, + ) + verify(mockPirRepository).setCustomStatsPixelsLastSentMs(expectedEndDate) + } + + @Test + fun whenMultipleBrokersWithMixedSuccessRatesThenFiresPixelsForNonNullRates() = runTest { + val now = baseTime + whenever(mockPirRepository.getCustomStatsPixelsLastSentMs()).thenReturn(0L) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn( + listOf( + testBroker1, + testBroker2, + ), + ) + whenever(mockPirRepository.getAllUserProfileQueries()).thenReturn(listOf(testProfileQuery)) + whenever( + mockOptOutSubmitRateCalculator.calculateOptOutSubmitRate( + testBroker1.name, + 0L, + now - twentyFourHours, + ), + ) + .thenReturn(null) + whenever( + mockOptOutSubmitRateCalculator.calculateOptOutSubmitRate( + testBroker2.name, + 0L, + now - twentyFourHours, + ), + ) + .thenReturn(0.9) + + worker.doWork() + verify(mockPirPixelSender, never()).reportBrokerCustomStateOptOutSubmitRate( + brokerUrl = eq(testBroker1.url), + optOutSuccessRate = any(), + ) + verify(mockPirPixelSender).reportBrokerCustomStateOptOutSubmitRate( + brokerUrl = testBroker2.url, + optOutSuccessRate = 0.9, + ) + verify(mockPirRepository).setCustomStatsPixelsLastSentMs(now - twentyFourHours) + } + + @Test + fun whenShouldFirePixelButNoBrokersAndNoProfilesThenReturnsSuccess() = runTest { + val now = baseTime + whenever(mockPirRepository.getCustomStatsPixelsLastSentMs()).thenReturn(0L) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(emptyList()) + whenever(mockPirRepository.getAllUserProfileQueries()).thenReturn(emptyList()) + + worker.doWork() + + verify(mockOptOutSubmitRateCalculator, never()).calculateOptOutSubmitRate( + any(), + any(), + any(), + ) + verify(mockPirPixelSender, never()).reportBrokerCustomStateOptOutSubmitRate(any(), any()) + verify(mockPirRepository, never()).setCustomStatsPixelsLastSentMs(any()) + } + + @Test + fun whenShouldFirePixelButNoBrokersWithProfilesThenReturnsSuccess() = runTest { + val now = baseTime + whenever(mockPirRepository.getCustomStatsPixelsLastSentMs()).thenReturn(0L) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(emptyList()) + whenever(mockPirRepository.getAllUserProfileQueries()).thenReturn(listOf(testProfileQuery)) + + worker.doWork() + + verify(mockOptOutSubmitRateCalculator, never()).calculateOptOutSubmitRate( + any(), + any(), + any(), + ) + verify(mockPirPixelSender, never()).reportBrokerCustomStateOptOutSubmitRate(any(), any()) + verify(mockPirRepository, never()).setCustomStatsPixelsLastSentMs(any()) + } + + @Test + fun whenShouldFirePixelButNoProfilesWithBrokersThenReturnsSuccess() = runTest { + val now = baseTime + whenever(mockPirRepository.getCustomStatsPixelsLastSentMs()).thenReturn(0L) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker1)) + whenever(mockPirRepository.getAllUserProfileQueries()).thenReturn(emptyList()) + + worker.doWork() + + verify(mockOptOutSubmitRateCalculator, never()).calculateOptOutSubmitRate( + any(), + any(), + any(), + ) + verify(mockPirPixelSender, never()).reportBrokerCustomStateOptOutSubmitRate(any(), any()) + verify(mockPirRepository, never()).setCustomStatsPixelsLastSentMs(any()) + } +} diff --git a/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/pixels/RealOptOutSubmitRateCalculatorTest.kt b/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/pixels/RealOptOutSubmitRateCalculatorTest.kt new file mode 100644 index 000000000000..f51f6fa8523b --- /dev/null +++ b/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/pixels/RealOptOutSubmitRateCalculatorTest.kt @@ -0,0 +1,828 @@ +/* + * Copyright (c) 2025 DuckDuckGo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.duckduckgo.pir.impl.pixels + +import com.duckduckgo.common.test.CoroutineTestRule +import com.duckduckgo.pir.impl.models.scheduling.JobRecord.OptOutJobRecord +import com.duckduckgo.pir.impl.models.scheduling.JobRecord.OptOutJobRecord.OptOutJobStatus +import com.duckduckgo.pir.impl.store.PirSchedulingRepository +import kotlinx.coroutines.test.runTest +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNull +import org.junit.Before +import org.junit.Rule +import org.junit.Test +import org.mockito.kotlin.mock +import org.mockito.kotlin.whenever +import java.util.concurrent.TimeUnit + +class RealOptOutSubmitRateCalculatorTest { + + @get:Rule + val coroutineRule = CoroutineTestRule() + + private lateinit var testee: RealOptOutSubmitRateCalculator + + private val mockSchedulingRepository: PirSchedulingRepository = mock() + + @Before + fun setUp() { + testee = RealOptOutSubmitRateCalculator( + dispatcherProvider = coroutineRule.testDispatcherProvider, + schedulingRepository = mockSchedulingRepository, + ) + } + + // Test data + private val testBrokerName = "test-broker" + private val testBrokerName2 = "test-broker-2" + + // January 15, 2024 10:00:00 UTC + private val baseTime = 1705309200000L + private val oneHour = TimeUnit.HOURS.toMillis(1) + private val oneDay = TimeUnit.DAYS.toMillis(1) + private val twentyFourHours = TimeUnit.HOURS.toMillis(24) + + @Test + fun whenNoRecordsInDateRangeThenReturnNull() = runTest { + val startDate = baseTime + val endDate = baseTime + oneDay + + whenever(mockSchedulingRepository.getAllValidOptOutJobRecordsForBroker(testBrokerName)) + .thenReturn(emptyList()) + + val result = testee.calculateOptOutSubmitRate(testBrokerName, startDate, endDate) + + assertNull(result) + } + + @Test + fun whenRecordsExistButNoneInDateRangeThenReturnNull() = runTest { + val startDate = baseTime + val endDate = baseTime + oneDay + + val recordBeforeRange = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBrokerName, + dateCreatedInMillis = baseTime - oneDay, + ) + val recordAfterRange = createOptOutJobRecord( + extractedProfileId = 2L, + brokerName = testBrokerName, + dateCreatedInMillis = baseTime + oneDay + oneHour, + ) + + whenever(mockSchedulingRepository.getAllValidOptOutJobRecordsForBroker(testBrokerName)) + .thenReturn(listOf(recordBeforeRange, recordAfterRange)) + + val result = testee.calculateOptOutSubmitRate(testBrokerName, startDate, endDate) + + assertNull(result) + } + + @Test + fun whenRecordsInDateRangeButNoneRequestedThenReturnZero() = runTest { + val startDate = baseTime + val endDate = baseTime + oneDay + + val record1 = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBrokerName, + status = OptOutJobStatus.NOT_EXECUTED, + dateCreatedInMillis = baseTime + oneHour, + ) + val record2 = createOptOutJobRecord( + extractedProfileId = 2L, + brokerName = testBrokerName, + status = OptOutJobStatus.ERROR, + dateCreatedInMillis = baseTime + 2 * oneHour, + ) + + whenever(mockSchedulingRepository.getAllValidOptOutJobRecordsForBroker(testBrokerName)) + .thenReturn(listOf(record1, record2)) + + val result = testee.calculateOptOutSubmitRate(testBrokerName, startDate, endDate) + + assertEquals(0.0, result!!, 0.0) + } + + @Test + fun whenAllRecordsInDateRangeAreRequestedThenReturnOne() = runTest { + val startDate = baseTime + val endDate = baseTime + oneDay + val dateCreated = baseTime + oneHour + + val record1 = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = dateCreated, + optOutRequestedDateInMillis = dateCreated + oneHour, // Within 24 hours + ) + val record2 = createOptOutJobRecord( + extractedProfileId = 2L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = dateCreated + 2 * oneHour, + optOutRequestedDateInMillis = dateCreated + 3 * oneHour, // Within 24 hours + ) + + whenever(mockSchedulingRepository.getAllValidOptOutJobRecordsForBroker(testBrokerName)) + .thenReturn(listOf(record1, record2)) + + val result = testee.calculateOptOutSubmitRate(testBrokerName, startDate, endDate) + + assertEquals(1.0, result!!, 0.0) + } + + @Test + fun whenHalfRecordsInDateRangeAreRequestedThenReturnHalf() = runTest { + val startDate = baseTime + val endDate = baseTime + oneDay + val dateCreated = baseTime + oneHour + + val requestedRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = dateCreated, + optOutRequestedDateInMillis = dateCreated + oneHour, // Within 24 hours + ) + val notExecutedRecord = createOptOutJobRecord( + extractedProfileId = 2L, + brokerName = testBrokerName, + status = OptOutJobStatus.NOT_EXECUTED, + dateCreatedInMillis = dateCreated + 2 * oneHour, + ) + + whenever(mockSchedulingRepository.getAllValidOptOutJobRecordsForBroker(testBrokerName)) + .thenReturn(listOf(requestedRecord, notExecutedRecord)) + + val result = testee.calculateOptOutSubmitRate(testBrokerName, startDate, endDate) + + assertEquals(0.5, result!!, 0.0) + } + + @Test + fun whenRequestedRecordOutside24HourWindowThenNotCounted() = runTest { + val startDate = baseTime + val endDate = baseTime + oneDay + val dateCreated = baseTime + oneHour + + val requestedWithinWindow = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = dateCreated, + optOutRequestedDateInMillis = dateCreated + oneHour, // Within 24 hours + ) + val requestedOutsideWindow = createOptOutJobRecord( + extractedProfileId = 2L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = dateCreated + 2 * oneHour, + optOutRequestedDateInMillis = dateCreated + 2 * oneHour + twentyFourHours + oneHour, // Outside 24 hours + ) + + whenever(mockSchedulingRepository.getAllValidOptOutJobRecordsForBroker(testBrokerName)) + .thenReturn(listOf(requestedWithinWindow, requestedOutsideWindow)) + + val result = testee.calculateOptOutSubmitRate(testBrokerName, startDate, endDate) + + assertEquals(0.5, result!!, 0.0) + } + + @Test + fun whenRequestedRecordExactlyAt24HourWindowThenCounted() = runTest { + val startDate = baseTime + val endDate = baseTime + oneDay + val dateCreated = baseTime + oneHour + + val requestedAt24Hours = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = dateCreated, + optOutRequestedDateInMillis = dateCreated + twentyFourHours, // Exactly 24 hours + ) + val notExecutedRecord = createOptOutJobRecord( + extractedProfileId = 2L, + brokerName = testBrokerName, + status = OptOutJobStatus.NOT_EXECUTED, + dateCreatedInMillis = dateCreated + 2 * oneHour, + ) + + whenever(mockSchedulingRepository.getAllValidOptOutJobRecordsForBroker(testBrokerName)) + .thenReturn(listOf(requestedAt24Hours, notExecutedRecord)) + + val result = testee.calculateOptOutSubmitRate(testBrokerName, startDate, endDate) + + assertEquals(0.5, result!!, 0.0) + } + + @Test + fun whenRecordsFromDifferentBrokersThenOnlyCountMatchingBroker() = runTest { + val startDate = baseTime + val endDate = baseTime + oneDay + val dateCreated = baseTime + oneHour + + val recordForTestBroker = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = dateCreated, + optOutRequestedDateInMillis = dateCreated + oneHour, + ) + val recordForOtherBroker = createOptOutJobRecord( + extractedProfileId = 2L, + brokerName = testBrokerName2, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = dateCreated + 2 * oneHour, + optOutRequestedDateInMillis = dateCreated + 3 * oneHour, + ) + + whenever(mockSchedulingRepository.getAllValidOptOutJobRecordsForBroker(testBrokerName)) + .thenReturn(listOf(recordForTestBroker)) + whenever(mockSchedulingRepository.getAllValidOptOutJobRecordsForBroker(testBrokerName2)) + .thenReturn(listOf(recordForOtherBroker)) + + val result = testee.calculateOptOutSubmitRate(testBrokerName, startDate, endDate) + + assertEquals(1.0, result!!, 0.0) + } + + @Test + fun whenStartDateIsZeroThenUseDefaultStartDate() = runTest { + val endDate = baseTime + oneDay + val dateCreated = baseTime + oneHour + + val record = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = dateCreated, + optOutRequestedDateInMillis = dateCreated + oneHour, + ) + + whenever(mockSchedulingRepository.getAllValidOptOutJobRecordsForBroker(testBrokerName)) + .thenReturn(listOf(record)) + + val result = testee.calculateOptOutSubmitRate(testBrokerName, endDateMs = endDate) + + assertEquals(1.0, result!!, 0.0) + } + + @Test + fun whenRecordAtStartDateBoundaryThenIncluded() = runTest { + val startDate = baseTime + val endDate = baseTime + oneDay + + val recordAtStart = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = startDate, // Exactly at start + optOutRequestedDateInMillis = startDate + oneHour, + ) + + whenever(mockSchedulingRepository.getAllValidOptOutJobRecordsForBroker(testBrokerName)) + .thenReturn(listOf(recordAtStart)) + + val result = testee.calculateOptOutSubmitRate(testBrokerName, startDate, endDate) + + assertEquals(1.0, result!!, 0.0) + } + + @Test + fun whenRecordAtEndDateBoundaryThenIncluded() = runTest { + val startDate = baseTime + val endDate = baseTime + oneDay + + val recordAtEnd = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = endDate, // Exactly at end + optOutRequestedDateInMillis = endDate + oneHour, + ) + + whenever(mockSchedulingRepository.getAllValidOptOutJobRecordsForBroker(testBrokerName)) + .thenReturn(listOf(recordAtEnd)) + + val result = testee.calculateOptOutSubmitRate(testBrokerName, startDate, endDate) + + assertEquals(1.0, result!!, 0.0) + } + + @Test + fun whenMultipleStatusesThenOnlyRequestedCounted() = runTest { + val startDate = baseTime + val endDate = baseTime + oneDay + val dateCreated = baseTime + oneHour + + val requested = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = dateCreated, + optOutRequestedDateInMillis = dateCreated + oneHour, + ) + val removed = createOptOutJobRecord( + extractedProfileId = 2L, + brokerName = testBrokerName, + status = OptOutJobStatus.REMOVED, + dateCreatedInMillis = dateCreated + 2 * oneHour, + ) + val error = createOptOutJobRecord( + extractedProfileId = 3L, + brokerName = testBrokerName, + status = OptOutJobStatus.ERROR, + dateCreatedInMillis = dateCreated + 3 * oneHour, + ) + val pendingEmail = createOptOutJobRecord( + extractedProfileId = 4L, + brokerName = testBrokerName, + status = OptOutJobStatus.PENDING_EMAIL_CONFIRMATION, + dateCreatedInMillis = dateCreated + 4 * oneHour, + ) + + whenever(mockSchedulingRepository.getAllValidOptOutJobRecordsForBroker(testBrokerName)) + .thenReturn(listOf(requested, removed, error, pendingEmail)) + + val result = testee.calculateOptOutSubmitRate(testBrokerName, startDate, endDate) + + assertEquals(0.25, result!!, 0.0) + } + + @Test + fun whenComplexScenarioThenCalculateCorrectly() = runTest { + val startDate = baseTime + val endDate = baseTime + oneDay + val dateCreated = baseTime + oneHour + + // 5 records in range + val requested1 = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = dateCreated, + optOutRequestedDateInMillis = dateCreated + oneHour, // Within 24h + ) + val requested2 = createOptOutJobRecord( + extractedProfileId = 2L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = dateCreated + 2 * oneHour, + optOutRequestedDateInMillis = dateCreated + 2 * oneHour + twentyFourHours + oneHour, // Outside 24h + ) + val requested3 = createOptOutJobRecord( + extractedProfileId = 3L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = dateCreated + 3 * oneHour, + optOutRequestedDateInMillis = dateCreated + 3 * oneHour + twentyFourHours, // Exactly 24h + ) + val notExecuted = createOptOutJobRecord( + extractedProfileId = 4L, + brokerName = testBrokerName, + status = OptOutJobStatus.NOT_EXECUTED, + dateCreatedInMillis = dateCreated + 4 * oneHour, + ) + val error = createOptOutJobRecord( + extractedProfileId = 5L, + brokerName = testBrokerName, + status = OptOutJobStatus.ERROR, + dateCreatedInMillis = dateCreated + 5 * oneHour, + ) + + whenever(mockSchedulingRepository.getAllValidOptOutJobRecordsForBroker(testBrokerName)) + .thenReturn(listOf(requested1, requested2, requested3, notExecuted, error)) + + val result = testee.calculateOptOutSubmitRate(testBrokerName, startDate, endDate) + + // Only requested1 and requested3 count (2 out of 5) + assertEquals(0.4, result!!, 0.0) + } + + @Test + fun whenOptOutRequestedDateEqualsDateCreatedThenNotCounted() = runTest { + val startDate = baseTime + val endDate = baseTime + oneDay + val dateCreated = baseTime + oneHour + + val requestedAtSameTime = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = dateCreated, + optOutRequestedDateInMillis = dateCreated, // Exactly equal (should not be counted) + ) + val validRequested = createOptOutJobRecord( + extractedProfileId = 2L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = dateCreated + 2 * oneHour, + optOutRequestedDateInMillis = dateCreated + 2 * oneHour + oneHour, // After creation + ) + + whenever(mockSchedulingRepository.getAllValidOptOutJobRecordsForBroker(testBrokerName)) + .thenReturn(listOf(requestedAtSameTime, validRequested)) + + val result = testee.calculateOptOutSubmitRate(testBrokerName, startDate, endDate) + + // Only validRequested counts (1 out of 2) + assertEquals(0.5, result!!, 0.0) + } + + @Test + fun whenOptOutRequestedDateBeforeDateCreatedThenNotCounted() = runTest { + val startDate = baseTime + val endDate = baseTime + oneDay + val dateCreated = baseTime + oneHour + + val requestedBeforeCreation = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = dateCreated, + optOutRequestedDateInMillis = dateCreated - oneHour, // Before creation (should not be counted) + ) + val validRequested = createOptOutJobRecord( + extractedProfileId = 2L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = dateCreated + 2 * oneHour, + optOutRequestedDateInMillis = dateCreated + 3 * oneHour, // After creation + ) + + whenever(mockSchedulingRepository.getAllValidOptOutJobRecordsForBroker(testBrokerName)) + .thenReturn(listOf(requestedBeforeCreation, validRequested)) + + val result = testee.calculateOptOutSubmitRate(testBrokerName, startDate, endDate) + + // Only validRequested counts (1 out of 2) + assertEquals(0.5, result!!, 0.0) + } + + @Test + fun whenOptOutRequestedDateJustAfterDateCreatedThenCounted() = runTest { + val startDate = baseTime + val endDate = baseTime + oneDay + val dateCreated = baseTime + oneHour + val oneMillisecond = 1L + + val requestedJustAfter = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = dateCreated, + optOutRequestedDateInMillis = dateCreated + oneMillisecond, // Just 1ms after (should be counted) + ) + + whenever(mockSchedulingRepository.getAllValidOptOutJobRecordsForBroker(testBrokerName)) + .thenReturn(listOf(requestedJustAfter)) + + val result = testee.calculateOptOutSubmitRate(testBrokerName, startDate, endDate) + + assertEquals(1.0, result!!, 0.0) + } + + @Test + fun whenOptOutRequestedDateJustBefore24HourLimitThenCounted() = runTest { + val startDate = baseTime + val endDate = baseTime + oneDay + val dateCreated = baseTime + oneHour + val oneMillisecond = 1L + + val requestedJustBefore24h = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = dateCreated, + optOutRequestedDateInMillis = dateCreated + twentyFourHours - oneMillisecond, // Just before 24h limit + ) + + whenever(mockSchedulingRepository.getAllValidOptOutJobRecordsForBroker(testBrokerName)) + .thenReturn(listOf(requestedJustBefore24h)) + + val result = testee.calculateOptOutSubmitRate(testBrokerName, startDate, endDate) + + assertEquals(1.0, result!!, 0.0) + } + + @Test + fun whenOptOutRequestedDateJustAfter24HourLimitThenNotCounted() = runTest { + val startDate = baseTime + val endDate = baseTime + oneDay + val dateCreated = baseTime + oneHour + val oneMillisecond = 1L + + val requestedJustAfter24h = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = dateCreated, + optOutRequestedDateInMillis = dateCreated + twentyFourHours + oneMillisecond, // Just after 24h limit + ) + val notExecuted = createOptOutJobRecord( + extractedProfileId = 2L, + brokerName = testBrokerName, + status = OptOutJobStatus.NOT_EXECUTED, + dateCreatedInMillis = dateCreated + 2 * oneHour, + ) + + whenever(mockSchedulingRepository.getAllValidOptOutJobRecordsForBroker(testBrokerName)) + .thenReturn(listOf(requestedJustAfter24h, notExecuted)) + + val result = testee.calculateOptOutSubmitRate(testBrokerName, startDate, endDate) + + // requestedJustAfter24h doesn't count, so 0 out of 2 + assertEquals(0.0, result!!, 0.0) + } + + @Test + fun whenResultNeedsRoundingThenRoundsToTwoDecimalPlaces() = runTest { + val startDate = baseTime + val endDate = baseTime + oneDay + val dateCreated = baseTime + oneHour + + // 1 out of 3 = 0.333... should round to 0.33 + val requested = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = dateCreated, + optOutRequestedDateInMillis = dateCreated + oneHour, + ) + val notExecuted1 = createOptOutJobRecord( + extractedProfileId = 2L, + brokerName = testBrokerName, + status = OptOutJobStatus.NOT_EXECUTED, + dateCreatedInMillis = dateCreated + 2 * oneHour, + ) + val notExecuted2 = createOptOutJobRecord( + extractedProfileId = 3L, + brokerName = testBrokerName, + status = OptOutJobStatus.NOT_EXECUTED, + dateCreatedInMillis = dateCreated + 3 * oneHour, + ) + + whenever(mockSchedulingRepository.getAllValidOptOutJobRecordsForBroker(testBrokerName)) + .thenReturn(listOf(requested, notExecuted1, notExecuted2)) + + val result = testee.calculateOptOutSubmitRate(testBrokerName, startDate, endDate) + + // 1/3 = 0.333... rounded to 0.33 + assertEquals(0.33, result!!, 0.0) + } + + @Test + fun whenResultNeedsRoundingUpThenRoundsCorrectly() = runTest { + val startDate = baseTime + val endDate = baseTime + oneDay + val dateCreated = baseTime + oneHour + + // 2 out of 3 = 0.666... should round to 0.67 + val requested1 = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = dateCreated, + optOutRequestedDateInMillis = dateCreated + oneHour, + ) + val requested2 = createOptOutJobRecord( + extractedProfileId = 2L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = dateCreated + 2 * oneHour, + optOutRequestedDateInMillis = dateCreated + 3 * oneHour, + ) + val notExecuted = createOptOutJobRecord( + extractedProfileId = 3L, + brokerName = testBrokerName, + status = OptOutJobStatus.NOT_EXECUTED, + dateCreatedInMillis = dateCreated + 4 * oneHour, + ) + + whenever(mockSchedulingRepository.getAllValidOptOutJobRecordsForBroker(testBrokerName)) + .thenReturn(listOf(requested1, requested2, notExecuted)) + + val result = testee.calculateOptOutSubmitRate(testBrokerName, startDate, endDate) + + // 2/3 = 0.666... rounded to 0.67 + assertEquals(0.67, result!!, 0.0) + } + + @Test + fun whenRepositoryReturnsWrongBrokerNameThenExcluded() = runTest { + val startDate = baseTime + val endDate = baseTime + oneDay + val dateCreated = baseTime + oneHour + + // Repository might return records with wrong broker name (should be filtered out) + val recordWithWrongBroker = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBrokerName2, // Wrong broker + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = dateCreated, + optOutRequestedDateInMillis = dateCreated + oneHour, + ) + val recordWithCorrectBroker = createOptOutJobRecord( + extractedProfileId = 2L, + brokerName = testBrokerName, // Correct broker + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = dateCreated + 2 * oneHour, + optOutRequestedDateInMillis = dateCreated + 3 * oneHour, + ) + + // Repository returns both, but only correct broker should be counted + whenever(mockSchedulingRepository.getAllValidOptOutJobRecordsForBroker(testBrokerName)) + .thenReturn(listOf(recordWithWrongBroker, recordWithCorrectBroker)) + + val result = testee.calculateOptOutSubmitRate(testBrokerName, startDate, endDate) + + // Only recordWithCorrectBroker counts (1 out of 1 after filtering) + assertEquals(1.0, result!!, 0.0) + } + + @Test + fun whenSingleRecordRequestedThenReturnOne() = runTest { + val startDate = baseTime + val endDate = baseTime + oneDay + val dateCreated = baseTime + oneHour + + val singleRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = dateCreated, + optOutRequestedDateInMillis = dateCreated + oneHour, + ) + + whenever(mockSchedulingRepository.getAllValidOptOutJobRecordsForBroker(testBrokerName)) + .thenReturn(listOf(singleRecord)) + + val result = testee.calculateOptOutSubmitRate(testBrokerName, startDate, endDate) + + assertEquals(1.0, result!!, 0.0) + } + + @Test + fun whenSingleRecordNotRequestedThenReturnZero() = runTest { + val startDate = baseTime + val endDate = baseTime + oneDay + val dateCreated = baseTime + oneHour + + val singleRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBrokerName, + status = OptOutJobStatus.NOT_EXECUTED, + dateCreatedInMillis = dateCreated, + ) + + whenever(mockSchedulingRepository.getAllValidOptOutJobRecordsForBroker(testBrokerName)) + .thenReturn(listOf(singleRecord)) + + val result = testee.calculateOptOutSubmitRate(testBrokerName, startDate, endDate) + + assertEquals(0.0, result!!, 0.0) + } + + @Test + fun whenLargeDateRangeThenFiltersCorrectly() = runTest { + val startDate = baseTime + val endDate = baseTime + TimeUnit.DAYS.toMillis(365) // 1 year + val dateCreated = baseTime + oneDay + + val record1 = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = dateCreated, + optOutRequestedDateInMillis = dateCreated + oneHour, + ) + val record2 = createOptOutJobRecord( + extractedProfileId = 2L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = dateCreated + TimeUnit.DAYS.toMillis(180), // 6 months later + optOutRequestedDateInMillis = dateCreated + TimeUnit.DAYS.toMillis(180) + oneHour, + ) + val record3 = createOptOutJobRecord( + extractedProfileId = 3L, + brokerName = testBrokerName, + status = OptOutJobStatus.NOT_EXECUTED, + dateCreatedInMillis = dateCreated + TimeUnit.DAYS.toMillis(300), // 10 months later + ) + + whenever(mockSchedulingRepository.getAllValidOptOutJobRecordsForBroker(testBrokerName)) + .thenReturn(listOf(record1, record2, record3)) + + val result = testee.calculateOptOutSubmitRate(testBrokerName, startDate, endDate) + + // 2 out of 3 + assertEquals(0.67, result!!, 0.0) + } + + @Test + fun whenRequestedRecordHasZeroOptOutRequestedDateThenNotCounted() = runTest { + val startDate = baseTime + val endDate = baseTime + oneDay + val dateCreated = baseTime + oneHour + + val requestedWithZeroDate = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = dateCreated, + optOutRequestedDateInMillis = 0L, // Zero (should not be counted) + ) + val validRequested = createOptOutJobRecord( + extractedProfileId = 2L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = dateCreated + 2 * oneHour, + optOutRequestedDateInMillis = dateCreated + 3 * oneHour, + ) + + whenever(mockSchedulingRepository.getAllValidOptOutJobRecordsForBroker(testBrokerName)) + .thenReturn(listOf(requestedWithZeroDate, validRequested)) + + val result = testee.calculateOptOutSubmitRate(testBrokerName, startDate, endDate) + + // Only validRequested counts (1 out of 2) + assertEquals(0.5, result!!, 0.0) + } + + @Test + fun whenFractionalResultRoundsDownThenRoundsCorrectly() = runTest { + val startDate = baseTime + val endDate = baseTime + oneDay + val dateCreated = baseTime + oneHour + + // 1 out of 7 = 0.142857... should round to 0.14 + val requested = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBrokerName, + status = OptOutJobStatus.REQUESTED, + dateCreatedInMillis = dateCreated, + optOutRequestedDateInMillis = dateCreated + oneHour, + ) + val notExecutedRecords = (2L..7L).map { id -> + createOptOutJobRecord( + extractedProfileId = id, + brokerName = testBrokerName, + status = OptOutJobStatus.NOT_EXECUTED, + dateCreatedInMillis = dateCreated + id * oneHour, + ) + } + + whenever(mockSchedulingRepository.getAllValidOptOutJobRecordsForBroker(testBrokerName)) + .thenReturn(listOf(requested) + notExecutedRecords) + + val result = testee.calculateOptOutSubmitRate(testBrokerName, startDate, endDate) + + // 1/7 = 0.142857... rounded to 0.14 + assertEquals(0.14, result!!, 0.0) + } + + private fun createOptOutJobRecord( + extractedProfileId: Long, + brokerName: String = testBrokerName, + userProfileId: Long = 1L, + status: OptOutJobStatus = OptOutJobStatus.NOT_EXECUTED, + dateCreatedInMillis: Long = baseTime, + optOutRequestedDateInMillis: Long = 0L, + optOutRemovedDateInMillis: Long = 0L, + attemptCount: Int = 0, + lastOptOutAttemptDateInMillis: Long = 0L, + deprecated: Boolean = false, + ): OptOutJobRecord { + return OptOutJobRecord( + brokerName = brokerName, + userProfileId = userProfileId, + extractedProfileId = extractedProfileId, + status = status, + attemptCount = attemptCount, + lastOptOutAttemptDateInMillis = lastOptOutAttemptDateInMillis, + optOutRequestedDateInMillis = optOutRequestedDateInMillis, + optOutRemovedDateInMillis = optOutRemovedDateInMillis, + deprecated = deprecated, + dateCreatedInMillis = dateCreatedInMillis, + ) + } +} From 2c7f79d516aceefbe57994e8687195559a1efdf7 Mon Sep 17 00:00:00 2001 From: Karl Dimla Date: Tue, 11 Nov 2025 16:36:19 +0100 Subject: [PATCH 5/7] Fix pixel description --- PixelDefinitions/pixels/personal_information_removal.json5 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PixelDefinitions/pixels/personal_information_removal.json5 b/PixelDefinitions/pixels/personal_information_removal.json5 index f0901e130abd..dd0b36814b67 100644 --- a/PixelDefinitions/pixels/personal_information_removal.json5 +++ b/PixelDefinitions/pixels/personal_information_removal.json5 @@ -131,7 +131,7 @@ ] }, "dbp_databroker_custom_stats_optoutsubmit": { - "description": "Fired every 24 hours for every broker to measure opt-out request success rate.", + "description": "Pixel that contains a broker's opt-out 24h submission success rate.", "owners": ["karlenDimla", "landomen"], "triggers": ["other"], "suffixes": ["form_factor"], From 30df5b006838d68d553b9b6e82d2707a378994a1 Mon Sep 17 00:00:00 2001 From: Karl Dimla Date: Tue, 11 Nov 2025 16:54:28 +0100 Subject: [PATCH 6/7] Extract 24hour optout submission reporter --- ...tOut24HourSubmissionSuccessRateReporter.kt | 81 +++++++++++++++++++ .../pir/impl/pixels/PirCustomStatsWorker.kt | 58 +------------ ...4HourSubmissionSuccessRateReporterTest.kt} | 55 +++++++------ 3 files changed, 114 insertions(+), 80 deletions(-) create mode 100644 pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/OptOut24HourSubmissionSuccessRateReporter.kt rename pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/pixels/{PirCustomStatsWorkerTest.kt => RealOptOut24HourSubmissionSuccessRateReporterTest.kt} (94%) diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/OptOut24HourSubmissionSuccessRateReporter.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/OptOut24HourSubmissionSuccessRateReporter.kt new file mode 100644 index 000000000000..774a8df2551a --- /dev/null +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/OptOut24HourSubmissionSuccessRateReporter.kt @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2025 DuckDuckGo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.duckduckgo.pir.impl.pixels + +import com.duckduckgo.common.utils.CurrentTimeProvider +import com.duckduckgo.di.scopes.AppScope +import com.duckduckgo.pir.impl.store.PirRepository +import com.squareup.anvil.annotations.ContributesBinding +import java.util.concurrent.TimeUnit +import javax.inject.Inject +import kotlin.math.abs + +interface OptOut24HourSubmissionSuccessRateReporter { + suspend fun attemptFirePixel() +} + +@ContributesBinding(AppScope::class) +class RealOptOut24HourSubmissionSuccessRateReporter @Inject constructor( + private val optOutSubmitRateCalculator: OptOutSubmitRateCalculator, + private val pirRepository: PirRepository, + private val currentTimeProvider: CurrentTimeProvider, + private val pirPixelSender: PirPixelSender, +) : OptOut24HourSubmissionSuccessRateReporter { + override suspend fun attemptFirePixel() { + val startDate = pirRepository.getCustomStatsPixelsLastSentMs() + val now = currentTimeProvider.currentTimeMillis() + + if (shouldFirePixel(startDate, now)) { + val endDate = now - TimeUnit.HOURS.toMillis(24) + val activeBrokers = pirRepository.getAllActiveBrokerObjects() + val hasUserProfiles = pirRepository.getAllUserProfileQueries().isNotEmpty() + + if (activeBrokers.isNotEmpty() && hasUserProfiles) { + activeBrokers.forEach { + val successRate = optOutSubmitRateCalculator.calculateOptOutSubmitRate( + it.name, + startDate, + endDate, + ) + + if (successRate != null) { + pirPixelSender.reportBrokerCustomStateOptOutSubmitRate( + brokerUrl = it.url, + optOutSuccessRate = successRate, + ) + } + } + + pirRepository.setCustomStatsPixelsLastSentMs(endDate) + } + } + } + + private fun shouldFirePixel( + startDate: Long, + now: Long, + ): Boolean { + return if (startDate == 0L) { + // IF first run, we emit the custom stats pixel + true + } else { + // Else we check if at least 24 hours have passed since last emission + val nowDiffFromStart = abs(now - startDate) + nowDiffFromStart > TimeUnit.HOURS.toMillis(24) + } + } +} diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirCustomStatsWorker.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirCustomStatsWorker.kt index ddda63414b51..d7dfe9e13c0f 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirCustomStatsWorker.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirCustomStatsWorker.kt @@ -20,12 +20,8 @@ import android.content.Context import androidx.work.CoroutineWorker import androidx.work.WorkerParameters import com.duckduckgo.anvil.annotations.ContributesWorker -import com.duckduckgo.common.utils.CurrentTimeProvider import com.duckduckgo.di.scopes.AppScope -import com.duckduckgo.pir.impl.store.PirRepository -import java.util.concurrent.TimeUnit import javax.inject.Inject -import kotlin.math.abs @ContributesWorker(AppScope::class) class PirCustomStatsWorker( @@ -33,63 +29,13 @@ class PirCustomStatsWorker( workerParameters: WorkerParameters, ) : CoroutineWorker(context, workerParameters) { @Inject - lateinit var optOutSubmitRateCalculator: OptOutSubmitRateCalculator - - @Inject - lateinit var pirRepository: PirRepository - - @Inject - lateinit var currentTimeProvider: CurrentTimeProvider - - @Inject - lateinit var pirPixelSender: PirPixelSender + lateinit var optOutSubmissionSuccessRateReporter: OptOut24HourSubmissionSuccessRateReporter override suspend fun doWork(): Result { - val startDate = pirRepository.getCustomStatsPixelsLastSentMs() - val now = currentTimeProvider.currentTimeMillis() - - if (shouldFirePixel(startDate, now)) { - val endDate = now - TimeUnit.HOURS.toMillis(24) - val activeBrokers = pirRepository.getAllActiveBrokerObjects() - val hasUserProfiles = pirRepository.getAllUserProfileQueries().isNotEmpty() - - if (activeBrokers.isNotEmpty() && hasUserProfiles) { - activeBrokers.forEach { - val successRate = optOutSubmitRateCalculator.calculateOptOutSubmitRate( - it.name, - startDate, - endDate, - ) - - if (successRate != null) { - pirPixelSender.reportBrokerCustomStateOptOutSubmitRate( - brokerUrl = it.url, - optOutSuccessRate = successRate, - ) - } - } - - pirRepository.setCustomStatsPixelsLastSentMs(endDate) - } - } - + optOutSubmissionSuccessRateReporter.attemptFirePixel() return Result.success() } - private fun shouldFirePixel( - startDate: Long, - now: Long, - ): Boolean { - return if (startDate == 0L) { - // IF first run, we emit the custom stats pixel - true - } else { - // Else we check if at least 24 hours have passed since last emission - val nowDiffFromStart = abs(now - startDate) - nowDiffFromStart > TimeUnit.HOURS.toMillis(24) - } - } - companion object { const val TAG_PIR_RECURRING_CUSTOM_STATS = "TAG_PIR_RECURRING_CUSTOM_STATS" } diff --git a/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/pixels/PirCustomStatsWorkerTest.kt b/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/pixels/RealOptOut24HourSubmissionSuccessRateReporterTest.kt similarity index 94% rename from pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/pixels/PirCustomStatsWorkerTest.kt rename to pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/pixels/RealOptOut24HourSubmissionSuccessRateReporterTest.kt index 1b502ff2b595..e0dd3f314159 100644 --- a/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/pixels/PirCustomStatsWorkerTest.kt +++ b/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/pixels/RealOptOut24HourSubmissionSuccessRateReporterTest.kt @@ -17,7 +17,6 @@ package com.duckduckgo.pir.impl.pixels import android.content.Context -import androidx.work.testing.TestListenableWorkerBuilder import com.duckduckgo.common.test.CoroutineTestRule import com.duckduckgo.common.utils.CurrentTimeProvider import com.duckduckgo.pir.impl.models.Broker @@ -35,7 +34,7 @@ import org.mockito.kotlin.verify import org.mockito.kotlin.whenever import java.util.concurrent.TimeUnit -class PirCustomStatsWorkerTest { +class RealOptOut24HourSubmissionSuccessRateReporterTest { @get:Rule var coroutineRule = CoroutineTestRule() @@ -46,7 +45,7 @@ class PirCustomStatsWorkerTest { private val mockPirPixelSender: PirPixelSender = mock() private val context: Context = mock() - private lateinit var worker: PirCustomStatsWorker + private lateinit var toTest: RealOptOut24HourSubmissionSuccessRateReporter // Test data // January 15, 2024 10:00:00 UTC @@ -89,13 +88,12 @@ class PirCustomStatsWorkerTest { @Before fun setUp() { - worker = TestListenableWorkerBuilder - .from(context, PirCustomStatsWorker::class.java) - .build() - worker.pirRepository = mockPirRepository - worker.currentTimeProvider = mockCurrentTimeProvider - worker.optOutSubmitRateCalculator = mockOptOutSubmitRateCalculator - worker.pirPixelSender = mockPirPixelSender + toTest = RealOptOut24HourSubmissionSuccessRateReporter( + optOutSubmitRateCalculator = mockOptOutSubmitRateCalculator, + pirRepository = mockPirRepository, + currentTimeProvider = mockCurrentTimeProvider, + pirPixelSender = mockPirPixelSender, + ) } @Test @@ -113,7 +111,7 @@ class PirCustomStatsWorkerTest { ), ).thenReturn(0.5) - worker.doWork() + toTest.attemptFirePixel() verify(mockPirPixelSender).reportBrokerCustomStateOptOutSubmitRate( brokerUrl = testBroker1.url, @@ -130,7 +128,8 @@ class PirCustomStatsWorkerTest { whenever(mockPirRepository.getCustomStatsPixelsLastSentMs()).thenReturn(startDate) whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) - worker.doWork() + toTest.attemptFirePixel() + verify(mockPirRepository, never()).getAllActiveBrokerObjects() verify(mockPirPixelSender, never()).reportBrokerCustomStateOptOutSubmitRate(any(), any()) verify(mockPirRepository, never()).setCustomStatsPixelsLastSentMs(any()) @@ -153,7 +152,8 @@ class PirCustomStatsWorkerTest { ), ).thenReturn(0.75) - worker.doWork() + toTest.attemptFirePixel() + verify(mockPirPixelSender).reportBrokerCustomStateOptOutSubmitRate( brokerUrl = testBroker1.url, optOutSuccessRate = 0.75, @@ -169,7 +169,8 @@ class PirCustomStatsWorkerTest { whenever(mockPirRepository.getCustomStatsPixelsLastSentMs()).thenReturn(startDate) whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) - worker.doWork() + toTest.attemptFirePixel() + verify(mockPirRepository, never()).getAllActiveBrokerObjects() verify(mockPirPixelSender, never()).reportBrokerCustomStateOptOutSubmitRate(any(), any()) } @@ -182,7 +183,8 @@ class PirCustomStatsWorkerTest { whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(emptyList()) whenever(mockPirRepository.getAllUserProfileQueries()).thenReturn(listOf(testProfileQuery)) - worker.doWork() + toTest.attemptFirePixel() + verify(mockOptOutSubmitRateCalculator, never()).calculateOptOutSubmitRate( any(), any(), @@ -200,7 +202,8 @@ class PirCustomStatsWorkerTest { whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker1)) whenever(mockPirRepository.getAllUserProfileQueries()).thenReturn(emptyList()) - worker.doWork() + toTest.attemptFirePixel() + verify(mockOptOutSubmitRateCalculator, never()).calculateOptOutSubmitRate( any(), any(), @@ -239,7 +242,8 @@ class PirCustomStatsWorkerTest { ) .thenReturn(0.8) - worker.doWork() + toTest.attemptFirePixel() + verify(mockPirPixelSender).reportBrokerCustomStateOptOutSubmitRate( brokerUrl = testBroker1.url, optOutSuccessRate = 0.5, @@ -280,7 +284,8 @@ class PirCustomStatsWorkerTest { ) .thenReturn(null) - worker.doWork() + toTest.attemptFirePixel() + verify(mockPirPixelSender).reportBrokerCustomStateOptOutSubmitRate( brokerUrl = testBroker1.url, optOutSuccessRate = 0.5, @@ -307,7 +312,8 @@ class PirCustomStatsWorkerTest { ), ).thenReturn(null) - worker.doWork() + toTest.attemptFirePixel() + verify(mockPirPixelSender, never()).reportBrokerCustomStateOptOutSubmitRate(any(), any()) verify(mockPirRepository).setCustomStatsPixelsLastSentMs(now - twentyFourHours) } @@ -330,7 +336,7 @@ class PirCustomStatsWorkerTest { ), ).thenReturn(0.5) - worker.doWork() + toTest.attemptFirePixel() verify(mockOptOutSubmitRateCalculator).calculateOptOutSubmitRate( brokerName = testBroker1.name, @@ -369,7 +375,8 @@ class PirCustomStatsWorkerTest { ) .thenReturn(0.9) - worker.doWork() + toTest.attemptFirePixel() + verify(mockPirPixelSender, never()).reportBrokerCustomStateOptOutSubmitRate( brokerUrl = eq(testBroker1.url), optOutSuccessRate = any(), @@ -389,7 +396,7 @@ class PirCustomStatsWorkerTest { whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(emptyList()) whenever(mockPirRepository.getAllUserProfileQueries()).thenReturn(emptyList()) - worker.doWork() + toTest.attemptFirePixel() verify(mockOptOutSubmitRateCalculator, never()).calculateOptOutSubmitRate( any(), @@ -408,7 +415,7 @@ class PirCustomStatsWorkerTest { whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(emptyList()) whenever(mockPirRepository.getAllUserProfileQueries()).thenReturn(listOf(testProfileQuery)) - worker.doWork() + toTest.attemptFirePixel() verify(mockOptOutSubmitRateCalculator, never()).calculateOptOutSubmitRate( any(), @@ -427,7 +434,7 @@ class PirCustomStatsWorkerTest { whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker1)) whenever(mockPirRepository.getAllUserProfileQueries()).thenReturn(emptyList()) - worker.doWork() + toTest.attemptFirePixel() verify(mockOptOutSubmitRateCalculator, never()).calculateOptOutSubmitRate( any(), From dcbd7ba977727285549f8891efa32fa4f973fe91 Mon Sep 17 00:00:00 2001 From: Karl Dimla Date: Mon, 17 Nov 2025 12:09:33 +0100 Subject: [PATCH 7/7] PIR: Implement pixels for breakage KPIs (#7108) Task/Issue URL: https://app.asana.com/1/137249556945/project/72649045549333/task/1211848227486283?focus=true ### Description See attached task description ### Steps to test this PR https://app.asana.com/1/137249556945/project/72649045549333/task/1211863510648442?focus=true --- .../pixels/personal_information_removal.json5 | 112 ++++ .../15.json | 30 +- .../pir/impl/common/PirJobConstants.kt | 1 + .../pir/impl/models/scheduling/JobRecord.kt | 4 + ...tOut24HourSubmissionSuccessRateReporter.kt | 30 +- .../impl/pixels/OptOutConfirmationReporter.kt | 152 +++++ .../impl/pixels/OptOutSubmitRateCalculator.kt | 16 +- .../pir/impl/pixels/PirCustomStatsWorker.kt | 7 + .../duckduckgo/pir/impl/pixels/PirPixel.kt | 38 ++ .../pir/impl/pixels/PirPixelSender.kt | 112 ++++ .../pir/impl/scan/PirScanScheduler.kt | 4 +- .../pir/impl/store/PirSchedulingRepository.kt | 67 +++ .../pir/impl/store/db/JobSchedulingDao.kt | 48 ++ .../impl/store/db/JobSchedulingEntities.kt | 10 + ...24HourSubmissionSuccessRateReporterTest.kt | 170 +++++- .../RealOptOutConfirmationReporterTest.kt | 526 ++++++++++++++++++ .../RealOptOutSubmitRateCalculatorTest.kt | 216 ++----- .../store/RealPirSchedulingRepositoryTest.kt | 6 + 18 files changed, 1323 insertions(+), 226 deletions(-) create mode 100644 pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/OptOutConfirmationReporter.kt create mode 100644 pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/pixels/RealOptOutConfirmationReporterTest.kt diff --git a/PixelDefinitions/pixels/personal_information_removal.json5 b/PixelDefinitions/pixels/personal_information_removal.json5 index dd0b36814b67..61b6ae2749f2 100644 --- a/PixelDefinitions/pixels/personal_information_removal.json5 +++ b/PixelDefinitions/pixels/personal_information_removal.json5 @@ -148,5 +148,117 @@ "type": "string" } ] + }, + "dbp_optoutjob_at-7-days_confirmed": { + "description": "Pixel that contains if any submitted opt-out has been confirmed 7 days after submission.", + "owners": ["karlenDimla", "landomen"], + "triggers": ["other"], + "suffixes": ["form_factor"], + "parameters": [ + "appVersion", + { + "key": "data_broker", + "description": "The URL of the data broker that has a confirmed opt-out after 7 days", + "type": "string" + } + ] + }, + "dbp_optoutjob_at-7-days_unconfirmed": { + "description": "Pixel that contains if any submitted opt-out are still unconfirmed 7 days after submission.", + "owners": ["karlenDimla", "landomen"], + "triggers": ["other"], + "suffixes": ["form_factor"], + "parameters": [ + "appVersion", + { + "key": "data_broker", + "description": "The URL of the data broker that has an unconfirmed opt-out after 7 days", + "type": "string" + } + ] + }, + "dbp_optoutjob_at-14-days_confirmed": { + "description": "Pixel that contains if any submitted opt-out has been confirmed 14 days after submission.", + "owners": ["karlenDimla", "landomen"], + "triggers": ["other"], + "suffixes": ["form_factor"], + "parameters": [ + "appVersion", + { + "key": "data_broker", + "description": "The URL of the data broker that has a confirmed opt-out after 14 days", + "type": "string" + } + ] + }, + "dbp_optoutjob_at-14-days_unconfirmed": { + "description": "Pixel that contains if any submitted opt-out are still unconfirmed 14 days after submission.", + "owners": ["karlenDimla", "landomen"], + "triggers": ["other"], + "suffixes": ["form_factor"], + "parameters": [ + "appVersion", + { + "key": "data_broker", + "description": "The URL of the data broker that has an unconfirmed opt-out after 14 days", + "type": "string" + } + ] + }, + "dbp_optoutjob_at-21-days_confirmed": { + "description": "Pixel that contains if any submitted opt-out has been confirmed 21 days after submission.", + "owners": ["karlenDimla", "landomen"], + "triggers": ["other"], + "suffixes": ["form_factor"], + "parameters": [ + "appVersion", + { + "key": "data_broker", + "description": "The URL of the data broker that has a confirmed opt-out after 21 days", + "type": "string" + } + ] + }, + "dbp_optoutjob_at-21-days_unconfirmed": { + "description": "Pixel that contains if any submitted opt-out are still unconfirmed 21 days after submission.", + "owners": ["karlenDimla", "landomen"], + "triggers": ["other"], + "suffixes": ["form_factor"], + "parameters": [ + "appVersion", + { + "key": "data_broker", + "description": "The URL of the data broker that has an unconfirmed opt-out after 21 days", + "type": "string" + } + ] + }, + "dbp_optoutjob_at-42-days_confirmed": { + "description": "Pixel that contains if any submitted opt-out has been confirmed 42 days after submission.", + "owners": ["karlenDimla", "landomen"], + "triggers": ["other"], + "suffixes": ["form_factor"], + "parameters": [ + "appVersion", + { + "key": "data_broker", + "description": "The URL of the data broker that has a confirmed opt-out after 42 days", + "type": "string" + } + ] + }, + "dbp_optoutjob_at-42-days_unconfirmed": { + "description": "Pixel that contains if any submitted opt-out are still unconfirmed 42 days after submission.", + "owners": ["karlenDimla", "landomen"], + "triggers": ["other"], + "suffixes": ["form_factor"], + "parameters": [ + "appVersion", + { + "key": "data_broker", + "description": "The URL of the data broker that has an unconfirmed opt-out after 42 days", + "type": "string" + } + ] } } diff --git a/pir/pir-impl/schemas/com.duckduckgo.pir.impl.store.PirDatabase/15.json b/pir/pir-impl/schemas/com.duckduckgo.pir.impl.store.PirDatabase/15.json index 1c354877cbb9..a858f45c66f9 100644 --- a/pir/pir-impl/schemas/com.duckduckgo.pir.impl.store.PirDatabase/15.json +++ b/pir/pir-impl/schemas/com.duckduckgo.pir.impl.store.PirDatabase/15.json @@ -2,7 +2,7 @@ "formatVersion": 1, "database": { "version": 15, - "identityHash": "584648b8b3065521786fb33f44214f2e", + "identityHash": "a5782d19654dee4cad932b458037bb37", "entities": [ { "tableName": "pir_broker_json_etag", @@ -696,7 +696,7 @@ }, { "tableName": "pir_optout_job_record", - "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`extractedProfileId` INTEGER NOT NULL, `brokerName` TEXT NOT NULL, `userProfileId` INTEGER NOT NULL, `status` TEXT NOT NULL, `attemptCount` INTEGER NOT NULL, `lastOptOutAttemptDate` INTEGER, `optOutRequestedDate` INTEGER NOT NULL, `optOutRemovedDate` INTEGER NOT NULL, `deprecated` INTEGER NOT NULL, `dateCreatedInMillis` INTEGER NOT NULL, PRIMARY KEY(`extractedProfileId`))", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`extractedProfileId` INTEGER NOT NULL, `brokerName` TEXT NOT NULL, `userProfileId` INTEGER NOT NULL, `status` TEXT NOT NULL, `attemptCount` INTEGER NOT NULL, `lastOptOutAttemptDate` INTEGER, `optOutRequestedDate` INTEGER NOT NULL, `optOutRemovedDate` INTEGER NOT NULL, `deprecated` INTEGER NOT NULL, `dateCreatedInMillis` INTEGER NOT NULL, `reporting_sevenDayConfirmationReportSentDateMs` INTEGER NOT NULL, `reporting_fourteenDayConfirmationReportSentDateMs` INTEGER NOT NULL, `reporting_twentyOneDayConfirmationReportSentDateMs` INTEGER NOT NULL, `reporting_fortyTwoDayConfirmationReportSentDateMs` INTEGER NOT NULL, PRIMARY KEY(`extractedProfileId`))", "fields": [ { "fieldPath": "extractedProfileId", @@ -757,6 +757,30 @@ "columnName": "dateCreatedInMillis", "affinity": "INTEGER", "notNull": true + }, + { + "fieldPath": "reporting.sevenDayConfirmationReportSentDateMs", + "columnName": "reporting_sevenDayConfirmationReportSentDateMs", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "reporting.fourteenDayConfirmationReportSentDateMs", + "columnName": "reporting_fourteenDayConfirmationReportSentDateMs", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "reporting.twentyOneDayConfirmationReportSentDateMs", + "columnName": "reporting_twentyOneDayConfirmationReportSentDateMs", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "reporting.fortyTwoDayConfirmationReportSentDateMs", + "columnName": "reporting_fortyTwoDayConfirmationReportSentDateMs", + "affinity": "INTEGER", + "notNull": true } ], "primaryKey": { @@ -946,7 +970,7 @@ "views": [], "setupQueries": [ "CREATE TABLE IF NOT EXISTS room_master_table (id INTEGER PRIMARY KEY,identity_hash TEXT)", - "INSERT OR REPLACE INTO room_master_table (id,identity_hash) VALUES(42, '584648b8b3065521786fb33f44214f2e')" + "INSERT OR REPLACE INTO room_master_table (id,identity_hash) VALUES(42, 'a5782d19654dee4cad932b458037bb37')" ] } } \ No newline at end of file diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/common/PirJobConstants.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/common/PirJobConstants.kt index 9c3f0ac50ba8..53674d6ddbd5 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/common/PirJobConstants.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/common/PirJobConstants.kt @@ -22,4 +22,5 @@ object PirJobConstants { const val MAX_DETACHED_WEBVIEW_COUNT = 20 const val SCHEDULED_SCAN_INTERVAL_HOURS = 12L const val EMAIL_CONFIRMATION_INTERVAL_HOURS = 8L + const val CUSTOM_PIXEL_INTERVAL_HOURS = 5L } diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/models/scheduling/JobRecord.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/models/scheduling/JobRecord.kt index 95f7b6735ed8..965afbf75fe4 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/models/scheduling/JobRecord.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/models/scheduling/JobRecord.kt @@ -56,6 +56,10 @@ sealed class JobRecord( val optOutRemovedDateInMillis: Long = 0L, val deprecated: Boolean = false, val dateCreatedInMillis: Long = 0L, + val confirmation7dayReportSentDateMs: Long = 0L, + val confirmation14dayReportSentDateMs: Long = 0L, + val confirmation21dayReportSentDateMs: Long = 0L, + val confirmation42dayReportSentDateMs: Long = 0L, ) : JobRecord(brokerName, userProfileId) { enum class OptOutJobStatus { /** Opt-out has not been executed yet and should be executed when possible */ diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/OptOut24HourSubmissionSuccessRateReporter.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/OptOut24HourSubmissionSuccessRateReporter.kt index 774a8df2551a..a81adfda76f7 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/OptOut24HourSubmissionSuccessRateReporter.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/OptOut24HourSubmissionSuccessRateReporter.kt @@ -17,9 +17,13 @@ package com.duckduckgo.pir.impl.pixels import com.duckduckgo.common.utils.CurrentTimeProvider +import com.duckduckgo.common.utils.DispatcherProvider import com.duckduckgo.di.scopes.AppScope import com.duckduckgo.pir.impl.store.PirRepository +import com.duckduckgo.pir.impl.store.PirSchedulingRepository import com.squareup.anvil.annotations.ContributesBinding +import kotlinx.coroutines.withContext +import logcat.logcat import java.util.concurrent.TimeUnit import javax.inject.Inject import kotlin.math.abs @@ -34,32 +38,44 @@ class RealOptOut24HourSubmissionSuccessRateReporter @Inject constructor( private val pirRepository: PirRepository, private val currentTimeProvider: CurrentTimeProvider, private val pirPixelSender: PirPixelSender, + private val pirSchedulingRepository: PirSchedulingRepository, + private val dispatcherProvider: DispatcherProvider, ) : OptOut24HourSubmissionSuccessRateReporter { override suspend fun attemptFirePixel() { - val startDate = pirRepository.getCustomStatsPixelsLastSentMs() - val now = currentTimeProvider.currentTimeMillis() + withContext(dispatcherProvider.io()) { + logcat { "PIR-CUSTOM-STATS: Attempt to fire 24hour submission pixels" } + val startDate = pirRepository.getCustomStatsPixelsLastSentMs() + val now = currentTimeProvider.currentTimeMillis() - if (shouldFirePixel(startDate, now)) { + if (!shouldFirePixel(startDate, now)) return@withContext + logcat { "PIR-CUSTOM-STATS: Should fire pixel - 24hrs passed since last send" } val endDate = now - TimeUnit.HOURS.toMillis(24) val activeBrokers = pirRepository.getAllActiveBrokerObjects() val hasUserProfiles = pirRepository.getAllUserProfileQueries().isNotEmpty() + val activeOptOutJobRecords = pirSchedulingRepository.getAllValidOptOutJobRecords() + + if (activeBrokers.isNotEmpty() && activeOptOutJobRecords.isNotEmpty() && hasUserProfiles) { + activeBrokers.forEach { broker -> + val activeJobRecordsForBroker = activeOptOutJobRecords.filter { it.brokerName == broker.name } + + if (activeJobRecordsForBroker.isEmpty()) return@forEach - if (activeBrokers.isNotEmpty() && hasUserProfiles) { - activeBrokers.forEach { val successRate = optOutSubmitRateCalculator.calculateOptOutSubmitRate( - it.name, + activeJobRecordsForBroker, startDate, endDate, ) + logcat { "PIR-CUSTOM-STATS: 24hr submission ${broker.name} : $successRate" } if (successRate != null) { pirPixelSender.reportBrokerCustomStateOptOutSubmitRate( - brokerUrl = it.url, + brokerUrl = broker.url, optOutSuccessRate = successRate, ) } } + logcat { "PIR-CUSTOM-STATS: Updating last send date to $endDate" } pirRepository.setCustomStatsPixelsLastSentMs(endDate) } } diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/OptOutConfirmationReporter.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/OptOutConfirmationReporter.kt new file mode 100644 index 000000000000..5e7a36029d28 --- /dev/null +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/OptOutConfirmationReporter.kt @@ -0,0 +1,152 @@ +/* + * Copyright (c) 2025 DuckDuckGo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.duckduckgo.pir.impl.pixels + +import com.duckduckgo.common.utils.CurrentTimeProvider +import com.duckduckgo.common.utils.DispatcherProvider +import com.duckduckgo.di.scopes.AppScope +import com.duckduckgo.pir.impl.models.Broker +import com.duckduckgo.pir.impl.models.scheduling.JobRecord.OptOutJobRecord +import com.duckduckgo.pir.impl.models.scheduling.JobRecord.OptOutJobRecord.OptOutJobStatus.REMOVED +import com.duckduckgo.pir.impl.models.scheduling.JobRecord.OptOutJobRecord.OptOutJobStatus.REQUESTED +import com.duckduckgo.pir.impl.store.PirRepository +import com.duckduckgo.pir.impl.store.PirSchedulingRepository +import com.squareup.anvil.annotations.ContributesBinding +import kotlinx.coroutines.withContext +import logcat.logcat +import java.util.concurrent.TimeUnit +import javax.inject.Inject + +interface OptOutConfirmationReporter { + suspend fun attemptFirePixel() +} + +@ContributesBinding(AppScope::class) +class RealOptOutConfirmationReporter @Inject constructor( + private val dispatcherProvider: DispatcherProvider, + private val pirSchedulingRepository: PirSchedulingRepository, + private val pirRepository: PirRepository, + private val currentTimeProvider: CurrentTimeProvider, + private val pixelSender: PirPixelSender, +) : OptOutConfirmationReporter { + override suspend fun attemptFirePixel() { + logcat { "PIR-CUSTOM-STATS: Attempt to fire confirmation pixels" } + + withContext(dispatcherProvider.io()) { + val activeBrokers = pirRepository.getAllActiveBrokerObjects().associateBy { it.name } + val allValidRequestedOptOutJobs = pirSchedulingRepository.getAllValidOptOutJobRecords().filter { + it.status == REQUESTED || it.status == REMOVED // TODO: Filter out removed by user + } + + if (activeBrokers.isEmpty() || allValidRequestedOptOutJobs.isEmpty()) return@withContext + + logcat { "PIR-CUSTOM-STATS: Will fire confirmation pixels for ${allValidRequestedOptOutJobs.size} jobs" } + allValidRequestedOptOutJobs.also { + // Fire 7 day pixel + it.attemptFirePixelForConfirmationDay( + activeBrokers, + INTERVAL_DAY_7, + { jobRecord -> jobRecord.confirmation7dayReportSentDateMs == 0L }, + { brokerUrl -> pixelSender.reportBrokerOptOutConfirmed7Days(brokerUrl) }, + { brokerUrl -> pixelSender.reportBrokerOptOutUnconfirmed7Days(brokerUrl) }, + { jobRecord, now -> + pirSchedulingRepository.markOptOutDay7ConfirmationPixelSent(jobRecord.extractedProfileId, now) + }, + ) + + // Fire 14 day pixel + it.attemptFirePixelForConfirmationDay( + activeBrokers, + INTERVAL_DAY_14, + { jobRecord -> jobRecord.confirmation14dayReportSentDateMs == 0L }, + { brokerUrl -> pixelSender.reportBrokerOptOutConfirmed14Days(brokerUrl) }, + { brokerUrl -> pixelSender.reportBrokerOptOutUnconfirmed14Days(brokerUrl) }, + { jobRecord, now -> + pirSchedulingRepository.markOptOutDay14ConfirmationPixelSent(jobRecord.extractedProfileId, now) + }, + ) + + // Fire 21 day pixel + it.attemptFirePixelForConfirmationDay( + activeBrokers, + INTERVAL_DAY_21, + { jobRecord -> jobRecord.confirmation21dayReportSentDateMs == 0L }, + { brokerUrl -> pixelSender.reportBrokerOptOutConfirmed21Days(brokerUrl) }, + { brokerUrl -> pixelSender.reportBrokerOptOutUnconfirmed21Days(brokerUrl) }, + { jobRecord, now -> + pirSchedulingRepository.markOptOutDay21ConfirmationPixelSent(jobRecord.extractedProfileId, now) + }, + ) + + // Fire 42 day pixel + it.attemptFirePixelForConfirmationDay( + activeBrokers, + INTERVAL_DAY_42, + { jobRecord -> jobRecord.confirmation42dayReportSentDateMs == 0L }, + { brokerUrl -> pixelSender.reportBrokerOptOutConfirmed42Days(brokerUrl) }, + { brokerUrl -> pixelSender.reportBrokerOptOutUnconfirmed42Days(brokerUrl) }, + { jobRecord, now -> + pirSchedulingRepository.markOptOutDay42ConfirmationPixelSent(jobRecord.extractedProfileId, now) + }, + ) + } + } + } + + private suspend fun List.attemptFirePixelForConfirmationDay( + activeBrokers: Map, + confirmationDay: Long, + jobRecordFilter: (OptOutJobRecord) -> Boolean, + emitConfirmPixel: (String) -> Unit, + emitUnconfirmPixel: (String) -> Unit, + markOptOutJobRecordReporting: suspend (OptOutJobRecord, Long) -> Unit, + ) { + val now = currentTimeProvider.currentTimeMillis() + val optOutsForPixel = this.filter { + it.daysPassedSinceSubmission(now, confirmationDay) && jobRecordFilter(it) + } + + logcat { "PIR-CUSTOM-STATS: Firing $confirmationDay day confirmation pixels for ${optOutsForPixel.size} jobs" } + optOutsForPixel.forEach { optOutJobRecord -> + val broker = activeBrokers[optOutJobRecord.brokerName] ?: return@forEach + + if (optOutJobRecord.status == REMOVED) { + logcat { "PIR-CUSTOM-STATS: Firing $confirmationDay day confirmation pixels for ${broker.name}" } + emitConfirmPixel(broker.url) + } else { + logcat { "PIR-CUSTOM-STATS: Firing $confirmationDay day unconfirmation pixels for ${broker.name}" } + emitUnconfirmPixel(broker.url) + } + + markOptOutJobRecordReporting(optOutJobRecord, now) + } + } + + private fun OptOutJobRecord.daysPassedSinceSubmission( + now: Long, + interval: Long, + ): Boolean { + return now >= this.optOutRequestedDateInMillis + TimeUnit.DAYS.toMillis(interval) + } + + companion object { + private const val INTERVAL_DAY_7 = 7L + private const val INTERVAL_DAY_14 = 14L + private const val INTERVAL_DAY_21 = 21L + private const val INTERVAL_DAY_42 = 42L + } +} diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/OptOutSubmitRateCalculator.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/OptOutSubmitRateCalculator.kt index 23840aacd614..688fa2578b64 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/OptOutSubmitRateCalculator.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/OptOutSubmitRateCalculator.kt @@ -18,9 +18,9 @@ package com.duckduckgo.pir.impl.pixels import com.duckduckgo.common.utils.DispatcherProvider import com.duckduckgo.di.scopes.AppScope +import com.duckduckgo.pir.impl.models.scheduling.JobRecord import com.duckduckgo.pir.impl.models.scheduling.JobRecord.OptOutJobRecord.OptOutJobStatus.REMOVED import com.duckduckgo.pir.impl.models.scheduling.JobRecord.OptOutJobRecord.OptOutJobStatus.REQUESTED -import com.duckduckgo.pir.impl.store.PirSchedulingRepository import com.squareup.anvil.annotations.ContributesBinding import kotlinx.coroutines.withContext import java.util.concurrent.TimeUnit @@ -31,12 +31,12 @@ interface OptOutSubmitRateCalculator { /** * Calculates the opt-out 24h submit rate for a given broker within the specified date range. * - * @param brokerName name of the broker to calculate the opt-out submit rate for. + * @param allActiveOptOutJobsForBroker all active opt-out job records for the broker. * @param startDateMs The opt-out records to include should be created on or after this date. Default is 0L (epoch). * @param endDateMs tThe opt-out records to include should be created on or before this date. Default is 0L (epoch). */ suspend fun calculateOptOutSubmitRate( - brokerName: String, + allActiveOptOutJobsForBroker: List, startDateMs: Long = 0L, endDateMs: Long, ): Double? @@ -45,16 +45,15 @@ interface OptOutSubmitRateCalculator { @ContributesBinding(AppScope::class) class RealOptOutSubmitRateCalculator @Inject constructor( private val dispatcherProvider: DispatcherProvider, - private val schedulingRepository: PirSchedulingRepository, ) : OptOutSubmitRateCalculator { override suspend fun calculateOptOutSubmitRate( - brokerName: String, + allActiveOptOutJobsForBroker: List, startDateMs: Long, endDateMs: Long, ): Double? = withContext(dispatcherProvider.io()) { // Get all opt out job records created within the given range for the specified broker - val recordsCreatedWithinRange = schedulingRepository.getAllValidOptOutJobRecordsForBroker(brokerName).filter { - it.brokerName == brokerName && it.dateCreatedInMillis in startDateMs..endDateMs + val recordsCreatedWithinRange = allActiveOptOutJobsForBroker.filter { + it.dateCreatedInMillis in startDateMs..endDateMs } // We don't need to calculate the rate if there are no records @@ -68,7 +67,8 @@ class RealOptOutSubmitRateCalculator @Inject constructor( ) } - val optOutSuccessRate = requestedRecordsWithinRange.size.toDouble() / recordsCreatedWithinRange.size.toDouble() + val optOutSuccessRate = + requestedRecordsWithinRange.size.toDouble() / recordsCreatedWithinRange.size.toDouble() val roundedOptOutSuccessRate = round(optOutSuccessRate * 100) / 100 return@withContext roundedOptOutSuccessRate } diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirCustomStatsWorker.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirCustomStatsWorker.kt index d7dfe9e13c0f..b0bc5e2733ca 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirCustomStatsWorker.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirCustomStatsWorker.kt @@ -21,6 +21,7 @@ import androidx.work.CoroutineWorker import androidx.work.WorkerParameters import com.duckduckgo.anvil.annotations.ContributesWorker import com.duckduckgo.di.scopes.AppScope +import logcat.logcat import javax.inject.Inject @ContributesWorker(AppScope::class) @@ -31,8 +32,14 @@ class PirCustomStatsWorker( @Inject lateinit var optOutSubmissionSuccessRateReporter: OptOut24HourSubmissionSuccessRateReporter + @Inject + lateinit var optOutConfirmationReporter: OptOutConfirmationReporter + override suspend fun doWork(): Result { + logcat { "PIR-CUSTOM-STATS: Attempt to fire custom pixels" } optOutSubmissionSuccessRateReporter.attemptFirePixel() + optOutConfirmationReporter.attemptFirePixel() + return Result.success() } diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirPixel.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirPixel.kt index 1b70031b8118..a676af8f6b66 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirPixel.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirPixel.kt @@ -140,6 +140,44 @@ enum class PirPixel( PIR_BROKER_CUSTOM_STATS_OPTOUT_SUBMIT_SUCCESSRATE( baseName = "dbp_databroker_custom_stats_optoutsubmit", type = Count, + ), + + PIR_BROKER_CUSTOM_STATS_7DAY_CONFIRMED_OPTOUT( + baseName = "dbp_optoutjob_at-7-days_confirmed", + type = Count, + ), + + PIR_BROKER_CUSTOM_STATS_7DAY_UNCONFIRMED_OPTOUT( + baseName = "dbp_optoutjob_at-7-days_unconfirmed", + type = Count, + ), + PIR_BROKER_CUSTOM_STATS_14DAY_CONFIRMED_OPTOUT( + baseName = "dbp_optoutjob_at-14-days_confirmed", + type = Count, + ), + + PIR_BROKER_CUSTOM_STATS_14DAY_UNCONFIRMED_OPTOUT( + baseName = "dbp_optoutjob_at-14-days_unconfirmed", + type = Count, + ), + + PIR_BROKER_CUSTOM_STATS_21DAY_CONFIRMED_OPTOUT( + baseName = "dbp_optoutjob_at-21-days_confirmed", + type = Count, + ), + + PIR_BROKER_CUSTOM_STATS_21DAY_UNCONFIRMED_OPTOUT( + baseName = "dbp_optoutjob_at-21-days_unconfirmed", + type = Count, + ), + PIR_BROKER_CUSTOM_STATS_42DAY_CONFIRMED_OPTOUT( + baseName = "dbp_optoutjob_at-42-days_confirmed", + type = Count, + ), + + PIR_BROKER_CUSTOM_STATS_42DAY_UNCONFIRMED_OPTOUT( + baseName = "dbp_optoutjob_at-42-days_unconfirmed", + type = Count, ), ; constructor( diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirPixelSender.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirPixelSender.kt index 53ff572fabaa..933c7bbe6930 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirPixelSender.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/pixels/PirPixelSender.kt @@ -18,6 +18,14 @@ package com.duckduckgo.pir.impl.pixels import com.duckduckgo.app.statistics.pixels.Pixel import com.duckduckgo.di.scopes.AppScope +import com.duckduckgo.pir.impl.pixels.PirPixel.PIR_BROKER_CUSTOM_STATS_14DAY_CONFIRMED_OPTOUT +import com.duckduckgo.pir.impl.pixels.PirPixel.PIR_BROKER_CUSTOM_STATS_14DAY_UNCONFIRMED_OPTOUT +import com.duckduckgo.pir.impl.pixels.PirPixel.PIR_BROKER_CUSTOM_STATS_21DAY_CONFIRMED_OPTOUT +import com.duckduckgo.pir.impl.pixels.PirPixel.PIR_BROKER_CUSTOM_STATS_21DAY_UNCONFIRMED_OPTOUT +import com.duckduckgo.pir.impl.pixels.PirPixel.PIR_BROKER_CUSTOM_STATS_42DAY_CONFIRMED_OPTOUT +import com.duckduckgo.pir.impl.pixels.PirPixel.PIR_BROKER_CUSTOM_STATS_42DAY_UNCONFIRMED_OPTOUT +import com.duckduckgo.pir.impl.pixels.PirPixel.PIR_BROKER_CUSTOM_STATS_7DAY_CONFIRMED_OPTOUT +import com.duckduckgo.pir.impl.pixels.PirPixel.PIR_BROKER_CUSTOM_STATS_7DAY_UNCONFIRMED_OPTOUT import com.duckduckgo.pir.impl.pixels.PirPixel.PIR_BROKER_CUSTOM_STATS_OPTOUT_SUBMIT_SUCCESSRATE import com.duckduckgo.pir.impl.pixels.PirPixel.PIR_EMAIL_CONFIRMATION_ATTEMPT_FAILED import com.duckduckgo.pir.impl.pixels.PirPixel.PIR_EMAIL_CONFIRMATION_ATTEMPT_START @@ -333,6 +341,46 @@ interface PirPixelSender { brokerUrl: String, optOutSuccessRate: Double, ) + + /** + * Emits a pixel when an opt-out has been confirmed within 7 days. + */ + fun reportBrokerOptOutConfirmed7Days(brokerUrl: String) + + /** + * Emits a pixel when an opt-out is unconfirmed within 7 days. + */ + fun reportBrokerOptOutUnconfirmed7Days(brokerUrl: String) + + /** + * Emits a pixel when an opt-out has been confirmed within 14 days. + */ + fun reportBrokerOptOutConfirmed14Days(brokerUrl: String) + + /** + * Emits a pixel when an opt-out is unconfirmed within 14 days. + */ + fun reportBrokerOptOutUnconfirmed14Days(brokerUrl: String) + + /** + * Emits a pixel when an opt-out has been confirmed within 21 days. + */ + fun reportBrokerOptOutConfirmed21Days(brokerUrl: String) + + /** + * Emits a pixel when an opt-out is unconfirmed within 21 days. + */ + fun reportBrokerOptOutUnconfirmed21Days(brokerUrl: String) + + /** + * Emits a pixel when an opt-out has been confirmed within 42 days. + */ + fun reportBrokerOptOutConfirmed42Days(brokerUrl: String) + + /** + * Emits a pixel when an opt-out is unconfirmed within 42 days. + */ + fun reportBrokerOptOutUnconfirmed42Days(brokerUrl: String) } @ContributesBinding(AppScope::class) @@ -627,6 +675,70 @@ class RealPirPixelSender @Inject constructor( fire(PIR_BROKER_CUSTOM_STATS_OPTOUT_SUBMIT_SUCCESSRATE, params) } + override fun reportBrokerOptOutConfirmed7Days(brokerUrl: String) { + val params = mapOf( + PARAM_KEY_BROKER to brokerUrl, + ) + + fire(PIR_BROKER_CUSTOM_STATS_7DAY_CONFIRMED_OPTOUT, params) + } + + override fun reportBrokerOptOutUnconfirmed7Days(brokerUrl: String) { + val params = mapOf( + PARAM_KEY_BROKER to brokerUrl, + ) + + fire(PIR_BROKER_CUSTOM_STATS_7DAY_UNCONFIRMED_OPTOUT, params) + } + + override fun reportBrokerOptOutConfirmed14Days(brokerUrl: String) { + val params = mapOf( + PARAM_KEY_BROKER to brokerUrl, + ) + + fire(PIR_BROKER_CUSTOM_STATS_14DAY_CONFIRMED_OPTOUT, params) + } + + override fun reportBrokerOptOutUnconfirmed14Days(brokerUrl: String) { + val params = mapOf( + PARAM_KEY_BROKER to brokerUrl, + ) + + fire(PIR_BROKER_CUSTOM_STATS_14DAY_UNCONFIRMED_OPTOUT, params) + } + + override fun reportBrokerOptOutConfirmed21Days(brokerUrl: String) { + val params = mapOf( + PARAM_KEY_BROKER to brokerUrl, + ) + + fire(PIR_BROKER_CUSTOM_STATS_21DAY_CONFIRMED_OPTOUT, params) + } + + override fun reportBrokerOptOutUnconfirmed21Days(brokerUrl: String) { + val params = mapOf( + PARAM_KEY_BROKER to brokerUrl, + ) + + fire(PIR_BROKER_CUSTOM_STATS_21DAY_UNCONFIRMED_OPTOUT, params) + } + + override fun reportBrokerOptOutConfirmed42Days(brokerUrl: String) { + val params = mapOf( + PARAM_KEY_BROKER to brokerUrl, + ) + + fire(PIR_BROKER_CUSTOM_STATS_42DAY_CONFIRMED_OPTOUT, params) + } + + override fun reportBrokerOptOutUnconfirmed42Days(brokerUrl: String) { + val params = mapOf( + PARAM_KEY_BROKER to brokerUrl, + ) + + fire(PIR_BROKER_CUSTOM_STATS_42DAY_UNCONFIRMED_OPTOUT, params) + } + private fun fire( pixel: PirPixel, params: Map = emptyMap(), diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/scan/PirScanScheduler.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/scan/PirScanScheduler.kt index 28f3a414ce38..50fe401d48c9 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/scan/PirScanScheduler.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/scan/PirScanScheduler.kt @@ -31,6 +31,7 @@ import com.duckduckgo.app.di.AppCoroutineScope import com.duckduckgo.appbuildconfig.api.AppBuildConfig import com.duckduckgo.common.utils.CurrentTimeProvider import com.duckduckgo.di.scopes.AppScope +import com.duckduckgo.pir.impl.common.PirJobConstants.CUSTOM_PIXEL_INTERVAL_HOURS import com.duckduckgo.pir.impl.common.PirJobConstants.EMAIL_CONFIRMATION_INTERVAL_HOURS import com.duckduckgo.pir.impl.common.PirJobConstants.SCHEDULED_SCAN_INTERVAL_HOURS import com.duckduckgo.pir.impl.email.PirEmailConfirmationRemoteWorker @@ -134,8 +135,9 @@ class RealPirScanScheduler @Inject constructor( } private fun scheduleRecurringPixelStats() { - val periodicWorkRequest = PeriodicWorkRequestBuilder(5, TimeUnit.HOURS) + val periodicWorkRequest = PeriodicWorkRequestBuilder(CUSTOM_PIXEL_INTERVAL_HOURS, TimeUnit.HOURS) .addTag(TAG_PIR_RECURRING_CUSTOM_STATS) + .setInitialDelay(CUSTOM_PIXEL_INTERVAL_HOURS, TimeUnit.HOURS) .build() workManager.enqueueUniquePeriodicWork( diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/PirSchedulingRepository.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/PirSchedulingRepository.kt index d5461c69f7ed..676953b49bb1 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/PirSchedulingRepository.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/PirSchedulingRepository.kt @@ -28,6 +28,7 @@ import com.duckduckgo.pir.impl.models.scheduling.JobRecord.ScanJobRecord.ScanJob import com.duckduckgo.pir.impl.store.db.EmailConfirmationJobRecordEntity import com.duckduckgo.pir.impl.store.db.JobSchedulingDao import com.duckduckgo.pir.impl.store.db.OptOutJobRecordEntity +import com.duckduckgo.pir.impl.store.db.ReportingRecord import com.duckduckgo.pir.impl.store.db.ScanJobRecordEntity import com.duckduckgo.pir.impl.store.secure.PirSecureStorageDatabaseFactory import com.squareup.anvil.annotations.ContributesBinding @@ -124,6 +125,26 @@ interface PirSchedulingRepository { suspend fun deleteEmailConfirmationJobRecord(extractedProfileId: Long) suspend fun deleteAllEmailConfirmationJobRecords() + + suspend fun markOptOutDay7ConfirmationPixelSent( + extractedProfileId: Long, + timestampMs: Long, + ) + + suspend fun markOptOutDay14ConfirmationPixelSent( + extractedProfileId: Long, + timestampMs: Long, + ) + + suspend fun markOptOutDay21ConfirmationPixelSent( + extractedProfileId: Long, + timestampMs: Long, + ) + + suspend fun markOptOutDay42ConfirmationPixelSent( + extractedProfileId: Long, + timestampMs: Long, + ) } @ContributesBinding( @@ -319,6 +340,42 @@ class RealPirSchedulingRepository @Inject constructor( } } + override suspend fun markOptOutDay7ConfirmationPixelSent( + extractedProfileId: Long, + timestampMs: Long, + ) { + withContext(dispatcherProvider.io()) { + jobSchedulingDao()?.updateSevenDayConfirmationReportSentDate(extractedProfileId, timestampMs) + } + } + + override suspend fun markOptOutDay14ConfirmationPixelSent( + extractedProfileId: Long, + timestampMs: Long, + ) { + withContext(dispatcherProvider.io()) { + jobSchedulingDao()?.update14DayConfirmationReportSentDate(extractedProfileId, timestampMs) + } + } + + override suspend fun markOptOutDay21ConfirmationPixelSent( + extractedProfileId: Long, + timestampMs: Long, + ) { + withContext(dispatcherProvider.io()) { + jobSchedulingDao()?.update21DayConfirmationReportSentDate(extractedProfileId, timestampMs) + } + } + + override suspend fun markOptOutDay42ConfirmationPixelSent( + extractedProfileId: Long, + timestampMs: Long, + ) { + withContext(dispatcherProvider.io()) { + jobSchedulingDao()?.update42DayConfirmationReportSentDate(extractedProfileId, timestampMs) + } + } + private fun ScanJobRecordEntity.toRecord(): ScanJobRecord = ScanJobRecord( brokerName = this.brokerName, @@ -355,6 +412,10 @@ class RealPirSchedulingRepository @Inject constructor( optOutRemovedDateInMillis = this.optOutRemovedDate, deprecated = this.deprecated, dateCreatedInMillis = this.dateCreatedInMillis, + confirmation7dayReportSentDateMs = this.reporting.sevenDayConfirmationReportSentDateMs, + confirmation14dayReportSentDateMs = this.reporting.fourteenDayConfirmationReportSentDateMs, + confirmation21dayReportSentDateMs = this.reporting.twentyOneDayConfirmationReportSentDateMs, + confirmation42dayReportSentDateMs = this.reporting.fortyTwoDayConfirmationReportSentDateMs, ) private fun OptOutJobRecord.toEntity(): OptOutJobRecordEntity = @@ -373,6 +434,12 @@ class RealPirSchedulingRepository @Inject constructor( } else { currentTimeProvider.currentTimeMillis() }, + reporting = ReportingRecord( + sevenDayConfirmationReportSentDateMs = this.confirmation7dayReportSentDateMs, + fourteenDayConfirmationReportSentDateMs = this.confirmation14dayReportSentDateMs, + twentyOneDayConfirmationReportSentDateMs = this.confirmation21dayReportSentDateMs, + fortyTwoDayConfirmationReportSentDateMs = this.confirmation42dayReportSentDateMs, + ), ) private fun EmailConfirmationJobRecord.toEntity(): EmailConfirmationJobRecordEntity = diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/db/JobSchedulingDao.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/db/JobSchedulingDao.kt index e45f1bbad106..8e836a706502 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/db/JobSchedulingDao.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/db/JobSchedulingDao.kt @@ -118,4 +118,52 @@ interface JobSchedulingDao { deleteOptOutJobRecordsForProfiles(profileQueryIds) deleteEmailConfirmationJobRecordsForProfiles(profileQueryIds) } + + @Query( + """ + UPDATE pir_optout_job_record + SET reporting_sevenDayConfirmationReportSentDateMs = :newDate + WHERE extractedProfileId = :extractedProfileId + """, + ) + fun updateSevenDayConfirmationReportSentDate( + extractedProfileId: Long, + newDate: Long, + ) + + @Query( + """ + UPDATE pir_optout_job_record + SET reporting_fourteenDayConfirmationReportSentDateMs = :newDate + WHERE extractedProfileId = :extractedProfileId + """, + ) + fun update14DayConfirmationReportSentDate( + extractedProfileId: Long, + newDate: Long, + ) + + @Query( + """ + UPDATE pir_optout_job_record + SET reporting_twentyOneDayConfirmationReportSentDateMs = :newDate + WHERE extractedProfileId = :extractedProfileId + """, + ) + fun update21DayConfirmationReportSentDate( + extractedProfileId: Long, + newDate: Long, + ) + + @Query( + """ + UPDATE pir_optout_job_record + SET reporting_fortyTwoDayConfirmationReportSentDateMs = :newDate + WHERE extractedProfileId = :extractedProfileId + """, + ) + fun update42DayConfirmationReportSentDate( + extractedProfileId: Long, + newDate: Long, + ) } diff --git a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/db/JobSchedulingEntities.kt b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/db/JobSchedulingEntities.kt index 62e3bc1092be..69c7836009e1 100644 --- a/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/db/JobSchedulingEntities.kt +++ b/pir/pir-impl/src/main/java/com/duckduckgo/pir/impl/store/db/JobSchedulingEntities.kt @@ -16,6 +16,7 @@ package com.duckduckgo.pir.impl.store.db +import androidx.room.Embedded import androidx.room.Entity import androidx.room.PrimaryKey @@ -44,6 +45,15 @@ data class OptOutJobRecordEntity( val optOutRemovedDate: Long = 0L, val deprecated: Boolean = false, val dateCreatedInMillis: Long, + @Embedded(prefix = "reporting_") + val reporting: ReportingRecord, +) + +data class ReportingRecord( + val sevenDayConfirmationReportSentDateMs: Long = 0L, + val fourteenDayConfirmationReportSentDateMs: Long = 0L, + val twentyOneDayConfirmationReportSentDateMs: Long = 0L, + val fortyTwoDayConfirmationReportSentDateMs: Long = 0L, ) @Entity(tableName = "pir_email_confirmation_job_record") diff --git a/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/pixels/RealOptOut24HourSubmissionSuccessRateReporterTest.kt b/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/pixels/RealOptOut24HourSubmissionSuccessRateReporterTest.kt index e0dd3f314159..274404334961 100644 --- a/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/pixels/RealOptOut24HourSubmissionSuccessRateReporterTest.kt +++ b/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/pixels/RealOptOut24HourSubmissionSuccessRateReporterTest.kt @@ -16,12 +16,14 @@ package com.duckduckgo.pir.impl.pixels -import android.content.Context import com.duckduckgo.common.test.CoroutineTestRule import com.duckduckgo.common.utils.CurrentTimeProvider import com.duckduckgo.pir.impl.models.Broker import com.duckduckgo.pir.impl.models.ProfileQuery +import com.duckduckgo.pir.impl.models.scheduling.JobRecord.OptOutJobRecord +import com.duckduckgo.pir.impl.models.scheduling.JobRecord.OptOutJobRecord.OptOutJobStatus import com.duckduckgo.pir.impl.store.PirRepository +import com.duckduckgo.pir.impl.store.PirSchedulingRepository import kotlinx.coroutines.test.runTest import org.junit.Before import org.junit.Rule @@ -43,7 +45,7 @@ class RealOptOut24HourSubmissionSuccessRateReporterTest { private val mockCurrentTimeProvider: CurrentTimeProvider = mock() private val mockOptOutSubmitRateCalculator: OptOutSubmitRateCalculator = mock() private val mockPirPixelSender: PirPixelSender = mock() - private val context: Context = mock() + private val mockSchedulingRepository: PirSchedulingRepository = mock() private lateinit var toTest: RealOptOut24HourSubmissionSuccessRateReporter @@ -93,16 +95,24 @@ class RealOptOut24HourSubmissionSuccessRateReporterTest { pirRepository = mockPirRepository, currentTimeProvider = mockCurrentTimeProvider, pirPixelSender = mockPirPixelSender, + pirSchedulingRepository = mockSchedulingRepository, + dispatcherProvider = coroutineRule.testDispatcherProvider, ) } @Test fun whenFirstRunThenShouldFirePixel() = runTest { val now = baseTime + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker1.name, + ) + whenever(mockPirRepository.getCustomStatsPixelsLastSentMs()).thenReturn(0L) whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker1)) whenever(mockPirRepository.getAllUserProfileQueries()).thenReturn(listOf(testProfileQuery)) + whenever(mockSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) whenever( mockOptOutSubmitRateCalculator.calculateOptOutSubmitRate( any(), @@ -139,11 +149,16 @@ class RealOptOut24HourSubmissionSuccessRateReporterTest { fun whenMoreThan24HoursPassedThenShouldFirePixel() = runTest { val startDate = baseTime val now = baseTime + twentyFourHours + oneHour // 25 hours passed + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker1.name, + ) whenever(mockPirRepository.getCustomStatsPixelsLastSentMs()).thenReturn(startDate) whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker1)) whenever(mockPirRepository.getAllUserProfileQueries()).thenReturn(listOf(testProfileQuery)) + whenever(mockSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) whenever( mockOptOutSubmitRateCalculator.calculateOptOutSubmitRate( any(), @@ -201,6 +216,7 @@ class RealOptOut24HourSubmissionSuccessRateReporterTest { whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker1)) whenever(mockPirRepository.getAllUserProfileQueries()).thenReturn(emptyList()) + whenever(mockSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(emptyList()) toTest.attemptFirePixel() @@ -216,6 +232,15 @@ class RealOptOut24HourSubmissionSuccessRateReporterTest { @Test fun whenMultipleBrokersThenShouldFirePixelForEach() = runTest { val now = baseTime + val jobRecord1 = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker1.name, + ) + val jobRecord2 = createOptOutJobRecord( + extractedProfileId = 2L, + brokerName = testBroker2.name, + ) + whenever(mockPirRepository.getCustomStatsPixelsLastSentMs()).thenReturn(0L) whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn( @@ -225,19 +250,20 @@ class RealOptOut24HourSubmissionSuccessRateReporterTest { ), ) whenever(mockPirRepository.getAllUserProfileQueries()).thenReturn(listOf(testProfileQuery)) + whenever(mockSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord1, jobRecord2)) whenever( mockOptOutSubmitRateCalculator.calculateOptOutSubmitRate( - testBroker1.name, - 0L, - now - twentyFourHours, + eq(listOf(jobRecord1)), + eq(0L), + eq(now - twentyFourHours), ), ) .thenReturn(0.5) whenever( mockOptOutSubmitRateCalculator.calculateOptOutSubmitRate( - testBroker2.name, - 0L, - now - twentyFourHours, + eq(listOf(jobRecord2)), + eq(0L), + eq(now - twentyFourHours), ), ) .thenReturn(0.8) @@ -258,6 +284,15 @@ class RealOptOut24HourSubmissionSuccessRateReporterTest { @Test fun whenSuccessRateIsNullThenShouldNotFirePixelForThatBroker() = runTest { val now = baseTime + val jobRecord1 = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker1.name, + ) + val jobRecord2 = createOptOutJobRecord( + extractedProfileId = 2L, + brokerName = testBroker2.name, + ) + whenever(mockPirRepository.getCustomStatsPixelsLastSentMs()).thenReturn(0L) whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn( @@ -267,19 +302,20 @@ class RealOptOut24HourSubmissionSuccessRateReporterTest { ), ) whenever(mockPirRepository.getAllUserProfileQueries()).thenReturn(listOf(testProfileQuery)) + whenever(mockSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord1, jobRecord2)) whenever( mockOptOutSubmitRateCalculator.calculateOptOutSubmitRate( - testBroker1.name, - 0L, - now - twentyFourHours, + eq(listOf(jobRecord1)), + eq(0L), + eq(now - twentyFourHours), ), ) .thenReturn(0.5) whenever( mockOptOutSubmitRateCalculator.calculateOptOutSubmitRate( - testBroker2.name, - 0L, - now - twentyFourHours, + eq(listOf(jobRecord2)), + eq(0L), + eq(now - twentyFourHours), ), ) .thenReturn(null) @@ -300,10 +336,16 @@ class RealOptOut24HourSubmissionSuccessRateReporterTest { @Test fun whenAllSuccessRatesAreNullThenShouldNotFireAnyPixels() = runTest { val now = baseTime + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker1.name, + ) + whenever(mockPirRepository.getCustomStatsPixelsLastSentMs()).thenReturn(0L) whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker1)) whenever(mockPirRepository.getAllUserProfileQueries()).thenReturn(listOf(testProfileQuery)) + whenever(mockSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) whenever( mockOptOutSubmitRateCalculator.calculateOptOutSubmitRate( any(), @@ -323,11 +365,16 @@ class RealOptOut24HourSubmissionSuccessRateReporterTest { val startDate = baseTime val now = baseTime + twentyFourHours + oneHour val expectedEndDate = now - twentyFourHours + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker1.name, + ) whenever(mockPirRepository.getCustomStatsPixelsLastSentMs()).thenReturn(startDate) whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker1)) whenever(mockPirRepository.getAllUserProfileQueries()).thenReturn(listOf(testProfileQuery)) + whenever(mockSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) whenever( mockOptOutSubmitRateCalculator.calculateOptOutSubmitRate( any(), @@ -339,9 +386,9 @@ class RealOptOut24HourSubmissionSuccessRateReporterTest { toTest.attemptFirePixel() verify(mockOptOutSubmitRateCalculator).calculateOptOutSubmitRate( - brokerName = testBroker1.name, - startDateMs = startDate, - endDateMs = expectedEndDate, + allActiveOptOutJobsForBroker = eq(listOf(jobRecord)), + startDateMs = eq(startDate), + endDateMs = eq(expectedEndDate), ) verify(mockPirRepository).setCustomStatsPixelsLastSentMs(expectedEndDate) } @@ -349,6 +396,15 @@ class RealOptOut24HourSubmissionSuccessRateReporterTest { @Test fun whenMultipleBrokersWithMixedSuccessRatesThenFiresPixelsForNonNullRates() = runTest { val now = baseTime + val jobRecord1 = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker1.name, + ) + val jobRecord2 = createOptOutJobRecord( + extractedProfileId = 2L, + brokerName = testBroker2.name, + ) + whenever(mockPirRepository.getCustomStatsPixelsLastSentMs()).thenReturn(0L) whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn( @@ -358,19 +414,20 @@ class RealOptOut24HourSubmissionSuccessRateReporterTest { ), ) whenever(mockPirRepository.getAllUserProfileQueries()).thenReturn(listOf(testProfileQuery)) + whenever(mockSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord1, jobRecord2)) whenever( mockOptOutSubmitRateCalculator.calculateOptOutSubmitRate( - testBroker1.name, - 0L, - now - twentyFourHours, + eq(listOf(jobRecord1)), + eq(0L), + eq(now - twentyFourHours), ), ) .thenReturn(null) whenever( mockOptOutSubmitRateCalculator.calculateOptOutSubmitRate( - testBroker2.name, - 0L, - now - twentyFourHours, + eq(listOf(jobRecord2)), + eq(0L), + eq(now - twentyFourHours), ), ) .thenReturn(0.9) @@ -433,6 +490,7 @@ class RealOptOut24HourSubmissionSuccessRateReporterTest { whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker1)) whenever(mockPirRepository.getAllUserProfileQueries()).thenReturn(emptyList()) + whenever(mockSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(emptyList()) toTest.attemptFirePixel() @@ -444,4 +502,70 @@ class RealOptOut24HourSubmissionSuccessRateReporterTest { verify(mockPirPixelSender, never()).reportBrokerCustomStateOptOutSubmitRate(any(), any()) verify(mockPirRepository, never()).setCustomStatsPixelsLastSentMs(any()) } + + @Test + fun whenBrokerHasNoJobRecordsThenSkipsThatBroker() = runTest { + val now = baseTime + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker1.name, + ) + + whenever(mockPirRepository.getCustomStatsPixelsLastSentMs()).thenReturn(0L) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn( + listOf( + testBroker1, + testBroker2, // This broker has no job records + ), + ) + whenever(mockPirRepository.getAllUserProfileQueries()).thenReturn(listOf(testProfileQuery)) + whenever(mockSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) // Only for broker1 + whenever( + mockOptOutSubmitRateCalculator.calculateOptOutSubmitRate( + eq(listOf(jobRecord)), + eq(0L), + eq(now - twentyFourHours), + ), + ).thenReturn(0.5) + + toTest.attemptFirePixel() + + // Only broker1 should fire pixel, broker2 should be skipped + verify(mockPirPixelSender).reportBrokerCustomStateOptOutSubmitRate( + brokerUrl = testBroker1.url, + optOutSuccessRate = 0.5, + ) + verify(mockPirPixelSender, never()).reportBrokerCustomStateOptOutSubmitRate( + brokerUrl = eq(testBroker2.url), + optOutSuccessRate = any(), + ) + verify(mockPirRepository).setCustomStatsPixelsLastSentMs(now - twentyFourHours) + } + + private fun createOptOutJobRecord( + extractedProfileId: Long, + brokerName: String = testBroker1.name, + userProfileId: Long = 1L, + status: OptOutJobStatus = OptOutJobStatus.NOT_EXECUTED, + dateCreatedInMillis: Long = baseTime, + optOutRequestedDateInMillis: Long = 0L, + optOutRemovedDateInMillis: Long = 0L, + attemptCount: Int = 0, + lastOptOutAttemptDateInMillis: Long = 0L, + deprecated: Boolean = false, + ): OptOutJobRecord { + return OptOutJobRecord( + brokerName = brokerName, + userProfileId = userProfileId, + extractedProfileId = extractedProfileId, + status = status, + attemptCount = attemptCount, + lastOptOutAttemptDateInMillis = lastOptOutAttemptDateInMillis, + optOutRequestedDateInMillis = optOutRequestedDateInMillis, + optOutRemovedDateInMillis = optOutRemovedDateInMillis, + deprecated = deprecated, + dateCreatedInMillis = dateCreatedInMillis, + ) + } } diff --git a/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/pixels/RealOptOutConfirmationReporterTest.kt b/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/pixels/RealOptOutConfirmationReporterTest.kt new file mode 100644 index 000000000000..f00ff0f098da --- /dev/null +++ b/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/pixels/RealOptOutConfirmationReporterTest.kt @@ -0,0 +1,526 @@ +/* + * Copyright (c) 2025 DuckDuckGo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.duckduckgo.pir.impl.pixels + +import com.duckduckgo.common.test.CoroutineTestRule +import com.duckduckgo.common.utils.CurrentTimeProvider +import com.duckduckgo.pir.impl.models.Broker +import com.duckduckgo.pir.impl.models.scheduling.JobRecord.OptOutJobRecord +import com.duckduckgo.pir.impl.models.scheduling.JobRecord.OptOutJobRecord.OptOutJobStatus +import com.duckduckgo.pir.impl.store.PirRepository +import com.duckduckgo.pir.impl.store.PirSchedulingRepository +import kotlinx.coroutines.test.runTest +import org.junit.Before +import org.junit.Rule +import org.junit.Test +import org.mockito.kotlin.any +import org.mockito.kotlin.mock +import org.mockito.kotlin.never +import org.mockito.kotlin.verify +import org.mockito.kotlin.verifyNoInteractions +import org.mockito.kotlin.whenever +import java.util.concurrent.TimeUnit + +class RealOptOutConfirmationReporterTest { + + @get:Rule + val coroutineRule = CoroutineTestRule() + + private lateinit var testee: RealOptOutConfirmationReporter + + private val mockPirSchedulingRepository: PirSchedulingRepository = mock() + private val mockPirRepository: PirRepository = mock() + private val mockCurrentTimeProvider: CurrentTimeProvider = mock() + private val mockPixelSender: PirPixelSender = mock() + + // Test data + // January 15, 2024 10:00:00 UTC + private val baseTime = 1705309200000L + private val oneDay = TimeUnit.DAYS.toMillis(1) + private val sevenDays = TimeUnit.DAYS.toMillis(7) + private val fourteenDays = TimeUnit.DAYS.toMillis(14) + private val twentyOneDays = TimeUnit.DAYS.toMillis(21) + private val fortyTwoDays = TimeUnit.DAYS.toMillis(42) + + private val testBroker = Broker( + name = "test-broker", + fileName = "test-broker.json", + url = "https://test-broker.com", + version = "1.0", + parent = null, + addedDatetime = baseTime, + removedAt = 0L, + ) + + @Before + fun setUp() { + testee = RealOptOutConfirmationReporter( + dispatcherProvider = coroutineRule.testDispatcherProvider, + pirSchedulingRepository = mockPirSchedulingRepository, + pirRepository = mockPirRepository, + currentTimeProvider = mockCurrentTimeProvider, + pixelSender = mockPixelSender, + ) + } + + @Test + fun whenNoActiveBrokersThenDoesNotFirePixels() = runTest { + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(emptyList()) + whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(emptyList()) + + testee.attemptFirePixel() + + verifyNoInteractions(mockPixelSender) + verify(mockPirSchedulingRepository, never()).markOptOutDay7ConfirmationPixelSent(any(), any()) + } + + @Test + fun whenNoOptOutJobsThenDoesNotFirePixels() = runTest { + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker)) + whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(emptyList()) + + testee.attemptFirePixel() + + verifyNoInteractions(mockPixelSender) + verify(mockPirSchedulingRepository, never()).markOptOutDay7ConfirmationPixelSent(any(), any()) + } + + @Test + fun whenOptOutJobNotRequestedOrRemovedThenDoesNotFirePixels() = runTest { + val now = baseTime + sevenDays + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker.name, + status = OptOutJobStatus.NOT_EXECUTED, + optOutRequestedDateInMillis = baseTime, + ) + + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker)) + whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + + testee.attemptFirePixel() + + verifyNoInteractions(mockPixelSender) + verify(mockPirSchedulingRepository, never()).markOptOutDay7ConfirmationPixelSent(any(), any()) + } + + @Test + fun when7DaysPassedAndStatusIsRemovedThenFiresConfirmed7dayPixel() = runTest { + val now = baseTime + sevenDays + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker.name, + status = OptOutJobStatus.REMOVED, + optOutRequestedDateInMillis = baseTime, + ) + + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker)) + whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + + testee.attemptFirePixel() + + verify(mockPixelSender).reportBrokerOptOutConfirmed7Days(testBroker.url) + verify(mockPirSchedulingRepository).markOptOutDay7ConfirmationPixelSent(1L, now) + } + + @Test + fun when7DaysPassedAndStatusIsRequestedThenFiresUnconfirmed7dayPixel() = runTest { + val now = baseTime + sevenDays + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker.name, + status = OptOutJobStatus.REQUESTED, + optOutRequestedDateInMillis = baseTime, + ) + + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker)) + whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + + testee.attemptFirePixel() + + verify(mockPixelSender).reportBrokerOptOutUnconfirmed7Days(testBroker.url) + verify(mockPirSchedulingRepository).markOptOutDay7ConfirmationPixelSent(1L, now) + } + + @Test + fun when7DaysPixelAlreadySentThenDoesNotFireAgain() = runTest { + val now = baseTime + sevenDays + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker.name, + status = OptOutJobStatus.REMOVED, + optOutRequestedDateInMillis = baseTime, + confirmation7dayReportSentDateMs = baseTime + oneDay, // Already sent + ) + + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker)) + whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + + testee.attemptFirePixel() + + verify(mockPixelSender, never()).reportBrokerOptOutConfirmed7Days(any()) + verify(mockPirSchedulingRepository, never()).markOptOutDay7ConfirmationPixelSent(any(), any()) + } + + @Test + fun whenLessThan7DaysPassedThenDoesNotFire7DayPixel() = runTest { + val now = baseTime + sevenDays - TimeUnit.HOURS.toMillis(1) // 1 hour before 7 days + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker.name, + status = OptOutJobStatus.REMOVED, + optOutRequestedDateInMillis = baseTime, + ) + + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker)) + whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + + testee.attemptFirePixel() + + verify(mockPixelSender, never()).reportBrokerOptOutConfirmed7Days(any()) + verify(mockPirSchedulingRepository, never()).markOptOutDay7ConfirmationPixelSent(any(), any()) + } + + @Test + fun whenExactly7DaysPassedThenFires7DayPixel() = runTest { + val now = baseTime + sevenDays // Exactly 7 days + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker.name, + status = OptOutJobStatus.REMOVED, + optOutRequestedDateInMillis = baseTime, + ) + + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker)) + whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + + testee.attemptFirePixel() + + verify(mockPixelSender).reportBrokerOptOutConfirmed7Days(testBroker.url) + verify(mockPirSchedulingRepository).markOptOutDay7ConfirmationPixelSent(1L, now) + } + + @Test + fun when14DaysPassedThenFires14DayPixel() = runTest { + val now = baseTime + fourteenDays + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker.name, + status = OptOutJobStatus.REMOVED, + optOutRequestedDateInMillis = baseTime, + ) + + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker)) + whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + + testee.attemptFirePixel() + + verify(mockPixelSender).reportBrokerOptOutConfirmed14Days(testBroker.url) + verify(mockPirSchedulingRepository).markOptOutDay14ConfirmationPixelSent(1L, now) + } + + @Test + fun when21DaysPassedThenFires21DayPixel() = runTest { + val now = baseTime + twentyOneDays + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker.name, + status = OptOutJobStatus.REMOVED, + optOutRequestedDateInMillis = baseTime, + ) + + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker)) + whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + + testee.attemptFirePixel() + + verify(mockPixelSender).reportBrokerOptOutConfirmed21Days(testBroker.url) + verify(mockPirSchedulingRepository).markOptOutDay21ConfirmationPixelSent(1L, now) + } + + @Test + fun when42DaysPassedThenFires42DayPixel() = runTest { + val now = baseTime + fortyTwoDays + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker.name, + status = OptOutJobStatus.REMOVED, + optOutRequestedDateInMillis = baseTime, + ) + + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker)) + whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + + testee.attemptFirePixel() + + verify(mockPixelSender).reportBrokerOptOutConfirmed42Days(testBroker.url) + verify(mockPirSchedulingRepository).markOptOutDay42ConfirmationPixelSent(1L, now) + } + + @Test + fun whenMultipleIntervalsPassedThenFiresAllApplicablePixels() = runTest { + val now = baseTime + fortyTwoDays // 42 days passed + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker.name, + status = OptOutJobStatus.REMOVED, + optOutRequestedDateInMillis = baseTime, + ) + + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker)) + whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + + testee.attemptFirePixel() + + verify(mockPixelSender).reportBrokerOptOutConfirmed7Days(testBroker.url) + verify(mockPixelSender).reportBrokerOptOutConfirmed14Days(testBroker.url) + verify(mockPixelSender).reportBrokerOptOutConfirmed21Days(testBroker.url) + verify(mockPixelSender).reportBrokerOptOutConfirmed42Days(testBroker.url) + } + + @Test + fun whenBrokerNotFoundThenSkipsJobRecord() = runTest { + val now = baseTime + sevenDays + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = "unknown-broker", + status = OptOutJobStatus.REMOVED, + optOutRequestedDateInMillis = baseTime, + ) + + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker)) + whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + + testee.attemptFirePixel() + + verify(mockPixelSender, never()).reportBrokerOptOutConfirmed7Days(any()) + verify(mockPirSchedulingRepository, never()).markOptOutDay7ConfirmationPixelSent(any(), any()) + } + + @Test + fun whenMultipleJobRecordsThenFiresPixelsForEach() = runTest { + val now = baseTime + sevenDays + val jobRecord1 = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker.name, + status = OptOutJobStatus.REMOVED, + optOutRequestedDateInMillis = baseTime, + ) + val jobRecord2 = createOptOutJobRecord( + extractedProfileId = 2L, + brokerName = testBroker.name, + status = OptOutJobStatus.REQUESTED, + optOutRequestedDateInMillis = baseTime, + ) + + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker)) + whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord1, jobRecord2)) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + + testee.attemptFirePixel() + + verify(mockPixelSender).reportBrokerOptOutConfirmed7Days(testBroker.url) + verify(mockPixelSender).reportBrokerOptOutUnconfirmed7Days(testBroker.url) + verify(mockPirSchedulingRepository).markOptOutDay7ConfirmationPixelSent(1L, now) + verify(mockPirSchedulingRepository).markOptOutDay7ConfirmationPixelSent(2L, now) + } + + @Test + fun when14DayPixelAlreadySentThenDoesNotFireAgain() = runTest { + val now = baseTime + fourteenDays + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker.name, + status = OptOutJobStatus.REMOVED, + optOutRequestedDateInMillis = baseTime, + confirmation14dayReportSentDateMs = baseTime + oneDay, // Already sent + ) + + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker)) + whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + + testee.attemptFirePixel() + + verify(mockPixelSender, never()).reportBrokerOptOutConfirmed14Days(any()) + verify(mockPirSchedulingRepository, never()).markOptOutDay14ConfirmationPixelSent(any(), any()) + } + + @Test + fun when21DayPixelAlreadySentThenDoesNotFireAgain() = runTest { + val now = baseTime + twentyOneDays + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker.name, + status = OptOutJobStatus.REMOVED, + optOutRequestedDateInMillis = baseTime, + confirmation21dayReportSentDateMs = baseTime + oneDay, // Already sent + ) + + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker)) + whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + + testee.attemptFirePixel() + + verify(mockPixelSender, never()).reportBrokerOptOutConfirmed21Days(any()) + verify(mockPirSchedulingRepository, never()).markOptOutDay21ConfirmationPixelSent(any(), any()) + } + + @Test + fun when42DayPixelAlreadySentThenDoesNotFireAgain() = runTest { + val now = baseTime + fortyTwoDays + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker.name, + status = OptOutJobStatus.REMOVED, + optOutRequestedDateInMillis = baseTime, + confirmation42dayReportSentDateMs = baseTime + oneDay, // Already sent + ) + + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker)) + whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + + testee.attemptFirePixel() + + verify(mockPixelSender, never()).reportBrokerOptOutConfirmed42Days(any()) + verify(mockPirSchedulingRepository, never()).markOptOutDay42ConfirmationPixelSent(any(), any()) + } + + @Test + fun when7DaysPassedBut14DaysNotPassedThenOnlyFires7DayPixel() = runTest { + val now = baseTime + sevenDays + oneDay // 8 days passed + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker.name, + status = OptOutJobStatus.REMOVED, + optOutRequestedDateInMillis = baseTime, + ) + + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker)) + whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + + testee.attemptFirePixel() + + verify(mockPixelSender).reportBrokerOptOutConfirmed7Days(testBroker.url) + verify(mockPixelSender, never()).reportBrokerOptOutConfirmed14Days(any()) + } + + @Test + fun when14DaysPassedBut21DaysNotPassedThenFires7And14DayPixels() = runTest { + val now = baseTime + fourteenDays + oneDay // 15 days passed + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker.name, + status = OptOutJobStatus.REMOVED, + optOutRequestedDateInMillis = baseTime, + ) + + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker)) + whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + + testee.attemptFirePixel() + + verify(mockPixelSender).reportBrokerOptOutConfirmed7Days(testBroker.url) + verify(mockPixelSender).reportBrokerOptOutConfirmed14Days(testBroker.url) + verify(mockPixelSender, never()).reportBrokerOptOutConfirmed21Days(any()) + } + + @Test + fun whenRequestedStatusThenFiresUnconfirmedPixels() = runTest { + val now = baseTime + sevenDays + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker.name, + status = OptOutJobStatus.REQUESTED, + optOutRequestedDateInMillis = baseTime, + ) + + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker)) + whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + + testee.attemptFirePixel() + + verify(mockPixelSender).reportBrokerOptOutUnconfirmed7Days(testBroker.url) + verify(mockPixelSender, never()).reportBrokerOptOutConfirmed7Days(any()) + } + + @Test + fun whenRemovedStatusThenFiresConfirmedPixels() = runTest { + val now = baseTime + sevenDays + val jobRecord = createOptOutJobRecord( + extractedProfileId = 1L, + brokerName = testBroker.name, + status = OptOutJobStatus.REMOVED, + optOutRequestedDateInMillis = baseTime, + ) + + whenever(mockPirRepository.getAllActiveBrokerObjects()).thenReturn(listOf(testBroker)) + whenever(mockPirSchedulingRepository.getAllValidOptOutJobRecords()).thenReturn(listOf(jobRecord)) + whenever(mockCurrentTimeProvider.currentTimeMillis()).thenReturn(now) + + testee.attemptFirePixel() + + verify(mockPixelSender).reportBrokerOptOutConfirmed7Days(testBroker.url) + verify(mockPixelSender, never()).reportBrokerOptOutUnconfirmed7Days(any()) + } + + private fun createOptOutJobRecord( + extractedProfileId: Long, + brokerName: String = testBroker.name, + userProfileId: Long = 1L, + status: OptOutJobStatus = OptOutJobStatus.REQUESTED, + optOutRequestedDateInMillis: Long = baseTime, + optOutRemovedDateInMillis: Long = 0L, + confirmation7dayReportSentDateMs: Long = 0L, + confirmation14dayReportSentDateMs: Long = 0L, + confirmation21dayReportSentDateMs: Long = 0L, + confirmation42dayReportSentDateMs: Long = 0L, + ): OptOutJobRecord { + return OptOutJobRecord( + brokerName = brokerName, + userProfileId = userProfileId, + extractedProfileId = extractedProfileId, + status = status, + attemptCount = 0, + lastOptOutAttemptDateInMillis = 0L, + optOutRequestedDateInMillis = optOutRequestedDateInMillis, + optOutRemovedDateInMillis = optOutRemovedDateInMillis, + deprecated = false, + dateCreatedInMillis = baseTime, + confirmation7dayReportSentDateMs = confirmation7dayReportSentDateMs, + confirmation14dayReportSentDateMs = confirmation14dayReportSentDateMs, + confirmation21dayReportSentDateMs = confirmation21dayReportSentDateMs, + confirmation42dayReportSentDateMs = confirmation42dayReportSentDateMs, + ) + } +} diff --git a/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/pixels/RealOptOutSubmitRateCalculatorTest.kt b/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/pixels/RealOptOutSubmitRateCalculatorTest.kt index f51f6fa8523b..6f3cc7407121 100644 --- a/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/pixels/RealOptOutSubmitRateCalculatorTest.kt +++ b/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/pixels/RealOptOutSubmitRateCalculatorTest.kt @@ -19,15 +19,12 @@ package com.duckduckgo.pir.impl.pixels import com.duckduckgo.common.test.CoroutineTestRule import com.duckduckgo.pir.impl.models.scheduling.JobRecord.OptOutJobRecord import com.duckduckgo.pir.impl.models.scheduling.JobRecord.OptOutJobRecord.OptOutJobStatus -import com.duckduckgo.pir.impl.store.PirSchedulingRepository import kotlinx.coroutines.test.runTest import org.junit.Assert.assertEquals import org.junit.Assert.assertNull import org.junit.Before import org.junit.Rule import org.junit.Test -import org.mockito.kotlin.mock -import org.mockito.kotlin.whenever import java.util.concurrent.TimeUnit class RealOptOutSubmitRateCalculatorTest { @@ -37,19 +34,15 @@ class RealOptOutSubmitRateCalculatorTest { private lateinit var testee: RealOptOutSubmitRateCalculator - private val mockSchedulingRepository: PirSchedulingRepository = mock() - @Before fun setUp() { testee = RealOptOutSubmitRateCalculator( dispatcherProvider = coroutineRule.testDispatcherProvider, - schedulingRepository = mockSchedulingRepository, ) } // Test data private val testBrokerName = "test-broker" - private val testBrokerName2 = "test-broker-2" // January 15, 2024 10:00:00 UTC private val baseTime = 1705309200000L @@ -62,10 +55,7 @@ class RealOptOutSubmitRateCalculatorTest { val startDate = baseTime val endDate = baseTime + oneDay - whenever(mockSchedulingRepository.getAllValidOptOutJobRecordsForBroker(testBrokerName)) - .thenReturn(emptyList()) - - val result = testee.calculateOptOutSubmitRate(testBrokerName, startDate, endDate) + val result = testee.calculateOptOutSubmitRate(emptyList(), startDate, endDate) assertNull(result) } @@ -86,10 +76,11 @@ class RealOptOutSubmitRateCalculatorTest { dateCreatedInMillis = baseTime + oneDay + oneHour, ) - whenever(mockSchedulingRepository.getAllValidOptOutJobRecordsForBroker(testBrokerName)) - .thenReturn(listOf(recordBeforeRange, recordAfterRange)) - - val result = testee.calculateOptOutSubmitRate(testBrokerName, startDate, endDate) + val result = testee.calculateOptOutSubmitRate( + listOf(recordBeforeRange, recordAfterRange), + startDate, + endDate, + ) assertNull(result) } @@ -111,11 +102,7 @@ class RealOptOutSubmitRateCalculatorTest { status = OptOutJobStatus.ERROR, dateCreatedInMillis = baseTime + 2 * oneHour, ) - - whenever(mockSchedulingRepository.getAllValidOptOutJobRecordsForBroker(testBrokerName)) - .thenReturn(listOf(record1, record2)) - - val result = testee.calculateOptOutSubmitRate(testBrokerName, startDate, endDate) + val result = testee.calculateOptOutSubmitRate(listOf(record1, record2), startDate, endDate) assertEquals(0.0, result!!, 0.0) } @@ -140,11 +127,7 @@ class RealOptOutSubmitRateCalculatorTest { dateCreatedInMillis = dateCreated + 2 * oneHour, optOutRequestedDateInMillis = dateCreated + 3 * oneHour, // Within 24 hours ) - - whenever(mockSchedulingRepository.getAllValidOptOutJobRecordsForBroker(testBrokerName)) - .thenReturn(listOf(record1, record2)) - - val result = testee.calculateOptOutSubmitRate(testBrokerName, startDate, endDate) + val result = testee.calculateOptOutSubmitRate(listOf(record1, record2), startDate, endDate) assertEquals(1.0, result!!, 0.0) } @@ -168,11 +151,11 @@ class RealOptOutSubmitRateCalculatorTest { status = OptOutJobStatus.NOT_EXECUTED, dateCreatedInMillis = dateCreated + 2 * oneHour, ) - - whenever(mockSchedulingRepository.getAllValidOptOutJobRecordsForBroker(testBrokerName)) - .thenReturn(listOf(requestedRecord, notExecutedRecord)) - - val result = testee.calculateOptOutSubmitRate(testBrokerName, startDate, endDate) + val result = testee.calculateOptOutSubmitRate( + listOf(requestedRecord, notExecutedRecord), + startDate, + endDate, + ) assertEquals(0.5, result!!, 0.0) } @@ -197,11 +180,7 @@ class RealOptOutSubmitRateCalculatorTest { dateCreatedInMillis = dateCreated + 2 * oneHour, optOutRequestedDateInMillis = dateCreated + 2 * oneHour + twentyFourHours + oneHour, // Outside 24 hours ) - - whenever(mockSchedulingRepository.getAllValidOptOutJobRecordsForBroker(testBrokerName)) - .thenReturn(listOf(requestedWithinWindow, requestedOutsideWindow)) - - val result = testee.calculateOptOutSubmitRate(testBrokerName, startDate, endDate) + val result = testee.calculateOptOutSubmitRate(listOf(requestedWithinWindow, requestedOutsideWindow), startDate, endDate) assertEquals(0.5, result!!, 0.0) } @@ -225,46 +204,11 @@ class RealOptOutSubmitRateCalculatorTest { status = OptOutJobStatus.NOT_EXECUTED, dateCreatedInMillis = dateCreated + 2 * oneHour, ) - - whenever(mockSchedulingRepository.getAllValidOptOutJobRecordsForBroker(testBrokerName)) - .thenReturn(listOf(requestedAt24Hours, notExecutedRecord)) - - val result = testee.calculateOptOutSubmitRate(testBrokerName, startDate, endDate) + val result = testee.calculateOptOutSubmitRate(listOf(requestedAt24Hours, notExecutedRecord), startDate, endDate) assertEquals(0.5, result!!, 0.0) } - @Test - fun whenRecordsFromDifferentBrokersThenOnlyCountMatchingBroker() = runTest { - val startDate = baseTime - val endDate = baseTime + oneDay - val dateCreated = baseTime + oneHour - - val recordForTestBroker = createOptOutJobRecord( - extractedProfileId = 1L, - brokerName = testBrokerName, - status = OptOutJobStatus.REQUESTED, - dateCreatedInMillis = dateCreated, - optOutRequestedDateInMillis = dateCreated + oneHour, - ) - val recordForOtherBroker = createOptOutJobRecord( - extractedProfileId = 2L, - brokerName = testBrokerName2, - status = OptOutJobStatus.REQUESTED, - dateCreatedInMillis = dateCreated + 2 * oneHour, - optOutRequestedDateInMillis = dateCreated + 3 * oneHour, - ) - - whenever(mockSchedulingRepository.getAllValidOptOutJobRecordsForBroker(testBrokerName)) - .thenReturn(listOf(recordForTestBroker)) - whenever(mockSchedulingRepository.getAllValidOptOutJobRecordsForBroker(testBrokerName2)) - .thenReturn(listOf(recordForOtherBroker)) - - val result = testee.calculateOptOutSubmitRate(testBrokerName, startDate, endDate) - - assertEquals(1.0, result!!, 0.0) - } - @Test fun whenStartDateIsZeroThenUseDefaultStartDate() = runTest { val endDate = baseTime + oneDay @@ -277,11 +221,7 @@ class RealOptOutSubmitRateCalculatorTest { dateCreatedInMillis = dateCreated, optOutRequestedDateInMillis = dateCreated + oneHour, ) - - whenever(mockSchedulingRepository.getAllValidOptOutJobRecordsForBroker(testBrokerName)) - .thenReturn(listOf(record)) - - val result = testee.calculateOptOutSubmitRate(testBrokerName, endDateMs = endDate) + val result = testee.calculateOptOutSubmitRate(listOf(record), endDateMs = endDate) assertEquals(1.0, result!!, 0.0) } @@ -298,11 +238,7 @@ class RealOptOutSubmitRateCalculatorTest { dateCreatedInMillis = startDate, // Exactly at start optOutRequestedDateInMillis = startDate + oneHour, ) - - whenever(mockSchedulingRepository.getAllValidOptOutJobRecordsForBroker(testBrokerName)) - .thenReturn(listOf(recordAtStart)) - - val result = testee.calculateOptOutSubmitRate(testBrokerName, startDate, endDate) + val result = testee.calculateOptOutSubmitRate(listOf(recordAtStart), startDate, endDate) assertEquals(1.0, result!!, 0.0) } @@ -319,11 +255,7 @@ class RealOptOutSubmitRateCalculatorTest { dateCreatedInMillis = endDate, // Exactly at end optOutRequestedDateInMillis = endDate + oneHour, ) - - whenever(mockSchedulingRepository.getAllValidOptOutJobRecordsForBroker(testBrokerName)) - .thenReturn(listOf(recordAtEnd)) - - val result = testee.calculateOptOutSubmitRate(testBrokerName, startDate, endDate) + val result = testee.calculateOptOutSubmitRate(listOf(recordAtEnd), startDate, endDate) assertEquals(1.0, result!!, 0.0) } @@ -359,11 +291,7 @@ class RealOptOutSubmitRateCalculatorTest { status = OptOutJobStatus.PENDING_EMAIL_CONFIRMATION, dateCreatedInMillis = dateCreated + 4 * oneHour, ) - - whenever(mockSchedulingRepository.getAllValidOptOutJobRecordsForBroker(testBrokerName)) - .thenReturn(listOf(requested, removed, error, pendingEmail)) - - val result = testee.calculateOptOutSubmitRate(testBrokerName, startDate, endDate) + val result = testee.calculateOptOutSubmitRate(listOf(requested, removed, error, pendingEmail), startDate, endDate) assertEquals(0.25, result!!, 0.0) } @@ -408,11 +336,7 @@ class RealOptOutSubmitRateCalculatorTest { status = OptOutJobStatus.ERROR, dateCreatedInMillis = dateCreated + 5 * oneHour, ) - - whenever(mockSchedulingRepository.getAllValidOptOutJobRecordsForBroker(testBrokerName)) - .thenReturn(listOf(requested1, requested2, requested3, notExecuted, error)) - - val result = testee.calculateOptOutSubmitRate(testBrokerName, startDate, endDate) + val result = testee.calculateOptOutSubmitRate(listOf(requested1, requested2, requested3, notExecuted, error), startDate, endDate) // Only requested1 and requested3 count (2 out of 5) assertEquals(0.4, result!!, 0.0) @@ -438,11 +362,7 @@ class RealOptOutSubmitRateCalculatorTest { dateCreatedInMillis = dateCreated + 2 * oneHour, optOutRequestedDateInMillis = dateCreated + 2 * oneHour + oneHour, // After creation ) - - whenever(mockSchedulingRepository.getAllValidOptOutJobRecordsForBroker(testBrokerName)) - .thenReturn(listOf(requestedAtSameTime, validRequested)) - - val result = testee.calculateOptOutSubmitRate(testBrokerName, startDate, endDate) + val result = testee.calculateOptOutSubmitRate(listOf(requestedAtSameTime, validRequested), startDate, endDate) // Only validRequested counts (1 out of 2) assertEquals(0.5, result!!, 0.0) @@ -468,11 +388,7 @@ class RealOptOutSubmitRateCalculatorTest { dateCreatedInMillis = dateCreated + 2 * oneHour, optOutRequestedDateInMillis = dateCreated + 3 * oneHour, // After creation ) - - whenever(mockSchedulingRepository.getAllValidOptOutJobRecordsForBroker(testBrokerName)) - .thenReturn(listOf(requestedBeforeCreation, validRequested)) - - val result = testee.calculateOptOutSubmitRate(testBrokerName, startDate, endDate) + val result = testee.calculateOptOutSubmitRate(listOf(requestedBeforeCreation, validRequested), startDate, endDate) // Only validRequested counts (1 out of 2) assertEquals(0.5, result!!, 0.0) @@ -493,10 +409,7 @@ class RealOptOutSubmitRateCalculatorTest { optOutRequestedDateInMillis = dateCreated + oneMillisecond, // Just 1ms after (should be counted) ) - whenever(mockSchedulingRepository.getAllValidOptOutJobRecordsForBroker(testBrokerName)) - .thenReturn(listOf(requestedJustAfter)) - - val result = testee.calculateOptOutSubmitRate(testBrokerName, startDate, endDate) + val result = testee.calculateOptOutSubmitRate(listOf(requestedJustAfter), startDate, endDate) assertEquals(1.0, result!!, 0.0) } @@ -515,11 +428,7 @@ class RealOptOutSubmitRateCalculatorTest { dateCreatedInMillis = dateCreated, optOutRequestedDateInMillis = dateCreated + twentyFourHours - oneMillisecond, // Just before 24h limit ) - - whenever(mockSchedulingRepository.getAllValidOptOutJobRecordsForBroker(testBrokerName)) - .thenReturn(listOf(requestedJustBefore24h)) - - val result = testee.calculateOptOutSubmitRate(testBrokerName, startDate, endDate) + val result = testee.calculateOptOutSubmitRate(listOf(requestedJustBefore24h), startDate, endDate) assertEquals(1.0, result!!, 0.0) } @@ -544,11 +453,7 @@ class RealOptOutSubmitRateCalculatorTest { status = OptOutJobStatus.NOT_EXECUTED, dateCreatedInMillis = dateCreated + 2 * oneHour, ) - - whenever(mockSchedulingRepository.getAllValidOptOutJobRecordsForBroker(testBrokerName)) - .thenReturn(listOf(requestedJustAfter24h, notExecuted)) - - val result = testee.calculateOptOutSubmitRate(testBrokerName, startDate, endDate) + val result = testee.calculateOptOutSubmitRate(listOf(requestedJustAfter24h, notExecuted), startDate, endDate) // requestedJustAfter24h doesn't count, so 0 out of 2 assertEquals(0.0, result!!, 0.0) @@ -580,11 +485,7 @@ class RealOptOutSubmitRateCalculatorTest { status = OptOutJobStatus.NOT_EXECUTED, dateCreatedInMillis = dateCreated + 3 * oneHour, ) - - whenever(mockSchedulingRepository.getAllValidOptOutJobRecordsForBroker(testBrokerName)) - .thenReturn(listOf(requested, notExecuted1, notExecuted2)) - - val result = testee.calculateOptOutSubmitRate(testBrokerName, startDate, endDate) + val result = testee.calculateOptOutSubmitRate(listOf(requested, notExecuted1, notExecuted2), startDate, endDate) // 1/3 = 0.333... rounded to 0.33 assertEquals(0.33, result!!, 0.0) @@ -617,48 +518,12 @@ class RealOptOutSubmitRateCalculatorTest { status = OptOutJobStatus.NOT_EXECUTED, dateCreatedInMillis = dateCreated + 4 * oneHour, ) - - whenever(mockSchedulingRepository.getAllValidOptOutJobRecordsForBroker(testBrokerName)) - .thenReturn(listOf(requested1, requested2, notExecuted)) - - val result = testee.calculateOptOutSubmitRate(testBrokerName, startDate, endDate) + val result = testee.calculateOptOutSubmitRate(listOf(requested1, requested2, notExecuted), startDate, endDate) // 2/3 = 0.666... rounded to 0.67 assertEquals(0.67, result!!, 0.0) } - @Test - fun whenRepositoryReturnsWrongBrokerNameThenExcluded() = runTest { - val startDate = baseTime - val endDate = baseTime + oneDay - val dateCreated = baseTime + oneHour - - // Repository might return records with wrong broker name (should be filtered out) - val recordWithWrongBroker = createOptOutJobRecord( - extractedProfileId = 1L, - brokerName = testBrokerName2, // Wrong broker - status = OptOutJobStatus.REQUESTED, - dateCreatedInMillis = dateCreated, - optOutRequestedDateInMillis = dateCreated + oneHour, - ) - val recordWithCorrectBroker = createOptOutJobRecord( - extractedProfileId = 2L, - brokerName = testBrokerName, // Correct broker - status = OptOutJobStatus.REQUESTED, - dateCreatedInMillis = dateCreated + 2 * oneHour, - optOutRequestedDateInMillis = dateCreated + 3 * oneHour, - ) - - // Repository returns both, but only correct broker should be counted - whenever(mockSchedulingRepository.getAllValidOptOutJobRecordsForBroker(testBrokerName)) - .thenReturn(listOf(recordWithWrongBroker, recordWithCorrectBroker)) - - val result = testee.calculateOptOutSubmitRate(testBrokerName, startDate, endDate) - - // Only recordWithCorrectBroker counts (1 out of 1 after filtering) - assertEquals(1.0, result!!, 0.0) - } - @Test fun whenSingleRecordRequestedThenReturnOne() = runTest { val startDate = baseTime @@ -672,11 +537,7 @@ class RealOptOutSubmitRateCalculatorTest { dateCreatedInMillis = dateCreated, optOutRequestedDateInMillis = dateCreated + oneHour, ) - - whenever(mockSchedulingRepository.getAllValidOptOutJobRecordsForBroker(testBrokerName)) - .thenReturn(listOf(singleRecord)) - - val result = testee.calculateOptOutSubmitRate(testBrokerName, startDate, endDate) + val result = testee.calculateOptOutSubmitRate(listOf(singleRecord), startDate, endDate) assertEquals(1.0, result!!, 0.0) } @@ -693,11 +554,7 @@ class RealOptOutSubmitRateCalculatorTest { status = OptOutJobStatus.NOT_EXECUTED, dateCreatedInMillis = dateCreated, ) - - whenever(mockSchedulingRepository.getAllValidOptOutJobRecordsForBroker(testBrokerName)) - .thenReturn(listOf(singleRecord)) - - val result = testee.calculateOptOutSubmitRate(testBrokerName, startDate, endDate) + val result = testee.calculateOptOutSubmitRate(listOf(singleRecord), startDate, endDate) assertEquals(0.0, result!!, 0.0) } @@ -729,10 +586,7 @@ class RealOptOutSubmitRateCalculatorTest { dateCreatedInMillis = dateCreated + TimeUnit.DAYS.toMillis(300), // 10 months later ) - whenever(mockSchedulingRepository.getAllValidOptOutJobRecordsForBroker(testBrokerName)) - .thenReturn(listOf(record1, record2, record3)) - - val result = testee.calculateOptOutSubmitRate(testBrokerName, startDate, endDate) + val result = testee.calculateOptOutSubmitRate(listOf(record1, record2, record3), startDate, endDate) // 2 out of 3 assertEquals(0.67, result!!, 0.0) @@ -759,10 +613,7 @@ class RealOptOutSubmitRateCalculatorTest { optOutRequestedDateInMillis = dateCreated + 3 * oneHour, ) - whenever(mockSchedulingRepository.getAllValidOptOutJobRecordsForBroker(testBrokerName)) - .thenReturn(listOf(requestedWithZeroDate, validRequested)) - - val result = testee.calculateOptOutSubmitRate(testBrokerName, startDate, endDate) + val result = testee.calculateOptOutSubmitRate(listOf(requestedWithZeroDate, validRequested), startDate, endDate) // Only validRequested counts (1 out of 2) assertEquals(0.5, result!!, 0.0) @@ -791,10 +642,7 @@ class RealOptOutSubmitRateCalculatorTest { ) } - whenever(mockSchedulingRepository.getAllValidOptOutJobRecordsForBroker(testBrokerName)) - .thenReturn(listOf(requested) + notExecutedRecords) - - val result = testee.calculateOptOutSubmitRate(testBrokerName, startDate, endDate) + val result = testee.calculateOptOutSubmitRate(listOf(requested) + notExecutedRecords, startDate, endDate) // 1/7 = 0.142857... rounded to 0.14 assertEquals(0.14, result!!, 0.0) diff --git a/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/store/RealPirSchedulingRepositoryTest.kt b/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/store/RealPirSchedulingRepositoryTest.kt index 301541d36c7b..770d39f032fe 100644 --- a/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/store/RealPirSchedulingRepositoryTest.kt +++ b/pir/pir-impl/src/test/kotlin/com/duckduckgo/pir/impl/store/RealPirSchedulingRepositoryTest.kt @@ -27,6 +27,7 @@ import com.duckduckgo.pir.impl.store.db.BrokerJsonDao import com.duckduckgo.pir.impl.store.db.EmailConfirmationJobRecordEntity import com.duckduckgo.pir.impl.store.db.JobSchedulingDao import com.duckduckgo.pir.impl.store.db.OptOutJobRecordEntity +import com.duckduckgo.pir.impl.store.db.ReportingRecord import com.duckduckgo.pir.impl.store.db.ScanJobRecordEntity import com.duckduckgo.pir.impl.store.secure.PirSecureStorageDatabaseFactory import kotlinx.coroutines.runBlocking @@ -103,6 +104,7 @@ class RealPirSchedulingRepositoryTest { optOutRequestedDate = 2000L, optOutRemovedDate = 0L, dateCreatedInMillis = 100L, + reporting = ReportingRecord(), ) private val deprecatedOptOutJobEntity = @@ -117,6 +119,7 @@ class RealPirSchedulingRepositoryTest { optOutRequestedDate = 4000L, optOutRemovedDate = 0L, dateCreatedInMillis = 100L, + reporting = ReportingRecord(), ) private val scanJobRecord = @@ -405,6 +408,7 @@ class RealPirSchedulingRepositoryTest { optOutRequestedDate = 2000L, optOutRemovedDate = 0L, dateCreatedInMillis = 9000L, + reporting = ReportingRecord(), ), ) } @@ -439,6 +443,7 @@ class RealPirSchedulingRepositoryTest { optOutRequestedDate = 2000L, optOutRemovedDate = 0L, dateCreatedInMillis = 9000L, + reporting = ReportingRecord(), ), OptOutJobRecordEntity( extractedProfileId = 999L, @@ -450,6 +455,7 @@ class RealPirSchedulingRepositoryTest { optOutRequestedDate = 4000L, optOutRemovedDate = 5000L, dateCreatedInMillis = 9000L, + reporting = ReportingRecord(), ), ), )