Support deduplication key for aggregate attribution

WICG/attribution-reporting-api#527

This CL adds aggregatable deduplication key to trigger registration to
allow deduplication mechanism for aggregatable reports. The
deduplication keys are separate for event-level and aggregatable
reports.

Bug: 1368147
Change-Id: Ic6871436e2b3a95e81d8de29f3ad58c7021d7482
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3918652
Reviewed-by: Andrew Paseltiner <apaseltiner@chromium.org>
Commit-Queue: Nan Lin <linnan@chromium.org>
Reviewed-by: John Delaney <johnidel@chromium.org>
Reviewed-by: Nate Chapin <japhet@chromium.org>
Reviewed-by: Robert Sesek <rsesek@chromium.org>
Cr-Commit-Position: refs/heads/main@{#1052902}

content/browser/attribution_reporting/attribution_data_host_manager_impl.cc

@@ -561,6 +561,9 @@ void AttributionDataHostManagerImpl::TriggerDataAvailable(
       std::move(*not_filters),
       data->debug_key ? absl::make_optional(data->debug_key->value)
                       : absl::nullopt,
+      data->aggregatable_dedup_key
+          ? absl::make_optional(data->aggregatable_dedup_key->value)
+          : absl::nullopt,
       std::move(event_triggers), std::move(*aggregatable_trigger_data),
       std::move(*aggregatable_values));
 

content/browser/attribution_reporting/attribution_data_host_manager_impl_unittest.cc

@@ -481,7 +481,8 @@ TEST_F(AttributionDataHostManagerImplTest, TriggerDataHost_TriggerRegistered) {
                           }))),
                       EventTriggerDataMatches(EventTriggerDataMatcherConfig(
                           4, 5, Eq(absl::nullopt), AttributionFilterData(),
-                          AttributionFilterData())))))));
+                          AttributionFilterData()))),
+          Optional(123)))));
 
   {
     RemoteDataHost data_host_remote{.task_environment = task_environment_};
@@ -514,6 +515,9 @@ TEST_F(AttributionDataHostManagerImplTest, TriggerDataHost_TriggerRegistered) {
         /*filters=*/blink::mojom::AttributionFilterData::New(),
         /*not_filters=*/blink::mojom::AttributionFilterData::New()));
 
+    trigger_data->aggregatable_dedup_key =
+        blink::mojom::AttributionTriggerDedupKey::New(123);
+
     data_host_remote.data_host->TriggerDataAvailable(std::move(trigger_data));
     data_host_remote.data_host.FlushForTesting();
   }

content/browser/attribution_reporting/attribution_internals.mojom

@@ -159,9 +159,9 @@ struct WebUITrigger {
     kExcessiveAttributions,
     kExcessiveReportingOrigins,
     kProhibitedByBrowserPolicy,
+    kDeduplicated,
 
     // Event-level statuses:
-    kDeduplicated,
     kLowPriority,
     kNoised,
     kNoMatchingConfigurations,

content/browser/attribution_reporting/attribution_internals_browsertest.cc

@@ -882,6 +882,7 @@ IN_PROC_BROWSER_TEST_F(AttributionInternalsWebUiBrowserTest,
       /*filters=*/AttributionFilterData::CreateForTesting({{"a", {"b"}}}),
       /*not_filters=*/AttributionFilterData::CreateForTesting({{"g", {"h"}}}),
       /*debug_key=*/1,
+      /*aggregatable_dedup_key=*/absl::nullopt,
       {
           AttributionTrigger::EventTriggerData(
               /*data=*/2,

content/browser/attribution_reporting/attribution_internals_handler_impl.cc

@@ -400,6 +400,8 @@ WebUITriggerStatus GetWebUITriggerStatus(AggregatableStatus status) {
       return WebUITriggerStatus::kNotRegistered;
     case AggregatableStatus::kProhibitedByBrowserPolicy:
       return WebUITriggerStatus::kProhibitedByBrowserPolicy;
+    case AggregatableStatus::kDeduplicated:
+      return WebUITriggerStatus::kDeduplicated;
   }
 }
 
@@ -450,6 +452,8 @@ void AttributionInternalsHandlerImpl::OnTriggerHandled(
 
   web_ui_trigger->aggregatable_values = trigger.aggregatable_values().values();
 
+  // TODO(crbug.com/1368147): Display aggregatable dedup key in internals UI.
+
   for (auto& observer : observers_) {
     observer->OnTriggerHandled(web_ui_trigger.Clone());
   }

content/browser/attribution_reporting/attribution_src_browsertest.cc

@@ -922,6 +922,7 @@ IN_PROC_BROWSER_TEST_P(AttributionSrcBasicTriggerBrowserTest,
       IsEmpty());
   EXPECT_THAT(trigger_data.front()->aggregatable_trigger_data, IsEmpty());
   EXPECT_THAT(trigger_data.front()->aggregatable_values, IsEmpty());
+  EXPECT_FALSE(trigger_data.front()->aggregatable_dedup_key);
 }
 
 IN_PROC_BROWSER_TEST_F(AttributionSrcBrowserTest,
@@ -1004,6 +1005,8 @@ IN_PROC_BROWSER_TEST_F(AttributionSrcBrowserTest,
 
   EXPECT_THAT(trigger_data.front()->aggregatable_values,
               ElementsAre(Pair("key", 123)));
+  EXPECT_EQ(trigger_data.front()->aggregatable_dedup_key,
+            blink::mojom::AttributionTriggerDedupKey::New(123));
 }
 
 IN_PROC_BROWSER_TEST_F(

content/browser/attribution_reporting/attribution_storage_sql.cc

@@ -60,11 +60,11 @@
 namespace content {
 
 // Version number of the database.
-const int AttributionStorageSql::kCurrentVersionNumber = 36;
+const int AttributionStorageSql::kCurrentVersionNumber = 37;
 
 // Earliest version which can use a |kCurrentVersionNumber| database
 // without failing.
-const int AttributionStorageSql::kCompatibleVersionNumber = 36;
+const int AttributionStorageSql::kCompatibleVersionNumber = 37;
 
 // Latest version of the database that cannot be upgraded to
 // |kCurrentVersionNumber| without razing the database.
@@ -238,6 +238,10 @@ absl::optional<AttributionSourceType> DeserializeSourceType(int val) {
   }
 }
 
+int SerializeReportType(AttributionReport::Type val) {
+  return static_cast<int>(val);
+}
+
 std::string SerializePotentiallyTrustworthyOrigin(const url::Origin& origin) {
   DCHECK(network::IsOriginPotentiallyTrustworthy(origin));
   return SerializeOrigin(origin);
@@ -883,7 +887,8 @@ CreateReportResult AttributionStorageSql::MaybeCreateAndStoreReport(
     DCHECK(new_aggregatable_report.has_value());
     store_aggregatable_status = MaybeStoreAggregatableAttributionReport(
         *new_aggregatable_report,
-        source_to_attribute->source.aggregatable_budget_consumed());
+        source_to_attribute->source.aggregatable_budget_consumed(),
+        trigger.aggregatable_dedup_key());
   }
 
   if (store_event_level_status == EventLevelResult::kInternalError ||
@@ -1034,7 +1039,8 @@ EventLevelResult AttributionStorageSql::MaybeCreateEventLevelReport(
     return EventLevelResult::kNoMatchingConfigurations;
 
   switch (ReportAlreadyStored(attribution_info.source.source_id(),
-                              event_trigger->dedup_key)) {
+                              event_trigger->dedup_key,
+                              AttributionReport::Type::kEventLevel)) {
     case ReportAlreadyStoredStatus::kNotStored:
       break;
     case ReportAlreadyStoredStatus::kStored:
@@ -1138,16 +1144,10 @@ EventLevelResult AttributionStorageSql::MaybeStoreEventLevelReport(
   // If a dedup key is present, store it. We do this regardless of whether
   // `create_report` is true to avoid leaking whether the report was actually
   // stored.
-  if (dedup_key.has_value()) {
-    static constexpr char kInsertDedupKeySql[] =
-        "INSERT INTO dedup_keys(source_id,dedup_key)VALUES(?,?)";
-    sql::Statement insert_dedup_key_statement(
-        db_->GetCachedStatement(SQL_FROM_HERE, kInsertDedupKeySql));
-    insert_dedup_key_statement.BindInt64(0,
-                                         *attribution_info.source.source_id());
-    insert_dedup_key_statement.BindInt64(1, SerializeUint64(*dedup_key));
-    if (!insert_dedup_key_statement.Run())
-      return EventLevelResult::kInternalError;
+  if (dedup_key.has_value() &&
+      !StoreDedupKey(attribution_info.source.source_id(), *dedup_key,
+                     AttributionReport::Type::kEventLevel)) {
+    return EventLevelResult::kInternalError;
   }
 
   // Only increment the number of conversions associated with the source if
@@ -1784,18 +1784,21 @@ bool AttributionStorageSql::HasCapacityForStoringSource(
 }
 
 AttributionStorageSql::ReportAlreadyStoredStatus
-AttributionStorageSql::ReportAlreadyStored(StoredSource::Id source_id,
-                                           absl::optional<uint64_t> dedup_key) {
+AttributionStorageSql::ReportAlreadyStored(
+    StoredSource::Id source_id,
+    absl::optional<uint64_t> dedup_key,
+    AttributionReport::Type report_type) {
   if (!dedup_key.has_value())
     return ReportAlreadyStoredStatus::kNotStored;
 
   static constexpr char kCountReportsSql[] =
       "SELECT COUNT(*)FROM dedup_keys "
-      "WHERE source_id=? AND dedup_key=?";
+      "WHERE source_id=? AND report_type=? AND dedup_key=?";
   sql::Statement statement(
       db_->GetCachedStatement(SQL_FROM_HERE, kCountReportsSql));
   statement.BindInt64(0, *source_id);
-  statement.BindInt64(1, SerializeUint64(*dedup_key));
+  statement.BindInt(1, SerializeReportType(report_type));
+  statement.BindInt64(2, SerializeUint64(*dedup_key));
 
   // If there's an error, return true so `MaybeCreateAndStoreReport()`
   // returns early.
@@ -1867,22 +1870,31 @@ std::vector<StoredSource> AttributionStorageSql::GetActiveSources(int limit) {
 
   for (auto& source : sources) {
     absl::optional<std::vector<uint64_t>> dedup_keys =
-        ReadDedupKeys(source.source_id());
+        ReadDedupKeys(source.source_id(), AttributionReport::Type::kEventLevel);
     if (!dedup_keys.has_value())
       return {};
     source.SetDedupKeys(std::move(*dedup_keys));
+
+    absl::optional<std::vector<uint64_t>> aggregatable_dedup_keys =
+        ReadDedupKeys(source.source_id(),
+                      AttributionReport::Type::kAggregatableAttribution);
+    if (!aggregatable_dedup_keys.has_value())
+      return {};
+    source.SetAggregatableDedupKeys(std::move(*aggregatable_dedup_keys));
   }
 
   return sources;
 }
 
 absl::optional<std::vector<uint64_t>> AttributionStorageSql::ReadDedupKeys(
-    StoredSource::Id source_id) {
+    StoredSource::Id source_id,
+    AttributionReport::Type report_type) {
   static constexpr char kDedupKeySql[] =
-      "SELECT dedup_key FROM dedup_keys WHERE source_id=?";
+      "SELECT dedup_key FROM dedup_keys WHERE source_id=? AND report_type=?";
   sql::Statement statement(
       db_->GetCachedStatement(SQL_FROM_HERE, kDedupKeySql));
   statement.BindInt64(0, *source_id);
+  statement.BindInt(1, SerializeReportType(report_type));
 
   std::vector<uint64_t> dedup_keys;
   while (statement.Step()) {
@@ -1894,6 +1906,19 @@ absl::optional<std::vector<uint64_t>> AttributionStorageSql::ReadDedupKeys(
   return dedup_keys;
 }
 
+bool AttributionStorageSql::StoreDedupKey(StoredSource::Id source_id,
+                                          uint64_t dedup_key,
+                                          AttributionReport::Type report_type) {
+  static constexpr char kInsertDedupKeySql[] =
+      "INSERT INTO dedup_keys(source_id,report_type,dedup_key)VALUES(?,?,?)";
+  sql::Statement statement(
+      db_->GetCachedStatement(SQL_FROM_HERE, kInsertDedupKeySql));
+  statement.BindInt64(0, *source_id);
+  statement.BindInt(1, SerializeReportType(report_type));
+  statement.BindInt64(2, SerializeUint64(dedup_key));
+  return statement.Run();
+}
+
 void AttributionStorageSql::HandleInitializationFailure(
     const InitStatus status) {
   RecordInitializationStatus(status);
@@ -2154,8 +2179,9 @@ bool AttributionStorageSql::CreateSchema() {
   static constexpr char kDedupKeyTableSql[] =
       "CREATE TABLE dedup_keys("
       "source_id INTEGER NOT NULL,"
+      "report_type INTEGER NOT NULL,"
       "dedup_key INTEGER NOT NULL,"
-      "PRIMARY KEY(source_id,dedup_key))WITHOUT ROWID";
+      "PRIMARY KEY(source_id,report_type,dedup_key))WITHOUT ROWID";
   if (!db_->Execute(kDedupKeyTableSql))
     return false;
 
@@ -2625,6 +2651,17 @@ AttributionStorageSql::MaybeCreateAggregatableAttributionReport(
   if (contributions.empty())
     return AggregatableResult::kNoHistograms;
 
+  switch (ReportAlreadyStored(
+      attribution_info.source.source_id(), trigger.aggregatable_dedup_key(),
+      AttributionReport::Type::kAggregatableAttribution)) {
+    case ReportAlreadyStoredStatus::kNotStored:
+      break;
+    case ReportAlreadyStoredStatus::kStored:
+      return AggregatableResult::kDeduplicated;
+    case ReportAlreadyStoredStatus::kError:
+      return AggregatableResult::kInternalError;
+  }
+
   switch (CapacityForStoringReport(
       trigger, AttributionReport::Type::kAggregatableAttribution)) {
     case ConversionCapacityStatus::kHasCapacity:
@@ -2709,7 +2746,8 @@ bool AttributionStorageSql::StoreAggregatableAttributionReport(
 AggregatableResult
 AttributionStorageSql::MaybeStoreAggregatableAttributionReport(
     AttributionReport& report,
-    int64_t aggregatable_budget_consumed) {
+    int64_t aggregatable_budget_consumed,
+    absl::optional<uint64_t> dedup_key) {
   const auto* aggregatable_attribution =
       absl::get_if<AttributionReport::AggregatableAttributionData>(
           &report.data());
@@ -2732,14 +2770,20 @@ AttributionStorageSql::MaybeStoreAggregatableAttributionReport(
   if (!StoreAggregatableAttributionReport(report))
     return AggregatableResult::kInternalError;
 
+  StoredSource::Id source_id = report.attribution_info().source.source_id();
+
   base::CheckedNumeric<int64_t> budget_required =
       aggregatable_attribution->BudgetRequired();
   // The value was already validated by
   // `AggregatableAttributionAllowedForBudgetLimit()` above.
   DCHECK(budget_required.IsValid());
-  if (!AdjustBudgetConsumedForSource(
-          report.attribution_info().source.source_id(),
-          budget_required.ValueOrDie())) {
+  if (!AdjustBudgetConsumedForSource(source_id, budget_required.ValueOrDie())) {
+    return AggregatableResult::kInternalError;
+  }
+
+  if (dedup_key.has_value() &&
+      !StoreDedupKey(source_id, *dedup_key,
+                     AttributionReport::Type::kAggregatableAttribution)) {
     return AggregatableResult::kInternalError;
   }
 

content/browser/attribution_reporting/attribution_storage_sql.h

@@ -157,7 +157,8 @@ class CONTENT_EXPORT AttributionStorageSql : public AttributionStorage {
 
   ReportAlreadyStoredStatus ReportAlreadyStored(
       StoredSource::Id source_id,
-      absl::optional<uint64_t> dedup_key)
+      absl::optional<uint64_t> dedup_key,
+      AttributionReport::Type report_type)
       VALID_CONTEXT_REQUIRED(sequence_checker_);
 
   enum class ConversionCapacityStatus {
@@ -190,7 +191,14 @@ class CONTENT_EXPORT AttributionStorageSql : public AttributionStorage {
       VALID_CONTEXT_REQUIRED(sequence_checker_);
 
   absl::optional<std::vector<uint64_t>> ReadDedupKeys(
-      StoredSource::Id source_id) VALID_CONTEXT_REQUIRED(sequence_checker_);
+      StoredSource::Id source_id,
+      AttributionReport::Type report_type)
+      VALID_CONTEXT_REQUIRED(sequence_checker_);
+
+  bool StoreDedupKey(StoredSource::Id source_id,
+                     uint64_t dedup_key,
+                     AttributionReport::Type report_type)
+      VALID_CONTEXT_REQUIRED(sequence_checker_);
 
   [[nodiscard]] RateLimitResult
   HasCapacityForUniqueDestinationLimitForPendingSource(
@@ -348,7 +356,8 @@ class CONTENT_EXPORT AttributionStorageSql : public AttributionStorage {
 
   AttributionTrigger::AggregatableResult
   MaybeStoreAggregatableAttributionReport(AttributionReport& report,
-                                          int64_t aggregatable_budget_consumed)
+                                          int64_t aggregatable_budget_consumed,
+                                          absl::optional<uint64_t> dedup_key)
       VALID_CONTEXT_REQUIRED(sequence_checker_);
 
   [[nodiscard]] bool StoreAggregatableAttributionReport(

content/browser/attribution_reporting/attribution_storage_sql_migrations.cc

@@ -7,6 +7,7 @@
 #include "base/check.h"
 #include "base/metrics/histogram_functions.h"
 #include "base/time/time.h"
+#include "content/browser/attribution_reporting/attribution_report.h"
 #include "content/browser/attribution_reporting/common_source_info.h"
 #include "content/browser/attribution_reporting/rate_limit_table.h"
 #include "sql/database.h"
@@ -224,6 +225,48 @@ bool MigrateToVersion36(sql::Database* db, sql::MetaTable* meta_table) {
   return transaction.Commit();
 }
 
+bool MigrateToVersion37(sql::Database* db, sql::MetaTable* meta_table) {
+  // Wrap each migration in its own transaction. See comment in
+  // `MigrateToVersion34`.
+  sql::Transaction transaction(db);
+  if (!transaction.Begin())
+    return false;
+
+  static constexpr char kNewDedupKeyTableSql[] =
+      "CREATE TABLE IF NOT EXISTS new_dedup_keys"
+      "(source_id INTEGER NOT NULL,"
+      "report_type INTEGER NOT NULL,"
+      "dedup_key INTEGER NOT NULL,"
+      "PRIMARY KEY(source_id,report_type,dedup_key))WITHOUT ROWID";
+  if (!db->Execute(kNewDedupKeyTableSql))
+    return false;
+
+  static_assert(static_cast<int>(AttributionReport::Type::kEventLevel) == 0,
+                "update the report type value `0` below");
+
+  // Transfer the existing rows to the new table, inserting
+  // `Attribution::Type::kEventLevel` as default values for the
+  // report_type column.
+  static constexpr char kPopulateNewDedupKeyTableSql[] =
+      "INSERT INTO new_dedup_keys SELECT "
+      "source_id,0,dedup_key "
+      "FROM dedup_keys";
+  if (!db->Execute(kPopulateNewDedupKeyTableSql))
+    return false;
+
+  static constexpr char kDropOldDedupKeyTableSql[] = "DROP TABLE dedup_keys";
+  if (!db->Execute(kDropOldDedupKeyTableSql))
+    return false;
+
+  static constexpr char kRenameDedupKeyTableSql[] =
+      "ALTER TABLE new_dedup_keys RENAME TO dedup_keys";
+  if (!db->Execute(kRenameDedupKeyTableSql))
+    return false;
+
+  meta_table->SetVersionNumber(37);
+  return transaction.Commit();
+}
+
 }  // namespace
 
 bool UpgradeAttributionStorageSqlSchema(sql::Database* db,
@@ -245,6 +288,10 @@ bool UpgradeAttributionStorageSqlSchema(sql::Database* db,
     if (!MigrateToVersion36(db, meta_table))
       return false;
   }
+  if (meta_table->GetVersionNumber() == 36) {
+    if (!MigrateToVersion37(db, meta_table))
+      return false;
+  }
   // Add similar if () blocks for new versions here.
 
   base::UmaHistogramMediumTimes("Conversions.Storage.MigrationTime",